import app from "../../scrape_it_cloud.app.mjs";
import utils from "../../common/utils.mjs";
export default {
key: "scrape_it_cloud-start-scraping",
name: "Start Scraping",
description: "Initiate the scraping process for a specific endpoint. [See the documentation here](https://scrape-it.cloud/docs/api-features/basic-request).",
type: "action",
version: "0.0.1",
props: {
app,
url: {
type: "string",
label: "URL",
description: "Specify the URL of the web page you would like to scrape.",
},
headers: {
type: "object",
label: "Headers",
description: "Specify custom HTTP headers to be passed to the request. For example, in order to override the User-Agent header, use `User-Agent` as a key and `Teapot` as a value.",
optional: true,
},
blockResources: {
type: "boolean",
label: "Block Resources",
description: "By default Scrape-it.cloud do not block images and CSS in the scraped page. To speed up requests and block images and CSS set this parameter value to true.",
optional: true,
},
blockUrls: {
type: "string[]",
label: "Block URLs",
description: "If you want to block some resources except images and CSS, for example, analytics scripts you can add part of the urls to be blocked. Follow [Blocking URLs](https://scrape-it.cloud/docs/api-features/blocking-urls) page to get more information.",
optional: true,
},
wait: {
type: "integer",
label: "Wait",
description: "Some websites may use javascript frameworks that may require a few extra seconds to load their content. This parameters specifies the time in milliseconds to wait for the website. Recommended values are in the interval `5000` - `10000`.",
optional: true,
},
waitFor: {
type: "string",
label: "Wait For",
description: "Specify a CSS selector and the API will wait 30 seconds until the selector appears.",
optional: true,
},
screenshot: {
type: "boolean",
label: "Screenshot",
description: "If you want to get a screenshot of the page you want to scrape, set screenshot parameter value to `true`.",
optional: true,
},
jsScenario: {
type: "string[]",
label: "JS Scenario",
description: "If you want to do some acts on the pages you want to scrape, for example, go to the next one, set JavaScript scenario to API call. Follow [JavaScript Execution](https://scrape-it.cloud/docs/api-features/javascript-execution) page to get more information.",
optional: true,
},
},
methods: {
scrape(args = {}) {
return this.app.post({
path: "/scrape",
...args,
});
},
},
async run({ $: step }) {
const {
url, headers, blockResources, blockUrls, wait, waitFor, screenshot, jsScenario,
} = this;
const response = await this.scrape({
step,
data: {
url,
headers: utils.parse(headers),
block_resources: blockResources,
block_urls: utils.parseArray(blockUrls),
wait,
wait_for: waitFor,
screenshot,
js_scenario: utils.parseArray(jsScenario)?.map(utils.parse),
},
});
step.export("$summary", `Successfully scraped ${url}.`);
return response;
},
};
Chase Roberts@chsrbrts@benedictevans If you haven’t used @pipedream yet, then you haven’t lived.
✨Ellie Day✨@heyelliedayEvaluation update: @pipedream has quite literally been a dream to work with! I’m excited to leverage this tool for all the various workflows I need to write. I’m currently at 11k invocations a day from the initial workflows I’ve written in the past couple weeks.
Michael Braedley@MBraedleyUpdate: I got it working properly, and it's working so well that I'm dropping IFTTT. @pipedream can do everything that IFTTT basic can, and most (if not all things) IFTTT pro can for free or at a reasonable price if you need it. I am recommending it for basically any power user.
Thomas Cutting@mrthomascuttingWant quick+dirty integrations for a serverless workflow - @pipedream is my new go-to 😃
Matthew Roberts@mattdotrobertsday 013 - finally hit node js. This is the secret sauce of taking #nocode projects that one step further. Pumped about getting deeper into @pipedream now
Kenneth Auchenberg 💭@auchenbergYahoo Pipes is back! Kinda 😍 @pipedream
Raymond Camden 🥑@raymondcamdenAwesome video by the @pipedream folks showing real time twitter sentiment analysis integrated with Google Sheets. This is where Pipedream *really* shines, connecting systems together in easy workflows.
Nacho Caballero@nachocaballeroI couldn't recommend @pipedream more. It's an amazing service to integrate different APIs. Much more powerful than Zapier and more user-friendly than AWS Lambda. I'm very proud to wear this t-shirt #NoCode
Jason Snow@jyksnwDeveloped a working prototype environmental sensor IoT solution with @particle Photon, @pipedream, and @MongoDB with full graphing and alerting in less than a day! All amazing technology, will def. be exploring these more.
Steven Terrana@steven_terrana@burgwyn you've inspired me to finally set up my own blog. I'll make sure my first blog post explains the tech behind the setup. think @obsdmd + @GatsbyJS + @pipedream.
🚄 James Augeri, PhD@DotDotJamesWant to low-code your back end, need more horsepower than @Bubble / @KnackHQ, or just miss Yahoo! Pipes? Check out @PipeDream
Sébastien Chopin@AtinuxGitHub issues should be like @linear_app for maintainers. Looking forward more integrations with GH actions or tools like @pipedream 👀
Raul@raul_predescuIf you're a dev and not using @pipedream, you're missing out. Been using it for months, daily. FREE for devs. Plenty of integrations and good limits. Absolutely love it.
Bruno Skvorc@bitfallsSo @pipedream is pretty amazing. In 3 minutes I just made a flow which adds @rickastley's Never Gonna Give You Up to my @spotify playlist whenever a new pull request arrives in an old repo of mine.
Zach Lanich@ZachLanichUm, wow 🤯 @pipedream
Steven Bell@bellontechI just used @pipedream to build a Shopify App. Wow, they make small backed tasks easy.
Jay Hack 🎩🇺🇸@_jayhack_Very impressed with this bad boi - it reminds me of a @PalantirTech internal tool, but geared towards integrations instead of data analysis and far more customizable. Great expectations here 🚀🤩
Tree Sturgeon 🔥🚴♂️🌳@philsturgeonFor context this is day 2 of a really challenging and stupid migration from Notion to @airtable with disparate/missing data. It's going better than expected and thanks to @pipedream I don't have to bother the iOS dev to add W3W.