import { ConfigurationError } from "@pipedream/platform";
import {
FORMAT_OPTIONS,
PROXY_COUNTRY_OPTIONS,
PROXY_POOL_OPTIONS,
} from "../../common/constants.mjs";
import { parseObject } from "../../common/utils.mjs";
import scrapfly from "../../scrapfly.app.mjs";
export default {
key: "scrapfly-scrape-page",
name: "Scrape Page",
description: "Extract data from a specified web page. [See the documentation](https://scrapfly.io/docs/scrape-api/getting-started)",
version: "0.0.1",
type: "action",
props: {
scrapfly,
url: {
propDefinition: [
scrapfly,
"url",
],
},
headers: {
type: "object",
label: "Headers",
description: "Pass custom headers to the request.",
optional: true,
},
lang: {
type: "string",
label: "Language",
description: "Select page language. By default it uses the language of the selected proxy location. Behind the scenes, it configures the `Accept-Language` HTTP header. If the website support the language, the content will be in that lang. **Note: you cannot set headers `Accept-Language` header manually**. [See the documentation](https://scrapfly.io/docs/scrape-api/getting-started#spec)",
optional: true,
},
os: {
type: "string",
label: "Operating System",
description: "Operating System, if not selected it's random. **Note: you cannot set os parameter and `User-Agent` header at the same time.** [See the documentation](https://scrapfly.io/docs/scrape-api/getting-started#spec)",
optional: true,
},
timeout: {
type: "integer",
label: "Timeout",
description: "Timeout in milliseconds. It represents the maximum time allowed for Scrapfly to perform the scrape. Since `timeout` is not trivial to understand see our [extended documentation on timeouts](https://scrapfly.io/docs/scrape-api/understand-timeout)",
optional: true,
},
format: {
type: "string",
label: "Format",
description: "Format of the response.",
options: FORMAT_OPTIONS,
optional: true,
},
retry: {
type: "boolean",
label: "Retry",
description: "Improve reliability with retries on failure.",
optional: true,
},
proxifiedResponse: {
type: "boolean",
label: "Proxified Response",
description: "Return the content of the page directly.",
optional: true,
},
debug: {
type: "boolean",
label: "Debug",
description: "Store the API result and take a screenshot if rendering js is enabled.",
optional: true,
},
correlationId: {
type: "string",
label: "Correlation ID",
description: "Helper ID for correlating a group of scrapes.",
optional: true,
},
tags: {
type: "string[]",
label: "Tags",
description: "Add tags to your scrapes to group them.",
optional: true,
},
dns: {
type: "boolean",
label: "DNS",
description: "Query and retrieve target DNS information.",
optional: true,
},
ssl: {
type: "boolean",
label: "SSL",
description: "SSL option.",
optional: true,
},
proxyPool: {
type: "string",
label: "Proxy Pool",
description: "Select the proxy pool to use.",
optional: true,
options: PROXY_POOL_OPTIONS,
reloadProps: true,
},
},
async additionalProps() {
const props = {};
props.country = {
type: "string",
label: "Country",
description: "Proxy country location. If not set it chooses a random location available. A reference to a country must be ISO 3166 alpha-2 (2 letters). The available countries are defined by the proxy pool you use. [See the documentation](https://scrapfly.io/docs/scrape-api/getting-started#spec)",
optional: true,
options: PROXY_COUNTRY_OPTIONS[this.proxyPool],
};
return props;
},
async run({ $ }) {
try {
let headers = "";
if (this.headers) {
headers = Object.keys(parseObject(this.headers))
.reduce((acc, key) => {
acc.push(`headers[${key}]=${encodeURIComponent(this.headers[key])}`);
return acc;
}, []);
}
const params = {
url: this.url,
proxy_pool: this.proxyPool,
country: this.country,
lang: this.lang,
os: this.os,
timeout: this.timeout,
format: this.format,
retry: this.retry,
proxified_response: this.proxifiedResponse,
debug: this.debug,
correlation_id: this.correlationId,
tags: parseObject(this.tags),
dns: this.dns,
ssl: this.ssl,
headers,
};
const response = await this.scrapfly.extractWebPageContent({
$,
params,
});
$.export("$summary", `Successfully scraped content from ${this.url}`);
return response;
} catch ({ response: { data: { message } } }) {
throw new ConfigurationError(message);
}
},
};
Chase Roberts@chsrbrts@benedictevans If you haven’t used @pipedream yet, then you haven’t lived.
✨Ellie Day✨@heyelliedayEvaluation update: @pipedream has quite literally been a dream to work with! I’m excited to leverage this tool for all the various workflows I need to write. I’m currently at 11k invocations a day from the initial workflows I’ve written in the past couple weeks.
Michael Braedley@MBraedleyUpdate: I got it working properly, and it's working so well that I'm dropping IFTTT. @pipedream can do everything that IFTTT basic can, and most (if not all things) IFTTT pro can for free or at a reasonable price if you need it. I am recommending it for basically any power user.
Thomas Cutting@mrthomascuttingWant quick+dirty integrations for a serverless workflow - @pipedream is my new go-to 😃
Matthew Roberts@mattdotrobertsday 013 - finally hit node js. This is the secret sauce of taking #nocode projects that one step further. Pumped about getting deeper into @pipedream now
Kenneth Auchenberg 💭@auchenbergYahoo Pipes is back! Kinda 😍 @pipedream
Raymond Camden 🥑@raymondcamdenAwesome video by the @pipedream folks showing real time twitter sentiment analysis integrated with Google Sheets. This is where Pipedream *really* shines, connecting systems together in easy workflows.
Nacho Caballero@nachocaballeroI couldn't recommend @pipedream more. It's an amazing service to integrate different APIs. Much more powerful than Zapier and more user-friendly than AWS Lambda. I'm very proud to wear this t-shirt #NoCode
Jason Snow@jyksnwDeveloped a working prototype environmental sensor IoT solution with @particle Photon, @pipedream, and @MongoDB with full graphing and alerting in less than a day! All amazing technology, will def. be exploring these more.
Steven Terrana@steven_terrana@burgwyn you've inspired me to finally set up my own blog. I'll make sure my first blog post explains the tech behind the setup. think @obsdmd + @GatsbyJS + @pipedream.
🚄 James Augeri, PhD@DotDotJamesWant to low-code your back end, need more horsepower than @Bubble / @KnackHQ, or just miss Yahoo! Pipes? Check out @PipeDream
Sébastien Chopin@AtinuxGitHub issues should be like @linear_app for maintainers. Looking forward more integrations with GH actions or tools like @pipedream 👀
Raul@raul_predescuIf you're a dev and not using @pipedream, you're missing out. Been using it for months, daily. FREE for devs. Plenty of integrations and good limits. Absolutely love it.
Bruno Skvorc@bitfallsSo @pipedream is pretty amazing. In 3 minutes I just made a flow which adds @rickastley's Never Gonna Give You Up to my @spotify playlist whenever a new pull request arrives in an old repo of mine.
Zach Lanich@ZachLanichUm, wow 🤯 @pipedream
Steven Bell@bellontechI just used @pipedream to build a Shopify App. Wow, they make small backed tasks easy.
Jay Hack 🎩🇺🇸@_jayhack_Very impressed with this bad boi - it reminds me of a @PalantirTech internal tool, but geared towards integrations instead of data analysis and far more customizable. Great expectations here 🚀🤩
Tree Sturgeon 🔥🚴♂️🌳@philsturgeonFor context this is day 2 of a really challenging and stupid migration from Notion to @airtable with disparate/missing data. It's going better than expected and thanks to @pipedream I don't have to bother the iOS dev to add W3W.