import { ConfigurationError } from "@pipedream/platform";
import {
clearObj,
parseError, parseObject,
} from "../../common/utils.mjs";
import scrapeninja from "../../scrapeninja.app.mjs";
export default {
key: "scrapeninja-scrape-with-js-rendering",
name: "Scrape with JS Rendering",
description: "Uses the ScrapeNinja real Chrome browser engine to scrape pages that require JS rendering. [See the documentation](https://scrapeninja.net/docs/api-reference/scrape-js/)",
version: "0.0.1",
type: "action",
props: {
scrapeninja,
url: {
propDefinition: [
scrapeninja,
"url",
],
},
waitForSelector: {
propDefinition: [
scrapeninja,
"waitForSelector",
],
optional: true,
},
postWaitTime: {
propDefinition: [
scrapeninja,
"postWaitTime",
],
optional: true,
},
dumpIframe: {
propDefinition: [
scrapeninja,
"dumpIframe",
],
optional: true,
},
waitForSelectorIframe: {
propDefinition: [
scrapeninja,
"waitForSelectorIframe",
],
optional: true,
},
extractorTargetIframe: {
propDefinition: [
scrapeninja,
"extractorTargetIframe",
],
optional: true,
},
headers: {
propDefinition: [
scrapeninja,
"headers",
],
optional: true,
},
retryNum: {
propDefinition: [
scrapeninja,
"retryNum",
],
optional: true,
},
geo: {
propDefinition: [
scrapeninja,
"geo",
],
optional: true,
},
proxy: {
propDefinition: [
scrapeninja,
"proxy",
],
optional: true,
},
timeout: {
propDefinition: [
scrapeninja,
"timeout",
],
optional: true,
},
textNotExpected: {
propDefinition: [
scrapeninja,
"textNotExpected",
],
optional: true,
},
statusNotExpected: {
propDefinition: [
scrapeninja,
"statusNotExpected",
],
optional: true,
},
blockImages: {
propDefinition: [
scrapeninja,
"blockImages",
],
optional: true,
},
blockMedia: {
propDefinition: [
scrapeninja,
"blockMedia",
],
optional: true,
},
screenshot: {
propDefinition: [
scrapeninja,
"screenshot",
],
optional: true,
},
catchAjaxHeadersUrlMask: {
propDefinition: [
scrapeninja,
"catchAjaxHeadersUrlMask",
],
optional: true,
},
viewportWidth: {
propDefinition: [
scrapeninja,
"viewportWidth",
],
optional: true,
},
viewportHeight: {
propDefinition: [
scrapeninja,
"viewportHeight",
],
optional: true,
},
viewportDeviceScaleFactor: {
propDefinition: [
scrapeninja,
"viewportDeviceScaleFactor",
],
optional: true,
},
viewportHasTouch: {
propDefinition: [
scrapeninja,
"viewportHasTouch",
],
optional: true,
},
viewportIsMobile: {
propDefinition: [
scrapeninja,
"viewportIsMobile",
],
optional: true,
},
viewportIsLandscape: {
propDefinition: [
scrapeninja,
"viewportIsLandscape",
],
optional: true,
},
extractor: {
propDefinition: [
scrapeninja,
"extractor",
],
optional: true,
},
},
async run({ $ }) {
try {
const viewport = clearObj({
width: this.viewportWidth,
height: this.viewportHeight,
deviceScaleFactor: this.viewportDeviceScaleFactor,
hasTouch: this.viewportHasTouch,
isMobile: this.viewportIsMobile,
isLandscape: this.viewportIsLandscape,
});
const data = clearObj({
url: this.url,
waitForSelector: this.waitForSelector,
postWaitTime: this.postWaitTime,
dumpIframe: this.dumpIframe,
waitForSelectorIframe: this.waitForSelectorIframe,
extractorTargetIframe: this.extractorTargetIframe,
headers: parseObject(this.headers),
retryNum: this.retryNum,
geo: this.geo,
proxy: this.proxy,
timeout: this.timeout,
textNotExpected: parseObject(this.textNotExpected),
statusNotExpected: parseObject(this.statusNotExpected),
blockImages: this.blockImages,
blockMedia: this.blockMedia,
screenshot: this.screenshot,
catchAjaxHeadersUrlMask: this.catchAjaxHeadersUrlMask,
extractor: this.extractor,
});
if (Object.entries(viewport).length) {
data.viewport = viewport;
}
const response = await this.scrapeninja.scrapeJs({
$,
data,
});
$.export("$summary", `Successfully scraped ${this.url} with JS rendering`);
return response;
} catch ({ response: { data } }) {
throw new ConfigurationError(parseError(data));
}
},
};
Chase Roberts@chsrbrts@benedictevans If you haven’t used @pipedream yet, then you haven’t lived.
✨Ellie Day✨@heyelliedayEvaluation update: @pipedream has quite literally been a dream to work with! I’m excited to leverage this tool for all the various workflows I need to write. I’m currently at 11k invocations a day from the initial workflows I’ve written in the past couple weeks.
Michael Braedley@MBraedleyUpdate: I got it working properly, and it's working so well that I'm dropping IFTTT. @pipedream can do everything that IFTTT basic can, and most (if not all things) IFTTT pro can for free or at a reasonable price if you need it. I am recommending it for basically any power user.
Thomas Cutting@mrthomascuttingWant quick+dirty integrations for a serverless workflow - @pipedream is my new go-to 😃
Matthew Roberts@mattdotrobertsday 013 - finally hit node js. This is the secret sauce of taking #nocode projects that one step further. Pumped about getting deeper into @pipedream now
Kenneth Auchenberg 💭@auchenbergYahoo Pipes is back! Kinda 😍 @pipedream
Raymond Camden 🥑@raymondcamdenAwesome video by the @pipedream folks showing real time twitter sentiment analysis integrated with Google Sheets. This is where Pipedream *really* shines, connecting systems together in easy workflows.
Nacho Caballero@nachocaballeroI couldn't recommend @pipedream more. It's an amazing service to integrate different APIs. Much more powerful than Zapier and more user-friendly than AWS Lambda. I'm very proud to wear this t-shirt #NoCode
Jason Snow@jyksnwDeveloped a working prototype environmental sensor IoT solution with @particle Photon, @pipedream, and @MongoDB with full graphing and alerting in less than a day! All amazing technology, will def. be exploring these more.
Steven Terrana@steven_terrana@burgwyn you've inspired me to finally set up my own blog. I'll make sure my first blog post explains the tech behind the setup. think @obsdmd + @GatsbyJS + @pipedream.
🚄 James Augeri, PhD@DotDotJamesWant to low-code your back end, need more horsepower than @Bubble / @KnackHQ, or just miss Yahoo! Pipes? Check out @PipeDream
Sébastien Chopin@AtinuxGitHub issues should be like @linear_app for maintainers. Looking forward more integrations with GH actions or tools like @pipedream 👀
Raul@raul_predescuIf you're a dev and not using @pipedream, you're missing out. Been using it for months, daily. FREE for devs. Plenty of integrations and good limits. Absolutely love it.
Bruno Skvorc@bitfallsSo @pipedream is pretty amazing. In 3 minutes I just made a flow which adds @rickastley's Never Gonna Give You Up to my @spotify playlist whenever a new pull request arrives in an old repo of mine.
Zach Lanich@ZachLanichUm, wow 🤯 @pipedream
Steven Bell@bellontechI just used @pipedream to build a Shopify App. Wow, they make small backed tasks easy.
Jay Hack 🎩🇺🇸@_jayhack_Very impressed with this bad boi - it reminds me of a @PalantirTech internal tool, but geared towards integrations instead of data analysis and far more customizable. Great expectations here 🚀🤩
Tree Sturgeon 🔥🚴♂️🌳@philsturgeonFor context this is day 2 of a really challenging and stupid migration from Notion to @airtable with disparate/missing data. It's going better than expected and thanks to @pipedream I don't have to bother the iOS dev to add W3W.
