Scrape Website Text

import { PipedreamClient } from "@pipedream/sdk"

const pd = new PipedreamClient({
  projectId: process.env.PIPEDREAM_PROJECT_ID!,
  clientId: process.env.PIPEDREAM_CLIENT_ID!,
  clientSecret: process.env.PIPEDREAM_CLIENT_SECRET!,
  projectEnvironment: "production",
})

const result = await pd.actions.run({
  id: "webscraping_ai-scrape-website-text",
  externalUserId: "{external_user_id}", // any stable ID for this user in your system
  configuredProps: {
    webscraping_ai: { authProvisionId: "apn_xxxxxxx" },
    targetUrl: "Target URL",
    headers: "Headers",
  },
})

console.log(result)

curl -X POST https://api.pipedream.com/v1/connect/{project_id}/actions/run \
  -H "Content-Type: application/json" \
  -H "X-PD-Environment: production" \
  -H "Authorization: Bearer {access_token}" \
  -d '{
    "external_user_id": "{external_user_id}",
    "id": "webscraping_ai-scrape-website-text",
    "configured_props": {
      "webscraping_ai": { "authProvisionId": "apn_xxxxxxx" },
      "targetUrl": "Target URL",
      "headers": "Headers"
    }
  }'

// accessToken: mint a short-lived token with the Connect SDK — see the MCP guide
const transport = new StreamableHTTPClientTransport(
  new URL("https://remote.mcp.pipedream.net/v3"),
  {
    requestInit: {
      headers: {
        Authorization: `Bearer ${accessToken}`,
        "x-pd-project-id": "{project_id}",
        "x-pd-environment": "production",
        "x-pd-external-user-id": "{external_user_id}", // any stable ID for this user in your system
        "x-pd-app-slug": "webscraping_ai",
      },
    },
  },
)

const mcp = new Client({ name: "my-agent", version: "1.0.0" })
await mcp.connect(transport)

const { tools } = await mcp.listTools()

// listTools() hands your model this tool's input schema, so it can
// fill the arguments itself:
const result = await mcp.callTool({
  name: "webscraping_ai-scrape-website-text",
  arguments: {
    targetUrl: "Target URL",
    headers: "Headers",
  },
})