with FireCrawl and RabbitMQ?
Crawls a given URL and returns the contents of sub-pages. See the documentation
Extract structured data from one or multiple URLs. See the documentation
Obtains the status and data from a previous crawl operation. See the documentation
Obtains the status and data from a previous extract operation. See the documentation
Scrapes a URL and returns content from that page. See the documentation
import { axios } from "@pipedream/platform"
export default defineComponent({
props: {
firecrawl: {
type: "app",
app: "firecrawl",
}
},
async run({steps, $}) {
const data = {
"url": "https://pipedream.com",
}
return await axios($, {
method: "post",
url: `https://api.firecrawl.dev/v0/crawl`,
headers: {
Authorization: `Bearer ${this.firecrawl.$auth.api_key}`,
},
data,
})
},
})
import amqp from "amqplib";
export default defineComponent({
props: {
rabbitmq: {
type: "app",
app: "rabbitmq",
}
},
async run({ steps, $ }) {
const url = `amqp://${this.rabbitmq.$auth.username}:${this.rabbitmq.$auth.password}@${this.rabbitmq.$auth.host}:${this.rabbitmq.$auth.port}`;
const connection = await amqp.connect(url);
const channel = await connection.createChannel();
const queue = 'Sample Queue';
await channel.assertQueue(queue, { durable: true });
const message = 'Welcome RabbitMQ + Pipedream users! ' + new Date().toISOString()
channel.sendToQueue(queue, Buffer.from(message), { persistent: true });
console.log(`Sent: ${message}`);
const queueInfo = await channel.checkQueue(queue);
return queueInfo;
},
})