with Kafka and Puppeteer?
Get the HTML of a webpage using Puppeteer. See the documentation for details.
Get the title of a webpage using Puppeteer. See the documentation
Captures a screenshot of a page using Puppeteer. See the documentation
import { Kafka } from "kafkajs"
export default defineComponent({
props: {
kafka: {
type: "app",
app: "kafka",
}
},
async run({steps, $}) {
const kafka = new Kafka({
brokers: [`${this.kafka.$auth.host}:${this.kafka.$auth.port}`],
});
const consumer = kafka.consumer({ groupId: 'TestGroup' });
await consumer.connect()
await consumer.subscribe({ topic: 'SampleTopic', fromBeginning: true });
let consumedMessage = "";
const eachMessage = async function({ topic, partition, message }){
consumedMessage = message.value.toString();
return consumedMessage;
};
await consumer.run({
eachMessage,
});
const producer = kafka.producer();
await producer.connect()
await producer.send({
topic: 'SampleTopic',
messages: [
{ value: 'Welcome KafkaJS + Pipedream users! '+ new Date().toISOString() },
],
});
await producer.disconnect();
const data = await consumer.describeGroup();
return { consumedMessage, groupDescription: data };
},
})
Puppeteer is a Node.js library which provides a high-level API to control Chrome/Chromium over the DevTools Protocol. Puppeteer runs in headless mode on Chromium on Pipedream.
Using Puppeteer you can perform tasks including:
import { puppeteer } from '@pipedream/browsers';
export default defineComponent({
async run({steps, $}) {
const browser = await puppeteer.browser();
// Interact with the web page programmatically
// See Puppeeter's Page documentation for available methods:
// https://pptr.dev/api/puppeteer.page
const page = await browser.newPage();
await page.goto('https://pipedream.com/');
const title = await page.title();
const content = await page.content();
// The browser needs to be closed, otherwise the step will hang
await browser.close();
return { title, content }
},
})