Integrate the Automatic Data Extraction API with the MongoDB API

import { axios } from "@pipedream/platform"
export default defineComponent({
  props: {
    automatic_data_extraction: {
      type: "app",
      app: "automatic_data_extraction",
    }
  },
  async run({steps, $}) {
    const data = JSON.stringify([{
      'url': 'http://books.toscrape.com/catalogue/a-light-in-the-attic_1000/index.html',
      'pageType': 'product',
    }]);
    return await axios($, {
      method: "post",
      url: `https://autoextract.scrapinghub.com/v1/extract`,
      headers: {
        "Content-Type": `application/json`,
      },
      auth: {
        username: `${this.automatic_data_extraction.$auth.api_key}`,
        password: ``,
      },
      data,
    })
  },
})

import mongodb from 'mongodb'

export default defineComponent({
  props: {
    mongodb: {
      type: "app",
      app: "mongodb",
    },
    collection: {
      type: "string"
    },
    filter: {
      type: "object"
    }
  },
  async run({steps, $}) {
    const MongoClient = mongodb.MongoClient
    
    const {
      database,
      hostname,
      username,
      password,
    } = this.mongodb.$auth
    
    const url = `mongodb+srv://${username}:${password}@${hostname}/test?retryWrites=true&w=majority`
    const client = await MongoClient.connect(url, { 
      useNewUrlParser: true, 
      useUnifiedTopology: true 
    })
    
    const db = client.db(database)

    const results = await db.collection(this.collection).find(this.filter).toArray();
    $.export('results', results);
    
    await client.close()
  },
})