with FireCrawl and Google Identity?
Crawls a given URL and returns the contents of sub-pages. See the documentation
Extract structured data from one or multiple URLs. See the documentation
Obtains the status and data from a previous crawl operation. See the documentation
Obtains the status and data from a previous extract operation. See the documentation
Scrapes a URL and returns content from that page. See the documentation
import { axios } from "@pipedream/platform"
export default defineComponent({
props: {
firecrawl: {
type: "app",
app: "firecrawl",
}
},
async run({steps, $}) {
const data = {
"url": "https://pipedream.com",
}
return await axios($, {
method: "post",
url: `https://api.firecrawl.dev/v0/crawl`,
headers: {
Authorization: `Bearer ${this.firecrawl.$auth.api_key}`,
},
data,
})
},
})
import axios from "axios";
import * as jsrsasign from "jsrsasign";
export default defineComponent({
props: {
google_identity: {
type: "app",
app: "google_identity",
}
},
async run({ steps, $ }) {
const serviceAccountKey = JSON.parse(this.google_identity.$auth.service_account_key);
const GOOGLE_OAUTH = 'https://www.googleapis.com/oauth2/v4/token';
// add/remove your own scopes as needed
const SCOPES = [
'https://www.googleapis.com/auth/userinfo.email',
'https://www.googleapis.com/auth/userinfo.profile',
'https://www.googleapis.com/auth/cloud-platform',
'https://www.googleapis.com/auth/cloud-identity',
];
const getJwt = async function ({ client_email, private_key }, iat) {
const header = {
typ: 'JWT',
alg: 'RS256',
};
const exp = iat + 3600;
const payload = {
aud: GOOGLE_OAUTH,
iss: client_email,
scope: SCOPES.join(' '),
iat,
exp,
};
const jwt = jsrsasign.KJUR.jws.JWS.sign(null, header, payload, private_key);
return { jwt, exp };
};
const getToken = async function (serviceAccountKey) {
const now = Math.floor(Date.now() / 1000);
const jwtInfo = await getJwt(serviceAccountKey, now);
const params = new URLSearchParams();
params.append('grant_type', 'urn:ietf:params:oauth:grant-type:jwt-bearer');
params.append('assertion', jwtInfo.jwt);
const response = await axios.post(GOOGLE_OAUTH, params, {
headers: {
'Content-Type': 'application/x-www-form-urlencoded'
}
});
const accessToken = response.data.access_token;
return accessToken;
};
const generateOAuthToken = async function () {
return await getToken(serviceAccountKey);
};
// Call the function to generate the token
const accessToken = await generateOAuthToken();
// Call the Google Identity API to search for groups
const response = await axios({
method: "GET",
url: "https://cloudidentity.googleapis.com/v1/groups:search",
headers: {
Authorization: `Bearer ${accessToken}`,
"Content-Type": "application/json"
},
params: {
query: "parent == 'customers/C046psxkn'"
}
})
return response.data
},
})