import crypto from "crypto";
import { isString } from "lodash-es";
import googleCloud from "../../google_cloud.app.mjs";
import common from "../common/bigquery.mjs";
export default {
...common,
key: "google_cloud-bigquery-new-row",
name: "BigQuery - New Row",
description: "Emit new events when a new row is added to a table",
version: "0.1.6",
dedupe: "unique",
type: "source",
props: {
...common.props,
tableId: {
propDefinition: [
googleCloud,
"tableId",
({ datasetId }) => ({
datasetId,
}),
],
},
uniqueKey: {
type: "string",
label: "Unique Key",
description: "The name of a column in the table to use for deduplication. See [the docs](https://github.com/PipedreamHQ/pipedream/tree/master/components/google_cloud/sources/bigquery-new-row#technical-details) for more info.",
async options(context) {
const { page } = context;
if (page !== 0) {
return [];
}
const columnNames = await this._getColumnNames();
return columnNames.sort();
},
},
},
hooks: {
...common.hooks,
async deploy() {
await this._validateColumn(this.uniqueKey);
const lastResultId = await this._getIdOfLastRow(this.getInitialEventCount());
this._setLastResultId(lastResultId);
},
async activate() {
if (this._getLastResultId()) {
return;
}
await this._validateColumn(this.uniqueKey);
const lastResultId = await this._getIdOfLastRow();
this._setLastResultId(lastResultId);
},
deactivate() {
this._setLastResultId(null);
},
},
methods: {
...common.methods,
_getLastResultId() {
return this.db.get("lastResultId");
},
_setLastResultId(lastResultId) {
this.db.set("lastResultId", lastResultId);
console.log(`
Next scan of table '${this.tableId}' will start at ${this.uniqueKey}=${lastResultId}
`);
},
async _validateColumn(columnNameToValidate) {
if (!isString(columnNameToValidate)) {
throw new Error("columnNameToValidate must be a string");
}
const columnNames = await this._getColumnNames();
if (!columnNames.includes(columnNameToValidate)) {
throw new Error(`Nonexistent column: ${columnNameToValidate}`);
}
},
async _getColumnNames() {
const table = this.googleCloud
.getBigQueryClient()
.dataset(this.datasetId)
.table(this.tableId);
const [
metadata,
] = await table.getMetadata();
const { fields } = metadata.schema;
return fields.map(({ name }) => name);
},
async _getIdOfLastRow(offset = 0) {
const limit = offset + 1;
const query = `
SELECT *
FROM \`${this.tableId}\`
ORDER BY \`${this.uniqueKey}\` DESC
LIMIT @limit
`;
const queryOpts = {
query,
params: {
limit,
},
};
const rows = await this.getRowsForQuery(queryOpts, this.datasetId);
if (rows.length === 0) {
console.log(`
No records found in the target table, will start scanning from the beginning
`);
return;
}
const startingRow = rows.pop();
return startingRow[this.uniqueKey];
},
getQueryOpts() {
const lastResultId = this._getLastResultId();
const query = `
SELECT *
FROM \`${this.tableId}\`
WHERE \`${this.uniqueKey}\` >= @lastResultId
ORDER BY \`${this.uniqueKey}\` ASC
`;
const params = {
lastResultId,
};
return {
query,
params,
};
},
generateMeta(row, ts) {
const id = row[this.uniqueKey];
const summary = `New row: ${id}`;
return {
id,
summary,
ts,
};
},
generateMetaForCollection(rows, ts) {
const hash = crypto.createHash("sha1");
rows
.map((i) => i[this.uniqueKey])
.map((i) => i.toString())
.forEach((i) => hash.update(i));
const id = hash.digest("base64");
const rowCount = rows.length;
const entity = rowCount === 1
? "row"
: "rows";
const summary = `${rowCount} new ${entity}`;
return {
id,
summary,
ts,
};
},
},
};
Chase Roberts@chsrbrts@benedictevans If you haven’t used @pipedream yet, then you haven’t lived.
✨Ellie Day✨@heyelliedayEvaluation update: @pipedream has quite literally been a dream to work with! I’m excited to leverage this tool for all the various workflows I need to write. I’m currently at 11k invocations a day from the initial workflows I’ve written in the past couple weeks.
Michael Braedley@MBraedleyUpdate: I got it working properly, and it's working so well that I'm dropping IFTTT. @pipedream can do everything that IFTTT basic can, and most (if not all things) IFTTT pro can for free or at a reasonable price if you need it. I am recommending it for basically any power user.
Thomas Cutting@mrthomascuttingWant quick+dirty integrations for a serverless workflow - @pipedream is my new go-to 😃
Matthew Roberts@mattdotrobertsday 013 - finally hit node js. This is the secret sauce of taking #nocode projects that one step further. Pumped about getting deeper into @pipedream now
Kenneth Auchenberg 💭@auchenbergYahoo Pipes is back! Kinda 😍 @pipedream
Raymond Camden 🥑@raymondcamdenAwesome video by the @pipedream folks showing real time twitter sentiment analysis integrated with Google Sheets. This is where Pipedream *really* shines, connecting systems together in easy workflows.
Nacho Caballero@nachocaballeroI couldn't recommend @pipedream more. It's an amazing service to integrate different APIs. Much more powerful than Zapier and more user-friendly than AWS Lambda. I'm very proud to wear this t-shirt #NoCode
Jason Snow@jyksnwDeveloped a working prototype environmental sensor IoT solution with @particle Photon, @pipedream, and @MongoDB with full graphing and alerting in less than a day! All amazing technology, will def. be exploring these more.
Steven Terrana@steven_terrana@burgwyn you've inspired me to finally set up my own blog. I'll make sure my first blog post explains the tech behind the setup. think @obsdmd + @GatsbyJS + @pipedream.
🚄 James Augeri, PhD@DotDotJamesWant to low-code your back end, need more horsepower than @Bubble / @KnackHQ, or just miss Yahoo! Pipes? Check out @PipeDream
Sébastien Chopin@AtinuxGitHub issues should be like @linear_app for maintainers. Looking forward more integrations with GH actions or tools like @pipedream 👀
Raul@raul_predescuIf you're a dev and not using @pipedream, you're missing out. Been using it for months, daily. FREE for devs. Plenty of integrations and good limits. Absolutely love it.
Bruno Skvorc@bitfallsSo @pipedream is pretty amazing. In 3 minutes I just made a flow which adds @rickastley's Never Gonna Give You Up to my @spotify playlist whenever a new pull request arrives in an old repo of mine.
Zach Lanich@ZachLanichUm, wow 🤯 @pipedream
Steven Bell@bellontechI just used @pipedream to build a Shopify App. Wow, they make small backed tasks easy.
Jay Hack 🎩🇺🇸@_jayhack_Very impressed with this bad boi - it reminds me of a @PalantirTech internal tool, but geared towards integrations instead of data analysis and far more customizable. Great expectations here 🚀🤩
Tree Sturgeon 🔥🚴♂️🌳@philsturgeonFor context this is day 2 of a really challenging and stupid migration from Notion to @airtable with disparate/missing data. It's going better than expected and thanks to @pipedream I don't have to bother the iOS dev to add W3W.