import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform";
import googleDrive from "../../google_drive.app.mjs";
import { getListFilesOpts } from "../../common/utils.mjs";
import { GOOGLE_DRIVE_FOLDER_MIME_TYPE } from "../../common/constants.mjs";
import sampleEmit from "./test-event.mjs";
export default {
key: "google_drive-new-files-instant-polling",
name: "New Files (Polling)",
description: "Emit new event when a new file is added in your linked Google Drive",
version: "0.0.1",
type: "source",
dedupe: "unique",
props: {
googleDrive,
db: "$.service.db",
timer: {
type: "$.interface.timer",
default: {
intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL,
},
},
drive: {
propDefinition: [
googleDrive,
"watchedDrive",
],
description: "Defaults to My Drive. To select a [Shared Drive](https://support.google.com/a/users/answer/9310351) instead, select it from this list.",
optional: false,
},
folders: {
propDefinition: [
googleDrive,
"folderId",
({ drive }) => ({
drive,
baseOpts: {
q: `mimeType = '${GOOGLE_DRIVE_FOLDER_MIME_TYPE}' and trashed = false`,
},
}),
],
type: "string[]",
label: "Folders",
description: "The specific folder(s) to watch for new files. Leave blank to watch all files in the Drive.",
optional: true,
default: [],
},
},
hooks: {
async deploy() {
const driveId = this.getDriveId();
const startPageToken = await this.googleDrive.getPageToken(driveId);
this._setPageToken(startPageToken);
this._setLastRunTimestamp(Date.now());
const daysAgo = new Date();
daysAgo.setDate(daysAgo.getDate() - 30);
const timeString = daysAgo.toISOString();
const args = this.getListFilesOpts({
q: `mimeType != "application/vnd.google-apps.folder" and createdTime > "${timeString}" and trashed = false`,
orderBy: "createdTime desc",
fields: "*",
pageSize: 5,
});
const { files } = await this.googleDrive.listFilesInPage(null, args);
for (const file of files) {
if (this.shouldProcess(file)) {
await this.emitFile(file);
}
}
},
},
methods: {
_getPageToken() {
return this.db.get("pageToken");
},
_setPageToken(pageToken) {
this.db.set("pageToken", pageToken);
},
_getLastRunTimestamp() {
return this.db.get("lastRunTimestamp");
},
_setLastRunTimestamp(timestamp) {
this.db.set("lastRunTimestamp", timestamp);
},
getDriveId(drive = this.drive) {
return this.googleDrive.getDriveId(drive);
},
getListFilesOpts(args = {}) {
return getListFilesOpts(this.drive, {
q: "mimeType != 'application/vnd.google-apps.folder' and trashed = false",
...args,
});
},
shouldProcess(file) {
if (file.mimeType === GOOGLE_DRIVE_FOLDER_MIME_TYPE) {
return false;
}
if (this.folders?.length > 0) {
const watchedFolders = new Set(this.folders);
if (!file.parents || !file.parents.some((p) => watchedFolders.has(p))) {
return false;
}
}
return true;
},
generateMeta(file) {
const {
id: fileId,
name: summary,
createdTime: tsString,
} = file;
const ts = Date.parse(tsString);
return {
id: `${fileId}-${ts}`,
summary,
ts,
};
},
async emitFile(file) {
const meta = this.generateMeta(file);
this.$emit(file, meta);
},
},
async run() {
const currentRunTimestamp = Date.now();
const lastRunTimestamp = this._getLastRunTimestamp();
const pageToken = this._getPageToken();
const driveId = this.getDriveId();
const changedFilesStream = this.googleDrive.listChanges(pageToken, driveId);
for await (const changedFilesPage of changedFilesStream) {
console.log("Changed files page:", changedFilesPage);
const {
changedFiles,
nextPageToken,
} = changedFilesPage;
console.log(changedFiles.length
? `Processing ${changedFiles.length} changed files`
: "No changed files since last run");
for (const file of changedFiles) {
if (file.mimeType === GOOGLE_DRIVE_FOLDER_MIME_TYPE) {
continue;
}
const fullFile = await this.googleDrive.getFile(file.id, {
fields: "*",
});
const fileCreatedTime = Date.parse(fullFile.createdTime);
if (fileCreatedTime <= lastRunTimestamp) {
console.log(`Skipping existing file ${fullFile.name || fullFile.id}`);
continue;
}
if (!this.shouldProcess(fullFile)) {
console.log(`Skipping file ${fullFile.name || fullFile.id}`);
continue;
}
await this.emitFile(fullFile);
}
this._setPageToken(nextPageToken);
}
this._setLastRunTimestamp(currentRunTimestamp);
},
sampleEmit,
};
Chase Roberts@chsrbrts@benedictevans If you haven’t used @pipedream yet, then you haven’t lived.
✨Ellie Day✨@heyelliedayEvaluation update: @pipedream has quite literally been a dream to work with! I’m excited to leverage this tool for all the various workflows I need to write. I’m currently at 11k invocations a day from the initial workflows I’ve written in the past couple weeks.
Michael Braedley@MBraedleyUpdate: I got it working properly, and it's working so well that I'm dropping IFTTT. @pipedream can do everything that IFTTT basic can, and most (if not all things) IFTTT pro can for free or at a reasonable price if you need it. I am recommending it for basically any power user.
Thomas Cutting@mrthomascuttingWant quick+dirty integrations for a serverless workflow - @pipedream is my new go-to 😃
Matthew Roberts@mattdotrobertsday 013 - finally hit node js. This is the secret sauce of taking #nocode projects that one step further. Pumped about getting deeper into @pipedream now
Kenneth Auchenberg 💭@auchenbergYahoo Pipes is back! Kinda 😍 @pipedream
Raymond Camden 🥑@raymondcamdenAwesome video by the @pipedream folks showing real time twitter sentiment analysis integrated with Google Sheets. This is where Pipedream *really* shines, connecting systems together in easy workflows.
Nacho Caballero@nachocaballeroI couldn't recommend @pipedream more. It's an amazing service to integrate different APIs. Much more powerful than Zapier and more user-friendly than AWS Lambda. I'm very proud to wear this t-shirt #NoCode
Jason Snow@jyksnwDeveloped a working prototype environmental sensor IoT solution with @particle Photon, @pipedream, and @MongoDB with full graphing and alerting in less than a day! All amazing technology, will def. be exploring these more.
Steven Terrana@steven_terrana@burgwyn you've inspired me to finally set up my own blog. I'll make sure my first blog post explains the tech behind the setup. think @obsdmd + @GatsbyJS + @pipedream.
🚄 James Augeri, PhD@DotDotJamesWant to low-code your back end, need more horsepower than @Bubble / @KnackHQ, or just miss Yahoo! Pipes? Check out @PipeDream
Sébastien Chopin@AtinuxGitHub issues should be like @linear_app for maintainers. Looking forward more integrations with GH actions or tools like @pipedream 👀
Raul@raul_predescuIf you're a dev and not using @pipedream, you're missing out. Been using it for months, daily. FREE for devs. Plenty of integrations and good limits. Absolutely love it.
Bruno Skvorc@bitfallsSo @pipedream is pretty amazing. In 3 minutes I just made a flow which adds @rickastley's Never Gonna Give You Up to my @spotify playlist whenever a new pull request arrives in an old repo of mine.
Zach Lanich@ZachLanichUm, wow 🤯 @pipedream
Steven Bell@bellontechI just used @pipedream to build a Shopify App. Wow, they make small backed tasks easy.
Jay Hack 🎩🇺🇸@_jayhack_Very impressed with this bad boi - it reminds me of a @PalantirTech internal tool, but geared towards integrations instead of data analysis and far more customizable. Great expectations here 🚀🤩
Tree Sturgeon 🔥🚴♂️🌳@philsturgeonFor context this is day 2 of a really challenging and stupid migration from Notion to @airtable with disparate/missing data. It's going better than expected and thanks to @pipedream I don't have to bother the iOS dev to add W3W.