import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform";
import googleDrive from "../../google_drive.app.mjs";
import { getListFilesOpts } from "../../common/utils.mjs";
import sampleEmit from "./test-event.mjs";
import { GOOGLE_DRIVE_FOLDER_MIME_TYPE } from "../../common/constants.mjs";
export default {
key: "google_drive-new-or-modified-files-polling",
name: "New or Modified Files (Polling)",
description: "Emit new event when a file in the selected Drive is created, modified or trashed. [See the documentation](https://developers.google.com/drive/api/v3/reference/changes/list)",
version: "0.0.1",
type: "source",
dedupe: "unique",
props: {
googleDrive,
db: "$.service.db",
timer: {
type: "$.interface.timer",
default: {
intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL,
},
},
drive: {
propDefinition: [
googleDrive,
"watchedDrive",
],
description: "Defaults to My Drive. To select a [Shared Drive](https://support.google.com/a/users/answer/9310351) instead, select it from this list.",
optional: false,
},
files: {
type: "string[]",
label: "Files",
description: "The specific file(s) you want to watch for changes. Leave blank to watch all files in the Drive. Note that events will only be emitted for files directly in these folders (not in their subfolders, unless also selected here)",
optional: true,
default: [],
options({ prevContext }) {
const { nextPageToken } = prevContext;
return this.googleDrive.listFilesOptions(nextPageToken, this.getListFilesOpts());
},
},
folders: {
type: "string[]",
label: "Folders",
description: "The specific folder(s) to watch for changes. Leave blank to watch all folders in the Drive.",
optional: true,
default: [],
options({ prevContext }) {
const { nextPageToken } = prevContext;
const baseOpts = {
q: "mimeType = 'application/vnd.google-apps.folder' and trashed = false",
};
const isMyDrive = this.googleDrive.isMyDrive(this.drive);
const opts = isMyDrive
? baseOpts
: {
...baseOpts,
corpora: "drive",
driveId: this.getDriveId(),
includeItemsFromAllDrives: true,
supportsAllDrives: true,
};
return this.googleDrive.listFilesOptions(nextPageToken, opts);
},
},
newFilesOnly: {
type: "boolean",
label: "New Files Only",
description: "If enabled, only emit events for newly created files (based on `createdTime`)",
default: false,
optional: true,
},
},
hooks: {
async deploy() {
const driveId = this.getDriveId();
const startPageToken = await this.googleDrive.getPageToken(driveId);
this._setPageToken(startPageToken);
this._setLastRunTimestamp(Date.now());
const daysAgo = new Date();
daysAgo.setDate(daysAgo.getDate() - 30);
const timeString = daysAgo.toISOString();
const args = this.getListFilesOpts({
q: `mimeType != "application/vnd.google-apps.folder" and modifiedTime > "${timeString}" and trashed = false`,
orderBy: "modifiedTime desc",
fields: "*",
pageSize: 5,
});
const { files } = await this.googleDrive.listFilesInPage(null, args);
for (const file of files) {
if (this.shouldProcess(file)) {
await this.emitFile(file);
}
}
},
},
methods: {
_getPageToken() {
return this.db.get("pageToken");
},
_setPageToken(pageToken) {
this.db.set("pageToken", pageToken);
},
_getLastRunTimestamp() {
return this.db.get("lastRunTimestamp");
},
_setLastRunTimestamp(timestamp) {
this.db.set("lastRunTimestamp", timestamp);
},
getDriveId(drive = this.drive) {
return this.googleDrive.getDriveId(drive);
},
getListFilesOpts(args = {}) {
return getListFilesOpts(this.drive, {
q: "mimeType != 'application/vnd.google-apps.folder' and trashed = false",
...args,
});
},
shouldProcess(file, lastRunTimestamp) {
if (file.mimeType === GOOGLE_DRIVE_FOLDER_MIME_TYPE) {
return false;
}
if (this.newFilesOnly) {
const fileCreatedTime = Date.parse(file.createdTime);
if (fileCreatedTime <= lastRunTimestamp) {
return false;
}
}
if (this.files?.length > 0) {
if (!this.files.includes(file.id)) {
return false;
}
}
if (this.folders?.length > 0) {
const watchedFolders = new Set(this.folders);
if (!file.parents || !file.parents.some((p) => watchedFolders.has(p))) {
return false;
}
}
return true;
},
generateMeta(file) {
const {
id: fileId,
name: summary,
modifiedTime: tsString,
} = file;
const ts = Date.parse(tsString);
return {
id: `${fileId}-${ts}`,
summary,
ts,
};
},
async emitFile(file) {
const meta = this.generateMeta(file);
this.$emit(file, meta);
},
},
async run() {
const currentRunTimestamp = Date.now();
const lastRunTimestamp = this._getLastRunTimestamp();
const pageToken = this._getPageToken();
const driveId = this.getDriveId();
const changedFilesStream = this.googleDrive.listChanges(pageToken, driveId);
for await (const changedFilesPage of changedFilesStream) {
console.log("Changed files page:", changedFilesPage);
const {
changedFiles,
nextPageToken,
} = changedFilesPage;
console.log(changedFiles.length
? `Processing ${changedFiles.length} changed files`
: "No changed files since last run");
for (const file of changedFiles) {
const fullFile = await this.googleDrive.getFile(file.id, {
fields: "*",
});
if (!this.shouldProcess(fullFile, lastRunTimestamp)) {
console.log(`Skipping file ${fullFile.name || fullFile.id}`);
continue;
}
await this.emitFile(fullFile);
}
this._setPageToken(nextPageToken);
}
this._setLastRunTimestamp(currentRunTimestamp);
},
sampleEmit,
};
Chase Roberts@chsrbrts@benedictevans If you haven’t used @pipedream yet, then you haven’t lived.
✨Ellie Day✨@heyelliedayEvaluation update: @pipedream has quite literally been a dream to work with! I’m excited to leverage this tool for all the various workflows I need to write. I’m currently at 11k invocations a day from the initial workflows I’ve written in the past couple weeks.
Michael Braedley@MBraedleyUpdate: I got it working properly, and it's working so well that I'm dropping IFTTT. @pipedream can do everything that IFTTT basic can, and most (if not all things) IFTTT pro can for free or at a reasonable price if you need it. I am recommending it for basically any power user.
Thomas Cutting@mrthomascuttingWant quick+dirty integrations for a serverless workflow - @pipedream is my new go-to 😃
Matthew Roberts@mattdotrobertsday 013 - finally hit node js. This is the secret sauce of taking #nocode projects that one step further. Pumped about getting deeper into @pipedream now
Kenneth Auchenberg 💭@auchenbergYahoo Pipes is back! Kinda 😍 @pipedream
Raymond Camden 🥑@raymondcamdenAwesome video by the @pipedream folks showing real time twitter sentiment analysis integrated with Google Sheets. This is where Pipedream *really* shines, connecting systems together in easy workflows.
Nacho Caballero@nachocaballeroI couldn't recommend @pipedream more. It's an amazing service to integrate different APIs. Much more powerful than Zapier and more user-friendly than AWS Lambda. I'm very proud to wear this t-shirt #NoCode
Jason Snow@jyksnwDeveloped a working prototype environmental sensor IoT solution with @particle Photon, @pipedream, and @MongoDB with full graphing and alerting in less than a day! All amazing technology, will def. be exploring these more.
Steven Terrana@steven_terrana@burgwyn you've inspired me to finally set up my own blog. I'll make sure my first blog post explains the tech behind the setup. think @obsdmd + @GatsbyJS + @pipedream.
🚄 James Augeri, PhD@DotDotJamesWant to low-code your back end, need more horsepower than @Bubble / @KnackHQ, or just miss Yahoo! Pipes? Check out @PipeDream
Sébastien Chopin@AtinuxGitHub issues should be like @linear_app for maintainers. Looking forward more integrations with GH actions or tools like @pipedream 👀
Raul@raul_predescuIf you're a dev and not using @pipedream, you're missing out. Been using it for months, daily. FREE for devs. Plenty of integrations and good limits. Absolutely love it.
Bruno Skvorc@bitfallsSo @pipedream is pretty amazing. In 3 minutes I just made a flow which adds @rickastley's Never Gonna Give You Up to my @spotify playlist whenever a new pull request arrives in an old repo of mine.
Zach Lanich@ZachLanichUm, wow 🤯 @pipedream
Steven Bell@bellontechI just used @pipedream to build a Shopify App. Wow, they make small backed tasks easy.
Jay Hack 🎩🇺🇸@_jayhack_Very impressed with this bad boi - it reminds me of a @PalantirTech internal tool, but geared towards integrations instead of data analysis and far more customizable. Great expectations here 🚀🤩
Tree Sturgeon 🔥🚴♂️🌳@philsturgeonFor context this is day 2 of a really challenging and stupid migration from Notion to @airtable with disparate/missing data. It's going better than expected and thanks to @pipedream I don't have to bother the iOS dev to add W3W.