import sftpApp from "../../sftp.app.mjs";
import { DEFAULT_POLLING_SOURCE_TIMER_INTERVAL } from "@pipedream/platform";
export default {
key: "sftp-watch-remote-directory",
name: "New Remote Directory Watcher",
description: "Emit new events when files get created, changed or deleted from a remote directory. [See the docs](https://github.com/theophilusx/ssh2-sftp-client#orgfac43d1)",
version: "0.0.4",
type: "source",
dedupe: "unique",
props: {
sftpApp,
timer: {
type: "$.interface.timer",
default: {
intervalSeconds: DEFAULT_POLLING_SOURCE_TIMER_INTERVAL,
},
},
db: "$.service.db",
rootDirectory: {
label: "Root directory",
description: "Root directory to be watched. example: `/public`",
type: "string",
default: "/",
},
maxDepth: {
label: "Maximum watcher depth",
description: "Watch all subdirectories within of the root directory, considering the selected maximum depth.",
type: "integer",
min: 0,
},
},
methods: {
setFiles(files) {
this.db.set("files", files);
},
getFiles() {
return this.db.get("files") || [];
},
getChangesWithEvent(currentFiles, previousFiles) {
const fileChanges = [];
for (const prvFile of previousFiles) {
const file = currentFiles.find((p) => p.path === prvFile.path);
if (!file) {
fileChanges.push({
...prvFile,
event: "deleted",
});
}
}
for (const file of currentFiles) {
const prvFile = previousFiles.find((p) => p.path === file.path);
if (prvFile) {
if (prvFile.size === file.size && prvFile.modifyTime === file.modifyTime) {
continue;
}
fileChanges.push({
...file,
event: "updated",
});
} else {
fileChanges.push({
...file,
event: "created",
});
}
}
return fileChanges;
},
async listDirectories(sftp, parent, currDepth) {
const files = await sftp.list(parent);
return files.filter((file) => file.type === "d")
.map((directory) => ({
...directory,
parent,
path: `${parent}/${directory.name}`,
depth: currDepth,
}));
},
async listDirectoriesDeep(sftp, parent, maxDepth, currDepth) {
if (currDepth > maxDepth) {
return [];
}
const nextDepth = currDepth + 1;
const rootDirectories = await this.listDirectories(sftp, parent, currDepth);
const childrenDirectories = [];
for (const item of rootDirectories) {
const path = `${parent}/${item.name}`;
const directories = await this.listDirectoriesDeep(sftp, path, maxDepth, nextDepth);
childrenDirectories.push(...directories);
}
rootDirectories.push(...childrenDirectories);
return rootDirectories;
},
async listAllFilesFromDirectories(sftp, directories) {
const allFiles = [];
for (const directory of directories) {
const listingResult = await sftp.list(directory.path);
const files = listingResult.filter((file) => file.type !== "d")
.map((file) => ({
...file,
directory: directory.path,
path: `${directory.path}/${file.name}`,
depth: directory.depth || 0,
}));
allFiles.push(...files);
}
return allFiles;
},
validateRootDirectory(rootDirectory) {
if (!rootDirectory) {
throw new Error("Must provide root directory");
}
if (!rootDirectory.startsWith("/")) {
throw new Error("The root directory must to be the absolute path and start with a slash, such as: '/public'");
}
},
getEventId(event) {
return `${event.event}|${event.path}|${event.size}|${event.modifyTime}`;
},
emitEvents(events) {
for (const fileChangeEvent of events) {
this.$emit(fileChangeEvent, {
id: this.getEventId(fileChangeEvent),
summary: `${fileChangeEvent.event} ${fileChangeEvent.path}`,
ts: new Date(fileChangeEvent.modifyTime),
});
}
},
async connect() {
return await this.sftpApp.connect();
},
async disconnect(sftp) {
await sftp.end();
},
},
async run() {
this.validateRootDirectory(this.rootDirectory);
const sftp = await this.connect();
let directories = [
{
path: this.rootDirectory,
},
];
directories = directories.concat(
await this.listDirectoriesDeep(sftp, this.rootDirectory, this.maxDepth, 0),
);
const currentFiles = await this.listAllFilesFromDirectories(sftp, directories);
const previousFiles = this.getFiles();
const filesChangesWithEvent = this.getChangesWithEvent(currentFiles, previousFiles);
this.emitEvents(filesChangesWithEvent);
this.setFiles(currentFiles);
await this.disconnect(sftp);
},
};
Chase Roberts@chsrbrts@benedictevans If you haven’t used @pipedream yet, then you haven’t lived.
✨Ellie Day✨@heyelliedayEvaluation update: @pipedream has quite literally been a dream to work with! I’m excited to leverage this tool for all the various workflows I need to write. I’m currently at 11k invocations a day from the initial workflows I’ve written in the past couple weeks.
Michael Braedley@MBraedleyUpdate: I got it working properly, and it's working so well that I'm dropping IFTTT. @pipedream can do everything that IFTTT basic can, and most (if not all things) IFTTT pro can for free or at a reasonable price if you need it. I am recommending it for basically any power user.
Thomas Cutting@mrthomascuttingWant quick+dirty integrations for a serverless workflow - @pipedream is my new go-to 😃
Matthew Roberts@mattdotrobertsday 013 - finally hit node js. This is the secret sauce of taking #nocode projects that one step further. Pumped about getting deeper into @pipedream now
Kenneth Auchenberg 💭@auchenbergYahoo Pipes is back! Kinda 😍 @pipedream
Raymond Camden 🥑@raymondcamdenAwesome video by the @pipedream folks showing real time twitter sentiment analysis integrated with Google Sheets. This is where Pipedream *really* shines, connecting systems together in easy workflows.
Nacho Caballero@nachocaballeroI couldn't recommend @pipedream more. It's an amazing service to integrate different APIs. Much more powerful than Zapier and more user-friendly than AWS Lambda. I'm very proud to wear this t-shirt #NoCode
Jason Snow@jyksnwDeveloped a working prototype environmental sensor IoT solution with @particle Photon, @pipedream, and @MongoDB with full graphing and alerting in less than a day! All amazing technology, will def. be exploring these more.
Steven Terrana@steven_terrana@burgwyn you've inspired me to finally set up my own blog. I'll make sure my first blog post explains the tech behind the setup. think @obsdmd + @GatsbyJS + @pipedream.
🚄 James Augeri, PhD@DotDotJamesWant to low-code your back end, need more horsepower than @Bubble / @KnackHQ, or just miss Yahoo! Pipes? Check out @PipeDream
Sébastien Chopin@AtinuxGitHub issues should be like @linear_app for maintainers. Looking forward more integrations with GH actions or tools like @pipedream 👀
Raul@raul_predescuIf you're a dev and not using @pipedream, you're missing out. Been using it for months, daily. FREE for devs. Plenty of integrations and good limits. Absolutely love it.
Bruno Skvorc@bitfallsSo @pipedream is pretty amazing. In 3 minutes I just made a flow which adds @rickastley's Never Gonna Give You Up to my @spotify playlist whenever a new pull request arrives in an old repo of mine.
Zach Lanich@ZachLanichUm, wow 🤯 @pipedream
Steven Bell@bellontechI just used @pipedream to build a Shopify App. Wow, they make small backed tasks easy.
Jay Hack 🎩🇺🇸@_jayhack_Very impressed with this bad boi - it reminds me of a @PalantirTech internal tool, but geared towards integrations instead of data analysis and far more customizable. Great expectations here 🚀🤩
Tree Sturgeon 🔥🚴♂️🌳@philsturgeonFor context this is day 2 of a really challenging and stupid migration from Notion to @airtable with disparate/missing data. It's going better than expected and thanks to @pipedream I don't have to bother the iOS dev to add W3W.