import app from "../../lamini.app.mjs";
import constants from "../../common/constants.mjs";
import utils from "../../common/utils.mjs";
export default {
key: "lamini-create-fine-tune-job",
name: "Create Fine-Tune Job",
description: "Create a fine-tuning job with a dataset. [See the documentation](https://docs.lamini.ai/api/).",
version: "0.0.2",
type: "action",
props: {
app,
modelName: {
description: "Base model to be fine-tuned.",
propDefinition: [
app,
"modelName",
() => ({
includeFineTunedModels: false,
}),
],
},
datasetId: {
type: "string",
label: "Dataset ID",
description: "Previously uploaded dataset to use for training. Please use the **Upload Dataset** action to upload a dataset.",
},
fineTuneArgs: {
type: "object",
label: "Finetune Arguments",
description: "Optional hyperparameters for fine-tuning. Each property is optional:\n- `index_pq_m`: Number of subquantizers for PQ (eg. 8)\n- `index_max_size`: Maximum index size (eg. 65536)\n- `max_steps`: Maximum number of training steps (eg. 60)\n- `batch_size`: Training batch size (eg. 1)\n- `learning_rate`: Learning rate (eg. 0.0003)\n- `index_pq_nbits`: Number of bits per subquantizer (eg. 8)\n- `max_length`: Maximum sequence length (eg. 2048)\n- `index_ivf_nlist`: Number of IVF lists (eg. 2048)\n- `save_steps`: Steps between checkpoints (eg. 60)\n- `args_name`: Name for the argument set (eg. \"demo\")\n- `r_value`: R value for LoRA (eg. 32)\n- `index_hnsw_m`: Number of neighbors in HNSW (eg. 32)\n- `index_method`: Indexing method (eg. \"IndexIVFPQ\")\n- `optim`: Optimizer to use (eg. \"adafactor\")\n- `index_hnsw_efConstruction`: HNSW construction parameter (eg. 16)\n- `index_hnsw_efSearch`: HNSW search parameter (eg. 8)\n- `index_k`: Number of nearest neighbors (eg. 2)\n- `index_ivf_nprobe`: Number of IVF probes (eg. 48)\n- `eval_steps`: Steps between evaluations (eg. 30)\n[See the documentation](https://docs.lamini.ai/tuning/hyperparameters/#finetune_args).",
optional: true,
},
gpuConfig: {
type: "object",
label: "GPU Config",
description: "Optional GPU configuration for fine-tuning. [See the documentation](https://docs.lamini.ai/tuning/hyperparameters/#gpu_config).",
optional: true,
},
isPublic: {
type: "boolean",
label: "Is Public",
description: "Whether this fine-tuning job and dataset should be publicly accessible.",
optional: true,
},
customModelName: {
type: "string",
label: "Custom Model Name",
description: "A human-readable name for the fine-tuned model.",
optional: true,
},
waitForCompletion: {
type: "boolean",
label: "Wait for Completion",
description: "If set to `true`, the action will wait and poll until the fine-tuning job is `COMPLETED`. If is set to `false`, it will return immediately after creating the job.",
default: false,
optional: true,
},
},
methods: {
createFineTuneJob(args = {}) {
return this.app.post({
versionPath: constants.VERSION_PATH.V1,
path: "/train",
...args,
});
},
},
async run({ $ }) {
const {
app,
createFineTuneJob,
modelName,
datasetId,
fineTuneArgs,
gpuConfig,
isPublic,
customModelName,
waitForCompletion,
} = this;
const MAX_RETRIES = 15;
const DELAY = 1000 * 30;
const { run } = $.context;
if (run.runs === 1) {
const { upload_base_path: uploadBasePath } =
await app.getUploadBasePath({
$,
});
await app.getExistingDataset({
$,
data: {
dataset_id: datasetId,
upload_base_path: uploadBasePath,
},
});
const response = await createFineTuneJob({
$,
data: {
model_name: modelName,
dataset_id: datasetId,
upload_file_path: `${uploadBasePath}/${datasetId}.jsonlines`,
finetune_args: utils.parseJson(fineTuneArgs),
gpu_config: utils.parseJson(gpuConfig),
is_public: isPublic,
custom_model_name: customModelName,
},
});
$.export("$summary", `Successfully created a fine-tune job with ID \`${response.job_id}\`.`);
if (!waitForCompletion) {
return response;
}
$.flow.rerun(DELAY, {
jobId: response.job_id,
}, MAX_RETRIES);
return response;
}
if (run.runs > MAX_RETRIES) {
throw new Error("Max retries exceeded - fine-tuning job may still be running");
}
const { jobId } = run.context;
const statusResponse = await app.getJobStatus({
$,
jobId,
});
if (statusResponse.status === "COMPLETED") {
$.export("$summary", `Fine-tuning job \`${jobId}\` completed successfully.`);
return statusResponse;
}
if (statusResponse.status === "FAILED") {
throw new Error(`Fine-tuning job \`${jobId}\` failed.`);
}
$.flow.rerun(DELAY, {
jobId,
}, MAX_RETRIES);
return {
status: statusResponse.status,
jobId,
message: `Job is still running. Current status: ${statusResponse.status}`,
};
},
};
Chase Roberts@chsrbrts@benedictevans If you haven’t used @pipedream yet, then you haven’t lived.
✨Ellie Day✨@heyelliedayEvaluation update: @pipedream has quite literally been a dream to work with! I’m excited to leverage this tool for all the various workflows I need to write. I’m currently at 11k invocations a day from the initial workflows I’ve written in the past couple weeks.
Michael Braedley@MBraedleyUpdate: I got it working properly, and it's working so well that I'm dropping IFTTT. @pipedream can do everything that IFTTT basic can, and most (if not all things) IFTTT pro can for free or at a reasonable price if you need it. I am recommending it for basically any power user.
Thomas Cutting@mrthomascuttingWant quick+dirty integrations for a serverless workflow - @pipedream is my new go-to 😃
Matthew Roberts@mattdotrobertsday 013 - finally hit node js. This is the secret sauce of taking #nocode projects that one step further. Pumped about getting deeper into @pipedream now
Kenneth Auchenberg 💭@auchenbergYahoo Pipes is back! Kinda 😍 @pipedream
Raymond Camden 🥑@raymondcamdenAwesome video by the @pipedream folks showing real time twitter sentiment analysis integrated with Google Sheets. This is where Pipedream *really* shines, connecting systems together in easy workflows.
Nacho Caballero@nachocaballeroI couldn't recommend @pipedream more. It's an amazing service to integrate different APIs. Much more powerful than Zapier and more user-friendly than AWS Lambda. I'm very proud to wear this t-shirt #NoCode
Jason Snow@jyksnwDeveloped a working prototype environmental sensor IoT solution with @particle Photon, @pipedream, and @MongoDB with full graphing and alerting in less than a day! All amazing technology, will def. be exploring these more.
Steven Terrana@steven_terrana@burgwyn you've inspired me to finally set up my own blog. I'll make sure my first blog post explains the tech behind the setup. think @obsdmd + @GatsbyJS + @pipedream.
🚄 James Augeri, PhD@DotDotJamesWant to low-code your back end, need more horsepower than @Bubble / @KnackHQ, or just miss Yahoo! Pipes? Check out @PipeDream
Sébastien Chopin@AtinuxGitHub issues should be like @linear_app for maintainers. Looking forward more integrations with GH actions or tools like @pipedream 👀
Raul@raul_predescuIf you're a dev and not using @pipedream, you're missing out. Been using it for months, daily. FREE for devs. Plenty of integrations and good limits. Absolutely love it.
Bruno Skvorc@bitfallsSo @pipedream is pretty amazing. In 3 minutes I just made a flow which adds @rickastley's Never Gonna Give You Up to my @spotify playlist whenever a new pull request arrives in an old repo of mine.
Zach Lanich@ZachLanichUm, wow 🤯 @pipedream
Steven Bell@bellontechI just used @pipedream to build a Shopify App. Wow, they make small backed tasks easy.
Jay Hack 🎩🇺🇸@_jayhack_Very impressed with this bad boi - it reminds me of a @PalantirTech internal tool, but geared towards integrations instead of data analysis and far more customizable. Great expectations here 🚀🤩
Tree Sturgeon 🔥🚴♂️🌳@philsturgeonFor context this is day 2 of a really challenging and stupid migration from Notion to @airtable with disparate/missing data. It's going better than expected and thanks to @pipedream I don't have to bother the iOS dev to add W3W.