workflow and speech job support

README.md

@@ -20,14 +20,15 @@ npm install fluent-ai zod@next
 
 fluent-ai includes support for multiple AI providers and modalities.
 
-| provider  | chat completion    | embedding          | image generation   | list models        |
-| --------- | ------------------ | ------------------ | ------------------ | ------------------ |
-| anthropic | :white_check_mark: |                    |                    | :white_check_mark: |
-| fal       |                    |                    | :white_check_mark: |                    |
-| google    | :white_check_mark: |                    |                    |                    |
-| ollama    | :white_check_mark: | :white_check_mark: |                    | :white_check_mark: |
-| openai    | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |
-| voyage    |                    | :white_check_mark: |                    |                    |
+| provider   | chat completion    | embedding          | image generation   | list models        | text to speech     |
+| ---------- | ------------------ | ------------------ | ------------------ | ------------------ | ------------------ |
+| anthropic  | :white_check_mark: |                    |                    | :white_check_mark: |                    |
+| elevenlabs |                    |                    |                    |                    | :white_check_mark: |
+| fal        |                    |                    | :white_check_mark: |                    |                    |
+| google     | :white_check_mark: |                    |                    |                    |                    |
+| ollama     | :white_check_mark: | :white_check_mark: |                    | :white_check_mark: |                    |
+| openai     | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: |
+| voyage     |                    | :white_check_mark: |                    |                    |                    |
 
 By default, API keys for providers are read from environment variable (`process.env`) following the format `<PROVIDER>_API_KEY` (e.g., `OPENAI_API_KEY`, `ANTHROPIC_API_KEY`).
 
@@ -193,6 +194,15 @@ import { openai } from "fluent-ai";
 const models = await openai().models().run();
 ```
 
+## Text to Speech
+
+```ts
+import { openai } from "fluent-ai";
+
+const job = openai().model("tts-1").text("hi");
+const result = await job.run();
+```
+
 ## Support
 
 Feel free to [open an issue](https://github.com/modalityml/fluent-ai/issues) or [start a discussion](https://github.com/modalityml/fluent-ai/discussions) if you have any questions. If you would like to request support for a new AI provider, please create an issue with details about the provider's API. [Join our Discord community](https://discord.gg/HzGZWbY8Fx) for help and updates.

bun.lock

@@ -10,6 +10,7 @@
       "devDependencies": {
         "@types/bun": "latest",
         "bun-plugin-dts": "^0.3.0",
+        "prettier": "^3.5.3",
       },
       "peerDependencies": {
         "typescript": "^5.0.0",
@@ -58,6 +59,8 @@
 
     "partial-json": ["[email protected]", "", {}, "sha512-Njv/59hHaokb/hRUjce3Hdv12wd60MtM9Z5Olmn+nehe0QDAsRtRbJPvJ0Z91TusF0SuZRIvnM+S4l6EIP8leA=="],
 
+    "prettier": ["[email protected]", "", { "bin": { "prettier": "bin/prettier.cjs" } }, "sha512-QQtaxnoDJeAkDvDKWCLiwIXkTgRhwYDEQCghU9Z6q03iyek/rxRh/2lC3HB7P8sWT2xC/y5JDctPLBIGzHKbhw=="],
+
     "require-directory": ["[email protected]", "", {}, "sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q=="],
 
     "resolve-pkg-maps": ["[email protected]", "", {}, "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw=="],

examples/openai-chat-stream.ts

@@ -2,9 +2,8 @@ import { text, openai } from "../src";
 
 const job = openai()
   .chat("gpt-4o-mini")
-  .messages([{ role: "user", content: "generate a 50 words text" }])
+  .prompt("generate a 50 words text")
   .stream();
-const stream = await job.run();
-for await (const chunk of stream) {
-  process.stdout.write(text(chunk));
+for await (const chunk of job) {
+  console.log(chunk?.message);
 }

examples/openai-chat-tool-stream.ts

@@ -7,7 +7,7 @@ const weatherTool = tool("get_current_weather")
     z.object({
       location: z.string(),
       unit: z.enum(["celsius", "fahrenheit"]).optional(),
-    })
+    }),
   );
 const job = openai()
   .chat("gpt-4o-mini")

examples/openai-chat-tool.ts

@@ -7,7 +7,7 @@ const weatherTool = tool("get_current_weather")
     z.object({
       location: z.string(),
       unit: z.enum(["celsius", "fahrenheit"]).optional(),
-    })
+    }),
   );
 const job = openai()
   .chat("gpt-4o-mini")

examples/openai-chat.ts

@@ -4,4 +4,4 @@ const job = openai({})
   .chat("gpt-4o-mini")
   .messages([system("you are a helpful assistant"), user("hi")]);
 const result = await job.run();
-console.log(text(result));
+console.log(result?.message);

examples/openai-embedding.ts

@@ -2,4 +2,4 @@ import { openai } from "../src";
 
 const job = openai().embedding("text-embedding-3-small").value("hello");
 const result = await job.run();
-console.log(result.embedding);
+console.log(result!.embedding);

examples/openai-image-edit.ts

@@ -0,0 +1,14 @@
+import { openai } from "../src";
+import { readFileSync, writeFileSync } from "node:fs";
+
+const job = openai()
+  .image("gpt-image-1") // TODO: add support for dall-e-2
+  .edit(
+    new File([readFileSync("./cat.jpg")], "cat.jpg", { type: "image/jpeg" }),
+  )
+  .prompt("add a hat to the cat")
+  .size("1024x1024");
+
+const result = await job.run();
+const buffer = Buffer.from(result!.raw.data[0].b64_json, "base64");
+writeFileSync("cat_edit.jpg", buffer);

examples/openai-image.ts

@@ -1,9 +1,13 @@
 import { openai } from "../src";
+import { writeFileSync } from "node:fs";
 
 const job = openai()
-  .image("dalle-2")
+  .image("dall-e-2")
   .prompt("a cat")
-  .size({ width: 512, height: 512 });
-const result = await job.run();
+  .size("512x512")
+  .outputFormat("jpeg")
+  .responseFormat("b64_json");
 
-console.log(result);
+const result = await job.run();
+const buffer = Buffer.from(result!.raw.data[0].b64_json, "base64");
+writeFileSync("cat.jpg", buffer);

examples/openai-models.ts

@@ -2,5 +2,6 @@ import { openai } from "../src";
 
 const job = openai().models();
 const result = await job.run();
-
-console.log(result);
+for (const model of result!) {
+  console.log(model);
+}

examples/workflow.ts

@@ -0,0 +1,31 @@
+import { z } from "zod";
+import { openai, workflow } from "../src";
+
+const flow = workflow("workflow1")
+  .input(
+    z.object({
+      description: z.string(),
+    }),
+  )
+  .step("step1", ({ context }) => {
+    return openai()
+      .chat("gpt-4o-mini")
+      .prompt(
+        `generate a story based on following description: ${context.input.description}`,
+      )
+      .jsonSchema(
+        z.object({
+          story: z.string(),
+        }),
+      );
+  })
+  .step("step2", ({ context }) => {
+    return elevenlabs()
+      .tts("eleven_multilingual_v2")
+      .text(context.steps.step1.story);
+  });
+
+const result = await flow.run({
+  input: { description: "fire engine and a cat" },
+});
+console.log(result);

package.json

@@ -33,7 +33,8 @@
   ],
   "devDependencies": {
     "@types/bun": "latest",
-    "bun-plugin-dts": "^0.3.0"
+    "bun-plugin-dts": "^0.3.0",
+    "prettier": "^3.5.3"
   },
   "peerDependencies": {
     "typescript": "^5.0.0",
@@ -43,5 +44,8 @@
   "repository": {
     "type": "git",
     "url": "git+https://github.com/modalityml/fluent-ai.git"
+  },
+  "prettier": {
+    "trailingComma": "all"
   }
 }

src/client.ts

@@ -0,0 +1,70 @@
+import type { Job } from "./jobs/load";
+
+export interface ClientOptions {
+  url: string;
+  apiKey: string;
+}
+
+export class Client {
+  url: string;
+  apiKey: string;
+
+  constructor(options: ClientOptions) {
+    this.url = options.url;
+    this.apiKey = options.apiKey;
+  }
+
+  async createJob(job: Job) {
+    // TODO: reuse fetch error handling
+    const response = await fetch(`${this.url}/api/jobs`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${this.apiKey}`,
+      },
+      body: JSON.stringify(job),
+    });
+
+    const data = await response.json();
+    return data;
+  }
+
+  async streamJob(jobId: string) {
+    const response = await fetch(`${this.url}/api/jobs/${jobId}/stream`, {
+      headers: {
+        Authorization: `Bearer ${this.apiKey}`,
+      },
+    });
+
+    const reader = response.body!.getReader();
+    const decoder = new TextDecoder();
+
+    async function* streamGenerator() {
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+
+        const chunk = decoder.decode(value, { stream: true });
+        const lines = chunk.split("\n").filter((line) => line.trim());
+
+        for (const line of lines) {
+          if (line.startsWith("data: ")) {
+            const jsonStr = line.slice(6);
+            try {
+              const data = JSON.parse(jsonStr);
+              yield data;
+            } catch (e) {
+              console.error("Error parsing SSE data:", e);
+            }
+          }
+        }
+      }
+    }
+
+    return streamGenerator();
+  }
+}
+
+export function createClient(options: ClientOptions): Client {
+  return new Client(options);
+}

src/index.ts

@@ -6,6 +6,7 @@ export * from "./jobs/embedding";
 export * from "./jobs/models";
 
 export * from "./providers/anthropic";
+export * from "./providers/elevenlabs";
 export * from "./providers/deepseek";
 export * from "./providers/fal";
 export * from "./providers/fireworks";
@@ -15,3 +16,7 @@ export * from "./providers/ollama";
 export * from "./providers/openai";
 export * from "./providers/together";
 export * from "./providers/voyage";
+
+export * from "./workflow";
+
+export * from "./client";

src/jobs/builder.ts

@@ -1,37 +1,36 @@
 import { version } from "../../package.json";
-import type { Job } from "./load";
-import type {
-  JobCost,
-  JobOptions,
-  JobPerformance,
-  JobProvider,
-  JobType,
-} from "./schema";
+import type { BaseJob } from "./schema";
 
 export class HTTPError extends Error {
   status: number;
   json?: any;
 
   constructor(message: string, status: number, json?: any) {
+    if (json && json.error && json.error.message) {
+      message = json.error.message;
+    }
     super(message);
     this.status = status;
     this.json = json;
   }
 }
 
-export class JobBuilder<Input, Output> {
-  provider!: JobProvider;
-  options!: JobOptions;
-  type!: JobType;
-  input?: Input;
-  output?: Output;
-  cost?: JobCost;
-  performance?: JobPerformance; // TODO: track job performance
+export abstract class JobBuilder<Job extends BaseJob> {
+  provider!: Job["provider"];
+  options!: Job["options"];
+  type!: Job["type"];
+  input?: Job["input"];
+  output?: Job["output"];
+  cost?: Job["cost"];
+  performance?: Job["performance"]; // TODO: track job performance
+
+  abstract makeRequest(): Request;
 
-  makeRequest?: () => Request;
-  handleResponse?: (response: Response) => any;
+  async handleResponse(response: Response): Promise<Job["output"]> {
+    throw new Error("Not implemented");
+  }
 
-  async run(): Promise<Output> {
+  async run(): Promise<Job["output"]> {
     const request = this.makeRequest!();
     const response = await fetch(request);
     if (!response.ok) {
@@ -41,9 +40,9 @@ export class JobBuilder<Input, Output> {
       } catch (e) {}
 
       throw new HTTPError(
-        `Fetch error: ${response.statusText}`,
+        `HTTP error: ${response.statusText}`,
         response.status,
-        json
+        json,
       );
     }
     return await this.handleResponse!(response);
@@ -56,9 +55,9 @@ export class JobBuilder<Input, Output> {
       options: this.options,
       type: this.type,
       input: this.input!,
-      output: this.output as any,
+      output: this.output,
       cost: this.cost,
       performance: this.performance,
-    } as Job;
+    };
   }
 }