blog: telemetry v2 blogpost

alchemy-web/src/content/docs/blog/2025-09-30-reworking-alchemys-telemetry.md

@@ -0,0 +1,179 @@
+# Reworking Alchemy's Telemetry
+
+## Why we left posthog
+
+Alchemy used to use posthog for our platform analytics. While posthot was really easy to set up (see our example here) it simply didn't meet our needs. One of our most important metrics is the number of projects that use alchemy. In order to get quality analytics from posthog we would need to give each alchemy project a dedicated project id and maintain that id as a project grows; this proves to be quite a challenging issue. We can't just use a UUID and store it somewhere as alchemy doesn't have a config file, and we don't expect the `.alchemy` directory to be committed. We explored using various other solutions such as the root commit hash (unavailable for partial clone) or the git upstream url (breaks if origin changes) but none of these solutions were reliably enough. Ultimately we decided to identify projects based on multiple factors instead of having a single id.
+
+Between not having a consistent project id, and multiple of our team members having experience with datalakes and large scale analytics solutions, we decided to switch to a more developer-oriented solutions. Just having all of our own data in-house so we can do whatever we want with it as we please.
+
+That being said posthog is great at what it does, and we will continue to use posthog for our web analytics where its still a great fit!
+
+## Why We Chose Clickhouse Cloud
+
+We started with a few criteria
+- we wanted an OLAP database since they are great for analytics
+- we wanted something SQL based so we didn't have to learn a new query language
+- we wanted something to avoid the big cloud providers as we don't support GCP or Azure yet, and we are in the middle of revamping our AWS resources as part of our [effect](https://effect.website/) based rewrite.
+- we wanted something quick as the team was getting frustrated with our current analytics solution.
+- preferrably a nice controlplane api
+
+After looking at the options we decided to go with Clickhouse Cloud. It had a great controlplane api so making a resource was easy. First we generate a typescript api from Clickhouse's OpenAPI spec, then we write our alchemy resource.
+
+This is a simplified example, but we're omitting clickhouse's plethora of customization options for brevity, but the full resource is available [here](https://github.com/alchemy-framework/alchemy/blob/main/alchemy/src/clickhouse/service.ts).
+
+```ts
+export const Service = Resource(
+  "clickhouse::Service",
+  async function (
+    this: Context<Service>,
+    id: string,
+    props: ServiceProps,
+  ): Promise<Service> {
+    const api = createClickhouseApi();
+    const minReplicaMemoryGb = props.minReplicaMemoryGb ?? 8;
+    const maxReplicaMemoryGb = props.maxReplicaMemoryGb ?? 356;
+    const numReplicas = props.numReplicas ?? 3;
+    const name = this.scope.createPhysicalName(id);
+
+    if (this.phase === "delete") {
+      await api.deleteService({
+        path: {
+          organizationId: props.organization.id,
+          serviceId: this.output.clickhouseId,
+        },
+      });
+      return this.destroy();
+    }
+    if (this.phase === "update") {
+      const resourceDiff = diff(props,this.output,);
+      const updates: Partial<Service> = {};
+
+      if (resourceDiff.some((prop) => prop === "name" && prop === "minReplicaMemoryGb" && prop === "maxReplicaMemoryGb" && prop === "numReplicas")) { return this.replace(); }
+
+      if (resourceDiff.some((prop) => prop === "name")) {
+        const response = (
+          await api.updateServiceBasicDetails({
+            path: {
+              organizationId: props.organization.id,
+              serviceId: this.output.clickhouseId,
+            },
+            body: { name },
+          })
+        ).data.result;
+
+        updates.name = response.name;
+        updates.mysqlEndpoint = response.endpoints.find(
+          (endpoint) => endpoint.protocol === "mysql",
+        ) as any;
+        updates.httpsEndpoint = response.endpoints.find(
+          (endpoint) => endpoint.protocol === "https",
+        ) as any;
+      }
+
+      if (
+        resourceDiff.some(
+          (prop) => prop === "minReplicaMemoryGb" || prop === "maxReplicaMemoryGb" || prop === "numReplicas",
+        )
+      ) {
+        const response = (
+          await api.updateServiceAutoScalingSettings2({
+            path: {
+              organizationId: props.organization.id,
+              serviceId: this.output.clickhouseId,
+            },
+            body: {
+              minReplicaMemoryGb: props.minReplicaMemoryGb,
+              maxReplicaMemoryGb: props.maxReplicaMemoryGb,
+              numReplicas: props.numReplicas,
+            },
+          })
+        ).data.result;
+
+        updates.minReplicaMemoryGb = response.minReplicaMemoryGb;
+        updates.maxReplicaMemoryGb = response.maxReplicaMemoryGb;
+        updates.numReplicas = response.numReplicas;
+      }
+
+      return {
+        ...this.output,
+        ...updates,
+      };
+    }
+
+    const response = (
+      await api.createNewService({
+        path: {
+          organizationId: props.organization.id,
+        },
+        body: {
+          name,
+          provider: props.provider,
+          region: props.region,
+          minReplicaMemoryGb: minReplicaMemoryGb,
+          maxReplicaMemoryGb: maxReplicaMemoryGb,
+          numReplicas: numReplicas,
+        },
+      })
+    ).data.result;
+
+    return {
+      organizationId: props.organization.id,
+      name: response.service.name,
+      clickhouseId: response.service.id,
+      password: secret(response.password),
+      provider: response.service.provider,
+      region: response.service.region,
+      minReplicaMemoryGb: response.service.minReplicaMemoryGb,
+      maxReplicaMemoryGb: response.service.maxReplicaMemoryGb,
+      numReplicas: response.service.numReplicas,
+      mysqlEndpoint: response.service.endpoints.find(
+        (endpoint) => endpoint.protocol === "mysql",
+      ) as any,
+      httpsEndpoint: response.service.endpoints.find(
+        (endpoint) => endpoint.protocol === "https",
+      ) as any,
+    };
+  },
+);
+```
+
+Its only about 100 lines of code and now we have an alchemy resource for clickhouse.
+
+## Using the resource
+
+While our telemetry backend isn't open source, the `alchemy.run.ts` file is less than 25 lines.
+
+```ts
+// imports
+export const app = await alchemy("alchemy-telemetry");
+const organization = await getOrganizationByName(alchemy.env.CLICKHOUSE_ORG);
+
+const clickhouse = await Service("clickhouse", {
+	organization,
+	provider: "aws",
+	region: "us-east-1",
+	minReplicaMemoryGb: 8,
+	maxReplicaMemoryGb: 356,
+	numReplicas: 3,
+});
+
+await Exec("migrations", {
+	command: `bunx clickhouse-migrations migrate --db default --host https://${clickhouse.httpsEndpoint?.host}:${clickhouse.httpsEndpoint?.port} --user ${clickhouse.mysqlEndpoint?.username} --password ${clickhouse.password.unencrypted} --migrations-home ${join(import.meta.dirname, "migrations")}`,
+});
+
+export const ingestWorker = await Worker("ingest-worker", {
+	adopt: true,
+	entrypoint: "./deployments/telemetry.ts",
+	bindings: {
+		CLICKHOUSE_URL: `https://${clickhouse.httpsEndpoint?.host}:${clickhouse.httpsEndpoint?.port}`,
+		CLICKHOUSE_PASSWORD: clickhouse.password,
+	},
+	domains: ["telemetry.alchemy.run"],
+});
+
+await app.finalize();
+```
+
+## Future Improvements
+
+Just dumping data straight to clickhouse is by no means the best solution, we understand that! Our goal here was to quickly spin up some data storage and fix our analytics; We'll share more in depth technical details in the future on how to bring down costs and build a more enterprise level data solution on top of alchemy.

alchemy/bin/trpc.ts

@@ -1,7 +1,7 @@
 import { cancel, log } from "@clack/prompts";
 import pc from "picocolors";
 import { trpcServer, type TrpcCliMeta } from "trpc-cli";
-import { TelemetryClient } from "../src/util/telemetry/client.ts";
+import { createAndSendEvent } from "../src/util/telemetry/v2.ts";
 
 export const t = trpcServer.initTRPC.meta<TrpcCliMeta>().create();
 
@@ -15,25 +15,21 @@ export class ExitSignal extends Error {
 export class CancelSignal extends Error {}
 
 const loggingMiddleware = t.middleware(async ({ path, next }) => {
-  const telemetry = TelemetryClient.create({
-    enabled: true,
-    quiet: true,
-  });
-  telemetry.record({
+  createAndSendEvent({
     event: "cli.start",
     command: path,
   });
   let exitCode = 0;
 
   try {
     const result = await next();
-    telemetry.record({
+    createAndSendEvent({
       event: "cli.success",
       command: path,
     });
     return result;
   } catch (error) {
-    telemetry.record({
+    createAndSendEvent({
       event:
         error instanceof ExitSignal && error.code === 0
           ? "cli.success"
@@ -46,7 +42,6 @@ const loggingMiddleware = t.middleware(async ({ path, next }) => {
       throw error;
     }
   } finally {
-    await telemetry.finalize();
     //* this is a node issue https://github.com/nodejs/node/issues/56645
     await new Promise((resolve) => setTimeout(resolve, 100));
     process.exit(exitCode);

alchemy/src/alchemy.ts

@@ -17,7 +17,6 @@ import { secret } from "./secret.ts";
 import type { StateStoreType } from "./state.ts";
 import type { LoggerApi } from "./util/cli.ts";
 import { ALCHEMY_ROOT } from "./util/root-dir.ts";
-import { TelemetryClient } from "./util/telemetry/client.ts";
 
 /**
  * Type alias for semantic highlighting of `alchemy` as a type keyword
@@ -162,20 +161,13 @@ If this is a mistake, you can disable this check by setting the ALCHEMY_CI_STATE
   }
 
   const phase = mergedOptions?.phase ?? "up";
-  const telemetryClient =
-    mergedOptions?.parent?.telemetryClient ??
-    TelemetryClient.create({
-      phase,
-      enabled: mergedOptions?.telemetry ?? true,
-      quiet: mergedOptions?.quiet ?? false,
-    });
   const root = new Scope({
     ...mergedOptions,
     parent: undefined,
     scopeName: appName,
     phase,
     password: mergedOptions?.password ?? process.env.ALCHEMY_PASSWORD,
-    telemetryClient,
+    noTrack: mergedOptions?.noTrack ?? false,
     isSelected: app === undefined ? undefined : app === appName,
   });
   onExit((code) => {
@@ -280,12 +272,12 @@ export interface AlchemyOptions {
    */
   password?: string;
   /**
-   * Whether to send anonymous telemetry data to the Alchemy team.
+   * Whether to stop sending anonymous telemetry data to the Alchemy team.
    * You can also opt out by setting the `DO_NOT_TRACK` or `ALCHEMY_TELEMETRY_DISABLED` environment variables to a truthy value.
    *
-   * @default true
+   * @default false
    */
-  telemetry?: boolean;
+  noTrack?: boolean;
   /**
    * A custom logger instance to use for this scope.
    * If not provided, the default fallback logger will be used.
@@ -353,18 +345,11 @@ async function run<T>(
           RunOptions,
           (this: Scope, scope: Scope) => Promise<T>,
         ]);
-  const telemetryClient =
-    options?.parent?.telemetryClient ??
-    TelemetryClient.create({
-      phase: options?.phase ?? "up",
-      enabled: options?.telemetry ?? true,
-      quiet: options?.quiet ?? false,
-    });
   const _scope = new Scope({
     ...options,
     parent: options?.parent,
     scopeName: id,
-    telemetryClient,
+    noTrack: options?.noTrack ?? false,
   });
   let noop = options?.noop ?? false;
   try {

alchemy/src/apply.ts

@@ -19,7 +19,7 @@ import { serialize } from "./serde.ts";
 import type { State } from "./state.ts";
 import { formatFQN } from "./util/cli.ts";
 import { logger } from "./util/logger.ts";
-import type { Telemetry } from "./util/telemetry/index.ts";
+import { createAndSendEvent } from "./util/telemetry/v2.ts";
 
 export interface ApplyOptions {
   quiet?: boolean;
@@ -83,9 +83,13 @@ async function _apply<Out extends ResourceAttributes>(
         // we are running in a monorepo and are not the selected app, so we need to wait for the process to be consistent
         state = await waitForConsistentState();
       }
-      scope.telemetryClient.record({
+      createAndSendEvent({
         event: "resource.read",
+        phase: scope.phase,
+        duration: performance.now() - start,
+        status: state.status,
         resource: resource[ResourceKind],
+        replaced: false,
       });
       return state.output as Awaited<Out> & Resource;
 
@@ -179,10 +183,13 @@ async function _apply<Out extends ResourceAttributes>(
             status: "success",
           });
         }
-        scope.telemetryClient.record({
+        createAndSendEvent({
           event: "resource.skip",
           resource: resource[ResourceKind],
           status: state.status,
+          phase: scope.phase,
+          duration: performance.now() - start,
+          replaced: false,
         });
         return state.output as Awaited<Out> & Resource;
       }
@@ -202,10 +209,13 @@ async function _apply<Out extends ResourceAttributes>(
       });
     }
 
-    scope.telemetryClient.record({
+    createAndSendEvent({
       event: "resource.start",
       resource: resource[ResourceKind],
       status: state.status,
+      phase: scope.phase,
+      duration: performance.now() - start,
+      replaced: false,
     });
 
     await scope.state.set(resource[ResourceID], state);
@@ -326,11 +336,12 @@ async function _apply<Out extends ResourceAttributes>(
     }
 
     const status = phase === "create" ? "created" : "updated";
-    scope.telemetryClient.record({
+    createAndSendEvent({
       event: "resource.success",
       resource: resource[ResourceKind],
       status,
-      elapsed: performance.now() - start,
+      phase: scope.phase,
+      duration: performance.now() - start,
       replaced: isReplaced,
     });
 
@@ -347,12 +358,17 @@ async function _apply<Out extends ResourceAttributes>(
     });
     return output as Awaited<Out> & Resource;
   } catch (error) {
-    scope.telemetryClient.record({
-      event: "resource.error",
-      resource: resource[ResourceKind],
-      error: error as Telemetry.ErrorInput,
-      elapsed: performance.now() - start,
-    });
+    createAndSendEvent(
+      {
+        event: "resource.error",
+        resource: resource[ResourceKind],
+        duration: performance.now() - start,
+        phase: scope.phase,
+        status: "unknown",
+        replaced: false,
+      },
+      error as Error | undefined,
+    );
     scope.fail();
     throw error;
   }