From 2daa7ffd47ce93fdd427aa047cde5d6f1c65a102 Mon Sep 17 00:00:00 2001 From: jherrera-jump Date: Tue, 10 Jun 2025 19:22:17 +0000 Subject: [PATCH] gui: compress ws messages --- package-lock.json | 301 ++++++++++++++++++++++++++++++ package.json | 3 + src/api/ws/ConnectionProvider.tsx | 28 ++- src/api/ws/connectWebSocket.ts | 20 +- vite.config.ts | 4 + 5 files changed, 342 insertions(+), 14 deletions(-) diff --git a/package-lock.json b/package-lock.json index 48e335a..a552772 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,6 +17,7 @@ "@fontsource/roboto-mono": "^5.1.0", "@nivo/core": "^0.88.0", "@nivo/pie": "^0.88.0", + "@oneidentity/zstd-js": "^1.0.3", "@radix-ui/react-icons": "^1.3.0", "@radix-ui/themes": "^3.1.3", "@react-spring/web": "^9.7.4", @@ -70,6 +71,8 @@ "typescript-plugin-css-modules": "^5.1.0", "vite": "^5.4.8", "vite-plugin-checker": "^0.8.0", + "vite-plugin-top-level-await": "^1.5.0", + "vite-plugin-wasm": "^3.4.1", "vitest": "^3.0.9" } }, @@ -1328,6 +1331,15 @@ "node": ">= 8" } }, + "node_modules/@oneidentity/zstd-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@oneidentity/zstd-js/-/zstd-js-1.0.3.tgz", + "integrity": "sha512-Jm6sawqxLzBrjC4sg2BeXToa33yPzUmq20CKsehKY2++D/gHb/oSwVjNgT+RH4vys+r8FynrgcNzGwhZWMLzfQ==", + "license": "SEE LICENSE IN LICENSE", + "dependencies": { + "@types/emscripten": "^1.39.4" + } + }, "node_modules/@parcel/watcher": { "version": "2.5.1", "resolved": "https://registry.npmjs.org/@parcel/watcher/-/watcher-2.5.1.tgz", @@ -3147,6 +3159,24 @@ "react-dom": "^16.8.0 || ^17.0.0 || ^18.0.0" } }, + "node_modules/@rollup/plugin-virtual": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/@rollup/plugin-virtual/-/plugin-virtual-3.0.2.tgz", + "integrity": "sha512-10monEYsBp3scM4/ND4LNH5Rxvh3e/cVeL3jWTgZ2SrQ+BmUoQcopVQvnaMcOnykb1VkxUFuDAN+0FnpTFRy2A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=14.0.0" + }, + "peerDependencies": { + "rollup": "^1.20.0||^2.0.0||^3.0.0||^4.0.0" + }, + "peerDependenciesMeta": { + "rollup": { + "optional": true + } + } + }, "node_modules/@rollup/rollup-android-arm-eabi": { "version": "4.36.0", "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.36.0.tgz", @@ -3413,6 +3443,232 @@ "win32" ] }, + "node_modules/@swc/core": { + "version": "1.12.6", + "resolved": "https://registry.npmjs.org/@swc/core/-/core-1.12.6.tgz", + "integrity": "sha512-TEpta6Gi02X1b2yDIzBOIr7dFprvq9jD8RbEVI2OcMrwklbCUx0Dz9TrAnklSOwRvYvH5JjCx8ht9E94oWiG7A==", + "dev": true, + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "@swc/counter": "^0.1.3", + "@swc/types": "^0.1.23" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/swc" + }, + "optionalDependencies": { + "@swc/core-darwin-arm64": "1.12.6", + "@swc/core-darwin-x64": "1.12.6", + "@swc/core-linux-arm-gnueabihf": "1.12.6", + "@swc/core-linux-arm64-gnu": "1.12.6", + "@swc/core-linux-arm64-musl": "1.12.6", + "@swc/core-linux-x64-gnu": "1.12.6", + "@swc/core-linux-x64-musl": "1.12.6", + "@swc/core-win32-arm64-msvc": "1.12.6", + "@swc/core-win32-ia32-msvc": "1.12.6", + "@swc/core-win32-x64-msvc": "1.12.6" + }, + "peerDependencies": { + "@swc/helpers": ">=0.5.17" + }, + "peerDependenciesMeta": { + "@swc/helpers": { + "optional": true + } + } + }, + "node_modules/@swc/core-darwin-arm64": { + "version": "1.12.6", + "resolved": "https://registry.npmjs.org/@swc/core-darwin-arm64/-/core-darwin-arm64-1.12.6.tgz", + "integrity": "sha512-yLiw+XzG+MilfFh0ON7qt67bfIr7UxB9JprhYReVOmLTBDmDVQSC3T4/vIuc+GwlX08ydnHy0ud4lIjTNW4uWg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0 AND MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-darwin-x64": { + "version": "1.12.6", + "resolved": "https://registry.npmjs.org/@swc/core-darwin-x64/-/core-darwin-x64-1.12.6.tgz", + "integrity": "sha512-qwg8ux5x5Gd1LmSUtL4s9mbyfzAjr5M6OtjO281dKHwc/GYiSc4j1urb2jNSo9FcMkfT78oAOW2L6HQiWv+j1A==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0 AND MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-linux-arm-gnueabihf": { + "version": "1.12.6", + "resolved": "https://registry.npmjs.org/@swc/core-linux-arm-gnueabihf/-/core-linux-arm-gnueabihf-1.12.6.tgz", + "integrity": "sha512-pnkqH59JXBZu+MedaykMAC2or7tlUKeya7GKjzub+hkwxBP0ywWoFd+QYEdzp7QSziOt1VIHc4Wb9iZ2EfnzmA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "Apache-2.0", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-linux-arm64-gnu": { + "version": "1.12.6", + "resolved": "https://registry.npmjs.org/@swc/core-linux-arm64-gnu/-/core-linux-arm64-gnu-1.12.6.tgz", + "integrity": "sha512-h8+Ltx0NSEzIFHetkOYoQ+UQ59unYLuJ4wF6kCpxzS4HskRLjcngr1HgN0F/PRpptnrmJUPVQmfms/vjN8ndAQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0 AND MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-linux-arm64-musl": { + "version": "1.12.6", + "resolved": "https://registry.npmjs.org/@swc/core-linux-arm64-musl/-/core-linux-arm64-musl-1.12.6.tgz", + "integrity": "sha512-GZu3MnB/5qtBxKEH46hgVDaplEe4mp3ZmQ1O2UpFCv/u/Ji3Gar5w5g2wHCZoT5AOouAhP1bh7IAEqjG/fbVfg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0 AND MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-linux-x64-gnu": { + "version": "1.12.6", + "resolved": "https://registry.npmjs.org/@swc/core-linux-x64-gnu/-/core-linux-x64-gnu-1.12.6.tgz", + "integrity": "sha512-WwJLQFzMW9ufVjM6k3le4HUgBFNunyt2oghjcgn2YjnKj0Ka2LrrBHCxfS7lgFSCQh/shib2wIlKXUnlTEWQJw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0 AND MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-linux-x64-musl": { + "version": "1.12.6", + "resolved": "https://registry.npmjs.org/@swc/core-linux-x64-musl/-/core-linux-x64-musl-1.12.6.tgz", + "integrity": "sha512-rVGPNpI/sm8VVAhnB09Z/23OJP3ymouv6F4z4aYDbq/2JIwxqgpnl8gtMYP+Jw3XqabaFNjQmPiL15TvKCQaxQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0 AND MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-win32-arm64-msvc": { + "version": "1.12.6", + "resolved": "https://registry.npmjs.org/@swc/core-win32-arm64-msvc/-/core-win32-arm64-msvc-1.12.6.tgz", + "integrity": "sha512-EKDJ1+8vaIlJGMl2yvd2HklV4GNbpKKwNQcUQid6j91tFYz4/aByw+9vh/sDVG7ZNqdmdywSnLRo317UTt0zFg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "Apache-2.0 AND MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-win32-ia32-msvc": { + "version": "1.12.6", + "resolved": "https://registry.npmjs.org/@swc/core-win32-ia32-msvc/-/core-win32-ia32-msvc-1.12.6.tgz", + "integrity": "sha512-jnULikZkR2fpZgFUQs7NsNIztavM1JdX+8Y+8FsfChBvMvziKxXtvUPGjeVJ8nzU1wgMnaeilJX9vrwuDGkA0Q==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "Apache-2.0 AND MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/core-win32-x64-msvc": { + "version": "1.12.6", + "resolved": "https://registry.npmjs.org/@swc/core-win32-x64-msvc/-/core-win32-x64-msvc-1.12.6.tgz", + "integrity": "sha512-jL2Dcdcc/QZiQnwByP1uIE4k/mTlapzUng7owtLD2tSBBi1d+jPIdXIefdv+nccYJKRA+lKG3rRB6Tk9GrC7Kg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "Apache-2.0 AND MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=10" + } + }, + "node_modules/@swc/counter": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@swc/counter/-/counter-0.1.3.tgz", + "integrity": "sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ==", + "dev": true, + "license": "Apache-2.0" + }, + "node_modules/@swc/types": { + "version": "0.1.23", + "resolved": "https://registry.npmjs.org/@swc/types/-/types-0.1.23.tgz", + "integrity": "sha512-u1iIVZV9Q0jxY+yM2vw/hZGDNudsN85bBpTqzAQ9rzkxW9D+e3aEM4Han+ow518gSewkXgjmEK0BD79ZcNVgPw==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "@swc/counter": "^0.1.3" + } + }, "node_modules/@tanstack/history": { "version": "1.115.0", "resolved": "https://registry.npmjs.org/@tanstack/history/-/history-1.115.0.tgz", @@ -3815,6 +4071,12 @@ "integrity": "sha512-yuzZug1nkAAaBlBBikKZTgzCeA+k1uy4ZFwWANOfKw5z5LRhV0gNA7gNkKm7HoK+HRN0wX3EkxGk0fpbWhmB7g==", "license": "MIT" }, + "node_modules/@types/emscripten": { + "version": "1.40.1", + "resolved": "https://registry.npmjs.org/@types/emscripten/-/emscripten-1.40.1.tgz", + "integrity": "sha512-sr53lnYkQNhjHNN0oJDdUm5564biioI5DuOpycufDVK7D3y+GR3oUswe2rlwY1nPNyusHbrJ9WoTyIHl4/Bpwg==", + "license": "MIT" + }, "node_modules/@types/estree": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz", @@ -10225,6 +10487,20 @@ "dev": true, "license": "MIT" }, + "node_modules/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", + "dev": true, + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/vite": { "version": "5.4.14", "resolved": "https://registry.npmjs.org/vite/-/vite-5.4.14.tgz", @@ -10374,6 +10650,31 @@ } } }, + "node_modules/vite-plugin-top-level-await": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/vite-plugin-top-level-await/-/vite-plugin-top-level-await-1.5.0.tgz", + "integrity": "sha512-r/DtuvHrSqUVk23XpG2cl8gjt1aATMG5cjExXL1BUTcSNab6CzkcPua9BPEc9fuTP5UpwClCxUe3+dNGL0yrgQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@rollup/plugin-virtual": "^3.0.2", + "@swc/core": "^1.10.16", + "uuid": "^10.0.0" + }, + "peerDependencies": { + "vite": ">=2.8" + } + }, + "node_modules/vite-plugin-wasm": { + "version": "3.4.1", + "resolved": "https://registry.npmjs.org/vite-plugin-wasm/-/vite-plugin-wasm-3.4.1.tgz", + "integrity": "sha512-ja3nSo2UCkVeitltJGkS3pfQHAanHv/DqGatdI39ja6McgABlpsZ5hVgl6wuR8Qx5etY3T5qgDQhOWzc5RReZA==", + "dev": true, + "license": "MIT", + "peerDependencies": { + "vite": "^2 || ^3 || ^4 || ^5 || ^6" + } + }, "node_modules/vite/node_modules/@esbuild/aix-ppc64": { "version": "0.21.5", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.21.5.tgz", diff --git a/package.json b/package.json index 8df33b0..6ee8839 100644 --- a/package.json +++ b/package.json @@ -27,6 +27,7 @@ "@fontsource/roboto-mono": "^5.1.0", "@nivo/core": "^0.88.0", "@nivo/pie": "^0.88.0", + "@oneidentity/zstd-js": "^1.0.3", "@radix-ui/react-icons": "^1.3.0", "@radix-ui/themes": "^3.1.3", "@react-spring/web": "^9.7.4", @@ -80,6 +81,8 @@ "typescript-plugin-css-modules": "^5.1.0", "vite": "^5.4.8", "vite-plugin-checker": "^0.8.0", + "vite-plugin-top-level-await": "^1.5.0", + "vite-plugin-wasm": "^3.4.1", "vitest": "^3.0.9" } } diff --git a/src/api/ws/ConnectionProvider.tsx b/src/api/ws/ConnectionProvider.tsx index ed2531d..6d4f414 100644 --- a/src/api/ws/ConnectionProvider.tsx +++ b/src/api/ws/ConnectionProvider.tsx @@ -4,6 +4,7 @@ import { PropsWithChildren, useCallback, useEffect, useState } from "react"; import connectWebSocket from "./connectWebSocket"; import { ConnectionStatus } from "./types"; import { socketStateAtom } from "./atoms"; +import { ZstdInit } from "@oneidentity/zstd-js/decompress"; import UpdateAtoms from "../UpdateAtoms"; import { ConnectionContext, @@ -59,18 +60,27 @@ export function ConnectionProvider({ children }: PropsWithChildren) { setSocketState(connectionStatus.socketState); }; - const [sendMessage, dispose] = connectWebSocket( - websocketUrl, - onMessage, - onConnectionStatusChanged, - ); + const disposePromise = (async () => { + const zstd = await ZstdInit(); - updateContext({ sendMessage, emitter, isActive: true }); + const [sendMessage, dispose] = connectWebSocket( + websocketUrl, + onMessage, + onConnectionStatusChanged, + zstd, + ); + + updateContext({ sendMessage, emitter, isActive: true }); + + return dispose; + })(); return () => { - emitter.removeAllListeners(); - resetContext(); - dispose(); + void (async () => { + emitter.removeAllListeners(); + resetContext(); + (await disposePromise)(); + })(); }; }, [resetContext, updateContext, setSocketState, websocketUrl]); diff --git a/src/api/ws/connectWebSocket.ts b/src/api/ws/connectWebSocket.ts index bc1fd6a..89f37a4 100644 --- a/src/api/ws/connectWebSocket.ts +++ b/src/api/ws/connectWebSocket.ts @@ -1,5 +1,6 @@ import { ClientMessage, ConnectionStatus, SocketState } from "./types"; import { logDebug, logWarning } from "../../logger"; +import { ZstdDec } from "@oneidentity/zstd-js/decompress"; const RECONNECT_DELAY = 3_000; @@ -9,6 +10,7 @@ export default function connectWebSocket( url: string | URL, onMessage: (message: unknown) => void, onConnectionStatusChanged: (connectionStatus: ConnectionStatus) => void, + zstd: ZstdDec, ) { let ws: WebSocket; let isDisposing = false; @@ -16,7 +18,7 @@ export default function connectWebSocket( function connect() { logDebug("WS", `Connecting to API WebSocket ${url.toString()}`); onConnectionStatusChanged({ socketState: SocketState.Connecting }); - ws = new WebSocket(url); + ws = new WebSocket(url, ["compress-zstd"]); ws.binaryType = "arraybuffer"; ws.onopen = function onopen() { @@ -39,14 +41,22 @@ export default function connectWebSocket( reconnectTimer = setTimeout(connect, RECONNECT_DELAY); }; - ws.onmessage = function onmessage(ev: MessageEvent) { - if (this !== ws || isDisposing) return; + const decoder = new TextDecoder(); + ws.onmessage = function onmessage(ev: MessageEvent) { + if (this !== ws || isDisposing || !zstd) return; try { - const json = JSON.parse(ev.data) as unknown; + const message = ev.data; + let json = undefined; + if (typeof message === "string") { + json = JSON.parse(message) as unknown; + } else if (message instanceof ArrayBuffer) { + json = JSON.parse( + decoder.decode(zstd.ZstdStream.decompress(new Uint8Array(message))), + ) as unknown; + } onMessage(json); } catch (e) { console.error(e); - console.error(ev.data); } }; } diff --git a/vite.config.ts b/vite.config.ts index 9bb932e..7e5caed 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -3,6 +3,8 @@ import react from "@vitejs/plugin-react"; import checker from "vite-plugin-checker"; import { TanStackRouterVite } from "@tanstack/router-vite-plugin"; import license from "rollup-plugin-license"; +import wasm from "vite-plugin-wasm"; +import topLevelAwait from "vite-plugin-top-level-await"; // https://vitejs.dev/config/ export default defineConfig({ @@ -35,6 +37,8 @@ export default defineConfig({ plugins: [ react(), TanStackRouterVite(), + wasm(), + topLevelAwait(), checker({ typescript: true,