diff --git a/.github/workflows/make-and-test.yml b/.github/workflows/make-and-test.yml index 088354e702f..a51af692607 100644 --- a/.github/workflows/make-and-test.yml +++ b/.github/workflows/make-and-test.yml @@ -39,7 +39,7 @@ jobs: detached: true - uses: actions/checkout@v4 - name: Install python3 requests - run: sudo apt-get install python3-requests + run: sudo apt-get install python3-requests python3-yapf - name: Check doc links run: cd src/scripts && python3 check_doc_urls.py || sleep 5 || python3 check_doc_urls.py @@ -91,19 +91,15 @@ jobs: # cache: "pnpm" # cache-dependency-path: "src/packages/pnpm-lock.yaml" - - name: Download and install Valkey - run: | - VALKEY_VERSION=8.1.2 - curl -LOq https://download.valkey.io/releases/valkey-${VALKEY_VERSION}-jammy-x86_64.tar.gz - tar -xzf valkey-${VALKEY_VERSION}-jammy-x86_64.tar.gz - sudo cp valkey-${VALKEY_VERSION}-jammy-x86_64/bin/valkey-server /usr/local/bin/ + - name: Install btrfs-progs and bup for @cocalc/file-server + run: sudo apt-get update && sudo apt-get install -y btrfs-progs bup - name: Set up Python venv and Jupyter kernel run: | python3 -m pip install --upgrade pip virtualenv python3 -m virtualenv venv source venv/bin/activate - pip install ipykernel + pip install ipykernel yapf python -m ipykernel install --prefix=./jupyter-local --name python3-local --display-name "Python 3 (Local)" @@ -128,30 +124,30 @@ jobs: name: "test-results-node-${{ matrix.node-version }}-pg-${{ matrix.pg-version }}" path: 'src/packages/*/junit.xml' - report: - runs-on: ubuntu-latest +# report: +# runs-on: ubuntu-latest - needs: [test] +# needs: [test] - if: ${{ !cancelled() }} +# if: ${{ !cancelled() }} - steps: - - name: Checkout code - uses: actions/checkout@v4 +# steps: +# - name: Checkout code +# uses: actions/checkout@v4 - - name: Download all test artifacts - uses: actions/download-artifact@v4 - with: - pattern: "test-results-*" - merge-multiple: true - path: test-results/ +# - name: Download all test artifacts +# uses: actions/download-artifact@v4 +# with: +# pattern: "test-results-*" +# merge-multiple: true +# path: test-results/ - - name: Test Report - uses: dorny/test-reporter@v2 - with: - name: CoCalc Jest Tests - path: 'test-results/**/junit.xml' - reporter: jest-junit - use-actions-summary: 'true' - fail-on-error: false +# - name: Test Report +# uses: dorny/test-reporter@v2 +# with: +# name: CoCalc Jest Tests +# path: 'test-results/**/junit.xml' +# reporter: jest-junit +# use-actions-summary: 'true' +# fail-on-error: false diff --git a/src/package.json b/src/package.json index d0f07012776..8f4ce63e09a 100644 --- a/src/package.json +++ b/src/package.json @@ -18,7 +18,7 @@ "version-check": "pip3 install typing_extensions mypy || pip3 install --break-system-packages typing_extensions mypy && ./workspaces.py version-check && mypy scripts/check_npm_packages.py", "test-parallel": "unset DEBUG && pnpm run version-check && cd packages && pnpm run -r --parallel test", "test": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test", - "test-github-ci": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test --exclude=jupyter,file-server --retries=1", + "test-github-ci": "unset DEBUG && pnpm run depcheck && pnpm run version-check && ./workspaces.py test --test-github-ci --exclude=jupyter --retries=1", "depcheck": "cd packages && pnpm run -r --parallel depcheck", "prettier-all": "cd packages/", "local-ci": "./scripts/ci.sh", diff --git a/src/packages/backend/conat/files/local-path.ts b/src/packages/backend/conat/files/local-path.ts new file mode 100644 index 00000000000..f432001403c --- /dev/null +++ b/src/packages/backend/conat/files/local-path.ts @@ -0,0 +1,72 @@ +import { fsServer, DEFAULT_FILE_SERVICE } from "@cocalc/conat/files/fs"; +import { SandboxedFilesystem } from "@cocalc/backend/sandbox"; +import { isValidUUID } from "@cocalc/util/misc"; +import { mkdir } from "fs/promises"; +import { join } from "path"; +import { type Client } from "@cocalc/conat/core/client"; +import { conat } from "@cocalc/backend/conat/conat"; +import { client as createFileClient } from "@cocalc/conat/files/file-server"; + +export async function localPathFileserver({ + path, + service = DEFAULT_FILE_SERVICE, + client, + project_id, + unsafeMode, +}: { + service?: string; + client?: Client; + // if project_id is specified, only serve this one project_id + project_id?: string; + // - if path is given, serve projects from `${path}/${project_id}` + // - if path not given, connect to the file-server service on the conat network. + path?: string; + unsafeMode?: boolean; +} = {}) { + client ??= conat(); + + const getPath = async (project_id2: string) => { + if (project_id != null && project_id != project_id2) { + throw Error(`only serves ${project_id}`); + } + if (path != null) { + const p = join(path, project_id2); + try { + await mkdir(p); + } catch {} + return p; + } else { + const fsclient = createFileClient({ client }); + return (await fsclient.mount({ project_id: project_id2 })).path; + } + }; + + const server = await fsServer({ + service, + client, + project_id, + fs: async (subject: string) => { + const project_id = getProjectId(subject); + return new SandboxedFilesystem(await getPath(project_id), { + unsafeMode, + host: project_id, + }); + }, + }); + return { server, client, path, service, close: () => server.close() }; +} + +function getProjectId(subject: string) { + const v = subject.split("."); + if (v.length != 2) { + throw Error("subject must have 2 segments"); + } + if (!v[1].startsWith("project-")) { + throw Error("second segment of subject must start with 'project-'"); + } + const project_id = v[1].slice("project-".length); + if (!isValidUUID(project_id)) { + throw Error("not a valid project id"); + } + return project_id; +} diff --git a/src/packages/backend/conat/files/test/listing.test.ts b/src/packages/backend/conat/files/test/listing.test.ts new file mode 100644 index 00000000000..4f223a3f59e --- /dev/null +++ b/src/packages/backend/conat/files/test/listing.test.ts @@ -0,0 +1,93 @@ +import { SandboxedFilesystem } from "@cocalc/backend/sandbox"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "path"; +import { randomId } from "@cocalc/conat/names"; +import listing from "@cocalc/conat/files/listing"; + +let tmp; +beforeAll(async () => { + tmp = await mkdtemp(join(tmpdir(), `cocalc-${randomId()}0`)); +}); + +afterAll(async () => { + try { + await rm(tmp, { force: true, recursive: true }); + } catch {} +}); + +describe("creating a listing monitor starting with an empty directory", () => { + let fs, dir; + it("creates sandboxed filesystem", async () => { + fs = new SandboxedFilesystem(tmp); + dir = await listing({ path: "", fs }); + }); + + it("initial listing is empty", () => { + expect(Object.keys(dir.files)).toEqual([]); + }); + + let iter; + it("create a file and get an update", async () => { + iter = dir.iter(); + await fs.writeFile("a.txt", "hello"); + let { value } = await iter.next(); + expect(value).toEqual({ + mtime: value.mtime, + name: "a.txt", + size: value.size, + }); + // it's possible that the file isn't written completely above. + if (value.size != 5) { + ({ value } = await iter.next()); + } + const stat = await fs.stat("a.txt"); + expect(stat.mtimeMs).toEqual(value.mtime); + expect(dir.files["a.txt"]).toEqual({ mtime: value.mtime, size: 5 }); + }); + + it("modify the file and get two updates -- one when it starts and another when done", async () => { + await fs.appendFile("a.txt", " there"); + const { value } = await iter.next(); + expect(value).toEqual({ mtime: value.mtime, name: "a.txt", size: 5 }); + const { value: value2 } = await iter.next(); + expect(value2).toEqual({ mtime: value2.mtime, name: "a.txt", size: 11 }); + const stat = await fs.stat("a.txt"); + expect(stat.mtimeMs).toEqual(value2.mtime); + expect(dir.files["a.txt"]).toEqual({ mtime: value2.mtime, size: 11 }); + }); + + it("create another monitor starting with the now nonempty directory", async () => { + const dir2 = await listing({ path: "", fs }); + expect(Object.keys(dir2.files!)).toEqual(["a.txt"]); + expect(dir.files["a.txt"].mtime).toBeCloseTo(dir2.files!["a.txt"].mtime); + dir2.close(); + }); + + const count = 500; + it(`creates ${count} files and see they are found`, async () => { + const n = Object.keys(dir.files).length; + + for (let i = 0; i < count; i++) { + await fs.writeFile(`${i}`, ""); + } + const values: string[] = []; + while (true) { + const { value } = await iter.next(); + if (value == "a.txt") { + continue; + } + values.push(value); + if (value.name == `${count - 1}`) { + break; + } + } + expect(new Set(values).size).toEqual(count); + + expect(Object.keys(dir.files).length).toEqual(n + count); + }); + + it("cleans up", () => { + dir.close(); + }); +}); diff --git a/src/packages/backend/conat/files/test/local-path.test.ts b/src/packages/backend/conat/files/test/local-path.test.ts new file mode 100644 index 00000000000..88bbeaf783d --- /dev/null +++ b/src/packages/backend/conat/files/test/local-path.test.ts @@ -0,0 +1,477 @@ +import { link, readFile, symlink, writeFile } from "node:fs/promises"; +import { join } from "path"; +import { fsClient } from "@cocalc/conat/files/fs"; +import { randomId } from "@cocalc/conat/names"; +import { before, after } from "@cocalc/backend/conat/test/setup"; +import { uuid } from "@cocalc/util/misc"; +import { + createPathFileserver, + cleanupFileservers, +} from "@cocalc/backend/conat/files/test/util"; +import { TextDecoder } from "node:util"; + +beforeAll(before); + +describe("use all the standard api functions of fs", () => { + let server; + it("creates the simple fileserver service", async () => { + server = await createPathFileserver(); + }); + + const project_id = uuid(); + let fs; + it("create a client", () => { + fs = fsClient({ subject: `${server.service}.project-${project_id}` }); + }); + + it("appendFile works", async () => { + await fs.writeFile("a", ""); + await fs.appendFile("a", "foo"); + expect(await fs.readFile("a", "utf8")).toEqual("foo"); + }); + + it("chmod works", async () => { + await fs.writeFile("b", "hi"); + await fs.chmod("b", 0o755); + const s = await fs.stat("b"); + expect(s.mode.toString(8)).toBe("100755"); + }); + + it("constants work", async () => { + const constants = await fs.constants(); + expect(constants.O_RDONLY).toBe(0); + expect(constants.O_WRONLY).toBe(1); + expect(constants.O_RDWR).toBe(2); + }); + + it("copyFile works", async () => { + await fs.writeFile("c", "hello"); + await fs.copyFile("c", "d.txt"); + expect(await fs.readFile("d.txt", "utf8")).toEqual("hello"); + }); + + it("cp works on a directory", async () => { + await fs.mkdir("folder"); + await fs.writeFile("folder/a.txt", "hello"); + await fs.cp("folder", "folder2", { recursive: true }); + expect(await fs.readFile("folder2/a.txt", "utf8")).toEqual("hello"); + }); + + it("exists works", async () => { + expect(await fs.exists("does-not-exist")).toBe(false); + await fs.writeFile("does-exist", ""); + expect(await fs.exists("does-exist")).toBe(true); + }); + + it("creating a hard link works", async () => { + await fs.writeFile("source", "the source"); + await fs.link("source", "target"); + expect(await fs.readFile("target", "utf8")).toEqual("the source"); + // hard link, not symlink + expect(await fs.realpath("target")).toBe("target"); + + await fs.appendFile("source", " and more"); + expect(await fs.readFile("target", "utf8")).toEqual("the source and more"); + }); + + it("mkdir works", async () => { + await fs.mkdir("xyz"); + const s = await fs.stat("xyz"); + expect(s.isDirectory()).toBe(true); + expect(s.isFile()).toBe(false); + }); + + it("readFile works", async () => { + await fs.writeFile("a", Buffer.from([1, 2, 3])); + const s = await fs.readFile("a"); + expect(s).toEqual(Buffer.from([1, 2, 3])); + + await fs.writeFile("b.txt", "conat"); + const t = await fs.readFile("b.txt", "utf8"); + expect(t).toEqual("conat"); + }); + + it("the full error message structure is preserved exactly as in the nodejs library", async () => { + const path = randomId(); + try { + await fs.readFile(path); + } catch (err) { + expect(err.message).toEqual( + `ENOENT: no such file or directory, open '${path}'`, + ); + expect(err.message).toContain(path); + expect(err.code).toEqual("ENOENT"); + expect(err.errno).toEqual(-2); + expect(err.path).toEqual(path); + expect(err.syscall).toEqual("open"); + } + }); + + let fire; + it("readdir works", async () => { + await fs.mkdir("dirtest"); + for (let i = 0; i < 5; i++) { + await fs.writeFile(`dirtest/${i}`, `${i}`); + } + fire = "🔥.txt"; + await fs.writeFile(join("dirtest", fire), "this is ️‍🔥!"); + const v = await fs.readdir("dirtest"); + expect(v).toEqual(["0", "1", "2", "3", "4", fire]); + }); + + it("readdir with the withFileTypes option", async () => { + const path = "readdir-1"; + await fs.mkdir(path); + expect(await fs.readdir(path, { withFileTypes: true })).toEqual([]); + await fs.writeFile(join(path, "a.txt"), ""); + + { + const v = await fs.readdir(path, { withFileTypes: true }); + expect(v.map(({ name }) => name)).toEqual(["a.txt"]); + expect(v.map((x) => x.isFile())).toEqual([true]); + } + { + await fs.mkdir(join(path, "co")); + const v = await fs.readdir(path, { withFileTypes: true }); + expect(v.map(({ name }) => name)).toEqual(["a.txt", "co"]); + expect(v.map((x) => x.isFile())).toEqual([true, false]); + expect(v.map((x) => x.isDirectory())).toEqual([false, true]); + } + + { + await fs.symlink(join(path, "a.txt"), join(path, "link")); + const v = await fs.readdir(path, { withFileTypes: true }); + expect(v.map(({ name }) => name)).toEqual(["a.txt", "co", "link"]); + expect(v[2].isSymbolicLink()).toEqual(true); + } + }); + + it("readdir with the recursive option", async () => { + const path = "readdir-2"; + await fs.mkdir(path); + expect(await fs.readdir(path, { recursive: true })).toEqual([]); + await fs.mkdir(join(path, "subdir")); + await fs.writeFile(join(path, "subdir", "b.txt"), "x"); + const v = await fs.readdir(path, { recursive: true }); + expect(v).toEqual(["subdir", "subdir/b.txt"]); + + // and withFileTypes + const w = await fs.readdir(path, { recursive: true, withFileTypes: true }); + expect(w.map(({ name }) => name)).toEqual(["subdir", "b.txt"]); + expect(w[0]).toEqual( + expect.objectContaining({ + name: "subdir", + parentPath: path, + path, + }), + ); + expect(w[0].isDirectory()).toBe(true); + expect(w[1]).toEqual( + expect.objectContaining({ + name: "b.txt", + parentPath: join(path, "subdir"), + path: join(path, "subdir"), + }), + ); + expect(w[1].isFile()).toBe(true); + expect(await fs.readFile(join(w[1].path, w[1].name), "utf8")).toEqual("x"); + }); + + it("readdir works with non-utf8 filenames in the path", async () => { + // this test uses internal implementation details (kind of crappy) + const path = "readdir-3"; + await fs.mkdir(path); + const fullPath = join(server.path, project_id, path); + + const buf = Buffer.from([0xff, 0xfe, 0xfd]); + expect(() => { + const decoder = new TextDecoder("utf-8", { fatal: true }); + decoder.decode(buf); + }).toThrow("not valid"); + const c = process.cwd(); + process.chdir(fullPath); + await writeFile(buf, "hi"); + process.chdir(c); + const w = await fs.readdir(path, { encoding: "buffer" }); + expect(w[0]).toEqual(buf); + }); + + it("use the find command instead of readdir", async () => { + const { stdout } = await fs.find("dirtest", { + options: ["-maxdepth", "1", "-mindepth", "1", "-printf", "%f\n"], + }); + const v = stdout.toString().trim().split("\n"); + // output of find is NOT in alphabetical order: + expect(new Set(v)).toEqual(new Set(["0", "1", "2", "3", "4", fire])); + }); + + it("realpath works", async () => { + await fs.writeFile("file0", "file0"); + await fs.symlink("file0", "file1"); + expect(await fs.readFile("file1", "utf8")).toBe("file0"); + const r = await fs.realpath("file1"); + expect(r).toBe("file0"); + + await fs.writeFile("file2", "file2"); + await fs.link("file2", "file3"); + expect(await fs.readFile("file3", "utf8")).toBe("file2"); + const r3 = await fs.realpath("file3"); + expect(r3).toBe("file3"); + }); + + it("rename a file", async () => { + await fs.writeFile("bella", "poo"); + await fs.rename("bella", "bells"); + expect(await fs.readFile("bells", "utf8")).toBe("poo"); + await fs.mkdir("x"); + await fs.rename("bells", "x/belltown"); + }); + + it("rm a file", async () => { + await fs.writeFile("bella-to-rm", "poo"); + await fs.rm("bella-to-rm"); + expect(await fs.exists("bella-to-rm")).toBe(false); + }); + + it("rm a directory", async () => { + await fs.mkdir("rm-dir"); + expect(async () => { + await fs.rm("rm-dir"); + }).rejects.toThrow("Path is a directory"); + await fs.rm("rm-dir", { recursive: true }); + expect(await fs.exists("rm-dir")).toBe(false); + }); + + it("rm a nonempty directory", async () => { + await fs.mkdir("rm-dir2"); + await fs.writeFile("rm-dir2/a", "a"); + await fs.rm("rm-dir2", { recursive: true }); + expect(await fs.exists("rm-dir2")).toBe(false); + }); + + it("rmdir empty directory", async () => { + await fs.mkdir("rm-dir3"); + await fs.rmdir("rm-dir3"); + expect(await fs.exists("rm-dir3")).toBe(false); + }); + + it("stat not existing path", async () => { + expect(async () => { + await fs.stat(randomId()); + }).rejects.toThrow("no such file or directory"); + }); + + it("stat a file", async () => { + await fs.writeFile("abc.txt", "hi"); + const stat = await fs.stat("abc.txt"); + expect(stat.size).toBe(2); + expect(stat.isFile()).toBe(true); + expect(stat.isSymbolicLink()).toBe(false); + expect(stat.isDirectory()).toBe(false); + expect(stat.isBlockDevice()).toBe(false); + expect(stat.isCharacterDevice()).toBe(false); + expect(stat.isSymbolicLink()).toBe(false); + expect(stat.isFIFO()).toBe(false); + expect(stat.isSocket()).toBe(false); + }); + + it("stat a directory", async () => { + await fs.mkdir("my-stat-dir"); + const stat = await fs.stat("my-stat-dir"); + expect(stat.isFile()).toBe(false); + expect(stat.isSymbolicLink()).toBe(false); + expect(stat.isDirectory()).toBe(true); + expect(stat.isBlockDevice()).toBe(false); + expect(stat.isCharacterDevice()).toBe(false); + expect(stat.isSymbolicLink()).toBe(false); + expect(stat.isFIFO()).toBe(false); + expect(stat.isSocket()).toBe(false); + }); + + it("stat a symlink", async () => { + await fs.writeFile("sl2", "the source"); + await fs.symlink("sl2", "target-sl2"); + const stat = await fs.stat("target-sl2"); + // this is how stat works! + expect(stat.isFile()).toBe(true); + expect(stat.isSymbolicLink()).toBe(false); + // so use lstat + const lstat = await fs.lstat("target-sl2"); + expect(lstat.isFile()).toBe(false); + expect(lstat.isSymbolicLink()).toBe(true); + }); + + it("truncate a file", async () => { + await fs.writeFile("t", ""); + await fs.truncate("t", 10); + const s = await fs.stat("t"); + expect(s.size).toBe(10); + }); + + it("delete a file with unlink", async () => { + await fs.writeFile("to-unlink", ""); + await fs.unlink("to-unlink"); + expect(await fs.exists("to-unlink")).toBe(false); + }); + + it("sets times of a file", async () => { + await fs.writeFile("my-times", ""); + const atime = Date.now() - 100_000; + const mtime = Date.now() - 10_000_000; + // NOTE: fs.utimes in nodejs takes *seconds*, not ms, hence + // dividing by 1000 here: + await fs.utimes("my-times", atime / 1000, mtime / 1000); + const s = await fs.stat("my-times"); + expect(s.atimeMs).toBeCloseTo(atime); + expect(s.mtimeMs).toBeCloseTo(mtime); + expect(s.atime.valueOf()).toBeCloseTo(atime); + expect(s.mtime.valueOf()).toBeCloseTo(mtime); + }); + + it("creating a symlink works (as does using lstat)", async () => { + await fs.writeFile("source1", "the source"); + await fs.symlink("source1", "target1"); + expect(await fs.readFile("target1", "utf8")).toEqual("the source"); + // symlink, not hard + expect(await fs.realpath("target1")).toBe("source1"); + await fs.appendFile("source1", " and more"); + expect(await fs.readFile("target1", "utf8")).toEqual("the source and more"); + const stats = await fs.stat("target1"); + expect(stats.isSymbolicLink()).toBe(false); + + const lstats = await fs.lstat("target1"); + expect(lstats.isSymbolicLink()).toBe(true); + + const stats0 = await fs.stat("source1"); + expect(stats0.isSymbolicLink()).toBe(false); + }); + + it("watch a file", async () => { + await fs.writeFile("a.txt", "hi"); + const w = await fs.watch("a.txt"); + await fs.appendFile("a.txt", " there"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "change", filename: "a.txt" }, + }); + }); + + it("watch a directory", async () => { + const FOLDER = randomId(); + await fs.mkdir(FOLDER); + const w = await fs.watch(FOLDER); + + await fs.writeFile(join(FOLDER, "x"), "hi"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "rename", filename: "x" }, + }); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "change", filename: "x" }, + }); + + await fs.appendFile(join(FOLDER, "x"), "xxx"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "change", filename: "x" }, + }); + + await fs.writeFile(join(FOLDER, "z"), "there"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "rename", filename: "z" }, + }); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "change", filename: "z" }, + }); + + // this is correct -- from the node docs "On most platforms, 'rename' is emitted whenever a filename appears or disappears in the directory." + await fs.unlink(join(FOLDER, "z")); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "rename", filename: "z" }, + }); + }); +}); + +describe("SECURITY CHECKS: dangerous symlinks can't be followed", () => { + let server; + let tempDir; + it("creates the simple fileserver service", async () => { + server = await createPathFileserver(); + tempDir = server.path; + }); + + const project_id = uuid(); + const project_id2 = uuid(); + let fs, fs2; + it("create two clients", () => { + fs = fsClient({ subject: `${server.service}.project-${project_id}` }); + fs2 = fsClient({ subject: `${server.service}.project-${project_id2}` }); + }); + + it("create a secret in one", async () => { + await fs.writeFile("password", "s3cr3t"); + await fs2.writeFile("a", "init"); + }); + + // This is setup bypassing security and is part of our threat model, due to users + // having full access internally to their sandbox fs. + it("directly create a dangerous file that is a symlink outside of the sandbox -- this should work", async () => { + await symlink( + join(tempDir, project_id, "password"), + join(tempDir, project_id2, "danger"), + ); + const s = await readFile(join(tempDir, project_id2, "danger"), "utf8"); + expect(s).toBe("s3cr3t"); + }); + + it("fails to read the symlink content via the api", async () => { + await expect(async () => { + await fs2.readFile("danger", "utf8"); + }).rejects.toThrow("outside of sandbox"); + }); + + it("directly create a dangerous relative symlink ", async () => { + await symlink( + join("..", project_id, "password"), + join(tempDir, project_id2, "danger2"), + ); + const s = await readFile(join(tempDir, project_id2, "danger2"), "utf8"); + expect(s).toBe("s3cr3t"); + }); + + it("fails to read the relative symlink content via the api", async () => { + await expect(async () => { + await fs2.readFile("danger2", "utf8"); + }).rejects.toThrow("outside of sandbox"); + }); + + // This is not a vulnerability, because there's no way for the user + // to create a hard link like this from within an nfs mount (say) + // of their own folder. + it("directly create a hard link", async () => { + await link( + join(tempDir, project_id, "password"), + join(tempDir, project_id2, "danger3"), + ); + const s = await readFile(join(tempDir, project_id2, "danger3"), "utf8"); + expect(s).toBe("s3cr3t"); + }); + + it("a hardlink *can* get outside the sandbox", async () => { + const s = await fs2.readFile("danger3", "utf8"); + expect(s).toBe("s3cr3t"); + }); + + it("closes the server", () => { + server.close(); + }); +}); + +afterAll(async () => { + await after(); + await cleanupFileservers(); +}); diff --git a/src/packages/backend/conat/files/test/util.ts b/src/packages/backend/conat/files/test/util.ts new file mode 100644 index 00000000000..b50a114c5da --- /dev/null +++ b/src/packages/backend/conat/files/test/util.ts @@ -0,0 +1,30 @@ +import { localPathFileserver } from "../local-path"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "path"; +import { client } from "@cocalc/backend/conat/test/setup"; +import { randomId } from "@cocalc/conat/names"; + +const tempDirs: string[] = []; +const servers: any[] = []; +export async function createPathFileserver({ + service = `fs-${randomId()}`, +}: { service?: string } = {}) { + const tempDir = await mkdtemp(join(tmpdir(), `cocalc-${randomId()}0`)); + tempDirs.push(tempDir); + const server = await localPathFileserver({ client, service, path: tempDir }); + servers.push(server); + return server; +} + +// clean up any +export async function cleanupFileservers() { + for (const server of servers) { + server.close(); + } + for (const tempDir of tempDirs) { + try { + await rm(tempDir, { force: true, recursive: true }); + } catch {} + } +} diff --git a/src/packages/backend/conat/files/test/watch.test.ts b/src/packages/backend/conat/files/test/watch.test.ts new file mode 100644 index 00000000000..c1cc6d608a3 --- /dev/null +++ b/src/packages/backend/conat/files/test/watch.test.ts @@ -0,0 +1,75 @@ +import { before, after, client, wait } from "@cocalc/backend/conat/test/setup"; +import { watchServer, watchClient } from "@cocalc/conat/files/watch"; +import { SandboxedFilesystem } from "@cocalc/backend/sandbox"; +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "path"; +import { randomId } from "@cocalc/conat/names"; + +let tmp; +beforeAll(async () => { + await before(); + tmp = await mkdtemp(join(tmpdir(), `cocalc-${randomId()}0`)); +}); +afterAll(async () => { + await after(); + try { + await rm(tmp, { force: true, recursive: true }); + } catch {} +}); + +describe("basic core of the async path watch functionality", () => { + let fs; + it("creates sandboxed filesystem", () => { + fs = new SandboxedFilesystem(tmp); + }); + + let server; + it("create watch server", () => { + server = watchServer({ client, subject: "foo", watch: fs.watch }); + }); + + it("create a file", async () => { + await fs.writeFile("a.txt", "hi"); + }); + + let w; + it("create a watcher client for 'a.txt'", async () => { + w = await watchClient({ client, subject: "foo", path: "a.txt" }); + }); + + it("observe watch works", async () => { + await fs.appendFile("a.txt", "foo"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "change", filename: "a.txt" }, + }); + + await fs.appendFile("a.txt", "bar"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "change", filename: "a.txt" }, + }); + }); + + it("close the watcher client frees up a server socket", async () => { + expect(Object.keys(server.sockets).length).toEqual(1); + w.close(); + await wait({ until: () => Object.keys(server.sockets).length == 0 }); + expect(Object.keys(server.sockets).length).toEqual(0); + }); + + it("trying to watch file that does not exist throws error", async () => { + await expect(async () => { + await watchClient({ client, subject: "foo", path: "b.txt" }); + }).rejects.toThrow( + "Error: ENOENT: no such file or directory, watch 'b.txt'", + ); + + try { + await watchClient({ client, subject: "foo", path: "b.txt" }); + } catch (err) { + expect(err.code).toEqual("ENOENT"); + } + }); +}); diff --git a/src/packages/backend/conat/sync.ts b/src/packages/backend/conat/sync.ts index 3bccd54978a..50963ac6960 100644 --- a/src/packages/backend/conat/sync.ts +++ b/src/packages/backend/conat/sync.ts @@ -7,7 +7,6 @@ import { dkv as createDKV, type DKV, type DKVOptions } from "@cocalc/conat/sync/ import { dko as createDKO, type DKO } from "@cocalc/conat/sync/dko"; import { akv as createAKV, type AKV } from "@cocalc/conat/sync/akv"; import { astream as createAStream, type AStream } from "@cocalc/conat/sync/astream"; -import { createOpenFiles, type OpenFiles } from "@cocalc/conat/sync/open-files"; export { inventory } from "@cocalc/conat/sync/inventory"; import "./index"; @@ -35,6 +34,3 @@ export async function dko(opts: DKVOptions): Promise> { return await createDKO(opts); } -export async function openFiles(project_id: string, opts?): Promise { - return await createOpenFiles({ project_id, ...opts }); -} diff --git a/src/packages/backend/conat/test/files/file-server.test.ts b/src/packages/backend/conat/test/files/file-server.test.ts new file mode 100644 index 00000000000..e31e8354796 --- /dev/null +++ b/src/packages/backend/conat/test/files/file-server.test.ts @@ -0,0 +1,145 @@ +import { before, after, connect } from "@cocalc/backend/conat/test/setup"; +import { + server as createFileServer, + client as createFileClient, +} from "@cocalc/conat/files/file-server"; +import { uuid } from "@cocalc/util/misc"; + +beforeAll(before); + +describe("create basic mocked file server and test it out", () => { + let client1, client2; + it("create two clients", () => { + client1 = connect(); + client2 = connect(); + }); + + const volumes = new Set(); + const quotaSize: { [project_id: string]: number } = {}; + it("create file server", async () => { + await createFileServer({ + client: client1, + mount: async ({ project_id }): Promise<{ path: string }> => { + volumes.add(project_id); + return { path: `/mnt/${project_id}` }; + }, + + // create project_id as an exact lightweight clone of src_project_id + clone: async (opts: { + project_id: string; + src_project_id: string; + }): Promise => { + volumes.add(opts.project_id); + }, + + getUsage: async (_opts: { + project_id: string; + }): Promise<{ + size: number; + used: number; + free: number; + }> => { + return { size: 0, used: 0, free: 0 }; + }, + + getQuota: async (_opts: { + project_id: string; + }): Promise<{ + size: number; + used: number; + }> => { + return { size: quotaSize[project_id] ?? 0, used: 0 }; + }, + + setQuota: async ({ + project_id, + size, + }: { + project_id: string; + size: number | string; + }): Promise => { + quotaSize[project_id] = typeof size == "string" ? parseInt(size) : size; + }, + + cp: async (_opts: { + // the src paths are relative to the src volume + src: { project_id: string; path: string | string[] }; + // the dest path is relative to the dest volume + dest: { project_id: string; path: string }; + options?; + }): Promise => {}, + + backup: async (_opts: { + project_id: string; + }): Promise<{ time: Date; id: string }> => { + return { time: new Date(), id: "0" }; + }, + + restore: async (_opts: { + project_id: string; + id: string; + path?: string; + dest?: string; + }): Promise => {}, + + deleteBackup: async (_opts: { + project_id: string; + id: string; + }): Promise => {}, + + getBackups: async (_opts: { + project_id: string; + }): Promise< + { + id: string; + time: Date; + }[] + > => { + return []; + }, + + getBackupFiles: async (_opts: { + project_id: string; + id: string; + }): Promise => { + return []; + }, + }); + }); + + let project_id; + it("make a client and test the file server", async () => { + project_id = uuid(); + const fileClient = createFileClient({ + client: client2, + }); + const { path } = await fileClient.mount({ project_id }); + expect(path).toEqual(`/mnt/${project_id}`); + expect(volumes.has(project_id)); + + expect(await fileClient.getUsage({ project_id })).toEqual({ + size: 0, + used: 0, + free: 0, + }); + + expect(await fileClient.getQuota({ project_id })).toEqual({ + size: 0, + used: 0, + }); + + await fileClient.setQuota({ project_id, size: 10 }); + + expect(await fileClient.getQuota({ project_id })).toEqual({ + size: 10, + used: 0, + }); + + await fileClient.cp({ + src: { project_id, path: "x" }, + dest: { project_id, path: "y" }, + }); + }); +}); + +afterAll(after); diff --git a/src/packages/backend/conat/test/llm.test.ts b/src/packages/backend/conat/test/llm.test.ts index 9ac69af07fb..ab0eeb8ece8 100644 --- a/src/packages/backend/conat/test/llm.test.ts +++ b/src/packages/backend/conat/test/llm.test.ts @@ -18,7 +18,7 @@ beforeAll(before); describe("create an llm server, client, and stub evaluator, and run an evaluation", () => { // define trivial evaluate - const OUTPUT = "Thanks for asing about "; + const OUTPUT = "Thanks for asking about "; async function evaluate({ input, stream }) { stream(OUTPUT); stream(input); @@ -52,7 +52,7 @@ describe("create an llm server, client, and stub evaluator, and run an evaluatio describe("test an evaluate that throws an error half way through", () => { // define trivial evaluate - const OUTPUT = "Thanks for asing about "; + const OUTPUT = "Thanks for asking about "; const ERROR = "I give up"; async function evaluate({ stream }) { stream(OUTPUT); diff --git a/src/packages/backend/conat/test/project/jupyter/run-code.test.ts b/src/packages/backend/conat/test/project/jupyter/run-code.test.ts new file mode 100644 index 00000000000..460442324b8 --- /dev/null +++ b/src/packages/backend/conat/test/project/jupyter/run-code.test.ts @@ -0,0 +1,272 @@ +/* + +DEVELOPMENT: + +pnpm test `pwd`/run-code.test.ts + +*/ + +import { + before, + after, + connect, + delay, + wait, +} from "@cocalc/backend/conat/test/setup"; +import { + jupyterClient, + jupyterServer, +} from "@cocalc/conat/project/jupyter/run-code"; +import { uuid } from "@cocalc/util/misc"; + +// it's really 100+, but tests fails if less than this. +const MIN_EVALS_PER_SECOND = 10; + +beforeAll(before); + +describe("create very simple mocked jupyter runner and test evaluating code", () => { + let client1, client2; + it("create two clients", async () => { + client1 = connect(); + client2 = connect(); + }); + + let server; + const project_id = uuid(); + it("create jupyter code run server", () => { + // running code with this just results in two responses: the path and the cells + async function run({ path, cells }) { + async function* runner() { + yield { path, id: "0" }; + yield { cells, id: "0" }; + } + return runner(); + } + + server = jupyterServer({ client: client1, project_id, run }); + }); + + let client; + const path = "a.ipynb"; + const cells = [{ id: "a", input: "2+3" }]; + it("create a jupyter client, then run some code", async () => { + client = jupyterClient({ path, project_id, client: client2 }); + const iter = await client.run(cells); + const v: any[] = []; + for await (const output of iter) { + v.push(...output); + } + expect(v).toEqual([ + { path, id: "0" }, + { cells, id: "0" }, + ]); + }); + + it("start iterating over the output after waiting", async () => { + // this is the same as the previous test, except we insert a + // delay from when we create the iterator, and when we start + // reading values out of it. This is important to test, because + // it was broken in my first implementation, and is a common mistake + // when implementing async iterators. + client.verbose = true; + const iter = await client.run(cells); + const v: any[] = []; + await delay(500); + for await (const output of iter) { + v.push(...output); + } + expect(v).toEqual([ + { path, id: "0" }, + { cells, id: "0" }, + ]); + }); + + const count = 100; + it(`run ${count} evaluations to ensure that the speed is reasonable (and also everything is kept properly ordered, etc.)`, async () => { + const start = Date.now(); + for (let i = 0; i < count; i++) { + const v: any[] = []; + const cells = [{ id: `${i}`, input: `${i} + ${i}` }]; + for await (const output of await client.run(cells)) { + v.push(...output); + } + expect(v).toEqual([ + { path, id: "0" }, + { cells, id: "0" }, + ]); + } + const evalsPerSecond = Math.floor((1000 * count) / (Date.now() - start)); + if (process.env.BENCH) { + console.log({ evalsPerSecond }); + } + expect(evalsPerSecond).toBeGreaterThan(MIN_EVALS_PER_SECOND); + }); + + it("cleans up", () => { + server.close(); + client.close(); + }); +}); + +describe("create simple mocked jupyter runner that does actually eval an expression", () => { + let client1, client2; + it("create two clients", async () => { + client1 = connect(); + client2 = connect(); + }); + + let server; + const project_id = uuid(); + const compute_server_id = 3; + it("create jupyter code run server", () => { + // running code with this just results in two responses: the path and the cells + async function run({ cells }) { + async function* runner() { + for (const { id, input } of cells) { + yield { id, output: eval(input) }; + } + } + return runner(); + } + + server = jupyterServer({ + client: client1, + project_id, + run, + compute_server_id, + }); + }); + + let client; + const path = "b.ipynb"; + const cells = [ + { id: "a", input: "2+3" }, + { id: "b", input: "3**5" }, + ]; + it("create a jupyter client, then run some code", async () => { + client = jupyterClient({ + path, + project_id, + client: client2, + compute_server_id, + }); + const iter = await client.run(cells); + const v: any[] = []; + for await (const output of iter) { + v.push(...output); + } + expect(v).toEqual([ + { id: "a", output: 5 }, + { id: "b", output: 243 }, + ]); + }); + + it("run code that FAILS and see error is visible to client properly", async () => { + const iter = await client.run([ + { id: "a", input: "2+3" }, + { id: "b", input: "2+invalid" }, + ]); + try { + for await (const _ of iter) { + } + } catch (err) { + expect(`${err}`).toContain("ReferenceError: invalid is not defined"); + } + }); + + it("cleans up", () => { + server.close(); + client.close(); + }); +}); + +describe("create mocked jupyter runner that does failover to backend output management when client disconnects", () => { + let client1, client2; + it("create two clients", async () => { + client1 = connect(); + client2 = connect(); + }); + + const path = "b.ipynb"; + const cells = [ + { id: "a", input: "10*(2+3)" }, + { id: "b", input: "100" }, + ]; + let server; + const project_id = uuid(); + let handler: any = null; + + it("create jupyter code run server that also takes as long as the output to run", () => { + async function run({ cells }) { + async function* runner() { + for (const { id, input } of cells) { + const output = eval(input); + await delay(output); + yield { id, output }; + } + } + return runner(); + } + + class OutputHandler { + messages: any[] = []; + + constructor(public cells) {} + + process = (mesg: any) => { + this.messages.push(mesg); + }; + done = () => { + this.messages.push({ done: true }); + }; + } + + function outputHandler({ path: path0, cells }) { + if (path0 != path) { + throw Error(`path must be ${path}`); + } + handler = new OutputHandler(cells); + return handler; + } + + server = jupyterServer({ + client: client1, + project_id, + run, + outputHandler, + }); + }); + + let client; + it("create a jupyter client, then run some code (doesn't use output handler)", async () => { + client = jupyterClient({ + path, + project_id, + client: client2, + }); + const iter = await client.run(cells); + const v: any[] = []; + for await (const output of iter) { + v.push(output); + } + expect(v).toEqual([[{ id: "a", output: 50 }], [{ id: "b", output: 100 }]]); + }); + + it("starts code running then closes the client, which causes output to have to be placed in the handler instead.", async () => { + await client.run(cells); + client.close(); + await wait({ until: () => handler.messages.length >= 3 }); + expect(handler.messages).toEqual([ + { id: "a", output: 50 }, + { id: "b", output: 100 }, + { done: true }, + ]); + }); + + it("cleans up", () => { + server.close(); + client.close(); + }); +}); + +afterAll(after); diff --git a/src/packages/backend/conat/test/project/runner/load-balancer-2.test.ts b/src/packages/backend/conat/test/project/runner/load-balancer-2.test.ts new file mode 100644 index 00000000000..2e4209e8f2d --- /dev/null +++ b/src/packages/backend/conat/test/project/runner/load-balancer-2.test.ts @@ -0,0 +1,93 @@ +/* + +DEVELOPMENT: + + +pnpm test `pwd`/load-balancer-2.test.ts + + +*/ + +import { before, after, connect } from "@cocalc/backend/conat/test/setup"; +import { + server as projectRunnerServer, +} from "@cocalc/conat/project/runner/run"; +import { + server as lbServer, + client as lbClient, +} from "@cocalc/conat/project/runner/load-balancer"; +import { uuid } from "@cocalc/util/misc"; + +beforeAll(before); + +describe("create runner and load balancer with getConfig function", () => { + let client1, client2; + it("create two clients", () => { + client1 = connect(); + client2 = connect(); + }); + + const running: { [project_id: string]: any } = {}; + const projectState: { [project_id: string]: any } = {}; + + it("create project runner server and load balancer with getConfig and setState functions", async () => { + await projectRunnerServer({ + id: "0", + client: client1, + start: async ({ project_id, config }) => { + running[project_id] = { ...config }; + }, + stop: async ({ project_id }) => { + delete running[project_id]; + }, + status: async ({ project_id }) => { + return running[project_id] != null + ? { state: "running" } + : { state: "opened" }; + }, + }); + await lbServer({ + client: client1, + getConfig: async ({ project_id }) => { + return { name: project_id }; + }, + setState: async ({ project_id, state }) => { + projectState[project_id] = state; + }, + }); + }); + + it("make a client for the load balancer, and test the runner via the load balancer", async () => { + const project_id = uuid(); + const lbc = lbClient({ + subject: `project.${project_id}.run`, + client: client2, + }); + await lbc.start(); + + expect(projectState).toEqual({ [project_id]: "running" }); + expect(running[project_id]).toEqual({ name: project_id }); + + expect(await lbc.status()).toEqual({ + server: "0", + state: "running", + }); + + const lbc2 = lbClient({ + subject: `project.${uuid()}.run`, + client: client2, + }); + expect(await lbc2.status()).toEqual({ + server: "0", + state: "opened", + }); + + await lbc.stop(); + expect(await lbc.status()).toEqual({ + server: "0", + state: "opened", + }); + }); +}); + +afterAll(after); diff --git a/src/packages/backend/conat/test/project/runner/load-balancer.test.ts b/src/packages/backend/conat/test/project/runner/load-balancer.test.ts new file mode 100644 index 00000000000..5ea5934833b --- /dev/null +++ b/src/packages/backend/conat/test/project/runner/load-balancer.test.ts @@ -0,0 +1,103 @@ +/* + +DEVELOPMENT: + +pnpm test `pwd`/load-balancer.test.ts + +*/ + +import { before, after, connect } from "@cocalc/backend/conat/test/setup"; +import { + server as projectRunnerServer, + client as projectRunnerClient, +} from "@cocalc/conat/project/runner/run"; +import { + server as lbServer, + client as lbClient, +} from "@cocalc/conat/project/runner/load-balancer"; +import { uuid } from "@cocalc/util/misc"; + +beforeAll(before); + +describe("create basic mocked project runner service and test", () => { + let client1, client2; + it("create two clients", () => { + client1 = connect(); + client2 = connect(); + }); + + it("create project runner server", async () => { + const running = new Set(); + await projectRunnerServer({ + id: "0", + client: client1, + start: async ({ project_id }) => { + running.add(project_id); + }, + stop: async ({ project_id }) => { + running.delete(project_id); + }, + status: async ({ project_id }) => { + return running.has(project_id) + ? { state: "running" } + : { state: "opened" }; + }, + }); + }); + + it("make a client and test the server", async () => { + const project_id = uuid(); + const runClient = projectRunnerClient({ + subject: "project-runner.0", + client: client2, + }); + await runClient.start({ project_id }); + expect(await runClient.status({ project_id })).toEqual({ + server: "0", + state: "running", + }); + expect(await runClient.status({ project_id: uuid() })).toEqual({ + server: "0", + state: "opened", + }); + await runClient.stop({ project_id }); + expect(await runClient.status({ project_id })).toEqual({ + server: "0", + state: "opened", + }); + }); + + it("make a load balancer", async () => { + await lbServer({ client: client1 }); + }); + + it("make a client for the load balancer, and test the runner via the load balancer", async () => { + const project_id = uuid(); + const lbc = lbClient({ + subject: `project.${project_id}.run`, + client: client2, + }); + await lbc.start(); + expect(await lbc.status()).toEqual({ + server: "0", + state: "running", + }); + + const lbc2 = lbClient({ + subject: `project.${uuid()}.run`, + client: client2, + }); + expect(await lbc2.status()).toEqual({ + server: "0", + state: "opened", + }); + + await lbc.stop(); + expect(await lbc.status()).toEqual({ + server: "0", + state: "opened", + }); + }); +}); + +afterAll(after); diff --git a/src/packages/backend/conat/test/project/runner/run.test.ts b/src/packages/backend/conat/test/project/runner/run.test.ts new file mode 100644 index 00000000000..877e4d8dce4 --- /dev/null +++ b/src/packages/backend/conat/test/project/runner/run.test.ts @@ -0,0 +1,104 @@ +/* + +DEVELOPMENT: + +pnpm test `pwd`/run-code.test.ts + +*/ + +import { before, after, connect, wait } from "@cocalc/backend/conat/test/setup"; +import { + server as projectRunnerServer, + client as projectRunnerClient, +} from "@cocalc/conat/project/runner/run"; +import { uuid } from "@cocalc/util/misc"; +import state from "@cocalc/conat/project/runner/state"; + +beforeAll(before); + +describe("create basic mocked project runner service and test", () => { + let client1, client2; + it("create two clients", () => { + client1 = connect(); + client2 = connect(); + }); + + it("create project runner server", async () => { + const running = new Set(); + await projectRunnerServer({ + id: "0", + client: client1, + start: async ({ project_id }) => { + running.add(project_id); + }, + stop: async ({ project_id }) => { + running.delete(project_id); + }, + status: async ({ project_id }) => + running.has(project_id) ? { state: "running" } : { state: "opened" }, + }); + }); + + let project_id; + it("make a client and test the server", async () => { + project_id = uuid(); + const runClient = projectRunnerClient({ + subject: "project-runner.0", + client: client2, + }); + await runClient.start({ project_id }); + expect(await runClient.status({ project_id })).toEqual({ + state: "running", + server: "0", + }); + expect(await runClient.status({ project_id: uuid() })).toEqual({ + state: "opened", + server: "0", + }); + await runClient.stop({ project_id }); + expect(await runClient.status({ project_id })).toEqual({ + state: "opened", + server: "0", + }); + }); + + it("get the status of the runner", async () => { + const { projects, runners } = await state({ client: client2 }); + expect(runners.getAll()).toEqual({ "0": { time: runners.get("0")?.time } }); + await wait({ until: () => projects.get(project_id)?.state == "opened" }); + expect(projects.get(project_id)).toEqual({ state: "opened", server: "0" }); + }); + + it("add another runner and observe it appears", async () => { + const running = new Set(); + await projectRunnerServer({ + id: "1", + client: client1, + start: async ({ project_id }) => { + running.add(project_id); + }, + stop: async ({ project_id }) => { + running.delete(project_id); + }, + status: async ({ project_id }) => + running.has(project_id) ? { state: "running" } : { state: "opened" }, + }); + + const { runners } = await state({ client: client2 }); + await wait({ + until: () => runners.get("1") != null, + }); + }); + + it("run a projects on server 1", async () => { + const runClient = projectRunnerClient({ + subject: "project-runner.1", + client: client2, + }); + const project_id = uuid(); + const x = await runClient.start({ project_id }); + expect(x.server).toEqual("1"); + }); +}); + +afterAll(after); diff --git a/src/packages/backend/conat/test/setup.ts b/src/packages/backend/conat/test/setup.ts index 151eff2227a..e36ba0ee05f 100644 --- a/src/packages/backend/conat/test/setup.ts +++ b/src/packages/backend/conat/test/setup.ts @@ -21,6 +21,7 @@ import { once } from "@cocalc/util/async-utils"; import { until } from "@cocalc/util/async-utils"; import { randomId } from "@cocalc/conat/names"; import { isEqual } from "lodash"; +import { setConatServer } from "@cocalc/backend/data"; export { wait, delay, once }; @@ -150,6 +151,7 @@ export async function before( } server = await createServer(); + setConatServer(server.address()); client = connect(); persistServer = createPersistServer({ client }); setConatClient({ @@ -299,7 +301,15 @@ export async function waitForConsistentState( export async function after() { persistServer?.close(); - await rm(tempDir, { force: true, recursive: true }); + while (true) { + try { + await rm(tempDir, { force: true, recursive: true }); + break; + } catch (err) { + console.log(err); + await delay(1000); + } + } try { server?.close(); } catch {} diff --git a/src/packages/backend/conat/test/socket/basic.test.ts b/src/packages/backend/conat/test/socket/basic.test.ts index 1ae6e364b2f..36e4467b2ac 100644 --- a/src/packages/backend/conat/test/socket/basic.test.ts +++ b/src/packages/backend/conat/test/socket/basic.test.ts @@ -168,7 +168,7 @@ describe("create a client first and write more messages than the queue size resu }); it("wait for client to drain; then we can now send another message without an error", async () => { - await client.waitUntilDrain(); + await client.drain(); client.write("foo"); }); diff --git a/src/packages/backend/conat/test/sync-doc/conflict.test.ts b/src/packages/backend/conat/test/sync-doc/conflict.test.ts new file mode 100644 index 00000000000..739b33da7e6 --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/conflict.test.ts @@ -0,0 +1,277 @@ +/* +Illustrate and test behavior when there is conflict. + +TODO: we must get noAutosave to fully work so we can make +the tests of conflicts, etc., better. + +E.g, the test below WILL RANDOMLY FAIL right now due to autosave randomness... +*/ + +import { + before, + after, + uuid, + connect, + server, + once, + delay, + waitUntilSynced, +} from "./setup"; +import { split } from "@cocalc/util/misc"; + +beforeAll(before); +afterAll(after); + +const GAP_DELAY = 50; + +describe("synchronized editing with branching and merging", () => { + const project_id = uuid(); + let s1, s2, client1, client2; + + it("creates two clients", async () => { + client1 = connect(); + client2 = connect(); + s1 = client1.sync.string({ + project_id, + path: "a.txt", + service: server.service, + noAutosave: true, + }); + await once(s1, "ready"); + + s2 = client2.sync.string({ + project_id, + path: "a.txt", + service: server.service, + noAutosave: true, + }); + await once(s2, "ready"); + expect(s1.to_str()).toBe(""); + expect(s2.to_str()).toBe(""); + expect(s1 === s2).toBe(false); + }); + + it("both clients set the first version independently and inconsistently", async () => { + s2.from_str("y"); + s1.from_str("x"); + s1.commit(); + // delay so s2's time is always bigger than s1's so our unit test + // is well defined + await delay(GAP_DELAY); + s2.commit(); + await s1.save(); + await s2.save(); + }); + + it("wait until both clients see two heads", async () => { + await waitUntilSynced([s1, s2]); + const heads1 = s1.patch_list.getHeads(); + const heads2 = s2.patch_list.getHeads(); + expect(heads1.length).toBe(2); + expect(heads2.length).toBe(2); + expect(heads1).toEqual(heads2); + }); + + it("get the current value, which is a merge", () => { + const v1 = s1.to_str(); + const v2 = s2.to_str(); + expect(v1).toEqual("xy"); + expect(v2).toEqual("xy"); + }); + + it("commit current value and see that there is a new single head that both share, thus resolving the merge in this way", async () => { + s1.commit(); + await s1.save(); + await waitUntilSynced([s1, s2]); + const heads1 = s1.patch_list.getHeads(); + const heads2 = s2.patch_list.getHeads(); + expect(heads1.length).toBe(1); + expect(heads2.length).toBe(1); + expect(heads1).toEqual(heads2); + }); + + it("set values inconsistently again and explicitly resolve the merge conflict in a way that is different than the default", async () => { + s1.from_str("xy1"); + s1.commit(); + await delay(GAP_DELAY); + s2.from_str("xy2"); + s2.commit(); + await s1.save(); + await s2.save(); + + await waitUntilSynced([s1, s2]); + expect(s1.to_str()).toEqual("xy12"); + expect(s2.to_str()).toEqual("xy12"); + + // resolve the conflict in our own way + s1.from_str("xy3"); + s1.commit(); + await s1.save(); + await waitUntilSynced([s1, s2]); + + // everybody has this state now + expect(s1.to_str()).toEqual("xy3"); + expect(s2.to_str()).toEqual("xy3"); + }); +}); + +describe("do the example in the blog post 'Lies I was Told About Collaborative Editing, Part 1: Algorithms for offline editing' -- https://www.moment.dev/blog/lies-i-was-told-pt-1", () => { + const project_id = uuid(); + let client1, client2; + + async function getInitialState(path: string) { + client1 ??= connect(); + client2 ??= connect(); + client1 + .fs({ project_id, service: server.service }) + .writeFile(path, "The Color of Pomegranates"); + const alice = client1.sync.string({ + project_id, + path, + service: server.service, + noAutosave: true, + }); + await once(alice, "ready"); + await alice.save(); + + const bob = client2.sync.string({ + project_id, + path, + service: server.service, + noAutosave: true, + }); + await once(bob, "ready"); + await bob.save(); + await waitUntilSynced([bob, alice]); + + return { alice, bob }; + } + + let alice, bob; + it("creates two clients", async () => { + ({ alice, bob } = await getInitialState("first.txt")); + expect(alice.to_str()).toEqual("The Color of Pomegranates"); + expect(bob.to_str()).toEqual("The Color of Pomegranates"); + }); + + it("Bob changes the spelling of Color to the British Colour and unaware Alice deletes all of the text.", async () => { + bob.from_str("The Colour of Pomegranates"); + bob.commit(); + alice.from_str(""); + alice.commit(); + }); + + it("Both come back online -- the resolution is the empty (with either order above) string because the **best effort** application of inserting the u (with context) to either is a no-op.", async () => { + await bob.save(); + await alice.save(); + await waitUntilSynced([bob, alice]); + expect(alice.to_str()).toEqual(""); + expect(bob.to_str()).toEqual(""); + }); + + it("the important thing about the cocalc approach is that a consistent history is saved, so everybody knows precisely what happened. **I.e., the fact that at one point Bob adding a British u is not lost to either party!**", () => { + const v = alice.versions(); + const x = v.map((t) => alice.version(t).to_str()); + expect(new Set(x)).toEqual( + new Set(["The Color of Pomegranates", "The Colour of Pomegranates", ""]), + ); + + const w = alice.versions(); + const y = w.map((t) => bob.version(t).to_str()); + expect(y).toEqual(x); + }); + + it("reset -- create alicea and bob again", async () => { + ({ alice, bob } = await getInitialState("second.txt")); + }); + + // opposite order this time + it("Bob changes the spelling of Color to the British Colour and unaware Alice deletes all of the text.", async () => { + alice.from_str(""); + alice.commit(); + bob.from_str("The Colour of Pomegranates"); + bob.commit(); + }); + + it("both empty again", async () => { + await bob.save(); + await alice.save(); + await waitUntilSynced([bob, alice]); + expect(alice.to_str()).toEqual(""); + expect(bob.to_str()).toEqual(""); + }); + + it("There are two heads; either client can resolve the merge conflict.", async () => { + expect(alice.patch_list.getHeads().length).toBe(2); + expect(bob.patch_list.getHeads().length).toBe(2); + bob.from_str("The Colour of Pomegranates"); + bob.commit(); + await bob.save(); + + await waitUntilSynced([bob, alice]); + expect(alice.to_str()).toEqual("The Colour of Pomegranates"); + expect(bob.to_str()).toEqual("The Colour of Pomegranates"); + }); +}); + +const numHeads = 15; +describe(`create editing conflict with ${numHeads} heads`, () => { + const project_id = uuid(); + let docs: any[] = [], + clients: any[] = []; + + it(`create ${numHeads} clients`, async () => { + const v: any[] = []; + for (let i = 0; i < numHeads; i++) { + const client = connect(); + clients.push(client); + const doc = client.sync.string({ + project_id, + path: "a.txt", + service: server.service, + noAutosave: true, + }); + docs.push(doc); + v.push(once(doc, "ready")); + } + await Promise.all(v); + }); + + it("every client writes a different value all at once", async () => { + for (let i = 0; i < numHeads; i++) { + docs[i].from_str(`${i} `); + docs[i].commit(); + docs[i].save(); + } + await waitUntilSynced(docs); + const heads = docs[0].patch_list.getHeads(); + expect(heads.length).toBe(docs.length); + }); + + it("merge -- order is random, but value is consistent", async () => { + const value = docs[0].to_str(); + let v = new Set(); + for (let i = 0; i < numHeads; i++) { + v.add(`${i}`); + expect(docs[i].to_str()).toEqual(value); + } + const t = new Set(split(docs[0].to_str())); + expect(t).toEqual(v); + }); + + it(`resolve the merge conflict -- all ${numHeads} clients then see the resolution`, async () => { + let r = ""; + for (let i = 0; i < numHeads; i++) { + r += `${i} `; + } + docs[0].from_str(r); + docs[0].commit(); + await docs[0].save(); + + await waitUntilSynced(docs); + for (let i = 0; i < numHeads; i++) { + expect(docs[i].to_str()).toEqual(r); + } + // docs[0].show_history(); + }); +}); diff --git a/src/packages/backend/conat/test/sync-doc/delete.test.ts b/src/packages/backend/conat/test/sync-doc/delete.test.ts new file mode 100644 index 00000000000..0d1854be748 --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/delete.test.ts @@ -0,0 +1,119 @@ +import { before, after, uuid, connect, server, once, delay } from "./setup"; + +beforeAll(before); +afterAll(after); + +describe("deleting a file that is open as a syncdoc", () => { + const project_id = uuid(); + const path = "a.txt"; + let client1, client2, s1, s2, fs; + const deletedThreshold = 50; // make test faster + const watchRecreateWait = 100; + + it("creates two clients editing 'a.txt'", async () => { + client1 = connect(); + client2 = connect(); + fs = client1.fs({ project_id, service: server.service }); + await fs.writeFile(path, "my existing file"); + s1 = client1.sync.string({ + project_id, + path, + fs, + deletedThreshold, + watchRecreateWait, + }); + + await once(s1, "ready"); + + s2 = client2.sync.string({ + project_id, + path, + service: server.service, + deletedThreshold, + watchRecreateWait, + }); + await once(s2, "ready"); + }); + + it(`delete 'a.txt' from disk and both clients emit 'deleted' event in about ${deletedThreshold}ms`, async () => { + const start = Date.now(); + const d1 = once(s1, "deleted"); + const d2 = once(s2, "deleted"); + await fs.unlink("a.txt"); + await d1; + await d2; + expect(Date.now() - start).toBeLessThan(deletedThreshold + 1000); + }); + + it("clients still work (clients can ignore 'deleted' if they want)", async () => { + expect(s1.isClosed()).toBe(false); + expect(s2.isClosed()).toBe(false); + s1.from_str("back"); + const d1 = once(s1, "watching"); + const d2 = once(s2, "watching"); + await s1.save_to_disk(); + expect(await fs.readFile("a.txt", "utf8")).toEqual("back"); + await d1; + await d2; + }); + + it(`deleting 'a.txt' again -- still triggers deleted events`, async () => { + const start = Date.now(); + const d1 = once(s1, "deleted"); + const d2 = once(s2, "deleted"); + await fs.unlink("a.txt"); + await d1; + await d2; + expect(Date.now() - start).toBeLessThan(deletedThreshold + 1000); + }); + + // it("disconnect one client, delete file, then reconnect client", async () => { + // console.log(1); + // client2.disconnect(); + // const d1 = once(s1, "deleted"); + // const d2 = once(s2, "deleted"); + // console.log(2); + // await fs.unlink("a.txt"); + // console.log(3); + // client2.connect(); + // console.log(4); + // await d1; + // console.log(5); + // await d2; + // expect(Date.now() - start).toBeLessThan(deletedThreshold + 1000); + // }); +}); + +describe("deleting a file then recreate it quickly does NOT trigger a 'deleted' event", () => { + const project_id = uuid(); + const path = "a.txt"; + let client1, s1, fs; + const deletedThreshold = 250; + + it("creates two clients editing 'a.txt'", async () => { + client1 = connect(); + fs = client1.fs({ project_id, service: server.service }); + await fs.writeFile(path, "my existing file"); + s1 = client1.sync.string({ + project_id, + path, + fs, + service: server.service, + deletedThreshold, + }); + + await once(s1, "ready"); + }); + + it(`delete 'a.txt' from disk and both clients emit 'deleted' event in about ${deletedThreshold}ms`, async () => { + let c1 = 0; + s1.once("deleted", () => { + c1++; + }); + await fs.unlink("a.txt"); + await delay(deletedThreshold - 100); + await fs.writeFile(path, "I'm back!"); + await delay(deletedThreshold); + expect(c1).toBe(0); + }); +}); diff --git a/src/packages/backend/conat/test/sync-doc/no-autosave.test.ts b/src/packages/backend/conat/test/sync-doc/no-autosave.test.ts new file mode 100644 index 00000000000..032526f381a --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/no-autosave.test.ts @@ -0,0 +1,88 @@ +import { + before, + after, + uuid, + connect, + server, + once, + delay, + waitUntilSynced, +} from "./setup"; + +beforeAll(before); +afterAll(after); + +describe("confirm noAutosave works", () => { + const project_id = uuid(); + const path = "a.txt"; + let client1, client2, s1, s2; + + it("creates two clients with noAutosave enabled", async () => { + client1 = connect(); + client2 = connect(); + await client1 + .fs({ project_id, service: server.service }) + .writeFile(path, ""); + s1 = client1.sync.string({ + project_id, + path, + service: server.service, + noAutosave: true, + }); + + await once(s1, "ready"); + + s2 = client2.sync.string({ + project_id, + path, + service: server.service, + noAutosave: true, + }); + await once(s2, "ready"); + expect(s1.noAutosave).toEqual(true); + expect(s2.noAutosave).toEqual(true); + }); + + const howLong = 750; + it(`write a change to s1 and commit it, but observe s2 does not see it even after ${howLong}ms (which should be plenty of time)`, async () => { + s1.from_str("new-ver"); + s1.commit(); + + expect(s2.to_str()).toEqual(""); + await delay(howLong); + expect(s2.to_str()).toEqual(""); + }); + + it("explicitly save and see s2 does get the change", async () => { + await s1.save(); + await waitUntilSynced([s1, s2]); + expect(s2.to_str()).toEqual("new-ver"); + }); + + it("make a change resulting in two heads", async () => { + s2.from_str("new-ver-1"); + s2.commit(); + // no background saving happening: + await delay(100); + s1.from_str("new-ver-2"); + s1.commit(); + await Promise.all([s1.save(), s2.save()]); + }); + + it("there are two heads and value is merged", async () => { + await waitUntilSynced([s1, s2]); + expect(s1.to_str()).toEqual("new-ver-1-2"); + expect(s2.to_str()).toEqual("new-ver-1-2"); + expect(s1.patch_list.getHeads().length).toBe(2); + expect(s2.patch_list.getHeads().length).toBe(2); + }); + + it("string state info matches", async () => { + const a1 = s1.syncstring_table_get_one().toJS(); + const a2 = s2.syncstring_table_get_one().toJS(); + expect(a1).toEqual(a2); + expect(new Set(a1.users)).toEqual( + new Set([s1.client.client_id(), s2.client.client_id()]), + ); + }); +}); diff --git a/src/packages/backend/conat/test/sync-doc/setup.ts b/src/packages/backend/conat/test/sync-doc/setup.ts new file mode 100644 index 00000000000..b6ce38042d2 --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/setup.ts @@ -0,0 +1,43 @@ +import { + before as before0, + after as after0, + client as client0, + wait, +} from "@cocalc/backend/conat/test/setup"; +export { connect, wait, once, delay } from "@cocalc/backend/conat/test/setup"; +import { + createPathFileserver, + cleanupFileservers, +} from "@cocalc/backend/conat/files/test/util"; +export { uuid } from "@cocalc/util/misc"; + +export { client0 as client }; + +export let server, fs; + +export async function before() { + await before0(); + server = await createPathFileserver(); +} + +export async function after() { + await cleanupFileservers(); + await after0(); +} + +// wait until the state of several syncdocs all have same heads- they may have multiple +// heads, but they all have the same heads +export async function waitUntilSynced(syncdocs: any[]) { + await wait({ + until: () => { + const X = new Set(); + for (const s of syncdocs) { + X.add(JSON.stringify(s.patch_list.getHeads()?.sort())); + if (X.size > 1) { + return false; + } + } + return true; + }, + }); +} diff --git a/src/packages/backend/conat/test/sync-doc/syncdb.test.ts b/src/packages/backend/conat/test/sync-doc/syncdb.test.ts new file mode 100644 index 00000000000..24e2706f52f --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/syncdb.test.ts @@ -0,0 +1,99 @@ +import { + before, + after, + uuid, + wait, + connect, + server, + once, +} from "./setup"; + +beforeAll(before); +afterAll(after); + +describe("loading/saving syncstring to disk and setting values", () => { + let s; + const project_id = uuid(); + let client; + + it("creates a client", () => { + client = connect(); + }); + + it("a syncdb associated to a file that does not exist on disk is initialized to empty", async () => { + s = client.sync.db({ + project_id, + path: "new.syncdb", + service: server.service, + primary_keys: ["name"], + }); + await once(s, "ready"); + expect(s.to_str()).toBe(""); + expect(s.versions().length).toBe(0); + }); + + it("store a record", async () => { + s.set({ name: "cocalc", value: 10 }); + expect(s.to_str()).toBe('{"name":"cocalc","value":10}'); + const t = s.get_one({ name: "cocalc" }).toJS(); + expect(t).toEqual({ name: "cocalc", value: 10 }); + await s.commit(); + await s.save(); + // [ ] TODO: this save to disk definitely should NOT be needed + await s.save_to_disk(); + }); + + let client2, s2; + it("connect another client", async () => { + client2 = connect(); + // [ ] loading this resets the state if we do not save above. + s2 = client2.sync.db({ + project_id, + path: "new.syncdb", + service: server.service, + primary_keys: ["name"], + }); + await once(s2, "ready"); + expect(s2).not.toBe(s); + expect(s2.to_str()).toBe('{"name":"cocalc","value":10}'); + const t = s2.get_one({ name: "cocalc" }).toJS(); + expect(t).toEqual({ name: "cocalc", value: 10 }); + + s2.set({ name: "conat", date: new Date() }); + s2.commit(); + await s2.save(); + }); + + it("verifies the change on s2 is seen by s (and also that Date objects do NOT work)", async () => { + await wait({ until: () => s.get_one({ name: "conat" }) != null }); + const t = s.get_one({ name: "conat" }).toJS(); + expect(t).toEqual({ name: "conat", date: t.date }); + // They don't work because we're storing syncdb's in jsonl format, + // so json is used. We should have a new format called + // msgpackl and start using that. + expect(t.date instanceof Date).toBe(false); + }); + + const count = 1000; + it(`store ${count} records`, async () => { + const before = s.get().size; + for (let i = 0; i < count; i++) { + s.set({ name: i }); + } + s.commit(); + await s.save(); + expect(s.get().size).toBe(count + before); + }); + + it("confirm file saves to disk with many lines", async () => { + await s.save_to_disk(); + const v = (await s.fs.readFile("new.syncdb", "utf8")).split("\n"); + expect(v.length).toBe(s.get().size); + }); + + it("verifies lookups are not too slow (there is an index)", () => { + for (let i = 0; i < count; i++) { + expect(s.get_one({ name: i }).get("name")).toEqual(i); + } + }); +}); diff --git a/src/packages/backend/conat/test/sync-doc/syncstring-bench.test.ts b/src/packages/backend/conat/test/sync-doc/syncstring-bench.test.ts new file mode 100644 index 00000000000..2e5a64ef2a5 --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/syncstring-bench.test.ts @@ -0,0 +1,32 @@ +import { before, after, uuid, client, server, once } from "./setup"; + +beforeAll(before); +afterAll(after); + +const log = process.env.BENCH ? console.log : (..._args) => {}; + +describe("loading/saving syncstring to disk and setting values", () => { + let s; + const project_id = uuid(); + let fs; + it("time opening a syncstring for editing a file that already exists on disk", async () => { + fs = client.fs({ project_id, service: server.service }); + await fs.writeFile("a.txt", "hello"); + + const t0 = Date.now(); + await fs.readFile("a.txt", "utf8"); + log("lower bound: time to read file", Date.now() - t0, "ms"); + + const start = Date.now(); + s = client.sync.string({ + project_id, + path: "a.txt", + service: server.service, + }); + await once(s, "ready"); + const total = Date.now() - start; + log("actual time to open sync document", total); + + expect(s.to_str()).toBe("hello"); + }); +}); diff --git a/src/packages/backend/conat/test/sync-doc/syncstring.test.ts b/src/packages/backend/conat/test/sync-doc/syncstring.test.ts new file mode 100644 index 00000000000..3990baaa55c --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/syncstring.test.ts @@ -0,0 +1,127 @@ +import { before, after, uuid, wait, connect, server, once } from "./setup"; + +beforeAll(before); +afterAll(after); + +describe("loading/saving syncstring to disk and setting values", () => { + let s; + const project_id = uuid(); + let client; + + it("creates the client", () => { + client = connect(); + }); + + it("a syncstring associated to a file that does not exist on disk is initialized to the empty string", async () => { + s = client.sync.string({ + project_id, + path: "new.txt", + service: server.service, + }); + await once(s, "ready"); + expect(s.to_str()).toBe(""); + expect(s.versions().length).toBe(0); + s.close(); + }); + + let fs; + it("a syncstring for editing a file that already exists on disk is initialized to that file", async () => { + fs = client.fs({ project_id, service: server.service }); + await fs.writeFile("a.txt", "hello"); + s = client.sync.string({ + project_id, + path: "a.txt", + service: server.service, + }); + await once(s, "ready"); + expect(s.fs).not.toEqual(undefined); + }); + + it("initially it is 'hello'", () => { + expect(s.to_str()).toBe("hello"); + expect(s.versions().length).toBe(1); + }); + + it("set the value", () => { + s.from_str("test"); + expect(s.to_str()).toBe("test"); + expect(s.versions().length).toBe(1); + }); + + it("save value to disk", async () => { + await s.save_to_disk(); + const disk = await fs.readFile("a.txt", "utf8"); + expect(disk).toEqual("test"); + }); + + it("commit the value", () => { + s.commit(); + expect(s.versions().length).toBe(2); + }); + + it("change the value and commit a second time", () => { + s.from_str("bar"); + s.commit(); + expect(s.versions().length).toBe(3); + }); + + it("get first version", () => { + expect(s.version(s.versions()[0]).to_str()).toBe("hello"); + expect(s.version(s.versions()[1]).to_str()).toBe("test"); + }); +}); + +describe("synchronized editing with two copies of a syncstring", () => { + const project_id = uuid(); + let s1, s2, client1, client2; + + it("creates the fs client and two copies of a syncstring", async () => { + client1 = connect(); + client2 = connect(); + s1 = client1.sync.string({ + project_id, + path: "a.txt", + service: server.service, + }); + await once(s1, "ready"); + + s2 = client2.sync.string({ + project_id, + path: "a.txt", + service: server.service, + }); + await once(s2, "ready"); + expect(s1.to_str()).toBe(""); + expect(s2.to_str()).toBe(""); + expect(s1 === s2).toBe(false); + }); + + it("change one, commit and save, and see change reflected in the other", async () => { + s1.from_str("hello world"); + s1.commit(); + await s1.save(); + await wait({ + until: () => { + return s2.to_str() == "hello world"; + }, + }); + }); + + it("change second and see change reflected in first", async () => { + s2.from_str("hello world!"); + s2.commit(); + await s2.save(); + await wait({ until: () => s1.to_str() == "hello world!" }); + }); + + it("view the history from each", async () => { + expect(s1.versions().length).toEqual(2); + expect(s2.versions().length).toEqual(2); + + const v1: string[] = [], + v2: string[] = []; + s1.show_history({ log: (x) => v1.push(x) }); + s2.show_history({ log: (x) => v2.push(x) }); + expect(v1).toEqual(v2); + }); +}); \ No newline at end of file diff --git a/src/packages/backend/conat/test/sync-doc/watch-file.test.ts b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts new file mode 100644 index 00000000000..8f1238dbc03 --- /dev/null +++ b/src/packages/backend/conat/test/sync-doc/watch-file.test.ts @@ -0,0 +1,171 @@ +import { + before, + after, + uuid, + connect, + server, + once, + wait, + delay, + waitUntilSynced, +} from "./setup"; + +beforeAll(before); +afterAll(after); + +describe("basic watching of file on disk happens automatically", () => { + const project_id = uuid(); + const path = "a.txt"; + let client, s, fs; + + it("creates client", async () => { + client = connect(); + fs = client.fs({ project_id, service: server.service }); + await fs.writeFile(path, "init"); + s = client.sync.string({ + project_id, + path, + service: server.service, + }); + await once(s, "ready"); + expect(s.to_str()).toEqual("init"); + }); + + it("changes the file on disk and call readFile to immediately update", async () => { + await fs.writeFile(path, "modified"); + await s.readFile(); + expect(s.to_str()).toEqual("modified"); + }); + + it("change file on disk and it automatically updates with no explicit call needed", async () => { + await fs.writeFile(path, "changed again!"); + await wait({ + until: () => { + return s.to_str() == "changed again!"; + }, + }); + }); + + it("change file on disk should not trigger a load from disk", async () => { + const orig = s.readFileDebounced; + let c = 0; + s.readFileDebounced = () => { + c += 1; + }; + s.from_str("a different value"); + await s.save_to_disk(); + expect(c).toBe(0); + await delay(100); + expect(c).toBe(0); + s.readFileDebounced = orig; + // disable the ignore that happens as part of save_to_disk, + // or the tests below won't work + await s.fileWatcher?.ignore(0); + }); + + let client2, s2; + it("file watching also works if there are multiple clients, with only one handling the change", async () => { + client2 = connect(); + s2 = client2.sync.string({ + project_id, + path, + service: server.service, + }); + await once(s2, "ready"); + let c = 0, + c2 = 0; + s.on("handle-file-change", () => c++); + s2.on("handle-file-change", () => c2++); + + await fs.writeFile(path, "version3"); + expect(await fs.readFile(path, "utf8")).toEqual("version3"); + await wait({ + until: () => { + return s2.to_str() == "version3" && s.to_str() == "version3"; + }, + }); + expect(s.to_str()).toEqual("version3"); + expect(s2.to_str()).toEqual("version3"); + expect(c + c2).toBe(1); + }); + + it("file watching must still work if either client is closed", async () => { + s.close(); + await fs.writeFile(path, "version4"); + await wait({ + until: () => { + return s2.to_str() == "version4"; + }, + }); + expect(s2.to_str()).toEqual("version4"); + }); + + let client3, s3; + it("add a third client and close client2 and have file watching still work", async () => { + client3 = connect(); + s3 = client3.sync.string({ + project_id, + path, + service: server.service, + }); + await once(s3, "ready"); + s2.close(); + + await fs.writeFile(path, "version5"); + + await wait({ + until: () => { + return s3.to_str() == "version5"; + }, + }); + expect(s3.to_str()).toEqual("version5"); + }); +}); + +describe("has unsaved changes", () => { + const project_id = uuid(); + let s1, s2, client1, client2; + + it("creates two clients and opens a new file (does not exist on disk yet)", async () => { + client1 = connect(); + client2 = connect(); + s1 = client1.sync.string({ + project_id, + path: "a.txt", + service: server.service, + }); + await once(s1, "ready"); + // definitely has unsaved changes, since it doesn't even exist + expect(s1.has_unsaved_changes()).toBe(true); + + s2 = client2.sync.string({ + project_id, + path: "a.txt", + service: server.service, + }); + await once(s2, "ready"); + expect(s1.to_str()).toBe(""); + expect(s2.to_str()).toBe(""); + expect(s1 === s2).toBe(false); + expect(s2.has_unsaved_changes()).toBe(true); + }); + + it("save empty file to disk -- now no unsaved changes", async () => { + await s1.save_to_disk(); + expect(s1.has_unsaved_changes()).toBe(false); + // but s2 doesn't know anything + expect(s2.has_unsaved_changes()).toBe(true); + }); + + it("make a change via s2 and save", async () => { + s2.from_str("i am s2"); + await s2.save_to_disk(); + expect(s2.has_unsaved_changes()).toBe(false); + }); + + it("as soon as s1 learns that there was a change to the file on disk, it doesn't know", async () => { + await waitUntilSynced([s1, s2]); + expect(s1.has_unsaved_changes()).toBe(true); + expect(s1.to_str()).toEqual("i am s2"); + }); +}); diff --git a/src/packages/backend/conat/test/sync/open-files.test.ts b/src/packages/backend/conat/test/sync/open-files.test.ts deleted file mode 100644 index 371ba6a4769..00000000000 --- a/src/packages/backend/conat/test/sync/open-files.test.ts +++ /dev/null @@ -1,132 +0,0 @@ -/* -Unit test basic functionality of the openFiles distributed key:value -store. Projects and compute servers use this to know what files -to open so they can fulfill their backend responsibilities: - - computation - - save to disk - - load from disk when file changes - -DEVELOPMENT: - -pnpm test ./open-files.test.ts - -*/ - -import { openFiles as createOpenFiles } from "@cocalc/backend/conat/sync"; -import { once } from "@cocalc/util/async-utils"; -import { delay } from "awaiting"; -import { before, after, wait } from "@cocalc/backend/conat/test/setup"; - -beforeAll(before); - -const project_id = "00000000-0000-4000-8000-000000000000"; -async function create() { - return await createOpenFiles(project_id, { noAutosave: true, noCache: true }); -} - -describe("create open file tracker and do some basic operations", () => { - let o1, o2; - let file1 = `${Math.random()}.txt`; - let file2 = `${Math.random()}.txt`; - - it("creates two open files trackers (tracking same project) and clear them", async () => { - o1 = await create(); - o2 = await create(); - // ensure caching disabled so our sync tests are real - expect(o1.getKv() === o2.getKv()).toBe(false); - o1.clear(); - await o1.save(); - expect(o1.hasUnsavedChanges()).toBe(false); - o2.clear(); - while (o2.hasUnsavedChanges() || o1.hasUnsavedChanges()) { - try { - // expected due to merge conflict and autosave being disabled. - await o2.save(); - } catch { - await delay(10); - } - } - }); - - it("confirm they are cleared", async () => { - expect(o1.getAll()).toEqual([]); - expect(o2.getAll()).toEqual([]); - }); - - it("touch file in one and observe change and timestamp getting assigned by server", async () => { - o1.touch(file1); - expect(o1.get(file1).time).toBeCloseTo(Date.now(), -3); - }); - - it("touches file in one and observes change by OTHER", async () => { - o1.touch(file2); - expect(o1.get(file2)?.path).toBe(file2); - expect(o2.get(file2)).toBe(undefined); - await o1.save(); - if (o2.get(file2) == null) { - await once(o2, "change", 250); - expect(o2.get(file2).path).toBe(file2); - expect(o2.get(file2).time == null).toBe(false); - } - }); - - it("get all in o2 sees both file1 and file2", async () => { - const v = o2.getAll(); - expect(v[0].path).toBe(file1); - expect(v[1].path).toBe(file2); - expect(v.length).toBe(2); - }); - - it("delete file1 and verify fact that it is deleted is sync'd", async () => { - o1.delete(file1); - expect(o1.get(file1)).toBe(undefined); - expect(o1.getAll().length).toBe(1); - await o1.save(); - - // verify file is gone in o2, at least after waiting (if necessary) - await wait({ - until: () => { - return o2.getAll().length == 1; - }, - }); - expect(o2.get(file1)).toBe(undefined); - // should be 1 due to file2 still being there: - expect(o2.getAll().length).toBe(1); - - // Also confirm file1 is gone in a newly opened one: - const o3 = await create(); - expect(o3.get(file1)).toBe(undefined); - // should be 1 due to file2 still being there, but not file1. - expect(o3.getAll().length).toBe(1); - o3.close(); - }); - - it("sets an error", async () => { - o2.setError(file2, Error("test error")); - expect(o2.get(file2).error.error).toBe("Error: test error"); - expect(typeof o2.get(file2).error.time == "number").toBe(true); - expect(Math.abs(Date.now() - o2.get(file2).error.time)).toBeLessThan(10000); - try { - // get a conflict due to above so resolve it... - await o2.save(); - } catch { - await o2.save(); - } - if (!o1.get(file2).error) { - await once(o1, "change", 250); - } - expect(o1.get(file2).error.error).toBe("Error: test error"); - }); - - it("clears the error", async () => { - o1.setError(file2); - expect(o1.get(file2).error).toBe(undefined); - await o1.save(); - if (o2.get(file2).error) { - await once(o2, "change", 250); - } - expect(o2.get(file2).error).toBe(undefined); - }); -}); - -afterAll(after); diff --git a/src/packages/backend/conat/test/util.ts b/src/packages/backend/conat/test/util.ts index a6a9adb4375..d3fa39ac63e 100644 --- a/src/packages/backend/conat/test/util.ts +++ b/src/packages/backend/conat/test/util.ts @@ -4,12 +4,14 @@ export async function wait({ until: f, start = 5, decay = 1.2, + min = 5, max = 300, timeout = 10000, }: { until: Function; start?: number; decay?: number; + min?: number; max?: number; timeout?: number; }) { @@ -25,7 +27,7 @@ export async function wait({ start, decay, max, - min: 5, + min, timeout, }, ); diff --git a/src/packages/backend/data.ts b/src/packages/backend/data.ts index a95c19dd7ff..08982ab6d3f 100644 --- a/src/packages/backend/data.ts +++ b/src/packages/backend/data.ts @@ -180,6 +180,8 @@ export const pgdatabase: string = export const projects: string = process.env.PROJECTS ?? join(data, "projects", "[project_id]"); export const secrets: string = process.env.SECRETS ?? join(data, "secrets"); +export const rusticRepo: string = + process.env.RUSTIC_REPO ?? join(data, "rustic"); // Where the sqlite database files used for sync are stored. // The idea is there is one very fast *ephemeral* directory @@ -252,6 +254,10 @@ export let conatPersistCount = parseInt(process.env.CONAT_PERSIST_COUNT ?? "1"); // number of api servers (if configured to run) export let conatApiCount = parseInt(process.env.CONAT_API_COUNT ?? "1"); +export let conatProjectRunnerCount = parseInt( + process.env.CONAT_PROJECT_RUNNER_COUNT ?? "1", +); + // if configured, will create a socketio cluster using // the cluster adapter, listening on the given port. diff --git a/src/packages/backend/execute-code.test.ts b/src/packages/backend/execute-code.test.ts index 8cca0e920da..2d7378751e1 100644 --- a/src/packages/backend/execute-code.test.ts +++ b/src/packages/backend/execute-code.test.ts @@ -40,7 +40,7 @@ describe("tests involving bash mode", () => { it("reports missing executable in non-bash mode", async () => { try { await executeCode({ - command: "this_does_not_exist", + command: "/usr/bin/this_does_not_exist", args: ["nothing"], bash: false, }); @@ -52,7 +52,7 @@ describe("tests involving bash mode", () => { it("reports missing executable in non-bash mode when ignoring on exit", async () => { try { await executeCode({ - command: "this_does_not_exist", + command: "/usr/bin/this_does_not_exist", args: ["nothing"], err_on_exit: false, bash: false, @@ -376,7 +376,7 @@ describe("await", () => { it("deal with unknown executables", async () => { const c = await executeCode({ - command: "random123unknown99", + command: "/usr/bin/random123unknown99", err_on_exit: false, async_call: true, }); diff --git a/src/packages/backend/get-listing.ts b/src/packages/backend/get-listing.ts index 02bcdbc8355..90c66c3c8db 100644 --- a/src/packages/backend/get-listing.ts +++ b/src/packages/backend/get-listing.ts @@ -3,10 +3,12 @@ * License: MS-RSL – see LICENSE.md for details */ +// DEPRECATABLE? + /* This is used by backends to serve directory listings to clients: -{files:[..., {size:?,name:?,mtime:?,isdir:?}]} +{files:[..., {size:?,name:?,mtime:?,isDir:?}]} where mtime is integer SECONDS since epoch, size is in bytes, and isdir is only there if true. @@ -46,7 +48,7 @@ const getListing = reuseInFlight( if (!hidden && file.name[0] === ".") { continue; } - let entry: DirectoryListingEntry; + let entry: Partial; try { // I don't actually know if file.name can fail to be JSON-able with node.js -- is there // even a string in Node.js that cannot be dumped to JSON? With python @@ -61,14 +63,14 @@ const getListing = reuseInFlight( try { let stats: Stats; if (file.isSymbolicLink()) { - // Optimization: don't explicitly set issymlink if it is false - entry.issymlink = true; + // Optimization: don't explicitly set isSymLink if it is false + entry.isSymLink = true; } - if (entry.issymlink) { + if (entry.isSymLink) { // at least right now we only use this symlink stuff to display // information to the user in a listing, and nothing else. try { - entry.link_target = await readlink(dir + "/" + entry.name); + entry.linkTarget = await readlink(dir + "/" + entry.name); } catch (err) { // If we don't know the link target for some reason; just ignore this. } @@ -81,7 +83,7 @@ const getListing = reuseInFlight( } entry.mtime = stats.mtime.valueOf() / 1000; if (stats.isDirectory()) { - entry.isdir = true; + entry.isDir = true; const v = await readdir(dir + "/" + entry.name); if (hidden) { entry.size = v.length; @@ -100,7 +102,7 @@ const getListing = reuseInFlight( } catch (err) { entry.error = `${entry.error ? entry.error : ""}${err}`; } - files.push(entry); + files.push(entry as DirectoryListingEntry); } return files; }, diff --git a/src/packages/backend/logger.ts b/src/packages/backend/logger.ts index 6d1dcea6a3b..4b89029caad 100644 --- a/src/packages/backend/logger.ts +++ b/src/packages/backend/logger.ts @@ -71,8 +71,6 @@ function myFormat(...args): string { function defaultTransports(): { console?: boolean; file?: string } { if (process.env.SMC_TEST) { return {}; - } else if (process.env.COCALC_DOCKER) { - return { file: "/var/log/hub/log" }; } else if (process.env.NODE_ENV == "production") { return { console: true }; } else { diff --git a/src/packages/backend/package.json b/src/packages/backend/package.json index 7c5c50eb3ed..6ddf99246a4 100644 --- a/src/packages/backend/package.json +++ b/src/packages/backend/package.json @@ -6,6 +6,9 @@ "./*": "./dist/*.js", "./database": "./dist/database/index.js", "./conat": "./dist/conat/index.js", + "./files/*": "./dist/files/*.js", + "./sandbox": "./dist/sandbox/index.js", + "./conat/sync/*": "./dist/conat/sync/*.js", "./server-settings": "./dist/server-settings/index.js", "./auth/*": "./dist/auth/*.js", "./auth/tokens/*": "./dist/auth/tokens/*.js" @@ -17,11 +20,11 @@ "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf dist node_modules", - "build": "pnpm exec tsc --build", + "install-sandbox-tools": "echo 'require(\"@cocalc/backend/sandbox/install\").install()' | node", + "build": "pnpm exec tsc --build && pnpm install-sandbox-tools", "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", "test": "pnpm exec jest --forceExit", "test-conat": " pnpm exec jest --forceExit conat", - "testp": "pnpm exec jest --forceExit", "depcheck": "pnpx depcheck --ignores events", "prepublishOnly": "pnpm test", "conat-watch": "node ./bin/conat-watch.cjs", @@ -43,13 +46,11 @@ "@cocalc/backend": "workspace:*", "@cocalc/conat": "workspace:*", "@cocalc/util": "workspace:*", - "@types/debug": "^4.1.12", - "@types/jest": "^29.5.14", "awaiting": "^3.0.0", - "better-sqlite3": "^11.10.0", + "better-sqlite3": "^12.2.0", "chokidar": "^3.6.0", "debug": "^4.4.0", - "fs-extra": "^11.2.0", + "fs-extra": "^11.3.1", "lodash": "^4.17.21", "lru-cache": "^7.18.3", "password-hash": "^1.2.2", @@ -65,6 +66,8 @@ }, "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/backend", "devDependencies": { + "@types/debug": "^4.1.12", + "@types/jest": "^29.5.14", "@types/node": "^18.16.14" } } diff --git a/src/packages/backend/sandbox/cp.ts b/src/packages/backend/sandbox/cp.ts new file mode 100644 index 00000000000..bc3301afb19 --- /dev/null +++ b/src/packages/backend/sandbox/cp.ts @@ -0,0 +1,77 @@ +/* +Implement cp with same API as node, but using a spawned subprocess, because +Node's cp does NOT have reflink support, but we very much want it to get +the full benefit of btrfs's copy-on-write functionality. +*/ + +import exec from "./exec"; +import { type CopyOptions } from "@cocalc/conat/files/fs"; +export { type CopyOptions }; +import { exists } from "./install"; + +export default async function cp( + src: string[] | string, + dest: string, + options: CopyOptions = {}, +): Promise { + const args: string[] = []; + if (typeof src == "string") { + args.push("-T", src); + // Why the -T? When the src is string instead of an array, + // we maintain the + // cp semantics of ndoejs, where always dest is exactly what gets + // created/written, **not a file in dest**. E.g., if a is a directory, + // then doing + // cp('a','b',{recursive:true}) + // twice is idempotent, whereas with /usr/bin/cp without the -T option, + // then second call creates b/a. + } else { + args.push(...src); + } + args.push(dest); + const opts: string[] = []; + if (!options.dereference) { + opts.push("-d"); + } + if (!(options.force ?? true)) { + // according to node docs, when force=true: + // "overwrite existing file or directory" + // The -n (or --no-clobber) docs to cp: "do not overwrite an existing file", + // so I think force=false is same as --update. + if (options.errorOnExist) { + // If moreover errorOnExist is set, then node's cp will also throw an error + // with code "ERR_FS_CP_EEXIST" + // SystemError [ERR_FS_CP_EEXIST]: Target already exists + // /usr/bin/cp doesn't really have such an option, so we use exist directly. + if (await exists(dest)) { + const e = Error( + "SystemError [ERR_FS_CP_EEXIST]: Target already exists", + ); + // @ts-ignore + e.code = "ERR_FS_CP_EEXIST"; + throw e; + } + } + // silently does nothing if target exists + opts.push("-n"); + } + + if (options.preserveTimestamps) { + opts.push("-p"); + } + if (options.recursive) { + opts.push("-r"); + } + if (options.reflink) { + opts.push("--reflink=auto"); + } + + const { code, stderr } = await exec({ + cmd: "/usr/bin/cp", + safety: [...opts, ...args], + timeout: options.timeout, + }); + if (code) { + throw Error(stderr.toString()); + } +} diff --git a/src/packages/backend/sandbox/dust.test.ts b/src/packages/backend/sandbox/dust.test.ts new file mode 100644 index 00000000000..cd69a56951b --- /dev/null +++ b/src/packages/backend/sandbox/dust.test.ts @@ -0,0 +1,39 @@ +import dust from "./dust"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +let tempDir; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc")); +}); +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); +}); + +describe("dust works", () => { + it("directory starts empty - no results", async () => { + const { stdout, truncated } = await dust(tempDir, { options: ["-j"] }); + const s = JSON.parse(Buffer.from(stdout).toString()); + expect(s).toEqual({ children: [], name: tempDir, size: s.size }); + expect(truncated).toBe(false); + }); + + it("create a file and see it appears in the dust result", async () => { + await writeFile(join(tempDir, "a.txt"), "hello"); + const { stdout, truncated } = await dust(tempDir, { options: ["-j"] }); + const s = JSON.parse(Buffer.from(stdout).toString()); + expect(s).toEqual({ + size: s.size, + name: tempDir, + children: [ + { + size: s.children[0].size, + name: join(tempDir, "a.txt"), + children: [], + }, + ], + }); + expect(truncated).toBe(false); + }); +}); diff --git a/src/packages/backend/sandbox/dust.ts b/src/packages/backend/sandbox/dust.ts new file mode 100644 index 00000000000..1e5043ccc15 --- /dev/null +++ b/src/packages/backend/sandbox/dust.ts @@ -0,0 +1,121 @@ +import exec, { type ExecOutput, validate } from "./exec"; +import { type DustOptions } from "@cocalc/conat/files/fs"; +export { type DustOptions }; +import { dust as dustPath } from "./install"; + +export default async function dust( + path: string, + { options, darwin, linux, timeout, maxSize }: DustOptions = {}, +): Promise { + if (path == null) { + throw Error("path must be specified"); + } + + return await exec({ + cmd: dustPath, + cwd: path, + positionalArgs: [path], + options, + darwin, + linux, + maxSize, + timeout, + whitelist, + }); +} + +const whitelist = { + "-d": validate.int, + "--depth": validate.int, + + "-n": validate.int, + "--number-of-lines": validate.int, + + "-p": true, + "--full-paths": true, + + "-X": validate.str, + "--ignore-directory": validate.str, + + "-x": true, + "--limit-filesystem": true, + + "-s": true, + "--apparent-size": true, + + "-r": true, + "--reverse": true, + + "-c": true, + "--no-colors": true, + "-C": true, + "--force-colors": true, + + "-b": true, + "--no-percent-bars": true, + + "-B": true, + "--bars-on-right": true, + + "-z": validate.str, + "--min-size": validate.str, + + "-R": true, + "--screen-reader": true, + + "--skip-total": true, + + "-f": true, + "--filecount": true, + + "-i": true, + "--ignore-hidden": true, + + "-v": validate.str, + "--invert-filter": validate.str, + + "-e": validate.str, + "--filter": validate.str, + + "-t": validate.str, + "--file-types": validate.str, + + "-w": validate.int, + "--terminal-width": validate.int, + + "-P": true, + "--no-progress": true, + + "--print-errors": true, + + "-D": true, + "--only-dir": true, + + "-F": true, + "--only-file": true, + + "-o": validate.str, + "--output-format": validate.str, + + "-j": true, + "--output-json": true, + + "-M": validate.str, + "--mtime": validate.str, + + "-A": validate.str, + "--atime": validate.str, + + "-y": validate.str, + "--ctime": validate.str, + + "--collapse": validate.str, + + "-m": validate.set(["a", "c", "m"]), + "--filetime": validate.set(["a", "c", "m"]), + + "-h": true, + "--help": true, + "-V": true, + "--version": true, +} as const; diff --git a/src/packages/backend/sandbox/exec.test.ts b/src/packages/backend/sandbox/exec.test.ts new file mode 100644 index 00000000000..509cc604946 --- /dev/null +++ b/src/packages/backend/sandbox/exec.test.ts @@ -0,0 +1,124 @@ +/* +Test the exec command. +*/ + +import exec from "./exec"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { nsjail, exists } from "./install"; + +const log = process.env.VERBOSE ? console.log : (..._args) => {}; + +let haveJail; +let tempDir; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc")); + haveJail = await exists(nsjail); +}); +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); +}); + +describe("exec works", () => { + it(`create file and run ls command`, async () => { + await writeFile(join(tempDir, "a.txt"), "hello"); + const { stderr, stdout, truncated, code } = await exec({ + cmd: "ls", + cwd: tempDir, + }); + expect(code).toBe(0); + expect(truncated).toBe(false); + expect(stdout.toString()).toEqual("a.txt\n"); + expect(stderr.toString()).toEqual(""); + }); + + if (haveJail) { + it("run ls in a jail", async () => { + const { stdout, truncated, code } = await exec({ + cmd: "/usr/bin/ls", + nsjail: [ + "-Mo", + "-R", + "/lib64", + "-R", + "/lib", + "-R", + "/usr", + "-B", + tempDir, + "--cwd", + tempDir, + ], + }); + expect(code).toBe(0); + expect(truncated).toBe(false); + expect(stdout.toString()).toEqual("a.txt\n"); + }); + + it("ls in a jail sees only a small amount of the filesystem", async () => { + const { stdout, truncated, code } = await exec({ + cmd: "/usr/bin/ls", + positionalArgs: ["/"], + nsjail: [ + "-Mo", + "-R", + "/lib64", + "-R", + "/lib", + "-R", + "/usr", + "-B", + tempDir, + "--cwd", + tempDir, + ], + }); + expect(code).toBe(0); + expect(truncated).toBe(false); + expect(stdout.toString()).toEqual("lib\nlib64\nproc\ntmp\nusr\n"); + }); + + const jailCount = 200; + it(`Benchmark: run ls in a jail and not in a jail ${jailCount} times and check that the overhead is minimal`, async () => { + const t0 = Date.now(); + for (let i = 0; i < jailCount; i++) { + await exec({ + cmd: "ls", + cwd: tempDir, + }); + } + const nonJailTime = Date.now() - t0; + const nonJailRate = Math.ceil((jailCount / nonJailTime) * 1000); + log(`No Jail: ${nonJailRate} calls per second`); + expect(nonJailRate).toBeGreaterThan(10); + + const t1 = Date.now(); + for (let i = 0; i < jailCount; i++) { + await exec({ + cmd: "/usr/bin/ls", + nsjail: [ + "-Mo", + "-R", + "/lib64", + "-R", + "/lib", + "-R", + "/usr", + "-B", + tempDir, + "--cwd", + tempDir, + ], + }); + } + const jailTime = Date.now() - t1; + const jailRate = Math.ceil((jailCount / jailTime) * 1000); + log(`Jail: ${jailRate} calls per second`); + expect(jailRate).toBeGreaterThan(10); + + const jailOverhead = (jailTime - nonJailTime) / jailCount; + log(`Jail Overhead is ${jailOverhead} ms per call`); + }); + } +}); diff --git a/src/packages/backend/sandbox/exec.ts b/src/packages/backend/sandbox/exec.ts new file mode 100644 index 00000000000..062292dc78a --- /dev/null +++ b/src/packages/backend/sandbox/exec.ts @@ -0,0 +1,248 @@ +import { execFile, spawn } from "node:child_process"; +import { arch } from "node:os"; +import { type ExecOutput } from "@cocalc/conat/files/fs"; +export { type ExecOutput }; +import getLogger from "@cocalc/backend/logger"; +import { nsjail as nsjailPath } from "./install"; + +const logger = getLogger("files:sandbox:exec"); + +const DEFAULT_TIMEOUT = 3_000; +const DEFAULT_MAX_SIZE = 10_000_000; + +export interface Options { + // the path to the command + cmd: string; + // position args *before* any options; these are not sanitized + prefixArgs?: string[]; + // positional arguments; these are not sanitized, but are given after '--' for safety + positionalArgs?: string[]; + // whitelisted args flags; these are checked according to the whitelist specified below + options?: string[]; + // if given, use these options when os.arch()=='darwin' (i.e., macOS); these must match whitelist + darwin?: string[]; + // if given, use these options when os.arch()=='linux'; these must match whitelist + linux?: string[]; + // when total size of stdout and stderr hits this amount, command is terminated, and + // truncated is set. The total amount of output may thus be slightly larger than maxOutput + maxSize?: number; + // command is terminated after this many ms + timeout?: number; + // each command line option that is explicitly whitelisted + // should be a key in the following whitelist map. + // The value can be either: + // - true: in which case the option does not take a argument, or + // - a function: in which the option takes exactly one argument; the function should validate that argument + // and throw an error if the argument is not allowed. + whitelist?: { [option: string]: true | ValidateFunction }; + // where to launch command + cwd?: string; + + // options that are always included first for safety and need NOT match whitelist + safety?: string[]; + + // if nodejs is running as root and give this username, then cmd runs as this + // user instead. + username?: string; + + // run command under nsjail with these options, which are not sanitized + // in any way. + nsjail?: string[]; +} + +type ValidateFunction = (value: string) => void; + +export default async function exec({ + cmd, + positionalArgs = [], + prefixArgs = [], + options = [], + linux = [], + darwin = [], + safety = [], + maxSize = DEFAULT_MAX_SIZE, + timeout = DEFAULT_TIMEOUT, + whitelist = {}, + cwd, + username, + nsjail, +}: Options): Promise { + if (arch() == "darwin") { + options = options.concat(darwin); + } else if (arch() == "linux") { + options = options.concat(linux); + } + options = safety.concat(parseAndValidateOptions(options, whitelist)); + const userId = username ? await getUserIds(username) : undefined; + + return new Promise((resolve, reject) => { + const stdoutChunks: Buffer[] = []; + const stderrChunks: Buffer[] = []; + let truncated = false; + let stdoutSize = 0; + let stderrSize = 0; + + let args = prefixArgs.concat(options); + if (positionalArgs.length > 0) { + args.push("--", ...positionalArgs); + } + + logger.debug({ cmd, args }); + if (nsjail) { + args = [...nsjail, "--", cmd, ...args]; + cmd = nsjailPath; + } + + // console.log(`${cmd} ${args.join(" ")}`, { cwd }); + const child = spawn(cmd, args, { + stdio: ["ignore", "pipe", "pipe"], + env: {} as any, // sometimes next complains about this + cwd, + ...userId, + }); + + let timeoutHandle: NodeJS.Timeout | null = null; + + if (timeout > 0) { + timeoutHandle = setTimeout(() => { + truncated = true; + child.kill("SIGTERM"); + // Force kill after grace period + setTimeout(() => { + if (!child.killed) { + child.kill("SIGKILL"); + } + }, 1000); + }, timeout); + } + + child.stdout.on("data", (chunk: Buffer) => { + stdoutSize += chunk.length; + if (stdoutSize + stderrSize >= maxSize) { + truncated = true; + child.kill("SIGTERM"); + return; + } + stdoutChunks.push(chunk); + }); + + child.stderr.on("data", (chunk: Buffer) => { + stderrSize += chunk.length; + if (stdoutSize + stderrSize > maxSize) { + truncated = true; + child.kill("SIGTERM"); + return; + } + stderrChunks.push(chunk); + }); + + child.on("error", (err) => { + if (timeoutHandle) { + clearTimeout(timeoutHandle); + } + reject(err); + }); + + child.once("close", (code) => { + if (timeoutHandle) { + clearTimeout(timeoutHandle); + } + + resolve({ + stdout: Buffer.concat(stdoutChunks), + stderr: Buffer.concat(stderrChunks), + code, + truncated, + }); + }); + }); +} + +export function parseAndValidateOptions( + options: string[], + whitelist, +): string[] { + const validatedOptions: string[] = []; + let i = 0; + + while (i < options.length) { + const opt = options[i]; + + // Check if this is a safe option + const validate = whitelist[opt]; + if (!validate) { + throw new Error(`Disallowed option: ${opt}`); + } + validatedOptions.push(opt); + + // Handle options that take values + if (validate !== true) { + i++; + if (i >= options.length) { + throw new Error(`Option ${opt} requires a value`); + } + const value = String(options[i]); + validate(value); + // didn't throw, so good to go + validatedOptions.push(value); + } + i++; + } + return validatedOptions; +} + +export const validate = { + str: () => {}, + set: (allowed) => { + allowed = new Set(allowed); + return (value: string) => { + if (!allowed.includes(value)) { + throw Error("invalid value"); + } + }; + }, + int: (value: string) => { + const x = parseInt(value); + if (!isFinite(x)) { + throw Error("argument must be a number"); + } + }, + float: (value: string) => { + const x = parseFloat(value); + if (!isFinite(x)) { + throw Error("argument must be a number"); + } + }, +}; + +async function getUserIds( + username: string, +): Promise<{ uid: number; gid: number }> { + return Promise.all([ + new Promise((resolve, reject) => { + execFile("id", ["-u", username], (err, stdout) => { + if (err) return reject(err); + resolve(parseInt(stdout.trim(), 10)); + }); + }), + new Promise((resolve, reject) => { + execFile("id", ["-g", username], (err, stdout) => { + if (err) return reject(err); + resolve(parseInt(stdout.trim(), 10)); + }); + }), + ]).then(([uid, gid]) => ({ uid, gid })); +} + +// take the output of exec and convert stdout, stderr to strings. If code is nonzero, +// instead throw an error with message stderr. +export function parseOutput({ stdout, stderr, code, truncated }: ExecOutput) { + if (code) { + throw new Error(Buffer.from(stderr).toString()); + } + return { + stdout: Buffer.from(stdout).toString(), + stderr: Buffer.from(stderr).toString(), + truncated, + }; +} diff --git a/src/packages/backend/sandbox/fd.test.ts b/src/packages/backend/sandbox/fd.test.ts new file mode 100644 index 00000000000..f17132d9721 --- /dev/null +++ b/src/packages/backend/sandbox/fd.test.ts @@ -0,0 +1,27 @@ +import fd from "./fd"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +let tempDir; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc")); +}); +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); +}); + +describe("fd files", () => { + it("directory starts empty", async () => { + const { stdout, truncated } = await fd(tempDir); + expect(stdout.length).toBe(0); + expect(truncated).toBe(false); + }); + + it("create a file and see it appears via fd", async () => { + await writeFile(join(tempDir, "a.txt"), "hello"); + const { stdout, truncated } = await fd(tempDir); + expect(truncated).toBe(false); + expect(stdout.toString()).toEqual("a.txt\n"); + }); +}); diff --git a/src/packages/backend/sandbox/fd.ts b/src/packages/backend/sandbox/fd.ts new file mode 100644 index 00000000000..dfc92e8e1ea --- /dev/null +++ b/src/packages/backend/sandbox/fd.ts @@ -0,0 +1,119 @@ +import exec, { type ExecOutput, validate } from "./exec"; +import { type FdOptions } from "@cocalc/conat/files/fs"; +export { type FdOptions }; +import { fd as fdPath } from "./install"; + +export default async function fd( + path: string, + { options, darwin, linux, pattern, timeout, maxSize }: FdOptions = {}, +): Promise { + if (path == null) { + throw Error("path must be specified"); + } + + return await exec({ + cmd: fdPath, + cwd: path, + positionalArgs: pattern ? [pattern] : [], + options, + darwin, + linux, + safety: ["--no-follow"], + maxSize, + timeout, + whitelist, + }); +} + +const whitelist = { + "-H": true, + "--hidden": true, + + "-I": true, + "--no-ignore": true, + + "-u": true, + "--unrestricted": true, + + "--no-ignore-vcs": true, + "--no-require-git": true, + + "-s": true, + "--case-sensitive": true, + + "-i": true, + "--ignore-case": true, + + "-g": true, + "--glob": true, + + "--regex": true, + + "-F": true, + "--fixed-strings": true, + + "--and": validate.str, + + "-l": true, + "--list-details": true, + + "-p": true, + "--full-path": true, + + "-0": true, + "--print0": true, + + "--max-results": validate.int, + + "-1": true, + + "-q": true, + "--quite": true, + + "--show-errors": true, + + "--strip-cwd-prefix": validate.set(["never", "always", "auto"]), + + "--one-file-system": true, + "--mount": true, + "--xdev": true, + + "-h": true, + "--help": true, + + "-V": true, + "--version": true, + + "-d": validate.int, + "--max-depth": validate.int, + + "--min-depth": validate.int, + + "--exact-depth": validate.int, + + "--prune": true, + + "--type": validate.str, + + "-e": validate.str, + "--extension": validate.str, + + "-E": validate.str, + "--exclude": validate.str, + + "--ignore-file": validate.str, + + "-c": validate.set(["never", "always", "auto"]), + "--color": validate.set(["never", "always", "auto"]), + + "-S": validate.str, + "--size": validate.str, + + "--changed-within": validate.str, + "--changed-before": validate.str, + + "-o": validate.str, + "--owner": validate.str, + + "--format": validate.str, +} as const; diff --git a/src/packages/backend/sandbox/find.test.ts b/src/packages/backend/sandbox/find.test.ts new file mode 100644 index 00000000000..66574f5cac5 --- /dev/null +++ b/src/packages/backend/sandbox/find.test.ts @@ -0,0 +1,80 @@ +import find from "./find"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +let tempDir; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc")); +}); +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); +}); + +describe("find files", () => { + it("directory starts empty", async () => { + const { stdout, truncated } = await find(tempDir, { + options: ["-maxdepth", "1", "-mindepth", "1", "-printf", "%f\n"], + }); + expect(stdout.length).toBe(0); + expect(truncated).toBe(false); + }); + + it("create a file and see it appears in find", async () => { + await writeFile(join(tempDir, "a.txt"), "hello"); + const { stdout, truncated } = await find(tempDir, { + options: ["-maxdepth", "1", "-mindepth", "1", "-printf", "%f\n"], + }); + expect(truncated).toBe(false); + expect(stdout.toString()).toEqual("a.txt\n"); + }); + + it("find files matching a given pattern", async () => { + await writeFile(join(tempDir, "pattern"), ""); + await mkdir(join(tempDir, "blue")); + await writeFile(join(tempDir, "blue", "Patton"), ""); + const { stdout } = await find(tempDir, { + options: [ + "-maxdepth", + "1", + "-mindepth", + "1", + "-iname", + "patt*", + "-printf", + "%f\n", + ], + }); + const v = stdout.toString().trim().split("\n"); + expect(new Set(v)).toEqual(new Set(["pattern"])); + }); + + it("find file in a subdirectory too", async () => { + const { stdout } = await find(tempDir, { + options: ["-iname", "patt*", "-printf", "%P\n"], + }); + const w = stdout.toString().trim().split("\n"); + expect(new Set(w)).toEqual(new Set(["pattern", "blue/Patton"])); + }); + + // this is NOT a great test, unfortunately. + const count = 5000; + it(`hopefully exceed the timeout by creating ${count} files`, async () => { + for (let i = 0; i < count; i++) { + await writeFile(join(tempDir, `${i}`), ""); + } + const t = Date.now(); + const { stdout, truncated } = await find(tempDir, { + options: ["-printf", "%f\n"], + timeout: 0.1, + }); + + expect(truncated).toBe(true); + expect(Date.now() - t).toBeGreaterThan(1); + + const { stdout: stdout2 } = await find(tempDir, { + options: ["-maxdepth", "1", "-mindepth", "1", "-printf", "%f\n"], + }); + expect(stdout2.length).toBeGreaterThan(stdout.length); + }); +}); diff --git a/src/packages/backend/sandbox/find.ts b/src/packages/backend/sandbox/find.ts new file mode 100644 index 00000000000..8c1929ce705 --- /dev/null +++ b/src/packages/backend/sandbox/find.ts @@ -0,0 +1,117 @@ +/* + +NOTE: fd is a very fast parallel rust program for finding files matching +a pattern. It is complementary to find here though, because we mainly +use find to compute directory +listing info (e.g., file size, mtime, etc.), and fd does NOT do that; it can +exec ls, but that is slower than using find. So both find and fd are useful +for different tasks -- find is *better* for directory listings and fd is better +for finding filesnames in a directory tree that match a pattern. +*/ + +import type { FindOptions } from "@cocalc/conat/files/fs"; +export type { FindOptions }; +import exec, { type ExecOutput, validate } from "./exec"; + +export default async function find( + path: string, + { options, darwin, linux, timeout, maxSize }: FindOptions, +): Promise { + if (path == null) { + throw Error("path must be specified"); + } + return await exec({ + cmd: "find", + cwd: path, + prefixArgs: [path ? path : "."], + options, + darwin, + linux, + maxSize, + timeout, + whitelist, + safety: [], + }); +} + +const whitelist = { + // POSITIONAL OPTIONS + "-daystart": true, + "-regextype": validate.str, + "-warn": true, + "-nowarn": true, + + // GLOBAL OPTIONS + "-d": true, + "-depth": true, + "--help": true, + "-ignore_readdir_race": true, + "-maxdepth": validate.int, + "-mindepth": validate.int, + "-mount": true, + "-noignore_readdir_race": true, + "--version": true, + "-xdev": true, + + // TESTS + "-amin": validate.float, + "-anewer": validate.str, + "-atime": validate.float, + "-cmin": validate.float, + "-cnewer": validate.str, + "-ctime": validate.float, + "-empty": true, + "-executable": true, + "-fstype": validate.str, + "-gid": validate.int, + "-group": validate.str, + "-ilname": validate.str, + "-iname": validate.str, + "-inum": validate.int, + "-ipath": validate.str, + "-iregex": validate.str, + "-iwholename": validate.str, + "-links": validate.int, + "-lname": validate.str, + "-mmin": validate.int, + "-mtime": validate.int, + "-name": validate.str, + "-newer": validate.str, + "-newerXY": validate.str, + "-nogroup": true, + "-nouser": true, + "-path": validate.str, + "-perm": validate.str, + "-readable": true, + "-regex": validate.str, + "-samefile": validate.str, + "-size": validate.str, + "-true": true, + "-type": validate.str, + "-uid": validate.int, + "-used": validate.float, + "-user": validate.str, + "-wholename": validate.str, + "-writable": true, + "-xtype": validate.str, + "-context": validate.str, + + // ACTIONS: obviously many are not whitelisted! + "-ls": true, + "-print": true, + "-print0": true, + "-printf": validate.str, + "-prune": true, + "-quit": true, + + // OPERATORS + "(": true, + ")": true, + "!": true, + "-not": true, + "-a": true, + "-and": true, + "-o": true, + "-or": true, + ",": true, +} as const; diff --git a/src/packages/backend/sandbox/index.ts b/src/packages/backend/sandbox/index.ts new file mode 100644 index 00000000000..72e84b7a315 --- /dev/null +++ b/src/packages/backend/sandbox/index.ts @@ -0,0 +1,500 @@ +/* +Given a path to a folder on the filesystem, this provides +a wrapper class with an API similar to the fs/promises modules, +but which only allows access to files in that folder. +It's a bit simpler with return data that is always +serializable. + +Absolute and relative paths are considered as relative to the input folder path. + +REFERENCE: We don't use https://github.com/metarhia/sandboxed-fs, but did +look at the code. + + + +SECURITY: + +The following could be a big problem -- user somehow create or change path to +be a dangerous symlink *after* the realpath check below, but before we do an fs *read* +operation. If they did that, then we would end up reading the target of the +symlink. I.e., if they could somehow create the file *as an unsafe symlink* +right after we confirm that it does not exist and before we read from it. This +would only happen via something not involving this sandbox, e.g., the filesystem +mounted into a container some other way. + +In short, I'm worried about: + +1. Request to read a file named "link" which is just a normal file. We confirm this using realpath + in safeAbsPath. +2. Somehow delete "link" and replace it by a new file that is a symlink to "../{project_id}/.ssh/id_ed25519" +3. Read the file "link" and get the contents of "../{project_id}/.ssh/id_ed25519". + +The problem is that 1 and 3 happen microseconds apart as separate calls to the filesystem. + +**[ ] TODO -- NOT IMPLEMENTED YET: This is why we have to uses file descriptors!** + +1. User requests to read a file named "link" which is just a normal file. +2. We wet file descriptor fd for whatever "link" is. Then confirm this is OK using realpath in safeAbsPath. +3. user somehow deletes "link" and replace it by a new file that is a symlink to "../{project_id}/.ssh/id_ed25519" +4. We read from the file descriptor fd and get the contents of original "link" (or error). + +*/ + +import { + appendFile, + chmod, + cp, + constants, + copyFile, + link, + lstat, + readdir, + readFile, + readlink, + realpath, + rename, + rm, + rmdir, + mkdir, + stat, + symlink, + truncate, + writeFile, + unlink, + utimes, +} from "node:fs/promises"; +import { move } from "fs-extra"; +import { watch } from "node:fs"; +import { exists } from "@cocalc/backend/misc/async-utils-node"; +import { basename, dirname, join, resolve } from "path"; +import { replace_all } from "@cocalc/util/misc"; +import { EventIterator } from "@cocalc/util/event-iterator"; +import { type WatchOptions } from "@cocalc/conat/files/watch"; +import find, { type FindOptions } from "./find"; +import ripgrep, { type RipgrepOptions } from "./ripgrep"; +import fd, { type FdOptions } from "./fd"; +import dust, { type DustOptions } from "./dust"; +import rustic from "./rustic"; +import { type ExecOutput } from "./exec"; +import { rusticRepo } from "@cocalc/backend/data"; +import ouch, { type OuchOptions } from "./ouch"; +import cpExec from "./cp"; +import { type CopyOptions } from "@cocalc/conat/files/fs"; +export { type CopyOptions }; + +// max time code can run (in safe mode), e.g., for find, +// ripgrep, fd, and dust. +const MAX_TIMEOUT = 5000; + +interface Options { + // unsafeMode -- if true, assume security model where user is running this + // themself, e.g., in a project, so no security is needed at all. + unsafeMode?: boolean; + // readonly -- only allow operations that don't change files + readonly?: boolean; + host?: string; + rusticRepo?: string; +} + +// If you add any methods below that are NOT for the public api +// be sure to exclude them here! +const INTERNAL_METHODS = new Set([ + "safeAbsPath", + "safeAbsPaths", + "constructor", + "path", + "unsafeMode", + "readonly", + "assertWritable", + "rusticRepo", + "host", +]); + +export class SandboxedFilesystem { + public readonly unsafeMode: boolean; + public readonly readonly: boolean; + public rusticRepo: string; + private host?: string; + constructor( + // path should be the path to a FOLDER on the filesystem (not a file) + public readonly path: string, + { + unsafeMode = false, + readonly = false, + host = "global", + rusticRepo: repo, + }: Options = {}, + ) { + this.unsafeMode = !!unsafeMode; + this.readonly = !!readonly; + this.host = host; + this.rusticRepo = repo ?? rusticRepo; + for (const f in this) { + if (INTERNAL_METHODS.has(f)) { + continue; + } + const orig = this[f]; + // @ts-ignore + this[f] = async (...args) => { + try { + // @ts-ignore + return await orig(...args); + } catch (err) { + if (err.path) { + err.path = err.path.slice(this.path.length + 1); + } + err.message = replace_all(err.message, this.path + "/", ""); + throw err; + } + }; + } + } + + private assertWritable = (path: string) => { + if (this.readonly) { + throw new SandboxError( + `EACCES: permission denied -- read only filesystem, open '${path}'`, + { errno: -13, code: "EACCES", syscall: "open", path }, + ); + } + }; + + safeAbsPaths = async (path: string[] | string): Promise => { + return await Promise.all( + (typeof path == "string" ? [path] : path).map(this.safeAbsPath), + ); + }; + + safeAbsPath = async (path: string): Promise => { + if (typeof path != "string") { + throw Error(`path must be a string but is of type ${typeof path}`); + } + // pathInSandbox is *definitely* a path in the sandbox: + const pathInSandbox = join(this.path, resolve("/", path)); + if (this.unsafeMode) { + // not secure -- just convenient. + return pathInSandbox; + } + // However, there is still one threat, which is that it could + // be a path to an existing link that goes out of the sandbox. So + // we resolve to the realpath: + try { + const p = await realpath(pathInSandbox); + if (p != this.path && !p.startsWith(this.path + "/")) { + throw Error( + `realpath of '${path}' resolves to a path outside of sandbox`, + ); + } + // don't return the result of calling realpath -- what's important is + // their path's realpath is in the sandbox. + return pathInSandbox; + } catch (err) { + if (err.code == "ENOENT") { + return pathInSandbox; + } else { + throw err; + } + } + }; + + appendFile = async (path: string, data: string | Buffer, encoding?) => { + this.assertWritable(path); + return await appendFile(await this.safeAbsPath(path), data, encoding); + }; + + chmod = async (path: string, mode: string | number) => { + this.assertWritable(path); + await chmod(await this.safeAbsPath(path), mode); + }; + + constants = async (): Promise<{ [key: string]: number }> => { + return constants; + }; + + copyFile = async (src: string, dest: string) => { + this.assertWritable(dest); + await copyFile(await this.safeAbsPath(src), await this.safeAbsPath(dest)); + }; + + cp = async (src: string | string[], dest: string, options?: CopyOptions) => { + this.assertWritable(dest); + dest = await this.safeAbsPath(dest); + + // ensure containing directory of destination exists -- node cp doesn't + // do this but for cocalc this is very convenient and saves some network + // round trips. + const destDir = dirname(dest); + if (destDir != this.path && !(await exists(destDir))) { + await mkdir(destDir, { recursive: true }); + } + + const v = await this.safeAbsPaths(src); + if (!options?.reflink) { + // can use node cp: + for (const path of v) { + if (typeof src == "string") { + await cp(path, dest, options); + } else { + // copying multiple files to a directory + await cp(path, join(dest, basename(path)), options); + } + } + } else { + // /usr/bin/cp. NOte that behavior depends on string versus string[], + // so we pass the absolute paths v in that way. + await cpExec( + typeof src == "string" ? v[0] : v, + dest, + capTimeout(options, MAX_TIMEOUT), + ); + } + }; + + exists = async (path: string) => { + return await exists(await this.safeAbsPath(path)); + }; + + find = async (path: string, options?: FindOptions): Promise => { + return await find( + await this.safeAbsPath(path), + capTimeout(options, MAX_TIMEOUT), + ); + }; + + // find files + fd = async (path: string, options?: FdOptions): Promise => { + return await fd( + await this.safeAbsPath(path), + capTimeout(options, MAX_TIMEOUT), + ); + }; + + // disk usage + dust = async (path: string, options?: DustOptions): Promise => { + return await dust( + await this.safeAbsPath(path), + // dust reasonably takes longer than the other commands and is used less, + // so for now we give it more breathing room. + capTimeout(options, 4 * MAX_TIMEOUT), + ); + }; + + // compression + ouch = async (args: string[], options?: OuchOptions): Promise => { + options = { ...options }; + if (options.cwd) { + options.cwd = await this.safeAbsPath(options.cwd); + } + return await ouch( + [args[0]].concat(await Promise.all(args.slice(1).map(this.safeAbsPath))), + capTimeout(options, 6 * MAX_TIMEOUT), + ); + }; + + // backups + rustic = async ( + args: string[], + { + timeout = 120_000, + maxSize = 10_000_000, // the json output can be quite large + cwd, + }: { timeout?: number; maxSize?: number; cwd?: string } = {}, + ): Promise => { + return await rustic(args, { + repo: this.rusticRepo, + safeAbsPath: this.safeAbsPath, + timeout, + maxSize, + host: this.host, + cwd, + }); + }; + + ripgrep = async ( + path: string, + pattern: string, + options?: RipgrepOptions, + ): Promise => { + return await ripgrep( + await this.safeAbsPath(path), + pattern, + capTimeout(options, MAX_TIMEOUT), + ); + }; + + // hard link + link = async (existingPath: string, newPath: string) => { + this.assertWritable(newPath); + return await link( + await this.safeAbsPath(existingPath), + await this.safeAbsPath(newPath), + ); + }; + + lstat = async (path: string) => { + return await lstat(await this.safeAbsPath(path)); + }; + + mkdir = async (path: string, options?) => { + this.assertWritable(path); + await mkdir(await this.safeAbsPath(path), options); + }; + + readFile = async (path: string, encoding?: any): Promise => { + return await readFile(await this.safeAbsPath(path), encoding); + }; + + readdir = async (path: string, options?) => { + const x = (await readdir(await this.safeAbsPath(path), options)) as any[]; + if (options?.withFileTypes) { + // each entry in x has a path and parentPath field, which refers to the + // absolute paths to the directory that contains x or the target of x (if + // it is a link). This is an absolute path on the fileserver, which we try + // not to expose from the sandbox, hence we modify them all if possible. + for (const a of x) { + if (a.path.startsWith(this.path)) { + a.path = a.path.slice(this.path.length + 1); + } + if (a.parentPath.startsWith(this.path)) { + a.parentPath = a.parentPath.slice(this.path.length + 1); + } + } + } + + return x; + }; + + readlink = async (path: string): Promise => { + return await readlink(await this.safeAbsPath(path)); + }; + + realpath = async (path: string): Promise => { + const x = await realpath(await this.safeAbsPath(path)); + return x.slice(this.path.length + 1); + }; + + rename = async (oldPath: string, newPath: string) => { + this.assertWritable(oldPath); + await rename( + await this.safeAbsPath(oldPath), + await this.safeAbsPath(newPath), + ); + }; + + move = async ( + src: string, + dest: string, + options?: { overwrite?: boolean }, + ) => { + this.assertWritable(dest); + await move( + await this.safeAbsPath(src), + await this.safeAbsPath(dest), + options, + ); + }; + + rm = async (path: string | string[], options?) => { + const v = await this.safeAbsPaths(path); + const f = async (path) => { + this.assertWritable(path); + await rm(path, options); + }; + await Promise.all(v.map(f)); + }; + + rmdir = async (path: string, options?) => { + this.assertWritable(path); + await rmdir(await this.safeAbsPath(path), options); + }; + + stat = async (path: string) => { + return await stat(await this.safeAbsPath(path)); + }; + + symlink = async (target: string, path: string) => { + this.assertWritable(target); + return await symlink( + await this.safeAbsPath(target), + await this.safeAbsPath(path), + ); + }; + + truncate = async (path: string, len?: number) => { + this.assertWritable(path); + await truncate(await this.safeAbsPath(path), len); + }; + + unlink = async (path: string) => { + this.assertWritable(path); + await unlink(await this.safeAbsPath(path)); + }; + + utimes = async ( + path: string, + atime: number | string | Date, + mtime: number | string | Date, + ) => { + this.assertWritable(path); + await utimes(await this.safeAbsPath(path), atime, mtime); + }; + + watch = async (filename: string, options?: WatchOptions) => { + // NOTE: in node v24 they fixed the fs/promises watch to have a queue, but previous + // versions were clearly badly implemented so we reimplement it from scratch + // using the non-promise watch. + const watcher = watch(await this.safeAbsPath(filename), options as any); + const iter = new EventIterator(watcher, "change", { + maxQueue: options?.maxQueue ?? 2048, + overflow: options?.overflow, + map: (args) => { + // exact same api as new fs/promises watch + return { eventType: args[0], filename: args[1] }; + }, + onEnd: () => { + watcher.close(); + }, + }); + // AbortController signal can cause this + watcher.once("close", () => { + iter.end(); + }); + return iter; + }; + + writeFile = async (path: string, data: string | Buffer) => { + this.assertWritable(path); + return await writeFile(await this.safeAbsPath(path), data); + }; +} + +export class SandboxError extends Error { + code: string; + errno: number; + syscall: string; + path: string; + constructor(mesg: string, { code, errno, syscall, path }) { + super(mesg); + this.code = code; + this.errno = errno; + this.syscall = syscall; + this.path = path; + } +} + +function capTimeout(options, max: number) { + if (options == null) { + return { timeout: max }; + } + + let timeout; + try { + timeout = parseFloat(options.timeout); + } catch { + return { ...options, timeout: max }; + } + if (!isFinite(timeout)) { + return { ...options, timeout: max }; + } + return { ...options, timeout: Math.min(timeout, max) }; +} diff --git a/src/packages/backend/sandbox/install.ts b/src/packages/backend/sandbox/install.ts new file mode 100644 index 00000000000..f98f6a1b390 --- /dev/null +++ b/src/packages/backend/sandbox/install.ts @@ -0,0 +1,299 @@ +/* +Download a ripgrep or fd binary for the operating system + +This supports x86_64/arm64 linux & macos + +This assumes tar is installed. + +NOTE: There are several npm modules that purport to install ripgrep. We do not use +https://www.npmjs.com/package/@vscode/ripgrep because it is not properly maintained, +e.g., + - security vulnerabilities: https://github.com/microsoft/ripgrep-prebuilt/issues/48 + - not updated to a major new release without a good reason: https://github.com/microsoft/ripgrep-prebuilt/issues/38 +*/ + +import { arch, platform } from "os"; +import { execFileSync, execSync } from "child_process"; +import { writeFile, stat, unlink, mkdir, chmod } from "fs/promises"; +import { join } from "path"; +import getLogger from "@cocalc/backend/logger"; + +const logger = getLogger("files:sandbox:install"); + +const i = __dirname.lastIndexOf("packages/backend"); +const binPath = join( + __dirname.slice(0, i + "packages/backend".length), + "node_modules/.bin", +); + +interface Spec { + nonFatal?: boolean; // true if failure to install is non-fatal + VERSION?: string; + BASE?: string; + binary?: string; + path: string; + stripComponents?: number; + pathInArchive?: string; + skip?: string[]; + script?: string; + platforms?: string[]; + fix?: string; +} + +const NSJAIL_VERSION = "3.4"; + +const SPEC = { + ripgrep: { + // See https://github.com/BurntSushi/ripgrep/releases + VERSION: "14.1.1", + BASE: "https://github.com/BurntSushi/ripgrep/releases/download", + binary: "rg", + path: join(binPath, "rg"), + }, + fd: { + // See https://github.com/sharkdp/fd/releases + VERSION: "v10.2.0", + BASE: "https://github.com/sharkdp/fd/releases/download", + binary: "fd", + path: join(binPath, "fd"), + }, + dust: { + // See https://github.com/bootandy/dust/releases + VERSION: "v1.2.3", + BASE: "https://github.com/bootandy/dust/releases/download", + binary: "dust", + path: join(binPath, "dust"), + }, + ouch: { + // See https://github.com/ouch-org/ouch/releases + VERSION: "0.6.1", + BASE: "https://github.com/ouch-org/ouch/releases/download", + binary: "ouch", + path: join(binPath, "ouch"), + // See https://github.com/ouch-org/ouch/issues/45; note that ouch is in home brew + // for this platform. + skip: ["aarch64-apple-darwin"], + }, + rustic: { + // See https://github.com/rustic-rs/rustic/releases + VERSION: "v0.9.5", + BASE: "https://github.com/rustic-rs/rustic/releases/download", + binary: "rustic", + path: join(binPath, "rustic"), + stripComponents: 0, + pathInArchive: "rustic", + }, + nsjail: { + nonFatal: true, + platforms: ["linux"], + VERSION: NSJAIL_VERSION, + BASE: "https://github.com/google/nsjail/releases", + path: join(binPath, "nsjail"), + fix: "sudo apt-get update && sudo apt-get install -y autoconf bison flex gcc g++ git libprotobuf-dev libnl-route-3-dev libtool make pkg-config protobuf-compiler libseccomp-dev", + script: `cd /tmp && rm -rf /tmp/nsjail && git clone --branch ${NSJAIL_VERSION} --depth 1 --single-branch https://github.com/google/nsjail.git && cd nsjail && make -j8 && strip nsjail && cp nsjail ${join(binPath, "nsjail")} && rm -rf /tmp/nsjail`, + }, +}; + +export const ripgrep = SPEC.ripgrep.path; +export const fd = SPEC.fd.path; +export const dust = SPEC.dust.path; +export const rustic = SPEC.rustic.path; +export const ouch = SPEC.ouch.path; +export const nsjail = SPEC.nsjail.path; + +type App = keyof typeof SPEC; + +// https://github.com/sharkdp/fd/releases/download/v10.2.0/fd-v10.2.0-x86_64-unknown-linux-musl.tar.gz +// https://github.com/BurntSushi/ripgrep/releases/download/14.1.1/ripgrep-14.1.1-x86_64-unknown-linux-musl.tar.gz + +export async function exists(path: string) { + try { + await stat(path); + return true; + } catch { + return false; + } +} + +async function alreadyInstalled(app: App) { + return await exists(SPEC[app].path); +} + +export async function install(app?: App) { + if (app == null) { + // @ts-ignore + await Promise.all(Object.keys(SPEC).map(install)); + return; + } + + if (await alreadyInstalled(app)) { + return; + } + + const spec = SPEC[app] as Spec; + + if (spec.platforms != null && !spec.platforms?.includes(platform())) { + return; + } + + const { script } = spec; + try { + if (script) { + try { + execSync(script); + } catch (err) { + if (spec.fix) { + console.warn(`BUILD OF ${app} FAILED: Suggested fix -- ${spec.fix}`); + } + throw err; + } + if (!(await alreadyInstalled(app))) { + throw Error(`failed to install ${app}`); + } + return; + } + + const url = getUrl(app); + if (!url) { + logger.debug("install: skipping ", app); + return; + } + logger.debug("install", { app, url }); + // - 1. Fetch the tarball from the github url (using the fetch library) + const response = await downloadFromGithub(url); + const tarballBuffer = Buffer.from(await response.arrayBuffer()); + + // - 2. Extract the file "rg" from the tarball to ${__dirname}/rg + // The tarball contains this one file "rg" at the top level, i.e., for + // ripgrep-14.1.1-x86_64-unknown-linux-musl.tar.gz + // we have "tar tvf ripgrep-14.1.1-x86_64-unknown-linux-musl.tar.gz" outputs + // ... + // ripgrep-14.1.1-x86_64-unknown-linux-musl/rg + + const { + VERSION, + binary, + path, + stripComponents = 1, + pathInArchive = app == "ouch" + ? `${app}-${getOS()}/${binary}` + : `${app}-${VERSION}-${getOS()}/${binary}`, + } = spec; + + const tmpFile = join(__dirname, `${app}-${VERSION}.tar.gz`); + try { + try { + if (!(await exists(binPath))) { + await mkdir(binPath); + } + } catch {} + await writeFile(tmpFile, tarballBuffer); + // sync is fine since this is run at *build time*. + execFileSync("tar", [ + "xzf", + tmpFile, + `--strip-components=${stripComponents}`, + `-C`, + binPath, + pathInArchive, + ]); + + // - 3. Make the file rg executable + await chmod(path, 0o755); + } finally { + try { + await unlink(tmpFile); + } catch {} + } + } catch (err) { + if (spec.nonFatal) { + console.log(`WARNING: unable to install ${app}`, err); + } else { + throw err; + } + } +} + +// Download from github, but aware of rate limits, the retry-after header, etc. +async function downloadFromGithub(url: string) { + const maxRetries = 10; + const baseDelay = 1000; // 1 second + + for (let attempt = 1; attempt <= maxRetries; attempt++) { + try { + const res = await fetch(url); + + if (res.status === 429) { + // Rate limit error + if (attempt === maxRetries) { + throw new Error("Rate limit exceeded after max retries"); + } + + const retryAfter = res.headers.get("retry-after"); + const delay = retryAfter + ? parseInt(retryAfter) * 1000 + : baseDelay * Math.pow(2, attempt - 1); // Exponential backoff + + console.log( + `Rate limited. Retrying in ${delay}ms (attempt ${attempt}/${maxRetries})`, + ); + await new Promise((resolve) => setTimeout(resolve, delay)); + continue; + } + + if (!res.ok) { + throw new Error(`HTTP ${res.status}: ${res.statusText}`); + } + return res; + } catch (error) { + if (attempt === maxRetries) { + throw error; + } + + const delay = baseDelay * Math.pow(2, attempt - 1); + console.log( + `Fetch ${url} failed. Retrying in ${delay}ms (attempt ${attempt}/${maxRetries})`, + ); + await new Promise((resolve) => setTimeout(resolve, delay)); + } + } + throw new Error("Should not reach here"); +} + +function getUrl(app: App) { + const { BASE, VERSION, skip } = SPEC[app] as Spec; + const os = getOS(); + if (skip?.includes(os)) { + return ""; + } + if (app == "ouch") { + return `${BASE}/${VERSION}/${app}-${os}.tar.gz`; + } else { + return `${BASE}/${VERSION}/${app}-${VERSION}-${os}.tar.gz`; + } +} + +function getOS() { + switch (platform()) { + case "linux": + switch (arch()) { + case "x64": + return "x86_64-unknown-linux-musl"; + case "arm64": + return "aarch64-unknown-linux-gnu"; + default: + throw Error(`unsupported arch '${arch()}'`); + } + case "darwin": + switch (arch()) { + case "x64": + return "x86_64-apple-darwin"; + case "arm64": + return "aarch64-apple-darwin"; + default: + throw Error(`unsupported arch '${arch()}'`); + } + default: + throw Error(`unsupported platform '${platform()}'`); + } +} diff --git a/src/packages/backend/sandbox/ouch.test.ts b/src/packages/backend/sandbox/ouch.test.ts new file mode 100644 index 00000000000..96ae332833e --- /dev/null +++ b/src/packages/backend/sandbox/ouch.test.ts @@ -0,0 +1,56 @@ +/* +Test the ouch compression api. +*/ + +import ouch from "./ouch"; +import { mkdtemp, mkdir, rm, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { exists } from "@cocalc/backend/misc/async-utils-node"; + +let tempDir, options; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc")); + options = { cwd: tempDir }; +}); +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); +}); + +describe("ouch works on a little file", () => { + for (const ext of [ + "zip", + "7z", + "tar.gz", + "tar.xz", + "tar.bz", + "tar.bz2", + "tar.bz3", + "tar.lz4", + "tar.sz", + "tar.zst", + "tar.br", + ]) { + it(`create file and compress it up using ${ext}`, async () => { + await writeFile(join(tempDir, "a.txt"), "hello"); + const { truncated, code } = await ouch( + ["compress", "a.txt", `a.${ext}`], + options, + ); + expect(code).toBe(0); + expect(truncated).toBe(false); + expect(await exists(join(tempDir, `a.${ext}`))).toBe(true); + }); + + it(`extract ${ext} in subdirectory`, async () => { + await mkdir(join(tempDir, `target-${ext}`)); + const { code } = await ouch(["decompress", join(tempDir, `a.${ext}`)], { + cwd: join(tempDir, `target-${ext}`), + }); + expect(code).toBe(0); + expect( + (await readFile(join(tempDir, `target-${ext}`, "a.txt"))).toString(), + ).toEqual("hello"); + }); + } +}); diff --git a/src/packages/backend/sandbox/ouch.ts b/src/packages/backend/sandbox/ouch.ts new file mode 100644 index 00000000000..69754330b42 --- /dev/null +++ b/src/packages/backend/sandbox/ouch.ts @@ -0,0 +1,69 @@ +/* + +https://github.com/ouch-org/ouch + +ouch stands for Obvious Unified Compression Helper. + +The .tar.gz support in 'ouch' is excellent -- super fast and memory efficient, +since it is fully parallel. + +*/ + +import exec, { type ExecOutput, validate } from "./exec"; +import { type OuchOptions } from "@cocalc/conat/files/fs"; +export { type OuchOptions }; +import { ouch as ouchPath } from "./install"; + +export default async function ouch( + args: string[], + { timeout, options, cwd }: OuchOptions = {}, +): Promise { + const command = args[0]; + if (!commands.includes(command)) { + throw Error(`first argument must be one of ${commands.join(", ")}`); + } + + return await exec({ + cmd: ouchPath, + cwd, + positionalArgs: args.slice(1), + safety: [command, "-y", "-q"], + timeout, + options, + whitelist, + }); +} + +const commands = ["compress", "c", "decompress", "d", "list", "l", "ls"]; + +const whitelist = { + // general options, + "-H": true, + "--hidden": true, + g: true, + "--gitignore": true, + "-f": validate.str, + "--format": validate.str, + "-p": validate.str, + "--password": validate.str, + "-h": true, + "--help": true, + "-V": true, + "--version": true, + + // compression-specific options + // do NOT enable '-S, --follow-symlinks' as that could escape the sandbox! + // It's off by default. + + "-l": validate.str, + "--level": validate.str, + "--fast": true, + "--slow": true, + + // decompress specific options + "-d": validate.str, + "--dir": validate.str, + r: true, + "--remove": true, + "--no-smart-unpack": true, +} as const; diff --git a/src/packages/backend/sandbox/ripgrep.test.ts b/src/packages/backend/sandbox/ripgrep.test.ts new file mode 100644 index 00000000000..34ba5e0d675 --- /dev/null +++ b/src/packages/backend/sandbox/ripgrep.test.ts @@ -0,0 +1,27 @@ +import ripgrep from "./ripgrep"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +let tempDir; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc")); +}); +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); +}); + +describe("ripgrep files", () => { + it("directory starts empty - no results", async () => { + const { stdout, truncated } = await ripgrep(tempDir, ""); + expect(stdout.length).toBe(0); + expect(truncated).toBe(false); + }); + + it("create a file and see it appears in the rigrep result", async () => { + await writeFile(join(tempDir, "a.txt"), "hello"); + const { stdout, truncated } = await ripgrep(tempDir, "he"); + expect(truncated).toBe(false); + expect(stdout.toString()).toEqual("a.txt:hello\n"); + }); +}); diff --git a/src/packages/backend/sandbox/ripgrep.ts b/src/packages/backend/sandbox/ripgrep.ts new file mode 100644 index 00000000000..9fbb362816b --- /dev/null +++ b/src/packages/backend/sandbox/ripgrep.ts @@ -0,0 +1,237 @@ +import exec, { type ExecOutput, validate } from "./exec"; +import type { RipgrepOptions } from "@cocalc/conat/files/fs"; +export type { RipgrepOptions }; +import { ripgrep as ripgrepPath } from "./install"; + +export default async function ripgrep( + path: string, + pattern: string, + { options, darwin, linux, timeout, maxSize }: RipgrepOptions = {}, +): Promise { + if (path == null) { + throw Error("path must be specified"); + } + if (pattern == null) { + throw Error("pattern must be specified"); + } + + return await exec({ + cmd: ripgrepPath, + cwd: path, + positionalArgs: [pattern], + options, + darwin, + linux, + maxSize, + timeout, + whitelist, + // if large memory usage is an issue, it might be caused by parallel interleaving; using + // -j1 below will prevent that, but will make ripgrep much slower (since not in parallel). + // See the ripgrep man page. + safety: ["--no-follow", "--block-buffered", "--no-config" /* "-j1"*/], + }); +} + +const whitelist = { + "-e": validate.str, + + "-s": true, + "--case-sensitive": true, + + "--crlf": true, + + "-E": validate.set([ + "utf-8", + "utf-16", + "utf-16le", + "utf-16be", + "ascii", + "latin-1", + ]), + "--encoding": validate.set([ + "utf-8", + "utf-16", + "utf-16le", + "utf-16be", + "ascii", + "latin-1", + ]), + + "--engine": validate.set(["default", "pcre2", "auto"]), + + "-F": true, + "--fixed-strings": true, + + "-i": true, + "--ignore-case": true, + + "-v": true, + "--invert-match": true, + + "-x": true, + "--line-regexp": true, + + "-m": validate.int, + "--max-count": validate.int, + + "-U": true, + "--multiline": true, + + "--multiline-dotall": true, + + "--no-unicode": true, + + "--null-data": true, + + "-P": true, + "--pcre2": true, + + "-S": true, + "--smart-case": true, + + "--stop-on-nonmatch": true, + + // this allows searching in binary files -- there is some danger of this + // using a lot more memory. Hence we do not allow it. + // "-a": true, + // "--text": true, + + "-w": true, + "--word-regexp": true, + + "--binary": true, + + "-g": validate.str, + "--glob": validate.str, + "--glob-case-insensitive": true, + + "-.": true, + "--hidden": true, + + "--iglob": validate.str, + + "--ignore-file-case-insensitive": true, + + "-d": validate.int, + "--max-depth": validate.int, + + "--max-filesize": validate.str, + + "--no-ignore": true, + "--no-ignore-dot": true, + "--no-ignore-exclude": true, + "--no-ignore-files": true, + "--no-ignore-global": true, + "--no-ignore-parent": true, + "--no-ignore-vcs": true, + "--no-require-git": true, + "--one-file-system": true, + + "-t": validate.str, + "--type": validate.str, + "-T": validate.str, + "--type-not": validate.str, + "--type-add": validate.str, + "--type-list": true, + "--type-clear": validate.str, + + "-u": true, + "--unrestricted": true, + + "-A": validate.int, + "--after-context": validate.int, + "-B": validate.int, + "--before-context": validate.int, + + "-b": true, + "--byte-offset": true, + + "--color": validate.set(["never", "auto", "always", "ansi"]), + "--colors": validate.str, + + "--column": true, + "-C": validate.int, + "--context": validate.int, + + "--context-separator": validate.str, + "--field-context-separator": validate.str, + "--field-match-separator": validate.str, + + "--heading": true, + "--no-heading": true, + + "-h": true, + "--help": true, + + "--include-zero": true, + + "-n": true, + "--line-number": true, + "-N": true, + "--no-line-number": true, + + "-M": validate.int, + "--max-columns": validate.int, + + "--max-columns-preview": validate.int, + + "-O": true, + "--null": true, + + "--passthru": true, + + "-p": true, + "--pretty": true, + + "-q": true, + "--quiet": true, + + // From the docs: "Neither this flag nor any other ripgrep flag will modify your files." + "-r": validate.str, + "--replace": validate.str, + + "--sort": validate.set(["none", "path", "modified", "accessed", "created"]), + "--sortr": validate.set(["none", "path", "modified", "accessed", "created"]), + + "--trim": true, + "--no-trim": true, + + "--vimgrep": true, + + "-H": true, + "--with-filename": true, + + "-I": true, + "--no-filename": true, + + "-c": true, + "--count": true, + + "--count-matches": true, + "-l": true, + "--files-with-matches": true, + "--files-without-match": true, + "--json": true, + + "--debug": true, + "--no-ignore-messages": true, + "--no-messages": true, + + "--stats": true, + + "--trace": true, + + "--files": true, + + "--generate": validate.set([ + "man", + "complete-bash", + "complete-zsh", + "complete-fish", + "complete-powershell", + ]), + + "--pcre2-version": true, + "-V": true, + "--version": true, +} as const; diff --git a/src/packages/backend/sandbox/rustic.test.ts b/src/packages/backend/sandbox/rustic.test.ts new file mode 100644 index 00000000000..0e9561c27ce --- /dev/null +++ b/src/packages/backend/sandbox/rustic.test.ts @@ -0,0 +1,80 @@ +/* +Test the rustic backup api. + +https://github.com/rustic-rs/rustic +*/ + +import rustic from "./rustic"; +import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { parseOutput } from "./exec"; + +let tempDir, options, home; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc")); + const repo = join(tempDir, "repo"); + home = join(tempDir, "home"); + await mkdir(home); + const safeAbsPath = (path: string) => join(home, resolve("/", path)); + options = { + host: "my-host", + repo, + safeAbsPath, + }; +}); +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); +}); + +describe("rustic does something", () => { + it("there are initially no backups", async () => { + const { stdout, truncated } = await rustic( + ["snapshots", "--json"], + options, + ); + const s = JSON.parse(Buffer.from(stdout).toString()); + expect(s).toEqual([]); + expect(truncated).toBe(false); + }); + + it("create a file and back it up", async () => { + await writeFile(join(tempDir, "a.txt"), "hello"); + const { stdout, truncated } = await rustic( + ["backup", "--json", "a.txt"], + options, + ); + const s = JSON.parse(Buffer.from(stdout).toString()); + expect(s.paths).toEqual(["a.txt"]); + expect(truncated).toBe(false); + }); + + it("use a .toml file instead of explicitly passing in a repo", async () => { + await mkdir(join(home, "x")); + await writeFile( + join(home, "x/a.toml"), + ` +[repository] +repository = "${options.repo}" +password = "" +`, + ); + const options2 = { ...options, repo: join(home, "x/a.toml") }; + const { stdout } = parseOutput( + await rustic(["snapshots", "--json"], options2), + ); + const s = JSON.parse(stdout); + expect(s.length).toEqual(1); + expect(s[0][0].hostname).toEqual("my-host"); + }); + + // it("it appears in the snapshots list", async () => { + // const { stdout, truncated } = await rustic( + // ["snapshots", "--json"], + // options, + // ); + // const s = JSON.parse(Buffer.from(stdout).toString()); + // expect(s).toEqual([]); + // expect(truncated).toBe(false); + // }); +}); diff --git a/src/packages/backend/sandbox/rustic.ts b/src/packages/backend/sandbox/rustic.ts new file mode 100644 index 00000000000..eb4d2d7d5e7 --- /dev/null +++ b/src/packages/backend/sandbox/rustic.ts @@ -0,0 +1,368 @@ +/* +Whitelist: + +The idea is that + - the client can only work with snapshots with exactly the given host. + - any snapshots they create have that host + - snapshots are only of data in their sandbox + - snapshots can only be restored to their sandbox + +The subcommands with some whitelisted support are: + + - backup + - snapshots + - ls + - restore + - find + - forget + +The source options are relative paths and the command is run from the +root of the sandbox_path. + + rustic backup --host=sandbox_path [whitelisted options]... [source]... + + rustic snapshots --filter-host=... [whitelisted options]... + + +Here the snapshot id will be checked to have the right host before +the command is run. Destination is relative to sandbox_path. + + rustic restore [whitelisted options] + + +Dump is used for viewing a version of a file via timetravel: + + rustic dump + +Find is used for getting info about all versions of a file that are backed up: + + rustic find --filter-host=... + + rustic find --filter-host=... --glob='foo/x.txt' -h + + +Delete snapshots: + +- delete snapshot with specific id, which must have the specified host. + + rustic forget [id] + +- + + +*/ + +import exec, { + type ExecOutput, + parseAndValidateOptions, + validate, +} from "./exec"; +import { rustic as rusticPath } from "./install"; +import { exists } from "@cocalc/backend/misc/async-utils-node"; +import { join } from "path"; +import { rusticRepo } from "@cocalc/backend/data"; +import LRU from "lru-cache"; + +export interface RusticOptions { + repo?: string; + timeout?: number; + maxSize?: number; + safeAbsPath?: (path: string) => Promise; + host?: string; + cwd?: string; +} + +export default async function rustic( + args: string[], + options: RusticOptions, +): Promise { + const { + timeout, + maxSize, + repo = rusticRepo, + safeAbsPath, + host = "host", + } = options; + + let common; + if (repo.endsWith(".toml")) { + common = ["-P", repo.slice(0, -".toml".length)]; + } else { + common = ["--password", "", "-r", repo]; + } + + await ensureInitialized(repo); + const cwd = await safeAbsPath?.(options.cwd ?? ""); + + const run = async (sanitizedArgs: string[]) => { + return await exec({ + cmd: rusticPath, + cwd, + safety: [...common, args[0], ...sanitizedArgs], + maxSize, + timeout, + }); + }; + + switch (args[0]) { + case "init": { + if (safeAbsPath != null) { + throw Error("init not allowed"); + } + return await run([]); + } + case "backup": { + if (safeAbsPath == null || cwd == null) { + throw Error("safeAbsPath must be specified when making a backup"); + } + if (args.length == 1) { + throw Error("missing backup source"); + } + const source = (await safeAbsPath(args.slice(-1)[0])).slice(cwd.length); + const options = parseAndValidateOptions( + args.slice(1, -1), + whitelist.backup, + ); + + return await run([ + ...options, + "--no-scan", + "--host", + host, + "--", + source ? source : ".", + ]); + } + case "snapshots": { + const options = parseAndValidateOptions( + args.slice(1), + whitelist.snapshots, + ); + return await run([...options, "--filter-host", host]); + } + case "ls": { + if (args.length <= 1) { + throw Error("missing "); + } + const snapshot = args.slice(-1)[0]; // + await assertValidSnapshot({ snapshot, host, repo }); + const options = parseAndValidateOptions(args.slice(1, -1), whitelist.ls); + return await run([...options, snapshot]); + } + case "restore": { + if (args.length <= 2) { + throw Error("missing "); + } + if (safeAbsPath == null) { + throw Error("safeAbsPath must be specified when restoring"); + } + const snapshot = args.slice(-2)[0]; // + await assertValidSnapshot({ snapshot, host, repo }); + const destination = await safeAbsPath(args.slice(-1)[0]); // + const options = parseAndValidateOptions( + args.slice(1, -2), + whitelist.restore, + ); + return await run([...options, snapshot, destination]); + } + case "find": { + const options = parseAndValidateOptions(args.slice(1), whitelist.find); + return await run([...options, "--filter-host", host]); + } + case "forget": { + if (args.length == 2 && !args[1].startsWith("-")) { + // delete exactly id + const snapshot = args[1]; + await assertValidSnapshot({ snapshot, host, repo }); + return await run([snapshot]); + } + // delete several defined by rules. + const options = parseAndValidateOptions(args.slice(1), whitelist.forget); + return await run([...options, "--filter-host", host]); + } + default: + throw Error(`subcommand not allowed: ${args[0]}`); + } +} + +const whitelist = { + backup: { + "--label": validate.str, + "--tag": validate.str, + "--description": validate.str, + "--time": validate.str, + "--delete-after": validate.str, + "--as-path": validate.str, + "--with-atime": true, + "--ignore-devid": true, + "--json": true, + "--long": true, + "--quiet": true, + "-h": true, + "--help": true, + "--glob": validate.str, + "--iglob": validate.str, + "--git-ignore": true, + "--no-require-git": true, + "-x": true, + "--one-file-system": true, + "--exclude-larger-than": validate.str, + }, + snapshots: { + "-g": validate.str, + "--group-by": validate.str, + "--long": true, + "--json": true, + "--all": true, + "-h": true, + "--help": true, + "--filter-label": validate.str, + "--filter-paths": validate.str, + "--filter-paths-exact": validate.str, + "--filter-after": validate.str, + "--filter-before": validate.str, + "--filter-size": validate.str, + "--filter-size-added": validate.str, + "--filter-jq": validate.str, + }, + restore: { + "--delete": true, + "--verify-existing": true, + "--recursive": true, + "-h": true, + "--help": true, + "--glob": validate.str, + "--iglob": validate.str, + }, + ls: { + "-s": true, + "--summary": true, + "-l": true, + "--long": true, + "--json": true, + "--recursive": true, + "-h": true, + "--help": true, + "--glob": validate.str, + "--iglob": validate.str, + }, + find: { + "--glob": validate.str, + "--iglob": validate.str, + "--path": validate.str, + "-g": validate.str, + "--group-by": validate.str, + "--all": true, + "--show-misses": true, + "-h": true, + "--help": true, + "--filter-label": validate.str, + "--filter-paths": validate.str, + "--filter-paths-exact": validate.str, + "--filter-after": validate.str, + "--filter-before": validate.str, + "--filter-size": validate.str, + "--filter-size-added": validate.str, + "--filter-jq": validate.str, + }, + forget: { + "--json": true, + "-g": validate.str, + "--group-by": validate.str, + "-h": true, + "--help": true, + "--filter-label": validate.str, + "--filter-paths": validate.str, + "--filter-paths-exact": validate.str, + "--filter-after": validate.str, + "--filter-before": validate.str, + "--filter-size": validate.str, + "--filter-size-added": validate.str, + "--filter-jq": validate.str, + "--keep-tags": validate.str, + "--keep-id": validate.str, + "-l": validate.int, + "--keep-last": validate.int, + "-M": validate.int, + "--keep-minutely": validate.int, + "-H": validate.int, + "--keep-hourly": validate.int, + "-d": validate.int, + "--keep-daily": validate.int, + "-w": validate.int, + "--keep-weekly": validate.int, + "-m": validate.int, + "--keep-monthly": validate.int, + "--keep-quarter-yearly": validate.int, + "--keep-half-yearly": validate.int, + "-y": validate.int, + "--keep-yearly": validate.int, + "--keep-within": validate.str, + "--keep-within-minutely": validate.str, + "--keep-within-hourly": validate.str, + "--keep-within-daily": validate.str, + "--keep-within-weekly": validate.str, + "--keep-within-monthly": validate.str, + "--keep-within-quarter-yearly": validate.str, + "--keep-within-half-yearly": validate.str, + "--keep-within-yearly": validate.str, + "--keep-none": validate.str, + }, +} as const; + +async function ensureInitialized(repo: string) { + if (repo.endsWith(".toml")) { + // nothing to do + return; + } + const config = join(repo, "config"); + if (!(await exists(config))) { + await exec({ + cmd: rusticPath, + safety: ["--no-progress", "--password", "", "-r", repo, "init"], + }); + } +} + +async function assertValidSnapshot({ snapshot, host, repo }) { + const id = snapshot.split(":")[0]; + if (id == "latest") { + // possible race condition so do not allow + throw Error("latest is not allowed"); + } + const actualHost = await getHost({ id, repo }); + if (actualHost != host) { + throw Error( + `host for snapshot with id ${id} must be '${host}' but it is ${actualHost}`, + ); + } +} + +// we do not allow changing host so this is safe to cache. +const hostCache = new LRU({ + max: 10000, +}); + +export async function getHost(opts) { + if (hostCache.has(opts.id)) { + return hostCache.get(opts.id); + } + const info = await getSnapshot(opts); + const hostname = info[0][1][0]["hostname"]; + hostCache.set(opts.id, hostname); + return hostname; +} + +export async function getSnapshot({ + id, + repo = rusticRepo, +}: { + id: string; + repo?: string; +}) { + const { stdout } = await exec({ + cmd: rusticPath, + safety: ["--password", "", "-r", repo, "snapshots", "--json", id], + }); + return JSON.parse(stdout.toString()); +} diff --git a/src/packages/backend/sandbox/sandbox.test.ts b/src/packages/backend/sandbox/sandbox.test.ts new file mode 100644 index 00000000000..b4eb78961d2 --- /dev/null +++ b/src/packages/backend/sandbox/sandbox.test.ts @@ -0,0 +1,257 @@ +import { SandboxedFilesystem } from "@cocalc/backend/sandbox"; +import { + mkdtemp, + mkdir, + rm, + readFile, + symlink, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "path"; + +let tempDir; +beforeAll(async () => { + tempDir = await mkdtemp(join(tmpdir(), "cocalc")); +}); + +describe("test using the filesystem sandbox to do a few standard things", () => { + let fs; + it("creates and reads file", async () => { + await mkdir(join(tempDir, "test-1")); + fs = new SandboxedFilesystem(join(tempDir, "test-1")); + await fs.writeFile("a", "hi"); + const r = await fs.readFile("a", "utf8"); + expect(r).toEqual("hi"); + expect(fs.unsafeMode).toBe(false); + }); + + it("truncate file", async () => { + await fs.writeFile("b", "hello"); + await fs.truncate("b", 4); + const r = await fs.readFile("b", "utf8"); + expect(r).toEqual("hell"); + }); +}); + +describe("make various attempts to break out of the sandbox", () => { + let fs; + it("creates sandbox", async () => { + await mkdir(join(tempDir, "test-2")); + fs = new SandboxedFilesystem(join(tempDir, "test-2")); + await fs.writeFile("x", "hi"); + }); + + it("obvious first attempt to escape fails", async () => { + const v = await fs.readdir(".."); + expect(v).toEqual(["x"]); + }); + + it("obvious first attempt to escape fails", async () => { + const v = await fs.readdir("a/../.."); + expect(v).toEqual(["x"]); + }); + + it("another attempt", async () => { + await fs.copyFile("/x", "/tmp"); + const v = await fs.readdir("a/../.."); + expect(v).toEqual(["tmp", "x"]); + + const r = await fs.readFile("tmp", "utf8"); + expect(r).toEqual("hi"); + }); +}); + +describe("test watching a file and a folder in the sandbox", () => { + let fs; + it("creates sandbox", async () => { + await mkdir(join(tempDir, "test-watch")); + fs = new SandboxedFilesystem(join(tempDir, "test-watch")); + await fs.writeFile("x", "hi"); + }); + + it("watches the file x for changes", async () => { + await fs.writeFile("x", "hi"); + const w = await fs.watch("x"); + await fs.appendFile("x", " there"); + const x = await w.next(); + expect(x).toEqual({ + value: { eventType: "change", filename: "x" }, + done: false, + }); + w.end(); + }); + + it("the maxQueue parameter limits the number of queue events", async () => { + await fs.writeFile("x", "hi"); + const w = await fs.watch("x", { maxQueue: 2 }); + expect(w.queueSize()).toBe(0); + // make many changes + await fs.appendFile("x", "0"); + await fs.appendFile("x", "0"); + await fs.appendFile("x", "0"); + await fs.appendFile("x", "0"); + // there will only be 2 available: + expect(w.queueSize()).toBe(2); + const x0 = await w.next(); + expect(x0).toEqual({ + value: { eventType: "change", filename: "x" }, + done: false, + }); + const x1 = await w.next(); + expect(x1).toEqual({ + value: { eventType: "change", filename: "x" }, + done: false, + }); + // one more next would hang... + expect(w.queueSize()).toBe(0); + w.end(); + }); + + it("maxQueue with overflow throw", async () => { + await fs.writeFile("x", "hi"); + const w = await fs.watch("x", { maxQueue: 2, overflow: "throw" }); + await fs.appendFile("x", "0"); + await fs.appendFile("x", "0"); + await fs.appendFile("x", "0"); + expect(async () => { + await w.next(); + }).rejects.toThrow("maxQueue overflow"); + w.end(); + }); + + it("AbortController works", async () => { + const ac = new AbortController(); + const { signal } = ac; + await fs.writeFile("x", "hi"); + const w = await fs.watch("x", { signal }); + await fs.appendFile("x", "0"); + const e = await w.next(); + expect(e.done).toBe(false); + + // now abort + ac.abort(); + const { done } = await w.next(); + expect(done).toBe(true); + }); + + it("watches a directory", async () => { + await fs.mkdir("folder"); + const w = await fs.watch("folder"); + + await fs.writeFile("folder/x", "hi"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "rename", filename: "x" }, + }); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "change", filename: "x" }, + }); + + await fs.appendFile("folder/x", "xxx"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "change", filename: "x" }, + }); + + await fs.writeFile("folder/z", "there"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "rename", filename: "z" }, + }); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "change", filename: "z" }, + }); + + // this is correct -- from the node docs "On most platforms, 'rename' is emitted whenever a filename appears or disappears in the directory." + await fs.unlink("folder/z"); + expect(await w.next()).toEqual({ + done: false, + value: { eventType: "rename", filename: "z" }, + }); + }); +}); + +describe("unsafe mode sandbox", () => { + let fs; + it("creates and reads file", async () => { + await mkdir(join(tempDir, "test-unsafe")); + fs = new SandboxedFilesystem(join(tempDir, "test-unsafe"), { + unsafeMode: true, + }); + expect(fs.unsafeMode).toBe(true); + await fs.writeFile("a", "hi"); + const r = await fs.readFile("a", "utf8"); + expect(r).toEqual("hi"); + }); + + it("directly create a dangerous file that is a symlink outside of the sandbox -- this should work", async () => { + await writeFile(join(tempDir, "password"), "s3cr3t"); + await symlink( + join(tempDir, "password"), + join(tempDir, "test-unsafe", "danger"), + ); + const s = await readFile(join(tempDir, "test-unsafe", "danger"), "utf8"); + expect(s).toBe("s3cr3t"); + }); + + it("can **UNSAFELY** read the symlink content via the api", async () => { + expect(await fs.readFile("danger", "utf8")).toBe("s3cr3t"); + }); +}); + +describe("safe mode sandbox", () => { + let fs; + it("creates and reads file", async () => { + await mkdir(join(tempDir, "test-safe")); + fs = new SandboxedFilesystem(join(tempDir, "test-safe"), { + unsafeMode: false, + }); + expect(fs.unsafeMode).toBe(false); + expect(fs.readonly).toBe(false); + await fs.writeFile("a", "hi"); + const r = await fs.readFile("a", "utf8"); + expect(r).toEqual("hi"); + }); + + it("directly create a dangerous file that is a symlink outside of the sandbox -- this should work", async () => { + await writeFile(join(tempDir, "password"), "s3cr3t"); + await symlink( + join(tempDir, "password"), + join(tempDir, "test-safe", "danger"), + ); + const s = await readFile(join(tempDir, "test-safe", "danger"), "utf8"); + expect(s).toBe("s3cr3t"); + }); + + it("cannot read the symlink content via the api", async () => { + await expect(async () => { + await fs.readFile("danger", "utf8"); + }).rejects.toThrow("outside of sandbox"); + }); +}); + +describe("read only sandbox", () => { + let fs; + it("creates and reads file", async () => { + await mkdir(join(tempDir, "test-ro")); + fs = new SandboxedFilesystem(join(tempDir, "test-ro"), { + readonly: true, + }); + expect(fs.readonly).toBe(true); + await expect(async () => { + await fs.writeFile("a", "hi"); + }).rejects.toThrow("permission denied -- read only filesystem"); + try { + await fs.writeFile("a", "hi"); + } catch (err) { + expect(err.code).toEqual("EACCES"); + } + }); +}); + +afterAll(async () => { + await rm(tempDir, { force: true, recursive: true }); +}); diff --git a/src/packages/conat/core/client.ts b/src/packages/conat/core/client.ts index 2e3aa8d1ae0..cdf1a9f89eb 100644 --- a/src/packages/conat/core/client.ts +++ b/src/packages/conat/core/client.ts @@ -244,6 +244,17 @@ import { } from "@cocalc/conat/sync/dstream"; import { akv, type AKV } from "@cocalc/conat/sync/akv"; import { astream, type AStream } from "@cocalc/conat/sync/astream"; +import { + syncstring, + type SyncString, + type SyncStringOptions, +} from "@cocalc/conat/sync-doc/syncstring"; +import { + syncdb, + type SyncDB, + type SyncDBOptions, +} from "@cocalc/conat/sync-doc/syncdb"; +import { fsClient, fsSubject } from "@cocalc/conat/files/fs"; import TTL from "@isaacs/ttlcache"; import { ConatSocketServer, @@ -262,6 +273,7 @@ export const MAX_INTEREST_TIMEOUT = 90_000; const DEFAULT_WAIT_FOR_INTEREST_TIMEOUT = 30_000; +// WARNING: do NOT change MSGPACK_ENCODER_OPTIONS unless you know what you're doing! const MSGPACK_ENCODER_OPTIONS = { // ignoreUndefined is critical so database queries work properly, and // also we have a lot of api calls with tons of wasted undefined values. @@ -561,6 +573,14 @@ export class Client extends EventEmitter { setTimeout(() => this.conn.io.disconnect(), 1); }; + connect = () => { + this.conn.io.connect(); + }; + + isConnected = () => this.state == "connected"; + + isSignedIn = () => !!(this.info?.user && !this.info?.user?.error); + // this has NO timeout by default waitUntilSignedIn = reuseInFlight( async ({ timeout }: { timeout?: number } = {}) => { @@ -1106,9 +1126,16 @@ export class Client extends EventEmitter { // good for services. await mesg.respond(result); } catch (err) { + let error = err.message; + if (!error) { + error = `${err}`.slice("Error: ".length); + } await mesg.respond(null, { - noThrow: true, // we're not catching this one - headers: { error: `${err}` }, + noThrow: true, // we're not catching this respond + headers: { + error, + error_attrs: JSON.parse(JSON.stringify(err)), + }, }); } }; @@ -1127,7 +1154,7 @@ export class Client extends EventEmitter { const call = async (name: string, args: any[]) => { const resp = await this.request(subject, [name, args], opts); if (resp.headers?.error) { - throw Error(`${resp.headers.error}`); + throw headerToError(resp.headers); } else { return resp.data; } @@ -1136,8 +1163,12 @@ export class Client extends EventEmitter { return new Proxy( {}, { - get: (_, name) => { - if (typeof name !== "string") { + get: (target, name) => { + const s = target[String(name)]; + if (s !== undefined) { + return s; + } + if (typeof name !== "string" || name == "then") { return undefined; } return async (...args) => await call(name, args); @@ -1447,6 +1478,17 @@ export class Client extends EventEmitter { return sub; }; + fs = (opts: { + project_id: string; + compute_server_id?: number; + service?: string; + }) => { + return fsClient({ + subject: fsSubject(opts), + client: this, + }); + }; + sync = { dkv: async (opts: DKVOptions): Promise> => await dkv({ ...opts, client: this }), @@ -1460,6 +1502,10 @@ export class Client extends EventEmitter { await astream({ ...opts, client: this }), synctable: async (opts: SyncTableOptions): Promise => await createSyncTable({ ...opts, client: this }), + string: (opts: Omit, "fs">): SyncString => + syncstring({ ...opts, client: this }), + db: (opts: Omit, "fs">): SyncDB => + syncdb({ ...opts, client: this }), }; socket = { @@ -1915,7 +1961,7 @@ export function messageData( export type Subscription = EventIterator; export class ConatError extends Error { - code: string | number; + code?: string | number; constructor(mesg: string, { code }) { super(mesg); this.code = code; @@ -1940,3 +1986,16 @@ function toConatError(socketIoError) { }); } } + +export function headerToError(headers): ConatError { + const err = Error(headers.error); + if (headers.error_attrs) { + for (const field in headers.error_attrs) { + err[field] = headers.error_attrs[field]; + } + } + if (err["code"] === undefined && headers.code) { + err["code"] = headers.code; + } + return err; +} diff --git a/src/packages/conat/core/cluster.ts b/src/packages/conat/core/cluster.ts index fbaaedad0e5..5e75f644f7e 100644 --- a/src/packages/conat/core/cluster.ts +++ b/src/packages/conat/core/cluster.ts @@ -7,7 +7,6 @@ import { type StickyUpdate, } from "@cocalc/conat/core/server"; import type { DStream } from "@cocalc/conat/sync/dstream"; -import { once } from "@cocalc/util/async-utils"; import { server as createPersistServer } from "@cocalc/conat/persist/server"; import { getLogger } from "@cocalc/conat/client"; import { hash_string } from "@cocalc/util/misc"; @@ -165,7 +164,7 @@ class ClusterLink { if (Date.now() - start >= timeout) { throw Error("timeout"); } - await once(this.interest, "change"); + await this.interest.waitForChange(); if ((this.state as any) == "closed" || signal?.aborted) { return false; } @@ -273,8 +272,8 @@ export async function trimClusterStreams( minAge: number, ): Promise<{ seqsInterest: number[]; seqsSticky: number[] }> { const { interest, sticky } = streams; - // First deal with interst - // we iterate over the interest stream checking for subjects + // First deal with interest. + // We iterate over the interest stream checking for subjects // with no current interest at all; in such cases it is safe // to purge them entirely from the stream. const seqs: number[] = []; diff --git a/src/packages/conat/core/patterns.ts b/src/packages/conat/core/patterns.ts index 79eada9e5e1..708df2afdef 100644 --- a/src/packages/conat/core/patterns.ts +++ b/src/packages/conat/core/patterns.ts @@ -2,6 +2,8 @@ import { isEqual } from "lodash"; import { getLogger } from "@cocalc/conat/client"; import { EventEmitter } from "events"; import { hash_string } from "@cocalc/util/misc"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { once } from "@cocalc/util/async-utils"; type Index = { [pattern: string]: Index | string }; @@ -13,9 +15,14 @@ export class Patterns extends EventEmitter { constructor() { super(); - this.setMaxListeners(1000); + this.setMaxListeners(100); } + // wait until one single change event fires. Throws an error if this gets closed first. + waitForChange = reuseInFlight(async (timeout?) => { + await once(this, "change", timeout); + }); + close = () => { this.emit("closed"); this.patterns = {}; diff --git a/src/packages/conat/core/server.ts b/src/packages/conat/core/server.ts index dc87dcf1911..6ba50af48cd 100644 --- a/src/packages/conat/core/server.ts +++ b/src/packages/conat/core/server.ts @@ -53,7 +53,7 @@ import { import { Patterns } from "./patterns"; import { is_array } from "@cocalc/util/misc"; import { UsageMonitor } from "@cocalc/conat/monitor/usage"; -import { once, until } from "@cocalc/util/async-utils"; +import { until } from "@cocalc/util/async-utils"; import { clusterLink, type ClusterLink, @@ -1703,8 +1703,8 @@ export class ConatServer extends EventEmitter { throw Error("timeout"); } try { - // if signal is set only wait for the change for up to 1 second. - await once(this.interest, "change", signal != null ? 1000 : undefined); + // if signal is set, only wait for the change for up to 1 second. + await this.interest.waitForChange(signal != null ? 1000 : undefined); } catch { continue; } @@ -1739,7 +1739,7 @@ function socketSubjectRoom({ socket, subject }) { return JSON.stringify({ id: socket.id, subject }); } -export function randomChoice(v: Set): string { +export function randomChoice(v: Set): T { if (v.size == 0) { throw Error("v must have size at least 1"); } diff --git a/src/packages/conat/files/file-server.ts b/src/packages/conat/files/file-server.ts new file mode 100644 index 00000000000..dbbd59787bc --- /dev/null +++ b/src/packages/conat/files/file-server.ts @@ -0,0 +1,114 @@ +/* +File server - managers where projects are stored. + +This is a conat service that runs directly on the btrfs file server. +Only admin processes (hubs) can talk directly to it, not normal users. +It handles: + +Core Functionality: + + - creating volume where a project's files are stored + - from scratch, and as a zero-cost clone of an existing project + - copy files between distinct volumes (with btrfs this is done via + highly efficient dedup'd cloning). + +Additional functionality: + - set a quota on project volume + - delete volume + - create snapshot + - update snapshots + - create backup + +*/ + +import { type Client } from "@cocalc/conat/core/client"; +import { conat } from "@cocalc/conat/client"; +import { type CopyOptions } from "./fs"; +export { type CopyOptions }; + +const SUBJECT = "file-server"; + +export interface Fileserver { + mount: (opts: { project_id: string }) => Promise<{ path: string }>; + + // create project_id as an exact lightweight clone of src_project_id + clone: (opts: { + project_id: string; + src_project_id: string; + }) => Promise; + + getUsage: (opts: { project_id: string }) => Promise<{ + size: number; + used: number; + free: number; + }>; + + getQuota: (opts: { project_id: string }) => Promise<{ + size: number; + used: number; + }>; + + setQuota: (opts: { + project_id: string; + size: number | string; + }) => Promise; + + cp: (opts: { + src: { project_id: string; path: string | string[] }; + dest: { project_id: string; path: string }; + options?: CopyOptions; + }) => Promise; + + // create new complete backup of the project; this first snapshots the + // project, makes a backup of the snapshot, then deletes the snapshot, so the + // backup is guranteed to be consistent. + backup: (opts: { project_id: string }) => Promise<{ time: Date; id: string }>; + + // restore the given path in the backup to the given dest. The default + // path is '' (the whole project) and the default destination is the + // same as the path. + restore: (opts: { + project_id: string; + id: string; + path?: string; + dest?: string; + }) => Promise; + + // delete the given backup + deleteBackup: (opts: { project_id: string; id: string }) => Promise; + + // Return list of id's and timestamps of all backups of this project. + getBackups: (opts: { project_id: string }) => Promise< + { + id: string; + time: Date; + }[] + >; + + // Return list of all files in the given backup. + getBackupFiles: (opts: { + project_id: string; + id: string; + }) => Promise; +} + +export interface Options extends Fileserver { + client?: Client; +} + +export async function server({ client, ...impl }: Options) { + client ??= conat(); + + const sub = await client.service(SUBJECT, impl); + + return { + close: () => { + sub.close(); + }, + }; +} + +export function client({ client }: { client?: Client } = {}): Fileserver { + client ??= conat(); + return client.call(SUBJECT); +} diff --git a/src/packages/conat/files/fs.ts b/src/packages/conat/files/fs.ts new file mode 100644 index 00000000000..0a71e6bb26d --- /dev/null +++ b/src/packages/conat/files/fs.ts @@ -0,0 +1,547 @@ +/* +Tests are in + +packages/backend/conat/files/test/local-path.test.ts + +*/ + +import { type Client } from "@cocalc/conat/core/client"; +import { conat } from "@cocalc/conat/client"; +import { + watchServer, + watchClient, + type WatchIterator, +} from "@cocalc/conat/files/watch"; +import listing, { type Listing, type FileTypeLabel } from "./listing"; +import { isValidUUID } from "@cocalc/util/misc"; + +export const DEFAULT_FILE_SERVICE = "fs"; + +export interface ExecOutput { + stdout: Buffer; + stderr: Buffer; + code: number | null; + // true if terminated early due to output size or time + truncated?: boolean; +} + +export interface RipgrepOptions { + options?: string[]; + darwin?: string[]; + linux?: string[]; + timeout?: number; + maxSize?: number; +} + +export interface FindOptions { + timeout?: number; + // all safe whitelisted options to the find command + options?: string[]; + darwin?: string[]; + linux?: string[]; + maxSize?: number; +} + +export interface FdOptions { + pattern?: string; + options?: string[]; + darwin?: string[]; + linux?: string[]; + timeout?: number; + maxSize?: number; +} + +export interface DustOptions { + options?: string[]; + darwin?: string[]; + linux?: string[]; + timeout?: number; + maxSize?: number; +} + +export interface OuchOptions { + cwd?: string; + options?: string[]; + timeout?: number; +} + +export const OUCH_FORMATS = [ + "zip", + "7z", + "tar.gz", + "tar.xz", + "tar.bz", + "tar.bz2", + "tar.bz3", + "tar.lz4", + "tar.sz", + "tar.zst", + "tar.br", +]; + +export interface CopyOptions { + dereference?: boolean; + errorOnExist?: boolean; + force?: boolean; + preserveTimestamps?: boolean; + recursive?: boolean; + verbatimSymlinks?: boolean; + // if true, will try to use copy-on-write - this spawns the operating system '/usr/bin/cp' command. + reflink?: boolean; + // when using /usr/bin/cp: + timeout?: number; +} + +export interface Filesystem { + appendFile: (path: string, data: string | Buffer, encoding?) => Promise; + chmod: (path: string, mode: string | number) => Promise; + constants: () => Promise<{ [key: string]: number }>; + copyFile: (src: string, dest: string) => Promise; + + cp: ( + // NOTE!: we also support any array of src's unlike node's cp; + // however, when src is an array, the target *must* be a directory and this works like + // /usr/bin/cp, where files are copied INTO that target. + // When src is a string, this is just normal node cp behavior. + src: string | string[], + dest: string, + options?: CopyOptions, + ) => Promise; + exists: (path: string) => Promise; + link: (existingPath: string, newPath: string) => Promise; + lstat: (path: string) => Promise; + mkdir: (path: string, options?) => Promise; + + // move from fs-extra -- https://github.com/jprichardson/node-fs-extra/blob/HEAD/docs/move.md + move: ( + src: string | string[], + dest: string, + options?: { overwrite?: boolean }, + ) => Promise; + + readFile: (path: string, encoding?: any) => Promise; + readdir(path: string, options?): Promise; + readdir(path: string, options: { withFileTypes?: false }): Promise; + readdir(path: string, options: { withFileTypes: true }): Promise; + readlink: (path: string) => Promise; + realpath: (path: string) => Promise; + rename: (oldPath: string, newPath: string) => Promise; + rm: (path: string | string[], options?) => Promise; + rmdir: (path: string, options?) => Promise; + stat: (path: string) => Promise; + symlink: (target: string, path: string) => Promise; + truncate: (path: string, len?: number) => Promise; + unlink: (path: string) => Promise; + utimes: ( + path: string, + atime: number | string | Date, + mtime: number | string | Date, + ) => Promise; + writeFile: (path: string, data: string | Buffer) => Promise; + // todo: typing + watch: (path: string, options?) => Promise; + + // compression + ouch: (args: string[], options?: OuchOptions) => Promise; + + // We add very little to the Filesystem api, but we have to add + // a sandboxed "find" command, since it is a 1-call way to get + // arbitrary directory listing info, which is just not possible + // with the fs API, but required in any serious application. + // find -P {path} -maxdepth 1 -mindepth 1 -printf {printf} + // For security reasons, this does not support all find arguments, + // and can only use limited resources. + find: (path: string, options?: FindOptions) => Promise; + + // Convenience function that uses the find and stat support to + // provide all files in a directory by using tricky options to find, + // and ensuring they are used by stat in a consistent way for updates. + listing?: (path: string) => Promise; + + // fd is a rust rewrite of find that is extremely fast at finding + // files that match an expression, e.g., + // options: { type: "name", pattern:"^\.DS_Store$" } + fd: (path: string, options?: FdOptions) => Promise; + + // dust is an amazing disk space tool + dust: (path: string, options?: DustOptions) => Promise; + + // We add ripgrep, as a 1-call way to very efficiently search in files + // directly on whatever is serving files. + // For security reasons, this does not support all ripgrep arguments, + // and can only use limited resources. + ripgrep: ( + path: string, + pattern: string, + options?: RipgrepOptions, + ) => Promise; + + rustic: (args: string[]) => Promise; +} + +interface IDirent { + name: string; + parentPath: string; + path: string; + type?: number; +} + +const DIRENT_TYPES = { + 0: "UNKNOWN", + 1: "FILE", + 2: "DIR", + 3: "LINK", + 4: "FIFO", + 5: "SOCKET", + 6: "CHAR", + 7: "BLOCK", +}; + +class Dirent { + constructor( + public name: string, + public parentPath: string, + public path: string, + public type: number, + ) {} + isFile = () => DIRENT_TYPES[this.type] == "FILE"; + isDirectory = () => DIRENT_TYPES[this.type] == "DIR"; + isSymbolicLink = () => DIRENT_TYPES[this.type] == "LINK"; + isFIFO = () => DIRENT_TYPES[this.type] == "FIFO"; + isSocket = () => DIRENT_TYPES[this.type] == "SOCKET"; + isCharacterDevice = () => DIRENT_TYPES[this.type] == "CHAR"; + isBlockDevice = () => DIRENT_TYPES[this.type] == "BLOCK"; +} + +interface IStats { + dev: number; + ino: number; + mode: number; + nlink: number; + uid: number; + gid: number; + rdev: number; + size: number; + blksize: number; + blocks: number; + atimeMs: number; + mtimeMs: number; + ctimeMs: number; + birthtimeMs: number; + atime: Date; + mtime: Date; + ctime: Date; + birthtime: Date; +} + +export class Stats { + dev: number; + ino: number; + mode: number; + nlink: number; + uid: number; + gid: number; + rdev: number; + size: number; + blksize: number; + blocks: number; + atimeMs: number; + mtimeMs: number; + ctimeMs: number; + birthtimeMs: number; + atime: Date; + mtime: Date; + ctime: Date; + birthtime: Date; + + constructor(private constants: { [key: string]: number }) {} + + isSymbolicLink = () => + (this.mode & this.constants.S_IFMT) === this.constants.S_IFLNK; + + isFile = () => (this.mode & this.constants.S_IFMT) === this.constants.S_IFREG; + + isDirectory = () => + (this.mode & this.constants.S_IFMT) === this.constants.S_IFDIR; + + isBlockDevice = () => + (this.mode & this.constants.S_IFMT) === this.constants.S_IFBLK; + + isCharacterDevice = () => + (this.mode & this.constants.S_IFMT) === this.constants.S_IFCHR; + + isFIFO = () => (this.mode & this.constants.S_IFMT) === this.constants.S_IFIFO; + + isSocket = () => + (this.mode & this.constants.S_IFMT) === this.constants.S_IFSOCK; + + get type(): FileTypeLabel { + switch (this.mode & this.constants.S_IFMT) { + case this.constants.S_IFLNK: + return "l"; + case this.constants.S_IFREG: + return "f"; + case this.constants.S_IFDIR: + return "d"; + case this.constants.S_IFBLK: + return "b"; + case this.constants.S_IFCHR: + return "c"; + case this.constants.S_IFSOCK: + return "s"; + case this.constants.S_IFIFO: + return "p"; + } + return "f"; + } +} + +interface Options { + service: string; + client?: Client; + fs: (subject?: string) => Promise; + // project-id: if given, ONLY serve files for this one project, and the + // path must be the home of the project + // If not given, serves files for all projects. + project_id?: string; +} + +export async function fsServer({ service, fs, client, project_id }: Options) { + client ??= conat(); + const subject = project_id + ? `${service}.project-${project_id}` + : `${service}.*`; + const watches: { [subject: string]: any } = {}; + const sub = await client.service(subject, { + async appendFile(path: string, data: string | Buffer, encoding?) { + await (await fs(this.subject)).appendFile(path, data, encoding); + }, + async chmod(path: string, mode: string | number) { + await (await fs(this.subject)).chmod(path, mode); + }, + async constants(): Promise<{ [key: string]: number }> { + return await (await fs(this.subject)).constants(); + }, + async copyFile(src: string, dest: string) { + await (await fs(this.subject)).copyFile(src, dest); + }, + async cp(src: string | string[], dest: string, options?) { + await (await fs(this.subject)).cp(src, dest, options); + }, + async dust(path: string, options?: DustOptions) { + return await (await fs(this.subject)).dust(path, options); + }, + async exists(path: string): Promise { + return await (await fs(this.subject)).exists(path); + }, + async fd(path: string, options?: FdOptions) { + return await (await fs(this.subject)).fd(path, options); + }, + async find(path: string, options?: FindOptions) { + return await (await fs(this.subject)).find(path, options); + }, + async link(existingPath: string, newPath: string) { + await (await fs(this.subject)).link(existingPath, newPath); + }, + async lstat(path: string): Promise { + return await (await fs(this.subject)).lstat(path); + }, + async mkdir(path: string, options?) { + await (await fs(this.subject)).mkdir(path, options); + }, + async ouch(args: string[], options?: OuchOptions) { + return await (await fs(this.subject)).ouch(args, options); + }, + async readFile(path: string, encoding?) { + return await (await fs(this.subject)).readFile(path, encoding); + }, + async readdir(path: string, options?) { + const files = await (await fs(this.subject)).readdir(path, options); + if (!options?.withFileTypes) { + return files; + } + // Dirent - change the [Symbol(type)] field to something serializable so client can use this: + return files.map((x) => { + // @ts-ignore + return { ...x, type: x[Object.getOwnPropertySymbols(x)[0]] }; + }); + }, + async readlink(path: string) { + return await (await fs(this.subject)).readlink(path); + }, + async realpath(path: string) { + return await (await fs(this.subject)).realpath(path); + }, + async rename(oldPath: string, newPath: string) { + await (await fs(this.subject)).rename(oldPath, newPath); + }, + async move( + src: string | string[], + dest: string, + options?: { overwrite?: boolean }, + ) { + return await (await fs(this.subject)).move(src, dest, options); + }, + async ripgrep(path: string, pattern: string, options?: RipgrepOptions) { + return await (await fs(this.subject)).ripgrep(path, pattern, options); + }, + async rustic(args: string[]) { + return await (await fs(this.subject)).rustic(args); + }, + async rm(path: string | string[], options?) { + await (await fs(this.subject)).rm(path, options); + }, + async rmdir(path: string, options?) { + await (await fs(this.subject)).rmdir(path, options); + }, + async stat(path: string): Promise { + const s = await (await fs(this.subject)).stat(path); + return { + ...s, + // for some reason these times get corrupted on transport from the nodejs datastructure, + // so we make them standard Date objects. + atime: new Date(s.atime), + mtime: new Date(s.mtime), + ctime: new Date(s.ctime), + birthtime: new Date(s.birthtime), + }; + }, + async symlink(target: string, path: string) { + await (await fs(this.subject)).symlink(target, path); + }, + async truncate(path: string, len?: number) { + await (await fs(this.subject)).truncate(path, len); + }, + async unlink(path: string) { + await (await fs(this.subject)).unlink(path); + }, + async utimes( + path: string, + atime: number | string | Date, + mtime: number | string | Date, + ) { + await (await fs(this.subject)).utimes(path, atime, mtime); + }, + async writeFile(path: string, data: string | Buffer) { + await (await fs(this.subject)).writeFile(path, data); + }, + // @ts-ignore + async watch() { + const subject = this.subject!; + if (watches[subject] != null) { + return; + } + const f = await fs(subject); + watches[subject] = watchServer({ + client, + subject: subject!, + watch: f.watch, + }); + }, + }); + return { + close: () => { + for (const subject in watches) { + watches[subject].close(); + delete watches[subject]; + } + sub.close(); + }, + }; +} + +export type FilesystemClient = Omit, "lstat"> & { + listing: (path: string) => Promise; + stat: (path: string) => Promise; + lstat: (path: string) => Promise; +}; + +export function getService({ + compute_server_id, + service = DEFAULT_FILE_SERVICE, +}: { + compute_server_id?: number; + service?: string; +}) { + return compute_server_id ? `${service}/${compute_server_id}` : service; +} + +export function fsSubject({ + project_id, + compute_server_id = 0, + service = DEFAULT_FILE_SERVICE, +}: { + project_id: string; + compute_server_id?: number; + service?: string; +}) { + if (!isValidUUID(project_id)) { + throw Error(`project_id must be a valid uuid -- ${project_id}`); + } + if (typeof compute_server_id != "number") { + throw Error("compute_server_id must be a number"); + } + if (typeof service != "string") { + throw Error("service must be a string"); + } + return `${getService({ service, compute_server_id })}.project-${project_id}`; +} + +const DEFAULT_FS_CALL_TIMEOUT = 5 * 60_000; + +export function fsClient({ + client, + subject, + timeout = DEFAULT_FS_CALL_TIMEOUT, +}: { + client?: Client; + subject: string; + timeout?: number; +}): FilesystemClient { + client ??= conat(); + let call = client.call(subject, { timeout }); + + const readdir0 = call.readdir.bind(call); + call.readdir = async (path: string, options?) => { + const files = await readdir0(path, options); + if (options?.withFileTypes) { + return files.map((x) => new Dirent(x.name, x.parentPath, x.path, x.type)); + } else { + return files; + } + }; + + let constants: any = null; + const stat0 = call.stat.bind(call); + call.stat = async (path: string) => { + const s = await stat0(path); + constants = constants ?? (await call.constants()); + const stats = new Stats(constants); + for (const k in s) { + stats[k] = s[k]; + } + return stats; + }; + + const lstat0 = call.lstat.bind(call); + call.lstat = async (path: string) => { + const s = await lstat0(path); + constants = constants ?? (await call.constants()); + const stats = new Stats(constants); + for (const k in s) { + stats[k] = s[k]; + } + return stats; + }; + + const ensureWatchServerExists = call.watch.bind(call); + call.watch = async (path: string, options?) => { + await ensureWatchServerExists(path, options); + return await watchClient({ client, subject, path, options }); + }; + call.listing = async (path: string) => { + return await listing({ fs: call, path }); + }; + + return call; +} diff --git a/src/packages/conat/files/listing.ts b/src/packages/conat/files/listing.ts new file mode 100644 index 00000000000..ff0ecdf4fe8 --- /dev/null +++ b/src/packages/conat/files/listing.ts @@ -0,0 +1,187 @@ +/* +Directory Listing + +Tests in packages/backend/conat/files/test/listing.test.ts + + +*/ + +import { EventEmitter } from "events"; +import { join } from "path"; +import { type FilesystemClient } from "./fs"; +import { EventIterator } from "@cocalc/util/event-iterator"; + +export type FileTypeLabel = "f" | "d" | "l" | "b" | "c" | "s" | "p"; + +export const typeDescription = { + f: "regular file", + d: "directory", + l: "symlink", + b: "block device", + c: "character device", + s: "socket", + p: "fifo", +}; + +interface FileData { + // last modification time as time since epoch in **milliseconds** (as is usual for javascript) + mtime: number; + size: number; + // isDir = mainly for backward compat: + isDir?: boolean; + // issymlink = mainly for backward compat: + isSymLink?: boolean; + linkTarget?: string; + // see typeDescription above. + type?: FileTypeLabel; +} + +export type Files = { [name: string]: FileData }; + +interface Options { + path: string; + fs: FilesystemClient; +} + +export default async function listing(opts: Options): Promise { + const listing = new Listing(opts); + await listing.init(); + return listing; +} + +export class Listing extends EventEmitter { + public files?: Files = {}; + public truncated?: boolean; + private watch?; + private iters: EventIterator[] = []; + constructor(public readonly opts: Options) { + super(); + } + + iter = () => { + const iter = new EventIterator(this, "change", { + map: (args) => { + return { name: args[0], ...args[1] }; + }, + }); + this.iters.push(iter); + return iter; + }; + + close = () => { + this.emit("closed"); + this.removeAllListeners(); + this.iters.map((iter) => iter.end()); + this.iters.length = 0; + this.watch?.close(); + delete this.files; + delete this.watch; + }; + + init = async () => { + const { fs, path } = this.opts; + this.watch = await fs.watch(path); + const { files, truncated } = await getListing(fs, path); + this.files = files; + this.truncated = truncated; + this.emit("ready"); + this.handleUpdates(); + }; + + private handleUpdates = async () => { + for await (const { filename } of this.watch) { + if (this.files == null || !filename) { + return; + } + this.update(filename); + } + }; + + private update = async (filename: string) => { + if (this.files == null) { + // closed or not initialized yet + return; + } + try { + const stats = await this.opts.fs.lstat(join(this.opts.path, filename)); + if (this.files == null) { + return; + } + const data: FileData = { + mtime: stats.mtimeMs, + size: stats.size, + type: stats.type, + }; + if (stats.isSymbolicLink()) { + // resolve target. + data.linkTarget = await this.opts.fs.readlink( + join(this.opts.path, filename), + ); + data.isSymLink = true; + } + if (stats.isDirectory()) { + data.isDir = true; + } + this.files[filename] = data; + } catch (err) { + if (this.files == null) { + return; + } + if (err.code == "ENOENT") { + // file deleted + delete this.files[filename]; + } else { + //if (!process.env.COCALC_TEST_MODE) { + console.warn("WARNING:", err); + // TODO: some other error -- e.g., network down or permissions, so we don't know anything. + // Should we retry (?). + //} + return; + } + } + this.emit("change", filename, this.files[filename]); + }; +} + +async function getListing( + fs: FilesystemClient, + path: string, +): Promise<{ files: Files; truncated?: boolean }> { + const { stdout, truncated } = await fs.find(path, { + options: [ + "-maxdepth", + "1", + "-mindepth", + "1", + "-printf", + "%f\\0%T@\\0%s\\0%y\\0%l\n", + ], + }); + const buf = Buffer.from(stdout); + const files: Files = {}; + // todo -- what about non-utf8...? + + const s = buf.toString().trim(); + if (!s) { + return { files, truncated }; + } + for (const line of s.split("\n")) { + try { + const v = line.split("\0"); + const name = v[0]; + const mtime = parseFloat(v[1]) * 1000; + const size = parseInt(v[2]); + files[name] = { mtime, size, type: v[3] as FileTypeLabel }; + if (v[3] == "l") { + files[name].isSymLink = true; + } + if (v[3] == "d") { + files[name].isDir = true; + } + if (v[4]) { + files[name].linkTarget = v[4]; + } + } catch {} + } + return { files, truncated }; +} diff --git a/src/packages/conat/files/watch.ts b/src/packages/conat/files/watch.ts new file mode 100644 index 00000000000..4c1beade5bf --- /dev/null +++ b/src/packages/conat/files/watch.ts @@ -0,0 +1,198 @@ +/* +Remotely proxying a fs.watch AsyncIterator over a Conat Socket. +*/ + +import { + type Client as ConatClient, + headerToError, +} from "@cocalc/conat/core/client"; +import { + type ConatSocketServer, + type ServerSocket, +} from "@cocalc/conat/socket"; +import { EventIterator } from "@cocalc/util/event-iterator"; +import { getLogger } from "@cocalc/conat/client"; + +const logger = getLogger("conat:files:watch"); + +// (path:string, options:WatchOptions) => AsyncIterator +type AsyncWatchFunction = any; + +// see https://nodejs.org/docs/latest/api/fs.html#fspromiseswatchfilename-options +export interface WatchOptions { + persistent?: boolean; + recursive?: boolean; + encoding?: string; + signal?: AbortSignal; + maxQueue?: number; + overflow?: "ignore" | "throw"; + + // if more than one client is actively watching the same path and has unique set, all but one may receive + // the extra field ignore:true in the update. Also, if there are multiple clients with unique set, the + // other options of all but the first are ignored. + unique?: boolean; +} + +export function watchServer({ + client, + subject, + watch, +}: { + client: ConatClient; + subject: string; + watch: AsyncWatchFunction; +}) { + const server: ConatSocketServer = client.socket.listen(subject); + logger.debug("server: listening on ", { subject }); + + const unique: { [path: string]: ServerSocket[] } = {}; + const ignores: { [path: string]: { ignoreUntil: number }[] } = {}; + async function handleUnique({ mesg, socket, path, options, ignore }) { + let w: any = undefined; + + socket.once("closed", () => { + // when this socket closes, remove it from recipient list + unique[path] = unique[path]?.filter((x) => x.id != socket.id); + if (unique[path] != null && unique[path].length == 0) { + // nobody listening + w?.close(); + w = undefined; + delete unique[path]; + delete ignores[path]; + } + }); + + if (unique[path] == null) { + // set it up + unique[path] = [socket]; + ignores[path] = [ignore]; + w = await watch(path, options); + await mesg.respond(); + for await (const event of w) { + const now = Date.now(); + let ignore = false; + for (const { ignoreUntil } of ignores[path]) { + if (ignoreUntil > now) { + // every client is told to ignore this change, i.e., not load based on it happening + ignore = true; + break; + } + } + for (const s of unique[path]) { + if (s.state == "ready") { + if (ignore) { + s.write({ ...event, ignore: true }); + } else { + s.write(event); + ignore = true; + } + } + } + } + } else { + unique[path].push(socket); + ignores[path].push(ignore); + await mesg.respond(); + } + } + + async function handleNonUnique({ mesg, socket, path, options, ignore }) { + const w = await watch(path, options); + socket.once("closed", () => { + w.close(); + }); + await mesg.respond(); + for await (const event of w) { + if (ignore.ignoreUntil >= Date.now()) { + continue; + } + socket.write(event); + } + } + + server.on("connection", (socket: ServerSocket) => { + logger.debug("server: got new connection", { + id: socket.id, + subject: socket.subject, + }); + let initialized = false; + const ignore = { ignoreUntil: 0 }; + socket.on("request", async (mesg) => { + const data = mesg.data; + if (data.ignore != null) { + ignore.ignoreUntil = data.ignore > 0 ? Date.now() + data.ignore : 0; + await mesg.respond(null, { noThrow: true }); + return; + } + try { + if (initialized) { + throw Error("already initialized"); + } + initialized = true; + const { path, options } = data; + logger.debug("got request", { path, options }); + if (options?.unique) { + await handleUnique({ mesg, socket, path, options, ignore }); + } else { + await handleNonUnique({ mesg, socket, path, options, ignore }); + } + } catch (err) { + mesg.respondSync(null, { + headers: { error: `${err}`, code: err.code }, + }); + } + }); + }); + + return server; +} + +export type WatchIterator = EventIterator & { + ignore?: (ignore: number) => Promise; +}; + +export interface ChangeEvent { + eventType: "change" | "rename"; + filename: string; +} + +export async function watchClient({ + client, + subject, + path, + options, +}: { + client: ConatClient; + subject: string; + path: string; + options?: WatchOptions; +}): Promise { + const socket = client.socket.connect(subject); + const iter = new EventIterator(socket, "data", { + map: (args) => args[0], + onEnd: () => { + socket.close(); + }, + }); + socket.on("closed", () => { + iter.end(); + delete iter2.ignore; + }); + // tell it what to watch + const resp = await socket.request({ + path, + options, + }); + if (resp.headers?.error) { + throw headerToError(resp.headers); + } + + const iter2 = iter as WatchIterator; + + // ignore events for ignore ms. + iter2.ignore = async (ignore: number) => { + await socket.request({ ignore }); + }; + + return iter2; +} diff --git a/src/packages/conat/hub/api/projects.ts b/src/packages/conat/hub/api/projects.ts index be1d473d0b6..c089a3ea258 100644 --- a/src/packages/conat/hub/api/projects.ts +++ b/src/packages/conat/hub/api/projects.ts @@ -1,6 +1,6 @@ import { authFirstRequireAccount } from "./util"; import { type CreateProjectOptions } from "@cocalc/util/db-schema/projects"; -import { type UserCopyOptions } from "@cocalc/util/db-schema/projects"; +import { type CopyOptions } from "@cocalc/conat/files/fs"; export const projects = { createProject: authFirstRequireAccount, @@ -10,6 +10,8 @@ export const projects = { inviteCollaborator: authFirstRequireAccount, inviteCollaboratorWithoutAccount: authFirstRequireAccount, setQuotas: authFirstRequireAccount, + + getDiskQuota: authFirstRequireAccount, }; export type AddCollaborator = @@ -30,7 +32,11 @@ export interface Projects { // request to have conat permissions to project subjects. createProject: (opts: CreateProjectOptions) => Promise; - copyPathBetweenProjects: (opts: UserCopyOptions) => Promise; + copyPathBetweenProjects: (opts: { + src: { project_id: string; path: string | string[] }; + dest: { project_id: string; path: string }; + options?: CopyOptions; + }) => Promise; removeCollaborator: ({ account_id, @@ -85,6 +91,7 @@ export interface Projects { }; }) => Promise; + // for admins only! setQuotas: (opts: { account_id?: string; project_id: string; @@ -98,4 +105,11 @@ export interface Projects { member_host?: number; always_running?: number; }) => Promise; + + getDiskQuota: (opts: { + account_id?: string; + project_id: string; + }) => Promise<{ used: number; size: number }>; + + } diff --git a/src/packages/conat/llm/server.ts b/src/packages/conat/llm/server.ts index 77afd7d86d1..c13ef765d20 100644 --- a/src/packages/conat/llm/server.ts +++ b/src/packages/conat/llm/server.ts @@ -14,6 +14,9 @@ how paying for that would work. import { conat } from "@cocalc/conat/client"; import { isValidUUID } from "@cocalc/util/misc"; import type { Subscription } from "@cocalc/conat/core/client"; +import { getLogger } from "@cocalc/conat/client"; + +const logger = getLogger("conat:llm:server"); export const SUBJECT = process.env.COCALC_TEST_MODE ? "llm-test" : "llm"; @@ -61,7 +64,7 @@ export async function close() { if (sub == null) { return; } - sub.drain(); + sub.close(); sub = null; } @@ -77,24 +80,36 @@ async function listen(evaluate) { async function handleMessage(mesg, evaluate) { const options = mesg.data; - let seq = 0; - const respond = ({ text, error }: { text?: string; error?: string }) => { - mesg.respondSync({ text, error, seq }); + let seq = -1; + const respond = async ({ + text, + error, + }: { + text?: string; + error?: string; + }) => { seq += 1; + try { + await mesg.respond({ text, error, seq }); + } catch (err) { + logger.debug("WARNING: error sending response -- ", err); + end(); + } }; let done = false; - const end = () => { + const end = async () => { if (done) return; done = true; - // end response stream with null payload. - mesg.respondSync(null); + // end response stream with null payload -- send sync, or it could + // get sent before the responses above, which would cancel them out! + await mesg.respond(null, { noThrow: true }); }; - const stream = (text?) => { + const stream = async (text?) => { if (done) return; if (text != null) { - respond({ text }); + await respond({ text }); } else { end(); } diff --git a/src/packages/conat/package.json b/src/packages/conat/package.json index 79bfe77778c..1d8dd0f6ca0 100644 --- a/src/packages/conat/package.json +++ b/src/packages/conat/package.json @@ -7,6 +7,8 @@ "./llm/*": "./dist/llm/*.js", "./socket": "./dist/socket/index.js", "./socket/*": "./dist/socket/*.js", + "./sync-doc": "./dist/sync-doc/index.js", + "./sync-doc/*": "./dist/sync-doc/*.js", "./hub/changefeeds": "./dist/hub/changefeeds/index.js", "./hub/api": "./dist/hub/api/index.js", "./hub/api/*": "./dist/hub/api/*.js", @@ -24,21 +26,14 @@ "test": "pnpm exec jest", "depcheck": "pnpx depcheck --ignores events" }, - "files": [ - "dist/**", - "README.md", - "package.json" - ], + "files": ["dist/**", "README.md", "package.json"], "author": "SageMath, Inc.", - "keywords": [ - "utilities", - "conat", - "cocalc" - ], + "keywords": ["utilities", "conat", "cocalc"], "license": "SEE LICENSE.md", "dependencies": { "@cocalc/comm": "workspace:*", "@cocalc/conat": "workspace:*", + "@cocalc/sync": "workspace:*", "@cocalc/util": "workspace:*", "@isaacs/ttlcache": "^1.4.1", "@msgpack/msgpack": "^3.1.1", @@ -56,7 +51,6 @@ }, "devDependencies": { "@types/better-sqlite3": "^7.6.13", - "@types/json-stable-stringify": "^1.0.32", "@types/lodash": "^4.14.202", "@types/node": "^18.16.14" }, diff --git a/src/packages/conat/persist/server.ts b/src/packages/conat/persist/server.ts index 5a270630e79..8ad17e4715a 100644 --- a/src/packages/conat/persist/server.ts +++ b/src/packages/conat/persist/server.ts @@ -59,7 +59,6 @@ import { type ConatSocketServer, type ServerSocket, } from "@cocalc/conat/socket"; -import { getLogger } from "@cocalc/conat/client"; import type { StoredMessage, PersistentStream, @@ -70,6 +69,7 @@ import { throttle } from "lodash"; import { type SetOptions } from "./client"; import { once } from "@cocalc/util/async-utils"; import { UsageMonitor } from "@cocalc/conat/monitor/usage"; +import { getLogger } from "@cocalc/conat/client"; const logger = getLogger("persist:server"); diff --git a/src/packages/conat/project/api/editor.ts b/src/packages/conat/project/api/editor.ts index cd70dc973dc..66750e2c511 100644 --- a/src/packages/conat/project/api/editor.ts +++ b/src/packages/conat/project/api/editor.ts @@ -1,17 +1,14 @@ -import type { NbconvertParams } from "@cocalc/util/jupyter/types"; -import type { RunNotebookOptions } from "@cocalc/util/jupyter/nbgrader-types"; import type { Options as FormatterOptions } from "@cocalc/util/code-formatter"; -import type { KernelSpec } from "@cocalc/util/jupyter/types"; export const editor = { newFile: true, - jupyterStripNotebook: true, - jupyterNbconvert: true, - jupyterRunNotebook: true, - jupyterKernelLogo: true, - jupyterKernels: true, - formatterString: true, + + formatString: true, + printSageWS: true, + sagewsStart: true, + sagewsStop: true, + createTerminalService: true, }; @@ -33,27 +30,16 @@ export interface Editor { // context of our editors. newFile: (path: string) => Promise; - jupyterStripNotebook: (path_ipynb: string) => Promise; - - jupyterNbconvert: (opts: NbconvertParams) => Promise; - - jupyterRunNotebook: (opts: RunNotebookOptions) => Promise; - - jupyterKernelLogo: ( - kernelName: string, - opts?: { noCache?: boolean }, - ) => Promise<{ filename: string; base64: string }>; - - jupyterKernels: (opts?: { noCache?: boolean }) => Promise; - - // returns a patch to transform str into formatted form. - formatterString: (opts: { + // returns formatted version of str. + formatString: (opts: { str: string; options: FormatterOptions; path?: string; // only used for CLANG }) => Promise; printSageWS: (opts) => Promise; + sagewsStart: (path_sagews: string) => Promise; + sagewsStop: (path_sagews: string) => Promise; createTerminalService: ( termPath: string, diff --git a/src/packages/conat/project/api/index.ts b/src/packages/conat/project/api/index.ts index 694229b2bcd..848b9aa09aa 100644 --- a/src/packages/conat/project/api/index.ts +++ b/src/packages/conat/project/api/index.ts @@ -1,17 +1,21 @@ import { type System, system } from "./system"; import { type Editor, editor } from "./editor"; +import { type Jupyter, jupyter } from "./jupyter"; import { type Sync, sync } from "./sync"; import { handleErrorMessage } from "@cocalc/conat/util"; +export { projectApiClient } from "./project-client"; export interface ProjectApi { system: System; editor: Editor; + jupyter: Jupyter; sync: Sync; } const ProjectApiStructure = { system, editor, + jupyter, sync, } as const; diff --git a/src/packages/conat/project/api/jupyter.ts b/src/packages/conat/project/api/jupyter.ts new file mode 100644 index 00000000000..22f7e46211c --- /dev/null +++ b/src/packages/conat/project/api/jupyter.ts @@ -0,0 +1,55 @@ +import type { NbconvertParams } from "@cocalc/util/jupyter/types"; +import type { RunNotebookOptions } from "@cocalc/util/jupyter/nbgrader-types"; +import type { KernelSpec } from "@cocalc/util/jupyter/types"; + +export const jupyter = { + start: true, + stop: true, + stripNotebook: true, + nbconvert: true, + runNotebook: true, + kernelLogo: true, + kernels: true, + introspect: true, + complete: true, + signal: true, + getConnectionFile: true, +}; + +// In the functions below path can be either the .ipynb or the .sage-jupyter2 path, and +// the correct backend kernel will get found/created automatically. +export interface Jupyter { + stripNotebook: (path_ipynb: string) => Promise; + + // path = the syncdb path (not *.ipynb) + start: (path: string) => Promise; + stop: (path: string) => Promise; + + nbconvert: (opts: NbconvertParams) => Promise; + + runNotebook: (opts: RunNotebookOptions) => Promise; + + kernelLogo: ( + kernelName: string, + opts?: { noCache?: boolean }, + ) => Promise<{ filename: string; base64: string }>; + + kernels: (opts?: { noCache?: boolean }) => Promise; + + introspect: (opts: { + path: string; + code: string; + cursor_pos: number; + detail_level: 0 | 1; + }) => Promise; + + complete: (opts: { + path: string; + code: string; + cursor_pos: number; + }) => Promise; + + getConnectionFile: (opts: { path: string }) => Promise; + + signal: (opts: { path: string; signal: string }) => Promise; +} diff --git a/src/packages/conat/project/api/project-client.ts b/src/packages/conat/project/api/project-client.ts new file mode 100644 index 00000000000..a2000406614 --- /dev/null +++ b/src/packages/conat/project/api/project-client.ts @@ -0,0 +1,66 @@ +/* +Create a client for the project's api. Anything that can publish to *.project-project_id... can use this. +*/ + +import { projectSubject } from "@cocalc/conat/names"; +import { type Client, connect } from "@cocalc/conat/core/client"; +import { isValidUUID } from "@cocalc/util/misc"; +import { type ProjectApi, initProjectApi } from "./index"; + +const DEFAULT_TIMEOUT = 15000; + +export function projectApiClient({ + project_id, + compute_server_id = 0, + client = connect(), + timeout = DEFAULT_TIMEOUT, +}: { + project_id: string; + compute_server_id?: number; + client?: Client; + timeout?: number; +}): ProjectApi { + if (!isValidUUID(project_id)) { + throw Error(`project_id = '${project_id}' must be a valid uuid`); + } + const callProjectApi = async ({ name, args }) => { + return await callProject({ + client, + project_id, + compute_server_id, + timeout, + service: "api", + name, + args, + }); + }; + return initProjectApi(callProjectApi); +} + +async function callProject({ + client, + service = "api", + project_id, + compute_server_id, + name, + args = [], + timeout = DEFAULT_TIMEOUT, +}: { + client: Client; + service?: string; + project_id: string; + compute_server_id?: number; + name: string; + args: any[]; + timeout?: number; +}) { + const subject = projectSubject({ project_id, compute_server_id, service }); + const resp = await client.request( + subject, + { name, args }, + // we use waitForInterest because often the project hasn't + // quite fully started. + { timeout, waitForInterest: true }, + ); + return resp.data; +} diff --git a/src/packages/conat/project/api/system.ts b/src/packages/conat/project/api/system.ts index 3e780381c86..a2fbfa9dc1b 100644 --- a/src/packages/conat/project/api/system.ts +++ b/src/packages/conat/project/api/system.ts @@ -15,7 +15,6 @@ export const system = { version: true, listing: true, - deleteFiles: true, moveFiles: true, renameFile: true, realpath: true, @@ -46,7 +45,6 @@ export interface System { path: string; hidden?: boolean; }) => Promise; - deleteFiles: (opts: { paths: string[] }) => Promise; moveFiles: (opts: { paths: string[]; dest: string }) => Promise; renameFile: (opts: { src: string; dest: string }) => Promise; realpath: (path: string) => Promise; diff --git a/src/packages/conat/project/jupyter/run-code.ts b/src/packages/conat/project/jupyter/run-code.ts new file mode 100644 index 00000000000..5643b21dd27 --- /dev/null +++ b/src/packages/conat/project/jupyter/run-code.ts @@ -0,0 +1,310 @@ +/* +A conat socket server that takes as input + +Tests are in + +packages/backend/conat/test/juypter/run-code.test.s + +*/ + +import { type Client as ConatClient } from "@cocalc/conat/core/client"; +import { + type ConatSocketServer, + type ServerSocket, +} from "@cocalc/conat/socket"; +import { EventIterator } from "@cocalc/util/event-iterator"; +import { getLogger } from "@cocalc/conat/client"; +import { Throttle } from "@cocalc/util/throttle"; +const MAX_MSGS_PER_SECOND = parseInt( + process.env.COCALC_JUPYTER_MAX_MSGS_PER_SECOND ?? "20", +); +const logger = getLogger("conat:project:jupyter:run-code"); + +function getSubject({ + project_id, + compute_server_id = 0, +}: { + project_id: string; + compute_server_id?: number; +}) { + return `jupyter.project-${project_id}.${compute_server_id}`; +} + +export interface InputCell { + id: string; + input: string; + output?: { [n: string]: OutputMessage | null } | null; + state?: "done" | "busy" | "run"; + exec_count?: number | null; + start?: number | null; + end?: number | null; + cell_type?: "code"; +} + +export interface OutputMessage { + // id = id of the cell + id: string; + // everything below is exactly from Jupyter + metadata?; + content?; + buffers?; + msg_type?: string; + done?: boolean; +} + +export interface RunOptions { + // syncdb path + path: string; + // array of input cells to run + cells: InputCell[]; + // if true do not halt running the cells, even if one fails with an error + noHalt?: boolean; + // the socket is used for raw_input, to communicate between the client + // that initiated the request and the server. + socket: ServerSocket; +} + +type JupyterCodeRunner = ( + opts: RunOptions, +) => Promise>; + +interface OutputHandler { + process: (mesg: OutputMessage) => void; + done: () => void; +} + +type CreateOutputHandler = (opts: { + path: string; + cells: InputCell[]; +}) => OutputHandler; + +export function jupyterServer({ + client, + project_id, + compute_server_id = 0, + // run takes a path and cells to run and returns an async iterator + // over the outputs. + run, + // outputHandler takes a path and returns an OutputHandler, which can be + // used to process the output and include it in the notebook. It is used + // as a fallback in case the client that initiated running cells is + // disconnected, so output won't be lost. + outputHandler, +}: { + client: ConatClient; + project_id: string; + compute_server_id?: number; + run: JupyterCodeRunner; + outputHandler?: CreateOutputHandler; +}) { + const subject = getSubject({ project_id, compute_server_id }); + const server: ConatSocketServer = client.socket.listen(subject, { + keepAlive: 5000, + keepAliveTimeout: 5000, + }); + logger.debug("server: listening on ", { subject }); + + server.on("connection", (socket: ServerSocket) => { + logger.debug("server: got new connection", { + id: socket.id, + subject: socket.subject, + }); + + socket.on("request", async (mesg) => { + const { path, cells, noHalt } = mesg.data; + try { + mesg.respondSync(null); + await handleRequest({ + socket, + run, + outputHandler, + path, + cells, + noHalt, + }); + } catch (err) { + logger.debug("server: failed to handle execute request -- ", err); + if (socket.state != "closed") { + try { + logger.debug("sending to client: ", { + headers: { error: `${err}` }, + }); + socket.write(null, { headers: { foo: "bar", error: `${err}` } }); + } catch (err) { + // an error trying to report an error shouldn't crash everything + logger.debug("WARNING: unable to send error to client", err); + } + } + } + }); + + socket.on("closed", () => { + logger.debug("socket closed", { id: socket.id }); + }); + }); + + return server; +} + +async function handleRequest({ + socket, + run, + outputHandler, + path, + cells, + noHalt, +}) { + const runner = await run({ path, cells, noHalt, socket }); + const output: OutputMessage[] = []; + + const throttle = new Throttle(MAX_MSGS_PER_SECOND); + let unhandledClientWriteError: any = undefined; + throttle.on("data", async (mesgs) => { + try { + socket.write(mesgs); + } catch (err) { + if (err.code == "ENOBUFS") { + // wait for the over-filled socket to finish writing out data. + await socket.drain(); + socket.write(mesgs); + } else { + unhandledClientWriteError = err; + } + } + }); + + try { + let handler: OutputHandler | null = null; + for await (const mesg of runner) { + if (socket.state == "closed") { + // client socket has closed -- the backend server must take over! + if (handler == null) { + logger.debug("socket closed -- server must handle output"); + if (outputHandler == null) { + throw Error("no output handler available"); + } + handler = outputHandler({ path, cells }); + if (handler == null) { + throw Error("bug -- outputHandler must return a handler"); + } + for (const prev of output) { + handler.process(prev); + } + output.length = 0; + } + handler.process(mesg); + } else { + if (unhandledClientWriteError) { + throw unhandledClientWriteError; + } + output.push(mesg); + throttle.write(mesg); + } + } + // no errors happened, so close up and flush and + // remaining data immediately: + handler?.done(); + if (socket.state != "closed") { + throttle.flush(); + socket.write(null); + } + } finally { + throttle.close(); + } +} + +class JupyterClient { + private iter?: EventIterator; + private socket; + constructor( + private client: ConatClient, + private subject: string, + private path: string, + private stdin: (opts: { + id: string; + prompt: string; + password?: boolean; + }) => Promise, + ) { + this.socket = this.client.socket.connect(this.subject); + this.socket.once("close", () => this.iter?.end()); + this.socket.on("request", async (mesg) => { + const { data } = mesg; + try { + switch (data.type) { + case "stdin": + await mesg.respond(await this.stdin(data)); + return; + default: + console.warn(`Jupyter: got unknown message type '${data.type}'`); + await mesg.respond( + new Error(`unknown message type '${data.type}'`), + ); + } + } catch (err) { + console.warn("error responding to jupyter request", err); + } + }); + } + + close = () => { + this.iter?.end(); + delete this.iter; + this.socket.close(); + }; + + run = async (cells: InputCell[], opts: { noHalt?: boolean } = {}) => { + if (this.iter) { + // one evaluation at a time -- starting a new one ends the previous one. + // Each client browser has a separate instance of JupyterClient, so + // a properly implemented frontend client would never hit this. + this.iter.end(); + delete this.iter; + } + this.iter = new EventIterator(this.socket, "data", { + map: (args) => { + if (args[1]?.error) { + this.iter?.throw(Error(args[1].error)); + return; + } + if (args[0] == null) { + this.iter?.end(); + return; + } else { + return args[0]; + } + }, + }); + // get rid of any fields except id and input from the cells, since, e.g., + // if there is a lot of output in a cell, there is no need to send that to the backend. + const cells1 = cells.map(({ id, input }) => { + return { id, input }; + }); + await this.socket.request({ + path: this.path, + cells: cells1, + noHalt: opts.noHalt, + }); + return this.iter; + }; +} + +export function jupyterClient(opts: { + path: string; + project_id: string; + compute_server_id?: number; + client: ConatClient; + stdin?: (opts: { + id: string; + prompt: string; + password?: boolean; + }) => Promise; +}) { + const subject = getSubject(opts); + return new JupyterClient( + opts.client, + subject, + opts.path, + opts.stdin ?? (async () => "stdin not implemented"), + ); +} diff --git a/src/packages/conat/project/runner/load-balancer.ts b/src/packages/conat/project/runner/load-balancer.ts new file mode 100644 index 00000000000..3b1143a6523 --- /dev/null +++ b/src/packages/conat/project/runner/load-balancer.ts @@ -0,0 +1,189 @@ +/* +Service to load balance running cocalc projects across the runners. + +Tests are in + + - packages/backend/conat/test/project + +*/ + +import { type Client } from "@cocalc/conat/core/client"; +import { randomChoice } from "@cocalc/conat/core/server"; +import { conat } from "@cocalc/conat/client"; +import { client as projectRunnerClient, UPDATE_INTERVAL } from "./run"; +import { getLogger } from "@cocalc/conat/client"; +import state, { type ProjectStatus, type ProjectState } from "./state"; +import { field_cmp } from "@cocalc/util/misc"; +import { delay } from "awaiting"; + +const logger = getLogger("conat:project:runner:load-balancer"); + +const MAX_STATUS_TRIES = 3; + +export interface Options { + subject?: string; + client?: Client; + setState?: (opts: { + project_id: string; + state: ProjectState; + }) => Promise; + getConfig?: ({ project_id }: { project_id: string }) => Promise; +} + +export interface API { + start: () => Promise; + stop: () => Promise; + status: () => Promise; +} + +export async function server({ + subject = "project.*.run", + client, + setState, + getConfig, +}: Options) { + client ??= conat(); + + // - [ ] get info about the runner's status (use a stream?) -- write that here. + // - [ ] connect to database to get quota for running a project -- via a function that is passed in + // - [ ] it will contact runner to run project -- write that here. + const { projects, runners } = await state({ client }); + + const getClient = async (project_id: string) => { + const cutoff = Date.now() - UPDATE_INTERVAL * 2.5; + + const cur = projects.get(project_id); + if (cur != null && cur.state != "opened") { + const { server } = cur; + if (server) { + const s = runners.get(server); + if ((s?.time ?? 0) > cutoff) { + return projectRunnerClient({ + client, + subject: `project-runner.${server}`, + }); + } + } + } + const v: { time: number; server: string }[] = []; + const k = runners.getAll(); + for (const server in k) { + if ((k[server].time ?? 0) <= cutoff) { + continue; + } + v.push({ ...k[server], server }); + } + v.sort(field_cmp("time")); + v.reverse(); + + if (v.length == 0) { + throw Error("no project runners available -- try again later"); + } + // this is a very dumb first attempt; it should also try another server in case a server isn't reachable + const server = randomChoice(new Set(v)).server; + logger.debug("getClient -- assigning to ", { project_id, server }); + return projectRunnerClient({ + client, + subject: `project-runner.${server}`, + }); + }; + + const getProjectId = (t: any) => { + const subject = t.subject as string; + const project_id = subject.split(".")[1]; + return project_id; + }; + + const setState1 = + setState == null + ? undefined + : async (opts: { project_id: string; state: ProjectState }) => { + if (setState == null) { + return; + } + try { + await setState(opts); + } catch (err) { + logger.debug(`WARNING: issue calling setState`, opts, err); + } + }; + + const sub = await client.service(subject, { + async start() { + const project_id = getProjectId(this); + const config = await getConfig?.({ project_id }); + const cur = projects.get(project_id); + if (cur?.state == "starting" || cur?.state == "running") { + return; + } + const runClient = await getClient(project_id); + await setState1?.({ project_id, state: "starting" }); + await runClient.start({ project_id, config }); + await setState1?.({ project_id, state: "running" }); + }, + + async stop() { + const project_id = getProjectId(this); + const runClient = await getClient(project_id); + try { + await runClient.stop({ project_id }); + await setState1?.({ project_id, state: "opened" }); + } catch (err) { + if (err.code == 503) { + // the runner is no longer running, so obviously project isn't running there. + await setState1?.({ project_id, state: "opened" }); + } + throw err; + } + }, + + async status() { + const project_id = getProjectId(this); + const runClient = await getClient(project_id); + for (let i = 0; i < MAX_STATUS_TRIES; i++) { + try { + logger.debug("status", { project_id }); + const s = await runClient.status({ project_id }); + logger.debug("status: got ", s); + await setState1?.({ project_id, ...s }); + return s; + } catch (err) { + logger.debug("status: got err", err); + if (i < MAX_STATUS_TRIES - 1) { + logger.debug("status: waiting 3s and trying again..."); + await delay(3000); + continue; + } + if (err.code == 503) { + logger.debug( + "status: running is no longer running -- giving up on project", + ); + // the runner is no longer running, so obviously project isn't running there. + await setState1?.({ project_id, state: "opened" }); + } + logger.debug("status: reporting error"); + throw err; + } + } + logger.debug("status: bug"); + throw Error("bug"); + }, + }); + + return { + close: () => { + sub.close(); + }, + }; +} + +export function client({ + client, + subject, +}: { + client?: Client; + subject: string; +}): API { + client ??= conat(); + return client.call(subject); +} diff --git a/src/packages/conat/project/runner/run.ts b/src/packages/conat/project/runner/run.ts new file mode 100644 index 00000000000..36d20cf971d --- /dev/null +++ b/src/packages/conat/project/runner/run.ts @@ -0,0 +1,94 @@ +/* +Service to run a CoCalc project. + +Tests are in + + - packages/backend/conat/test/project + +*/ + +import { type Client } from "@cocalc/conat/core/client"; +import { conat } from "@cocalc/conat/client"; +import { randomId } from "@cocalc/conat/names"; +import state, { type ProjectStatus } from "./state"; +import { until } from "@cocalc/util/async-utils"; + +export const UPDATE_INTERVAL = 10_000; + +export interface Options { + id?: string; + client?: Client; + start: (opts: { project_id: string; config?: any }) => Promise; + stop: (opts: { project_id: string }) => Promise; + status: (opts: { project_id: string }) => Promise; +} + +export interface API { + start: (opts: { project_id: string; config?: any }) => Promise; + stop: (opts: { project_id: string }) => Promise; + status: (opts: { project_id: string }) => Promise; +} + +export async function server({ + id = randomId(), + client, + start, + stop, + status, +}: Options) { + client ??= conat(); + const { projects, runners } = await state({ client }); + let running = true; + + until( + () => { + if (!running) { + return true; + } + runners.set(id, { time: Date.now() }); + return false; + }, + { min: UPDATE_INTERVAL, max: UPDATE_INTERVAL }, + ); + + const sub = await client.service(`project-runner.${id}`, { + async start(opts: { project_id: string; config?: any }) { + projects.set(opts.project_id, { server: id, state: "starting" } as const); + await start(opts); + const s = { server: id, state: "running" } as const; + projects.set(opts.project_id, s); + return s; + }, + async stop(opts: { project_id: string }) { + projects.set(opts.project_id, { server: id, state: "stopping" } as const); + await stop(opts); + const s = { server: id, state: "opened" } as const; + projects.set(opts.project_id, s); + return s; + }, + async status(opts: { project_id: string }) { + const s = { ...(await status(opts)), server: id }; + projects.set(opts.project_id, s); + return s; + }, + }); + + return { + close: () => { + running = false; + runners.delete(id); + sub.close(); + }, + }; +} + +export function client({ + client, + subject, +}: { + client?: Client; + subject: string; +}): API { + client ??= conat(); + return client.call(subject, { waitForInterest: true }); +} diff --git a/src/packages/conat/project/runner/state.ts b/src/packages/conat/project/runner/state.ts new file mode 100644 index 00000000000..aa677748c66 --- /dev/null +++ b/src/packages/conat/project/runner/state.ts @@ -0,0 +1,27 @@ +// get the shared state used by the load balancer and all the project runners + +import { dkv } from "@cocalc/conat/sync/dkv"; + +export interface RunnerStatus { + time: number; +} + +export type ProjectState = "running" | "opened" | "stopping" | "starting"; +export interface ProjectStatus { + server?: string; + state: ProjectState; + ip?: string; // the ip address when running +} + +export default async function state({ client }) { + return { + projects: await dkv({ + client, + name: "project-runner.projects", + }), + runners: await dkv({ + client, + name: "project-runner.runners", + }), + }; +} diff --git a/src/packages/conat/service/formatter.ts b/src/packages/conat/service/formatter.ts deleted file mode 100644 index 4fa5b5a4906..00000000000 --- a/src/packages/conat/service/formatter.ts +++ /dev/null @@ -1,46 +0,0 @@ -/* -Formatting services in a project. -*/ - -import { createServiceClient, createServiceHandler } from "./typed"; - -import type { - Options as FormatterOptions, - FormatResult, -} from "@cocalc/util/code-formatter"; - -// TODO: we may change it to NOT take compute server and have this listening from -// project and all compute servers... and have only the one with the file open -// actually reply. -interface FormatterApi { - formatter: (opts: { - path: string; - options: FormatterOptions; - }) => Promise; -} - -export function formatterClient({ compute_server_id = 0, project_id }) { - return createServiceClient({ - project_id, - compute_server_id, - service: "formatter", - }); -} - -export async function createFormatterService({ - compute_server_id = 0, - project_id, - impl, -}: { - project_id: string; - compute_server_id?: number; - impl: FormatterApi; -}) { - return await createServiceHandler({ - project_id, - compute_server_id, - service: "formatter", - description: "Code formatter API", - impl, - }); -} diff --git a/src/packages/conat/socket/base.ts b/src/packages/conat/socket/base.ts index 24d57ac1484..3f98df515f2 100644 --- a/src/packages/conat/socket/base.ts +++ b/src/packages/conat/socket/base.ts @@ -116,14 +116,16 @@ export abstract class ConatSocketBase extends EventEmitter { } if (this.reconnection) { setTimeout(() => { - this.connect(); + if (this.state != "closed") { + this.connect(); + } }, RECONNECT_DELAY); } }; connect = async () => { - if (this.state != "disconnected") { - // already connected + if (this.state != "disconnected" || !this.client) { + // already connected or closed return; } this.setState("connecting"); diff --git a/src/packages/conat/socket/client.ts b/src/packages/conat/socket/client.ts index bb107bdaeb3..caa1a45fb93 100644 --- a/src/packages/conat/socket/client.ts +++ b/src/packages/conat/socket/client.ts @@ -91,8 +91,8 @@ export class ConatSocketClient extends ConatSocketBase { }); } - waitUntilDrain = async () => { - await this.tcp?.send.waitUntilDrain(); + drain = async () => { + await this.tcp?.send.drain(); }; private sendCommandToServer = async ( @@ -225,7 +225,7 @@ export class ConatSocketClient extends ConatSocketBase { if (this.state == "closed") { throw Error("closed"); } - // console.log("sending request from client ", { subject, data, options }); + //console.log("sending request from client ", { subject, data, options }); return await this.client.request(subject, data, options); }; diff --git a/src/packages/conat/socket/server-socket.ts b/src/packages/conat/socket/server-socket.ts index edfdc225a11..0b678b05dbc 100644 --- a/src/packages/conat/socket/server-socket.ts +++ b/src/packages/conat/socket/server-socket.ts @@ -51,6 +51,7 @@ export class ServerSocket extends EventEmitter { this.initKeepAlive(); } + private firstPing = true; private initKeepAlive = () => { this.alive?.close(); this.alive = keepAlive({ @@ -59,10 +60,15 @@ export class ServerSocket extends EventEmitter { await this.request(null, { headers: { [SOCKET_HEADER_CMD]: "ping" }, timeout: this.conatSocket.keepAliveTimeout, - // waitForInterest is very important in a cluster -- also, obviously + // waitForInterest for the *first ping* is very important + // in a cluster -- also, obviously // if somebody just opened a socket, they probably exist. - waitForInterest: true, + // However, after the first ping, we want to fail + // very quickly if the client disappears (and hence no + // more interest). + waitForInterest: this.firstPing, }); + this.firstPing = false; }, disconnect: this.close, keepAlive: this.conatSocket.keepAlive, @@ -232,7 +238,7 @@ export class ServerSocket extends EventEmitter { } }); - waitUntilDrain = async () => { - await this.tcp?.send.waitUntilDrain(); + drain = async () => { + await this.tcp?.send.drain(); }; } diff --git a/src/packages/conat/socket/tcp.ts b/src/packages/conat/socket/tcp.ts index 7f0e2b49948..da55e76a6ac 100644 --- a/src/packages/conat/socket/tcp.ts +++ b/src/packages/conat/socket/tcp.ts @@ -275,7 +275,7 @@ export class Sender extends EventEmitter { } }; - waitUntilDrain = reuseInFlight(async () => { + drain = reuseInFlight(async () => { if (this.unsent == 0) { return; } diff --git a/src/packages/conat/socket/util.ts b/src/packages/conat/socket/util.ts index 9e81f08439f..5274ba12068 100644 --- a/src/packages/conat/socket/util.ts +++ b/src/packages/conat/socket/util.ts @@ -13,15 +13,16 @@ export type Role = "client" | "server"; // socketio and use those to manage things. This ping // is entirely a "just in case" backup if some event // were missed (e.g., a kill -9'd process...) -export const PING_PONG_INTERVAL = 90000; +export const PING_PONG_INTERVAL = 90_000; // We queue up unsent writes, but only up to a point (to not have a huge memory issue). // Any write beyond this size result in an exception. // NOTE: in nodejs the default for exactly this is "infinite=use up all RAM", so // maybe we should make this even larger (?). // Also note that this is just the *number* of messages, and a message can have -// any size. -export const DEFAULT_MAX_QUEUE_SIZE = 1000; +// any size. But determining message size is very difficult without serializing the +// message, which costs. +export const DEFAULT_MAX_QUEUE_SIZE = 1_000; export let DEFAULT_COMMAND_TIMEOUT = 10_000; export let DEFAULT_KEEP_ALIVE = 25_000; diff --git a/src/packages/conat/sync-doc/sync-client.ts b/src/packages/conat/sync-doc/sync-client.ts new file mode 100644 index 00000000000..6c284b24218 --- /dev/null +++ b/src/packages/conat/sync-doc/sync-client.ts @@ -0,0 +1,110 @@ +/* + * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. + * License: MS-RSL – see LICENSE.md for details + */ + +import { EventEmitter } from "events"; +import { Client as Client0 } from "@cocalc/sync/editor/generic/types"; +import { parseQueryWithOptions } from "@cocalc/sync/table/util"; +import { PubSub } from "@cocalc/conat/sync/pubsub"; +import { type Client as ConatClient } from "@cocalc/conat/core/client"; +import { type ConatSyncTable } from "@cocalc/conat/sync/synctable"; + +export class SyncClient extends EventEmitter implements Client0 { + private client: ConatClient; + constructor(client: ConatClient) { + super(); + if (client == null) { + throw Error("client must be specified"); + } + this.client = client; + this.client.once("closed", this.close); + } + + close = () => { + this.emit("closed"); + // @ts-ignore + delete this.client; + }; + + is_project = (): boolean => false; + is_browser = (): boolean => true; + is_compute_server = (): boolean => false; + + dbg = (_f: string) => { + return (..._) => {}; + }; + + is_connected = (): boolean => { + return this.client.isConnected(); + }; + + is_signed_in = (): boolean => { + return this.client.isSignedIn(); + }; + + touch_project = (_): void => {}; + + is_deleted = (_filename: string, _project_id?: string): boolean => { + return false; + }; + + set_deleted = (_filename: string, _project_id?: string): void => {}; + + synctable_conat = async (query0, options?): Promise => { + const { query } = parseQueryWithOptions(query0, options); + return await this.client.sync.synctable({ + ...options, + query, + }); + }; + + pubsub_conat = async (opts): Promise => { + return new PubSub({ client: this.client, ...opts }); + }; + + // account_id or project_id or hub_id or fallback client.id + client_id = (): string => { + const user = this.client.info?.user; + return ( + user?.account_id ?? user?.project_id ?? user?.hub_id ?? this.client.id + ); + }; + + server_time = (): Date => { + return new Date(); + }; + + ///////////////////////////////// + // EVERYTHING BELOW: TO REMOVE? + mark_file = (_): void => {}; + + alert_message = (_): void => {}; + + sage_session = (_): void => {}; + + shell = (_): void => {}; + + path_access = (opts): void => { + opts.cb(true); + }; + path_stat = (opts): void => { + console.log("path_state", opts.path); + opts.cb(true); + }; + + async path_read(opts): Promise { + opts.cb(true); + } + async write_file(opts): Promise { + opts.cb(true); + } + watch_file(_): any {} + + log_error = (_): void => {}; + + query = (_): void => { + throw Error("not implemented"); + }; + query_cancel = (_): void => {}; +} diff --git a/src/packages/conat/sync-doc/syncdb.ts b/src/packages/conat/sync-doc/syncdb.ts new file mode 100644 index 00000000000..4211ad84775 --- /dev/null +++ b/src/packages/conat/sync-doc/syncdb.ts @@ -0,0 +1,21 @@ +import { SyncClient } from "./sync-client"; +import { SyncDB, type SyncDBOpts0 } from "@cocalc/sync/editor/db"; +import { type Client as ConatClient } from "@cocalc/conat/core/client"; + +export type MakeOptional = Omit & + Partial>; + +export interface SyncDBOptions + extends MakeOptional, "fs"> { + client: ConatClient; + // name of the file service that hosts this file: + service?: string; +} + +export type { SyncDB }; + +export function syncdb({ client, service, ...opts }: SyncDBOptions): SyncDB { + const fs = opts.fs ?? client.fs({ service, project_id: opts.project_id }); + const syncClient = new SyncClient(client); + return new SyncDB({ ...opts, fs, client: syncClient }); +} diff --git a/src/packages/conat/sync-doc/syncstring.ts b/src/packages/conat/sync-doc/syncstring.ts new file mode 100644 index 00000000000..322a2621edb --- /dev/null +++ b/src/packages/conat/sync-doc/syncstring.ts @@ -0,0 +1,26 @@ +import { SyncClient } from "./sync-client"; +import { + SyncString, + type SyncStringOpts, +} from "@cocalc/sync/editor/string/sync"; +import { type Client as ConatClient } from "@cocalc/conat/core/client"; +import { type MakeOptional } from "./syncdb"; + +export interface SyncStringOptions + extends MakeOptional, "fs"> { + client: ConatClient; + // name of the file server that hosts this document: + service?: string; +} + +export type { SyncString }; + +export function syncstring({ + client, + service, + ...opts +}: SyncStringOptions): SyncString { + const fs = opts.fs ?? client.fs({ service, project_id: opts.project_id }); + const syncClient = new SyncClient(client); + return new SyncString({ ...opts, fs, client: syncClient }); +} diff --git a/src/packages/conat/sync/dko.ts b/src/packages/conat/sync/dko.ts index f4e773a90fa..c0c04c1b065 100644 --- a/src/packages/conat/sync/dko.ts +++ b/src/packages/conat/sync/dko.ts @@ -1,7 +1,7 @@ /* Distributed eventually consistent key:object store, where changes propogate sparsely. -The "values" MUST be objects and no keys or fields of objects can container the +The "values" MUST be objects and no keys or fields of objects can container the sep character, which is '|' by default. NOTE: Whenever you do a set, the lodash isEqual function is used to see which fields @@ -38,6 +38,7 @@ export class DKO extends EventEmitter { constructor(private opts: DKVOptions) { super(); + this.setMaxListeners(1000); return new Proxy(this, { deleteProperty(target, prop) { if (typeof prop == "string") { diff --git a/src/packages/conat/sync/dstream.ts b/src/packages/conat/sync/dstream.ts index 4bc6121c261..26d0c997f9d 100644 --- a/src/packages/conat/sync/dstream.ts +++ b/src/packages/conat/sync/dstream.ts @@ -290,6 +290,7 @@ export class DStream extends EventEmitter { }; save = reuseInFlight(async () => { + //console.log("save", this.noAutosave); await until( async () => { if (this.isClosed()) { diff --git a/src/packages/conat/sync/open-files.ts b/src/packages/conat/sync/open-files.ts deleted file mode 100644 index b82afd85786..00000000000 --- a/src/packages/conat/sync/open-files.ts +++ /dev/null @@ -1,302 +0,0 @@ -/* -Keep track of open files. - -We use the "dko" distributed key:value store because of the potential of merge -conflicts, e.g,. one client changes the compute server id and another changes -whether a file is deleted. By using dko, only the field that changed is sync'd -out, so we get last-write-wins on the level of fields. - -WARNINGS: -An old version use dkv with merge conflict resolution, but with multiple clients -and the project, feedback loops or something happened and it would start getting -slow -- basically, merge conflicts could take a few seconds to resolve, which would -make opening a file start to be slow. Instead we use DKO data type, where fields -are treated separately atomically by the storage system. A *subtle issue* is -that when you set an object, this is NOT treated atomically. E.g., if you -set 2 fields in a set operation, then 2 distinct changes are emitted as the -two fields get set. - -DEVELOPMENT: - -Change to packages/backend, since packages/conat doesn't have a way to connect: - -~/cocalc/src/packages/backend$ node - -> z = await require('@cocalc/backend/conat/sync').openFiles({project_id:cc.current().project_id}) -> z.touch({path:'a.txt'}) -> z.get({path:'a.txt'}) -{ open: true, count: 1, time:2025-02-09T16:37:20.713Z } -> z.touch({path:'a.txt'}) -> z.get({path:'a.txt'}) -{ open: true, count: 2 } -> z.time({path:'a.txt'}) -2025-02-09T16:36:58.510Z -> z.touch({path:'foo/b.md',id:0}) -> z.getAll() -{ - 'a.txt': { open: true, count: 3 }, - 'foo/b.md': { open: true, count: 1 } - -Frontend Dev in browser: - -z = await cc.client.conat_client.openFiles({project_id:cc.current().project_id)) -z.getAll() -} -*/ - -import { type State } from "@cocalc/conat/types"; -import { dko, type DKO } from "@cocalc/conat/sync/dko"; -import { EventEmitter } from "events"; -import getTime, { getSkew } from "@cocalc/conat/time"; - -// info about interest in open files (and also what was explicitly deleted) older -// than this is automatically purged. -const MAX_AGE_MS = 1000 * 60 * 60 * 24; - -interface Deleted { - // what deleted state is - deleted: boolean; - // when deleted state set - time: number; -} - -interface Backend { - // who has it opened -- the compute_server_id (0 for project) - id: number; - // when they last reported having it opened - time: number; -} - -export interface KVEntry { - // a web browser has the file open at this point in time (in ms) - time?: number; - // if the file was removed from disk (and not immmediately written back), - // then deleted gets set to the time when this happened (in ms since epoch) - // and the file is closed on the backend. It won't be re-opened until - // either (1) the file is created on disk again, or (2) deleted is cleared. - // Note: the actual time here isn't really important -- what matter is the number - // is nonzero. It's just used for a display to the user. - // We store the deleted state *and* when this was set, so that in case - // of merge conflict we can do something sensible. - deleted?: Deleted; - - // if file is actively opened on a compute server/project, then it sets - // this entry. Right when it closes the file, it clears this. - // If it gets killed/broken and doesn't have a chance to clear it, then - // backend.time can be used to decide this isn't valid. - backend?: Backend; - - // optional information - doctype?; -} - -export interface Entry extends KVEntry { - // path to file relative to HOME - path: string; -} - -interface Options { - project_id: string; - noAutosave?: boolean; - noCache?: boolean; -} - -export async function createOpenFiles(opts: Options) { - const openFiles = new OpenFiles(opts); - await openFiles.init(); - return openFiles; -} - -export class OpenFiles extends EventEmitter { - private project_id: string; - private noCache?: boolean; - private noAutosave?: boolean; - private kv?: DKO; - public state: "disconnected" | "connected" | "closed" = "disconnected"; - - constructor({ project_id, noAutosave, noCache }: Options) { - super(); - if (!project_id) { - throw Error("project_id must be specified"); - } - this.project_id = project_id; - this.noAutosave = noAutosave; - this.noCache = noCache; - } - - private setState = (state: State) => { - this.state = state; - this.emit(state); - }; - - private initialized = false; - init = async () => { - if (this.initialized) { - throw Error("init can only be called once"); - } - this.initialized = true; - const d = await dko({ - name: "open-files", - project_id: this.project_id, - config: { - max_age: MAX_AGE_MS, - }, - noAutosave: this.noAutosave, - noCache: this.noCache, - noInventory: true, - }); - this.kv = d; - d.on("change", this.handleChange); - // ensure clock is synchronized - await getSkew(); - this.setState("connected"); - }; - - private handleChange = ({ key: path }) => { - const entry = this.get(path); - if (entry != null) { - // not deleted and timestamp is set: - this.emit("change", entry as Entry); - } - }; - - close = () => { - if (this.kv == null) { - return; - } - this.setState("closed"); - this.removeAllListeners(); - this.kv.removeListener("change", this.handleChange); - this.kv.close(); - delete this.kv; - // @ts-ignore - delete this.project_id; - }; - - private getKv = () => { - const { kv } = this; - if (kv == null) { - throw Error("closed"); - } - return kv; - }; - - private set = (path, entry: KVEntry) => { - this.getKv().set(path, entry); - }; - - // When a client has a file open, they should periodically - // touch it to indicate that it is open. - // updates timestamp and ensures open=true. - touch = (path: string, doctype?) => { - if (!path) { - throw Error("path must be specified"); - } - const kv = this.getKv(); - const cur = kv.get(path); - const time = getTime(); - if (doctype) { - this.set(path, { - ...cur, - time, - doctype, - }); - } else { - this.set(path, { - ...cur, - time, - }); - } - }; - - setError = (path: string, err?: any) => { - const kv = this.getKv(); - if (!err) { - const current = { ...kv.get(path) }; - delete current.error; - this.set(path, current); - } else { - const current = { ...kv.get(path) }; - current.error = { time: Date.now(), error: `${err}` }; - this.set(path, current); - } - }; - - setDeleted = (path: string) => { - const kv = this.getKv(); - this.set(path, { - ...kv.get(path), - deleted: { deleted: true, time: getTime() }, - }); - }; - - isDeleted = (path: string) => { - return !!this.getKv().get(path)?.deleted?.deleted; - }; - - setNotDeleted = (path: string) => { - const kv = this.getKv(); - this.set(path, { - ...kv.get(path), - deleted: { deleted: false, time: getTime() }, - }); - }; - - // set that id is the backend with the file open. - // This should be called by that backend periodically - // when it has the file opened. - setBackend = (path: string, id: number) => { - const kv = this.getKv(); - this.set(path, { - ...kv.get(path), - backend: { id, time: getTime() }, - }); - }; - - // get current backend that has file opened. - getBackend = (path: string): Backend | undefined => { - return this.getKv().get(path)?.backend; - }; - - // ONLY if backend for path is currently set to id, then clear - // the backend field. - setNotBackend = (path: string, id: number) => { - const kv = this.getKv(); - const cur = { ...kv.get(path) }; - if (cur?.backend?.id == id) { - delete cur.backend; - this.set(path, cur); - } - }; - - getAll = (): Entry[] => { - const x = this.getKv().getAll(); - return Object.keys(x).map((path) => { - return { ...x[path], path }; - }); - }; - - get = (path: string): Entry | undefined => { - const x = this.getKv().get(path); - if (x == null) { - return x; - } - return { ...x, path }; - }; - - delete = (path) => { - this.getKv().delete(path); - }; - - clear = () => { - this.getKv().clear(); - }; - - save = async () => { - await this.getKv().save(); - }; - - hasUnsavedChanges = () => { - return this.getKv().hasUnsavedChanges(); - }; -} diff --git a/src/packages/conat/sync/synctable-kv.ts b/src/packages/conat/sync/synctable-kv.ts index 7950305ed44..9e56393fff5 100644 --- a/src/packages/conat/sync/synctable-kv.ts +++ b/src/packages/conat/sync/synctable-kv.ts @@ -33,6 +33,7 @@ export class SyncTableKV extends EventEmitter { private config?: Partial; private desc?: JSONValue; private ephemeral?: boolean; + private noAutosave?: boolean; constructor({ query, @@ -44,6 +45,7 @@ export class SyncTableKV extends EventEmitter { config, desc, ephemeral, + noAutosave, }: { query; client: Client; @@ -54,6 +56,7 @@ export class SyncTableKV extends EventEmitter { config?: Partial; desc?: JSONValue; ephemeral?: boolean; + noAutosave?: boolean; }) { super(); this.setMaxListeners(1000); @@ -64,6 +67,7 @@ export class SyncTableKV extends EventEmitter { this.client = client; this.desc = desc; this.ephemeral = ephemeral; + this.noAutosave = noAutosave; this.table = keys(query)[0]; if (query[this.table][0].string_id && query[this.table][0].project_id) { this.project_id = query[this.table][0].project_id; @@ -126,6 +130,7 @@ export class SyncTableKV extends EventEmitter { config: this.config, desc: this.desc, ephemeral: this.ephemeral, + noAutosave: this.noAutosave, }); } else { this.dkv = await createDko({ @@ -136,6 +141,7 @@ export class SyncTableKV extends EventEmitter { config: this.config, desc: this.desc, ephemeral: this.ephemeral, + noAutosave: this.noAutosave, }); } // For some reason this one line confuses typescript and break building the compute server package (nothing else similar happens). diff --git a/src/packages/conat/sync/synctable-stream.ts b/src/packages/conat/sync/synctable-stream.ts index 36b1b3e27eb..597cfc8d953 100644 --- a/src/packages/conat/sync/synctable-stream.ts +++ b/src/packages/conat/sync/synctable-stream.ts @@ -45,6 +45,7 @@ export class SyncTableStream extends EventEmitter { private config?: Partial; private start_seq?: number; private noInventory?: boolean; + private noAutosave?: boolean; private ephemeral?: boolean; constructor({ @@ -57,6 +58,7 @@ export class SyncTableStream extends EventEmitter { start_seq, noInventory, ephemeral, + noAutosave, }: { query; client: Client; @@ -67,10 +69,12 @@ export class SyncTableStream extends EventEmitter { start_seq?: number; noInventory?: boolean; ephemeral?: boolean; + noAutosave?: boolean; }) { super(); this.client = client; this.noInventory = noInventory; + this.noAutosave = noAutosave; this.ephemeral = ephemeral; this.setMaxListeners(1000); this.getHook = immutable ? fromJS : (x) => x; @@ -107,6 +111,7 @@ export class SyncTableStream extends EventEmitter { start_seq: this.start_seq, noInventory: this.noInventory, ephemeral: this.ephemeral, + noAutosave: this.noAutosave, }); this.dstream.on("change", (mesg) => { this.handle(mesg, true); diff --git a/src/packages/conat/sync/synctable.ts b/src/packages/conat/sync/synctable.ts index 8f69000eee9..95048b86d2f 100644 --- a/src/packages/conat/sync/synctable.ts +++ b/src/packages/conat/sync/synctable.ts @@ -43,6 +43,7 @@ export interface SyncTableOptions { start_seq?: number; noInventory?: boolean; ephemeral?: boolean; + noAutosave?: boolean; } export const createSyncTable = refCache({ diff --git a/src/packages/conat/tsconfig.json b/src/packages/conat/tsconfig.json index 687201523d0..5a8dd8655a1 100644 --- a/src/packages/conat/tsconfig.json +++ b/src/packages/conat/tsconfig.json @@ -6,5 +6,5 @@ }, "exclude": ["node_modules", "dist", "test"], "references_comment": "Do not define path:../comm because that causes a circular references.", - "references": [{ "path": "../util" }] + "references": [{ "path": "../util", "path": "../sync" }] } diff --git a/src/packages/database/postgres-server-queries.coffee b/src/packages/database/postgres-server-queries.coffee index 423fedbc844..e3f0eccb421 100644 --- a/src/packages/database/postgres-server-queries.coffee +++ b/src/packages/database/postgres-server-queries.coffee @@ -1517,7 +1517,7 @@ exports.extend_PostgreSQL = (ext) -> class PostgreSQL extends ext opts = defaults opts, project_id : required path : required - listing : required # files in path [{name:..., isdir:boolean, ....}, ...] + listing : required # files in path [{name:..., isDir:boolean, ....}, ...] cb : required # Get all public paths for the given project_id, then check if path is "in" one according # to the definition in misc. diff --git a/src/packages/file-server/btrfs/filesystem.ts b/src/packages/file-server/btrfs/filesystem.ts index 927fb23548f..25e258530c8 100644 --- a/src/packages/file-server/btrfs/filesystem.ts +++ b/src/packages/file-server/btrfs/filesystem.ts @@ -7,57 +7,44 @@ Start node, then: DEBUG="cocalc:*file-server*" DEBUG_CONSOLE=yes node -a = require('@cocalc/file-server/btrfs'); fs = await a.filesystem({device:'/tmp/btrfs.img', formatIfNeeded:true, mount:'/mnt/btrfs', uid:293597964}) +a = require('@cocalc/file-server/btrfs'); fs = await a.filesystem({image:'/tmp/btrfs.img', mount:'/mnt/btrfs', size:'2G'}) */ import refCache from "@cocalc/util/refcache"; -import { mkdirp, btrfs, sudo } from "./util"; -import { join } from "path"; +import { mkdirp, btrfs, sudo, ensureMoreLoopbackDevices } from "./util"; import { Subvolumes } from "./subvolumes"; import { mkdir } from "fs/promises"; import { exists } from "@cocalc/backend/misc/async-utils-node"; -import { executeCode } from "@cocalc/backend/execute-code"; - -// default size of btrfs filesystem if creating an image file. -const DEFAULT_FILESYSTEM_SIZE = "10G"; - -// default for newly created subvolumes -export const DEFAULT_SUBVOLUME_SIZE = "1G"; - -const MOUNT_ERROR = "wrong fs type, bad option, bad superblock"; +import rustic from "@cocalc/backend/sandbox/rustic"; export interface Options { - // the underlying block device. - // If this is a file (or filename) ending in .img, then it's a sparse file mounted as a loopback device. - // If this starts with "/dev" then it is a raw block device. - device: string; - // formatIfNeeded -- DANGEROUS! if true, format the device or image, - // if it doesn't mount with an error containing "wrong fs type, - // bad option, bad superblock". Never use this in production. Useful - // for testing and dev. - formatIfNeeded?: boolean; - // where the btrfs filesystem is mounted + // mount = root mountpoint of the btrfs filesystem. If you specify the image + // path below, then a btrfs filesystem will get automatically created (via sudo + // and a loopback device). mount: string; - // default size of newly created subvolumes - defaultSize?: string | number; - defaultFilesystemSize?: string | number; + // image = optioanlly use a image file at this location for the btrfs filesystem. + // This is used for development and in Docker. It will be created as a sparse image file + // with given size, and mounted at opts.mount if it does not exist. If you create + // it be sure to use mkfs.btrfs to format it. + image?: string; + size?: string | number; + + // rustic = the rustic backups path. + // If this path ends in .toml, it is the configuration file for rustic, e.g., you can + // configure rustic however you want by pointing this at a toml cofig file. + // Otherwise, if this path does not exist, it will be created a new rustic repo + // initialized here. + rustic: string; } export class Filesystem { public readonly opts: Options; - public readonly bup: string; public readonly subvolumes: Subvolumes; constructor(opts: Options) { - opts = { - defaultSize: DEFAULT_SUBVOLUME_SIZE, - defaultFilesystemSize: DEFAULT_FILESYSTEM_SIZE, - ...opts, - }; this.opts = opts; - this.bup = join(this.opts.mount, "bup"); this.subvolumes = new Subvolumes(this); } @@ -68,7 +55,12 @@ export class Filesystem { await btrfs({ args: ["quota", "enable", "--simple", this.opts.mount], }); - await this.initBup(); + await this.initRustic(); + await this.sync(); + }; + + sync = async () => { + await btrfs({ args: ["filesystem", "sync", this.opts.mount] }); }; unmount = async () => { @@ -82,15 +74,16 @@ export class Filesystem { close = () => {}; private initDevice = async () => { - if (!isImageFile(this.opts.device)) { - // raw block device -- nothing to do + if (!this.opts.image) { return; } - if (!(await exists(this.opts.device))) { + if (!(await exists(this.opts.image))) { + // we create and format the sparse image await sudo({ command: "truncate", - args: ["-s", `${this.opts.defaultFilesystemSize}`, this.opts.device], + args: ["-s", `${this.opts.size ?? "10G"}`, this.opts.image], }); + await sudo({ command: "mkfs.btrfs", args: [this.opts.image] }); } }; @@ -115,27 +108,15 @@ export class Filesystem { } catch {} const { stderr, exit_code } = await this._mountFilesystem(); if (exit_code) { - if (stderr.includes(MOUNT_ERROR)) { - if (this.opts.formatIfNeeded) { - await this.formatDevice(); - const { stderr, exit_code } = await this._mountFilesystem(); - if (exit_code) { - throw Error(stderr); - } else { - return; - } - } - } throw Error(stderr); } }; - private formatDevice = async () => { - await sudo({ command: "mkfs.btrfs", args: [this.opts.device] }); - }; - private _mountFilesystem = async () => { - const args: string[] = isImageFile(this.opts.device) ? ["-o", "loop"] : []; + if (!this.opts.image) { + throw Error(`there must be a btrfs filesystem at ${this.opts.mount}`); + } + const args: string[] = ["-o", "loop"]; args.push( "-o", "compress=zstd", @@ -145,19 +126,28 @@ export class Filesystem { "space_cache=v2", "-o", "autodefrag", - this.opts.device, + this.opts.image, "-t", "btrfs", this.opts.mount, ); { - const { stderr, exit_code } = await sudo({ + const { exit_code: failed } = await sudo({ command: "mount", args, err_on_exit: false, }); - if (exit_code) { - return { stderr, exit_code }; + if (failed) { + // try again with more loopback devices + await ensureMoreLoopbackDevices(); + const { stderr, exit_code } = await sudo({ + command: "mount", + args, + err_on_exit: false, + }); + if (exit_code) { + return { stderr, exit_code }; + } } } const { stderr, exit_code } = await sudo({ @@ -171,26 +161,21 @@ export class Filesystem { return { stderr, exit_code }; }; - private initBup = async () => { - if (!(await exists(this.bup))) { - await mkdir(this.bup); + private initRustic = async () => { + if (!this.opts.rustic) { + throw Error("rustic repo path or toml must be specified"); } - await executeCode({ - command: "bup", - args: ["init"], - env: { BUP_DIR: this.bup }, - }); + if (!this.opts.rustic || (await exists(this.opts.rustic))) { + return; + } + if (this.opts.rustic.endsWith(".toml")) { + throw Error(`file not found: ${this.opts.rustic}`); + } + await mkdir(this.opts.rustic); + await rustic(["init"], { repo: this.opts.rustic }); }; } -function isImageFile(name: string) { - if (name.startsWith("/dev")) { - return false; - } - // TODO: could probably check os for a device with given name? - return name.endsWith(".img"); -} - const cache = refCache({ name: "btrfs-filesystems", createObject: async (options: Options) => { diff --git a/src/packages/file-server/btrfs/index.ts b/src/packages/file-server/btrfs/index.ts index edfd27c3a9b..56cf377d20c 100644 --- a/src/packages/file-server/btrfs/index.ts +++ b/src/packages/file-server/btrfs/index.ts @@ -1 +1,2 @@ export { filesystem } from "./filesystem"; +export { type Filesystem } from "./filesystem"; diff --git a/src/packages/file-server/btrfs/snapshots.ts b/src/packages/file-server/btrfs/snapshots.ts index daf8e09212c..99d0856245f 100644 --- a/src/packages/file-server/btrfs/snapshots.ts +++ b/src/packages/file-server/btrfs/snapshots.ts @@ -50,9 +50,9 @@ export async function updateRollingSnapshots({ } // get exactly the iso timestamp snapshot names: - const snapshotNames = (await snapshots.ls()) - .map((x) => x.name) - .filter((name) => DATE_REGEXP.test(name)); + const snapshotNames = (await snapshots.readdir()).filter((name) => + DATE_REGEXP.test(name), + ); snapshotNames.sort(); if (snapshotNames.length > 0) { const age = Date.now() - new Date(snapshotNames.slice(-1)[0]).valueOf(); diff --git a/src/packages/file-server/btrfs/subvolume-bup.ts b/src/packages/file-server/btrfs/subvolume-bup.ts deleted file mode 100644 index 21cbbff3646..00000000000 --- a/src/packages/file-server/btrfs/subvolume-bup.ts +++ /dev/null @@ -1,192 +0,0 @@ -/* - -BUP Architecture: - -There is a single global dedup'd backup archive stored in the btrfs filesystem. -Obviously, admins should rsync this regularly to a separate location as a genuine -backup strategy. - -NOTE: we use bup instead of btrfs send/recv ! - -Not used. Instead we will rely on bup (and snapshots of the underlying disk) for backups, since: - - much easier to check they are valid - - decoupled from any btrfs issues - - not tied to any specific filesystem at all - - easier to offsite via incremental rsync - - much more space efficient with *global* dedup and compression - - bup is really just git, which is much more proven than even btrfs - -The drawback is speed, but that can be managed. -*/ - -import { type DirectoryListingEntry } from "@cocalc/util/types"; -import { type Subvolume } from "./subvolume"; -import { sudo, parseBupTime } from "./util"; -import { join, normalize } from "path"; -import getLogger from "@cocalc/backend/logger"; - -const BUP_SNAPSHOT = "temp-bup-snapshot"; - -const logger = getLogger("file-server:btrfs:subvolume-bup"); - -export class SubvolumeBup { - constructor(private subvolume: Subvolume) {} - - // create a new bup backup - save = async ({ - // timeout used for bup index and bup save commands - timeout = 30 * 60 * 1000, - }: { timeout?: number } = {}) => { - if (await this.subvolume.snapshots.exists(BUP_SNAPSHOT)) { - logger.debug(`createBupBackup: deleting existing ${BUP_SNAPSHOT}`); - await this.subvolume.snapshots.delete(BUP_SNAPSHOT); - } - try { - logger.debug( - `createBackup: creating ${BUP_SNAPSHOT} to get a consistent backup`, - ); - await this.subvolume.snapshots.create(BUP_SNAPSHOT); - const target = this.subvolume.normalize( - this.subvolume.snapshots.path(BUP_SNAPSHOT), - ); - - logger.debug(`createBupBackup: indexing ${BUP_SNAPSHOT}`); - await sudo({ - command: "bup", - args: [ - "-d", - this.subvolume.filesystem.bup, - "index", - "--exclude", - join(target, ".snapshots"), - "-x", - target, - ], - timeout, - }); - - logger.debug(`createBackup: saving ${BUP_SNAPSHOT}`); - await sudo({ - command: "bup", - args: [ - "-d", - this.subvolume.filesystem.bup, - "save", - "--strip", - "-n", - this.subvolume.name, - target, - ], - timeout, - }); - } finally { - logger.debug(`createBupBackup: deleting temporary ${BUP_SNAPSHOT}`); - await this.subvolume.snapshots.delete(BUP_SNAPSHOT); - } - }; - - restore = async (path: string) => { - // path -- branch/revision/path/to/dir - if (path.startsWith("/")) { - path = path.slice(1); - } - path = normalize(path); - // ... but to avoid potential data loss, we make a snapshot before deleting it. - await this.subvolume.snapshots.create(); - const i = path.indexOf("/"); // remove the commit name - // remove the target we're about to restore - await this.subvolume.fs.rm(path.slice(i + 1), { recursive: true }); - await sudo({ - command: "bup", - args: [ - "-d", - this.subvolume.filesystem.bup, - "restore", - "-C", - this.subvolume.path, - join(`/${this.subvolume.name}`, path), - "--quiet", - ], - }); - }; - - ls = async (path: string = ""): Promise => { - if (!path) { - const { stdout } = await sudo({ - command: "bup", - args: ["-d", this.subvolume.filesystem.bup, "ls", this.subvolume.name], - }); - const v: DirectoryListingEntry[] = []; - let newest = 0; - for (const x of stdout.trim().split("\n")) { - const name = x.split(" ").slice(-1)[0]; - if (name == "latest") { - continue; - } - const mtime = parseBupTime(name).valueOf() / 1000; - newest = Math.max(mtime, newest); - v.push({ name, isdir: true, mtime }); - } - if (v.length > 0) { - v.push({ name: "latest", isdir: true, mtime: newest }); - } - return v; - } - - path = normalize(path); - const { stdout } = await sudo({ - command: "bup", - args: [ - "-d", - this.subvolume.filesystem.bup, - "ls", - "--almost-all", - "--file-type", - "-l", - join(`/${this.subvolume.name}`, path), - ], - }); - const v: DirectoryListingEntry[] = []; - for (const x of stdout.split("\n")) { - // [-rw-------","6b851643360e435eb87ef9a6ab64a8b1/6b851643360e435eb87ef9a6ab64a8b1","5","2025-07-15","06:12","a.txt"] - const w = x.split(/\s+/); - if (w.length >= 6) { - let isdir, name; - if (w[5].endsWith("@") || w[5].endsWith("=") || w[5].endsWith("|")) { - w[5] = w[5].slice(0, -1); - } - if (w[5].endsWith("/")) { - isdir = true; - name = w[5].slice(0, -1); - } else { - name = w[5]; - isdir = false; - } - const size = parseInt(w[2]); - const mtime = new Date(w[3] + "T" + w[4]).valueOf() / 1000; - v.push({ name, size, mtime, isdir }); - } - } - return v; - }; - - prune = async ({ - dailies = "1w", - monthlies = "4m", - all = "3d", - }: { dailies?: string; monthlies?: string; all?: string } = {}) => { - await sudo({ - command: "bup", - args: [ - "-d", - this.subvolume.filesystem.bup, - "prune-older", - `--keep-dailies-for=${dailies}`, - `--keep-monthlies-for=${monthlies}`, - `--keep-all-for=${all}`, - "--unsafe", - this.subvolume.name, - ], - }); - }; -} diff --git a/src/packages/file-server/btrfs/subvolume-fs.ts b/src/packages/file-server/btrfs/subvolume-fs.ts deleted file mode 100644 index f1f2dd36772..00000000000 --- a/src/packages/file-server/btrfs/subvolume-fs.ts +++ /dev/null @@ -1,150 +0,0 @@ -import { - appendFile, - chmod, - cp, - copyFile, - link, - readFile, - realpath, - rename, - rm, - rmdir, - mkdir, - stat, - symlink, - truncate, - writeFile, - unlink, - utimes, - watch, -} from "node:fs/promises"; -import { exists } from "@cocalc/backend/misc/async-utils-node"; -import { type DirectoryListingEntry } from "@cocalc/util/types"; -import getListing from "@cocalc/backend/get-listing"; -import { type Subvolume } from "./subvolume"; -import { isdir, sudo } from "./util"; - -export class SubvolumeFilesystem { - constructor(private subvolume: Subvolume) {} - - private normalize = this.subvolume.normalize; - - ls = async ( - path: string, - { hidden, limit }: { hidden?: boolean; limit?: number } = {}, - ): Promise => { - return await getListing(this.normalize(path), hidden, { - limit, - home: "/", - }); - }; - - readFile = async (path: string, encoding?: any): Promise => { - return await readFile(this.normalize(path), encoding); - }; - - writeFile = async (path: string, data: string | Buffer) => { - return await writeFile(this.normalize(path), data); - }; - - appendFile = async (path: string, data: string | Buffer, encoding?) => { - return await appendFile(this.normalize(path), data, encoding); - }; - - unlink = async (path: string) => { - await unlink(this.normalize(path)); - }; - - stat = async (path: string) => { - return await stat(this.normalize(path)); - }; - - exists = async (path: string) => { - return await exists(this.normalize(path)); - }; - - // hard link - link = async (existingPath: string, newPath: string) => { - return await link(this.normalize(existingPath), this.normalize(newPath)); - }; - - symlink = async (target: string, path: string) => { - return await symlink(this.normalize(target), this.normalize(path)); - }; - - realpath = async (path: string) => { - const x = await realpath(this.normalize(path)); - return x.slice(this.subvolume.path.length + 1); - }; - - rename = async (oldPath: string, newPath: string) => { - await rename(this.normalize(oldPath), this.normalize(newPath)); - }; - - utimes = async ( - path: string, - atime: number | string | Date, - mtime: number | string | Date, - ) => { - await utimes(this.normalize(path), atime, mtime); - }; - - watch = (filename: string, options?) => { - return watch(this.normalize(filename), options); - }; - - truncate = async (path: string, len?: number) => { - await truncate(this.normalize(path), len); - }; - - copyFile = async (src: string, dest: string) => { - await copyFile(this.normalize(src), this.normalize(dest)); - }; - - cp = async (src: string, dest: string, options?) => { - await cp(this.normalize(src), this.normalize(dest), options); - }; - - chmod = async (path: string, mode: string | number) => { - await chmod(this.normalize(path), mode); - }; - - mkdir = async (path: string, options?) => { - await mkdir(this.normalize(path), options); - }; - - rsync = async ({ - src, - target, - args = ["-axH"], - timeout = 5 * 60 * 1000, - }: { - src: string; - target: string; - args?: string[]; - timeout?: number; - }): Promise<{ stdout: string; stderr: string; exit_code: number }> => { - let srcPath = this.normalize(src); - let targetPath = this.normalize(target); - if (!srcPath.endsWith("/") && (await isdir(srcPath))) { - srcPath += "/"; - if (!targetPath.endsWith("/")) { - targetPath += "/"; - } - } - return await sudo({ - command: "rsync", - args: [...args, srcPath, targetPath], - err_on_exit: false, - timeout: timeout / 1000, - }); - }; - - rmdir = async (path: string, options?) => { - await rmdir(this.normalize(path), options); - }; - - rm = async (path: string, options?) => { - await rm(this.normalize(path), options); - }; -} diff --git a/src/packages/file-server/btrfs/subvolume-rustic.ts b/src/packages/file-server/btrfs/subvolume-rustic.ts new file mode 100644 index 00000000000..32f0319242b --- /dev/null +++ b/src/packages/file-server/btrfs/subvolume-rustic.ts @@ -0,0 +1,122 @@ +/* +Rustic Architecture: + +The minimal option is a single global repo stored in the btrfs filesystem. +Obviously, admins should rsync this regularly to a separate location as a +genuine backup strategy. It's better to configure repo on separate +storage. Rustic has a very wide range of options. + +Instead of using btrfs send/recv for backups, we use Rustic because: + - much easier to check backups are valid + - globally compressed and dedup'd! btrfs send/recv is NOT globally dedupd + - decoupled from any btrfs issues + - rustic has full support for using cloud buckets as hot/cold storage + - not tied to any specific filesystem at all + - easier to offsite via incremental rsync + - much more space efficient with *global* dedup and compression + - rustic "is" restic, which is very mature and proven + - rustic is VERY fast, being parallel and in rust. +*/ + +import { type Subvolume } from "./subvolume"; +import getLogger from "@cocalc/backend/logger"; +import { parseOutput } from "@cocalc/backend/sandbox/exec"; +import { field_cmp } from "@cocalc/util/misc"; + +export const RUSTIC = "rustic"; + +const RUSTIC_SNAPSHOT = "temp-rustic-snapshot"; + +const logger = getLogger("file-server:btrfs:subvolume-rustic"); + +interface Snapshot { + id: string; + time: Date; +} + +export class SubvolumeRustic { + constructor(private subvolume: Subvolume) {} + + // create a new rustic backup + backup = async ({ + timeout = 30 * 60 * 1000, + }: { timeout?: number } = {}): Promise => { + if (await this.subvolume.snapshots.exists(RUSTIC_SNAPSHOT)) { + logger.debug(`backup: deleting existing ${RUSTIC_SNAPSHOT}`); + await this.subvolume.snapshots.delete(RUSTIC_SNAPSHOT); + } + const target = this.subvolume.snapshots.path(RUSTIC_SNAPSHOT); + try { + logger.debug( + `backup: creating ${RUSTIC_SNAPSHOT} to get a consistent backup`, + ); + await this.subvolume.snapshots.create(RUSTIC_SNAPSHOT); + logger.debug(`backup: backing up ${RUSTIC_SNAPSHOT} using rustic`); + const { stdout } = parseOutput( + await this.subvolume.fs.rustic(["backup", "-x", "--json", "."], { + timeout, + cwd: target, + }), + ); + const { time, id } = JSON.parse(stdout); + return { time: new Date(time), id }; + } finally { + logger.debug(`backup: deleting temporary ${RUSTIC_SNAPSHOT}`); + await this.subvolume.snapshots.delete(RUSTIC_SNAPSHOT); + } + }; + + restore = async ({ + id, + path = "", + dest, + timeout = 30 * 60 * 1000, + }: { + id: string; + path?: string; + dest?: string; + timeout?: number; + }) => { + dest ??= path; + const { stdout } = parseOutput( + await this.subvolume.fs.rustic( + ["restore", `${id}${path != null ? ":" + path : ""}`, dest], + { timeout }, + ), + ); + return stdout; + }; + + // returns list of snapshots sorted from oldest to newest + snapshots = async (): Promise => { + const { stdout } = parseOutput( + await this.subvolume.fs.rustic(["snapshots", "--json"]), + ); + const x = JSON.parse(stdout); + const v = x[0][1].map(({ time, id }) => { + return { time: new Date(time), id }; + }); + v.sort(field_cmp("time")); + return v; + }; + + // return list of paths of files in this backup, as paths relative + // to HOME, and sorted in alphabetical order. + ls = async ({ id }: { id: string }) => { + const { stdout } = parseOutput( + await this.subvolume.fs.rustic(["ls", "--json", id]), + ); + return JSON.parse(stdout).sort(); + }; + + // Delete this backup. It's genuinely not accessible anymore, though + // this doesn't actually clean up disk space -- purge must be done separately + // later. Rustic likes the purge to happen maybe a day later, so it + // can better support concurrent writes. + forget = async ({ id }: { id: string }) => { + const { stdout } = parseOutput( + await this.subvolume.fs.rustic(["forget", id]), + ); + return stdout; + }; +} diff --git a/src/packages/file-server/btrfs/subvolume-snapshots.ts b/src/packages/file-server/btrfs/subvolume-snapshots.ts index ffe71fe6fc7..ddcc3ca2e10 100644 --- a/src/packages/file-server/btrfs/subvolume-snapshots.ts +++ b/src/packages/file-server/btrfs/subvolume-snapshots.ts @@ -2,7 +2,6 @@ import { type Subvolume } from "./subvolume"; import { btrfs } from "./util"; import getLogger from "@cocalc/backend/logger"; import { join } from "path"; -import { type DirectoryListingEntry } from "@cocalc/util/types"; import { SnapshotCounts, updateRollingSnapshots } from "./snapshots"; export const SNAPSHOTS = ".snapshots"; @@ -27,7 +26,7 @@ export class SubvolumeSnapshots { return; } await this.subvolume.fs.mkdir(SNAPSHOTS); - await this.subvolume.fs.chmod(SNAPSHOTS, "0550"); + await this.subvolume.fs.chmod(SNAPSHOTS, "0700"); }; create = async (name?: string) => { @@ -48,9 +47,9 @@ export class SubvolumeSnapshots { }); }; - ls = async (): Promise => { + readdir = async (): Promise => { await this.makeSnapshotsDir(); - return await this.subvolume.fs.ls(SNAPSHOTS, { hidden: false }); + return await this.subvolume.fs.readdir(SNAPSHOTS); }; lock = async (name: string) => { @@ -85,18 +84,18 @@ export class SubvolumeSnapshots { // has newly written changes since last snapshot hasUnsavedChanges = async (): Promise => { - const s = await this.ls(); + const s = await this.readdir(); if (s.length == 0) { // more than just the SNAPSHOTS directory? - const v = await this.subvolume.fs.ls("", { hidden: true }); - if (v.length == 0 || (v.length == 1 && v[0].name == SNAPSHOTS)) { + const v = await this.subvolume.fs.readdir(""); + if (v.length == 0 || (v.length == 1 && v[0] == SNAPSHOTS)) { return false; } return true; } const pathGen = await getGeneration(this.subvolume.path); const snapGen = await getGeneration( - join(this.snapshotsDir, s[s.length - 1].name), + join(this.snapshotsDir, s[s.length - 1]), ); return snapGen < pathGen; }; diff --git a/src/packages/file-server/btrfs/subvolume.ts b/src/packages/file-server/btrfs/subvolume.ts index 3f77abd9c40..b3b061aed2f 100644 --- a/src/packages/file-server/btrfs/subvolume.ts +++ b/src/packages/file-server/btrfs/subvolume.ts @@ -2,14 +2,14 @@ A subvolume */ -import { type Filesystem, DEFAULT_SUBVOLUME_SIZE } from "./filesystem"; +import { type Filesystem } from "./filesystem"; import refCache from "@cocalc/util/refcache"; import { sudo } from "./util"; -import { join, normalize } from "path"; -import { SubvolumeFilesystem } from "./subvolume-fs"; -import { SubvolumeBup } from "./subvolume-bup"; +import { join } from "path"; +import { SubvolumeRustic } from "./subvolume-rustic"; import { SubvolumeSnapshots } from "./subvolume-snapshots"; import { SubvolumeQuota } from "./subvolume-quota"; +import { SandboxedFilesystem } from "@cocalc/backend/sandbox"; import { exists } from "@cocalc/backend/misc/async-utils-node"; import getLogger from "@cocalc/backend/logger"; @@ -23,11 +23,10 @@ interface Options { export class Subvolume { public readonly name: string; - public readonly filesystem: Filesystem; public readonly path: string; - public readonly fs: SubvolumeFilesystem; - public readonly bup: SubvolumeBup; + public readonly fs: SandboxedFilesystem; + public readonly rustic: SubvolumeRustic; public readonly snapshots: SubvolumeSnapshots; public readonly quota: SubvolumeQuota; @@ -35,8 +34,11 @@ export class Subvolume { this.filesystem = filesystem; this.name = name; this.path = join(filesystem.opts.mount, name); - this.fs = new SubvolumeFilesystem(this); - this.bup = new SubvolumeBup(this); + this.fs = new SandboxedFilesystem(this.path, { + rusticRepo: filesystem.opts.rustic, + host: this.name, + }); + this.rustic = new SubvolumeRustic(this); this.snapshots = new SubvolumeSnapshots(this); this.quota = new SubvolumeQuota(this); } @@ -49,9 +51,6 @@ export class Subvolume { args: ["subvolume", "create", this.path], }); await this.chown(this.path); - await this.quota.set( - this.filesystem.opts.defaultSize ?? DEFAULT_SUBVOLUME_SIZE, - ); } }; @@ -64,7 +63,7 @@ export class Subvolume { delete this.path; // @ts-ignore delete this.snapshotsDir; - for (const sub of ["fs", "bup", "snapshots", "quota"]) { + for (const sub of ["fs", "rustic", "snapshots", "quota"]) { this[sub].close?.(); delete this[sub]; } @@ -76,13 +75,6 @@ export class Subvolume { args: [`${process.getuid?.() ?? 0}:${process.getgid?.() ?? 0}`, path], }); }; - - // this should provide a path that is guaranteed to be - // inside this.path on the filesystem or throw error - // [ ] TODO: not sure if the code here is sufficient!! - normalize = (path: string) => { - return join(this.path, normalize(path)); - }; } const cache = refCache({ diff --git a/src/packages/file-server/btrfs/subvolumes.ts b/src/packages/file-server/btrfs/subvolumes.ts index 0f8da1468f5..a193b37a1a9 100644 --- a/src/packages/file-server/btrfs/subvolumes.ts +++ b/src/packages/file-server/btrfs/subvolumes.ts @@ -3,17 +3,22 @@ import { subvolume, type Subvolume } from "./subvolume"; import getLogger from "@cocalc/backend/logger"; import { SNAPSHOTS } from "./subvolume-snapshots"; import { exists } from "@cocalc/backend/misc/async-utils-node"; -import { join, normalize } from "path"; -import { btrfs, isdir } from "./util"; +import { join } from "path"; +import { btrfs } from "./util"; import { chmod, rename, rm } from "node:fs/promises"; -import { executeCode } from "@cocalc/backend/execute-code"; +import { SandboxedFilesystem } from "@cocalc/backend/sandbox"; +import { RUSTIC } from "./subvolume-rustic"; -const RESERVED = new Set(["bup", SNAPSHOTS]); +const RESERVED = new Set([RUSTIC, SNAPSHOTS]); const logger = getLogger("file-server:btrfs:subvolumes"); export class Subvolumes { - constructor(public filesystem: Filesystem) {} + public readonly fs: SandboxedFilesystem; + + constructor(public filesystem: Filesystem) { + this.fs = new SandboxedFilesystem(this.filesystem.opts.mount); + } get = async (name: string): Promise => { if (RESERVED.has(name)) { @@ -22,7 +27,6 @@ export class Subvolumes { return await subvolume({ filesystem: this.filesystem, name }); }; - // create a subvolume by cloning an existing one. clone = async (source: string, dest: string) => { logger.debug("clone ", { source, dest }); if (RESERVED.has(dest)) { @@ -79,37 +83,4 @@ export class Subvolumes { .filter((x) => x) .sort(); }; - - rsync = async ({ - src, - target, - args = ["-axH"], - timeout = 5 * 60 * 1000, - }: { - src: string; - target: string; - args?: string[]; - timeout?: number; - }): Promise<{ stdout: string; stderr: string; exit_code: number }> => { - let srcPath = normalize(join(this.filesystem.opts.mount, src)); - if (!srcPath.startsWith(this.filesystem.opts.mount)) { - throw Error("suspicious source"); - } - let targetPath = normalize(join(this.filesystem.opts.mount, target)); - if (!targetPath.startsWith(this.filesystem.opts.mount)) { - throw Error("suspicious target"); - } - if (!srcPath.endsWith("/") && (await isdir(srcPath))) { - srcPath += "/"; - if (!targetPath.endsWith("/")) { - targetPath += "/"; - } - } - return await executeCode({ - command: "rsync", - args: [...args, srcPath, targetPath], - err_on_exit: false, - timeout: timeout / 1000, - }); - }; } diff --git a/src/packages/file-server/btrfs/test/filesystem-stress.test.ts b/src/packages/file-server/btrfs/test/filesystem-stress.test.ts index a2e1de95318..2302785d14b 100644 --- a/src/packages/file-server/btrfs/test/filesystem-stress.test.ts +++ b/src/packages/file-server/btrfs/test/filesystem-stress.test.ts @@ -90,6 +90,11 @@ describe("stress operations with subvolumes", () => { `deleted ${Math.round((count2 / (Date.now() - t)) * 1000)} subvolumes per second in parallel`, ); }); + + it("everything should be gone except the clones", async () => { + const v = await fs.subvolumes.list(); + expect(v.length).toBe(count1 + count2); + }); }); afterAll(after); diff --git a/src/packages/file-server/btrfs/test/filesystem.test.ts b/src/packages/file-server/btrfs/test/filesystem.test.ts index 673980af897..c4eea6992a1 100644 --- a/src/packages/file-server/btrfs/test/filesystem.test.ts +++ b/src/packages/file-server/btrfs/test/filesystem.test.ts @@ -1,5 +1,6 @@ import { before, after, fs } from "./setup"; import { isValidUUID } from "@cocalc/util/misc"; +import { RUSTIC } from "@cocalc/file-server/btrfs/subvolume-rustic"; beforeAll(before); @@ -22,7 +23,7 @@ describe("some basic tests", () => { describe("operations with subvolumes", () => { it("can't use a reserved subvolume name", async () => { expect(async () => { - await fs.subvolumes.get("bup"); + await fs.subvolumes.get(RUSTIC); }).rejects.toThrow("is reserved"); }); @@ -30,7 +31,7 @@ describe("operations with subvolumes", () => { const vol = await fs.subvolumes.get("cocalc"); expect(vol.name).toBe("cocalc"); // it has no snapshots - expect(await vol.snapshots.ls()).toEqual([]); + expect(await vol.snapshots.readdir()).toEqual([]); }); it("our subvolume is in the list", async () => { @@ -62,24 +63,37 @@ describe("operations with subvolumes", () => { ]); }); - it("rsync from one volume to another", async () => { - await fs.subvolumes.rsync({ src: "sagemath", target: "cython" }); + it("cp from one volume to another", async () => { + await fs.subvolumes.fs.cp("sagemath", "cython", { + recursive: true, + reflink: true, + }); }); - it("rsync an actual file", async () => { + it("cp an actual file", async () => { const sagemath = await fs.subvolumes.get("sagemath"); const cython = await fs.subvolumes.get("cython"); - await sagemath.fs.writeFile("README.md", "hi"); - await fs.subvolumes.rsync({ src: "sagemath", target: "cython" }); + await sagemath.fs.writeFile("README.md", "hi5"); + await fs.subvolumes.fs.cp("sagemath/README.md", "cython/README.md", { + reflink: true, + }); const copy = await cython.fs.readFile("README.md", "utf8"); - expect(copy).toEqual("hi"); + expect(copy).toEqual("hi5"); + + // also one without reflink + await sagemath.fs.writeFile("README2.md", "hi2"); + await fs.subvolumes.fs.cp("sagemath/README2.md", "cython/README2.md", { + reflink: false, + }); + const copy2 = await cython.fs.readFile("README2.md", "utf8"); + expect(copy2).toEqual("hi2"); }); it("clone a subvolume with contents", async () => { await fs.subvolumes.clone("cython", "pyrex"); const pyrex = await fs.subvolumes.get("pyrex"); const clone = await pyrex.fs.readFile("README.md", "utf8"); - expect(clone).toEqual("hi"); + expect(clone).toEqual("hi5"); }); }); @@ -97,7 +111,7 @@ describe("clone of a subvolume with snapshots should have no snapshots", () => { it("clone has no snapshots", async () => { const clone = await fs.subvolumes.get("my-clone"); expect(await clone.fs.readFile("abc.txt", "utf8")).toEqual("hi"); - expect(await clone.snapshots.ls()).toEqual([]); + expect(await clone.snapshots.readdir()).toEqual([]); await clone.snapshots.create("my-clone-snap"); }); }); diff --git a/src/packages/file-server/btrfs/test/rustic-stress.test.ts b/src/packages/file-server/btrfs/test/rustic-stress.test.ts new file mode 100644 index 00000000000..2eb0514df71 --- /dev/null +++ b/src/packages/file-server/btrfs/test/rustic-stress.test.ts @@ -0,0 +1,45 @@ +import { before, after, fs } from "./setup"; +import { type Subvolume } from "../subvolume"; + +beforeAll(before); + +const count = 10; +describe(`make backups of ${count} different volumes at the same time`, () => { + const vols: Subvolume[] = []; + it(`creates ${count} volumes`, async () => { + for (let i = 0; i < count; i++) { + const vol = await fs.subvolumes.get(`rustic-multi-${i}`); + await vol.fs.writeFile(`a-${i}.txt`, `hello-${i}`); + vols.push(vol); + } + }); + + it(`create ${count} rustic backup in parallel`, async () => { + await Promise.all(vols.map((vol) => vol.rustic.backup())); + }); + + it("delete file from each volume, then restore them all in parallel and confirm restore worked", async () => { + const snapshots = await Promise.all( + vols.map((vol) => vol.rustic.snapshots()), + ); + const ids = snapshots.map((x) => x[0].id); + for (let i = 0; i < count; i++) { + await vols[i].fs.unlink(`a-${i}.txt`); + } + + const v: any[] = []; + for (let i = 0; i < count; i++) { + v.push(vols[i].rustic.restore({ id: ids[i] })); + } + await Promise.all(v); + + for (let i = 0; i < count; i++) { + const vol = vols[i]; + expect((await vol.fs.readFile(`a-${i}.txt`)).toString("utf8")).toEqual( + `hello-${i}`, + ); + } + }); +}); + +afterAll(after); diff --git a/src/packages/file-server/btrfs/test/rustic.test.ts b/src/packages/file-server/btrfs/test/rustic.test.ts new file mode 100644 index 00000000000..5c75e46dded --- /dev/null +++ b/src/packages/file-server/btrfs/test/rustic.test.ts @@ -0,0 +1,75 @@ +import { before, after, fs } from "./setup"; +import { type Subvolume } from "../subvolume"; + +beforeAll(before); + +describe("test rustic backups", () => { + let vol: Subvolume; + it("creates a volume", async () => { + vol = await fs.subvolumes.get("rustic-test"); + await vol.fs.writeFile("a.txt", "hello"); + }); + + let x; + it("create a rustic backup", async () => { + x = await vol.rustic.backup(); + }); + + it("confirm a.txt is in our backup", async () => { + const v = await vol.rustic.snapshots(); + expect(v.length == 1); + expect(v[0]).toEqual(x); + expect(Math.abs(Date.now() - v[0].time.valueOf())).toBeLessThan(10000); + const { id } = v[0]; + const w = await vol.rustic.ls({ id }); + expect(w).toEqual([".snapshots", "a.txt"]); + }); + + it("delete a.txt, then restore it from the backup", async () => { + await vol.fs.unlink("a.txt"); + const { id } = (await vol.rustic.snapshots())[0]; + await vol.rustic.restore({ id }); + expect((await vol.fs.readFile("a.txt")).toString("utf8")).toEqual("hello"); + }); + + it("create a directory, make second backup, delete directory, then restore it from backup, and also restore just one file", async () => { + await vol.fs.mkdir("my-dir"); + await vol.fs.writeFile("my-dir/file.txt", "hello"); + await vol.fs.writeFile("my-dir/file2.txt", "hello2"); + await vol.rustic.backup(); + const v = await vol.rustic.snapshots(); + expect(v.length == 2); + const { id } = v[1]; + const w = await vol.rustic.ls({ id }); + expect(w).toEqual([ + ".snapshots", + "a.txt", + "my-dir", + "my-dir/file.txt", + "my-dir/file2.txt", + ]); + await vol.fs.rm("my-dir", { recursive: true }); + + // rustic all, including the path we just deleted + await vol.rustic.restore({ id }); + expect((await vol.fs.readFile("my-dir/file.txt")).toString("utf8")).toEqual( + "hello", + ); + + // restore just one specific file overwriting current version + await vol.fs.unlink("my-dir/file2.txt"); + await vol.fs.writeFile("my-dir/file.txt", "changed"); + await vol.rustic.restore({ id, path: "my-dir/file2.txt" }); + expect( + (await vol.fs.readFile("my-dir/file2.txt")).toString("utf8"), + ).toEqual("hello2"); + + // forget the second snapshot + await vol.rustic.forget({ id }); + const v2 = await vol.rustic.snapshots(); + expect(v2.length).toBe(1); + expect(v2[0].id).not.toEqual(id); + }); +}); + +afterAll(after); diff --git a/src/packages/file-server/btrfs/test/setup.ts b/src/packages/file-server/btrfs/test/setup.ts index c9736c8b79b..72dc8939ffe 100644 --- a/src/packages/file-server/btrfs/test/setup.ts +++ b/src/packages/file-server/btrfs/test/setup.ts @@ -2,11 +2,11 @@ import { filesystem, type Filesystem, } from "@cocalc/file-server/btrfs/filesystem"; -import { chmod, mkdtemp, mkdir, rm, stat } from "node:fs/promises"; +import { chmod, mkdtemp, mkdir, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "path"; import { until } from "@cocalc/util/async-utils"; -import { sudo } from "../util"; +import { ensureMoreLoopbackDevices, sudo } from "../util"; export { sudo }; export { delay } from "awaiting"; @@ -15,25 +15,6 @@ let tempDir; const TEMP_PREFIX = "cocalc-test-btrfs-"; -async function ensureMoreLoops() { - // to run tests, this is helpful - //for i in $(seq 8 63); do sudo mknod -m660 /dev/loop$i b 7 $i; sudo chown root:disk /dev/loop$i; done - for (let i = 0; i < 64; i++) { - try { - await stat(`/dev/loop${i}`); - continue; - } catch {} - try { - // also try/catch this because ensureMoreLoops happens in parallel many times at once... - await sudo({ - command: "mknod", - args: ["-m660", `/dev/loop${i}`, "b", "7", `${i}`], - }); - } catch {} - await sudo({ command: "chown", args: ["root:disk", `/dev/loop${i}`] }); - } -} - export async function before() { try { const command = `umount ${join(tmpdir(), TEMP_PREFIX)}*/mnt`; @@ -41,7 +22,7 @@ export async function before() { // TODO: this could impact runs in parallel await sudo({ command, bash: true }); } catch {} - await ensureMoreLoops(); + await ensureMoreLoopbackDevices(); tempDir = await mkdtemp(join(tmpdir(), TEMP_PREFIX)); // Set world read/write/execute await chmod(tempDir, 0o777); @@ -49,10 +30,12 @@ export async function before() { await mkdir(mount); await chmod(mount, 0o777); fs = await filesystem({ - device: join(tempDir, "btrfs.img"), - formatIfNeeded: true, + image: join(tempDir, "btrfs.img"), + size: "1G", mount: join(tempDir, "mnt"), + rustic: join(tempDir, "rustic"), }); + return fs; } export async function after() { diff --git a/src/packages/file-server/btrfs/test/subvolume-stress.test.ts b/src/packages/file-server/btrfs/test/subvolume-stress.test.ts index 29bc048e695..69279d9b042 100644 --- a/src/packages/file-server/btrfs/test/subvolume-stress.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume-stress.test.ts @@ -29,16 +29,16 @@ describe(`stress test creating ${numSnapshots} snapshots`, () => { `created ${Math.round((numSnapshots / (Date.now() - start)) * 1000)} snapshots per second in serial`, ); snaps.sort(); - expect((await vol.snapshots.ls()).map(({ name }) => name).sort()).toEqual( - snaps.sort(), - ); + expect( + (await vol.snapshots.readdir()).filter((x) => !x.startsWith(".")).sort(), + ).toEqual(snaps.sort()); }); it(`delete our ${numSnapshots} snapshots`, async () => { for (let i = 0; i < numSnapshots; i++) { await vol.snapshots.delete(`snap${i}`); } - expect(await vol.snapshots.ls()).toEqual([]); + expect(await vol.snapshots.readdir()).toEqual([]); }); }); @@ -58,9 +58,8 @@ describe(`create ${numFiles} files`, () => { log( `created ${Math.round((numFiles / (Date.now() - start)) * 1000)} files per second in serial`, ); - const v = await vol.fs.ls(""); - const w = v.map(({ name }) => name); - expect(w.sort()).toEqual(names.sort()); + const v = await vol.fs.readdir(""); + expect(v.sort()).toEqual(names.sort()); }); it(`creates ${numFiles} files in parallel`, async () => { @@ -77,9 +76,8 @@ describe(`create ${numFiles} files`, () => { `created ${Math.round((numFiles / (Date.now() - start)) * 1000)} files per second in parallel`, ); const t0 = Date.now(); - const v = await vol.fs.ls("p"); + const w = await vol.fs.readdir("p"); log("get listing of files took", Date.now() - t0, "ms"); - const w = v.map(({ name }) => name); expect(w.sort()).toEqual(names.sort()); }); }); diff --git a/src/packages/file-server/btrfs/test/subvolume.test.ts b/src/packages/file-server/btrfs/test/subvolume.test.ts index e586cc656bf..7f1a03050be 100644 --- a/src/packages/file-server/btrfs/test/subvolume.test.ts +++ b/src/packages/file-server/btrfs/test/subvolume.test.ts @@ -1,6 +1,4 @@ import { before, after, fs, sudo } from "./setup"; -import { mkdir } from "fs/promises"; -import { join } from "path"; import { wait } from "@cocalc/backend/conat/test/util"; import { randomBytes } from "crypto"; import { type Subvolume } from "../subvolume"; @@ -19,7 +17,7 @@ describe("setting and getting quota of a subvolume", () => { }); it("get directory listing", async () => { - const v = await vol.fs.ls(""); + const v = await vol.fs.readdir(""); expect(v).toEqual([]); }); @@ -36,9 +34,8 @@ describe("setting and getting quota of a subvolume", () => { const { used } = await vol.quota.usage(); expect(used).toBeGreaterThan(0); - const v = await vol.fs.ls(""); - // size is potentially random, reflecting compression - expect(v).toEqual([{ name: "buf", mtime: v[0].mtime, size: v[0].size }]); + const v = await vol.fs.readdir(""); + expect(v).toEqual(["buf"]); }); it("fail to write a 50MB file (due to quota)", async () => { @@ -54,20 +51,20 @@ describe("the filesystem operations", () => { it("creates a volume and get empty listing", async () => { vol = await fs.subvolumes.get("fs"); - expect(await vol.fs.ls("")).toEqual([]); + expect(await vol.fs.readdir("")).toEqual([]); }); it("error listing non-existent path", async () => { vol = await fs.subvolumes.get("fs"); expect(async () => { - await vol.fs.ls("no-such-path"); + await vol.fs.readdir("no-such-path"); }).rejects.toThrow("ENOENT"); }); it("creates a text file to it", async () => { await vol.fs.writeFile("a.txt", "hello"); - const ls = await vol.fs.ls(""); - expect(ls).toEqual([{ name: "a.txt", mtime: ls[0].mtime, size: 5 }]); + const ls = await vol.fs.readdir(""); + expect(ls).toEqual(["a.txt"]); }); it("read the file we just created as utf8", async () => { @@ -87,17 +84,18 @@ describe("the filesystem operations", () => { let origStat; it("snapshot filesystem and see file is in snapshot", async () => { await vol.snapshots.create("snap"); - const s = await vol.fs.ls(vol.snapshots.path("snap")); - expect(s).toEqual([{ name: "a.txt", mtime: s[0].mtime, size: 5 }]); + const s = await vol.fs.readdir(vol.snapshots.path("snap")); + expect(s).toContain("a.txt"); - const stat = await vol.fs.stat("a.txt"); - origStat = stat; - expect(stat.mtimeMs / 1000).toBeCloseTo(s[0].mtime ?? 0); + const stat0 = await vol.fs.stat(vol.snapshots.path("snap")); + const stat1 = await vol.fs.stat("a.txt"); + origStat = stat1; + expect(stat1.mtimeMs).toBeCloseTo(stat0.mtimeMs, -2); }); it("unlink (delete) our file", async () => { await vol.fs.unlink("a.txt"); - expect(await vol.fs.ls("")).toEqual([]); + expect(await vol.fs.readdir("")).toEqual([".snapshots"]); }); it("snapshot still exists", async () => { @@ -126,6 +124,7 @@ describe("the filesystem operations", () => { it("make a file readonly, then change it back", async () => { await vol.fs.writeFile("c.txt", "hi"); await vol.fs.chmod("c.txt", "440"); + await fs.sync(); expect(async () => { await vol.fs.appendFile("c.txt", " there"); }).rejects.toThrow("EACCES"); @@ -143,18 +142,17 @@ describe("the filesystem operations", () => { await vol.fs.writeFile("w.txt", "hi"); const ac = new AbortController(); const { signal } = ac; - const watcher = vol.fs.watch("w.txt", { signal }); + const watcher = await vol.fs.watch("w.txt", { signal }); vol.fs.appendFile("w.txt", " there"); // @ts-ignore const { value, done } = await watcher.next(); expect(done).toBe(false); expect(value).toEqual({ eventType: "change", filename: "w.txt" }); ac.abort(); - - expect(async () => { - // @ts-ignore - await watcher.next(); - }).rejects.toThrow("aborted"); + { + const { done } = await watcher.next(); + expect(done).toBe(true); + } }); it("rename a file", async () => { @@ -185,9 +183,9 @@ describe("test snapshots", () => { }); it("snapshot the volume", async () => { - expect(await vol.snapshots.ls()).toEqual([]); + expect(await vol.snapshots.readdir()).toEqual([]); await vol.snapshots.create("snap1"); - expect((await vol.snapshots.ls()).map((x) => x.name)).toEqual(["snap1"]); + expect(await vol.snapshots.readdir()).toEqual(["snap1"]); expect(await vol.snapshots.hasUnsavedChanges()).toBe(false); }); @@ -219,77 +217,11 @@ describe("test snapshots", () => { }); it("unlock our snapshot and delete it", async () => { + await fs.sync(); await vol.snapshots.unlock("snap1"); await vol.snapshots.delete("snap1"); expect(await vol.snapshots.exists("snap1")).toBe(false); - expect(await vol.snapshots.ls()).toEqual([]); - }); -}); - -describe.only("test bup backups", () => { - let vol: Subvolume; - it("creates a volume", async () => { - vol = await fs.subvolumes.get("bup-test"); - await vol.fs.writeFile("a.txt", "hello"); - }); - - it("create a bup backup", async () => { - await vol.bup.save(); - }); - - it("list bup backups of this vol -- there are 2, one for the date and 'latest'", async () => { - const v = await vol.bup.ls(); - expect(v.length).toBe(2); - const t = (v[0].mtime ?? 0) * 1000; - expect(Math.abs(t.valueOf() - Date.now())).toBeLessThan(10_000); - }); - - it("confirm a.txt is in our backup", async () => { - const x = await vol.bup.ls("latest"); - expect(x).toEqual([ - { name: "a.txt", size: 5, mtime: x[0].mtime, isdir: false }, - ]); - }); - - it("restore a.txt from our backup", async () => { - await vol.fs.writeFile("a.txt", "hello2"); - await vol.bup.restore("latest/a.txt"); - expect(await vol.fs.readFile("a.txt", "utf8")).toEqual("hello"); - }); - - it("prune bup backups does nothing since we have so few", async () => { - await vol.bup.prune(); - expect((await vol.bup.ls()).length).toBe(2); - }); - - it("add a directory and back up", async () => { - await mkdir(join(vol.path, "mydir")); - await vol.fs.writeFile(join("mydir", "file.txt"), "hello3"); - expect((await vol.fs.ls("mydir"))[0].name).toBe("file.txt"); - await vol.bup.save(); - const x = await vol.bup.ls("latest"); - expect(x).toEqual([ - { name: "a.txt", size: 5, mtime: x[0].mtime, isdir: false }, - { name: "mydir", size: 0, mtime: x[1].mtime, isdir: true }, - ]); - expect(Math.abs((x[0].mtime ?? 0) * 1000 - Date.now())).toBeLessThan( - 60_000, - ); - }); - - it("change file in the directory, then restore from backup whole dir", async () => { - await vol.fs.writeFile(join("mydir", "file.txt"), "changed"); - await vol.bup.restore("latest/mydir"); - expect(await vol.fs.readFile(join("mydir", "file.txt"), "utf8")).toEqual( - "hello3", - ); - }); - - it("most recent snapshot has a backup before the restore", async () => { - const s = await vol.snapshots.ls(); - const recent = s.slice(-1)[0].name; - const p = vol.snapshots.path(recent, "mydir", "file.txt"); - expect(await vol.fs.readFile(p, "utf8")).toEqual("changed"); + expect(await vol.snapshots.readdir()).toEqual([]); }); }); diff --git a/src/packages/file-server/btrfs/util.ts b/src/packages/file-server/btrfs/util.ts index b6408f84409..6decbab3d5f 100644 --- a/src/packages/file-server/btrfs/util.ts +++ b/src/packages/file-server/btrfs/util.ts @@ -44,7 +44,7 @@ export async function btrfs( return await sudo({ ...opts, command: "btrfs" }); } -export async function isdir(path: string) { +export async function isDir(path: string) { return (await stat(path)).isDirectory(); } @@ -63,3 +63,22 @@ export function parseBupTime(s: string): Date { Number(seconds), ); } + +export async function ensureMoreLoopbackDevices() { + // to run tests, this is helpful + //for i in $(seq 8 63); do sudo mknod -m660 /dev/loop$i b 7 $i; sudo chown root:disk /dev/loop$i; done + for (let i = 0; i < 64; i++) { + try { + await stat(`/dev/loop${i}`); + continue; + } catch {} + try { + // also try/catch this because ensureMoreLoops happens in parallel many times at once... + await sudo({ + command: "mknod", + args: ["-m660", `/dev/loop${i}`, "b", "7", `${i}`], + }); + } catch {} + await sudo({ command: "chown", args: ["root:disk", `/dev/loop${i}`] }); + } +} diff --git a/src/packages/file-server/package.json b/src/packages/file-server/package.json index 58285c4bf6f..1a952b2369c 100644 --- a/src/packages/file-server/package.json +++ b/src/packages/file-server/package.json @@ -4,13 +4,16 @@ "description": "CoCalc File Server", "exports": { "./btrfs": "./dist/btrfs/index.js", - "./btrfs/*": "./dist/btrfs/*.js" + "./btrfs/*": "./dist/btrfs/*.js", + "./conat/*": "./dist/conat/*.js", + "./fs/*": "./dist/fs/*.js" }, "scripts": { "preinstall": "npx only-allow pnpm", "build": "pnpm exec tsc --build", "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", - "test": "pnpm exec jest", + "test": "pnpm exec jest --forceExit", + "test-github-ci": "pnpm exec jest --maxWorkers=1 --forceExit", "depcheck": "pnpx depcheck", "clean": "rm -rf node_modules dist" }, diff --git a/src/packages/frontend/account/other-settings.tsx b/src/packages/frontend/account/other-settings.tsx index bdf6fcb0ca5..dba71ce5092 100644 --- a/src/packages/frontend/account/other-settings.tsx +++ b/src/packages/frontend/account/other-settings.tsx @@ -320,24 +320,6 @@ export function OtherSettings(props: Readonly): React.JSX.Element { ); } - function render_page_size(): Rendered { - return ( - - on_change("page_size", n)} - min={1} - max={10000} - number={props.other_settings.get("page_size")} - /> - - ); - } - function render_no_free_warnings(): Rendered { let extra; if (!props.is_stripe_customer) { @@ -714,7 +696,6 @@ export function OtherSettings(props: Readonly): React.JSX.Element { {render_vertical_fixed_bar_options()} {render_new_filenames()} {render_default_file_sort()} - {render_page_size()} {render_standby_timeout()}
diff --git a/src/packages/frontend/admin/site-settings/row-entry.tsx b/src/packages/frontend/admin/site-settings/row-entry.tsx index 8c964e00ca3..22e5ed48588 100644 --- a/src/packages/frontend/admin/site-settings/row-entry.tsx +++ b/src/packages/frontend/admin/site-settings/row-entry.tsx @@ -168,9 +168,8 @@ function VersionHint({ value }: { value: string }) { // The production site works differently. // TODO: make this a more sophisticated data editor. function JsonEntry({ name, data, readonly, onJsonEntryChange }) { - const jval = JSON.parse(data ?? "{}") ?? {}; - const dflt = FIELD_DEFAULTS[name]; - const quotas = { ...dflt, ...jval }; + const jsonValue = JSON.parse(data ?? "{}") ?? {}; + const quotas = { ...FIELD_DEFAULTS[name], ...jsonValue }; const value = JSON.stringify(quotas); return ( { - const x = await this.conat_client.openFiles(project_id); - if (setNotDeleted) { - x.setNotDeleted(path); - } - x.touch(path, doctype); - }; } export const webapp_client = new Client(); diff --git a/src/packages/frontend/client/project.ts b/src/packages/frontend/client/project.ts index 79ed25713fe..c977d88a8f8 100644 --- a/src/packages/frontend/client/project.ts +++ b/src/packages/frontend/client/project.ts @@ -41,6 +41,8 @@ import { WebappClient } from "./client"; import { throttle } from "lodash"; import { writeFile, type WriteFileOptions } from "@cocalc/conat/files/write"; import { readFile, type ReadFileOptions } from "@cocalc/conat/files/read"; +import { type ProjectApi } from "@cocalc/conat/project/api"; +import { type CopyOptions } from "@cocalc/conat/files/fs"; export class ProjectClient { private client: WebappClient; @@ -50,8 +52,11 @@ export class ProjectClient { this.client = client; } - private conatApi = (project_id: string) => { - return this.client.conat_client.projectApi({ project_id }); + conatApi = (project_id: string, compute_server_id = 0): ProjectApi => { + return this.client.conat_client.projectApi({ + project_id, + compute_server_id, + }); }; // This can write small text files in one message. @@ -118,16 +123,10 @@ export class ProjectClient { return url; }; - copy_path_between_projects = async (opts: { - src_project_id: string; // id of source project - src_path: string; // relative path of director or file in the source project - target_project_id: string; // if of target project - target_path?: string; // defaults to src_path - overwrite_newer?: boolean; // overwrite newer versions of file at destination (destructive) - delete_missing?: boolean; // delete files in dest that are missing from source (destructive) - backup?: boolean; // make ~ backup files instead of overwriting changed files - timeout?: number; // **timeout in milliseconds** -- how long to wait for the copy to complete before reporting "error" (though it could still succeed) - exclude?: string[]; // list of patterns to exclude; this uses exactly the (confusing) rsync patterns + copyPathBetweenProjects = async (opts: { + src: { project_id: string; path: string | string[] }; + dest: { project_id: string; path: string }; + options?: CopyOptions; }): Promise => { await this.client.conat_client.hub.projects.copyPathBetweenProjects(opts); }; @@ -476,6 +475,8 @@ export class ProjectClient { license?: string; // never use pool noPool?: boolean; + // make exact clone of the files from this project: + src_project_id?: string; }): Promise => { const project_id = await this.client.conat_client.hub.projects.createProject(opts); @@ -493,7 +494,7 @@ export class ProjectClient { return (await this.api(opts.project_id)).realpath(opts.path); }; - isdir = async ({ + isDir = async ({ project_id, path, }: { diff --git a/src/packages/frontend/components/error-display.tsx b/src/packages/frontend/components/error-display.tsx index 377d02825f4..eae239d9d96 100644 --- a/src/packages/frontend/components/error-display.tsx +++ b/src/packages/frontend/components/error-display.tsx @@ -3,6 +3,8 @@ * License: MS-RSL – see LICENSE.md for details */ +// DEPRECATED -- the ShowError component in ./error.tsx is much better. + import { Alert } from "antd"; // use "style" to customize diff --git a/src/packages/frontend/components/error.tsx b/src/packages/frontend/components/error.tsx index b8cfd9efd0c..bb53bff9dc8 100644 --- a/src/packages/frontend/components/error.tsx +++ b/src/packages/frontend/components/error.tsx @@ -8,6 +8,7 @@ interface Props { style?: CSSProperties; message?; banner?; + noMarkdown?: boolean; } export default function ShowError({ message = "Error", @@ -15,9 +16,10 @@ export default function ShowError({ setError, style, banner, + noMarkdown, }: Props) { if (!error) return null; - const err = `${error}`.replace(/^Error:/, "").trim(); + const err = `${error}`.replace(/Error:/g, "").trim(); return ( - + {noMarkdown ? err : }
} onClose={() => setError?.("")} diff --git a/src/packages/frontend/components/fake-progress.tsx b/src/packages/frontend/components/fake-progress.tsx index d634e14ad5a..1b0f10ef854 100644 --- a/src/packages/frontend/components/fake-progress.tsx +++ b/src/packages/frontend/components/fake-progress.tsx @@ -23,7 +23,6 @@ export default function FakeProgress({ time }) { return ( null} percent={percent} strokeColor={{ "0%": "#108ee9", "100%": "#87d068" }} diff --git a/src/packages/frontend/components/loading.tsx b/src/packages/frontend/components/loading.tsx index 8e0dac07694..5b1578de9b5 100644 --- a/src/packages/frontend/components/loading.tsx +++ b/src/packages/frontend/components/loading.tsx @@ -19,9 +19,9 @@ export const Estimate = null; // webpack + TS es2020 modules need this interface Props { style?: CSSProperties; text?: string; - estimate?: Estimate; + estimate?: Estimate | number; theme?: "medium" | undefined; - delay?: number; // if given, don't show anything until after delay milliseconds. The component could easily unmount by then, and hence never annoyingly flicker on screen. + delay?: number; // (default:1000) don't show anything until after delay milliseconds. The component could easily unmount by then, and hence never annoyingly flicker on screen. transparent?: boolean; } @@ -40,7 +40,7 @@ export function Loading({ text, estimate, theme, - delay, + delay = 1000, transparent = false, }: Props) { const intl = useIntl(); @@ -64,7 +64,13 @@ export function Loading({ {estimate != undefined && (
- +
)} diff --git a/src/packages/frontend/components/search-input.tsx b/src/packages/frontend/components/search-input.tsx index e3839ca547b..88d017315a0 100644 --- a/src/packages/frontend/components/search-input.tsx +++ b/src/packages/frontend/components/search-input.tsx @@ -10,13 +10,7 @@ */ import { Input, InputRef } from "antd"; - -import { - React, - useEffect, - useRef, - useState, -} from "@cocalc/frontend/app-framework"; +import { CSSProperties, useEffect, useRef, useState } from "react"; interface Props { size?; @@ -31,21 +25,39 @@ interface Props { on_down?: () => void; on_up?: () => void; on_escape?: (value: string) => void; - style?: React.CSSProperties; - input_class?: string; + style?: CSSProperties; autoFocus?: boolean; autoSelect?: boolean; placeholder?: string; - focus?: number; // if this changes, focus the search box. + focus?; // if this changes, focus the search box. status?: "warning" | "error"; } -export const SearchInput: React.FC = React.memo((props) => { - const [value, setValue] = useState( - props.value ?? props.default_value ?? "", - ); +export function SearchInput({ + size, + default_value, + value: value0, + on_change, + on_clear, + on_submit, + buttonAfter, + disabled, + clear_on_submit, + on_down, + on_up, + on_escape, + style, + autoFocus, + autoSelect, + placeholder, + focus, + status, +}: Props) { + const [value, setValue] = useState(value0 ?? default_value ?? ""); // if value changes, we update as well! - useEffect(() => setValue(props.value ?? ""), [props.value]); + useEffect(() => { + setValue(value0 ?? ""); + }, [value0]); const [ctrl_down, set_ctrl_down] = useState(false); const [shift_down, set_shift_down] = useState(false); @@ -53,7 +65,7 @@ export const SearchInput: React.FC = React.memo((props) => { const input_ref = useRef(null); useEffect(() => { - if (props.autoSelect && input_ref.current) { + if (autoSelect && input_ref.current) { try { input_ref.current?.select(); } catch (_) {} @@ -71,20 +83,20 @@ export const SearchInput: React.FC = React.memo((props) => { function clear_value(): void { setValue(""); - props.on_change?.("", get_opts()); - props.on_clear?.(); + on_change?.("", get_opts()); + on_clear?.(); } function submit(e?): void { if (e != null) { e.preventDefault(); } - if (typeof props.on_submit === "function") { - props.on_submit(value, get_opts()); + if (typeof on_submit === "function") { + on_submit(value, get_opts()); } - if (props.clear_on_submit) { + if (clear_on_submit) { clear_value(); - props.on_change?.(value, get_opts()); + on_change?.(value, get_opts()); } } @@ -94,10 +106,10 @@ export const SearchInput: React.FC = React.memo((props) => { escape(); break; case 40: - props.on_down?.(); + on_down?.(); break; case 38: - props.on_up?.(); + on_up?.(); break; case 17: set_ctrl_down(true); @@ -120,35 +132,35 @@ export const SearchInput: React.FC = React.memo((props) => { } function escape(): void { - if (typeof props.on_escape === "function") { - props.on_escape(value); + if (typeof on_escape === "function") { + on_escape(value); } clear_value(); } return ( { e.preventDefault(); const value = e.target?.value ?? ""; setValue(value); - props.on_change?.(value, get_opts()); + on_change?.(value, get_opts()); if (!value) clear_value(); }} onKeyDown={key_down} onKeyUp={key_up} - disabled={props.disabled} - enterButton={props.buttonAfter} - status={props.status} + disabled={disabled} + enterButton={buttonAfter} + status={status} /> ); -}); +} diff --git a/src/packages/frontend/components/time-ago.tsx b/src/packages/frontend/components/time-ago.tsx index 7259a5a75a9..00352b38d2a 100644 --- a/src/packages/frontend/components/time-ago.tsx +++ b/src/packages/frontend/components/time-ago.tsx @@ -166,6 +166,9 @@ export const TimeAgoElement: React.FC = ({ } const d = is_date(date) ? (date as Date) : new Date(date); + if (!d.valueOf()) { + return null; + } try { d.toISOString(); } catch (error) { @@ -203,7 +206,7 @@ export const TimeAgo: React.FC = React.memo( }: TimeAgoElementProps) => { const { timeAgoAbsolute } = useAppContext(); - if (date == null) { + if (!date?.valueOf()) { return <>; } diff --git a/src/packages/frontend/components/virtuoso-scroll-hook.ts b/src/packages/frontend/components/virtuoso-scroll-hook.ts index fab8ed41c92..43a20506235 100644 --- a/src/packages/frontend/components/virtuoso-scroll-hook.ts +++ b/src/packages/frontend/components/virtuoso-scroll-hook.ts @@ -15,6 +15,8 @@ the upstream Virtuoso project: https://github.com/petyosi/react-virtuoso/blob/m import LRU from "lru-cache"; import { useCallback, useMemo, useRef } from "react"; +const DEFAULT_VIEWPORT = 1000; + export interface ScrollState { index: number; offset: number; @@ -46,7 +48,7 @@ export default function useVirtuosoScrollHook({ }, []); if (disabled) return {}; const lastScrollRef = useRef( - initialState ?? { index: 0, offset: 0 } + initialState ?? { index: 0, offset: 0 }, ); const recordScrollState = useMemo(() => { return (state: ScrollState) => { @@ -64,8 +66,9 @@ export default function useVirtuosoScrollHook({ }, [onScroll, cacheId]); return { + increaseViewportBy: DEFAULT_VIEWPORT, initialTopMostItemIndex: - (cacheId ? cache.get(cacheId) ?? initialState : initialState) ?? 0, + (cacheId ? (cache.get(cacheId) ?? initialState) : initialState) ?? 0, scrollerRef: handleScrollerRef, onScroll: () => { const scrollTop = scrollerRef.current?.scrollTop; diff --git a/src/packages/frontend/compute/clone.tsx b/src/packages/frontend/compute/clone.tsx index 6c7c2815260..c2c5dcc6961 100644 --- a/src/packages/frontend/compute/clone.tsx +++ b/src/packages/frontend/compute/clone.tsx @@ -78,7 +78,7 @@ export async function cloneConfiguration({ const server = servers[0] as ComputeServerUserInfo; if (!noChange) { let n = 1; - let title = `Clone of ${server.title}`; + let title = `Fork of ${server.title}`; const titles = new Set(servers.map((x) => x.title)); if (titles.has(title)) { while (titles.has(title + ` (${n})`)) { diff --git a/src/packages/frontend/compute/compute-server.tsx b/src/packages/frontend/compute/compute-server.tsx index a354cc29669..555308c8f56 100644 --- a/src/packages/frontend/compute/compute-server.tsx +++ b/src/packages/frontend/compute/compute-server.tsx @@ -1,6 +1,5 @@ import { Button, Card, Divider, Modal, Popconfirm, Spin } from "antd"; import { CSSProperties, useMemo, useState } from "react"; - import { useTypedRedux } from "@cocalc/frontend/app-framework"; import { Icon } from "@cocalc/frontend/components"; import ShowError from "@cocalc/frontend/components/error"; diff --git a/src/packages/frontend/conat/client.ts b/src/packages/frontend/conat/client.ts index 628522f8d20..f2302d0ca95 100644 --- a/src/packages/frontend/conat/client.ts +++ b/src/packages/frontend/conat/client.ts @@ -9,9 +9,8 @@ import { randomId, inboxPrefix } from "@cocalc/conat/names"; import { projectSubject } from "@cocalc/conat/names"; import { parseQueryWithOptions } from "@cocalc/sync/table/util"; import { type HubApi, initHubApi } from "@cocalc/conat/hub/api"; -import { type ProjectApi, initProjectApi } from "@cocalc/conat/project/api"; +import { type ProjectApi, projectApiClient } from "@cocalc/conat/project/api"; import { isValidUUID } from "@cocalc/util/misc"; -import { createOpenFiles, OpenFiles } from "@cocalc/conat/sync/open-files"; import { PubSub } from "@cocalc/conat/sync/pubsub"; import type { ChatOptions } from "@cocalc/util/types/llm"; import { dkv } from "@cocalc/conat/sync/dkv"; @@ -46,6 +45,7 @@ import { deleteRememberMe, setRememberMe, } from "@cocalc/frontend/misc/remember-me"; +import { client as projectRunnerClient } from "@cocalc/conat/project/runner/run"; export interface ConatConnectionStatus { state: "connected" | "disconnected"; @@ -62,7 +62,6 @@ export class ConatClient extends EventEmitter { client: WebappClient; public hub: HubApi; public sessionId = randomId(); - private openFilesCache: { [project_id: string]: OpenFiles } = {}; private clientWithState: ClientWithState; private _conatClient: null | ReturnType; public numConnectionAttempts = 0; @@ -262,7 +261,7 @@ export class ConatClient extends EventEmitter { console.log( `Connecting to ${this._conatClient?.options.address}: attempts ${attempts}`, ); - this._conatClient?.conn.io.connect(); + this._conatClient?.connect(); return false; }, { min: 3000, max: 15000 }, @@ -314,7 +313,13 @@ export class ConatClient extends EventEmitter { const resp = await cn.request(subject, data, { timeout }); return resp.data; } catch (err) { - err.message = `${err.message} - callHub: subject='${subject}', name='${name}', `; + try { + err.message = `${err.message} - callHub: subject='${subject}', name='${name}', `; + } catch { + err = new Error( + `${err.message} - callHub: subject='${subject}', name='${name}', `, + ); + } throw err; } }; @@ -348,54 +353,14 @@ export class ConatClient extends EventEmitter { compute_server_id = actions.getComputeServerId(); } } - const callProjectApi = async ({ name, args }) => { - return await this.callProject({ - project_id, - compute_server_id, - timeout, - service: "api", - name, - args, - }); - }; - return initProjectApi(callProjectApi); - }; - - private callProject = async ({ - service = "api", - project_id, - compute_server_id, - name, - args = [], - timeout = DEFAULT_TIMEOUT, - }: { - service?: string; - project_id: string; - compute_server_id?: number; - name: string; - args: any[]; - timeout?: number; - }) => { - const cn = this.conat(); - const subject = projectSubject({ project_id, compute_server_id, service }); - const resp = await cn.request( - subject, - { name, args }, - // we use waitForInterest because often the project hasn't - // quite fully started. - { timeout, waitForInterest: true }, - ); - return resp.data; + return projectApiClient({ project_id, compute_server_id, timeout }); }; synctable: ConatSyncTableFunction = async ( query0, options?, ): Promise => { - const { query, table } = parseQueryWithOptions(query0, options); - if (options?.project_id != null && query[table][0]["project_id"] === null) { - query[table][0]["project_id"] = options.project_id; - } + const { query } = parseQueryWithOptions(query0, options); return await this.conat().sync.synctable({ ...options, query, @@ -425,42 +390,6 @@ export class ConatClient extends EventEmitter { }); }; - openFiles = reuseInFlight(async (project_id: string) => { - if (this.openFilesCache[project_id] == null) { - const openFiles = await createOpenFiles({ - project_id, - }); - this.openFilesCache[project_id] = openFiles; - openFiles.on("closed", () => { - delete this.openFilesCache[project_id]; - }); - openFiles.on("change", (entry) => { - if (entry.deleted?.deleted) { - setDeleted({ - project_id, - path: entry.path, - deleted: entry.deleted.time, - }); - } else { - setNotDeleted({ project_id, path: entry.path }); - } - }); - const recentlyDeletedPaths: any = {}; - for (const { path, deleted } of openFiles.getAll()) { - if (deleted?.deleted) { - recentlyDeletedPaths[path] = deleted.time; - } - } - const store = redux.getProjectStore(project_id); - store.setState({ recentlyDeletedPaths }); - } - return this.openFilesCache[project_id]!; - }); - - closeOpenFiles = (project_id) => { - this.openFilesCache[project_id]?.close(); - }; - pubsub = async ({ project_id, path, @@ -515,22 +444,13 @@ export class ConatClient extends EventEmitter { }; refCacheInfo = () => refCacheInfo(); -} - -function setDeleted({ project_id, path, deleted }) { - if (!redux.hasProjectStore(project_id)) { - return; - } - const actions = redux.getProjectActions(project_id); - actions.setRecentlyDeleted(path, deleted); -} -function setNotDeleted({ project_id, path }) { - if (!redux.hasProjectStore(project_id)) { - return; - } - const actions = redux.getProjectActions(project_id); - actions?.setRecentlyDeleted(path, 0); + projectRunner = (project_id: string) => { + return projectRunnerClient({ + subject: `project.${project_id}.run`, + client: this.conat(), + }); + }; } async function waitForOnline(): Promise { diff --git a/src/packages/frontend/course/assignments/actions.ts b/src/packages/frontend/course/assignments/actions.ts index a50934da009..cd0197b558f 100644 --- a/src/packages/frontend/course/assignments/actions.ts +++ b/src/packages/frontend/course/assignments/actions.ts @@ -8,7 +8,7 @@ Actions involving working with assignments: - assigning, collecting, setting feedback, etc. */ -import { delay, map } from "awaiting"; +import { map } from "awaiting"; import { Map } from "immutable"; import { debounce } from "lodash"; import { join } from "path"; @@ -39,7 +39,6 @@ import { uuid, } from "@cocalc/util/misc"; import { CourseActions } from "../actions"; -import { COPY_TIMEOUT_MS } from "../consts"; import { export_assignment } from "../export/export-assignment"; import { export_student_file_use_times } from "../export/file-use-times"; import { grading_state } from "../nbgrader/util"; @@ -456,15 +455,13 @@ export class AssignmentsActions { desc: `Copying assignment from ${student_name}`, }); try { - await webapp_client.project_client.copy_path_between_projects({ - src_project_id: student_project_id, - src_path: assignment.get("target_path"), - target_project_id: store.get("course_project_id"), - target_path, - overwrite_newer: true, - backup: true, - delete_missing: false, - timeout: COPY_TIMEOUT_MS, + await webapp_client.project_client.copyPathBetweenProjects({ + src: { + project_id: student_project_id, + path: assignment.get("target_path"), + }, + dest: { project_id: store.get("course_project_id"), path: target_path }, + options: { recursive: true }, }); // write their name to a file const name = store.get_student_name_extra(student_id); @@ -615,17 +612,25 @@ ${details} path: src_path + "/GRADE.md", content, }); - await webapp_client.project_client.copy_path_between_projects({ - src_project_id: store.get("course_project_id"), - src_path, - target_project_id: student_project_id, - target_path: assignment.get("graded_path"), - overwrite_newer: true, - backup: true, - delete_missing: false, - exclude: peer_graded ? ["*GRADER*.txt"] : undefined, - timeout: COPY_TIMEOUT_MS, + await webapp_client.project_client.copyPathBetweenProjects({ + src: { project_id: store.get("course_project_id"), path: src_path }, + dest: { + project_id: student_project_id, + path: assignment.get("graded_path"), + }, + options: { + recursive: true, + }, }); + if (peer_graded) { + const actions = redux.getProjectActions(student_project_id); + await actions.deleteMatchingFiles({ + path: assignment.get("graded_path"), + recursive: true, + compute_server_id: 0, + filter: (p) => p.includes("GRADER"), + }); + } finish(""); } catch (err) { finish(err); @@ -853,21 +858,19 @@ ${details} desc: `Copying files to ${student_name}'s project`, }); const opts = { - src_project_id: store.get("course_project_id"), - src_path, - target_project_id: student_project_id, - target_path: assignment.get("target_path"), - overwrite_newer: !!overwrite, // default is "false" - delete_missing: !!overwrite, // default is "false" - backup: !!!overwrite, // default is "true" - timeout: COPY_TIMEOUT_MS, + src: { project_id: store.get("course_project_id"), path: src_path }, + dest: { + project_id: student_project_id, + path: assignment.get("target_path"), + }, + options: { recursive: true, force: !!overwrite }, }; - await webapp_client.project_client.copy_path_between_projects(opts); + await webapp_client.project_client.copyPathBetweenProjects(opts); await this.course_actions.compute.setComputeServerAssociations({ student_id, - src_path: opts.src_path, - target_project_id: opts.target_project_id, - target_path: opts.target_path, + src_path, + target_project_id: student_project_id, + target_path: assignment.get("target_path"), unit_id: assignment_id, }); @@ -1007,28 +1010,6 @@ ${details} }); }; - private start_all_for_peer_grading = async (): Promise => { - // On cocalc.com, if the student projects get started specifically - // for the purposes of copying files to/from them, then they stop - // around a minute later. This is very bad for peer grading, since - // so much copying occurs, and we end up with conflicts between - // projects starting to peer grade, then stop, then needing to be - // started again all at once. We thus request that they all start, - // wait a few seconds for that "reason" for them to be running to - // take effect, and then do the copy. This way the projects aren't - // automatically stopped after the copies happen. - const id = this.course_actions.set_activity({ - desc: "Warming up all student projects for peer grading...", - }); - this.course_actions.student_projects.action_all_student_projects("start"); - // We request to start all projects simultaneously, and the system - // will start doing that. I think it's not so much important that - // the projects are actually running, but that they were started - // before the copy operations started. - await delay(5 * 1000); - this.course_actions.clear_activity(id); - }; - async peer_copy_to_all_students( assignment_id: string, new_only: boolean, @@ -1049,7 +1030,6 @@ ${details} this.course_actions.set_error(`${short_desc} -- ${err}`); return; } - await this.start_all_for_peer_grading(); // OK, now do the assignment... in parallel. await this.assignment_action_all_students({ assignment_id, @@ -1070,7 +1050,6 @@ ${details} desc += " from whom we have not already copied it"; } const short_desc = "copy peer grading from students"; - await this.start_all_for_peer_grading(); await this.assignment_action_all_students({ assignment_id, new_only, @@ -1370,15 +1349,19 @@ ${details} // peer grading is anonymous; also, remove original // due date to avoid confusion. // copy the files to be peer graded into place for this student - await webapp_client.project_client.copy_path_between_projects({ - src_project_id: store.get("course_project_id"), - src_path, - target_project_id: student_project_id, - target_path, - overwrite_newer: false, - delete_missing: false, - exclude: ["*STUDENT*.txt", "*" + DUE_DATE_FILENAME + "*"], - timeout: COPY_TIMEOUT_MS, + await webapp_client.project_client.copyPathBetweenProjects({ + src: { project_id: store.get("course_project_id"), path: src_path }, + dest: { project_id: student_project_id, path: target_path }, + options: { recursive: true, force: false }, + }); + + const actions = redux.getProjectActions(student_project_id); + await actions.deleteMatchingFiles({ + path: target_path, + recursive: true, + compute_server_id: 0, + filter: (path) => + path.includes("STUDENT") || path.includes(DUE_DATE_FILENAME), }); }; @@ -1454,14 +1437,10 @@ ${details} } // copy the files over from the student who did the peer grading - await webapp_client.project_client.copy_path_between_projects({ - src_project_id, - src_path, - target_project_id: store.get("course_project_id"), - target_path, - overwrite_newer: false, - delete_missing: false, - timeout: COPY_TIMEOUT_MS, + await webapp_client.project_client.copyPathBetweenProjects({ + src: { project_id: src_project_id, path: src_path }, + dest: { project_id: store.get("course_project_id"), path: target_path }, + options: { force: false, recursive: true }, }); // write local file identifying the grader @@ -1598,7 +1577,7 @@ ${details} let has_student_subdir: boolean = false; for (const entry of listing) { - if (entry.isdir && entry.name == STUDENT_SUBDIR) { + if (entry.isDir && entry.name == STUDENT_SUBDIR) { has_student_subdir = true; break; } @@ -1641,10 +1620,8 @@ ${details} const project_id = store.get("course_project_id"); let files; try { - files = await redux - .getProjectStore(project_id) - .get_listings() - .getListingDirectly(path); + const { fs } = this.course_actions.syncdb; + files = await fs.readdir(path, { withFileTypes: true }); } catch (err) { // This happens, e.g., if the instructor moves the directory // that contains their version of the ipynb file. @@ -1658,7 +1635,7 @@ ${details} if (this.course_actions.is_closed()) return result; const to_read = files - .filter((entry) => !entry.isdir && endswith(entry.name, ".ipynb")) + .filter((entry) => entry.isFile() && endswith(entry.name, ".ipynb")) .map((entry) => entry.name); const f: (file: string) => Promise = async (file) => { @@ -2008,15 +1985,10 @@ ${details} // This is necessary because grading the assignment may depend on // data files that are sent as part of the assignment. Also, // student's might have some code in text files next to the ipynb. - await webapp_client.project_client.copy_path_between_projects({ - src_project_id: course_project_id, - src_path: student_path, - target_project_id: grade_project_id, - target_path: student_path, - overwrite_newer: true, - delete_missing: true, - backup: false, - timeout: COPY_TIMEOUT_MS, + await webapp_client.project_client.copyPathBetweenProjects({ + src: { project_id: course_project_id, path: student_path }, + dest: { project_id: grade_project_id, path: student_path }, + options: { recursive: true }, }); } else { ephemeralGradePath = false; diff --git a/src/packages/frontend/course/assignments/assignment.tsx b/src/packages/frontend/course/assignments/assignment.tsx index 276aaa1364b..6ae1f5a8c8c 100644 --- a/src/packages/frontend/course/assignments/assignment.tsx +++ b/src/packages/frontend/course/assignments/assignment.tsx @@ -407,7 +407,7 @@ export function Assignment({ ); } - function open_assignment_path(): void { + async function open_assignment_path() { if (assignment.get("listing")?.size == 0) { // there are no files yet, so we *close* the assignment // details panel. This is just **a hack** so that the user @@ -421,7 +421,7 @@ export function Assignment({ assignment.get("assignment_id"), ); } - return redux + await redux .getProjectActions(project_id) .open_directory(assignment.get("path")); } @@ -704,11 +704,8 @@ export function Assignment({ return ( This will recopy all of the files to them. CAUTION: if you update a - file that a student has also worked on, their work will get copied - to a backup file ending in a tilde, or possibly only be available in - snapshots. Select "Replace student files!" in case you do not{" "} - want to create any backups and also delete all other files in - the assignment folder of their projects.{" "} + file that a student has also worked on, their work will get + overwritten. They can use TimeTravel to get it back. ); case "collect": - return "This will recollect all of the homework from them. CAUTION: if you have graded/edited a file that a student has updated, your work will get copied to a backup file ending in a tilde, or possibly only be available in snapshots."; + return "This will recollect all of the homework from them. CAUTION: if you have graded/edited a file that a student has updated, your work will get overwritten. Use TimeTravel to get it back."; case "return_graded": return "This will rereturn all of the graded files to them."; case "peer_assignment": - return "This will recopy all of the files to them. CAUTION: if there is a file a student has also worked on grading, their work will get copied to a backup file ending in a tilde, or possibly be only available in snapshots."; + return "This will recopy all of the files to them. CAUTION: if there is a file a student has also worked on grading, their work will get overwritten. Use TimeTravel to get it back."; case "peer_collect": - return "This will recollect all of the peer-graded homework from the students. CAUTION: if you have graded/edited a previously collected file that a student has updated, your work will get copied to a backup file ending in a tilde, or possibly only be available in snapshots."; + return "This will recollect all of the peer-graded homework from the students. CAUTION: if you have graded/edited a previously collected file that a student has updated, your work will get overwritten. Use TimeTravel to get it back."; } } @@ -743,18 +740,8 @@ export function Assignment({ disabled={copy_assignment_confirm_overwrite} onClick={() => copy_assignment(step, false)} > - Yes, do it (with backup) + Yes. Replaces student files! - {step === "assignment" ? ( - - ) : undefined} {render_copy_assignment_confirm_overwrite(step)} diff --git a/src/packages/frontend/course/common/student-assignment-info.tsx b/src/packages/frontend/course/common/student-assignment-info.tsx index 507c6db86c8..17a06ad9461 100644 --- a/src/packages/frontend/course/common/student-assignment-info.tsx +++ b/src/packages/frontend/course/common/student-assignment-info.tsx @@ -10,7 +10,6 @@ import { FormattedMessage, useIntl } from "react-intl"; import { useActions } from "@cocalc/frontend/app-framework"; import { Gap, Icon, Markdown, Tip } from "@cocalc/frontend/components"; import ShowError from "@cocalc/frontend/components/error"; -import { COPY_TIMEOUT_MS } from "@cocalc/frontend/course/consts"; import { MarkdownInput } from "@cocalc/frontend/editors/markdown-input"; import { labels } from "@cocalc/frontend/i18n"; import { NotebookScores } from "@cocalc/frontend/jupyter/nbgrader/autograde"; @@ -95,7 +94,7 @@ export function StudentAssignmentInfo({ nbgrader_run_info, }: StudentAssignmentInfoProps) { const intl = useIntl(); - const clicked_nbgrader = useRef(undefined); + const clicked_nbgrader = useRef(undefined); const actions = useActions({ name }); const size = useButtonSize(); const [recopy, set_recopy] = useRecopy(); @@ -512,7 +511,7 @@ export function StudentAssignmentInfo({ const do_stop = () => stop(type, info.assignment_id, info.student_id); const v: React.JSX.Element[] = []; if (enable_copy) { - if (webapp_client.server_time() - (data.start ?? 0) < COPY_TIMEOUT_MS) { + if (webapp_client.server_time() - (data.start ?? 0) < 15_000) { v.push(render_open_copying(step, do_open, do_stop)); } else if (data.time) { v.push( diff --git a/src/packages/frontend/course/configuration/configuration-copying.tsx b/src/packages/frontend/course/configuration/configuration-copying.tsx index 4510ad0f238..41df5121e60 100644 --- a/src/packages/frontend/course/configuration/configuration-copying.tsx +++ b/src/packages/frontend/course/configuration/configuration-copying.tsx @@ -35,19 +35,15 @@ import { } from "antd"; import { useMemo, useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; - import { labels } from "@cocalc/frontend/i18n"; import { redux, useFrameContext, - useTypedRedux, } from "@cocalc/frontend/app-framework"; import { Icon } from "@cocalc/frontend/components"; import ShowError from "@cocalc/frontend/components/error"; import { COMMANDS } from "@cocalc/frontend/course/commands"; -import { exec } from "@cocalc/frontend/frame-editors/generic/client"; import { IntlMessage } from "@cocalc/frontend/i18n"; -import { pathExists } from "@cocalc/frontend/project/directory-selector"; import { ProjectTitle } from "@cocalc/frontend/projects/project-title"; import { isIntlMessage } from "@cocalc/util/i18n"; import { plural } from "@cocalc/util/misc"; @@ -446,10 +442,6 @@ function AddTarget({ settings, actions, project_id }) { const [path, setPath] = useState(""); const [error, setError] = useState(""); const [create, setCreate] = useState(""); - const directoryListings = useTypedRedux( - { project_id }, - "directory_listings", - )?.get(0); const add = async () => { try { @@ -458,19 +450,14 @@ function AddTarget({ settings, actions, project_id }) { throw Error(`'${path} is the current course'`); } setLoading(true); - const exists = await pathExists(project_id, path, directoryListings); - if (!exists) { + const projectActions = redux.getProjectActions(project_id); + const fs = projectActions.fs(); + if (!(await fs.exists(path))) { if (create) { - await exec({ - command: "touch", - args: [path], - project_id, - filesystem: true, - }); - } else { - setCreate(path); - return; + await fs.writeFile(path, ""); } + } else { + setCreate(path); } const copy_config_targets = getTargets(settings); copy_config_targets[`${project_id}/${path}`] = true; diff --git a/src/packages/frontend/course/consts.ts b/src/packages/frontend/course/consts.ts deleted file mode 100644 index 96e0001fb19..00000000000 --- a/src/packages/frontend/course/consts.ts +++ /dev/null @@ -1,6 +0,0 @@ -// All copy operations (e.g., assigning, collecting, etc.) is set to timeout after this long. -// Also in the UI displaying that a copy is ongoing also times out after this long, e.g, if -// the user refreshes their browser and nothing is going to update things again. -// TODO: make this a configurable parameter, e.g., maybe users have very large assignments -// or things are very slow. -export const COPY_TIMEOUT_MS = 2 * 60 * 1000; // 2 minutes, for now -- starting project can take time. diff --git a/src/packages/frontend/course/export/export-assignment.ts b/src/packages/frontend/course/export/export-assignment.ts index 0ab46b502ea..35e14ea09bb 100644 --- a/src/packages/frontend/course/export/export-assignment.ts +++ b/src/packages/frontend/course/export/export-assignment.ts @@ -82,7 +82,7 @@ async function export_one_directory( let x: any; const timeout = 60; // 60 seconds for (x of listing) { - if (x.isdir) continue; // we ignore subdirectories... + if (x.isDir) continue; // we ignore subdirectories... const { name } = x; if (startswith(name, "STUDENT")) continue; if (startswith(name, ".")) continue; diff --git a/src/packages/frontend/course/handouts/actions.ts b/src/packages/frontend/course/handouts/actions.ts index 302e074bab0..8f604f14c1f 100644 --- a/src/packages/frontend/course/handouts/actions.ts +++ b/src/packages/frontend/course/handouts/actions.ts @@ -16,7 +16,6 @@ import { map } from "awaiting"; import type { SyncDBRecordHandout } from "../types"; import { exec } from "../../frame-editors/generic/client"; import { export_student_file_use_times } from "../export/file-use-times"; -import { COPY_TIMEOUT_MS } from "../consts"; export class HandoutsActions { private course_actions: CourseActions; @@ -253,22 +252,20 @@ export class HandoutsActions { }); const opts = { - src_project_id: course_project_id, - src_path, - target_project_id: student_project_id, - target_path: handout.get("target_path"), - overwrite_newer: !!overwrite, // default is "false" - delete_missing: !!overwrite, // default is "false" - backup: !!!overwrite, // default is "true" - timeout: COPY_TIMEOUT_MS, + src: { project_id: course_project_id, path: src_path }, + dest: { + project_id: student_project_id, + path: handout.get("target_path"), + }, + options: { force: !!overwrite }, }; - await webapp_client.project_client.copy_path_between_projects(opts); + await webapp_client.project_client.copyPathBetweenProjects(opts); await this.course_actions.compute.setComputeServerAssociations({ student_id, - src_path: opts.src_path, - target_project_id: opts.target_project_id, - target_path: opts.target_path, + src_path, + target_project_id: student_project_id, + target_path: handout.get("target_path"), unit_id: handout_id, }); diff --git a/src/packages/frontend/course/handouts/handout.tsx b/src/packages/frontend/course/handouts/handout.tsx index aaf3ab001e2..be744611ee3 100644 --- a/src/packages/frontend/course/handouts/handout.tsx +++ b/src/packages/frontend/course/handouts/handout.tsx @@ -324,8 +324,7 @@ export function Handout({ case "handout": return `\ This will recopy all of the files to them. - CAUTION: if you update a file that a student has also worked on, their work will get copied to a backup file ending in a tilde, or possibly only be available in snapshots. - Select "Replace student files!" in case you do not want to create any backups and also delete all other files in the handout directory of their projects.\ + CAUTION: if you update a file that a student has also worked on, their work will get overwritten. They can recover it using TimeTravel.\ `; } } diff --git a/src/packages/frontend/course/handouts/handouts-info-panel.tsx b/src/packages/frontend/course/handouts/handouts-info-panel.tsx index cfdf4d09020..f4999c8f772 100644 --- a/src/packages/frontend/course/handouts/handouts-info-panel.tsx +++ b/src/packages/frontend/course/handouts/handouts-info-panel.tsx @@ -11,7 +11,6 @@ import { useIntl } from "react-intl"; import { Icon, Tip } from "@cocalc/frontend/components"; import ShowError from "@cocalc/frontend/components/error"; -import { COPY_TIMEOUT_MS } from "@cocalc/frontend/course/consts"; import { labels } from "@cocalc/frontend/i18n"; import { webapp_client } from "@cocalc/frontend/webapp-client"; import { CourseActions } from "../actions"; @@ -155,7 +154,7 @@ export function StudentHandoutInfo({ } const v: any[] = []; if (enable_copy) { - if (webapp_client.server_time() - (obj.start ?? 0) < COPY_TIMEOUT_MS) { + if (webapp_client.server_time() - (obj.start ?? 0) < 15_000) { v.push(render_open_copying(do_open, do_stop)); } else if (obj.time) { v.push(render_open_recopy(name, do_open, do_copy, copy_tip, open_tip)); diff --git a/src/packages/frontend/course/shared-project/actions.ts b/src/packages/frontend/course/shared-project/actions.ts index 3b6254fcd21..918ef34f6fe 100644 --- a/src/packages/frontend/course/shared-project/actions.ts +++ b/src/packages/frontend/course/shared-project/actions.ts @@ -176,10 +176,10 @@ export class SharedProjectActions { if (!shared_project_id) { return; // no shared project } - const dflt_img = await redux.getStore("customize").getDefaultComputeImage(); - const img_id = store.get("settings").get("custom_image") ?? dflt_img; + const defaultImage = await redux.getStore("customize").getDefaultComputeImage(); + const imageId = store.get("settings").get("custom_image") ?? defaultImage; const actions = redux.getProjectActions(shared_project_id); - await actions.set_compute_image(img_id); + await actions.set_compute_image(imageId); }; set_datastore_and_envvars = async (): Promise => { diff --git a/src/packages/frontend/course/student-projects/actions.ts b/src/packages/frontend/course/student-projects/actions.ts index aeb9ea37a3d..a173dda6c09 100644 --- a/src/packages/frontend/course/student-projects/actions.ts +++ b/src/packages/frontend/course/student-projects/actions.ts @@ -66,13 +66,13 @@ export class StudentProjectsActions { const id = this.course_actions.set_activity({ desc: `Create project for ${store.get_student_name(student_id)}.`, }); - const dflt_img = await redux.getStore("customize").getDefaultComputeImage(); + const defaultImage = await redux.getStore("customize").getDefaultComputeImage(); let project_id: string; try { project_id = await redux.getActions("projects").create_project({ title: store.get("settings").get("title"), description: store.get("settings").get("description"), - image: store.get("settings").get("custom_image") ?? dflt_img, + image: store.get("settings").get("custom_image") ?? defaultImage, noPool: true, // student is unlikely to use the project right *now* }); } catch (err) { @@ -607,8 +607,8 @@ export class StudentProjectsActions { ): Promise => { const store = this.get_store(); if (store == null) return; - const dflt_img = await redux.getStore("customize").getDefaultComputeImage(); - const img_id = store.get("settings").get("custom_image") ?? dflt_img; + const defaultImage = await redux.getStore("customize").getDefaultComputeImage(); + const img_id = store.get("settings").get("custom_image") ?? defaultImage; const actions = redux.getProjectActions(student_project_id); await actions.set_compute_image(img_id); }; diff --git a/src/packages/frontend/course/sync.ts b/src/packages/frontend/course/sync.ts index 69a470e1f81..6fbfe3df11c 100644 --- a/src/packages/frontend/course/sync.ts +++ b/src/packages/frontend/course/sync.ts @@ -31,7 +31,7 @@ export function create_sync_db( const path = store.get("course_filename"); actions.setState({ loading: true }); - const syncdb = webapp_client.sync_client.sync_db({ + const syncdb = webapp_client.conat_client.conat().sync.db({ project_id, path, primary_keys: ["table", "handout_id", "student_id", "assignment_id"], diff --git a/src/packages/frontend/cspell.json b/src/packages/frontend/cspell.json index 31ca5324a6c..d2cb00db40b 100644 --- a/src/packages/frontend/cspell.json +++ b/src/packages/frontend/cspell.json @@ -52,6 +52,7 @@ "immutablejs", "ipynb", "isdir", + "isDir", "kernelspec", "LLM", "LLMs", diff --git a/src/packages/frontend/custom-software/reset-bar.tsx b/src/packages/frontend/custom-software/reset-bar.tsx index 2442a38bbe9..445b9479007 100644 --- a/src/packages/frontend/custom-software/reset-bar.tsx +++ b/src/packages/frontend/custom-software/reset-bar.tsx @@ -6,15 +6,15 @@ import { Button as AntdButton, Card } from "antd"; import React from "react"; import { FormattedMessage, useIntl } from "react-intl"; - import { A, Icon } from "@cocalc/frontend/components"; import { labels } from "@cocalc/frontend/i18n"; import { ProjectMap } from "@cocalc/frontend/todo-types"; import { COLORS, SITE_NAME } from "@cocalc/util/theme"; - import { Available as AvailableFeatures } from "../project_configuration"; import { ComputeImages } from "./init"; import { props2img, RESET_ICON } from "./util"; +import { useTypedRedux } from "@cocalc/frontend/app-framework"; +import { type ProjectActions } from "@cocalc/frontend/project_store"; const doc_snap = "https://doc.cocalc.com/project-files.html#snapshots"; const doc_tt = "https://doc.cocalc.com/time-travel.html"; @@ -36,13 +36,13 @@ interface Props { project_id: string; images: ComputeImages; project_map?: ProjectMap; - actions: any; + actions: ProjectActions; available_features?: AvailableFeatures; - site_name?: string; } export const CustomSoftwareReset: React.FC = (props: Props) => { - const { actions, site_name } = props; + const { actions } = props; + const site_name = useTypedRedux("customize", "site_name"); const intl = useIntl(); diff --git a/src/packages/frontend/custom-software/selector.tsx b/src/packages/frontend/custom-software/selector.tsx index 7e2f716369e..5e7c9309d51 100644 --- a/src/packages/frontend/custom-software/selector.tsx +++ b/src/packages/frontend/custom-software/selector.tsx @@ -3,7 +3,7 @@ * License: MS-RSL – see LICENSE.md for details */ -// cSpell:ignore descr disp dflt +// cSpell:ignore descr disp import { Col, Form } from "antd"; import { FormattedMessage, useIntl } from "react-intl"; @@ -43,11 +43,11 @@ export async function derive_project_img_name( custom_software: SoftwareEnvironmentState, ): Promise { const { image_type, image_selected } = custom_software; - const dflt_software_img = await redux + const defaultSoftwareImage = await redux .getStore("customize") .getDefaultComputeImage(); if (image_selected == null || image_type == null) { - return dflt_software_img; + return defaultSoftwareImage; } switch (image_type) { case "custom": @@ -56,7 +56,7 @@ export async function derive_project_img_name( return image_selected; default: unreachable(image_type); - return dflt_software_img; // make TS happy + return defaultSoftwareImage; // make TS happy } } @@ -77,7 +77,7 @@ export function SoftwareEnvironment(props: Props) { const onCoCalcCom = customize_kucalc === KUCALC_COCALC_COM; const customize_software = useTypedRedux("customize", "software"); const organization_name = useTypedRedux("customize", "organization_name"); - const dflt_software_img = customize_software.get("default"); + const defaultSoftwareImage = customize_software.get("default"); const software_images = customize_software.get("environments"); const haveSoftwareImages: boolean = useMemo( @@ -109,7 +109,7 @@ export function SoftwareEnvironment(props: Props) { // initialize selection, if there is a default image set React.useEffect(() => { - if (default_image == null || default_image === dflt_software_img) { + if (default_image == null || default_image === defaultSoftwareImage) { // do nothing, that's the initial state already! } else if (is_custom_image(default_image)) { if (images == null) return; @@ -123,7 +123,7 @@ export function SoftwareEnvironment(props: Props) { } else { // must be standard image const img = software_images.get(default_image); - const display = img != null ? img.get("title") ?? "" : ""; + const display = img != null ? (img.get("title") ?? "") : ""; setState(default_image, display, "standard"); } }, []); @@ -170,7 +170,7 @@ export function SoftwareEnvironment(props: Props) { } function render_onprem() { - const selected = image_selected ?? dflt_software_img; + const selected = image_selected ?? defaultSoftwareImage; return ( <> @@ -219,7 +219,7 @@ export function SoftwareEnvironment(props: Props) { } function render_standard_image_selector() { - const isCustom = is_custom_image(image_selected ?? dflt_software_img); + const isCustom = is_custom_image(image_selected ?? defaultSoftwareImage); return ( <> @@ -230,7 +230,7 @@ export function SoftwareEnvironment(props: Props) { > { diff --git a/src/packages/frontend/editors/archive/component.tsx b/src/packages/frontend/editors/archive/component.tsx index 7828ab71a28..c7512d60624 100644 --- a/src/packages/frontend/editors/archive/component.tsx +++ b/src/packages/frontend/editors/archive/component.tsx @@ -4,7 +4,6 @@ */ import { Button, Card } from "antd"; - import { useActions, useRedux } from "@cocalc/frontend/app-framework"; import { A, ErrorDisplay, Icon, Loading } from "@cocalc/frontend/components"; import { ArchiveActions } from "./actions"; diff --git a/src/packages/frontend/editors/file-info-dropdown.tsx b/src/packages/frontend/editors/file-info-dropdown.tsx index bcbf3530a2a..e25de0188d0 100644 --- a/src/packages/frontend/editors/file-info-dropdown.tsx +++ b/src/packages/frontend/editors/file-info-dropdown.tsx @@ -11,7 +11,7 @@ import { CSS, React, useActions } from "@cocalc/frontend/app-framework"; import { DropdownMenu, Icon, IconName } from "@cocalc/frontend/components"; import { MenuItems } from "@cocalc/frontend/components/dropdown-menu"; import { useStudentProjectFunctionality } from "@cocalc/frontend/course"; -import { file_actions } from "@cocalc/frontend/project_store"; +import { file_actions, type FileAction } from "@cocalc/frontend/project_store"; import { capitalize, filename_extension } from "@cocalc/util/misc"; interface Props { @@ -57,9 +57,9 @@ const EditorFileInfoDropdown: React.FC = React.memo( } for (const key in file_actions) { if (key === name) { - actions.show_file_action_panel({ + actions.showFileActionPanel({ path: filename, - action: key, + action: key as FileAction, }); break; } diff --git a/src/packages/frontend/file-editors.ts b/src/packages/frontend/file-editors.ts index ed692f555f5..aed28b2fbc8 100644 --- a/src/packages/frontend/file-editors.ts +++ b/src/packages/frontend/file-editors.ts @@ -259,9 +259,9 @@ export async function remove( save(path, redux, project_id, is_public); } - if (!is_public) { + if (!is_public && project_id) { // Also free the corresponding side chat, if it was created. - require("./chat/register").remove( + (await import("./chat/register")).remove( meta_file(path, "chat"), redux, project_id, diff --git a/src/packages/frontend/file-upload.tsx b/src/packages/frontend/file-upload.tsx index 9e1d1a81f8d..8cf08cab194 100644 --- a/src/packages/frontend/file-upload.tsx +++ b/src/packages/frontend/file-upload.tsx @@ -19,6 +19,7 @@ import { labels } from "@cocalc/frontend/i18n"; import { BASE_URL } from "@cocalc/frontend/misc"; import { MAX_BLOB_SIZE } from "@cocalc/util/db-schema/blobs"; import { defaults, is_array, merge } from "@cocalc/util/misc"; +import { alert_message } from "@cocalc/frontend/alerts"; // very large upload limit -- should be plenty? // there is no cost for ingress, and as cocalc is a data plaform @@ -192,7 +193,7 @@ interface FileUploadWrapperProps { project_id: string; // The project to upload files to dest_path: string; // The path for files to be sent config?: object; // All supported dropzone.js config options - event_handlers: { + event_handlers?: { complete?: Function | Function[]; sending?: Function | Function[]; removedfile?: Function | Function[]; @@ -245,7 +246,7 @@ export function FileUploadWrapper({ previewTemplate: ReactDOMServer.renderToStaticMarkup( preview_template?.() ?? , ), - addRemoveLinks: event_handlers.removedfile != null, + addRemoveLinks: event_handlers?.removedfile != null, ...UPLOAD_OPTIONS, }, true, @@ -309,7 +310,7 @@ export function FileUploadWrapper({ // from the dropzone. This is true by default if there is // no "removedfile" handler, and false otherwise. function close_preview( - remove_all: boolean = event_handlers.removedfile == null, + remove_all: boolean = event_handlers?.removedfile == null, ) { if (typeof on_close === "function") { on_close(); @@ -381,7 +382,7 @@ export function FileUploadWrapper({ } function set_up_events(): void { - if (dropzone.current == null) { + if (dropzone.current == null || event_handlers == null) { return; } @@ -540,7 +541,20 @@ export function BlobUpload(props) { removedfile: props.event_handlers?.removedfile, complete: (file) => { if (file.xhr?.responseText) { - const { uuid } = JSON.parse(file.xhr.responseText); + let uuid; + try { + ({ uuid } = JSON.parse(file.xhr.responseText)); + } catch (err) { + // this will happen if the server is down/broken, e.g., instead of proper json, we get + // back an error from cloudflare. + console.warn("WARNING: upload failure", file.xhr.responseText); + alert_message({ + type: "error", + message: + "Failed to upload. Server may be down. Please try again later.", + }); + return; + } const url = `${BASE_URL}/blobs/${encodeURIComponent( file.upload.filename, )}?uuid=${uuid}`; diff --git a/src/packages/frontend/frame-editors/code-editor/actions.ts b/src/packages/frontend/frame-editors/code-editor/actions.ts index 17fc8b7c8ad..e0adf71f0e8 100644 --- a/src/packages/frontend/frame-editors/code-editor/actions.ts +++ b/src/packages/frontend/frame-editors/code-editor/actions.ts @@ -52,11 +52,11 @@ import { } from "@cocalc/frontend/misc/local-storage"; import { AvailableFeatures } from "@cocalc/frontend/project_configuration"; import { SyncDB } from "@cocalc/sync/editor/db"; -import { apply_patch } from "@cocalc/sync/editor/generic/util"; +import { apply_patch, make_patch } from "@cocalc/sync/editor/generic/util"; import type { SyncString } from "@cocalc/sync/editor/string/sync"; import { once } from "@cocalc/util/async-utils"; import { - Config as FormatterConfig, + Options as FormatterOptions, Exts as FormatterExts, Syntax as FormatterSyntax, Tool as FormatterTool, @@ -89,7 +89,6 @@ import { SetMap, } from "../frame-tree/types"; import { - formatter, get_default_font_size, log_error, syncdb2, @@ -112,6 +111,7 @@ import { misspelled_words } from "./spell-check"; import { log_opened_time } from "@cocalc/frontend/project/open-file"; import { ensure_project_running } from "@cocalc/frontend/project/project-start-warning"; import { alert_message } from "@cocalc/frontend/alerts"; +import { webapp_client } from "@cocalc/frontend/webapp-client"; interface gutterMarkerParams { line: number; @@ -299,57 +299,66 @@ export class Actions< } protected _init_syncstring(): void { - if (this.doctype == "none") { - this._syncstring = syncstring({ - project_id: this.project_id, - path: this.path, - cursors: !this.disable_cursors, - before_change_hook: () => this.set_syncstring_to_codemirror(), - after_change_hook: () => this.set_codemirror_to_syncstring(), - fake: true, - patch_interval: 500, - }) as SyncString; - } else if (this.doctype == "syncstring") { - this._syncstring = syncstring2({ - project_id: this.project_id, - path: this.path, - cursors: !this.disable_cursors, - }); - } else if (this.doctype == "syncdb") { - if ( - this.primary_keys == null || - this.primary_keys.length == null || - this.primary_keys.length <= 0 - ) { - throw Error("primary_keys must be array of positive length"); - } - this._syncstring = syncdb2({ - project_id: this.project_id, - path: this.path, - primary_keys: this.primary_keys, - string_cols: this.string_cols, - cursors: !this.disable_cursors, - }); - if (this.searchEmbeddings != null) { - if (!this.primary_keys.includes(this.searchEmbeddings.primaryKey)) { - throw Error( - `search embedding primaryKey must be in ${JSON.stringify( - this.primary_keys, - )}`, - ); + if (this._syncstring == null) { + // this._syncstring wasn't set in derived class so we set it here + if (this.doctype == "none") { + this._syncstring = syncstring({ + project_id: this.project_id, + path: this.path, + cursors: !this.disable_cursors, + before_change_hook: () => this.set_syncstring_to_codemirror(), + after_change_hook: () => this.set_codemirror_to_syncstring(), + fake: true, + patch_interval: 500, + }) as SyncString; + } else if (this.doctype == "syncstring") { + this._syncstring = syncstring2({ + project_id: this.project_id, + path: this.path, + cursors: !this.disable_cursors, + }); + } else if (this.doctype == "syncdb") { + if ( + this.primary_keys == null || + this.primary_keys.length == null || + this.primary_keys.length <= 0 + ) { + throw Error("primary_keys must be array of positive length"); } - if (!this.string_cols.includes(this.searchEmbeddings.textColumn)) { - throw Error( - `search embedding textColumn must be in ${JSON.stringify( - this.string_cols, - )}`, - ); + this._syncstring = syncdb2({ + project_id: this.project_id, + path: this.path, + primary_keys: this.primary_keys, + string_cols: this.string_cols, + cursors: !this.disable_cursors, + }); + if (this.searchEmbeddings != null) { + if (!this.primary_keys.includes(this.searchEmbeddings.primaryKey)) { + throw Error( + `search embedding primaryKey must be in ${JSON.stringify( + this.primary_keys, + )}`, + ); + } + if (!this.string_cols.includes(this.searchEmbeddings.textColumn)) { + throw Error( + `search embedding textColumn must be in ${JSON.stringify( + this.string_cols, + )}`, + ); + } } + } else { + throw Error(`invalid doctype="${this.doctype}"`); } - } else { - throw Error(`invalid doctype="${this.doctype}"`); } + this._syncstring.once("deleted", () => { + // the file was deleted + this._syncstring.close(); + this._get_project_actions().close_file(this.path); + }); + this._syncstring.once("ready", (err) => { if (this.doctype != "none") { // doctype = 'none' must be handled elsewhere, e.g., terminals. @@ -439,7 +448,7 @@ export class Actions< // Flag that there is activity (causes icon to turn orange). private activity = (): void => { - this._get_project_actions().flag_file_activity(this.path); + this._get_project_actions()?.flag_file_activity(this.path); }; // This is currently NOT used in this base class. It's used in other @@ -1494,7 +1503,7 @@ export class Actions< return this.terminals.get_terminal(id, parent); } - public set_terminal_cwd(id: string, cwd: string): void { + set_terminal_cwd(id: string, cwd: string): void { this.save_editor_state(id, { cwd }); } @@ -2259,10 +2268,6 @@ export class Actions< const cm = this._get_cm(id); if (!cm) return; - if (!(await this.ensure_latest_changes_are_saved())) { - return; - } - // Important: this function may be called even if there is no format support, // because it can be called via a keyboard shortcut. That's why we gracefully // handle this case -- see https://github.com/sagemathinc/cocalc/issues/4180 @@ -2270,35 +2275,41 @@ export class Actions< if (s == null) { return; } - // TODO: Using any here since TypeMap is just not working right... - if (!this.has_format_support(id, s.get("available_features"))) { - return; - } // Definitely have format support cm.focus(); const ext = filename_extension(this.path).toLowerCase() as FormatterExts; const syntax: FormatterSyntax = ext2syntax[ext]; - const config: FormatterConfig = { - syntax, + const parser = syntax2tool[syntax]; + if (!parser || !this.has_format_support(id, s.get("available_features"))) { + return; + } + const options: FormatterOptions = { + parser, tabWidth: cm.getOption("tabSize") as number, useTabs: cm.getOption("indentWithTabs") as boolean, lastChanged: this._syncstring.last_changed(), }; - this.set_status("Running code formatter..."); + const api = webapp_client.project_client.conatApi(this.project_id); + const str = cm.getValue(); + try { - const patch = await formatter(this.project_id, this.path, config); - if (patch != null) { - // Apply the patch. - // NOTE: old backends that haven't restarted just return {status:'ok'} - // and directly make the change. Delete this comment in a month or so. - // See https://github.com/sagemathinc/cocalc/issues/4335 - this.set_syncstring_to_codemirror(); - const new_val = apply_patch(patch, this._syncstring.to_str())[0]; - this._syncstring.from_str(new_val); - this._syncstring.commit(); - this.set_codemirror_to_syncstring(); + this.set_status("Running code formatter..."); + let formatted = await api.editor.formatString({ + str, + options, + path: this.path, + }); + if (formatted != str) { + const str2 = cm.getValue(); + if (str2 != str) { + // user made edits *during* formatting, so we "3-way merge" it in, rather + // than breaking what they did: + const patch = make_patch(str, formatted); + formatted = apply_patch(patch, str2)[0]; + } + cm.setValueNoJump(formatted); } this.setFormatError(""); } catch (err) { @@ -3187,4 +3198,15 @@ export class Actions< }); actions?.foldAllThreads(); } + + getComputeServerId = (): number | undefined => { + return this.redux + .getProjectActions(this.project_id) + .getComputeServerIdForFile(this.path); + }; + + fs = () => { + const a = this.redux.getProjectActions(this.project_id); + return a.fs(a.getComputeServerIdForFile({ path: this.path })); + }; } diff --git a/src/packages/frontend/frame-editors/frame-tree/commands/generic-commands.tsx b/src/packages/frontend/frame-editors/frame-tree/commands/generic-commands.tsx index 1de09b438a9..0bb95fecce1 100644 --- a/src/packages/frontend/frame-editors/frame-tree/commands/generic-commands.tsx +++ b/src/packages/frontend/frame-editors/frame-tree/commands/generic-commands.tsx @@ -7,7 +7,6 @@ import { Input } from "antd"; import { debounce } from "lodash"; import { useEffect, useRef } from "react"; import { defineMessage, IntlShape, useIntl } from "react-intl"; - import { set_account_table } from "@cocalc/frontend/account/util"; import { redux } from "@cocalc/frontend/app-framework"; import { Icon } from "@cocalc/frontend/components"; @@ -1467,7 +1466,7 @@ function fileAction(action) { alwaysShow: true, onClick: ({ props }) => { const actions = redux.getProjectActions(props.project_id); - actions.show_file_action_panel({ + actions.showFileActionPanel({ path: props.path, action, }); diff --git a/src/packages/frontend/frame-editors/frame-tree/editor.tsx b/src/packages/frontend/frame-editors/frame-tree/editor.tsx index 3cd75739f87..a6893001083 100644 --- a/src/packages/frontend/frame-editors/frame-tree/editor.tsx +++ b/src/packages/frontend/frame-editors/frame-tree/editor.tsx @@ -18,7 +18,7 @@ import { import { ErrorDisplay, Loading, - LoadingEstimate, + type LoadingEstimate, } from "@cocalc/frontend/components"; import { AvailableFeatures } from "@cocalc/frontend/project_configuration"; import { is_different } from "@cocalc/util/misc"; @@ -194,7 +194,7 @@ const FrameTreeEditor: React.FC = React.memo( if (is_loaded) return; return (
- +
); } diff --git a/src/packages/frontend/frame-editors/frame-tree/save-button.tsx b/src/packages/frontend/frame-editors/frame-tree/save-button.tsx index 4874a04c2b9..0a9585fe4d3 100644 --- a/src/packages/frontend/frame-editors/frame-tree/save-button.tsx +++ b/src/packages/frontend/frame-editors/frame-tree/save-button.tsx @@ -47,7 +47,7 @@ export function SaveButton({ const intl = useIntl(); const label = useMemo(() => { - if (!no_labels) { + if (!no_labels || read_only) { return intl.formatMessage(labels.frame_editors_title_bar_save_label, { type: is_public ? "is_public" : read_only ? "read_only" : "save", }); @@ -67,7 +67,7 @@ export function SaveButton({ ); function renderLabel() { - if (!no_labels && label) { + if (label) { return {` ${label}`}; } } diff --git a/src/packages/frontend/frame-editors/frame-tree/title-bar.tsx b/src/packages/frontend/frame-editors/frame-tree/title-bar.tsx index 67bb8337b8e..eed5a2bdce3 100644 --- a/src/packages/frontend/frame-editors/frame-tree/title-bar.tsx +++ b/src/packages/frontend/frame-editors/frame-tree/title-bar.tsx @@ -717,8 +717,8 @@ export function FrameTitleBar(props: FrameTitleBarProps) { label === APPLICATION_MENU ? manageCommands.applicationMenuTitle() : isIntlMessage(label) - ? intl.formatMessage(label) - : label + ? intl.formatMessage(label) + : label } items={v} /> diff --git a/src/packages/frontend/frame-editors/generic/client.ts b/src/packages/frontend/frame-editors/generic/client.ts index 7c2bdbfe19c..33a8159f044 100644 --- a/src/packages/frontend/frame-editors/generic/client.ts +++ b/src/packages/frontend/frame-editors/generic/client.ts @@ -10,9 +10,7 @@ Typescript async/await rewrite of @cocalc/util/client.coffee... import { Map } from "immutable"; import { redux } from "@cocalc/frontend/app-framework"; import { webapp_client } from "@cocalc/frontend/webapp-client"; -import { CompressedPatch } from "@cocalc/sync/editor/generic/types"; import { callback2 } from "@cocalc/util/async-utils"; -import { Config as FormatterConfig } from "@cocalc/util/code-formatter"; import { FakeSyncstring } from "./syncstring-fake"; import { type UserSearchResult as User } from "@cocalc/util/db-schema/accounts"; export { type User }; @@ -154,30 +152,6 @@ export async function write_text_file_to_project( await webapp_client.project_client.write_text_file(opts); } -export async function formatter( - project_id: string, - path: string, - config: FormatterConfig, -): Promise { - const api = await webapp_client.project_client.api(project_id); - const resp = await api.formatter(path, config); - - if (resp.status === "error") { - const loc = resp.error?.loc; - if (loc && loc.start) { - throw Error( - `Syntax error prevented formatting code (possibly on line ${loc.start.line} column ${loc.start.column}) -- fix and run again.`, - ); - } else if (resp.error) { - throw Error(resp.error); - } else { - throw Error("Syntax error prevented formatting code."); - } - } else { - return resp.patch; - } -} - export function log_error(error: string | object): void { webapp_client.tracking_client.log_error(error); } @@ -200,7 +174,8 @@ export function syncstring(opts: SyncstringOpts): any { delete opts.fake; } opts1.id = schema.client_db.sha1(opts.project_id, opts.path); - return webapp_client.sync_string(opts1); + return webapp_client.conat_client.conat().sync.string(opts1); + // return webapp_client.sync_string(opts1); } import { DataServer } from "@cocalc/sync/editor/generic/sync-doc"; @@ -218,9 +193,7 @@ interface SyncstringOpts2 { } export function syncstring2(opts: SyncstringOpts2): SyncString { - const opts1: any = opts; - opts1.client = webapp_client; - return webapp_client.sync_client.sync_string(opts1); + return webapp_client.conat_client.conat().sync.string(opts); } export interface SyncDBOpts { @@ -238,8 +211,7 @@ export interface SyncDBOpts { } export function syncdb(opts: SyncDBOpts): any { - const opts1: any = opts; - return webapp_client.sync_db(opts1); + return webapp_client.conat_client.conat().sync.db(opts); } import type { SyncDB } from "@cocalc/sync/editor/db/sync"; @@ -250,7 +222,7 @@ export function syncdb2(opts: SyncDBOpts): SyncDB { } const opts1: any = opts; opts1.client = webapp_client; - return webapp_client.sync_client.sync_db(opts1); + return webapp_client.conat_client.conat().sync.db(opts1); } interface QueryOpts { diff --git a/src/packages/frontend/frame-editors/generic/syncstring-fake.ts b/src/packages/frontend/frame-editors/generic/syncstring-fake.ts index 3e5b4e87cf3..f22ef045325 100644 --- a/src/packages/frontend/frame-editors/generic/syncstring-fake.ts +++ b/src/packages/frontend/frame-editors/generic/syncstring-fake.ts @@ -25,6 +25,8 @@ export class FakeSyncstring extends EventEmitter { this.emit("ready"); } + hasFullHistory = () => true; + close() {} from_str() {} diff --git a/src/packages/frontend/frame-editors/jupyter-editor/actions.ts b/src/packages/frontend/frame-editors/jupyter-editor/actions.ts index 4d5f394c2ff..9591a8b9f41 100644 --- a/src/packages/frontend/frame-editors/jupyter-editor/actions.ts +++ b/src/packages/frontend/frame-editors/jupyter-editor/actions.ts @@ -42,8 +42,13 @@ export class JupyterEditorActions extends BaseActions { return { type: "jupyter_cell_notebook" }; } - _init2(): void { + protected _init_syncstring(): void { this.create_jupyter_actions(); + this._syncstring = this.jupyter_actions.syncdb; + super._init_syncstring(); + } + + _init2(): void { this.init_new_frame(); this.init_changes_state(); @@ -112,26 +117,12 @@ export class JupyterEditorActions extends BaseActions { private watchJupyterStore = (): void => { const store = this.jupyter_actions.store; - let connection_file = store.get("connection_file"); store.on("change", () => { // sync read only state -- source of true is jupyter_actions.store.get('read_only') const read_only = store.get("read_only"); if (read_only != this.store.get("read_only")) { this.setState({ read_only }); } - // sync connection file - const c = store.get("connection_file"); - if (c == connection_file) { - return; - } - connection_file = c; - const id = this._get_most_recent_shell_id("jupyter"); - if (id == null) { - // There is no Jupyter console open right now... - return; - } - // This will update the connection file - this.shell(id, true); }); }; @@ -167,6 +158,7 @@ export class JupyterEditorActions extends BaseActions { this.path, this.project_id, ); + this.jupyter_actions.jupyterEditorActions = this; } private close_jupyter_actions(): void { @@ -290,14 +282,12 @@ export class JupyterEditorActions extends BaseActions { } protected async get_shell_spec( - id: string, - ): Promise { - id = id; // not used - const connection_file = this.jupyter_actions.store.get("connection_file"); - if (connection_file == null) return; + _id: string, + ): Promise<{ command: string; args: string[] }> { + const connectionFile = await this.jupyter_actions.getConnectionFile(); return { command: "jupyter", - args: ["console", "--existing", connection_file], + args: ["console", "--existing", connectionFile], }; } diff --git a/src/packages/frontend/frame-editors/jupyter-editor/cell-notebook/actions.ts b/src/packages/frontend/frame-editors/jupyter-editor/cell-notebook/actions.ts index 87929465ca2..c0cb91e9d5a 100644 --- a/src/packages/frontend/frame-editors/jupyter-editor/cell-notebook/actions.ts +++ b/src/packages/frontend/frame-editors/jupyter-editor/cell-notebook/actions.ts @@ -258,7 +258,12 @@ export class NotebookFrameActions { this.frame_tree_actions.save(explicit); } - public enable_key_handler(): void { + public enable_key_handler(force?): void { + if (!force && this.jupyter_actions.store.get("stdin")) { + // do not enable when getting input from stdin, since user may be typing + // and keyboard shortcuts would be very confusing. + return; + } if (this.is_closed()) { // should be a no op -- no point in enabling the key handler after CellNotebookActions are closed. return; @@ -325,36 +330,41 @@ export class NotebookFrameActions { } } - public run_selected_cells(v?: string[]): void { + public run_selected_cells(ids?: string[]): void { this.save_input_editor(); - if (v === undefined) { - v = this.store.get_selected_cell_ids_list(); + if (ids == null) { + ids = this.store.get_selected_cell_ids_list(); } // for whatever reason, any running of a cell deselects // in official jupyter this.unselect_all_cells(); - for (const id of v) { - const save = id === v[v.length - 1]; // save only last one. - this.run_cell(id, save); - } + this.runCells(ids); + } + + run_cell(id: string) { + this.runCells([id]); } // This is here since it depends on knowing the edit state // of markdown cells. - public run_cell(id: string, save: boolean = true): void { - const type = this.jupyter_actions.store.get_cell_type(id); - if (type === "markdown") { - if (this.store.get("md_edit_ids", Set()).contains(id)) { - this.set_md_cell_not_editing(id); + public runCells(ids: string[]): void { + const v: string[] = []; + for (const id of ids) { + const type = this.jupyter_actions.store.get_cell_type(id); + if (type === "markdown") { + if (this.store.get("md_edit_ids", Set()).contains(id)) { + this.set_md_cell_not_editing(id); + } + } else if (type === "code") { + v.push(id); } - return; + // running is a no-op for raw cells. } - if (type === "code") { - this.jupyter_actions.run_cell(id, save); + if (v.length > 0) { + this.jupyter_actions.runCells(v); } - // running is a no-op for raw cells. } /*** diff --git a/src/packages/frontend/frame-editors/qmd-editor/actions.ts b/src/packages/frontend/frame-editors/qmd-editor/actions.ts index 50920aa7cc6..17cea7239ca 100644 --- a/src/packages/frontend/frame-editors/qmd-editor/actions.ts +++ b/src/packages/frontend/frame-editors/qmd-editor/actions.ts @@ -7,12 +7,8 @@ Quarto Editor Actions */ -import { Set } from "immutable"; import { debounce } from "lodash"; - -import { redux } from "@cocalc/frontend/app-framework"; import { markdown_to_html_frontmatter } from "@cocalc/frontend/markdown"; -import { path_split } from "@cocalc/util/misc"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { Actions as BaseActions, @@ -21,7 +17,7 @@ import { import { FrameTree } from "../frame-tree/types"; import { ExecOutput } from "../generic/client"; import { Actions as MarkdownActions } from "../markdown-editor/actions"; -import { derive_rmd_output_filename } from "../rmd-editor/utils"; +import { checkProducedFiles } from "../rmd-editor/utils"; import { convert } from "./qmd-converter"; const custom_pdf_error_message: string = ` @@ -112,44 +108,7 @@ export class Actions extends MarkdownActions { } async _check_produced_files(): Promise { - const project_actions = redux.getProjectActions(this.project_id); - if (project_actions == undefined) { - return; - } - const path = path_split(this.path).head; - await project_actions.fetch_directory_listing({ path }); - - const project_store = project_actions.get_store(); - if (project_store == undefined) { - return; - } - // TODO: change the 0 to the compute server when/if we ever support QMD on a compute server (which we don't) - const dir_listings = project_store.getIn(["directory_listings", 0]); - if (dir_listings == undefined) { - return; - } - const listing = dir_listings.get(path); - if (listing == undefined) { - return; - } - - let existing = Set(); - for (const ext of ["pdf", "html", "nb.html"]) { - // full path – basename might change - const expected_fn = derive_rmd_output_filename(this.path, ext); - const fn_exists = listing.some((entry) => { - const name = entry.get("name"); - return name === path_split(expected_fn).tail; - }); - if (fn_exists) { - existing = existing.add(ext); - } - } - - // console.log("setting derived_file_types to", existing.toJS()); - this.setState({ - derived_file_types: existing as any, - }); + await checkProducedFiles(this); } private set_log(output?: ExecOutput | undefined): void { @@ -248,4 +207,4 @@ export class Actions extends MarkdownActions { this.build(); } } -} +} \ No newline at end of file diff --git a/src/packages/frontend/frame-editors/rmd-editor/actions.ts b/src/packages/frontend/frame-editors/rmd-editor/actions.ts index 2b92b449a07..517cef0558f 100644 --- a/src/packages/frontend/frame-editors/rmd-editor/actions.ts +++ b/src/packages/frontend/frame-editors/rmd-editor/actions.ts @@ -7,13 +7,9 @@ R Markdown Editor Actions */ -import { Set } from "immutable"; import { debounce } from "lodash"; - -import { redux } from "@cocalc/frontend/app-framework"; import { markdown_to_html_frontmatter } from "@cocalc/frontend/markdown"; import { open_new_tab } from "@cocalc/frontend/misc"; -import { path_split } from "@cocalc/util/misc"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import { Actions as BaseActions, @@ -23,7 +19,7 @@ import { FrameTree } from "../frame-tree/types"; import { ExecOutput } from "../generic/client"; import { Actions as MarkdownActions } from "../markdown-editor/actions"; import { convert } from "./rmd-converter"; -import { derive_rmd_output_filename } from "./utils"; +import { checkProducedFiles } from "./utils"; const HELP_URL = "https://doc.cocalc.com/frame-editor.html#edit-rmd"; const MINIMAL = `--- @@ -139,44 +135,7 @@ export class Actions extends MarkdownActions { } async _check_produced_files(): Promise { - const project_actions = redux.getProjectActions(this.project_id); - if (project_actions == undefined) { - return; - } - const path = path_split(this.path).head; - await project_actions.fetch_directory_listing({ path }); - - const project_store = project_actions.get_store(); - if (project_store == undefined) { - return; - } - // TODO: change the 0 to the compute server when/if we ever support RMD on a compute server (which we don't) - const dir_listings = project_store.getIn(["directory_listings", 0]); - if (dir_listings == undefined) { - return; - } - const listing = dir_listings.get(path); - if (listing == undefined) { - return; - } - - let existing = Set(); - for (const ext of ["pdf", "html", "nb.html"]) { - // full path – basename might change - const expected_fn = derive_rmd_output_filename(this.path, ext); - const fn_exists = listing.some((entry) => { - const name = entry.get("name"); - return name === path_split(expected_fn).tail; - }); - if (fn_exists) { - existing = existing.add(ext); - } - } - - // console.log("setting derived_file_types to", existing.toJS()); - this.setState({ - derived_file_types: existing as any, - }); + await checkProducedFiles(this); } private set_log(output?: ExecOutput | undefined): void { diff --git a/src/packages/frontend/frame-editors/rmd-editor/utils.ts b/src/packages/frontend/frame-editors/rmd-editor/utils.ts index d4c129650a8..63dfce78f10 100644 --- a/src/packages/frontend/frame-editors/rmd-editor/utils.ts +++ b/src/packages/frontend/frame-editors/rmd-editor/utils.ts @@ -5,6 +5,7 @@ import { change_filename_extension, path_split } from "@cocalc/util/misc"; import { join } from "path"; +import { Set } from "immutable"; // something in the rmarkdown source code replaces all spaces by dashes // [hsy] I think this is because of calling pandoc. @@ -17,3 +18,31 @@ export function derive_rmd_output_filename(path, ext) { // avoid a leading / if it's just a filename (i.e. head = '') return join(head, fn); } + +export async function checkProducedFiles(codeEditorActions) { + const project_actions = codeEditorActions.redux.getProjectActions( + codeEditorActions.project_id, + ); + if (project_actions == null) { + return; + } + + let existing = Set(); + const fs = codeEditorActions.fs(); + const f = async (ext: string) => { + const expectedFilename = derive_rmd_output_filename( + codeEditorActions.path, + ext, + ); + if (await fs.exists(expectedFilename)) { + existing = existing.add(ext); + } + }; + const v = ["pdf", "html", "nb.html"].map(f); + await Promise.all(v); + + // console.log("setting derived_file_types to", existing.toJS()); + codeEditorActions.setState({ + derived_file_types: existing as any, + }); +} diff --git a/src/packages/frontend/frame-editors/terminal-editor/commands-guide.tsx b/src/packages/frontend/frame-editors/terminal-editor/commands-guide.tsx index 3f99d6a5797..ad338c2fafa 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/commands-guide.tsx +++ b/src/packages/frontend/frame-editors/terminal-editor/commands-guide.tsx @@ -11,8 +11,7 @@ import { Table, Typography, } from "antd"; -import { List, Map } from "immutable"; - +import { Map } from "immutable"; import { ControlOutlined, FileOutlined, @@ -20,82 +19,45 @@ import { InfoCircleOutlined, QuestionCircleOutlined, } from "@ant-design/icons"; -import { - CSS, - React, - TypedMap, - useActions, - useEffect, - useState, - useTypedRedux, -} from "@cocalc/frontend/app-framework"; +import { createContext, useEffect, useState } from "react"; import { Icon } from "@cocalc/frontend/components"; -import { plural, round1 } from "@cocalc/util/misc"; -import { DirectoryListingEntry } from "../../project/explorer/types"; +import { human_readable_size, plural } from "@cocalc/util/misc"; import { TerminalActions } from "./actions"; import { Command, SelectFile } from "./commands-guide-components"; +import useFiles from "@cocalc/frontend/project/listing/use-files"; +import ShowError from "@cocalc/frontend/components/error"; const { Panel } = Collapse; interface Props { - font_size: number; - project_id: string; actions: TerminalActions; local_view_state: Map; } -export const TerminalActionsContext = React.createContext< +export const TerminalActionsContext = createContext< TerminalActions | undefined >(undefined); const ListingStatsInit = { - total: 0, num_files: 0, num_dirs: 0, - size_mib: 0, + size: 0, }; const info = "info"; -type ListingImm = List>; - -function listing2names(listing?): string[] { - if (listing == null) { - return []; - } else { - return listing - .map((val) => val.get("name")) - .sort() - .toJS(); - } -} - function cwd2path(cwd: string): string { return cwd.charAt(0) === "/" ? ".smc/root" + cwd : cwd; } -export const CommandsGuide: React.FC = React.memo((props: Props) => { - const { /*font_size,*/ actions, local_view_state, project_id } = props; - - const project_actions = useActions({ project_id }); - // TODO: for now just assuming in the project (not a compute server) -- problem - // is that the guide is general to the whole terminal not a particular frame, - // and each frame can be on a different compute server! Not worth solving if - // nobody is using either the guide or compute servers. - const directory_listings = useTypedRedux( - { project_id }, - "directory_listings", - )?.get(0); - - const [terminal_id, set_terminal_id] = useState(); +export function CommandsGuide({ actions, local_view_state }: Props) { + const [terminal_id, setTerminalId] = useState(); const [cwd, set_cwd] = useState(""); // default home directory const [hidden, set_hidden] = useState(false); // hidden files // empty immutable js list - const [listing, set_listing] = useState(List([])); - const [listing_stats, set_listing_stats] = useState(ListingStatsInit); - const [directorynames, set_directorynames] = useState([]); + const [directoryNames, set_directoryNames] = useState([]); const [filenames, set_filenames] = useState([]); // directory and filenames const [dir1, set_dir1] = useState(undefined); @@ -103,13 +65,17 @@ export const CommandsGuide: React.FC = React.memo((props: Props) => { const [fn2, set_fn2] = useState(undefined); useEffect(() => { - const tid = actions._get_most_recent_active_frame_id_of_type("terminal"); - if (tid == null) return; - if (terminal_id != tid) set_terminal_id(tid); + const terminalId = + actions._get_most_recent_active_frame_id_of_type("terminal"); + if (terminalId == null) { + return; + } + if (terminal_id != terminalId) { + setTerminalId(terminalId); + } }, [local_view_state]); useEffect(() => { - //const terminal = actions.get_terminal(tid); const next_cwd = local_view_state.getIn([ "editor_state", terminal_id, @@ -117,50 +83,45 @@ export const CommandsGuide: React.FC = React.memo((props: Props) => { ]) as string | undefined; if (next_cwd != null && cwd != next_cwd) { set_cwd(next_cwd); - project_actions?.fetch_directory_listing({ path: cwd2path(next_cwd) }); } }, [terminal_id, local_view_state]); - // if the working directory changes or the listing itself, recompute the listing we base the files on - useEffect(() => { - if (cwd == null) return; - set_listing(directory_listings?.get(cwd2path(cwd))); - }, [directory_listings, cwd]); + const { files, error, refresh } = useFiles({ + fs: actions.fs(), + path: cwd2path(cwd), + }); // finally, if the listing really did change – or show/hide hidden files toggled – recalculate everything useEffect(() => { - // a user reported a crash "Uncaught TypeError: listing.filter is not a function". - // This was because directory_listings is a map from path to either an immutable - // listing **or** an error, as you can see where it is set in the file frontend/project_actions.ts - // The typescript there just has an "any", because it's code that was partly converted from coffeescript. - // Fixing this by just doing listing?.filter==null instead of listing==null here, since dealing with - // an error isn't necessary for this command guide. - if ( - listing == null || - typeof listing == "string" || - listing?.filter == null - ) + if (files == null) { return; - const all_files = hidden - ? listing - : listing.filter((val) => !val.get("name").startsWith(".")); - const grouped = all_files.groupBy((val) => !!val.get("isdir")); - const dirnames = [".", "..", ...listing2names(grouped.get(true))]; - const filenames = listing2names(grouped.get(false)); - set_directorynames(dirnames); + } + const dirnames: string[] = [".", ".."]; + const filenames: string[] = []; + for (const name in files) { + if (!hidden && name.startsWith(".")) { + continue; + } + if (files[name].isDir) { + dirnames.push(name); + } else { + filenames.push(name); + } + } + dirnames.sort(); + filenames.sort(); + + set_directoryNames(dirnames); set_filenames(filenames); - const total = all_files.size; - const size_red = grouped - .get(false) - ?.reduce((cur, val) => cur + val.get("size", 0), 0); - const size = (size_red ?? 0) / (1024 * 1024); + const size = filenames + .map((name) => files[name].size ?? 0) + .reduce((a, b) => a + b, 0); set_listing_stats({ - total, num_files: filenames.length, - num_dirs: dirnames.length, - size_mib: size, + num_dirs: dirnames.length - 2, + size, }); - }, [listing, hidden]); + }, [files, hidden]); // we also clear selected files if they no longer exist useEffect(() => { @@ -170,16 +131,16 @@ export const CommandsGuide: React.FC = React.memo((props: Props) => { if (fn2 != null && !filenames.includes(fn2)) { set_fn2(undefined); } - if (dir1 != null && !directorynames.includes(dir1)) { + if (dir1 != null && !directoryNames.includes(dir1)) { set_dir1(undefined); } - }, [directorynames, filenames]); + }, [directoryNames, filenames]); function render_files() { - const dirs = directorynames.map((v) => ({ key: v, name: v, type: "dir" })); + const dirs = directoryNames.map((v) => ({ key: v, name: v, type: "dir" })); const fns = filenames.map((v) => ({ key: v, name: v, type: "file" })); const data = [...dirs, ...fns]; - const style: CSS = { cursor: "pointer" }; + const style = { cursor: "pointer" } as const; const columns = [ { title: "Name", @@ -306,7 +267,7 @@ export const CommandsGuide: React.FC = React.memo((props: Props) => { {plural(listing_stats.num_files, "file")},{" "} {listing_stats.num_dirs}{" "} {plural(listing_stats.num_dirs, "directory", "directories")},{" "} - {round1(listing_stats.size_mib)} MiB + {human_readable_size(listing_stats.size)} @@ -316,7 +277,7 @@ export const CommandsGuide: React.FC = React.memo((props: Props) => { - + @@ -430,7 +391,7 @@ export const CommandsGuide: React.FC = React.memo((props: Props) => { } function render() { - const style: CSS = { overflowY: "auto" }; + const style = { overflowY: "auto" } as const; return ( } key={info}> @@ -501,7 +462,8 @@ export const CommandsGuide: React.FC = React.memo((props: Props) => { return ( + {render()} ); -}); +} diff --git a/src/packages/frontend/frame-editors/terminal-editor/conat-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/conat-terminal.ts index 5bb2c6a5bc3..3734b56808a 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/conat-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/conat-terminal.ts @@ -10,7 +10,6 @@ import { SIZE_TIMEOUT_MS, createBrowserClient, } from "@cocalc/conat/service/terminal"; -import { CONAT_OPEN_FILE_TOUCH_INTERVAL } from "@cocalc/util/conat"; import { until } from "@cocalc/util/async-utils"; type State = "disconnected" | "init" | "running" | "closed"; @@ -24,7 +23,7 @@ export class ConatTerminal extends EventEmitter { private terminalResize; private openPaths; private closePaths; - private api: TerminalServiceApi; + public readonly api: TerminalServiceApi; private service?; private options?; private writeQueue: string = ""; @@ -58,7 +57,6 @@ export class ConatTerminal extends EventEmitter { this.path = path; this.termPath = termPath; this.options = options; - this.touchLoop({ project_id, path: termPath }); this.sizeLoop(measureSize); this.api = createTerminalClient({ project_id, termPath }); this.createBrowserService(); @@ -143,25 +141,6 @@ export class ConatTerminal extends EventEmitter { } }; - touchLoop = async ({ project_id, path }) => { - while (this.state != ("closed" as State)) { - try { - // this marks the path as being of interest for editing and starts - // the service; it doesn't actually create a file on disk. - await webapp_client.touchOpenFile({ - project_id, - path, - }); - } catch (err) { - console.warn(err); - } - if (this.state == ("closed" as State)) { - break; - } - await delay(CONAT_OPEN_FILE_TOUCH_INTERVAL); - } - }; - sizeLoop = async (measureSize) => { while (this.state != ("closed" as State)) { measureSize(); diff --git a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts index 95e9bf16fce..eaf068aac6e 100644 --- a/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts +++ b/src/packages/frontend/frame-editors/terminal-editor/connected-terminal.ts @@ -50,13 +50,19 @@ const MAX_DELAY = 15000; const ENABLE_WEBGL = false; +// ephemeral = faster, less load on servers, but if project and browser all +// close, the history is gone... which may be good and less confusing. +const EPHEMERAL = true; + interface Path { file?: string; directory?: string; } +type State = "ready" | "closed"; + export class Terminal { - private state: string = "ready"; + private state: State = "ready"; private actions: Actions | ConnectedTerminalInterface; private account_store: any; private project_actions: ProjectActions; @@ -191,6 +197,8 @@ export class Terminal { // this.terminal_resize = debounce(this.terminal_resize, 2000); } + isClosed = () => (this.state ?? "closed") === "closed"; + private get_xtermjs_options = (): any => { const rendererType = this.rendererType; const settings = this.account_store.get("terminal"); @@ -216,13 +224,13 @@ export class Terminal { }; private assert_not_closed = (): void => { - if (this.state === "closed") { + if (this.isClosed()) { throw Error("BUG -- Terminal is closed."); } }; close = (): void => { - if (this.state === "closed") { + if (this.isClosed()) { return; } this.set_connection_status("disconnected"); @@ -309,13 +317,11 @@ export class Terminal { cwd: this.workingDir, env: this.actions.get_term_env(), }, + ephemeral: EPHEMERAL, }); this.conn = conn as any; conn.on("close", this.connect); conn.on("kick", this.close_request); - conn.on("cwd", (cwd) => { - this.actions.set_terminal_cwd(this.id, cwd); - }); conn.on("data", this.handleDataFromProject); conn.on("init", this.render); conn.once("ready", () => { @@ -395,10 +401,9 @@ export class Terminal { }; private render = async (data: string): Promise => { - if (data == null) { + if (data == null || this.isClosed()) { return; } - this.assert_not_closed(); this.history += data; if (this.history.length > MAX_HISTORY_LENGTH) { this.history = this.history.slice( @@ -420,7 +425,7 @@ export class Terminal { await delay(0); this.ignoreData--; } - if (this.state == "done") return; + if (this.isClosed()) return; // tell anyone who waited for output coming back about this while (this.render_done.length > 0) { this.render_done.pop()?.(); @@ -438,7 +443,7 @@ export class Terminal { this.terminal.onTitleChange((title) => { if (title != null) { this.actions.set_title(this.id, title); - this.ask_for_cwd(); + this.update_cwd(); } }); }; @@ -450,7 +455,7 @@ export class Terminal { }; touch = async () => { - if (this.state === "closed") return; + if (this.isClosed()) return; if (Date.now() - this.last_active < 70000) { if (this.project_actions.isTabClosed()) { return; @@ -464,7 +469,7 @@ export class Terminal { }; init_keyhandler = (): void => { - if (this.state === "closed") { + if (this.isClosed()) { return; } if (this.keyhandler_initialized) { @@ -575,7 +580,7 @@ export class Terminal { // Stop ignoring terminal data... but ONLY once // the render buffer is also empty. no_ignore = async (): Promise => { - if (this.state === "closed") { + if (this.isClosed()) { return; } const g = (cb) => { @@ -587,7 +592,7 @@ export class Terminal { } // cause render to actually appear now. await delay(0); - if (this.state === "closed") { + if (this.isClosed()) { return; } try { @@ -720,9 +725,23 @@ export class Terminal { this.render_buffer = ""; }; - ask_for_cwd = debounce((): void => { - this.conn_write({ cmd: "cwd" }); - }); + update_cwd = debounce( + async () => { + if (this.isClosed()) return; + let cwd; + try { + cwd = await this.conn?.api.cwd(); + } catch { + return; + } + if (this.isClosed()) return; + if (cwd != null) { + this.actions.set_terminal_cwd(this.id, cwd); + } + }, + 1000, + { leading: true, trailing: true }, + ); kick_other_users_out(): void { // @ts-ignore @@ -760,14 +779,14 @@ export class Terminal { } focus(): void { - if (this.state === "closed") { + if (this.isClosed()) { return; } this.terminal.focus(); } refresh(): void { - if (this.state === "closed") { + if (this.isClosed()) { return; } this.terminal.refresh(0, this.terminal.rows - 1); @@ -777,7 +796,7 @@ export class Terminal { try { await open_init_file(this.actions._get_project_actions(), this.termPath); } catch (err) { - if (this.state === "closed") { + if (this.isClosed()) { return; } this.actions.set_error(`Problem opening init file -- ${err}`); diff --git a/src/packages/frontend/frame-editors/time-travel-editor/actions.ts b/src/packages/frontend/frame-editors/time-travel-editor/actions.ts index edbe6403912..ce1bf926e79 100644 --- a/src/packages/frontend/frame-editors/time-travel-editor/actions.ts +++ b/src/packages/frontend/frame-editors/time-travel-editor/actions.ts @@ -21,7 +21,6 @@ import { List } from "immutable"; import { once } from "@cocalc/util/async-utils"; import { filename_extension, path_split } from "@cocalc/util/misc"; import { SyncDoc } from "@cocalc/sync/editor/generic/sync-doc"; -import { webapp_client } from "../../webapp-client"; import { exec } from "@cocalc/frontend/frame-editors/generic/client"; import { ViewDocument } from "./view-document"; import { @@ -32,8 +31,8 @@ import { FrameTree } from "../frame-tree/types"; import { export_to_json } from "./export-to-json"; import type { Document } from "@cocalc/sync/editor/generic/types"; import LRUCache from "lru-cache"; -import { syncdbPath } from "@cocalc/util/jupyter/names"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { until } from "@cocalc/util/async-utils"; const EXTENSION = ".time-travel"; @@ -81,7 +80,6 @@ export class TimeTravelActions extends CodeEditorActions { protected doctype: string = "none"; // actual document is managed elsewhere private docpath: string; private docext: string; - private syncpath: string; syncdoc?: SyncDoc; private first_load: boolean = true; ambient_actions?: CodeEditorActions; @@ -96,11 +94,7 @@ export class TimeTravelActions extends CodeEditorActions { this.docpath = head + "/" + this.docpath; } // log("init", { path: this.path }); - this.syncpath = this.docpath; this.docext = filename_extension(this.docpath); - if (this.docext == "ipynb") { - this.syncpath = syncdbPath(this.docpath); - } this.setState({ versions: List([]), loading: true, @@ -118,27 +112,53 @@ export class TimeTravelActions extends CodeEditorActions { init_frame_tree = () => {}; - close = (): void => { - if (this.syncdoc != null) { - this.syncdoc.close(); - delete this.syncdoc; - } - super.close(); - }; - set_error = (error) => { this.setState({ error }); }; private init_syncdoc = async (): Promise => { - const persistent = this.docext == "ipynb" || this.docext == "sagews"; // ugly for now (?) - this.syncdoc = await webapp_client.sync_client.open_existing_sync_document({ - project_id: this.project_id, - path: this.syncpath, - persistent, + let mainFileActions: any = null; + await until(async () => { + if (this.isClosed()) { + return true; + } + mainFileActions = this.redux.getEditorActions( + this.project_id, + this.docpath, + ); + if (mainFileActions == null) { + // open the file that we're showing timetravel for, so that the + // actions are available + try { + await this.open_file({ foreground: false, explicit: false }); + } catch (err) { + console.warn(err); + } + // will try again above in the next loop + return false; + } else { + const doc = mainFileActions._syncstring; + if (doc == null || doc.get_state() == "closed") { + // file is closing + return false; + } + // got it! + return true; + } }); - if (this.syncdoc == null) return; - this.syncdoc.on("change", debounce(this.syncdoc_changed, 1000)); + if (this.isClosed() || mainFileActions == null) { + return; + } + this.syncdoc = mainFileActions._syncstring; + + if ( + this.syncdoc == null || + this.syncdoc.get_state() == "closed" || + // @ts-ignore + this.syncdoc.is_fake + ) { + return; + } if (this.syncdoc.get_state() != "ready") { try { await once(this.syncdoc, "ready"); @@ -146,14 +166,15 @@ export class TimeTravelActions extends CodeEditorActions { return; } } - if (this.syncdoc == null) return; + this.syncdoc.on("change", debounce(this.syncdoc_changed, 750)); // cause initial load -- we could be plugging into an already loaded syncdoc, // so there wouldn't be any change event, so we have to trigger this. this.syncdoc_changed(); this.syncdoc.on("close", () => { - // in our code we don't check if the state is closed, but instead - // that this.syncdoc is not null. + // in the actions in this file, we don't check if the state is closed, but instead + // that this.syncdoc is not null: delete this.syncdoc; + this.init_syncdoc(); }); this.setState({ @@ -289,10 +310,10 @@ export class TimeTravelActions extends CodeEditorActions { } }; - open_file = async (): Promise => { + open_file = async (opts?): Promise => { // log("open_file"); const actions = this.redux.getProjectActions(this.project_id); - await actions.open_file({ path: this.docpath, foreground: true }); + await actions.open_file({ path: this.docpath, foreground: true, ...opts }); }; // Revert the live version of the document to a specific version */ @@ -316,6 +337,17 @@ export class TimeTravelActions extends CodeEditorActions { syncdoc.revert(version); } await syncdoc.commit(true); + if (this.docpath.endsWith(".ipynb")) { + const a = this.redux.getEditorActions( + this.project_id, + this.docpath, + )?.jupyter_actions; + if (a != null) { + // make sure nothing is running or appears to be (due to it being running in history) + a.clear_all_cell_run_state(); + a.signal("SIGINT"); + } + } // Some editors, e.g., the code text editor, only update Codemirror when // "after-change" is emitted (not just "change"), and commit does NOT result diff --git a/src/packages/frontend/jupyter/browser-actions.ts b/src/packages/frontend/jupyter/browser-actions.ts index f760f30256b..e1eb10554a1 100644 --- a/src/packages/frontend/jupyter/browser-actions.ts +++ b/src/packages/frontend/jupyter/browser-actions.ts @@ -8,7 +8,7 @@ browser-actions: additional actions that are only available in the web browser frontend. */ import * as awaiting from "awaiting"; -import { fromJS, Map } from "immutable"; +import { fromJS, Map, Set as iSet } from "immutable"; import { debounce, isEqual } from "lodash"; import { jupyter, labels } from "@cocalc/frontend/i18n"; import { getIntl } from "@cocalc/frontend/i18n/get-intl"; @@ -18,7 +18,6 @@ import { get_local_storage, set_local_storage, } from "@cocalc/frontend/misc/local-storage"; -import track from "@cocalc/frontend/user-tracking"; import { webapp_client } from "@cocalc/frontend/webapp-client"; import { JupyterActions as JupyterActions0 } from "@cocalc/jupyter/redux/actions"; import { CellToolbarName } from "@cocalc/jupyter/types"; @@ -27,6 +26,8 @@ import { base64ToBuffer, bufferToBase64 } from "@cocalc/util/base64"; import { Config as FormatterConfig, Syntax } from "@cocalc/util/code-formatter"; import { closest_kernel_match, + cmp, + field_cmp, from_json, history_path, merge_copy, @@ -56,6 +57,20 @@ import { syncdbPath } from "@cocalc/util/jupyter/names"; import getKernelSpec from "@cocalc/frontend/jupyter/kernelspecs"; import { get as getUsageInfo } from "@cocalc/conat/project/usage-info"; import { delay } from "awaiting"; +import { until } from "@cocalc/util/async-utils"; +import { + jupyterClient, + type InputCell, +} from "@cocalc/conat/project/jupyter/run-code"; +import { OutputHandler } from "@cocalc/jupyter/execute/output-handler"; +import { throttle } from "lodash"; +import { + char_idx_to_js_idx, + codemirror_to_jupyter_pos, + js_idx_to_char_idx, +} from "@cocalc/jupyter/util/misc"; + +const OUTPUT_FPS = 29; // local cache: map project_id (string) -> kernels (immutable) let jupyter_kernels = Map(); @@ -66,11 +81,12 @@ export class JupyterActions extends JupyterActions0 { private cursor_manager: CursorManager; private account_change_editor_settings: any; private update_keyboard_shortcuts: any; - private syncdbPath: string; + public syncdbPath: string; + private lastCursorMoveTime: number = 0; + public jupyterEditorActions?; protected init2(): void { this.syncdbPath = syncdbPath(this.path); - this.update_contents = debounce(this.update_contents.bind(this), 2000); this.setState({ toolbar: !this.get_local_storage("hide_toolbar"), cell_toolbar: this.get_local_storage("cell_toolbar"), @@ -100,13 +116,16 @@ export class JupyterActions extends JupyterActions0 { this.syncdb.on("connected", this.sync_read_only); // first update - this.syncdb.once("change", this.updateContentsNow); - this.syncdb.once("change", this.updateRunProgress); + this.syncdb.once("change", () => { + this.updateContentsNow(); + this.updateRunProgress(); + this.ensurePositionsAreUnique(); + }); this.syncdb.on("change", () => { // And activity indicator this.activity(); - // Update table of contents + // Update table of contents -- this is debounced this.update_contents(); // run progress this.updateRunProgress(); @@ -114,21 +133,11 @@ export class JupyterActions extends JupyterActions0 { this.fetch_jupyter_kernels(); - // Load kernel (once ipynb file loads). - (async () => { - await this.set_kernel_after_load(); - if (!this.store) return; - track("jupyter", { - kernel: this.store.get("kernel"), - project_id: this.project_id, - path: this.path, - }); - })(); - // nbgrader support this.nbgrader_actions = new NBGraderActions(this, this.redux); this.syncdb.once("ready", () => { + this._syncdb_init_kernel(); const ipywidgets_state = this.syncdb.ipywidgets_state; if (ipywidgets_state == null) { throw Error("bug -- ipywidgets_state must be defined"); @@ -208,32 +217,6 @@ export class JupyterActions extends JupyterActions0 { } }; - public run_cell( - id: string, - save: boolean = true, - no_halt: boolean = false, - ): void { - if (this.store.get("read_only")) return; - const cell = this.store.getIn(["cells", id]); - if (cell == null) { - // it is trivial to run a cell that does not exist -- nothing needs to be done. - return; - } - - const cell_type = cell.get("cell_type", "code"); - if (cell_type === "code") { - const code = this.get_cell_input(id).trim(); - if (!code) { - this.clear_cell(id, save); - return; - } - this.run_code_cell(id, save, no_halt); - if (save) { - this.save_asap(); - } - } - } - private async api_call_formatter( str: string, config: FormatterConfig, @@ -328,7 +311,8 @@ export class JupyterActions extends JupyterActions0 { } public async close(): Promise { - if (this.is_closed()) return; + if (this.isClosed()) return; + this.jupyterClient?.close(); await super.close(); } @@ -354,10 +338,9 @@ export class JupyterActions extends JupyterActions0 { }; protected close_client_only(): void { - const account = this.redux.getStore("account"); - if (account != null) { - account.removeListener("change", this.account_change); - } + this.redux + ?.getStore("account") + ?.removeListener("change", this.account_change); } private syncdb_cursor_activity = (): void => { @@ -996,9 +979,10 @@ export class JupyterActions extends JupyterActions0 { this.setState({ contents }); }; - public update_contents(): void { + update_contents = debounce(() => { + if (this.isClosed()) return; this.updateContentsNow(); - } + }, 2000); protected __syncdb_change_post_hook(_doInit: boolean) { if (this._state === "init") { @@ -1147,22 +1131,22 @@ export class JupyterActions extends JupyterActions0 { }); }; - private set_kernel_after_load = async (): Promise => { - // Browser Client: Wait until the .ipynb file has actually been parsed into - // the (hidden, e.g. .a.ipynb.sage-jupyter2) syncdb file, - // then set the kernel, if necessary. - try { - await this.syncdb.wait((s) => !!s.get_one({ type: "file" }), 600); - } catch (err) { - if (this._state != "ready") { - // Probably user just closed the notebook before it finished - // loading, so we don't need to set the kernel. - return; - } - throw Error("error waiting for ipynb file to load"); - } - this._syncdb_init_kernel(); - }; + // private set_kernel_after_load = async (): Promise => { + // // Browser Client: Wait until the .ipynb file has actually been parsed into + // // the (hidden, e.g. .a.ipynb.sage-jupyter2) syncdb file, + // // then set the kernel, if necessary. + // try { + // await this.syncdb.wait((s) => !!s.get_one({ type: "file" }), 600); + // } catch (err) { + // if (this._state != "ready") { + // // Probably user just closed the notebook before it finished + // // loading, so we don't need to set the kernel. + // return; + // } + // throw Error("error waiting for ipynb file to load"); + // } + // this._syncdb_init_kernel(); + // }; private _syncdb_init_kernel = (): void => { // console.log("jupyter::_syncdb_init_kernel", this.store.get("kernel")); @@ -1171,8 +1155,8 @@ export class JupyterActions extends JupyterActions0 { // we either let the user select a kernel, or use a stored one let using_default_kernel = false; - const account_store = this.redux.getStore("account") as any; - const editor_settings = account_store.get("editor_settings") as any; + const account_store = this.redux.getStore("account"); + const editor_settings = account_store.get("editor_settings"); if ( editor_settings != null && !editor_settings.get("ask_jupyter_kernel") @@ -1205,7 +1189,7 @@ export class JupyterActions extends JupyterActions0 { } }; - set_kernel = (kernel: string | null) => { + set_kernel = async (kernel: string | null) => { if (this.syncdb.get_state() != "ready") { console.warn("Jupyter syncdb not yet ready -- not setting kernel"); return; @@ -1221,8 +1205,14 @@ export class JupyterActions extends JupyterActions0 { if (this.store.get("show_kernel_selector") || kernel === "") { this.hide_select_kernel(); } - if (kernel === "") { - this.halt(); // user "detaches" kernel from notebook, we stop the kernel + try { + if (kernel === "") { + await this.halt(); // user "detaches" kernel from notebook, we stop the kernel + } else { + await this.restart(); + } + } catch (err) { + console.warn(err); } }; @@ -1344,6 +1334,17 @@ export class JupyterActions extends JupyterActions0 { return export_to_ipynb({ ...options, blob_store: blob_store2 }); }; + private saveIpynb = async () => { + if (this.isClosed()) return; + const ipynb = await this.toIpynb(); + const serialize = JSON.stringify(ipynb, undefined, 2); + this.syncdb.fs.writeFile(this.path, serialize); + }; + + save = async () => { + await Promise.all([this.saveIpynb(), this.syncdb.save_to_disk()]); + }; + private getBase64Blobs = async (cells) => { const blobs: { [hash: string]: string } = {}; const failed = new Set(); @@ -1447,4 +1448,519 @@ export class JupyterActions extends JupyterActions0 { } return; }; + + // if the project or compute server is running and listening, this call + // tells them to open this jupyter notebook, so it can provide the compute + // functionality. + + private jupyterApi = async () => { + const compute_server_id = await this.getComputeServerId(); + const api = webapp_client.project_client.conatApi( + this.project_id, + compute_server_id, + ); + return api.jupyter; + }; + + initBackend = async () => { + await until( + async () => { + if (this.is_closed()) { + return true; + } + try { + const api = await this.jupyterApi(); + await api.start(this.syncdbPath); + return true; + } catch (err) { + console.log("failed to initialize ", this.path, err); + return false; + } + }, + { min: 3000 }, + ); + }; + + stopBackend = async () => { + const api = await this.jupyterApi(); + await api.stop(this.syncdbPath); + }; + + getOutputHandler = (cell) => { + const handler = new OutputHandler({ cell }); + + // save first time, so that other clients know this cell is running. + let first = true; + const f = throttle( + () => { + // we ONLY set certain fields; e.g., setting the input would be + // extremely annoying since the user can edit the input while the + // cell is running. + const { id, state, output, start, end, exec_count } = cell; + this._set({ id, state, output, start, end, exec_count }, first); + first = false; + }, + 1000 / OUTPUT_FPS, + { + leading: false, + trailing: true, + }, + ); + handler.on("change", f); + return handler; + }; + + private addPendingCells = (ids: string[]) => { + let pendingCells = this.store.get("pendingCells") ?? iSet(); + for (const id of ids) { + pendingCells = pendingCells.add(id); + } + this.store.setState({ pendingCells }); + }; + private deletePendingCells = (ids: string[]) => { + let pendingCells = this.store.get("pendingCells"); + if (pendingCells == null) { + return; + } + for (const id of ids) { + pendingCells = pendingCells.delete(id); + } + this.store.setState({ pendingCells }); + }; + + // uses inheritence so NOT arrow function + protected clearRunQueue() { + this.store?.setState({ pendingCells: iSet() }); + this.runQueue.length = 0; + } + + private jupyterClient?; + private runQueue: any[] = []; + private runningNow = false; + runCells = async (ids: string[], opts: { noHalt?: boolean } = {}) => { + if (this.store?.get("read_only")) { + return; + } + if (this.runningNow) { + this.runQueue.push([ids, opts]); + this.addPendingCells(ids); + return; + } + try { + this.runningNow = true; + if ( + this.jupyterClient == null || + this.jupyterClient.socket.state == "closed" + ) { + // [ ] **TODO: Must invalidate this when compute server changes!!!!!** + // and + const compute_server_id = await this.getComputeServerId(); + if (this.isClosed()) return; + this.jupyterClient = jupyterClient({ + path: this.syncdbPath, + client: webapp_client.conat_client.conat(), + project_id: this.project_id, + compute_server_id, + stdin: async ({ id, prompt, password }) => { + // set the redux store so that it is known we would like some stdin, + // wait for the user to respond, and return the result. + this.setState({ stdin: { id, prompt, password } }); + try { + const [input] = await once(this.store, "stdin"); + this.setState({ stdin: undefined }); + return input; + } catch (err) { + return `${err}`; + } + }, + }); + this.jupyterClient.socket.on("closed", () => { + delete this.jupyterClient; + // TODO: doing this is not ideal, but it's probably less confusing. + this.clearRunQueue(); + this.runningNow = false; + }); + } + const client = this.jupyterClient; + if (client == null) { + throw Error("bug"); + } + const cells: InputCell[] = []; + const kernel = this.store.get("kernel"); + + for (const id of ids) { + const cell = this.store.getIn(["cells", id])?.toJS() as InputCell; + if ((cell?.cell_type ?? "code") != "code") { + // code is the default type + continue; + } + if (!cell?.input?.trim()) { + // nothing to do + continue; + } + if (!kernel) { + this._set({ type: "cell", id, state: "done" }); + continue; + } + if (cell.output) { + // trick to avoid flicker + for (const n in cell.output) { + if (n == "0") continue; + cell.output[n] = null; + } + // time last evaluation took + const last = cell.start && cell.end ? cell.end - cell.start : null; + this._set({ id: cell.id, last, output: cell.output }, false); + } + cells.push(cell); + } + this.addPendingCells(cells.map(({ id }) => id)); + + // ensures cells run in order: + cells.sort(field_cmp("pos")); + + const runner = await client.run(cells, opts); + if (this.isClosed()) return; + let handler: null | OutputHandler = null; + let id: null | string = null; + for await (const mesgs of runner) { + if (this.isClosed()) return; + for (const mesg of mesgs) { + if (!opts.noHalt && mesg.msg_type == "error") { + this.clearRunQueue(); + } + if (mesg.id !== id || handler == null) { + id = mesg.id; + if (id == null) { + continue; + } + this.deletePendingCells([id]); + let cell = this.store.getIn(["cells", mesg.id])?.toJS(); + if (cell == null) { + // cell removed? + cell = { id }; + } + cell.kernel = kernel; + handler?.done(); + handler = this.getOutputHandler(cell); + } + handler.process(mesg); + } + } + handler?.done(); + if (this.isClosed()) { + return; + } + this.syncdb.save(); + setTimeout(() => { + if (!this.isClosed()) { + this.syncdb.save(); + } + }, 1000); + } catch (err) { + console.warn("runCells", err); + this.clearRunQueue(); + this.set_error(err); + } finally { + if (this.isClosed()) return; + this.runningNow = false; + if (this.runQueue.length > 0) { + const [ids, opts] = this.runQueue.shift(); + this.runCells(ids, opts); + } + } + }; + + is_introspecting(): boolean { + const actions = this.getFrameActions(); + return actions?.store?.get("introspect") != null; + } + + introspect_close = () => { + if (this.is_introspecting()) { + this.getFrameActions()?.setState({ introspect: undefined }); + } + }; + + introspect_at_pos = async ( + code: string, + detail_level: 0 | 1 = 0, + pos: { ch: number; line: number }, + ): Promise => { + if (code === "") return; // no-op if there is no code (should never happen) + await this.introspect( + code, + detail_level, + codemirror_to_jupyter_pos(code, pos), + ); + }; + + private introspectRequest: number = 0; + introspect = async ( + code: string, + detail_level: 0 | 1, + cursor_pos?: number, + ): Promise | undefined> => { + this.introspectRequest++; + const req = this.introspectRequest; + if (cursor_pos == null) { + cursor_pos = code.length; + } + cursor_pos = js_idx_to_char_idx(cursor_pos, code); + + let introspect; + try { + const api = await this.jupyterApi(); + introspect = await api.introspect({ + path: this.path, + code, + cursor_pos, + detail_level, + }); + if (introspect.status !== "ok") { + introspect = { error: "completion failed" }; + } + delete introspect.status; + } catch (err) { + introspect = { error: err }; + } + if (this.introspectRequest > req) return; + this.getFrameActions()?.setState({ introspect }); + return introspect; // convenient / useful, e.g., for use by whiteboard. + }; + + clear_introspect = (): void => { + this.introspectRequest = + (this.introspectRequest != null ? this.introspectRequest : 0) + 1; + this.getFrameActions()?.setState({ introspect: undefined }); + }; + + /* + complete: + + Attempt to fetch completions for give code and cursor_pos + If successful, the completions are put in store.get('completions') and looks + like this (as an immutable map): + cursor_end : 2 + cursor_start : 0 + matches : ['the', 'completions', ...] + status : "ok" + code : code + cursor_pos : cursor_pos + + If not successful, result is: + status : "error" + code : code + cursor_pos : cursor_pos + error : 'an error message' + + Only the most recent fetch has any impact, and calling + clear_complete() ensures any fetch made before that + is ignored. + + // Returns true if a dialog with options appears, and false otherwise. + */ + private completeRequest = 0; + complete = async ( + code: string, + pos?: { line: number; ch: number } | number, + id?: string, + offset?: any, + ): Promise => { + this.completeRequest++; + const req = this.completeRequest; + this.setState({ complete: undefined }); + + // pos can be either a {line:?, ch:?} object as in codemirror, + // or a number. + let cursor_pos; + if (pos == null || typeof pos == "number") { + cursor_pos = pos; + } else { + cursor_pos = codemirror_to_jupyter_pos(code, pos); + } + cursor_pos = js_idx_to_char_idx(cursor_pos, code); + + const start = Date.now(); + let complete; + try { + const api = await this.jupyterApi(); + complete = await api.complete({ + path: this.path, + code, + cursor_pos, + }); + } catch (err) { + if (this.completeRequest > req) return false; + this.setState({ complete: { error: err } }); + throw Error(`ignore -- ${err}`); + } + + if (this.lastCursorMoveTime >= start) { + // see https://github.com/sagemathinc/cocalc/issues/3611 + throw Error("ignore"); + } + if (this.completeRequest > req) { + // future completion or clear happened; so ignore this result. + throw Error("ignore"); + } + + if (complete.status !== "ok") { + this.setState({ + complete: { + error: complete.error ? complete.error : "completion failed", + }, + }); + return false; + } + + if (complete.matches == 0) { + return false; + } + + delete complete.status; + complete.base = code; + complete.code = code; + complete.pos = char_idx_to_js_idx(cursor_pos, code); + complete.cursor_start = char_idx_to_js_idx(complete.cursor_start, code); + complete.cursor_end = char_idx_to_js_idx(complete.cursor_end, code); + complete.id = id; + // Set the result so the UI can then react to the change. + if (offset != null) { + complete.offset = offset; + } + // For some reason, sometimes complete.matches are not unique, which is annoying/confusing, + // and breaks an assumption in our react code too. + // I think the reason is e.g., a filename and a variable could be the same. We're not + // worrying about that now. + complete.matches = Array.from(new Set(complete.matches)); + // sort in a way that matches how JupyterLab sorts completions, which + // is case insensitive with % magics at the bottom + complete.matches.sort((x, y) => { + const c = cmp(getCompletionGroup(x), getCompletionGroup(y)); + if (c) { + return c; + } + return cmp(x.toLowerCase(), y.toLowerCase()); + }); + const i_complete = fromJS(complete); + if (complete.matches && complete.matches.length === 1 && id != null) { + // special case -- a unique completion and we know id of cell in which completing is given. + this.select_complete(id, complete.matches[0], i_complete); + return false; + } else { + this.setState({ complete: i_complete }); + return true; + } + }; + + clear_complete = (): void => { + this.completeRequest = + (this.completeRequest != null ? this.completeRequest : 0) + 1; + this.setState({ complete: undefined }); + }; + + public select_complete( + id: string, + item: string, + complete?: Map, + ): void { + if (complete == null) { + complete = this.store.get("complete"); + } + this.clear_complete(); + if (complete == null) { + return; + } + const input = complete.get("code"); + if (input != null && complete.get("error") == null) { + const starting = input.slice(0, complete.get("cursor_start")); + const ending = input.slice(complete.get("cursor_end")); + const new_input = starting + item + ending; + const base = complete.get("base"); + this.complete_cell(id, base, new_input); + } + } + + complete_cell = (id: string, base: string, new_input: string): void => { + this.merge_cell_input(id, base, new_input); + }; + + set_cursor_locs = (locs: any[] = [], side_effect: boolean = false): void => { + this.lastCursorMoveTime = Date.now(); + if (this.syncdb == null) { + // syncdb not always set -- https://github.com/sagemathinc/cocalc/issues/2107 + return; + } + if (locs.length === 0) { + // don't remove on blur -- cursor will fade out just fine + return; + } + this._cursor_locs = locs; // remember our own cursors for splitting cell + this.syncdb.set_cursor_locs(locs, side_effect); + }; + + signal = async (signal = "SIGINT"): Promise => { + const api = await this.jupyterApi(); + try { + await api.signal({ path: this.path, signal }); + this.clear_all_cell_run_state(); + } catch (err) { + this.set_error(err); + } + }; + + // Kill the running kernel and does NOT start it up again. + halt = reuseInFlight(async (): Promise => { + if (this.restartKernelOnClose != null && this.jupyter_kernel != null) { + this.jupyter_kernel.removeListener("closed", this.restartKernelOnClose); + delete this.restartKernelOnClose; + } + this.clear_all_cell_run_state(); + await this.signal("SIGKILL"); + // Wait a little, since SIGKILL has to really happen on backend, + // and server has to respond and change state. + const not_running = (s): boolean => { + if (this._state === "closed") return true; + const t = s.get_one({ type: "settings" }); + return t != null && t.get("backend_state") != "running"; + }; + try { + await this.syncdb.wait(not_running, 30); + // worked -- and also no need to show "kernel got killed" message since this was intentional. + this.set_error(""); + } catch (err) { + // failed + this.set_error(err); + } + }); + + restart = reuseInFlight(async (): Promise => { + await this.halt(); + if (this.is_closed()) return; + this.clear_all_cell_run_state(); + }); + + shutdown = reuseInFlight(async (): Promise => { + if (this.is_closed()) return; + await this.signal("SIGKILL"); + if (this.is_closed()) return; + this.clear_all_cell_run_state(); + }); + + getConnectionFile = async (): Promise => { + const api = await this.jupyterApi(); + return await api.getConnectionFile({ path: this.path }); + }; +} + +function getCompletionGroup(x: string): number { + switch (x[0]) { + case "_": + return 1; + case "%": + return 2; + default: + return 0; + } } diff --git a/src/packages/frontend/jupyter/cell-input.tsx b/src/packages/frontend/jupyter/cell-input.tsx index cb7f1606669..c2455877116 100644 --- a/src/packages/frontend/jupyter/cell-input.tsx +++ b/src/packages/frontend/jupyter/cell-input.tsx @@ -71,6 +71,7 @@ export interface CellInputProps { computeServerId?: number; setShowAICellGen?: (show: Position) => void; dragHandle?: React.JSX.Element; + isPending?: boolean; } export const CellInput: React.FC = React.memo( @@ -94,7 +95,7 @@ export const CellInput: React.FC = React.memo( = React.memo( next.index !== cur.index || next.computeServerId != cur.computeServerId || next.dragHandle !== cur.dragHandle || + next.isPending !== cur.isPending || (next.cell_toolbar === "slideshow" && next.cell.get("slide") !== cur.cell.get("slide")) ), diff --git a/src/packages/frontend/jupyter/cell-list.tsx b/src/packages/frontend/jupyter/cell-list.tsx index d8a717943e8..4702558137b 100644 --- a/src/packages/frontend/jupyter/cell-list.tsx +++ b/src/packages/frontend/jupyter/cell-list.tsx @@ -67,6 +67,7 @@ const BOTTOM_PADDING_CELL = ( interface CellListProps { actions?: JupyterActions; // if not defined, then everything is read only cell_list: immutable.List; // list of ids of cells in order + stdin?; cell_toolbar?: string; cells: immutable.Map; cm_options: immutable.Map; @@ -91,12 +92,14 @@ interface CellListProps { llmTools?: LLMTools; computeServerId?: number; read_only?: boolean; + pendingCells?: immutable.Set; } export const CellList: React.FC = (props: CellListProps) => { const { actions, cell_list, + stdin, cell_toolbar, cells, cm_options, @@ -121,6 +124,7 @@ export const CellList: React.FC = (props: CellListProps) => { llmTools, computeServerId, read_only, + pendingCells, } = props; const cellListDivRef = useRef(null); @@ -434,7 +438,7 @@ export const CellList: React.FC = (props: CellListProps) => { if (index == null) { index = cell_list.indexOf(id) ?? 0; } - const dragHandle = actions?.store.is_cell_editable(id) ? ( + const dragHandle = actions?.store?.is_cell_editable(id) ? ( = (props: CellListProps) => {
= (props: CellListProps) => { dragHandle={dragHandle} read_only={read_only} isDragging={isDragging} + isPending={pendingCells?.has(id)} />
); diff --git a/src/packages/frontend/jupyter/cell-output-time.tsx b/src/packages/frontend/jupyter/cell-output-time.tsx index e4e7c031dcd..88ddaf5ee9e 100644 --- a/src/packages/frontend/jupyter/cell-output-time.tsx +++ b/src/packages/frontend/jupyter/cell-output-time.tsx @@ -20,8 +20,15 @@ interface CellTimingProps { kernel?: string; } -// make this small so smooth. -const DELAY_MS = 100; +const DELAY_MS = 1000; + +function humanReadableSeconds(s) { + if (s >= 0.9) { + return seconds2hms(s, true); + } else { + return `${Math.round(s * 1000)} ms`; + } +} export default function CellTiming({ start, @@ -53,10 +60,12 @@ export default function CellTiming({ - Evaluated using {capitalize(kernel)} and took - about {seconds2hms(ms / 1000, true)}. + Took about {humanReadableSeconds(ms / 1000)}. Evaluated{" "} + + {kernel ? " using " : ""} + {capitalize(kernel)}. {last != null ? ( - <> Previous run took {seconds2hms(last / 1000, true)}. + <> Previous run took {humanReadableSeconds(last / 1000)}. ) : undefined} } diff --git a/src/packages/frontend/jupyter/cell-output.tsx b/src/packages/frontend/jupyter/cell-output.tsx index 81d5fb2e03d..6d89a229ffa 100644 --- a/src/packages/frontend/jupyter/cell-output.tsx +++ b/src/packages/frontend/jupyter/cell-output.tsx @@ -16,6 +16,7 @@ import { CellHiddenPart } from "./cell-hidden-part"; import { CollapsedOutput, OutputToggle } from "./cell-output-toggle"; import { CellOutputMessages } from "./output-messages/message"; import { OutputPrompt } from "./prompt/output"; +import RawInput from "./raw-input"; interface CellOutputProps { actions?: JupyterActions; @@ -32,6 +33,7 @@ interface CellOutputProps { divRef?; llmTools?: LLMTools; isDragging?: boolean; + stdin?; } export function CellOutput({ @@ -49,6 +51,7 @@ export function CellOutput({ style, llmTools, isDragging, + stdin, }: CellOutputProps) { const minHeight = complete ? "60vh" : undefined; @@ -64,7 +67,7 @@ export function CellOutput({ ); } - if (cell.get("output") == null) { + if (cell.get("output") == null && !stdin) { return
; } @@ -84,25 +87,28 @@ export function CellOutput({ "cocalc-output-div" /* used by stable unsafe html for clipping */ } > - {!hidePrompt && ( - } +
+ - )} - + {stdin && ( + + )} +
); } @@ -212,11 +218,7 @@ function ControlColumn({ actions, cell, id }) { } if (actions != null) { return ( - + {prompt} ); diff --git a/src/packages/frontend/jupyter/cell.tsx b/src/packages/frontend/jupyter/cell.tsx index 3d31309d077..329a64e9c5f 100644 --- a/src/packages/frontend/jupyter/cell.tsx +++ b/src/packages/frontend/jupyter/cell.tsx @@ -32,12 +32,12 @@ import { INPUT_PROMPT_COLOR } from "./prompt/base"; interface Props { cell: Map; // TODO: types + stdin?; cm_options: Map; mode: "edit" | "escape"; font_size: number; id?: string; // redundant, since it's in the cell. actions?: JupyterActions; - name?: string; index?: number; // position of cell in the list of all cells; just used to optimize rendering and for no other reason. is_current?: boolean; is_selected?: boolean; @@ -62,12 +62,15 @@ interface Props { dragHandle?: React.JSX.Element; read_only?: boolean; isDragging?: boolean; + isPending?: boolean; + name?: string; } function areEqual(props: Props, nextProps: Props): boolean { // note: we assume project_id and directory don't change return !( nextProps.id !== props.id || + nextProps.stdin !== props.stdin || nextProps.index !== props.index || nextProps.cm_options !== props.cm_options || nextProps.cell !== props.cell || @@ -91,7 +94,8 @@ function areEqual(props: Props, nextProps: Props): boolean { (nextProps.is_current || props.is_current)) || nextProps.dragHandle !== props.dragHandle || nextProps.read_only !== props.read_only || - nextProps.isDragging !== props.isDragging + nextProps.isDragging !== props.isDragging || + nextProps.isPending !== props.isPending ); } @@ -138,6 +142,7 @@ export const Cell: React.FC = React.memo((props: Props) => { computeServerId={props.computeServerId} setShowAICellGen={setShowAICellGen} dragHandle={props.dragHandle} + isPending={props.isPending} /> ); } @@ -162,6 +167,7 @@ export const Cell: React.FC = React.memo((props: Props) => { complete={props.is_current && props.complete != null} llmTools={props.llmTools} isDragging={props.isDragging} + stdin={props.stdin} /> ); } diff --git a/src/packages/frontend/jupyter/edit-attachments.tsx b/src/packages/frontend/jupyter/edit-attachments.tsx index 7011c8d89a9..f08cbf2cbcc 100644 --- a/src/packages/frontend/jupyter/edit-attachments.tsx +++ b/src/packages/frontend/jupyter/edit-attachments.tsx @@ -11,8 +11,9 @@ import { Icon } from "../components"; import { Button, Modal } from "antd"; import { Map as ImmutableMap } from "immutable"; import { JupyterActions } from "./browser-actions"; +import { type JSX } from "react"; -const ROW_STYLE: React.CSSProperties = { +const ROW_STYLE = { display: "flex", border: "1px solid #ddd", padding: "7px", @@ -25,47 +26,45 @@ interface EditAttachmentsProps { } export function EditAttachments({ actions, cell }: EditAttachmentsProps) { - function close() { - actions.setState({ edit_attachments: undefined }); - actions.focus(true); + if (cell == null) { + return null; } - - function renderAttachments() { - const v: any[] = []; - if (cell) { - const attachments = cell.get("attachments"); - if (attachments) { - attachments.forEach((_, name) => { - if (v.length > 0) { - v.push(
); - } - return v.push( -
-
{name}
-
- -
-
- ); - }); + const v: JSX.Element[] = []; + const attachments = cell.get("attachments"); + if (attachments) { + attachments.forEach((_, name) => { + if (v.length > 0) { + v.push(
); } - } - if (v.length === 0) { - return ( - - There are no attachments. To attach images, use Edit → Insert - Image. - + return v.push( +
+
{name}
+
+ +
+
, ); - } - return v; + }); + } + if (v.length === 0) { + return ( + + There are no attachments. To attach images, use Edit → Insert + Image. + + ); + } + + function close() { + actions.setState({ edit_attachments: undefined }); + actions.focus(true); } return ( @@ -79,7 +78,7 @@ export function EditAttachments({ actions, cell }: EditAttachmentsProps) { } > - {renderAttachments()} + {v} ); } diff --git a/src/packages/frontend/jupyter/kernelspecs.ts b/src/packages/frontend/jupyter/kernelspecs.ts index 596e55b57a8..5a1db32a2b5 100644 --- a/src/packages/frontend/jupyter/kernelspecs.ts +++ b/src/packages/frontend/jupyter/kernelspecs.ts @@ -38,7 +38,7 @@ const getKernelSpec = reuseInFlight( compute_server_id, timeout: 7500, }); - const spec = await api.editor.jupyterKernels(); + const spec = await api.jupyter.kernels(); cache.set(key, spec); return spec; }, diff --git a/src/packages/frontend/jupyter/logo.tsx b/src/packages/frontend/jupyter/logo.tsx index fe26b5f15ec..963a47650e3 100644 --- a/src/packages/frontend/jupyter/logo.tsx +++ b/src/packages/frontend/jupyter/logo.tsx @@ -113,7 +113,7 @@ async function getLogo({ return cache[key]; } const api = client.conat_client.projectApi({ project_id }); - const { filename, base64 } = await api.editor.jupyterKernelLogo(kernel, { + const { filename, base64 } = await api.jupyter.kernelLogo(kernel, { noCache, }); if (!filename || !base64) { diff --git a/src/packages/frontend/jupyter/main.tsx b/src/packages/frontend/jupyter/main.tsx index 819b519e693..ea4f0ef578b 100644 --- a/src/packages/frontend/jupyter/main.tsx +++ b/src/packages/frontend/jupyter/main.tsx @@ -12,11 +12,10 @@ import { CSS, React, redux, - Rendered, useRedux, - useRef, useTypedRedux, } from "@cocalc/frontend/app-framework"; +import { useRef } from "react"; // Support for all the MIME types import { Button, Tooltip } from "antd"; import "./output-messages/mime-types/init-frontend"; @@ -117,7 +116,6 @@ export const JupyterEditor: React.FC = React.memo((props: Props) => { "show_kernel_selector", ]); // string name of the kernel - const kernels: undefined | KernelsType = useRedux([name, "kernels"]); const kernelspec = useRedux([name, "kernel_info"]); const error: undefined | KernelsType = useRedux([name, "error"]); // settings for all the codemirror editors @@ -136,6 +134,10 @@ export const JupyterEditor: React.FC = React.memo((props: Props) => { name, "cell_list", ]); + + // if there is a stdin request: + const stdin = useRedux([name, "stdin"]); + // map from ids to cells const cells: undefined | immutable.Map = useRedux([ name, @@ -183,6 +185,10 @@ export const JupyterEditor: React.FC = React.memo((props: Props) => { name, "check_select_kernel_init", ]); + const pendingCells: undefined | immutable.Set = useRedux([ + name, + "pendingCells", + ]); const computeServerId = path ? useTypedRedux({ project_id }, "compute_server_ids")?.get(syncdbPath(path)) @@ -257,25 +263,11 @@ export const JupyterEditor: React.FC = React.memo((props: Props) => { ); } - function render_loading(): Rendered { - return ( - - ); - } - function render_cells() { if ( cell_list == null || font_size == null || cm_options == null || - kernels == null || cells == null ) { return ( @@ -295,6 +287,7 @@ export const JupyterEditor: React.FC = React.memo((props: Props) => { actions={actions} read_only={read_only} cell_list={cell_list} + stdin={stdin} cell_toolbar={cell_toolbar} cells={cells} cm_options={cm_options} @@ -318,99 +311,25 @@ export const JupyterEditor: React.FC = React.memo((props: Props) => { use_windowed_list={useWindowedListRef.current} llmTools={llmTools} computeServerId={computeServerId} + pendingCells={pendingCells} /> ); } - function render_about() { - return ( - - ); - } - - function render_nbconvert() { - if (path == null || project_id == null) return; - return ( - - ); - } - - function render_edit_attachments() { - if (edit_attachments == null || cells == null) { - return; - } - const cell = cells.get(edit_attachments); - if (cell == null) { - return; - } - return ; - } - - function render_edit_cell_metadata() { - if (edit_cell_metadata == null) { - return; - } - return ( - - ); - } - - function render_find_and_replace() { - if (cells == null || cur_id == null) { - return; - } - return ( - - ); - } - - function render_confirm_dialog() { - if (confirm_dialog == null || actions == null) return; - return ; - } - function render_select_kernel() { return ; } - function render_keyboard_shortcuts() { - if (actions == null) return; - return ( - - ); - } - function render_main() { if (!check_select_kernel_init) { - return render_loading(); + ; } else if (show_kernel_selector) { return render_select_kernel(); } else { @@ -422,13 +341,58 @@ export const JupyterEditor: React.FC = React.memo((props: Props) => { if (!is_focused) return; return ( <> - {render_about()} - {render_nbconvert()} - {render_edit_attachments()} - {render_edit_cell_metadata()} - {render_find_and_replace()} - {render_keyboard_shortcuts()} - {render_confirm_dialog()} + + {path != null && project_id != null && ( + + )} + {edit_attachments != null && ( + + )} + {edit_cell_metadata != null && ( + + )} + {cells != null && cur_id != null && ( + + )} + {actions != null && ( + + )} + {actions != null && confirm_dialog != null && ( + + )} ); } @@ -451,7 +415,10 @@ export const JupyterEditor: React.FC = React.memo((props: Props) => { overflowY: "hidden", }} > - + {!read_only && } {render_error()} {render_modals()} diff --git a/src/packages/frontend/jupyter/output-messages/ipywidget.tsx b/src/packages/frontend/jupyter/output-messages/ipywidget.tsx index b0c1a6f50b8..d0815ffcadc 100644 --- a/src/packages/frontend/jupyter/output-messages/ipywidget.tsx +++ b/src/packages/frontend/jupyter/output-messages/ipywidget.tsx @@ -146,7 +146,7 @@ ax.plot(x, y)
@@ -590,24 +520,9 @@ function CreateDirectory({ export async function pathExists( project_id: string, path: string, - directoryListings?, computeServerId?, ): Promise { - const { head, tail } = path_split(path); - let known = directoryListings?.get(head); - if (known == null) { - const actions = redux.getProjectActions(project_id); - await actions.fetch_directory_listing({ - path: head, - compute_server_id: computeServerId, - }); - } - known = directoryListings?.get(head); - if (known == null) { - return false; - } - for (const x of known) { - if (x.get("name") == tail) return true; - } - return false; + const actions = redux.getProjectActions(project_id); + const fs = actions.fs(computeServerId); + return await fs.exists(path); } diff --git a/src/packages/frontend/project/explorer/action-bar.tsx b/src/packages/frontend/project/explorer/action-bar.tsx index 7cfc6f58d5a..5c38b38eee3 100644 --- a/src/packages/frontend/project/explorer/action-bar.tsx +++ b/src/packages/frontend/project/explorer/action-bar.tsx @@ -5,22 +5,24 @@ import { Space } from "antd"; import * as immutable from "immutable"; -import { throttle } from "lodash"; -import React, { useEffect, useRef, useState } from "react"; +import React, { useRef } from "react"; import { FormattedMessage, useIntl } from "react-intl"; - import { Button, ButtonToolbar } from "@cocalc/frontend/antd-bootstrap"; import { Gap, Icon } from "@cocalc/frontend/components"; import { useStudentProjectFunctionality } from "@cocalc/frontend/course"; import { CustomSoftwareInfo } from "@cocalc/frontend/custom-software/info-bar"; -import { ComputeImages } from "@cocalc/frontend/custom-software/init"; +import { type ComputeImages } from "@cocalc/frontend/custom-software/init"; import { IS_MOBILE } from "@cocalc/frontend/feature"; import { labels } from "@cocalc/frontend/i18n"; -import { file_actions, ProjectActions } from "@cocalc/frontend/project_store"; +import { + file_actions, + type ProjectActions, + type FileAction, +} from "@cocalc/frontend/project_store"; import * as misc from "@cocalc/util/misc"; import { COLORS } from "@cocalc/util/theme"; - -import { useProjectContext } from "../context"; +import { DirectoryListingEntry } from "@cocalc/util/types"; +import { VisibleMDLG } from "@cocalc/frontend/components"; const ROW_INFO_STYLE = { color: COLORS.GRAY, @@ -31,11 +33,9 @@ const ROW_INFO_STYLE = { interface Props { project_id?: string; checked_files: immutable.Set; - listing: { name: string; isdir: boolean }[]; - page_number: number; - page_size: number; + listing: DirectoryListingEntry[]; current_path?: string; - project_map?: immutable.Map; + project_map?; images?: ComputeImages; actions: ProjectActions; available_features?; @@ -43,100 +43,47 @@ interface Props { project_is_running?: boolean; } -export const ActionBar: React.FC = (props: Props) => { +export function ActionBar({ + project_id, + checked_files, + listing, + current_path, + project_map, + images, + actions, + available_features, + show_custom_software_reset, + project_is_running, +}: Props) { const intl = useIntl(); - const [showLabels, setShowLabels] = useState(true); - const { mainWidthPx } = useProjectContext(); const buttonRef = useRef(null); - const widthThld = useRef(0); - const [select_entire_directory, set_select_entire_directory] = useState< - "hidden" | "check" | "clear" - >("hidden"); const student_project_functionality = useStudentProjectFunctionality( - props.actions.project_id, + actions.project_id, ); if (student_project_functionality.disableActions) { return
; } - useEffect(() => { - // user changed directory, hide the "select entire directory" button - if (select_entire_directory !== "hidden") { - set_select_entire_directory("hidden"); - } - }, [props.current_path]); - - useEffect(() => { - if ( - props.checked_files.size === props.listing.length && - select_entire_directory === "check" - ) { - // user just clicked the "select entire directory" button, show the "clear" button - set_select_entire_directory("clear"); - } - }, [props.checked_files, props.listing, select_entire_directory]); - - useEffect(() => { - const btnbar = buttonRef.current; - if (btnbar == null) return; - const resizeObserver = new ResizeObserver( - throttle( - (entries) => { - if (entries.length > 0) { - const width = entries[0].contentRect.width; - // TODO: this "+100" is sloppy. This makes it much better than before - // (e.g. german buttons were cutoff all the time), but could need more tweaking - if (showLabels && width > mainWidthPx + 100) { - setShowLabels(false); - widthThld.current = width; - } else if (!showLabels && width < widthThld.current - 1) { - setShowLabels(true); - } - } - }, - 100, - { leading: false, trailing: true }, - ), - ); - resizeObserver.observe(btnbar); - return () => { - resizeObserver.disconnect(); - }; - }, [mainWidthPx, buttonRef.current]); - function clear_selection(): void { - props.actions.set_all_files_unchecked(); - if (select_entire_directory !== "hidden") { - set_select_entire_directory("hidden"); - } + actions.set_all_files_unchecked(); } function check_all_click_handler(): void { - if (props.checked_files.size === 0) { - const files_on_page = props.listing.slice( - props.page_size * props.page_number, - props.page_size * (props.page_number + 1), - ); - props.actions.set_file_list_checked( - files_on_page.map((file) => - misc.path_to_file(props.current_path ?? "", file.name), - ), + if (checked_files.size === 0) { + actions.set_file_list_checked( + listing.map((file) => misc.path_to_file(current_path ?? "", file.name)), ); - if (props.listing.length > props.page_size) { - // if there are more items than one page, show a button to select everything - set_select_entire_directory("check"); - } } else { clear_selection(); } } function render_check_all_button(): React.JSX.Element | undefined { - if (props.listing.length === 0) { + if (listing.length === 0) { return; } - const checked = props.checked_files.size > 0; + const checked = checked_files.size > 0; const button_text = intl.formatMessage( { id: "project.explorer.action-bar.check_all.button", @@ -148,10 +95,10 @@ export const ActionBar: React.FC = (props: Props) => { ); let button_icon; - if (props.checked_files.size === 0) { + if (checked_files.size === 0) { button_icon = "square-o"; } else { - if (props.checked_files.size >= props.listing.length) { + if (checked_files.size >= listing.length) { button_icon = "check-square-o"; } else { button_icon = "minus-square-o"; @@ -169,37 +116,12 @@ export const ActionBar: React.FC = (props: Props) => { ); } - function do_select_entire_directory(): void { - props.actions.set_file_list_checked( - props.listing.map((file) => - misc.path_to_file(props.current_path ?? "", file.name), - ), - ); - } - - function render_select_entire_directory(): React.JSX.Element | undefined { - switch (select_entire_directory) { - case "check": - return ( - - ); - case "clear": - return ( - - ); - } - } - function render_currently_selected(): React.JSX.Element | undefined { - if (props.listing.length === 0) { + if (listing.length === 0) { return; } - const checked = props.checked_files.size; - const total = props.listing.length; + const checked = checked_files.size; + const total = listing.length; const style = ROW_INFO_STYLE; if (checked === 0) { @@ -237,27 +159,26 @@ export const ActionBar: React.FC = (props: Props) => { )} - {render_select_entire_directory()}
); } } - function render_action_button(name: string): React.JSX.Element { + function render_action_button(name: FileAction): React.JSX.Element { const disabled = isDisabledSnapshots(name) && - (props.current_path != null - ? props.current_path.startsWith(".snapshots") + (current_path != null + ? current_path.startsWith(".snapshots") : undefined); const obj = file_actions[name]; const handle_click = (_e: React.MouseEvent) => { - props.actions.set_file_action(name); + actions.set_file_action(name); }; return ( ); } @@ -273,21 +194,18 @@ export const ActionBar: React.FC = (props: Props) => { | "copy" | "share" )[]; - if (!props.project_is_running) { - return; - } - if (props.checked_files.size === 0) { + if (checked_files.size === 0) { return; - } else if (props.checked_files.size === 1) { - let isdir; - const item = props.checked_files.first(); - for (const file of props.listing) { - if (misc.path_to_file(props.current_path ?? "", file.name) === item) { - ({ isdir } = file); + } else if (checked_files.size === 1) { + let isDir; + const item = checked_files.first(); + for (const file of listing) { + if (misc.path_to_file(current_path ?? "", file.name) === item) { + ({ isDir } = file); } } - if (isdir) { + if (isDir) { // one directory selected action_buttons = [...ACTION_BUTTONS_DIR]; } else { @@ -306,25 +224,25 @@ export const ActionBar: React.FC = (props: Props) => { } function render_button_area(): React.JSX.Element | undefined { - if (props.checked_files.size === 0) { + if (checked_files.size === 0) { if ( - props.project_id == null || - props.images == null || - props.project_map == null || - props.available_features == null + project_id == null || + images == null || + project_map == null || + available_features == null ) { return; } return ( ); @@ -332,25 +250,21 @@ export const ActionBar: React.FC = (props: Props) => { return render_action_buttons(); } } - if (props.checked_files.size === 0 && IS_MOBILE) { + if (checked_files.size === 0 && IS_MOBILE) { return null; } return (
- - {props.project_is_running ? render_check_all_button() : undefined} - + {render_check_all_button()} {render_button_area()}
-
- {props.project_is_running ? render_currently_selected() : undefined} -
+
{render_currently_selected()}
); -}; +} export const ACTION_BUTTONS_DIR = [ "download", diff --git a/src/packages/frontend/project/explorer/action-box.tsx b/src/packages/frontend/project/explorer/action-box.tsx index 949a87c17b3..6ccc3389de4 100644 --- a/src/packages/frontend/project/explorer/action-box.tsx +++ b/src/packages/frontend/project/explorer/action-box.tsx @@ -9,7 +9,6 @@ import { Button as AntdButton, Radio, Space } from "antd"; import * as immutable from "immutable"; import { useState } from "react"; import { useIntl } from "react-intl"; - import { Alert, Button, @@ -19,16 +18,18 @@ import { Well, } from "@cocalc/frontend/antd-bootstrap"; import { useRedux, useTypedRedux } from "@cocalc/frontend/app-framework"; -import { Icon, Loading, LoginLink } from "@cocalc/frontend/components"; +import { Icon, LoginLink } from "@cocalc/frontend/components"; import SelectServer from "@cocalc/frontend/compute/select-server"; import ComputeServerTag from "@cocalc/frontend/compute/server-tag"; import { useRunQuota } from "@cocalc/frontend/project/settings/run-quota/hooks"; -import { file_actions, ProjectActions } from "@cocalc/frontend/project_store"; +import { + file_actions, + type ProjectActions, +} from "@cocalc/frontend/project_store"; import { SelectProject } from "@cocalc/frontend/projects/select-project"; import ConfigureShare from "@cocalc/frontend/share/config"; import * as misc from "@cocalc/util/misc"; import { COLORS } from "@cocalc/util/theme"; -import { useProjectContext } from "../context"; import DirectorySelector from "../directory-selector"; import { in_snapshot_path } from "../utils"; import CreateArchive from "./create-archive"; @@ -48,21 +49,22 @@ export const PRE_STYLE = { type FileAction = undefined | keyof typeof file_actions; -interface ReactProps { +interface Props { checked_files: immutable.Set; file_action: FileAction; current_path: string; project_id: string; - file_map: object; actions: ProjectActions; - displayed_listing?: object; - //new_name?: string; - name: string; } -export function ActionBox(props: ReactProps) { +export function ActionBox({ + checked_files, + file_action, + current_path, + project_id, + actions, +}: Props) { const intl = useIntl(); - const { project_id } = useProjectContext(); const runQuota = useRunQuota(project_id, null); const get_user_type: () => string = useRedux("account", "get_user_type"); const compute_server_id = useTypedRedux({ project_id }, "compute_server_id"); @@ -74,17 +76,22 @@ export function ActionBox(props: ReactProps) { const [copy_from_compute_server_to, set_copy_from_compute_server_to] = useState<"compute-server" | "project">("compute-server"); const [move_destination, set_move_destination] = useState(""); - //const [new_name, set_new_name] = useState(props.new_name ?? ""); const [show_different_project, set_show_different_project] = useState(false); - const [overwrite_newer, set_overwrite_newer] = useState(); - const [delete_extra_files, set_delete_extra_files] = useState(); + const [overwrite, set_overwrite] = useState(true); const [dest_compute_server_id, set_dest_compute_server_id] = useState( compute_server_id ?? 0, ); + function clear() { + actions.set_all_files_unchecked(); + setTimeout(() => { + actions.set_file_action(); + }, 1); + } + function cancel_action(): void { - props.actions.set_file_action(); + clear(); } function action_key(e): void { @@ -93,7 +100,7 @@ export function ActionBox(props: ReactProps) { cancel_action(); break; case 13: - switch (props.file_action) { + switch (file_action) { case "move": submit_action_move(); break; @@ -104,10 +111,10 @@ export function ActionBox(props: ReactProps) { } } - function render_selected_files_list(): React.JSX.Element { + function render_selected_files_list() { return (
-        {props.checked_files.toArray().map((name) => (
+        {checked_files.toArray().map((name) => (
           
{misc.path_split(name).tail}
))}
@@ -115,18 +122,16 @@ export function ActionBox(props: ReactProps) { } function delete_click(): void { - const paths = props.checked_files.toArray(); + const paths = checked_files.toArray(); for (const path of paths) { - props.actions.close_tab(path); + actions.close_tab(path); } - props.actions.delete_files({ paths }); - props.actions.set_file_action(); - props.actions.set_all_files_unchecked(); - props.actions.fetch_directory_listing(); + actions.deleteFiles({ paths }); + clear(); } - function render_delete_warning(): React.JSX.Element | undefined { - if (props.current_path === ".trash") { + function render_delete_warning() { + if (current_path === ".trash") { return ( @@ -140,8 +145,8 @@ export function ActionBox(props: ReactProps) { } } - function render_delete(): React.JSX.Element | undefined { - const { size } = props.checked_files; + function render_delete() { + const { size } = checked_files; return (
@@ -162,7 +167,7 @@ export function ActionBox(props: ReactProps) { href="" onClick={(e) => { e.preventDefault(); - props.actions.open_directory(".snapshots"); + actions.open_directory(".snapshots"); }} > ~/.snapshots @@ -179,7 +184,7 @@ export function ActionBox(props: ReactProps) { Delete {size} {misc.plural(size, "Item")} @@ -191,16 +196,15 @@ export function ActionBox(props: ReactProps) { } function move_click(): void { - props.actions.move_files({ - src: props.checked_files.toArray(), + actions.moveFiles({ + src: checked_files.toArray(), dest: move_destination, }); - props.actions.set_file_action(); - props.actions.set_all_files_unchecked(); + clear(); } function valid_move_input(): boolean { - const src_path = misc.path_split(props.checked_files.first()).head; + const src_path = misc.path_split(checked_files.first()).head; let dest = move_destination.trim(); if (dest === src_path) { return false; @@ -211,11 +215,11 @@ export function ActionBox(props: ReactProps) { if (dest.charAt(dest.length - 1) === "/") { dest = dest.slice(0, dest.length - 1); } - return dest !== props.current_path; + return dest !== current_path; } - function render_move(): React.JSX.Element { - const { size } = props.checked_files; + function render_move() { + const { size } = checked_files; return (
@@ -244,9 +248,9 @@ export function ActionBox(props: ReactProps) { onSelect={(move_destination: string) => set_move_destination(move_destination) } - project_id={props.project_id} - startingPath={props.current_path} - isExcluded={(path) => props.checked_files.has(path)} + project_id={project_id} + startingPath={current_path} + isExcluded={(path) => checked_files.has(path)} style={{ width: "100%" }} bodyStyle={{ maxHeight: "250px" }} /> @@ -262,13 +266,13 @@ export function ActionBox(props: ReactProps) { } } - function render_different_project_dialog(): React.JSX.Element | undefined { + function render_different_project_dialog() { if (show_different_project) { return (

Target Project

set_copy_destination_project_id(copy_destination_project_id) @@ -280,19 +284,12 @@ export function ActionBox(props: ReactProps) { } } - function render_copy_different_project_options(): React.JSX.Element | undefined { - if (props.project_id !== copy_destination_project_id) { + function render_copy_different_project_options() { + if (project_id !== copy_destination_project_id) { return (
- set_delete_extra_files((e.target as any).checked)} - > - Delete extra files in target directory - - set_overwrite_newer((e.target as any).checked)} - > - Overwrite newer versions of files + set_overwrite((e.target as any).checked)}> + Overwrite existing files or directories
); @@ -308,30 +305,29 @@ export function ActionBox(props: ReactProps) { function copy_click(): void { const destination_project_id = copy_destination_project_id; const destination_directory = copy_destination_directory; - const paths = props.checked_files.toArray(); + const paths = checked_files.toArray(); if ( destination_project_id != undefined && - props.project_id !== destination_project_id + project_id !== destination_project_id ) { - props.actions.copy_paths_between_projects({ - public: false, - src_project_id: props.project_id, - src: paths, - target_project_id: destination_project_id, - target_path: destination_directory, - overwrite_newer, - delete_missing: delete_extra_files, + actions.copyPathBetweenProjects({ + src: { project_id, path: paths }, + dest: { + project_id: destination_project_id, + path: destination_directory, + }, + options: { force: overwrite, recursive: true }, }); } else { if (compute_server_id) { - props.actions.copy_paths({ + actions.copyPaths({ src: paths, dest: destination_directory, src_compute_server_id: compute_server_id, dest_compute_server_id: getDestinationComputeServerId(), }); } else { - props.actions.copy_paths({ + actions.copyPaths({ src: paths, dest: destination_directory, src_compute_server_id: 0, @@ -340,11 +336,11 @@ export function ActionBox(props: ReactProps) { } } - props.actions.set_file_action(); + clear(); } function valid_copy_input(): boolean { - const src_path = misc.path_split(props.checked_files.first()).head; + const src_path = misc.path_split(checked_files.first()).head; const input = copy_destination_directory; const src_compute_server_id = compute_server_id ?? 0; @@ -352,7 +348,7 @@ export function ActionBox(props: ReactProps) { if ( input === src_path && - props.project_id === copy_destination_project_id && + project_id === copy_destination_project_id && src_compute_server_id == dest_compute_server_id ) { return false; @@ -361,8 +357,8 @@ export function ActionBox(props: ReactProps) { return false; } if ( - input === props.current_path && - props.project_id === copy_destination_project_id && + input === current_path && + project_id === copy_destination_project_id && src_compute_server_id == dest_compute_server_id ) { return false; @@ -374,7 +370,7 @@ export function ActionBox(props: ReactProps) { } function render_copy_description() { - for (const path of props.checked_files) { + for (const path of checked_files) { if (in_snapshot_path(path)) { return ( <> @@ -430,8 +426,8 @@ export function ActionBox(props: ReactProps) { ); } - function render_copy(): React.JSX.Element { - const { size } = props.checked_files; + function render_copy() { + const { size } = checked_files; const signed_in = get_user_type() === "signed_in"; if (!signed_in) { return ( @@ -499,7 +495,7 @@ export function ActionBox(props: ReactProps) {
set_dest_compute_server_id(dest_compute_server_id) @@ -514,7 +510,7 @@ export function ActionBox(props: ReactProps) { set_copy_destination_directory(value) } key="copy_destination_directory" - startingPath={props.current_path} + startingPath={current_path} project_id={copy_destination_project_id} style={{ width: "100%" }} bodyStyle={{ maxHeight: "250px" }} @@ -539,49 +535,39 @@ export function ActionBox(props: ReactProps) { } } - function render_share(): React.JSX.Element { + function render_share() { // currently only works for a single selected file - const path: string = props.checked_files.first() ?? ""; + const path: string = checked_files.first() ?? ""; if (!path) { - return <>; - } - const public_data = props.file_map[misc.path_split(path).tail]; - if (public_data == undefined) { - // directory listing not loaded yet... (will get re-rendered when loaded) - return ; + return null; } return ( props.actions.set_public_path(path, opts)} + onKeyUp={action_key} + actions={actions} has_network_access={!!runQuota.network} /> ); } - function render_action_box(action: FileAction): React.JSX.Element | undefined { + function render_action_box(action: FileAction) { switch (action) { case "compress": - return ; + return ; case "copy": return render_copy(); case "delete": return render_delete(); case "download": - return ; + return ; case "rename": - return ; + return ; case "duplicate": - return ; + return ; case "move": return render_move(); case "share": @@ -591,49 +577,45 @@ export function ActionBox(props: ReactProps) { } } - const action = props.file_action; + const action = file_action; const action_button = file_actions[action || "undefined"]; if (action_button == undefined) { return
Undefined action
; } - if (props.file_map == undefined) { - return ; - } else { - return ( - - - - {" "} - {intl.formatMessage(action_button.name)} -
- - - -
- {!!compute_server_id && ( - - )} - - {render_action_box(action)} -
-
- ); - } + return ( + + + + {" "} + {intl.formatMessage(action_button.name)} +
+ + + +
+ {!!compute_server_id && ( + + )} + + {render_action_box(action)} +
+
+ ); } diff --git a/src/packages/frontend/project/explorer/compute-file-masks.ts b/src/packages/frontend/project/explorer/compute-file-masks.ts index 6b8e4d95d08..50e6d94b67d 100644 --- a/src/packages/frontend/project/explorer/compute-file-masks.ts +++ b/src/packages/frontend/project/explorer/compute-file-masks.ts @@ -4,10 +4,10 @@ */ import { derive_rmd_output_filename } from "@cocalc/frontend/frame-editors/rmd-editor/utils"; -import { dict, filename_extension, startswith } from "@cocalc/util/misc"; +import { dict, filename_extension } from "@cocalc/util/misc"; import { DirectoryListing, DirectoryListingEntry } from "./types"; -const MASKED_FILENAMES = ["__pycache__"] as const; +const MASKED_FILENAMES = new Set(["__pycache__"]); const MASKED_FILE_EXTENSIONS = { py: ["pyc"], @@ -54,18 +54,19 @@ const MASKED_FILE_EXTENSIONS = { * the general outcome of this function is to set for some file entry objects * in "listing" the attribute .mask=true */ -export function compute_file_masks(listing: DirectoryListing): void { +export function computeFileMasks(listing: DirectoryListing): void { // map filename to file for easier lookup const filename_map: { [name: string]: DirectoryListingEntry } = dict( listing.map((item) => [item.name, item]), ); for (const file of listing) { - // mask certain known directories - if (MASKED_FILENAMES.indexOf(file.name as any) >= 0) { + // mask certain known paths + if (MASKED_FILENAMES.has(file.name as any)) { filename_map[file.name].mask = true; } - // note: never skip already masked files, because of rnw/rtex->tex + // NOTE: never skip already masked files, because of rnw/rtex->tex + const ext = filename_extension(file.name).toLowerCase(); // some extensions like Rmd modify the basename during compilation @@ -75,29 +76,29 @@ export function compute_file_masks(listing: DirectoryListing): void { for (let mask_ext of MASKED_FILE_EXTENSIONS[ext] ?? []) { // check each possible compiled extension - let bn; // derived basename + let derivedBasename; // some uppercase-strings have special meaning - if (startswith(mask_ext, "NODOT")) { - bn = basename.slice(0, -1); // exclude the trailing dot + if (mask_ext.startsWith("NODOT")) { + derivedBasename = basename.slice(0, -1); // exclude the trailing dot mask_ext = mask_ext.slice("NODOT".length); - } else if (mask_ext.indexOf("FILENAME") >= 0) { - bn = mask_ext.replace("FILENAME", filename); + } else if (mask_ext.includes("FILENAME")) { + derivedBasename = mask_ext.replace("FILENAME", filename); mask_ext = ""; - } else if (mask_ext.indexOf("BASENAME") >= 0) { - bn = mask_ext.replace("BASENAME", basename.slice(0, -1)); + } else if (mask_ext.includes("BASENAME")) { + derivedBasename = mask_ext.replace("BASENAME", basename.slice(0, -1)); mask_ext = ""; - } else if (mask_ext.indexOf("BASEDASHNAME") >= 0) { + } else if (mask_ext.includes("BASEDASHNAME")) { // BASEDASHNAME is like BASENAME, but replaces spaces by dashes // https://github.com/sagemathinc/cocalc/issues/3229 const fragment = basename.slice(0, -1).replace(/ /g, "-"); - bn = mask_ext.replace("BASEDASHNAME", fragment); + derivedBasename = mask_ext.replace("BASEDASHNAME", fragment); mask_ext = ""; } else { - bn = basename; + derivedBasename = basename; } - const mask_fn = `${bn}${mask_ext}`; - if (filename_map[mask_fn] != null) { - filename_map[mask_fn].mask = true; + const maskFilename = `${derivedBasename}${mask_ext}`; + if (filename_map[maskFilename] != null) { + filename_map[maskFilename].mask = true; } } } diff --git a/src/packages/frontend/project/explorer/create-archive.tsx b/src/packages/frontend/project/explorer/create-archive.tsx index a18c2d9fa09..f09c4bb7e90 100644 --- a/src/packages/frontend/project/explorer/create-archive.tsx +++ b/src/packages/frontend/project/explorer/create-archive.tsx @@ -1,4 +1,4 @@ -import { Button, Card, Input, Space, Spin } from "antd"; +import { Button, Card, Input, Select, Space, Spin } from "antd"; import { useEffect, useRef, useState } from "react"; import { useIntl } from "react-intl"; import { default_filename } from "@cocalc/frontend/account"; @@ -8,8 +8,15 @@ import { labels } from "@cocalc/frontend/i18n"; import { useProjectContext } from "@cocalc/frontend/project/context"; import { path_split, plural } from "@cocalc/util/misc"; import CheckedFiles from "./checked-files"; +import { join } from "path"; +import { OUCH_FORMATS } from "@cocalc/conat/files/fs"; -export default function CreateArchive({}) { +export const defaultFormat = OUCH_FORMATS.includes("tar.gz") + ? "tar.gz" + : OUCH_FORMATS[0]; + +export default function CreateArchive({ clear }) { + const [format, setFormat] = useState(""); const intl = useIntl(); const inputRef = useRef(null); const { actions } = useProjectContext(); @@ -41,21 +48,14 @@ export default function CreateArchive({}) { setLoading(true); const files = checked_files.toArray(); const path = store.get("current_path"); - await actions.zip_files({ - src: path ? files.map((x) => x.slice(path.length + 1)) : files, - dest: target + ".zip", - path, - }); - await actions.fetch_directory_listing({ path }); + await createArchive({ path, files, target, format, actions }); + clear(); } catch (err) { setLoading(false); setError(err); } finally { setLoading(false); } - - actions.set_all_files_unchecked(); - actions.set_file_action(); }; if (actions == null) { @@ -65,8 +65,8 @@ export default function CreateArchive({}) { return ( - Create a zip file from the following {checked_files?.size} selected{" "} - {plural(checked_files?.size, "item")} + Create a downloadable {format} archive from the following{" "} + {checked_files?.size} selected {plural(checked_files?.size, "item")} > @@ -76,9 +76,9 @@ export default function CreateArchive({}) { autoFocus onChange={(e) => setTarget(e.target.value)} value={target} - placeholder="Name of zip archive..." + placeholder="Name of archive..." onPressEnter={doCompress} - suffix=".zip" + suffix={"." + format} />
+ - + ); } + +export async function createArchive({ path, files, target, format, actions }) { + const fs = actions.fs(); + const { code, stderr } = await fs.ouch([ + "compress", + ...files, + join(path, target + "." + format), + ]); + if (code) { + throw Error(Buffer.from(stderr).toString()); + } +} + +export function SelectFormat({ format, setFormat }) { + useEffect(() => { + if (!OUCH_FORMATS.includes(format)) { + if (OUCH_FORMATS.includes(localStorage.defaultCompressionFormat)) { + setFormat(localStorage.defaultCompressionFormat); + } else { + setFormat(defaultFormat); + } + } + }, [format]); + + return ( + { + setTitle(e.target.value); + titleRef.current = e.target.value; + }} + /> +
+ ); +} diff --git a/src/packages/frontend/project/explorer/misc-side-buttons.tsx b/src/packages/frontend/project/explorer/misc-side-buttons.tsx index 73772b92724..c8c8d36530c 100644 --- a/src/packages/frontend/project/explorer/misc-side-buttons.tsx +++ b/src/packages/frontend/project/explorer/misc-side-buttons.tsx @@ -5,7 +5,6 @@ import { Space } from "antd"; import { join } from "path"; -import React from "react"; import { defineMessage, useIntl } from "react-intl"; import { Button, ButtonToolbar } from "@cocalc/frontend/antd-bootstrap"; import { Icon, Tip, VisibleLG } from "@cocalc/frontend/components"; @@ -13,67 +12,50 @@ import LinkRetry from "@cocalc/frontend/components/link-retry"; import { useStudentProjectFunctionality } from "@cocalc/frontend/course"; import { labels } from "@cocalc/frontend/i18n"; import { serverURL, SPEC } from "@cocalc/frontend/project/named-server-panel"; -import { Available } from "@cocalc/frontend/project_configuration"; -import { ProjectActions } from "@cocalc/frontend/project_store"; import track from "@cocalc/frontend/user-tracking"; import { KUCALC_COCALC_COM } from "@cocalc/util/db-schema/site-defaults"; +import { useProjectContext } from "@cocalc/frontend/project/context"; +import { useTypedRedux } from "@cocalc/frontend/app-framework"; +import { type JSX, type MouseEvent } from "react"; const SHOW_SERVER_LAUNCHERS = false; import TourButton from "./tour/button"; +import ForkProject from "./fork"; const OPEN_MSG = defineMessage({ id: "project.explorer.misc-side-buttons.open_dir.tooltip", defaultMessage: `Opens the current directory in a {name} server instance, running inside this project.`, }); -interface Props { - actions: ProjectActions; - available_features?: Available; - current_path: string; - kucalc?: string; - project_id: string; - show_hidden?: boolean; - show_masked?: boolean; -} - -export const MiscSideButtons: React.FC = (props) => { - const { - actions, - available_features, - current_path, - kucalc, - project_id, - show_hidden, - show_masked, - } = props; - +export function MiscSideButtons() { + const { actions, project_id } = useProjectContext(); + const show_hidden = useTypedRedux({ project_id }, "show_hidden"); + const current_path = useTypedRedux({ project_id }, "current_path"); + const available_features = useTypedRedux( + { project_id }, + "available_features", + )?.toJS(); + const kucalc = useTypedRedux("customize", "kucalc"); const intl = useIntl(); const student_project_functionality = useStudentProjectFunctionality(project_id); - const handle_hidden_toggle = (e: React.MouseEvent): void => { + const handle_hidden_toggle = (e: MouseEvent): void => { e.preventDefault(); - return actions.setState({ + return actions?.setState({ show_hidden: !show_hidden, }); }; - const handle_masked_toggle = (e: React.MouseEvent): void => { - e.preventDefault(); - actions.setState({ - show_masked: !show_masked, - }); - }; - - const handle_backup = (e: React.MouseEvent): void => { + const handle_backup = (e: MouseEvent): void => { e.preventDefault(); - actions.open_directory(".snapshots"); + actions?.open_directory(".snapshots"); track("snapshots", { action: "open", where: "explorer" }); }; - function render_hidden_toggle(): React.JSX.Element { + function render_hidden_toggle(): JSX.Element { const icon = show_hidden ? "eye" : "eye-slash"; return ( - ); - } - - function render_backup(): React.JSX.Element | undefined { + function render_backup(): JSX.Element | undefined { // NOTE -- snapshots aren't available except in "kucalc" version // -- they are complicated nontrivial thing that isn't usually setup... if (kucalc !== KUCALC_COCALC_COM) { @@ -124,12 +87,12 @@ export const MiscSideButtons: React.FC = (props) => { ); } - const handle_library_click = (_e: React.MouseEvent): void => { + const handle_library_click = (_e: MouseEvent): void => { track("library", { action: "open" }); - actions.toggle_library(); + actions?.toggle_library(); }; - function render_library_button(): React.JSX.Element | undefined { + function render_library_button(): JSX.Element | undefined { if (student_project_functionality.disableLibrary) { return; } @@ -143,7 +106,7 @@ export const MiscSideButtons: React.FC = (props) => { ); } - function render_vscode_button(): React.JSX.Element | undefined { + function render_vscode_button(): JSX.Element | undefined { if (student_project_functionality.disableVSCodeServer) { return; } @@ -165,7 +128,7 @@ export const MiscSideButtons: React.FC = (props) => { ); } - function render_jupyterlab_button(): React.JSX.Element | undefined { + function render_jupyterlab_button(): JSX.Element | undefined { if (student_project_functionality.disableJupyterLabServer) { return; } @@ -188,7 +151,7 @@ export const MiscSideButtons: React.FC = (props) => { ); } - function render_upload_button(): React.JSX.Element | undefined { + function render_upload_button(): JSX.Element | undefined { if (student_project_functionality.disableUploads) { return; } @@ -222,11 +185,11 @@ export const MiscSideButtons: React.FC = (props) => {
{render_hidden_toggle()} - {render_masked_toggle()} {render_backup()} +
); -}; +} diff --git a/src/packages/frontend/project/explorer/new-button.tsx b/src/packages/frontend/project/explorer/new-button.tsx index 437821485fe..a9e37355c7f 100644 --- a/src/packages/frontend/project/explorer/new-button.tsx +++ b/src/packages/frontend/project/explorer/new-button.tsx @@ -10,7 +10,6 @@ import { DropdownMenu, Icon } from "@cocalc/frontend/components"; import { labels } from "@cocalc/frontend/i18n"; import { ProjectActions } from "@cocalc/frontend/project_store"; import { COLORS } from "@cocalc/util/theme"; -import { Configuration } from "./explorer"; import { EXTs as ALL_FILE_BUTTON_TYPES } from "./file-listing/utils"; const { file_options } = require("@cocalc/frontend/editor"); @@ -21,21 +20,18 @@ interface Props { actions: ProjectActions; create_folder: (switch_over?: boolean) => void; create_file: (ext?: string, switch_over?: boolean) => void; - configuration?: Configuration; + configuration?; disabled: boolean; } -export const NewButton: React.FC = (props: Props) => { - const { - file_search = "", - /*current_path,*/ - actions, - create_folder, - create_file, - configuration, - disabled, - } = props; - +export const NewButton: React.FC = ({ + file_search = "", + actions, + create_folder, + create_file, + configuration, + disabled, +}: Props) => { const intl = useIntl(); function new_file_button_types() { diff --git a/src/packages/frontend/project/explorer/path-navigator.tsx b/src/packages/frontend/project/explorer/path-navigator.tsx index c3e57ae19db..961af1f5661 100644 --- a/src/packages/frontend/project/explorer/path-navigator.tsx +++ b/src/packages/frontend/project/explorer/path-navigator.tsx @@ -25,13 +25,12 @@ interface Props { // This path consists of several PathSegmentLinks export const PathNavigator: React.FC = React.memo( - (props: Readonly) => { - const { - project_id, - style, - className = "cc-path-navigator", - mode = "files", - } = props; + ({ + project_id, + style, + className = "cc-path-navigator", + mode = "files", + }: Readonly) => { const currentPath = useTypedRedux({ project_id }, "current_path"); const historyPath = useTypedRedux({ project_id }, "history_path"); const actions = useActions({ project_id }); diff --git a/src/packages/frontend/project/explorer/path-segment-link.tsx b/src/packages/frontend/project/explorer/path-segment-link.tsx index de1fbb2cea7..63c4d56dd55 100644 --- a/src/packages/frontend/project/explorer/path-segment-link.tsx +++ b/src/packages/frontend/project/explorer/path-segment-link.tsx @@ -27,18 +27,16 @@ export interface PathSegmentItem { } // One segment of the directory links at the top of the files listing. -export function createPathSegmentLink(props: Readonly): PathSegmentItem { - const { - path = "", - display, - on_click, - full_name, - history, - active = false, - key, - style, - } = props; - +export function createPathSegmentLink({ + path = "", + display, + on_click, + full_name, + history, + active = false, + key, + style, +}: Readonly): PathSegmentItem { function render_content(): React.JSX.Element | string | undefined { if (full_name && full_name !== display) { return ( diff --git a/src/packages/frontend/project/explorer/rename-file.tsx b/src/packages/frontend/project/explorer/rename-file.tsx index 2b819ba9d5b..c99ad1e808b 100644 --- a/src/packages/frontend/project/explorer/rename-file.tsx +++ b/src/packages/frontend/project/explorer/rename-file.tsx @@ -18,9 +18,10 @@ const MAX_FILENAME_LENGTH = 4095; interface Props { duplicate?: boolean; + clear: () => void; } -export default function RenameFile({ duplicate }: Props) { +export default function RenameFile({ duplicate, clear }: Props) { const intl = useIntl(); const inputRef = useRef(null); const { actions } = useProjectContext(); @@ -71,7 +72,7 @@ export default function RenameFile({ duplicate }: Props) { dest: path_to_file(renameDir, target), }; if (duplicate) { - await actions.copy_paths({ + await actions.copyPaths({ src: [opts.src], dest: opts.dest, only_contents: true, @@ -79,15 +80,13 @@ export default function RenameFile({ duplicate }: Props) { } else { await actions.rename_file(opts); } - await actions.fetch_directory_listing({ path: renameDir }); } catch (err) { setLoading(false); setError(err); } finally { setLoading(false); } - actions.set_all_files_unchecked(); - actions.set_file_action(); + clear(); }; if (actions == null) { diff --git a/src/packages/frontend/project/explorer/search-bar.tsx b/src/packages/frontend/project/explorer/search-bar.tsx index a4a6359cd92..741f123c86b 100644 --- a/src/packages/frontend/project/explorer/search-bar.tsx +++ b/src/packages/frontend/project/explorer/search-bar.tsx @@ -3,18 +3,17 @@ * License: MS-RSL – see LICENSE.md for details */ +import { memo, useEffect, useRef, useState, type CSSProperties } from "react"; import { Alert, Flex } from "antd"; -import React from "react"; import { useIntl } from "react-intl"; -import { CSS, redux } from "@cocalc/frontend/app-framework"; +import { redux } from "@cocalc/frontend/app-framework"; import { Icon, SearchInput } from "@cocalc/frontend/components"; import { ProjectActions } from "@cocalc/frontend/project_store"; import { webapp_client } from "@cocalc/frontend/webapp-client"; -import { path_to_file } from "@cocalc/util/misc"; import { useProjectContext } from "../context"; import { TERM_MODE_CHAR } from "./file-listing"; -import { ListingItem } from "./types"; import { TerminalModeDisplay } from "@cocalc/frontend/project/explorer/file-listing/terminal-mode-display"; +import { useTypedRedux } from "@cocalc/frontend/app-framework"; const HelpStyle = { wordWrap: "break-word", @@ -27,7 +26,7 @@ const HelpStyle = { borderRadius: "15px", } as const; -export const outputMinitermStyle: React.CSSProperties = { +export const outputMinitermStyle: CSSProperties = { background: "white", position: "absolute", zIndex: 10, @@ -47,282 +46,264 @@ interface Props { actions: ProjectActions; create_file: (a, b) => void; create_folder: (a) => void; - selected_file?: ListingItem; // if given, file selected by cursor, which we open on pressing enter - selected_file_index?: number; file_creation_error?: string; - num_files_displayed?: number; disabled?: boolean; ext_selection?: string; } // Commands such as CD throw a setState error. // Search WARNING to find the line in this class. -export const SearchBar = React.memo((props: Props) => { - const { +export const SearchBar = memo( + ({ file_search = "", current_path, actions, create_file, create_folder, - selected_file, - selected_file_index = 0, file_creation_error, - num_files_displayed = 0, disabled = false, ext_selection, - } = props; + }: Props) => { + const intl = useIntl(); + const { project_id } = useProjectContext(); + const numDisplayedFiles = + useTypedRedux({ project_id }, "numDisplayedFiles") ?? 0; - const intl = useIntl(); - const { project_id } = useProjectContext(); + // edit → run → edit + // TODO use "state" to show a progress spinner while a command is running + // @ts-ignore + const [state, set_state] = useState<"edit" | "run">("edit"); + const [error, set_error] = useState(undefined); + const [stdout, set_stdout] = useState(undefined); - // edit → run → edit - // TODO use "state" to show a progress spinner while a command is running - // @ts-ignore - const [state, set_state] = React.useState<"edit" | "run">("edit"); - const [error, set_error] = React.useState(undefined); - const [stdout, set_stdout] = React.useState(undefined); + const _id = useRef(0); + const [cmd, set_cmd] = useState<{ input: string; id: number } | undefined>( + undefined, + ); - const _id = React.useRef(0); - const [cmd, set_cmd] = React.useState< - { input: string; id: number } | undefined - >(undefined); + useEffect(() => { + actions.set_file_search(""); + }, [current_path]); - React.useEffect(() => { - if (cmd == null) return; - const { input, id } = cmd; - const input0 = input + '\necho $HOME "`pwd`"'; - const compute_server_id = redux - .getProjectStore(project_id) - ?.get("compute_server_id"); - webapp_client.exec({ - project_id, - command: input0, - timeout: 10, - max_output: 100000, - bash: true, - path: current_path, - err_on_exit: false, - compute_server_id, - filesystem: true, - cb(err, output) { - if (id !== _id.current) { - // computation was canceled -- ignore result. - return; - } - if (err) { - set_error(JSON.stringify(err)); - set_state("edit"); - } else { - if (output.stdout) { - // Find the current path - // after the command is executed, and strip - // the output of "pwd" from the output: - // NOTE: for compute servers which can in theory use a totally different HOME, this won't work. - // However, by default on cocalc.com they use the same HOME, so it should work. - let s = output.stdout.trim(); - let i = s.lastIndexOf("\n"); - if (i === -1) { - output.stdout = ""; - } else { - s = s.slice(i + 1); - output.stdout = output.stdout.slice(0, i); + useEffect(() => { + if (cmd == null) return; + const { input, id } = cmd; + const input0 = input + '\necho $HOME "`pwd`"'; + const compute_server_id = redux + .getProjectStore(project_id) + ?.get("compute_server_id"); + webapp_client.exec({ + project_id, + command: input0, + timeout: 10, + max_output: 100000, + bash: true, + path: current_path, + err_on_exit: false, + compute_server_id, + filesystem: true, + cb(err, output) { + if (id !== _id.current) { + // computation was canceled -- ignore result. + return; + } + if (err) { + set_error(JSON.stringify(err)); + set_state("edit"); + } else { + if (output.stdout) { + // Find the current path + // after the command is executed, and strip + // the output of "pwd" from the output: + // NOTE: for compute servers which can in theory use a totally different HOME, this won't work. + // However, by default on cocalc.com they use the same HOME, so it should work. + let s = output.stdout.trim(); + let i = s.lastIndexOf("\n"); + if (i === -1) { + output.stdout = ""; + } else { + s = s.slice(i + 1); + output.stdout = output.stdout.slice(0, i); + } + i = s.indexOf(" "); + const full_path = s.slice(i + 1); + if (full_path.slice(0, i) === s.slice(0, i)) { + // only change if in project + const path = s.slice(2 * i + 2); + actions.open_directory(path); + } } - i = s.indexOf(" "); - const full_path = s.slice(i + 1); - if (full_path.slice(0, i) === s.slice(0, i)) { - // only change if in project - const path = s.slice(2 * i + 2); - actions.open_directory(path); + if (!output.stderr) { + // only log commands that worked... + actions.log({ event: "termInSearch", input }); + } + // WARNING: RENDER ERROR. Move state to redux store + set_state("edit"); + set_error(output.stderr); + set_stdout(output.stdout); + if (!output.stderr) { + actions.set_file_search(""); } } - if (!output.stderr) { - // only log commands that worked... - actions.log({ event: "termInSearch", input }); - } - // WARNING: RENDER ERROR. Move state to redux store - set_state("edit"); - set_error(output.stderr); - set_stdout(output.stdout); - if (!output.stderr) { - actions.set_file_search(""); - } - } - }, - }); - }, [cmd]); - - // Miniterm functionality - function execute_command(command: string): void { - set_error(""); - set_stdout(""); - const input = command.trim(); - if (!input) { - return; - } - set_state("run"); - _id.current = _id.current + 1; - set_cmd({ input, id: _id.current }); - } + }, + }); + }, [cmd]); - function render_help_info(): React.JSX.Element | undefined { - if (file_search[0] == TERM_MODE_CHAR) { - return ; - } - if (file_search.length > 0 && num_files_displayed > 0) { - let text; - const firstFolderPosition = file_search.indexOf("/"); - if (file_search === " /") { - text = "Showing all folders in this directory"; - } else if (firstFolderPosition === file_search.length - 1) { - text = `Showing folders matching ${file_search.slice( - 0, - file_search.length - 1, - )}`; - } else { - text = `Showing files matching "${file_search}"`; + // Miniterm functionality + function execute_command(command: string): void { + set_error(""); + set_stdout(""); + const input = command.trim(); + if (!input) { + return; } - return ; - } - } - - function render_file_creation_error(): React.JSX.Element | undefined { - if (file_creation_error) { - return ( - - ); + set_state("run"); + _id.current = _id.current + 1; + set_cmd({ input, id: _id.current }); } - } - // Miniterm functionality - function render_output( - x: string | undefined, - style: CSS, - ): React.JSX.Element | undefined { - if (x) { - return ( -
-           {
-              e.preventDefault();
-              set_stdout("");
-              set_error("");
-            }}
-            href=""
+    function render_help_info() {
+      if (file_search[0] == TERM_MODE_CHAR) {
+        return (
+          
-            
-          
-          {x}
-        
- ); + /> + ); + } + if (file_search.length > 0 && numDisplayedFiles > 0) { + let text; + const firstFolderPosition = file_search.indexOf("/"); + if (file_search === " /") { + text = "Showing all folders in this directory"; + } else if (firstFolderPosition === file_search.length - 1) { + text = `Showing folders matching ${file_search.slice( + 0, + file_search.length - 1, + )}`; + } else { + text = `Showing files matching "${file_search}"`; + } + return ; + } + } + + function render_file_creation_error() { + if (file_creation_error) { + return ( + + ); + } } - } - function dismiss_alert(): void { - actions.setState({ file_creation_error: "" }); - } + // Miniterm functionality + function render_output(x: string | undefined, style: CSSProperties) { + if (x) { + return ( +
+             {
+                e.preventDefault();
+                set_stdout("");
+                set_error("");
+              }}
+              href=""
+              style={{
+                right: "5px",
+                top: "0px",
+                color: "#666",
+                fontSize: "14pt",
+                position: "absolute",
+                background: "white",
+              }}
+            >
+              
+            
+            {x}
+          
+ ); + } + } - function search_submit( - value: string, - { ctrl_down, shift_down }: { ctrl_down: boolean; shift_down: boolean }, - ): void { - if (current_path == null) { - return; + function dismiss_alert(): void { + actions.setState({ file_creation_error: "" }); } - if (value.startsWith(TERM_MODE_CHAR)) { - const command = value.slice(1, value.length); - execute_command(command); - } else if (selected_file) { - const new_path = path_to_file(current_path, selected_file.name); - const opening_a_dir = selected_file.isdir; - if (opening_a_dir) { - actions.open_directory(new_path); - actions.setState({ page_number: 0 }); - } else { - actions.open_file({ - path: new_path, - foreground: !ctrl_down, - }); + + function search_submit( + value: string, + { ctrl_down, shift_down }: { ctrl_down: boolean; shift_down: boolean }, + ): void { + if (current_path == null) { + return; } - if (opening_a_dir || !ctrl_down) { - actions.set_file_search(""); + if (value.startsWith(TERM_MODE_CHAR)) { + const command = value.slice(1, value.length); + execute_command(command); + } else if (file_search.length > 0 && shift_down) { + // only create a file, if shift is pressed as well to avoid creating + // jupyter notebooks (default file-type) by accident. + if (file_search[file_search.length - 1] === "/") { + create_folder(!ctrl_down); + } else { + create_file(undefined, !ctrl_down); + } actions.clear_selected_file_index(); } - } else if (file_search.length > 0 && shift_down) { - // only create a file, if shift is pressed as well to avoid creating - // jupyter notebooks (default file-type) by accident. - if (file_search[file_search.length - 1] === "/") { - create_folder(!ctrl_down); - } else { - create_file(undefined, !ctrl_down); - } - actions.clear_selected_file_index(); } - } - function on_up_press(): void { - if (selected_file_index > 0) { - actions.decrement_selected_file_index(); + function on_change(search: string): void { + actions.zero_selected_file_index(); + actions.set_file_search(search); } - } - function on_down_press(): void { - if (selected_file_index < num_files_displayed - 1) { - actions.increment_selected_file_index(); + function on_clear(): void { + actions.clear_selected_file_index(); + //set_input(""); + set_stdout(""); + set_error(""); } - } - - function on_change(search: string): void { - actions.zero_selected_file_index(); - actions.set_file_search(search); - } - function on_clear(): void { - actions.clear_selected_file_index(); - //set_input(""); - set_stdout(""); - set_error(""); - } - - return ( - - - {render_file_creation_error()} - {render_help_info()} -
- {render_output(error, { - color: "darkred", - margin: 0, - })} - {render_output(stdout, { margin: 0 })} -
-
- ); -}); + return ( + + + {render_file_creation_error()} + {render_help_info()} +
+ {render_output(error, { + color: "darkred", + margin: 0, + })} + {render_output(stdout, { margin: 0 })} +
+
+ ); + }, +); diff --git a/src/packages/frontend/project/explorer/tour/tour.tsx b/src/packages/frontend/project/explorer/tour/tour.tsx index 8a8a0e41c11..6b7d4bd412d 100644 --- a/src/packages/frontend/project/explorer/tour/tour.tsx +++ b/src/packages/frontend/project/explorer/tour/tour.tsx @@ -1,14 +1,12 @@ import type { TourProps } from "antd"; import { Checkbox, Tour } from "antd"; - -import { redux } from "@cocalc/frontend/app-framework"; +import { redux, useTypedRedux } from "@cocalc/frontend/app-framework"; import { Paragraph, Text } from "@cocalc/frontend/components"; import { A } from "@cocalc/frontend/components/A"; import { Icon } from "@cocalc/frontend/components/icon"; import actionsImage from "./actions.png"; export default function ExplorerTour({ - open, project_id, newFileRef, searchAndTerminalBar, @@ -16,6 +14,7 @@ export default function ExplorerTour({ currentDirectoryRef, miscButtonsRef, }) { + const open = useTypedRedux({ project_id }, "explorerTour"); const steps: TourProps["steps"] = [ { title: ( diff --git a/src/packages/frontend/project/explorer/types.ts b/src/packages/frontend/project/explorer/types.ts index 9b85a77ad0b..cbab305a197 100644 --- a/src/packages/frontend/project/explorer/types.ts +++ b/src/packages/frontend/project/explorer/types.ts @@ -3,30 +3,12 @@ * License: MS-RSL – see LICENSE.md for details */ -// NOTE(hsy): I don't know if these two types are the same, maybe they should be merged. +import type { DirectoryListingEntry as DirectoryListingEntry0 } from "@cocalc/util/types"; -export interface ListingItem { - name: string; - isdir: boolean; - isopen?: boolean; - mtime?: number; - size?: number; // bytes -} - -// NOTE: there is also @cocalc/util/types/directory-listing::DirectoryListingEntry -// but ATM the relation ship to this one is unclear. Don't mix them up! -// This type here is used in the frontend, e.g. in Explorer and Flyout Files. -export interface DirectoryListingEntry { - display_name?: string; // unclear, if this even exists - name: string; - size?: number; - mtime?: number; - isdir?: boolean; +// fill in extra info used in the frontend, mainly for the UI +export interface DirectoryListingEntry extends DirectoryListingEntry0 { + // whether or not to mask this file in the UI mask?: boolean; - isopen?: boolean; // opened in an editor - isactive?: boolean; // opeend in the currently active editor - is_public?: boolean; // a shared file - public?: any; // some data about the shared file (TODO type?) } export type DirectoryListing = DirectoryListingEntry[]; diff --git a/src/packages/frontend/project/fetch-directory-listing.ts b/src/packages/frontend/project/fetch-directory-listing.ts deleted file mode 100644 index 026de8cbf3a..00000000000 --- a/src/packages/frontend/project/fetch-directory-listing.ts +++ /dev/null @@ -1,127 +0,0 @@ -import { is_running_or_starting } from "./project-start-warning"; -import type { ProjectActions } from "@cocalc/frontend/project_actions"; -import { trunc_middle, uuid } from "@cocalc/util/misc"; -import { get_directory_listing } from "./directory-listing"; -import { fromJS, Map } from "immutable"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; - -//const log = (...args) => console.log("fetchDirectoryListing", ...args); -const log = (..._args) => {}; - -interface FetchDirectoryListingOpts { - path?: string; - // WARNING: THINK VERY HARD BEFORE YOU USE force=true, due to efficiency! - force?: boolean; - // can be explicit here; otherwise will fall back to store.get('compute_server_id') - compute_server_id?: number; -} - -function getPath( - actions, - opts?: FetchDirectoryListingOpts, -): string | undefined { - return opts?.path ?? actions.get_store()?.get("current_path"); -} - -function getComputeServerId(actions, opts): number { - return ( - opts?.compute_server_id ?? - actions.get_store()?.get("compute_server_id") ?? - 0 - ); -} - -const fetchDirectoryListing = reuseInFlight( - async ( - actions: ProjectActions, - opts: FetchDirectoryListingOpts = {}, - ): Promise => { - let status; - let store = actions.get_store(); - if (store == null) { - return; - } - const { force } = opts; - const path = getPath(actions, opts); - const compute_server_id = getComputeServerId(actions, opts); - - if (force && path != null) { - // update our interest. - store.get_listings().watch(path, true); - } - log({ force, path, compute_server_id }); - - if (path == null) { - // nothing to do if path isn't defined -- there is no current path -- - // see https://github.com/sagemathinc/cocalc/issues/818 - return; - } - - const id = uuid(); - if (path) { - status = `Loading file list - ${trunc_middle(path, 30)}`; - } else { - status = "Loading file list"; - } - - let error = ""; - try { - // only show actions indicator, if the project is running or starting - // if it is stopped, we get a stale listing from the database, which is fine. - if (is_running_or_starting(actions.project_id)) { - log("show activity"); - actions.set_activity({ id, status }); - } - - log("make sure user is fully signed in"); - await actions.redux.getStore("account").async_wait({ - until: (s) => s.get("is_logged_in") && s.get("account_id"), - }); - - log("getting listing"); - const listing = await get_directory_listing({ - project_id: actions.project_id, - path, - hidden: true, - max_time_s: 15, - trigger_start_project: false, - group: "collaborator", // nothing else is implemented - compute_server_id, - }); - log("got ", listing.files); - const value = fromJS(listing.files); - log("saving result"); - store = actions.get_store(); - if (store == null) { - return; - } - const directory_listings = store.get("directory_listings"); - let listing2 = directory_listings.get(compute_server_id) ?? Map(); - if (listing.noRunning && (listing2.get(path)?.size ?? 0) > 0) { - // do not change it - return; - } - listing2 = listing2.set(path, value); - actions.setState({ - directory_listings: directory_listings.set(compute_server_id, listing2), - }); - } catch (err) { - log("error", err); - error = `${err}`; - } finally { - actions.set_activity({ id, stop: "", error }); - } - }, - { - createKey: (args) => { - const actions = args[0]; - // reuse in flight on the project id, compute server id and path - return `${actions.project_id}-${getComputeServerId( - actions, - args[1], - )}-${getPath(actions, args[1])}`; - }, - }, -); - -export default fetchDirectoryListing; diff --git a/src/packages/frontend/project/listing/filter-listing.ts b/src/packages/frontend/project/listing/filter-listing.ts new file mode 100644 index 00000000000..53e296da7f2 --- /dev/null +++ b/src/packages/frontend/project/listing/filter-listing.ts @@ -0,0 +1,23 @@ +import { DirectoryListingEntry } from "@cocalc/util/types"; + +export default function filterListing({ + listing, + search, + showHidden, +}: { + listing?: DirectoryListingEntry[] | null; + search?: string; + showHidden?: boolean; +}): DirectoryListingEntry[] | null { + if (listing == null) { + return null; + } + if (!showHidden) { + listing = listing.filter((x) => !x.name.startsWith(".")); + } + search = search?.trim()?.toLowerCase(); + if (!search || search.startsWith("/")) { + return listing; + } + return listing.filter((x) => x.name.toLowerCase().includes(search)); +} diff --git a/src/packages/frontend/project/listing/use-files.ts b/src/packages/frontend/project/listing/use-files.ts new file mode 100644 index 00000000000..4d681b5d0c8 --- /dev/null +++ b/src/packages/frontend/project/listing/use-files.ts @@ -0,0 +1,190 @@ +/* +Hook that provides all files in a directory via a Conat FilesystemClient. +This automatically updates when files change. + +TESTS: See packages/test/project/listing/ + +*/ + +import useAsyncEffect from "use-async-effect"; +import { useRef, useState } from "react"; +import { throttle } from "lodash"; +import { type Files } from "@cocalc/conat/files/listing"; +import { type FilesystemClient } from "@cocalc/conat/files/fs"; +import { type ConatError } from "@cocalc/conat/core/client"; +import useCounter from "@cocalc/frontend/app-framework/counter-hook"; +import LRU from "lru-cache"; +import type { JSONValue } from "@cocalc/util/types"; +import { dirname, join } from "path"; + +export { Files }; + +const DEFAULT_THROTTLE_FILE_UPDATE = 500; + +// max number of subdirs to cache right after computing the listing for a dir +// This makes it so clicking on a subdir for a listing is MUCH faster. +const MAX_SUBDIR_CACHE = 10; + +const CACHE_SIZE = 150; + +const cache = new LRU({ max: CACHE_SIZE }); + +export function getFiles({ + cacheId, + path, +}: { + cacheId?: JSONValue; + path: string; +}): Files | null { + if (cacheId == null) { + return null; + } + return cache.get(key(cacheId, path)) ?? null; +} + +export default function useFiles({ + fs, + path, + throttleUpdate = DEFAULT_THROTTLE_FILE_UPDATE, + cacheId, +}: { + // fs = undefined is supported and just waits until you provide a fs that is defined + fs?: FilesystemClient | null; + path: string; + throttleUpdate?: number; + // cacheId -- if given, save most recently loaded Files for a path in an in-memory LRU cache. + // An example cacheId could be {project_id, compute_server_id}. + // This is used to speed up the first load, and can also be fetched synchronously. + cacheId?: JSONValue; +}): { files: Files | null; error: null | ConatError; refresh: () => void } { + const [files, setFiles] = useState(getFiles({ cacheId, path })); + const [error, setError] = useState(null); + const { val: counter, inc: refresh } = useCounter(); + const listingRef = useRef(null); + + useAsyncEffect( + async () => { + if (fs == null) { + setError(null); + setFiles(null); + return; + } + let listing; + try { + setFiles(getFiles({ cacheId, path })); + listing = await fs.listing(path); + listingRef.current = listing; + setError(null); + } catch (err) { + setError(err); + setFiles(null); + return; + } + if (cacheId != null) { + cache.set(key(cacheId, path), listing.files); + if (listing.files != null) { + cacheNeighbors({ fs, cacheId, path, files: listing.files }); + } + } + const update = () => { + setFiles({ ...listing.files }); + }; + update(); + + listing.on( + "change", + throttle(update, throttleUpdate, { leading: true, trailing: true }), + ); + }, + () => { + listingRef.current?.close(); + delete listingRef.current; + }, + [fs, path, counter], + ); + + return { files, error, refresh }; +} + +function key(cacheId: JSONValue, path: string) { + return JSON.stringify({ cacheId, path }); +} + +// anything in failed we don't try to update -- this is +// purely a convenience so no need to worry. +const failed = new Set(); + +async function ensureCached({ + cacheId, + fs, + path, +}: { + fs: FilesystemClient; + cacheId: JSONValue; + path: string; +}) { + const k = key(cacheId, path); + if (cache.has(k) || failed.has(k)) { + return; + } + try { + const { files } = await fs.listing(path); + if (files) { + cache.set(k, files); + } else { + failed.add(k); + } + } catch { + failed.add(k); + } +} + +async function cacheNeighbors({ + fs, + cacheId, + path, + files, +}: { + fs: FilesystemClient; + cacheId: JSONValue; + path: string; + files: Files; +}) { + let v: string[] = []; + for (const dir in files) { + if (!dir.startsWith(".") && files[dir].isDir) { + const full = join(path, dir); + const k = key(cacheId, full); + if (!cache.has(k) && !failed.has(k)) { + v.push(full); + } + } + } + if (path) { + let parent = dirname(path); + if (parent == ".") { + parent = ""; + } + const k = key(cacheId, parent); + if (!cache.has(k) && !failed.has(k)) { + v.push(parent); + } + } + const f = async (path: string) => { + await ensureCached({ cacheId, fs, path }); + }; + v.sort(); + // grab up to MAX_SUBDIR_CACHE missing listings in parallel + v = v.slice(0, MAX_SUBDIR_CACHE); + await Promise.all(v.map(f)); +} + +export function getCacheId({ + project_id, + compute_server_id = 0, +}: { + project_id: string; + compute_server_id?: number; +}) { + return { project_id, compute_server_id }; +} diff --git a/src/packages/frontend/project/listing/use-fs.ts b/src/packages/frontend/project/listing/use-fs.ts new file mode 100644 index 00000000000..1bf01ab7628 --- /dev/null +++ b/src/packages/frontend/project/listing/use-fs.ts @@ -0,0 +1,29 @@ +/* +Hook for getting a FilesystemClient. +*/ +import { webapp_client } from "@cocalc/frontend/webapp-client"; +import { type FilesystemClient } from "@cocalc/conat/files/fs"; +import { useState } from "react"; + +// this will probably get more complicated temporarily when we +// are transitioning between filesystems (hence why we return null in +// the typing for now) +export default function useFs({ + project_id, + compute_server_id, + computeServerId, +}: { + project_id: string; + compute_server_id?: number; + computeServerId?: number; +}): FilesystemClient | null { + const [fs] = useState(() => + webapp_client.conat_client + .conat() + .fs({ + project_id, + compute_server_id: compute_server_id ?? computeServerId, + }), + ); + return fs; +} diff --git a/src/packages/frontend/project/listing/use-listing.ts b/src/packages/frontend/project/listing/use-listing.ts new file mode 100644 index 00000000000..e49cdfafcd2 --- /dev/null +++ b/src/packages/frontend/project/listing/use-listing.ts @@ -0,0 +1,113 @@ +/* +A directory listing hook. + +TESTS: See packages/test/project/listing/ +*/ + +import { useMemo } from "react"; +import { type DirectoryListingEntry } from "@cocalc/frontend/project/explorer/types"; +import { field_cmp } from "@cocalc/util/misc"; +import useFiles from "./use-files"; +import { type FilesystemClient } from "@cocalc/conat/files/fs"; +import { type ConatError } from "@cocalc/conat/core/client"; +import type { JSONValue } from "@cocalc/util/types"; +import { getFiles, type Files } from "./use-files"; +import { computeFileMasks } from "@cocalc/frontend/project/explorer/compute-file-masks"; + +export type SortField = "name" | "mtime" | "size" | "type"; +export type SortDirection = "asc" | "desc"; + +export function getListing({ + path, + cacheId, + sortField, + sortDirection, +}: { + path; + string; + cacheId?: JSONValue; + sortField?: SortField; + sortDirection?: SortDirection; +}): null | DirectoryListingEntry[] { + const files = getFiles({ cacheId, path }); + return filesToListing({ files, sortField, sortDirection }); +} + +export default function useListing({ + fs, + path, + sortField = "name", + sortDirection = "asc", + throttleUpdate, + cacheId, + mask, +}: { + // fs = undefined is supported and just waits until you provide a fs that is defined + fs?: FilesystemClient | null; + path: string; + sortField?: SortField; + sortDirection?: SortDirection; + throttleUpdate?: number; + cacheId?: JSONValue; + mask?: boolean; +}): { + listing: null | DirectoryListingEntry[]; + error: null | ConatError; + refresh: () => void; +} { + const { files, error, refresh } = useFiles({ + fs, + path, + throttleUpdate, + cacheId, + }); + + const listing = useMemo(() => { + return filesToListing({ files, sortField, sortDirection, mask }); + }, [sortField, sortDirection, files]); + + return { listing, error, refresh }; +} + +function filesToListing({ + files, + sortField = "name", + sortDirection = "asc", + mask, +}: { + files?: Files | null; + sortField?: SortField; + sortDirection?: SortDirection; + mask?: boolean; +}): null | DirectoryListingEntry[] { + if (files == null) { + return null; + } + if (files == null) { + return null; + } + const v: DirectoryListingEntry[] = []; + for (const name in files) { + v.push({ name, ...files[name] }); + } + if ( + sortField != "name" && + sortField != "mtime" && + sortField != "size" && + sortField != "type" + ) { + console.warn(`invalid sort field: '${sortField}'`); + } + v.sort(field_cmp(sortField)); + if (sortDirection == "desc") { + v.reverse(); + } else if (sortDirection == "asc") { + } else { + console.warn(`invalid sort direction: '${sortDirection}'`); + } + if (mask) { + // note -- this masking is as much time as everything above + computeFileMasks(v); + } + return v; +} diff --git a/src/packages/frontend/project/new/file-type-selector.tsx b/src/packages/frontend/project/new/file-type-selector.tsx index 846c5b2f4ee..0467d72daeb 100644 --- a/src/packages/frontend/project/new/file-type-selector.tsx +++ b/src/packages/frontend/project/new/file-type-selector.tsx @@ -7,7 +7,6 @@ import { Col, Flex, Modal, Row, Tag } from "antd"; import { Gutter } from "antd/es/grid/row"; import type { ReactNode } from "react"; import { FormattedMessage, useIntl } from "react-intl"; - import { Available } from "@cocalc/comm/project-configuration"; import { CSS } from "@cocalc/frontend/app-framework"; import { A } from "@cocalc/frontend/components/A"; diff --git a/src/packages/frontend/project/new/new-file-page.tsx b/src/packages/frontend/project/new/new-file-page.tsx index 08df80c968c..d8f9ace744c 100644 --- a/src/packages/frontend/project/new/new-file-page.tsx +++ b/src/packages/frontend/project/new/new-file-page.tsx @@ -6,7 +6,6 @@ import { Button, Input, Modal, Space } from "antd"; import { useEffect, useRef, useState } from "react"; import { defineMessage, FormattedMessage, useIntl } from "react-intl"; - import { default_filename } from "@cocalc/frontend/account"; import { Alert, Col, Row } from "@cocalc/frontend/antd-bootstrap"; import { @@ -25,7 +24,6 @@ import { SettingBox, Tip, } from "@cocalc/frontend/components"; -import FakeProgress from "@cocalc/frontend/components/fake-progress"; import ComputeServer from "@cocalc/frontend/compute/inline"; import { filenameIcon } from "@cocalc/frontend/file-associations"; import { FileUpload, UploadLink } from "@cocalc/frontend/file-upload"; @@ -252,11 +250,6 @@ export default function NewFilePage(props: Props) { { - getActions().fetch_directory_listing(); - }, - }} project_id={project_id} current_path={current_path} show_header={false} @@ -357,11 +350,7 @@ export default function NewFilePage(props: Props) { } values={{ upload: ( - getActions().fetch_directory_listing()} - /> + ), folder: (txt) => ( } >
- +
diff --git a/src/packages/frontend/project/open-file.ts b/src/packages/frontend/project/open-file.ts index 2f9a49912b0..9b5ee78917d 100644 --- a/src/packages/frontend/project/open-file.ts +++ b/src/packages/frontend/project/open-file.ts @@ -5,7 +5,6 @@ // Implement the open_file actions for opening one single file in a project. -import { callback } from "awaiting"; import { alert_message } from "@cocalc/frontend/alerts"; import { redux } from "@cocalc/frontend/app-framework"; import { local_storage } from "@cocalc/frontend/editor-local-storage"; @@ -28,6 +27,14 @@ import { syncdbPath as ipynbSyncdbPath } from "@cocalc/util/jupyter/names"; import { termPath } from "@cocalc/util/terminal/names"; import { excludeFromComputeServer } from "@cocalc/frontend/file-associations"; +// if true, PRELOAD_BACKGROUND_TABS makes it so all tabs have their file editing +// preloaded, even background tabs. This can make the UI much more responsive, +// since after refreshing your browser or opening a project that had tabs open, +// all files are ready to edit instantly. It uses more browser memory (of course), +// and increases server load. Most users have very few files open at once, +// so this is probably a major win for power users and has little impact on load. +const PRELOAD_BACKGROUND_TABS = true; + export interface OpenFileOpts { path: string; ext?: string; // if given, use editor for this extension instead of whatever extension path has. @@ -161,11 +168,6 @@ export async function open_file( } if (opts.path != realpath) { if (!actions.open_files) return; // closed - alert_message({ - type: "info", - message: `Opening normalized real path "${realpath}"`, - timeout: 10, - }); actions.open_files.delete(opts.path); opts.path = realpath; actions.open_files.set(opts.path, "component", {}); @@ -222,7 +224,7 @@ export async function open_file( // Wait for the project to start opening (only do this if not public -- public users don't // know anything about the state of the project). try { - await callback(actions._ensure_project_is_open.bind(actions)); + await actions.ensureProjectIsOpen(); if (!tabIsOpened()) { return; } @@ -345,6 +347,8 @@ export async function open_file( actions.set_active_tab(tab, { change_history: opts.change_history, }); + } else if (PRELOAD_BACKGROUND_TABS) { + await actions.initFileRedux(opts.path); } if (alreadyOpened && opts.fragmentId) { diff --git a/src/packages/frontend/project/page/file-tabs.tsx b/src/packages/frontend/project/page/file-tabs.tsx index 0e1eaab893c..c593217ef29 100644 --- a/src/packages/frontend/project/page/file-tabs.tsx +++ b/src/packages/frontend/project/page/file-tabs.tsx @@ -100,9 +100,11 @@ export default function FileTabs({ openFiles, project_id, activeTab }) { if (action == "add") { actions.set_active_tab("files"); } else { - const path = keyToPath(key); - // close given file - actions.close_tab(path); + if (key) { + const path = keyToPath(key); + // close given file + actions.close_tab(path); + } } }; @@ -135,11 +137,14 @@ export default function FileTabs({ openFiles, project_id, activeTab }) { function onDragStart(event) { if (actions == null) return; if (event?.active?.id != activeKey) { - actions.set_active_tab(path_to_tab(keyToPath(event?.active?.id)), { - // noFocus -- critical to not focus when dragging or codemirror focus breaks on end of drag. - // See https://github.com/sagemathinc/cocalc/issues/7029 - noFocus: true, - }); + const key = event?.active?.id; + if (key) { + actions.set_active_tab(path_to_tab(keyToPath(key)), { + // noFocus -- critical to not focus when dragging or codemirror focus breaks on end of drag. + // See https://github.com/sagemathinc/cocalc/issues/7029 + noFocus: true, + }); + } } } @@ -160,7 +165,7 @@ export default function FileTabs({ openFiles, project_id, activeTab }) { activeKey={activeKey} type={"editable-card"} onChange={(key) => { - if (actions == null) return; + if (actions == null || !key) return; actions.set_active_tab(path_to_tab(keyToPath(key))); }} popupClassName={"cocalc-files-tabs-more"} diff --git a/src/packages/frontend/project/page/flyouts/active-group.tsx b/src/packages/frontend/project/page/flyouts/active-group.tsx index e2841648884..763e512e976 100644 --- a/src/packages/frontend/project/page/flyouts/active-group.tsx +++ b/src/packages/frontend/project/page/flyouts/active-group.tsx @@ -75,7 +75,7 @@ export function Group({ const fileType = file_options(`foo.${group}`); return { iconName: - group === "" ? UNKNOWN_FILE_TYPE_ICON : fileType?.icon ?? "file", + group === "" ? UNKNOWN_FILE_TYPE_ICON : (fileType?.icon ?? "file"), display: (group === "" ? "No extension" : fileType?.name) || group, }; } @@ -83,7 +83,7 @@ export function Group({ switch (mode) { case "folder": const isHome = group === ""; - const isopen = openFilesGrouped[group].some((path) => + const isOpen = openFilesGrouped[group].some((path) => openFiles.includes(path), ); return ( @@ -93,9 +93,9 @@ export function Group({ mode="active" item={{ name: group, - isdir: true, - isopen, - isactive: current_path === group && activeTab === "files", + isDir: true, + isOpen, + isActive: current_path === group && activeTab === "files", }} multiline={false} displayedNameOverride={displayed} diff --git a/src/packages/frontend/project/page/flyouts/active.tsx b/src/packages/frontend/project/page/flyouts/active.tsx index d8b81fa76c0..8436d1cde9a 100644 --- a/src/packages/frontend/project/page/flyouts/active.tsx +++ b/src/packages/frontend/project/page/flyouts/active.tsx @@ -194,7 +194,7 @@ export function ActiveFlyout(props: Readonly): React.JSX.Element { filteredFiles.forEach((path) => { const { head, tail } = path_split(path); const group = - mode === "folder" ? head : filename_extension_notilde(tail) ?? ""; + mode === "folder" ? head : (filename_extension_notilde(tail) ?? ""); if (grouped[group] == null) grouped[group] = []; grouped[group].push(path); }); @@ -258,7 +258,7 @@ export function ActiveFlyout(props: Readonly): React.JSX.Element { group?: string, isLast?: boolean, ): React.JSX.Element { - const isactive: boolean = activePath === path; + const isActive: boolean = activePath === path; const style = group != null ? { @@ -267,12 +267,12 @@ export function ActiveFlyout(props: Readonly): React.JSX.Element { } : undefined; - const isdir = path.endsWith("/"); - const isopen = openFiles.includes(path); + const isDir = path.endsWith("/"); + const isOpen = openFiles.includes(path); // if it is a directory, remove the trailing slash // and if it starts with ".smc/root/", replace that by a "/" - const display = isdir + const display = isDir ? path.slice(0, -1).replace(/^\.smc\/root\//, "/") : undefined; @@ -280,7 +280,7 @@ export function ActiveFlyout(props: Readonly): React.JSX.Element { ): React.JSX.Element { // we only toggle star, if it is currently opened! // otherwise, when closed and accidentally clicking on the star // the file unstarred and just vanishes - if (isopen) { + if (isOpen) { setStarredPath(path, starState); } else { handleFileClick(undefined, path, "star"); diff --git a/src/packages/frontend/project/page/flyouts/body.tsx b/src/packages/frontend/project/page/flyouts/body.tsx index d71a3c2c3da..cc7883ddda6 100644 --- a/src/packages/frontend/project/page/flyouts/body.tsx +++ b/src/packages/frontend/project/page/flyouts/body.tsx @@ -4,7 +4,6 @@ */ import { debounce } from "lodash"; - import { CSS, redux, @@ -97,7 +96,7 @@ export function FlyoutBody({ flyout, flyoutWidth }: FlyoutBodyProps) { style={style} onFocus={() => { // Remove any active key handler that is next to this side chat. - // E.g, this is critical for taks lists... + // E.g, this is critical for task lists... redux.getActions("page").erase_active_key_handler(); }} > diff --git a/src/packages/frontend/project/page/flyouts/file-list-item.tsx b/src/packages/frontend/project/page/flyouts/file-list-item.tsx index 348ab4dd129..44c74426cec 100644 --- a/src/packages/frontend/project/page/flyouts/file-list-item.tsx +++ b/src/packages/frontend/project/page/flyouts/file-list-item.tsx @@ -6,7 +6,6 @@ import { Button, Dropdown, MenuProps, Tooltip } from "antd"; import immutable from "immutable"; import { useIntl } from "react-intl"; - import { CSS, React, @@ -43,7 +42,7 @@ const FILE_ITEM_SELECTED_STYLE: CSS = { backgroundColor: COLORS.BLUE_LLL, // bit darker than .cc-project-flyout-file-item:hover } as const; -const FILE_ITEM_OPENED_STYLE: CSS = { +export const FILE_ITEM_OPENED_STYLE: CSS = { fontWeight: "bold", backgroundColor: COLORS.GRAY_LL, color: COLORS.PROJECT.FIXED_LEFT_ACTIVE, @@ -108,13 +107,14 @@ const CLOSE_ICON_STYLE: CSS = { }; interface Item { - isopen?: boolean; - isdir?: boolean; - isactive?: boolean; - is_public?: boolean; + isOpen?: boolean; + isDir?: boolean; + isActive?: boolean; + isPublic?: boolean; name: string; size?: number; mask?: boolean; + linkTarget?: string; } interface FileListItemProps { @@ -184,7 +184,7 @@ export const FileListItem = React.memo((props: Readonly) => { const bodyRef = useRef(null); function renderCloseItem(item: Item): React.JSX.Element | null { - if (onClose == null || !item.isopen) return null; + if (onClose == null || !item.isOpen) return null; const { name } = item; return ( @@ -200,7 +200,7 @@ export const FileListItem = React.memo((props: Readonly) => { } function renderPublishedIcon(): React.JSX.Element | undefined { - if (!showPublish || !item.is_public) return undefined; + if (!showPublish || !item.isPublic) return undefined; return (
); } @@ -278,9 +283,9 @@ export const FileListItem = React.memo((props: Readonly) => { ? selected ? "check-square" : "square" - : item.isdir - ? "folder-open" - : file_options(item.name)?.icon ?? "file"); + : item.isDir + ? "folder-open" + : (file_options(item.name)?.icon ?? "file")); return ( ) => { name={icon} style={{ ...ICON_STYLE, - color: isStarred && item.isopen ? COLORS.STAR : COLORS.GRAY_L, + color: isStarred && item.isOpen ? COLORS.STAR : COLORS.GRAY_L, }} onClick={(e: React.MouseEvent) => { e?.stopPropagation(); @@ -324,8 +329,8 @@ export const FileListItem = React.memo((props: Readonly) => { const currentExtra = type === 1 ? extra : extra2; if (currentExtra == null) return; // calculate extra margin to align the columns. if there is no "onClose", no margin - const closeMargin = onClose != null ? (item.isopen ? 0 : 18) : 0; - const publishMargin = showPublish ? (item.is_public ? 0 : 20) : 0; + const closeMargin = onClose != null ? (item.isOpen ? 0 : 18) : 0; + const publishMargin = showPublish ? (item.isPublic ? 0 : 20) : 0; const marginRight = type === 1 ? publishMargin + closeMargin : undefined; const widthPx = FLYOUT_DEFAULT_WIDTH_PX * 0.33; // if the 2nd extra shows up, fix the width to align the columns @@ -400,14 +405,14 @@ export const FileListItem = React.memo((props: Readonly) => { item: Item, multiple: boolean, ) { - const { isdir, name: fileName } = item; + const { isDir, name: fileName } = item; const actionNames = multiple ? ACTION_BUTTONS_MULTI - : isdir - ? ACTION_BUTTONS_DIR - : ACTION_BUTTONS_FILE; + : isDir + ? ACTION_BUTTONS_DIR + : ACTION_BUTTONS_FILE; for (const key of actionNames) { - if (key === "download" && !item.isdir) continue; + if (key === "download" && !item.isDir) continue; const disabled = isDisabledSnapshots(key) && (current_path?.startsWith(".snapshots") ?? false); @@ -441,13 +446,13 @@ export const FileListItem = React.memo((props: Readonly) => { } function getContextMenu(): MenuProps["items"] { - const { name, isdir, is_public, size } = item; + const { name, isDir, isPublic, size } = item; const n = checked_files?.size ?? 0; const multiple = n > 1; const sizeStr = size ? human_readable_size(size) : ""; const nameStr = trunc_middle(item.name, 30); - const typeStr = isdir ? "Folder" : "File"; + const typeStr = isDir ? "Folder" : "File"; const ctx: NonNullable = []; @@ -461,7 +466,7 @@ export const FileListItem = React.memo((props: Readonly) => { } else { ctx.push({ key: "header", - icon: , + icon: , label: `${typeStr} ${nameStr}${sizeStr ? ` (${sizeStr})` : ""}`, title: `${name}`, style: { fontWeight: "bold" }, @@ -469,14 +474,14 @@ export const FileListItem = React.memo((props: Readonly) => { ctx.push({ key: "open", icon: , - label: isdir ? "Open folder" : "Open file", + label: isDir ? "Open folder" : "Open file", onClick: () => onClick?.(), }); } ctx.push({ key: "divider-header", type: "divider" }); - if (is_public && typeof onPublic === "function") { + if (isPublic && typeof onPublic === "function") { ctx.push({ key: "public", label: "Item is published", @@ -490,7 +495,7 @@ export const FileListItem = React.memo((props: Readonly) => { // view/download buttons at the bottom const showDownload = !student_project_functionality.disableActions; - if (name !== ".." && !isdir && showDownload && !multiple) { + if (name !== ".." && !isDir && showDownload && !multiple) { const full_path = path_to_file(current_path, name); const ext = (filename_extension(name) ?? "").toLowerCase(); const showView = VIEWABLE_FILE_EXT.includes(ext); @@ -523,14 +528,14 @@ export const FileListItem = React.memo((props: Readonly) => { // because all those files are opened const activeStyle: CSS = mode === "active" - ? item.isactive + ? item.isActive ? FILE_ITEM_ACTIVE_STYLE_2 : {} - : item.isopen - ? item.isactive - ? FILE_ITEM_ACTIVE_STYLE - : FILE_ITEM_OPENED_STYLE - : {}; + : item.isOpen + ? item.isActive + ? FILE_ITEM_ACTIVE_STYLE + : FILE_ITEM_OPENED_STYLE + : {}; return ( diff --git a/src/packages/frontend/project/page/flyouts/files-bottom.tsx b/src/packages/frontend/project/page/flyouts/files-bottom.tsx index ea186390a45..ef71d914db0 100644 --- a/src/packages/frontend/project/page/flyouts/files-bottom.tsx +++ b/src/packages/frontend/project/page/flyouts/files-bottom.tsx @@ -63,6 +63,7 @@ interface FilesBottomProps { clearAllSelections: (switchMode: boolean) => void; selectAllFiles: () => void; getFile: (path: string) => DirectoryListingEntry | undefined; + publicFiles: Set; } export function FilesBottom({ @@ -78,6 +79,7 @@ export function FilesBottom({ showFileSharingDialog, getFile, directoryFiles, + publicFiles, }: FilesBottomProps) { const [mode, setMode] = modeState; const current_path = useTypedRedux({ project_id }, "current_path"); @@ -185,8 +187,8 @@ export function FilesBottom({ function renderDownloadView() { if (!singleFile) return; - const { name, isdir, size = 0 } = singleFile; - if (isdir) return; + const { name, isDir, size = 0 } = singleFile; + if (isDir) return; const full_path = path_to_file(current_path, name); const ext = (filename_extension(name) ?? "").toLowerCase(); const showView = VIEWABLE_FILE_EXT.includes(ext); @@ -268,6 +270,7 @@ export function FilesBottom({ getFile={getFile} mode="bottom" activeFile={activeFile} + publicFiles={publicFiles} /> ); } @@ -276,7 +279,7 @@ export function FilesBottom({ if (checked_files.size === 0) { let totSize = 0; for (const f of directoryFiles) { - if (!f.isdir) totSize += f.size ?? 0; + if (!f.isDir) totSize += f.size ?? 0; } return (
@@ -292,7 +295,7 @@ export function FilesBottom({ if (checked_files.size === 0) { let [nFiles, nDirs] = [0, 0]; for (const f of directoryFiles) { - if (f.isdir) { + if (f.isDir) { nDirs++; } else { nFiles++; @@ -307,9 +310,9 @@ export function FilesBottom({ ); } else if (singleFile) { const name = singleFile.name; - const iconName = singleFile.isdir + const iconName = singleFile.isDir ? "folder" - : file_options(name)?.icon ?? "file"; + : (file_options(name)?.icon ?? "file"); return (
{trunc_middle(name, 20)} diff --git a/src/packages/frontend/project/page/flyouts/files-controls.tsx b/src/packages/frontend/project/page/flyouts/files-controls.tsx index 9669e1126b2..7246a736660 100644 --- a/src/packages/frontend/project/page/flyouts/files-controls.tsx +++ b/src/packages/frontend/project/page/flyouts/files-controls.tsx @@ -6,7 +6,6 @@ import { Button, Descriptions, Space, Tooltip } from "antd"; import immutable from "immutable"; import { useIntl } from "react-intl"; - import { useActions, useTypedRedux } from "@cocalc/frontend/app-framework"; import { Icon, TimeAgo } from "@cocalc/frontend/components"; import { @@ -15,7 +14,7 @@ import { ACTION_BUTTONS_MULTI, isDisabledSnapshots, } from "@cocalc/frontend/project/explorer/action-bar"; -import { +import type { DirectoryListing, DirectoryListingEntry, } from "@cocalc/frontend/project/explorer/types"; @@ -38,6 +37,7 @@ interface FilesSelectedControlsProps { skip?: boolean, ) => void; activeFile: DirectoryListingEntry | null; + publicFiles: Set; } export function FilesSelectedControls({ @@ -49,6 +49,7 @@ export function FilesSelectedControls({ project_id, showFileSharingDialog, activeFile, + publicFiles, }: FilesSelectedControlsProps) { const intl = useIntl(); const current_path = useTypedRedux({ project_id }, "current_path"); @@ -68,7 +69,7 @@ export function FilesSelectedControls({ const basename = path_split(file).tail; const index = directoryFiles.findIndex((f) => f.name === basename); // skipping directories, because it makes no sense to flip through them rapidly - if (skipDirs && getFile(file)?.isdir) { + if (skipDirs && getFile(file)?.isDir) { open(e, index, true); continue; } @@ -83,7 +84,7 @@ export function FilesSelectedControls({ let [nFiles, nDirs] = [0, 0]; for (const f of directoryFiles) { - if (f.isdir) { + if (f.isDir) { nDirs++; } else { nFiles++; @@ -100,8 +101,8 @@ export function FilesSelectedControls({ function renderFileInfoBottom() { if (singleFile != null) { - const { size, mtime, isdir } = singleFile; - const age = typeof mtime === "number" ? 1000 * mtime : null; + const { size, mtime, isDir } = singleFile; + const age = typeof mtime === "number" ? mtime : null; return ( {age ? ( @@ -109,7 +110,7 @@ export function FilesSelectedControls({ ) : undefined} - {isdir ? ( + {isDir ? ( {size} {plural(size, "item")} @@ -118,7 +119,7 @@ export function FilesSelectedControls({ {human_readable_size(size)} )} - {singleFile.is_public ? ( + {publicFiles.has(singleFile.name) ? (
diff --git a/src/packages/frontend/project/page/flyouts/files.tsx b/src/packages/frontend/project/page/flyouts/files.tsx index aca83cbf658..5834d2ec0c0 100644 --- a/src/packages/frontend/project/page/flyouts/files.tsx +++ b/src/packages/frontend/project/page/flyouts/files.tsx @@ -4,24 +4,17 @@ */ import { Alert, InputRef } from "antd"; -import { delay } from "awaiting"; -import { List, Map } from "immutable"; +import { List } from "immutable"; import { debounce, fromPairs } from "lodash"; import { Virtuoso, VirtuosoHandle } from "react-virtuoso"; - import { React, TypedMap, redux, - useEffect, - useIsMountedRef, - useLayoutEffect, - useMemo, usePrevious, - useRef, - useState, useTypedRedux, } from "@cocalc/frontend/app-framework"; +import { useEffect, useLayoutEffect, useMemo, useRef, useState } from "react"; import { Loading, TimeAgo } from "@cocalc/frontend/components"; import useVirtuosoScrollHook from "@cocalc/frontend/components/virtuoso-scroll-hook"; import { useStudentProjectFunctionality } from "@cocalc/frontend/course"; @@ -29,23 +22,22 @@ import { file_options } from "@cocalc/frontend/editor-tmp"; import { FileUploadWrapper } from "@cocalc/frontend/file-upload"; import { should_open_in_foreground } from "@cocalc/frontend/lib/should-open-in-foreground"; import { useProjectContext } from "@cocalc/frontend/project/context"; -import { compute_file_masks } from "@cocalc/frontend/project/explorer/compute-file-masks"; +import { computeFileMasks } from "@cocalc/frontend/project/explorer/compute-file-masks"; import { DirectoryListing, DirectoryListingEntry, FileMap, } from "@cocalc/frontend/project/explorer/types"; -import { WATCH_THROTTLE_MS } from "@cocalc/frontend/conat/listings"; -import { mutate_data_to_compute_public_files } from "@cocalc/frontend/project_store"; +import { + getPublicFiles, + useStrippedPublicPaths, +} from "@cocalc/frontend/project_store"; import track from "@cocalc/frontend/user-tracking"; import { capitalize, - copy_without, human_readable_size, path_split, path_to_file, - search_match, - search_split, separate_file_extension, tab_to_path, unreachable, @@ -59,6 +51,9 @@ import { FileListItem } from "./file-list-item"; import { FilesBottom } from "./files-bottom"; import { FilesHeader } from "./files-header"; import { fileItemStyle } from "./utils"; +import useFs from "@cocalc/frontend/project/listing/use-fs"; +import useListing from "@cocalc/frontend/project/listing/use-listing"; +import ShowError from "@cocalc/frontend/components/error"; type PartialClickEvent = Pick< React.MouseEvent | React.KeyboardEvent, @@ -77,19 +72,6 @@ export type ActiveFileSort = TypedMap<{ is_descending: boolean; }>; -// modeled after ProjectStore::stripped_public_paths -function useStrippedPublicPaths(project_id: string) { - const public_paths = useTypedRedux({ project_id }, "public_paths"); - return useMemo(() => { - if (public_paths == null) return List(); - return public_paths - .valueSeq() - .map((public_path: any) => - copy_without(public_path.toJS(), ["id", "project_id"]), - ); - }, [public_paths]); -} - export function FilesFlyout({ flyoutWidth, }: { @@ -100,7 +82,6 @@ export function FilesFlyout({ project_id, actions, } = useProjectContext(); - const isMountedRef = useIsMountedRef(); const rootRef = useRef(null as any); const refInput = useRef(null as any); const [rootHeightPx, setRootHeightPx] = useState(0); @@ -110,12 +91,6 @@ export function FilesFlyout({ const current_path = useTypedRedux({ project_id }, "current_path"); const strippedPublicPaths = useStrippedPublicPaths(project_id); const compute_server_id = useTypedRedux({ project_id }, "compute_server_id"); - const directoryListings: Map< - string, - TypedMap | null - > | null = useTypedRedux({ project_id }, "directory_listings")?.get( - compute_server_id, - ); const activeTab = useTypedRedux({ project_id }, "active_project_tab"); const activeFileSort: ActiveFileSort = useTypedRedux( { project_id }, @@ -125,7 +100,9 @@ export function FilesFlyout({ const show_masked = useTypedRedux({ project_id }, "show_masked"); const hidden = useTypedRedux({ project_id }, "show_hidden"); const checked_files = useTypedRedux({ project_id }, "checked_files"); - const openFiles = useTypedRedux({ project_id }, "open_files_order"); + const openFiles = new Set( + useTypedRedux({ project_id }, "open_files_order")?.toJS() ?? [], + ); // mainly controls what a single click does, plus additional UI elements const [mode, setMode] = useState<"open" | "select">("open"); const [prevSelected, setPrevSelected] = useState(null); @@ -143,25 +120,6 @@ export function FilesFlyout({ return tab_to_path(activeTab); }, [activeTab]); - // copied roughly from directory-selector.tsx - useEffect(() => { - // Run the loop below every 30s until project_id or current_path changes (or unmount) - // in which case loop stops. If not unmount, then get new loops for new values. - if (!project_id) return; - const state = { loop: true }; - (async () => { - while (state.loop && isMountedRef.current) { - // Component is mounted, so call watch on all expanded paths. - const listings = redux.getProjectStore(project_id).get_listings(); - listings.watch(current_path); - await delay(WATCH_THROTTLE_MS); - } - })(); - return () => { - state.loop = false; - }; - }, [project_id, current_path]); - // selecting files switches over to "select" mode or back to "open" useEffect(() => { if (mode === "open" && checked_files.size > 0) { @@ -172,6 +130,16 @@ export function FilesFlyout({ } }, [checked_files]); + const fs = useFs({ project_id, compute_server_id }); + const { + listing: directoryListing, + error: listingError, + refresh, + } = useListing({ + fs, + path: current_path, + }); + // active file: current editor is the file in the listing // empty: either no files, or just the ".." for the parent dir const [directoryFiles, fileMap, activeFile, isEmpty] = useMemo((): [ @@ -180,28 +148,19 @@ export function FilesFlyout({ DirectoryListingEntry | null, boolean, ] => { - if (directoryListings == null) return EMPTY_LISTING; - const filesStore = directoryListings.get(current_path); - if (filesStore == null) return EMPTY_LISTING; - - // TODO this is an error, process it - if (typeof filesStore === "string") return EMPTY_LISTING; - - const files: DirectoryListing | null = filesStore.toJS?.(); + const files = directoryListing; if (files == null) return EMPTY_LISTING; let activeFile: DirectoryListingEntry | null = null; - compute_file_masks(files); - const searchWords = search_split(file_search.trim().toLowerCase()); + computeFileMasks(files); + const searchWords = file_search.trim().toLowerCase(); - const procFiles = files + const processedFiles: DirectoryListingEntry[] = files .filter((file: DirectoryListingEntry) => { - file.name ??= ""; // sanitization - if (file_search === "") return true; - const fName = file.name.toLowerCase(); + const filename = file.name.toLowerCase(); return ( - search_match(fName, searchWords) || - ((file.isdir ?? false) && search_match(`${fName}/`, searchWords)) + filename.includes(searchWords) || + (file.isDir && `${filename}/`.includes(searchWords)) ); }) .filter( @@ -211,17 +170,7 @@ export function FilesFlyout({ (file: DirectoryListingEntry) => hidden || !file.name.startsWith("."), ); - // this shares the logic with what's in project_store.js - mutate_data_to_compute_public_files( - { - listing: procFiles, - public: {}, - }, - strippedPublicPaths, - current_path, - ); - - procFiles.sort((a, b) => { + processedFiles.sort((a, b) => { // This replicated what project_store is doing const col = activeFileSort.get("column_name"); switch (col) { @@ -232,8 +181,8 @@ export function FilesFlyout({ case "time": return (b.mtime ?? 0) - (a.mtime ?? 0); case "type": - const aDir = a.isdir ?? false; - const bDir = b.isdir ?? false; + const aDir = a.isDir ?? false; + const bDir = b.isDir ?? false; if (aDir && !bDir) return -1; if (!aDir && bDir) return 1; const aExt = a.name.split(".").pop() ?? ""; @@ -245,47 +194,47 @@ export function FilesFlyout({ } }); - for (const file of procFiles) { - const fullPath = path_to_file(current_path, file.name); - if (openFiles.some((path) => path == fullPath)) { - file.isopen = true; - } - if (activePath === fullPath) { - file.isactive = true; - activeFile = file; - } - } - if (activeFileSort.get("is_descending")) { - procFiles.reverse(); // inplace op + processedFiles.reverse(); // inplace op } - const isEmpty = procFiles.length === 0; + const isEmpty = processedFiles.length === 0; // the ".." dir does not change the isEmpty state // hide ".." if there is a search -- https://github.com/sagemathinc/cocalc/issues/6877 if (file_search === "" && current_path != "") { - procFiles.unshift({ + processedFiles.unshift({ name: "..", - isdir: true, + isDir: true, + size: -1, // not used and we don't know the size in bytes + mtime: 0, // also not known }); } // map each filename to it's entry in the directory listing - const fileMap = fromPairs(procFiles.map((file) => [file.name, file])); + const fileMap = fromPairs(processedFiles.map((file) => [file.name, file])); - return [procFiles, fileMap, activeFile, isEmpty]; + return [processedFiles, fileMap, activeFile, isEmpty]; }, [ - directoryListings, + directoryListing, activeFileSort, hidden, file_search, - openFiles, show_masked, current_path, strippedPublicPaths, ]); + const isOpen = (file) => openFiles.has(path_to_file(current_path, file.name)); + const isActive = (file) => + activePath == path_to_file(current_path, file.name); + + const publicFiles = getPublicFiles( + directoryFiles, + strippedPublicPaths, + current_path, + ); + const prev_current_path = usePrevious(current_path); useEffect(() => { @@ -309,7 +258,7 @@ export function FilesFlyout({ useEffect(() => { setShowCheckboxIndex(null); - }, [directoryListings, current_path]); + }, [directoryListing, current_path]); const triggerRootResize = debounce( () => setRootHeightPx(rootRef.current?.clientHeight ?? 0), @@ -353,27 +302,6 @@ export function FilesFlyout({ return fileMap[basename]; } - if (directoryListings == null) { - (async () => { - await delay(0); - // Ensure store gets initialized before redux - // E.g., for copy between projects you make this - // directory selector before even opening the project. - redux.getProjectStore(project_id); - })(); - } - - if (directoryListings?.get(current_path) == null) { - (async () => { - // Must happen in a different render loop, hence the delay, because - // fetch can actually update the store in the same render loop. - await delay(0); - redux - .getProjectActions(project_id) - ?.fetch_directory_listing({ path: current_path }); - })(); - } - function open( e: PartialClickEvent, index: number, @@ -386,7 +314,7 @@ export function FilesFlyout({ if (!skip) { const fullPath = path_to_file(current_path, file.name); - if (file.isdir) { + if (file.isDir) { // true: change history, false: do not show "files" page actions?.open_directory(fullPath, true, false); setSearchState(""); @@ -459,7 +387,7 @@ export function FilesFlyout({ } // similar, if in open mode and already opened, just switch to it as well - if (mode === "open" && file.isopen && !e.shiftKey && !e.ctrlKey) { + if (mode === "open" && isOpen(file) && !e.shiftKey && !e.ctrlKey) { setPrevSelected(index); open(e, index); return; @@ -523,13 +451,13 @@ export function FilesFlyout({ } function renderTimeAgo(item: DirectoryListingEntry) { - const { mtime, isopen = false } = item; + const { mtime } = item; if (typeof mtime === "number") { return ( ); } @@ -542,7 +470,7 @@ export function FilesFlyout({ case "time": return renderTimeAgo(item); case "type": - if (item.isdir) return "Folder"; + if (item.isDir) return "Folder"; const { ext } = separate_file_extension(item.name); return capitalize(file_options(item.name).name) || ext; case "name": @@ -570,7 +498,7 @@ export function FilesFlyout({ function renderListItem(index: number, item: DirectoryListingEntry) { const { mtime, mask = false } = item; - const age = typeof mtime === "number" ? 1000 * mtime : null; + const age = typeof mtime === "number" ? mtime : null; // either select by scrolling (and only scrolling!) or by clicks const isSelected = scrollIdx != null @@ -581,7 +509,12 @@ export function FilesFlyout({ return ( + {disableUploads ? ( renderListing() @@ -710,9 +644,6 @@ export function FilesFlyout({ actions?.fetch_directory_listing(), - }} style={{ flex: "1 0 auto", display: "flex", @@ -736,6 +667,7 @@ export function FilesFlyout({ open={open} showFileSharingDialog={showFileSharingDialog} getFile={getFile} + publicFiles={publicFiles} />
); diff --git a/src/packages/frontend/project/page/flyouts/header.tsx b/src/packages/frontend/project/page/flyouts/header.tsx index bacbf7573e8..ad40f53e1a0 100644 --- a/src/packages/frontend/project/page/flyouts/header.tsx +++ b/src/packages/frontend/project/page/flyouts/header.tsx @@ -6,7 +6,6 @@ import { Button, Tooltip } from "antd"; import { useEffect, useState } from "react"; import { FormattedMessage, useIntl } from "react-intl"; - import { TourName } from "@cocalc/frontend/account/tours"; import { redux, useTypedRedux } from "@cocalc/frontend/app-framework"; import { Icon } from "@cocalc/frontend/components"; diff --git a/src/packages/frontend/project/page/flyouts/log.tsx b/src/packages/frontend/project/page/flyouts/log.tsx index e9d297dab67..22b63efb7f5 100644 --- a/src/packages/frontend/project/page/flyouts/log.tsx +++ b/src/packages/frontend/project/page/flyouts/log.tsx @@ -347,13 +347,13 @@ export function LogFlyout({ const time = entry.time; const account_id = entry.account_id; const path = entry.filename; - const isOpened: boolean = openFiles.some((p) => p === path); + const isOpen: boolean = openFiles.some((p) => p === path); const isActive: boolean = activePath === path; return ( ; +export function SearchFlyout() { + return ; } diff --git a/src/packages/frontend/project/page/share-indicator.tsx b/src/packages/frontend/project/page/share-indicator.tsx index 36f9c5eaf8f..ad4d826349d 100644 --- a/src/packages/frontend/project/page/share-indicator.tsx +++ b/src/packages/frontend/project/page/share-indicator.tsx @@ -72,7 +72,7 @@ export const ShareIndicator: React.FC = React.memo(
- ); - } - - if (isFlyout) { - return ( -
- {renderContent()} -
- ); - } else { - return {renderContent()}; - } + return ( +
+ {renderHeader()} + +
+ ); }; interface ProjectSearchInputProps { project_id: string; - neural?: boolean; - git?: boolean; small?: boolean; + regexp?: boolean; } function ProjectSearchInput({ - neural, project_id, - git, small = false, + regexp, }: ProjectSearchInputProps) { const actions = useActions({ project_id }); const user_input = useTypedRedux({ project_id }, "user_input"); - return ( actions?.setState({ user_input: value })} on_submit={() => actions?.search()} on_clear={() => @@ -298,22 +275,7 @@ function ProjectSearchInput({ type="primary" onClick={() => actions?.search()} > - {neural ? ( - <> - - {small ? "" : " Neural Search"} - - ) : git ? ( - <> - - {small ? "" : " Git Grep Search"} - - ) : ( - <> - - {small ? "" : " Grep Search"} - - )} + Search } /> @@ -324,16 +286,16 @@ interface ProjectSearchOutputProps { project_id: string; wrap?: Function; mode?: "project" | "flyout"; + actions?; } function ProjectSearchOutput({ project_id, - wrap, mode = "project", + actions, }: ProjectSearchOutputProps) { const [filter, setFilter] = useState(""); const [currentFilter, setCurrentFilter] = useState(""); - const isFlyout = mode === "flyout"; const most_recent_search = useTypedRedux( { project_id }, "most_recent_search", @@ -346,10 +308,9 @@ function ProjectSearchOutput({ const search_error = useTypedRedux({ project_id }, "search_error"); const too_many_results = useTypedRedux({ project_id }, "too_many_results"); - useEffect(() => { - setFilter(""); - setCurrentFilter(""); - }, [unfiltered_search_results]); + const virtuosoScroll = useVirtuosoScrollHook({ + cacheId: `search-${project_id}`, + }); const search_results = useMemo(() => { const f = filter?.trim(); @@ -376,159 +337,109 @@ function ProjectSearchOutput({ } function render_get_results() { - if (search_error != null) { - return ( - - Search error: {search_error} Please try a different type of search or - a more restrictive search. - - ); - } - if (search_results?.size == 0) { + if (search_results?.size == 0 && !search_error) { return ( - There were no results for your search. + No results for your search. ); } - const v: React.JSX.Element[] = []; - let i = 0; - for (const result of search_results) { - v.push( - , - ); - i += 1; - } - return v; - } - - function renderResultList() { - if (isFlyout) { - return wrap?.( - - {render_get_results()} - , - { borderTop: `1px solid ${COLORS.GRAY_L}` }, - ); - } else { - return {render_get_results()}; - } + return ( + { + const result = search_results.get(index); + if (result == null) { + return null; + } + return ( + + ); + }} + {...virtuosoScroll} + /> + ); } return ( - <> +
setCurrentFilter(e.target.value)} - placeholder="Filter... (regexp in / /)" + placeholder="Filter results... (regexp in / /)" onSearch={setFilter} enterButton="Filter" style={{ width: "350px", maxWidth: "100%", marginBottom: "15px" }} /> {too_many_results && ( + + {search_results.size} {plural(search_results.size, "Result")}: + {" "} There were more results than displayed below. Try making your search more specific. )} - {renderResultList()} - + {!too_many_results && ( + + + {search_results.size} {plural(search_results.size, "Result")} + + + )} + { + actions?.setState({ search_error: undefined }); + }} + /> +
{render_get_results()}
+
); } function ProjectSearchOutputHeader({ project_id }: { project_id: string }) { const actions = useActions({ project_id }); - const info_visible = useTypedRedux({ project_id }, "info_visible"); - const search_results = useTypedRedux({ project_id }, "search_results"); - const command = useTypedRedux({ project_id }, "command"); const most_recent_search = useTypedRedux( { project_id }, "most_recent_search", ); const most_recent_path = useTypedRedux({ project_id }, "most_recent_path"); - function output_path() { - return !most_recent_path ? : most_recent_path; - } - - function render_get_info() { - return ( - -
    -
  • - Search command (in a terminal):
    {command}
    -
  • -
  • - Number of results:{" "} - {search_results ? search_results?.size : } -
  • -
-
- ); - } - if (most_recent_search == null || most_recent_path == null) { return ; } return ( -
- - -

- Results of searching in {output_path()} for "{most_recent_search}" - - -

- - {info_visible && render_get_info()} + ); } -const DESC_STYLE: React.CSSProperties = { - color: COLORS.GRAY_M, - marginBottom: "5px", - border: "1px solid #eee", - borderRadius: "5px", - maxHeight: "300px", - padding: "15px", - overflowY: "auto", -} as const; - interface ProjectSearchResultLineProps { project_id: string; filename: string; @@ -539,17 +450,14 @@ interface ProjectSearchResultLineProps { mode?: "project" | "flyout"; } -function ProjectSearchResultLine(_: Readonly) { - const { - project_id, - filename, - description, - line_number, - fragment_id, - most_recent_path, - mode = "project", - } = _; - const isFlyout = mode === "flyout"; +function ProjectSearchResultLine({ + project_id, + filename, + description, + line_number, + fragment_id, + most_recent_path, +}: Readonly) { const actions = useActions({ project_id }); const ext = filename_extension(filename); const icon = file_associations[ext]?.icon ?? "file"; @@ -595,40 +503,32 @@ function ProjectSearchResultLine(_: Readonly) { ); } - if (isFlyout) { - return ( - - } - > - - - ); - } else { - return ( -
- {renderFileLink()} -
- -
-
- ); - } + } + > + + + ); } const MARKDOWN_EXTS = ["tasks", "slides", "board", "sage-chat"] as const; diff --git a/src/packages/frontend/project/search/run.ts b/src/packages/frontend/project/search/run.ts new file mode 100644 index 00000000000..3dafa94d49e --- /dev/null +++ b/src/packages/frontend/project/search/run.ts @@ -0,0 +1,93 @@ +import { type FilesystemClient } from "@cocalc/conat/files/fs"; +import { trunc } from "@cocalc/util/misc"; + +// we get about this many bytes of results from the filesystem, then stop... +const MAX_SIZE = 1_000_000; + +// 2s to be responsive and minimize server load. +const TIMEOUT = 2_000; + +const MAX_LINE_LENGTH = 256; + +interface SearchResult { + filename: string; + description: string; + line_number: number; + filter: string; +} + +export async function search({ + query, + path, + setState, + fs, + options = {}, +}: { + query: string; + path: string; + setState: (any) => void; + fs: FilesystemClient; + options: { + case_sensitive?: boolean; + git_grep?: boolean; + subdirectories?: boolean; + hidden_files?: boolean; + regexp?: boolean; + }; +}) { + if (!query.trim()) { + return; + } + + const rgOptions = ["--json"]; // note that -M doesn't seem to combine with --json, so can't do -M {MAX_LINE_LENGTH} + if (!options.subdirectories) { + rgOptions.push("-d", "1"); + } + if (!options.case_sensitive) { + rgOptions.push("-i"); + } + if (options.hidden_files) { + rgOptions.push("-."); + } + if (!options.git_grep) { + rgOptions.push("--no-ignore"); + } + if (!options.regexp) { + rgOptions.push("-F"); + } + + const { stdout, stderr, truncated } = await fs.ripgrep(path, query, { + options: rgOptions, + maxSize: MAX_SIZE, + timeout: TIMEOUT, + }); + const lines = Buffer.from(stdout).toString().split("\n"); + + const search_results: SearchResult[] = []; + for (const line of lines) { + let result; + try { + result = JSON.parse(line); + } catch { + continue; + } + if (result.type == "match") { + const { line_number, lines, path } = result.data; + const description = trunc(lines?.text ?? "", MAX_LINE_LENGTH); + search_results.push({ + filename: path?.text ?? "-", + description: `${(line_number.toString() + ":").padEnd(8, " ")}${description}`, + filter: `${path?.text?.toLowerCase?.() ?? ""} ${description.toLowerCase()}`, + line_number, + }); + } + } + + setState({ + too_many_results: truncated, + search_results, + most_recent_search: query, + most_recent_path: path, + search_error: Buffer.from(stderr).toString(), + }); +} diff --git a/src/packages/frontend/project/search/search.tsx b/src/packages/frontend/project/search/search.tsx index c9b6a3d53d2..19bfa377aae 100644 --- a/src/packages/frontend/project/search/search.tsx +++ b/src/packages/frontend/project/search/search.tsx @@ -4,7 +4,7 @@ import { ProjectSearchHeader } from "./header"; export const ProjectSearch: React.FC = () => { return ( -
+
diff --git a/src/packages/frontend/project/settings/software-env-info.tsx b/src/packages/frontend/project/settings/software-env-info.tsx index 25ba23a6c8a..15616a227c8 100644 --- a/src/packages/frontend/project/settings/software-env-info.tsx +++ b/src/packages/frontend/project/settings/software-env-info.tsx @@ -3,7 +3,7 @@ * License: MS-RSL – see LICENSE.md for details */ -// cSpell:ignore descr disp dflt +// cSpell:ignore descr disp import { join } from "path"; import { FormattedMessage } from "react-intl"; diff --git a/src/packages/frontend/project/utils.ts b/src/packages/frontend/project/utils.ts index e1bfb9cbaf8..d0f2b5a1c12 100644 --- a/src/packages/frontend/project/utils.ts +++ b/src/packages/frontend/project/utils.ts @@ -78,7 +78,7 @@ export class NewFilenames { // generate a new filename, by optionally avoiding the keys in the dictionary public gen( type?: NewFilenameTypes, - avoid?: { [name: string]: boolean }, + avoid?: { [name: string]: any } | null, ): string { type = this.sanitize_type(type); // reset the enumeration if type changes diff --git a/src/packages/frontend/project/websocket/api.ts b/src/packages/frontend/project/websocket/api.ts index 00ed94d1a63..97de9a505e8 100644 --- a/src/packages/frontend/project/websocket/api.ts +++ b/src/packages/frontend/project/websocket/api.ts @@ -19,7 +19,6 @@ import { import type { Config as FormatterConfig, Options as FormatterOptions, - FormatResult, } from "@cocalc/util/code-formatter"; import { syntax2tool } from "@cocalc/util/code-formatter"; import { DirectoryListingEntry } from "@cocalc/util/types"; @@ -35,7 +34,6 @@ import type { ExecuteCodeOutput, ExecuteCodeOptions, } from "@cocalc/util/types/execute-code"; -import { formatterClient } from "@cocalc/conat/service/formatter"; import { syncFsClientClient } from "@cocalc/conat/service/syncfs-client"; const log = (...args) => { @@ -119,14 +117,6 @@ export class API { return await api.system.version(); }; - delete_files = async ( - paths: string[], - compute_server_id?: number, - ): Promise => { - const api = this.getApi({ compute_server_id, timeout: 60000 }); - return await api.system.deleteFiles({ paths }); - }; - // Move the given paths to the dest. The folder dest must exist // already and be a directory, or this is in an error. move_files = async ( @@ -259,33 +249,15 @@ export class API { } }; - // Returns { status: "ok", patch:... the patch} or - // { status: "error", phase: "format", error: err.message }. - // We return a patch rather than the entire file, since often - // the file is very large, but the formatting is tiny. This is purely - // a data compression technique. - formatter = async ( - path: string, - config: FormatterConfig, - compute_server_id?: number, - ): Promise => { - const options: FormatterOptions = this.check_formatter_available(config); - const client = formatterClient({ - project_id: this.project_id, - compute_server_id: compute_server_id ?? this.getComputeServerId(path), - }); - return await client.formatter({ path, options }); - }; - formatter_string = async ( str: string, config: FormatterConfig, timeout: number = 15000, compute_server_id?: number, - ): Promise => { + ): Promise => { const options: FormatterOptions = this.check_formatter_available(config); const api = this.getApi({ compute_server_id, timeout }); - return await api.editor.formatterString({ str, options }); + return await api.editor.formatString({ str, options }); }; exec = async (opts: ExecuteCodeOptions): Promise => { @@ -326,7 +298,7 @@ export class API { compute_server_id ?? this.getComputeServerId(opts.args[0]), timeout: (opts.timeout ?? 60) * 1000 + 5000, }); - return await api.editor.jupyterNbconvert(opts); + return await api.jupyter.nbconvert(opts); }; // Get contents of an ipynb file, but with output and attachments removed (to save space) @@ -338,7 +310,7 @@ export class API { compute_server_id: compute_server_id ?? this.getComputeServerId(ipynb_path), }); - return await api.editor.jupyterStripNotebook(ipynb_path); + return await api.jupyter.stripNotebook(ipynb_path); }; // Run the notebook filling in the output of all cells, then return the @@ -358,7 +330,7 @@ export class API { compute_server_id, timeout: 60 + 2 * max_total_time_ms, }); - return await api.editor.jupyterRunNotebook(opts); + return await api.jupyter.runNotebook(opts); }; // Get the x11 *channel* for the given '.x11' path. diff --git a/src/packages/frontend/project_actions.ts b/src/packages/frontend/project_actions.ts index c6ca0fed221..f196b2752c5 100644 --- a/src/packages/frontend/project_actions.ts +++ b/src/packages/frontend/project_actions.ts @@ -10,9 +10,8 @@ import * as async from "async"; import { callback } from "awaiting"; import { List, Map, fromJS, Set as immutableSet } from "immutable"; import { isEqual, throttle } from "lodash"; -import { join } from "path"; +import { basename, join } from "path"; import { defineMessage } from "react-intl"; - import { computeServerManager, type ComputeServerManager, @@ -39,7 +38,7 @@ import { exec, } from "@cocalc/frontend/frame-editors/generic/client"; import { set_url } from "@cocalc/frontend/history"; -import { IntlMessage, dialogs } from "@cocalc/frontend/i18n"; +import { dialogs } from "@cocalc/frontend/i18n"; import { getIntl } from "@cocalc/frontend/i18n/get-intl"; import { download_file, @@ -49,8 +48,6 @@ import { } from "@cocalc/frontend/misc"; import Fragment, { FragmentId } from "@cocalc/frontend/misc/fragment-id"; import * as project_file from "@cocalc/frontend/project-file"; -import { delete_files } from "@cocalc/frontend/project/delete-files"; -import fetchDirectoryListing from "@cocalc/frontend/project/fetch-directory-listing"; import { ProjectEvent, SoftwareEnvironmentEvent, @@ -81,7 +78,6 @@ import { transform_get_url } from "@cocalc/frontend/project/transform-get-url"; import { NewFilenames, download_href, - in_snapshot_path, normalize, url_href, } from "@cocalc/frontend/project/utils"; @@ -106,9 +102,16 @@ import { DEFAULT_NEW_FILENAMES, NEW_FILENAMES } from "@cocalc/util/db-schema"; import * as misc from "@cocalc/util/misc"; import { reduxNameToProjectId } from "@cocalc/util/redux/name"; import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { MARKERS } from "@cocalc/util/sagews"; import { client_db } from "@cocalc/util/schema"; import { get_editor } from "./editors/react-wrapper"; +import { type FilesystemClient } from "@cocalc/conat/files/fs"; +import { + getCacheId, + getFiles, + type Files, +} from "@cocalc/frontend/project/listing/use-files"; +import { search } from "@cocalc/frontend/project/search/run"; +import { type CopyOptions } from "@cocalc/conat/files/fs"; const { defaults, required } = misc; @@ -181,14 +184,7 @@ const must_define = function (redux) { const _init_library_index_ongoing = {}; const _init_library_index_cache = {}; -interface FileAction { - name: IntlMessage; - icon: IconName; - allows_multiple_files?: boolean; - hideFlyout?: boolean; -} - -export const FILE_ACTIONS: { [key: string]: FileAction } = { +export const FILE_ACTIONS = { compress: { name: defineMessage({ id: "file_actions.compress.name", @@ -197,6 +193,7 @@ export const FILE_ACTIONS: { [key: string]: FileAction } = { }), icon: "compress" as IconName, allows_multiple_files: true, + hideFlyout: false, }, delete: { name: defineMessage({ @@ -206,6 +203,7 @@ export const FILE_ACTIONS: { [key: string]: FileAction } = { }), icon: "trash" as IconName, allows_multiple_files: true, + hideFlyout: false, }, rename: { name: defineMessage({ @@ -215,6 +213,7 @@ export const FILE_ACTIONS: { [key: string]: FileAction } = { }), icon: "swap" as IconName, allows_multiple_files: false, + hideFlyout: false, }, duplicate: { name: defineMessage({ @@ -224,6 +223,7 @@ export const FILE_ACTIONS: { [key: string]: FileAction } = { }), icon: "clone" as IconName, allows_multiple_files: false, + hideFlyout: false, }, move: { name: defineMessage({ @@ -233,6 +233,7 @@ export const FILE_ACTIONS: { [key: string]: FileAction } = { }), icon: "move" as IconName, allows_multiple_files: true, + hideFlyout: false, }, copy: { name: defineMessage({ @@ -242,6 +243,7 @@ export const FILE_ACTIONS: { [key: string]: FileAction } = { }), icon: "files" as IconName, allows_multiple_files: true, + hideFlyout: false, }, share: { name: defineMessage({ @@ -251,6 +253,7 @@ export const FILE_ACTIONS: { [key: string]: FileAction } = { }), icon: "share-square" as IconName, allows_multiple_files: false, + hideFlyout: false, }, download: { name: defineMessage({ @@ -260,6 +263,7 @@ export const FILE_ACTIONS: { [key: string]: FileAction } = { }), icon: "cloud-download" as IconName, allows_multiple_files: true, + hideFlyout: false, }, upload: { name: defineMessage({ @@ -268,6 +272,7 @@ export const FILE_ACTIONS: { [key: string]: FileAction } = { description: "Upload a file", }), icon: "upload" as IconName, + allows_multiple_files: false, hideFlyout: true, }, create: { @@ -277,10 +282,13 @@ export const FILE_ACTIONS: { [key: string]: FileAction } = { description: "Create a file", }), icon: "plus-circle" as IconName, + allows_multiple_files: false, hideFlyout: true, }, } as const; +export type FileAction = keyof typeof FILE_ACTIONS; + export class ProjectActions extends Actions { public state: "ready" | "closed" = "ready"; public project_id: string; @@ -377,8 +385,6 @@ export class ProjectActions extends Actions { this.remove_table(table); } - webapp_client.conat_client.closeOpenFiles(this.project_id); - const store = this.get_store(); store?.close_all_tables(); }; @@ -396,6 +402,7 @@ export class ProjectActions extends Actions { this.open_files?.close(); delete this.open_files; this.state = "closed"; + this._filesystem = {}; }; private save_session(): void { @@ -422,18 +429,16 @@ export class ProjectActions extends Actions { } }; - public _ensure_project_is_open(cb): void { - const s: any = this.redux.getStore("projects"); + ensureProjectIsOpen = async () => { + const s = this.redux.getStore("projects"); if (!s.is_project_open(this.project_id)) { - (this.redux.getActions("projects") as any).open_project({ + this.redux.getActions("projects").open_project({ project_id: this.project_id, switch_to: true, }); - s.wait_until_project_is_open(this.project_id, 30, cb); - } else { - cb(); + await s.waitUntilProjectIsOpen(this.project_id, 30); } - } + }; public get_store(): ProjectStore | undefined { if (this.redux.hasStore(this.name)) { @@ -621,9 +626,6 @@ export class ProjectActions extends Actions { if (opts.change_history) { this.set_url_to_path(store.get("current_path") ?? "", ""); } - if (opts.update_file_listing) { - this.fetch_directory_listing(); - } break; case "new": @@ -1061,28 +1063,34 @@ export class ProjectActions extends Actions { }; /* Initialize the redux store and react component for editing - a particular file. + a particular file, if necessary. */ - initFileRedux = async ( - path: string, - is_public: boolean = false, - ext?: string, // use this extension even instead of path's extension. - ): Promise => { - // LAZY IMPORT, so that editors are only available - // when you are going to use them. Helps with code splitting. - await import("./editors/register-all"); - - // Initialize the file's store and actions - const name = await project_file.initializeAsync( - path, - this.redux, - this.project_id, - is_public, - undefined, - ext, - ); - return name; - }; + initFileRedux = reuseInFlight( + async ( + path: string, + is_public: boolean = false, + ext?: string, // use this extension even instead of path's extension. + ): Promise => { + const cur = redux.getEditorActions(this.project_id, path); + if (cur != null) { + return cur.name; + } + // LAZY IMPORT, so that editors are only available + // when you are going to use them. Helps with code splitting. + await import("./editors/register-all"); + + // Initialize the file's store and actions + const name = await project_file.initializeAsync( + path, + this.redux, + this.project_id, + is_public, + undefined, + ext, + ); + return name; + }, + ); private init_file_react_redux = async ( path: string, @@ -1408,59 +1416,56 @@ export class ProjectActions extends Actions { }; // Makes this project the active project tab - foreground_project = (change_history = true): void => { - this._ensure_project_is_open((err) => { - if (err) { - // TODO! - console.warn( - "error putting project in the foreground: ", - err, - this.project_id, - ); - } else { - (this.redux.getActions("projects") as any).foreground_project( - this.project_id, - change_history, - ); - } - }); + foreground_project = async (change_history = true) => { + try { + await this.ensureProjectIsOpen(); + } catch (err) { + console.warn( + "error putting project in the foreground: ", + err, + this.project_id, + ); + return; + } + this.redux + .getActions("projects") + .foreground_project(this.project_id, change_history); }; - open_directory(path, change_history = true, show_files = true): void { + open_directory = async (path, change_history = true, show_files = true) => { path = normalize(path); - this._ensure_project_is_open(async (err) => { - if (err) { - // TODO! - console.log( - "error opening directory in project: ", - err, - this.project_id, - path, - ); - } else { - if (path[path.length - 1] === "/") { - path = path.slice(0, -1); - } - this.foreground_project(change_history); - this.set_current_path(path); - const store = this.get_store(); - if (store == undefined) { - return; - } - if (show_files) { - this.set_active_tab("files", { - update_file_listing: false, - change_history: false, // see "if" below - }); - } - if (change_history) { - // i.e. regardless of show_files is true or false, we might want to record this in the history - this.set_url_to_path(store.get("current_path") ?? "", ""); - } - this.set_all_files_unchecked(); - } - }); - } + try { + await this.ensureProjectIsOpen(); + } catch (err) { + console.warn( + "error opening directory in project: ", + err, + this.project_id, + path, + ); + return; + } + if (path[path.length - 1] === "/") { + path = path.slice(0, -1); + } + this.foreground_project(change_history); + this.set_current_path(path); + const store = this.get_store(); + if (store == undefined) { + return; + } + if (show_files) { + this.set_active_tab("files", { + update_file_listing: false, + change_history: false, // see "if" below + }); + } + if (change_history) { + // i.e. regardless of show_files is true or false, we might want to record this in the history + this.set_url_to_path(store.get("current_path") ?? "", ""); + } + this.set_all_files_unchecked(); + }; // ONLY updates current path // Does not push to URL, browser history, or add to analytics @@ -1493,11 +1498,8 @@ export class ProjectActions extends Actions { this.setState({ current_path: path, history_path, - page_number: 0, most_recent_file_click: undefined, }); - - store.get_listings().watch(path, true); }; setComputeServerId = (compute_server_id: number) => { @@ -1514,7 +1516,6 @@ export class ProjectActions extends Actions { compute_server_id, checked_files: store.get("checked_files").clear(), // always clear on compute_server_id change }); - this.fetch_directory_listing({ compute_server_id }); set_local_storage( store.computeServerIdLocalStorageKey, `${compute_server_id}`, @@ -1548,44 +1549,12 @@ export class ProjectActions extends Actions { set_file_search(search): void { this.setState({ file_search: search, - page_number: 0, file_action: undefined, most_recent_file_click: undefined, create_file_alert: false, }); } - // Update the directory listing cache for the given path. - // Uses current path if path not provided. - fetch_directory_listing = async (opts?): Promise => { - await fetchDirectoryListing(this, opts); - }; - - public async fetch_directory_listing_directly( - path: string, - trigger_start_project?: boolean, - compute_server_id?: number, - ): Promise { - const store = this.get_store(); - if (store == null) return; - compute_server_id = this.getComputeServerId(compute_server_id); - const listings = store.get_listings(compute_server_id); - try { - const files = await listings.getListingDirectly( - path, - trigger_start_project, - ); - const directory_listings = store.get("directory_listings"); - let listing = directory_listings.get(compute_server_id) ?? Map(); - listing = listing.set(path, files); - this.setState({ - directory_listings: directory_listings.set(compute_server_id, listing), - }); - } catch (err) { - console.warn(`Unable to fetch directory listing -- "${err}"`); - } - } - // Sets the active file_sort to next_column_name set_sorted_file_column(column_name): void { let is_descending; @@ -1612,9 +1581,11 @@ export class ProjectActions extends Actions { if (store == undefined) { return; } - const selected_index = store.get("selected_file_index"); - const current_index = selected_index != null ? selected_index : -1; - this.setState({ selected_file_index: current_index + 1 }); + const selected_index = store.get("selected_file_index") ?? 0; + const numDisplayedFiles = store.get("numDisplayedFiles") ?? 0; + if (selected_index + 1 < numDisplayedFiles) { + this.setState({ selected_file_index: selected_index + 1 }); + } } // Decreases the selected file index by 1. @@ -1625,9 +1596,9 @@ export class ProjectActions extends Actions { if (store == undefined) { return; } - const current_index = store.get("selected_file_index"); - if (current_index != null && current_index > 0) { - this.setState({ selected_file_index: current_index - 1 }); + const selected_index = store.get("selected_file_index") ?? 0; + if (selected_index > 0) { + this.setState({ selected_file_index: selected_index - 1 }); } } @@ -1645,7 +1616,7 @@ export class ProjectActions extends Actions { } // Set the selected state of all files between the most_recent_file_click and the given file - set_selected_file_range(file: string, checked: boolean): void { + set_selected_file_range(file: string, checked: boolean, listing): void { let range; const store = this.get_store(); if (store == undefined) { @@ -1658,9 +1629,9 @@ export class ProjectActions extends Actions { } else { // get the range of files const current_path = store.get("current_path"); - const names = store - .get("displayed_listing") - .listing.map((a) => misc.path_to_file(current_path, a.name)); + const names = listing.map(({ name }) => + misc.path_to_file(current_path, name), + ); range = misc.get_array_range(names, most_recent, file); } @@ -1679,7 +1650,7 @@ export class ProjectActions extends Actions { } const changes: { checked_files?: immutableSet; - file_action?: string | undefined; + file_action?: FileAction | undefined; } = {}; if (checked) { changes.checked_files = store.get("checked_files").add(file); @@ -1709,7 +1680,7 @@ export class ProjectActions extends Actions { } const changes: { checked_files: immutableSet; - file_action?: string | undefined; + file_action?: FileAction | undefined; } = { checked_files: store.get("checked_files").union(file_list) }; const file_action = store.get("file_action"); if ( @@ -1731,7 +1702,7 @@ export class ProjectActions extends Actions { } const changes: { checked_files: immutableSet; - file_action?: string | undefined; + file_action?: FileAction | undefined; } = { checked_files: store.get("checked_files").subtract(file_list) }; if (changes.checked_files.size === 0) { @@ -1749,44 +1720,9 @@ export class ProjectActions extends Actions { } this.setState({ checked_files: store.get("checked_files").clear(), - file_action: undefined, }); } - // this isn't really an action, but very helpful! - public get_filenames_in_current_dir(): - | { [name: string]: boolean } - | undefined { - const store = this.get_store(); - if (store == undefined) { - return; - } - - const files_in_dir = {}; - // This will set files_in_dir to our current view of the files in the current - // directory (at least the visible ones) or do nothing in case we don't know - // anything about files (highly unlikely). Unfortunately (for this), our - // directory listings are stored as (immutable) lists, so we have to make - // a map out of them. - const compute_server_id = store.get("compute_server_id"); - const listing = store.getIn([ - "directory_listings", - compute_server_id, - store.get("current_path"), - ]); - - if (typeof listing === "string") { - // must be an error - return undefined; // simple fallback - } - if (listing != null) { - listing.map(function (x) { - files_in_dir[x.get("name")] = true; - }); - } - return files_in_dir; - } - suggestDuplicateFilenameInCurrentDirectory = ( name: string, ): string | undefined => { @@ -1807,21 +1743,31 @@ export class ProjectActions extends Actions { } }; - set_file_action(action?: string): void { + set_file_action = (action?: FileAction): void => { const store = this.get_store(); if (store == null) { return; } this.setState({ file_action: action }); - } + }; - show_file_action_panel(opts): void { - opts = defaults(opts, { - path: required, - action: required, - }); + showFileActionPanel = async ({ + path, + action, + }: { + path: string; + action: + | FileAction + | "open" + | "open_recent" + | "quit" + | "close" + | "new" + | "create" + | "upload"; + }) => { this.set_all_files_unchecked(); - if (opts.action == "new" || opts.action == "create") { + if (action == "new" || action == "create") { // special case because it isn't a normal "file action panel", // but it is convenient to still support this. if (this.get_store()?.get("flyout") != "new") { @@ -1829,13 +1775,13 @@ export class ProjectActions extends Actions { } this.setState({ default_filename: default_filename( - misc.filename_extension(opts.path), + misc.filename_extension(path), this.project_id, ), }); return; } - if (opts.action == "upload") { + if (action == "upload") { this.set_active_tab("files"); setTimeout(() => { // NOTE: I'm not proud of this, but right now our upload functionality @@ -1844,34 +1790,34 @@ export class ProjectActions extends Actions { }, 100); return; } - if (opts.action == "open") { + if (action == "open") { if (this.get_store()?.get("flyout") != "files") { this.toggleFlyout("files"); } return; } - if (opts.action == "open_recent") { + if (action == "open_recent") { if (this.get_store()?.get("flyout") != "log") { this.toggleFlyout("log"); } return; } - const path_splitted = misc.path_split(opts.path); - this.open_directory(path_splitted.head); + const path_splitted = misc.path_split(path); + await this.open_directory(path_splitted.head); - if (opts.action == "quit") { + if (action == "quit") { // TODO: for jupyter and terminal at least, should also do more! - this.close_tab(opts.path); + this.close_tab(path); return; } - if (opts.action == "close") { - this.close_tab(opts.path); + if (action == "close") { + this.close_tab(path); return; } - this.set_file_checked(opts.path, true); - this.set_file_action(opts.action); - } + this.set_file_checked(path, true); + this.set_file_action(action); + }; private async get_from_web(opts: { url: string; @@ -1906,7 +1852,6 @@ export class ProjectActions extends Actions { // returns a function that takes the err and output and // does the right activity logging stuff. return (err?, output?) => { - this.fetch_directory_listing(); if (err) { this.set_activity({ id, error: err }); } else if ( @@ -1922,61 +1867,29 @@ export class ProjectActions extends Actions { }; } - zip_files = async ({ - src, - dest, - path, - }: { - src: string[]; - dest: string; - path?: string; - }) => { - const args = ["-rq", dest, ...src]; - const id = misc.uuid(); - this.set_activity({ - id, - status: `Creating ${dest} from ${src.length} ${misc.plural( - src.length, - "file", - )}`, - }); - try { - this.log({ event: "file_action", action: "created", files: [dest] }); - await webapp_client.exec({ - project_id: this.project_id, - command: "zip", - args, - timeout: 10 * 60 /* compressing CAN take a while -- zip is slow! */, - err_on_exit: true, // this should fail if exit_code != 0 - compute_server_id: this.get_store()?.get("compute_server_id"), - filesystem: true, - path, - }); - } catch (err) { - this.set_activity({ id, error: `${err}` }); - throw err; - } finally { - this.set_activity({ id, stop: "" }); - this.fetch_directory_listing(); - } - }; - - // DANGER: ASSUMES PATH IS IN THE DISPLAYED LISTING - private _convert_to_displayed_path(path): string { - if (path.slice(-1) === "/") { - return path; - } else { - const store = this.get_store(); - const file_name = misc.path_split(path).tail; - if (store !== undefined && store.get("displayed_listing")) { - const file_data = store.get("displayed_listing").file_map[file_name]; - if (file_data !== undefined && file_data.isdir) { - return path + "/"; - } + private appendSlashToDirectoryPaths = async ( + paths: string[], + compute_server_id?: number, + ): Promise => { + const f = async (path: string) => { + if (path.endsWith("/")) { + return path; } - return path; - } - } + const isDir = this.isDirViaCache(path, compute_server_id); + if (isDir === false) { + return path; + } + if (isDir === true) { + return path + "/"; + } + if (await this.isDir(path, compute_server_id)) { + return path + "/"; + } else { + return path; + } + }; + return await Promise.all(paths.map(f)); + }; // this is called in "projects.cjsx" (more then once) // in turn, it is calling init methods just once, though @@ -2275,7 +2188,14 @@ export class ProjectActions extends Actions { this.setState({ library_is_copying: status }); } - copy_paths = async (opts: { + copyPaths = async ({ + src, + dest, + id, + only_contents, + src_compute_server_id = this.get_store()?.get("compute_server_id") ?? 0, + dest_compute_server_id = this.get_store()?.get("compute_server_id") ?? 0, + }: { src: string[]; dest: string; id?: string; @@ -2286,37 +2206,31 @@ export class ProjectActions extends Actions { dest_compute_server_id?: number; // NOTE: right now src_compute_server_id and dest_compute_server_id // must be the same or one of them must be 0. We don't implement - // copying directly from one compute server to another *yet*. + // copying directly from one compute server to another. }) => { - opts = defaults(opts, { - src: required, // Should be an array of source paths - dest: required, - id: undefined, - only_contents: false, - src_compute_server_id: this.get_store()?.get("compute_server_id") ?? 0, - dest_compute_server_id: this.get_store()?.get("compute_server_id") ?? 0, - }); // true for duplicating files - - const with_slashes = opts.src.map(this._convert_to_displayed_path); + const withSlashes = await this.appendSlashToDirectoryPaths( + src, + src_compute_server_id, + ); this.log({ event: "file_action", action: "copied", - files: with_slashes.slice(0, 3), - count: opts.src.length > 3 ? opts.src.length : undefined, - dest: opts.dest + (opts.only_contents ? "" : "/"), - ...(opts.src_compute_server_id != opts.dest_compute_server_id + files: withSlashes, + count: src.length, + dest: dest + (only_contents ? "" : "/"), + ...(src_compute_server_id != dest_compute_server_id ? { - src_compute_server_id: opts.src_compute_server_id, - dest_compute_server_id: opts.dest_compute_server_id, + src_compute_server_id: src_compute_server_id, + dest_compute_server_id: dest_compute_server_id, } - : opts.src_compute_server_id - ? { compute_server_id: opts.src_compute_server_id } + : src_compute_server_id + ? { compute_server_id: src_compute_server_id } : undefined), }); - if (opts.only_contents) { - opts.src = with_slashes; + if (only_contents) { + src = withSlashes; } // If files start with a -, make them interpretable by rsync (see https://github.com/sagemathinc/cocalc/issues/516) @@ -2326,18 +2240,18 @@ export class ProjectActions extends Actions { }; // Ensure that src files are not interpreted as an option to rsync - opts.src = opts.src.map(add_leading_dash); + src = src.map(add_leading_dash); - const id = opts.id != null ? opts.id : misc.uuid(); + id ??= misc.uuid(); this.set_activity({ id, - status: `Copying ${opts.src.length} ${misc.plural( - opts.src.length, + status: `Copying ${src.length} ${misc.plural( + src.length, "file", - )} to ${opts.dest}`, + )} to ${dest}`, }); - if (opts.src_compute_server_id != opts.dest_compute_server_id) { + if (src_compute_server_id != dest_compute_server_id) { // Copying from/to a compute server from/to a project. This uses // an api, which behind the scenes uses lz4 compression and tar // proxied via a websocket, but no use of rsync or ssh. @@ -2345,20 +2259,20 @@ export class ProjectActions extends Actions { // do it. try { const api = await this.api(); - if (opts.src_compute_server_id) { + if (src_compute_server_id) { // from compute server to project await api.copyFromComputeServerToHomeBase({ - compute_server_id: opts.src_compute_server_id, - paths: opts.src, - dest: opts.dest, + compute_server_id: src_compute_server_id, + paths: src, + dest: dest, timeout: 60 * 15 * 1000, }); - } else if (opts.dest_compute_server_id) { + } else if (dest_compute_server_id) { // from project to compute server await api.copyFromHomeBaseToComputeServer({ - compute_server_id: opts.dest_compute_server_id, - paths: opts.src, - dest: opts.dest, + compute_server_id: dest_compute_server_id, + paths: src, + dest: dest, timeout: 60 * 15 * 1000, }); } else { @@ -2375,91 +2289,47 @@ export class ProjectActions extends Actions { return; } - // Copying directly on project or on compute server. This just uses local rsync (no network). - - let args = ["-rltgoDxH"]; - - // We ensure the target copy is writable if *any* source path starts is inside of .snapshots. - // See https://github.com/sagemathinc/cocalc/issues/2497 and https://github.com/sagemathinc/cocalc/issues/4935 - for (const x of opts.src) { - if (in_snapshot_path(x)) { - args = args.concat(["--perms", "--chmod", "u+w"]); - break; - } + // Copying directly on project or on compute server. + const fs = this.fs(src_compute_server_id); + try { + await fs.cp(src, dest, { recursive: true, reflink: true }); + this._finish_exec(id)(); + } catch (err) { + this._finish_exec(id)(`${err}`); } - - args = args.concat(opts.src); - args = args.concat([add_leading_dash(opts.dest)]); - - webapp_client.exec({ - project_id: this.project_id, - command: "rsync", // don't use "a" option to rsync, since on snapshots results in destroying project access! - args, - timeout: 120, // how long rsync runs on client - err_on_exit: true, - path: ".", - compute_server_id: opts.src_compute_server_id, - filesystem: true, - cb: this._finish_exec(id), - }); }; - copy_paths_between_projects(opts) { - opts = defaults(opts, { - public: false, - src_project_id: required, // id of source project - src: required, // list of relative paths of directories or files in the source project - target_project_id: required, // id of target project - target_path: undefined, // defaults to src_path - overwrite_newer: false, // overwrite newer versions of file at destination (destructive) - delete_missing: false, // delete files in dest that are missing from source (destructive) - backup: false, // make ~ backup files instead of overwriting changed files - }); + // Copy 1 or more paths from one project to another (possibly the same) project. + copyPathBetweenProjects = async (opts: { + src: { project_id: string; path: string | string[] }; + dest: { project_id: string; path: string }; + options?: CopyOptions; + }) => { const id = misc.uuid(); + const files = + typeof opts.src.path == "string" ? [opts.src.path] : opts.src.path; this.set_activity({ id, - status: `Copying ${opts.src.length} ${misc.plural( - opts.src.length, + status: `Copying ${files.length} ${misc.plural( + files.length, "path", )} to a project`, }); - const { src } = opts; - delete opts.src; - const with_slashes = src.map(this._convert_to_displayed_path); - let dest: string | undefined = undefined; - if (opts.target_path != null) { - dest = opts.target_path; - if (!misc.endswith(dest, "/")) { - dest += "/"; - } - } + + await webapp_client.project_client.copyPathBetweenProjects(opts); + + const withSlashes = await this.appendSlashToDirectoryPaths(files, 0); this.log({ event: "file_action", action: "copied", - dest, - files: with_slashes.slice(0, 3), - count: src.length > 3 ? src.length : undefined, - project: opts.target_project_id, + dest: opts.dest.path, + files: withSlashes, + count: files.length, + project: opts.dest.project_id, }); - const f = async (src_path, cb) => { - const opts0 = misc.copy(opts); - delete opts0.cb; - opts0.src_path = src_path; - // we do this for consistent semantics with file copy - opts0.target_path = misc.path_to_file( - opts0.target_path, - misc.path_split(src_path).tail, - ); - opts0.timeout = 90 * 1000; - try { - await webapp_client.project_client.copy_path_between_projects(opts0); - cb(); - } catch (err) { - cb(err); - } - }; - async.mapLimit(src, 3, f, this._finish_exec(id, opts.cb)); - } + + this.set_activity({ id, stop: "" }); + }; public async rename_file(opts: { src: string; @@ -2486,7 +2356,7 @@ export class ProjectActions extends Actions { event: "file_action", action: "renamed", src: opts.src, - dest: opts.dest + ((await this.isdir(opts.dest)) ? "/" : ""), + dest: opts.dest + ((await this.isDir(opts.dest)) ? "/" : ""), compute_server_id, }); } catch (err) { @@ -2496,51 +2366,109 @@ export class ProjectActions extends Actions { } } - // return true if exists and is a directory - private async isdir(path: string): Promise { - if (path == "") return true; // easy special case - try { - await webapp_client.project_client.exec({ - project_id: this.project_id, - command: "test", - args: ["-d", path], - err_on_exit: true, - }); + // note: there is no need to explicitly close or await what is returned by + // fs(...) since it's just a lightweight wrapper object to format appropriate RPC calls. + private _filesystem: { [compute_server_id: number]: FilesystemClient } = {}; + fs = (compute_server_id?: number): FilesystemClient => { + compute_server_id ??= this.get_store()?.get("compute_server_id") ?? 0; + this._filesystem[compute_server_id] ??= webapp_client.conat_client + .conat() + .fs({ project_id: this.project_id, compute_server_id }); + return this._filesystem[compute_server_id]; + }; + + // if available in cache, this returns the filenames in the current directory, + // which is often useful, or null if they are not known. This is sync, so it + // can't query the backend. (Here Files is a map from path names to data about them.) + get_filenames_in_current_dir = (): Files | null => { + const store = this.get_store(); + if (store == undefined) { + return null; + } + const path = store.get("current_path"); + if (path == null) { + return null; + } + return this.getFilesCache(path); + }; + + getCacheId = (compute_server_id?: number) => { + return getCacheId({ + project_id: this.project_id, + compute_server_id: + compute_server_id ?? this.get_store()?.get("compute_server_id") ?? 0, + }); + }; + + private getFilesCache = ( + path: string, + compute_server_id?: number, + ): Files | null => { + return getFiles({ + cacheId: this.getCacheId(compute_server_id), + path, + }); + }; + + // using listings cache, attempt to tell if path is a directory; + // undefined if no data about path in the cache. + isDirViaCache = ( + path: string, + compute_server_id?: number, + ): boolean | undefined => { + if (!path) { return true; - } catch (_) { - return false; } - } + const { head: dir, tail: base } = misc.path_split(path); + const files = this.getFilesCache(dir, compute_server_id); + const data = files?.[base]; + if (data == null) { + return undefined; + } else { + return !!data.isDir; + } + }; + + // return true if exists and is a directory + // error if doesn't exist or can't find out. + // Use isDirViaCache for more of a fast hint. + isDir = async ( + path: string, + compute_server_id?: number, + ): Promise => { + if (path == "") return true; // easy special case + const stats = await this.fs(compute_server_id).stat(path); + return stats.isDirectory(); + }; - public async move_files(opts: { + public async moveFiles({ + src, + dest, + compute_server_id, + }: { src: string[]; dest: string; compute_server_id?: number; }): Promise { - if ( - !(await ensure_project_running( - this.project_id, - "move " + opts.src.join(", "), - )) - ) { - return; - } const id = misc.uuid(); - const status = `Moving ${opts.src.length} ${misc.plural( - opts.src.length, + const status = `Moving ${src.length} ${misc.plural( + src.length, "file", - )} to ${opts.dest}`; + )} to ${dest}`; this.set_activity({ id, status }); let error: any = undefined; try { - const api = await this.api(); - const compute_server_id = this.getComputeServerId(opts.compute_server_id); - await api.move_files(opts.src, opts.dest, compute_server_id); + const fs = this.fs(compute_server_id); + await Promise.all( + src.map(async (path) => + fs.move(path, join(dest, basename(path)), { overwrite: true }), + ), + ); this.log({ event: "file_action", action: "moved", - files: opts.src, - dest: opts.dest + "/" /* target is assumed to be a directory */, + files: src, + dest: dest + "/" /* target is assumed to be a directory */, compute_server_id, }); } catch (err) { @@ -2565,52 +2493,36 @@ export class ProjectActions extends Actions { return false; } - public async delete_files(opts: { + deleteFiles = async ({ + paths, + compute_server_id, + }: { paths: string[]; compute_server_id?: number; - }): Promise { - let mesg; - opts = defaults(opts, { - paths: required, - compute_server_id: this.get_store()?.get("compute_server_id") ?? 0, - }); - if (opts.paths.length === 0) { - return; - } - - if ( - this.checkForSandboxError( - "Deleting files is not allowed in a sandbox project. Create your own private project in the Projects tab in the upper left.", - ) - ) { - return; - } - - if ( - !(await ensure_project_running( - this.project_id, - `delete ${opts.paths.join(", ")}`, - )) - ) { + }): Promise => { + if (paths.length == 0) { + // nothing to do return; } - const id = misc.uuid(); - if (isEqual(opts.paths, [".trash"])) { + let mesg; + if (isEqual(paths, [".trash"])) { mesg = "the trash"; - } else if (opts.paths.length === 1) { - mesg = `${opts.paths[0]}`; + } else if (paths.length === 1) { + mesg = `${paths[0]}`; } else { - mesg = `${opts.paths.length} files`; + mesg = `${paths.length} files`; } this.set_activity({ id, status: `Deleting ${mesg}...` }); + try { - await delete_files(this.project_id, opts.paths, opts.compute_server_id); + const fs = this.fs(compute_server_id); + await fs.rm(paths, { force: true, recursive: true }); this.log({ event: "file_action", action: "deleted", - files: opts.paths, - compute_server_id: opts.compute_server_id, + files: paths, + compute_server_id, }); this.set_activity({ id, @@ -2624,7 +2536,41 @@ export class ProjectActions extends Actions { stop: "", }); } - } + }; + + // remove all files in the given path (or subtree of that path) + // for which filter(filename) returns true. + // - path should be relative to HOME + // - filname will also be relative to HOME and will end in a slash if it is a directory + // Returns the deleted paths. + deleteMatchingFiles = async ({ + path, + filter, + recursive, + compute_server_id, + }: { + path: string; + filter: (path: string) => boolean; + recursive?: boolean; + compute_server_id?: number; + }): Promise => { + const fs = this.fs(compute_server_id); + const options: string[] = ["-H", "-I"]; + if (!recursive) { + options.push("-d", "1"); + } + const { stdout } = await fs.fd(path, { options }); + const paths = Buffer.from(stdout) + .toString() + .split("\n") + .slice(0, -1) + .map((p) => join(path, p)) + .filter(filter); + if (paths.length > 0) { + await this.deleteFiles({ paths, compute_server_id }); + } + return paths; + }; download_file = async ({ path, @@ -2747,7 +2693,6 @@ export class ProjectActions extends Actions { }); return; } - this.fetch_directory_listing({ path: p, compute_server_id }); if (switch_over) { this.open_directory(p); } @@ -2848,8 +2793,6 @@ export class ProjectActions extends Actions { explicit: true, compute_server_id, }); - } else { - this.fetch_directory_listing(); } }; @@ -2875,7 +2818,6 @@ export class ProjectActions extends Actions { alert: true, }); } finally { - this.fetch_directory_listing(); this.set_activity({ id, stop: "" }); this.setState({ downloading_file: false }); this.set_active_tab("files", { update_file_listing: false }); @@ -2917,13 +2859,13 @@ export class ProjectActions extends Actions { const id = client_db.sha1(project_id, path); const projects_store = redux.getStore("projects"); - const dflt_compute_img = await redux + const defaultComputeImage = await redux .getStore("customize") .getDefaultComputeImage(); const compute_image: string = projects_store.getIn(["project_map", project_id, "compute_image"]) ?? - dflt_compute_img; + defaultComputeImage; const table = this.redux.getProjectTable(project_id, "public_paths"); let obj: undefined | Map = table._table.get(id); @@ -2986,7 +2928,7 @@ export class ProjectActions extends Actions { // can't just change always since we frequently update last_edited to get the share to get copied over. this.log({ event: "public_path", - path: path + ((await this.isdir(path)) ? "/" : ""), + path: path + ((await this.isDir(path)) ? "/" : ""), disabled: !!obj.get("disabled"), unlisted: !!obj.get("unlisted"), authenticated: !!obj.get("authenticated"), @@ -3025,6 +2967,7 @@ export class ProjectActions extends Actions { redux .getActions("account") ?.set_other_settings("find_subdirectories", subdirectories); + this.search(); }; toggle_search_checkbox_case_sensitive = () => { @@ -3037,6 +2980,7 @@ export class ProjectActions extends Actions { redux .getActions("account") ?.set_other_settings("find_case_sensitive", case_sensitive); + this.search(); }; toggle_search_checkbox_hidden_files() { @@ -3049,6 +2993,7 @@ export class ProjectActions extends Actions { redux .getActions("account") ?.set_other_settings("find_hidden_files", hidden_files); + this.search(); } toggle_search_checkbox_git_grep() { @@ -3059,169 +3004,20 @@ export class ProjectActions extends Actions { const git_grep = !store.get("git_grep"); this.setState({ git_grep }); redux.getActions("account")?.set_other_settings("find_git_grep", git_grep); + this.search(); } - process_search_results(err, output, max_results, max_output, cmd) { + toggle_search_checkbox_regexp() { const store = this.get_store(); if (store == undefined) { return; } - if (err) { - err = misc.to_user_string(err); - } - if ((err && output == null) || (output != null && output.stdout == null)) { - this.setState({ search_error: err }); - return; - } - - const results = output.stdout.split("\n"); - const too_many_results = !!( - output.stdout.length >= max_output || - results.length > max_results || - err - ); - let num_results = 0; - const search_results: {}[] = []; - for (const line of results) { - if (line.trim() === "") { - continue; - } - let i = line.indexOf(":"); - num_results += 1; - if (i !== -1) { - // all valid lines have a ':', the last line may have been truncated too early - let filename = line.slice(0, i); - if (filename.slice(0, 2) === "./") { - filename = filename.slice(2); - } - let context = line.slice(i + 1); - // strip codes in worksheet output - if (context.length > 0 && context[0] === MARKERS.output) { - i = context.slice(1).indexOf(MARKERS.output); - context = context.slice(i + 2, context.length - 1); - } - - const m = /^(\d+):/.exec(context); - let line_number: number | undefined; - if (m != null) { - try { - line_number = parseInt(m[1]); - } catch (e) {} - } - - search_results.push({ - filename, - description: context, - line_number, - filter: `${filename.toLowerCase()} ${context.toLowerCase()}`, - }); - } - if (num_results >= max_results) { - break; - } - } - - if (store.get("command") === cmd) { - // only update the state if the results are from the most recent command - this.setState({ - too_many_results, - search_results, - }); - } + const regexp = !store.get("regexp"); + this.setState({ regexp }); + redux.getActions("account")?.set_other_settings("regexp", regexp); + this.search(); } - search = () => { - let cmd, ins; - const store = this.get_store(); - if (store == undefined) { - return; - } - - const query = store.get("user_input").trim().replace(/"/g, '\\"'); - if (query === "") { - return; - } - const search_query = `"${query}"`; - this.setState({ - search_results: undefined, - search_error: undefined, - most_recent_search: query, - most_recent_path: store.get("current_path"), - too_many_results: false, - }); - const path = store.get("current_path"); - - track("search", { - project_id: this.project_id, - path, - query, - neural_search: store.get("neural_search"), - subdirectories: store.get("subdirectories"), - hidden_files: store.get("hidden_files"), - git_grep: store.get("git_grep"), - }); - - // generate the grep command for the given query with the given flags - if (store.get("case_sensitive")) { - ins = ""; - } else { - ins = " -i "; - } - - if (store.get("git_grep")) { - let max_depth; - if (store.get("subdirectories")) { - max_depth = ""; - } else { - max_depth = "--max-depth=0"; - } - // The || true is so that if git rev-parse has exit code 0, - // but "git grep" finds nothing (hence has exit code 1), we don't - // fall back to normal git (the other side of the ||). See - // https://github.com/sagemathinc/cocalc/issues/4276 - cmd = `git rev-parse --is-inside-work-tree && (git grep -n -I -H ${ins} ${max_depth} ${search_query} || true) || `; - } else { - cmd = ""; - } - if (store.get("subdirectories")) { - if (store.get("hidden_files")) { - cmd += `rgrep -n -I -H --exclude-dir=.smc --exclude-dir=.snapshots ${ins} ${search_query} -- *`; - } else { - cmd += `rgrep -n -I -H --exclude-dir='.*' --exclude='.*' ${ins} ${search_query} -- *`; - } - } else { - if (store.get("hidden_files")) { - cmd += `grep -n -I -H ${ins} ${search_query} -- .* *`; - } else { - cmd += `grep -n -I -H ${ins} ${search_query} -- *`; - } - } - - cmd += ` | grep -v ${MARKERS.cell}`; - const max_results = 1000; - const max_output = 110 * max_results; // just in case - - this.setState({ - command: cmd, - }); - - const compute_server_id = this.getComputeServerId(); - webapp_client.exec({ - project_id: this.project_id, - command: cmd + " | cut -c 1-256", // truncate horizontal line length (imagine a binary file that is one very long line) - timeout: 20, // how long grep runs on client - max_output, - bash: true, - err_on_exit: true, - compute_server_id, - filesystem: true, - path: store.get("current_path"), - cb: (err, output) => { - this.process_search_results(err, output, max_results, max_output, cmd); - }, - }); - }; - set_file_listing_scroll(scroll_top) { this.setState({ file_listing_scroll_top: scroll_top }); } @@ -3236,17 +3032,16 @@ export class ProjectActions extends Actions { // log // settings // search - async load_target( + load_target = async ( target, foreground = true, ignore_kiosk = false, change_history = true, fragmentId?: FragmentId, - ): Promise { + ): Promise => { const segments = target.split("/"); const full_path = segments.slice(1).join("/"); const parent_path = segments.slice(1, segments.length - 1).join("/"); - const last = segments.slice(-1).join(); const main_segment = segments[0] as FixedTab | "home"; switch (main_segment) { case "active": @@ -3265,28 +3060,10 @@ export class ProjectActions extends Actions { if (store == null) { return; // project closed already } - // We check whether the path is a directory or not, first by checking if - // we have a recent directory listing in our cache, and if not, by calling - // isdir, which is a single exec. - let isdir; - let { item, err } = store.get_item_in_path(last, parent_path); - if (item == null || err) { - try { - isdir = await webapp_client.project_client.isdir({ - project_id: this.project_id, - path: normalize(full_path), - }); - } catch (err) { - // TODO: e.g., project is not running? - // I've seen this, e.g., when trying to open a file when not running, and it just - // gets retried and works. - console.log(`Error opening '${target}' -- ${err}`); - return; - } - } else { - isdir = item.get("isdir"); - } - if (isdir) { + + // We check whether the path is a directory or not: + const isDir = await this.isDir(full_path); + if (isDir) { this.open_directory(full_path, change_history); } else { this.open_file({ @@ -3344,7 +3121,7 @@ export class ProjectActions extends Actions { misc.unreachable(main_segment); console.warn(`project/load_target: don't know segment ${main_segment}`); } - } + }; set_compute_image = async (compute_image: string): Promise => { const projects_store = this.redux.getStore("projects"); @@ -3405,6 +3182,13 @@ export class ProjectActions extends Actions { async show(): Promise { const store = this.get_store(); if (store == undefined) return; // project closed + try { + await this.redux + .getActions("projects") + .updateProjectState(this.project_id); + } catch { + // this can fail, e.g., if user is not a collab on the project, server down, etc. + } const a = store.get("active_project_tab"); if (!misc.startswith(a, "editor-")) return; this.show_file(misc.tab_to_path(a)); @@ -3553,21 +3337,19 @@ export class ProjectActions extends Actions { const store = this.get_store(); if (store == null) return; this.setRecentlyDeleted(path, 0); - (async () => { - try { - const o = await webapp_client.conat_client.openFiles(this.project_id); - o.setNotDeleted(path); - } catch (err) { - console.log("WARNING: issue undeleting file", err); - } - })(); }; private initProjectStatus = async () => { - this.projectStatusSub = await getProjectStatus({ - project_id: this.project_id, - compute_server_id: 0, - }); + try { + this.projectStatusSub = await getProjectStatus({ + project_id: this.project_id, + compute_server_id: 0, + }); + } catch (err) { + // happens if you open a project you are not a collab on + console.warn(`unable to subscribe to project status updates: `, err); + return; + } for await (const mesg of this.projectStatusSub) { const status = mesg.data; this.setState({ status }); @@ -3717,4 +3499,37 @@ export class ProjectActions extends Actions { project_id: this.project_id, }); }; + + private searchId = 0; + search = async () => { + const store = this.get_store(); + if (!store) { + return; + } + const searchId = ++this.searchId; + const setState = (x) => { + if (this.searchId != searchId) { + // there's a newer search + return; + } + this.setState(x); + }; + try { + await search({ + setState, + fs: this.fs(), + query: store.get("user_input").trim(), + path: store.get("current_path"), + options: { + case_sensitive: store.get("case_sensitive"), + git_grep: store.get("git_grep"), + subdirectories: store.get("subdirectories"), + hidden_files: store.get("hidden_files"), + regexp: store.get("regexp"), + }, + }); + } catch (err) { + setState({ search_error: `${err}` }); + } + }; } diff --git a/src/packages/frontend/project_store.ts b/src/packages/frontend/project_store.ts index 3b06e820c6e..f7b2a392ffb 100644 --- a/src/packages/frontend/project_store.ts +++ b/src/packages/frontend/project_store.ts @@ -16,7 +16,6 @@ if (typeof window !== "undefined" && window !== null) { } import * as immutable from "immutable"; - import { AppRedux, project_redux_name, @@ -24,8 +23,9 @@ import { Store, Table, TypedMap, + useTypedRedux, } from "@cocalc/frontend/app-framework"; -import { Listings, listings } from "@cocalc/frontend/conat/listings"; +import { useMemo } from "react"; import { fileURL } from "@cocalc/frontend/lib/cocalc-urls"; import { get_local_storage } from "@cocalc/frontend/misc"; import { QueryParams } from "@cocalc/frontend/misc/query-params"; @@ -35,15 +35,14 @@ import { FILE_ACTIONS, ProjectActions, QUERIES, + type FileAction, } from "@cocalc/frontend/project_actions"; import { Available as AvailableFeatures, isMainConfiguration, ProjectConfiguration, } from "@cocalc/frontend/project_configuration"; -import * as misc from "@cocalc/util/misc"; -import { compute_file_masks } from "./project/explorer/compute-file-masks"; -import { DirectoryListing } from "./project/explorer/types"; +import { containing_public_path, deep_copy } from "@cocalc/util/misc"; import { FixedTab } from "./project/page/file-tab"; import { FlyoutActiveMode, @@ -57,8 +56,9 @@ import { FLYOUT_LOG_FILTER_DEFAULT, FlyoutLogFilter, } from "./project/page/flyouts/utils"; - -export { FILE_ACTIONS as file_actions, ProjectActions }; +import { type PublicPath } from "@cocalc/util/db-schema/public-paths"; +import { DirectoryListing } from "@cocalc/frontend/project/explorer/types"; +export { FILE_ACTIONS as file_actions, type FileAction, ProjectActions }; export type ModalInfo = TypedMap<{ title: string | React.JSX.Element; @@ -74,13 +74,10 @@ export interface ProjectStoreState { open_files: immutable.Map>; open_files_order: immutable.List; just_closed_files: immutable.List; - public_paths?: immutable.Map>; + public_paths?: immutable.Map>; - // directory_listings is a map from compute_server_id to {path:[listing for that path on the compute server]} - directory_listings: immutable.Map; show_upload: boolean; create_file_alert: boolean; - displayed_listing?: any; // computed(object), configuration?: ProjectConfiguration; configuration_loading: boolean; // for UI feedback available_features?: TypedMap; @@ -98,14 +95,18 @@ export interface ProjectStoreState { // Project Files activity: any; // immutable, active_file_sort: TypedMap<{ column_name: string; is_descending: boolean }>; - page_number: number; - file_action?: string; // undefineds is meaningfully none here + file_action?: FileAction; file_search?: string; show_hidden?: boolean; show_masked?: boolean; error?: string; checked_files: immutable.Set; + selected_file_index?: number; // Index on file listing to highlight starting at 0. undefined means none highlighted + // the number of visible files in the listing for the current directory; this is needed + // for cursor based navigation by the search bar. This is the number after hiding hidden files and search filtering. + numDisplayedFiles?: number; + new_name?: string; most_recent_file_click?: string; show_library: boolean; @@ -138,8 +139,11 @@ export interface ProjectStoreState { command?: string; most_recent_search?: string; most_recent_path: string; + + // TODO: these shouldn't be top level! subdirectories?: boolean; case_sensitive?: boolean; + regexp?: boolean; hidden_files?: boolean; git_grep: boolean; info_visible?: boolean; @@ -147,7 +151,6 @@ export interface ProjectStoreState { // Project Settings get_public_path_id?: (path: string) => any; - stripped_public_paths: any; //computed(immutable.List) // Project Info show_project_info_explanation?: boolean; @@ -181,7 +184,6 @@ export interface ProjectStoreState { export class ProjectStore extends Store { public project_id: string; private previous_runstate: string | undefined; - private listings: { [compute_server_id: number]: Listings } = {}; public readonly computeServerIdLocalStorageKey: string; // Function to call to initialize one of the tables in this store. @@ -226,10 +228,6 @@ export class ProjectStore extends Store { if (projects_store !== undefined) { projects_store.removeListener("change", this._projects_store_change); } - for (const id in this.listings) { - this.listings[id].close(); - delete this.listings[id]; - } // close any open file tabs, properly cleaning up editor state: const open = this.get("open_files")?.toJS(); if (open != null) { @@ -296,10 +294,8 @@ export class ProjectStore extends Store { open_files: immutable.Map>({}), open_files_order: immutable.List([]), just_closed_files: immutable.List([]), - directory_listings: immutable.Map(), // immutable, show_upload: false, create_file_alert: false, - displayed_listing: undefined, // computed(object), show_masked: true, configuration: undefined, configuration_loading: false, // for UI feedback @@ -316,7 +312,6 @@ export class ProjectStore extends Store { // Project Files activity: undefined, - page_number: 0, checked_files: immutable.Set(), show_library: false, file_listing_scroll_top: undefined, @@ -336,12 +331,11 @@ export class ProjectStore extends Store { subdirectories: other_settings?.get("find_subdirectories"), case_sensitive: other_settings?.get("find_case_sensitive"), hidden_files: other_settings?.get("find_hidden_files"), + regexp: other_settings?.get("regexp"), most_recent_path: "", // Project Settings - stripped_public_paths: this.selectors.stripped_public_paths.fn, - other_settings: undefined, compute_server_id, @@ -370,177 +364,6 @@ export class ProjectStore extends Store { }; }, }, - - // cached pre-processed file listing, which should always be up to date when - // called, and properly depends on dependencies. - displayed_listing: { - dependencies: [ - "active_file_sort", - "current_path", - "directory_listings", - "stripped_public_paths", - "file_search", - "other_settings", - "show_hidden", - "show_masked", - "compute_server_id", - ] as const, - fn: () => { - const search_escape_char = "/"; - const listingStored = this.getIn([ - "directory_listings", - this.get("compute_server_id"), - this.get("current_path"), - ]); - if (typeof listingStored === "string") { - if ( - listingStored.indexOf("ECONNREFUSED") !== -1 || - listingStored.indexOf("ENOTFOUND") !== -1 - ) { - return { error: "no_instance" }; // the host VM is down - } else if (listingStored.indexOf("o such path") !== -1) { - return { error: "no_dir" }; - } else if (listingStored.indexOf("ot a directory") !== -1) { - return { error: "not_a_dir" }; - } else if (listingStored.indexOf("not running") !== -1) { - // yes, no underscore. - return { error: "not_running" }; - } else { - return { error: listingStored }; - } - } - if (listingStored == null) { - return {}; - } - try { - if (listingStored?.errno) { - return { error: misc.to_json(listingStored) }; - } - } catch (err) { - return { - error: "Error getting directory listing - please try again.", - }; - } - - if (listingStored?.toJS == null) { - return { - error: "Unable to get directory listing - please try again.", - }; - } - - // We can proceed and get the listing as a JS object. - let listing: DirectoryListing = listingStored.toJS(); - - if (this.get("other_settings").get("mask_files")) { - compute_file_masks(listing); - } - - if (this.get("current_path") === ".snapshots") { - compute_snapshot_display_names(listing); - } - - const search = this.get("file_search"); - if (search && search[0] !== search_escape_char) { - listing = _matched_files(search.toLowerCase(), listing); - } - - const sorter = (() => { - switch (this.get("active_file_sort").get("column_name")) { - case "name": - return _sort_on_string_field("name"); - case "time": - return _sort_on_numerical_field("mtime", -1); - case "size": - return _sort_on_numerical_field("size"); - case "type": - return (a, b) => { - if (a.isdir && !b.isdir) { - return -1; - } else if (b.isdir && !a.isdir) { - return 1; - } else { - return misc.cmp_array( - a.name.split(".").reverse(), - b.name.split(".").reverse(), - ); - } - }; - } - })(); - - listing.sort(sorter); - - if (this.get("active_file_sort").get("is_descending")) { - listing.reverse(); - } - - if (!this.get("show_hidden")) { - listing = (() => { - const result: DirectoryListing = []; - for (const l of listing) { - if (!l.name.startsWith(".")) { - result.push(l); - } - } - return result; - })(); - } - - if (!this.get("show_masked", true)) { - // if we do not gray out files (and hence haven't computed the file mask yet) - // we do it now! - if (!this.get("other_settings").get("mask_files")) { - compute_file_masks(listing); - } - - const filtered: DirectoryListing = []; - for (const f of listing) { - if (!f.mask) filtered.push(f); - } - listing = filtered; - } - - const file_map = {}; - for (const v of listing) { - file_map[v.name] = v; - } - - const data = { - listing, - public: {}, - path: this.get("current_path"), - file_map, - }; - - mutate_data_to_compute_public_files( - data, - this.get("stripped_public_paths"), - this.get("current_path"), - ); - - return data; - }, - }, - - stripped_public_paths: { - dependencies: ["public_paths"] as const, - fn: () => { - const public_paths = this.get("public_paths"); - if (public_paths != null) { - return immutable.fromJS( - (() => { - const result: any[] = []; - const object = public_paths.toJS(); - for (const id in object) { - const x = object[id]; - result.push(misc.copy_without(x, ["id", "project_id"])); - } - return result; - })(), - ); - } - }, - }, }; // Returns the cursor positions for the given project_id/path, if that @@ -565,20 +388,6 @@ export class ProjectStore extends Store { return this.getIn(["open_files", path]) != null; }; - get_item_in_path = (name, path) => { - const listing = this.get("directory_listings").get(path); - if (typeof listing === "string") { - // must be an error - return { err: listing }; - } - return { - item: - listing != null - ? listing.find((val) => val.get("name") === name) - : undefined, - }; - }; - fileURL = (path, compute_server_id?: number) => { return fileURL({ project_id: this.project_id, @@ -605,143 +414,43 @@ export class ProjectStore extends Store { // note that component is NOT an immutable.js object: return this.getIn(["open_files", path, "component"])?.Editor != null; } - - public get_listings(compute_server_id: number | null = null): Listings { - const computeServerId = compute_server_id ?? this.get("compute_server_id"); - if (this.listings[computeServerId] == null) { - const listingsTable = listings(this.project_id, computeServerId); - this.listings[computeServerId] = listingsTable; - listingsTable.watch(this.get("current_path") ?? "", true); - listingsTable.on("change", async (paths) => { - let directory_listings_for_server = - this.getIn(["directory_listings", computeServerId]) ?? - immutable.Map(); - - const missing: string[] = []; - for (const path of paths) { - if (listingsTable.getMissing(path)) { - missing.push(path); - } - const files = await listingsTable.getForStore(path); - directory_listings_for_server = directory_listings_for_server.set( - path, - files, - ); - } - const f = () => { - const actions = redux.getProjectActions(this.project_id); - const directory_listings = this.get("directory_listings").set( - computeServerId, - directory_listings_for_server, - ); - actions.setState({ directory_listings }); - }; - f(); - - if (missing.length > 0) { - for (const path of missing) { - try { - const files = immutable.fromJS( - await listingsTable.getListingDirectly(path), - ); - directory_listings_for_server = directory_listings_for_server.set( - path, - files, - ); - } catch { - // happens if e.g., the project is not running - continue; - } - } - f(); - } - }); - } - if (this.listings[computeServerId] == null) { - throw Error("bug"); - } - return this.listings[computeServerId]; - } } -function _matched_files(search: string, listing: DirectoryListing) { - if (listing == null) { - return []; +// Returns set of paths that are public in the given +// listing, because they are in a public folder or are themselves public. +// This is used entirely to put an extra "public" label in the row of the file, +// when displaying it in a listing. +export function getPublicFiles( + listing: DirectoryListing, + public_paths: PublicPath[], + current_path: string, +): Set { + if ((public_paths?.length ?? 0) == 0) { + return new Set(); } - const words = misc.search_split(search); - const v: DirectoryListing = []; - for (const x of listing) { - const name = (x.display_name ?? x.name ?? "").toLowerCase(); - if ( - misc.search_match(name, words) || - (x.isdir && misc.search_match(name + "/", words)) - ) { - v.push(x); - } + const paths = public_paths + .filter(({ disabled }) => !disabled) + .map(({ path }) => path); + + if (paths.length == 0) { + return new Set(); } - return v; -} -function compute_snapshot_display_names(listing): void { - for (const item of listing) { - const tm = misc.parse_bup_timestamp(item.name); - item.display_name = `${tm}`; - item.mtime = tm.valueOf() / 1000; + const head = current_path ? current_path + "/" : ""; + if (containing_public_path(current_path, paths)) { + // fast special case: *every* file is public + return new Set(listing.map(({ name }) => name)); } -} -// Mutates data to include info on public paths. -export function mutate_data_to_compute_public_files( - data, - public_paths, - current_path, -) { - const { listing } = data; - const pub = data.public; - if (public_paths != null && public_paths.size > 0) { - const head = current_path ? current_path + "/" : ""; - const paths: string[] = []; - const public_path_data = {}; - for (const x of public_paths.toJS()) { - if (x.disabled) { - // Do not include disabled paths. Otherwise, it causes this confusing bug: - // https://github.com/sagemathinc/cocalc/issues/6159 - continue; - } - public_path_data[x.path] = x; - paths.push(x.path); - } - for (const x of listing) { - const full = head + x.name; - const p = misc.containing_public_path(full, paths); - if (p != null) { - x.public = public_path_data[p]; - x.is_public = !x.public.disabled; - pub[x.name] = public_path_data[p]; - } + // maybe some files are public? + const X = new Set(); + for (const file of listing) { + const full = head + file.name; + if (containing_public_path(full, paths) != null) { + X.add(file.name); } } -} - -function _sort_on_string_field(field) { - return function (a, b) { - return misc.cmp( - a[field] !== undefined ? a[field].toLowerCase() : "", - b[field] !== undefined ? b[field].toLowerCase() : "", - ); - }; -} - -function _sort_on_numerical_field(field, factor = 1) { - return (a, b) => { - const c = misc.cmp( - (a[field] != null ? a[field] : -1) * factor, - (b[field] != null ? b[field] : -1) * factor, - ); - if (c) return c; - // break ties using the name, so well defined. - return misc.cmp(a.name, b.name) * factor; - }; + return X; } export function init(project_id: string, redux: AppRedux): ProjectStore { @@ -765,7 +474,7 @@ export function init(project_id: string, redux: AppRedux): ProjectStore { actions.project_id = project_id; // so actions can assume this is available on the object store._init(); - const queries = misc.deep_copy(QUERIES); + const queries = deep_copy(QUERIES); const create_table = function (table_name, q) { //console.log("create_table", table_name) @@ -822,3 +531,11 @@ export function init(project_id: string, redux: AppRedux): ProjectStore { return store; } + +export function useStrippedPublicPaths(project_id: string): PublicPath[] { + const public_paths = useTypedRedux({ project_id }, "public_paths"); + return useMemo(() => { + const rows = public_paths?.valueSeq()?.toJS() ?? []; + return rows as unknown as PublicPath[]; + }, [public_paths]); +} diff --git a/src/packages/frontend/projects/actions.ts b/src/packages/frontend/projects/actions.ts index 76aa6bb3c06..ba281d44653 100644 --- a/src/packages/frontend/projects/actions.ts +++ b/src/packages/frontend/projects/actions.ts @@ -5,7 +5,6 @@ import { Set } from "immutable"; import { isEqual } from "lodash"; - import { alert_message } from "@cocalc/frontend/alerts"; import { Actions, redux } from "@cocalc/frontend/app-framework"; import { set_window_title } from "@cocalc/frontend/browser"; @@ -32,6 +31,7 @@ import { SiteLicenseQuota } from "@cocalc/util/types/site-licenses"; import { Upgrades } from "@cocalc/util/upgrades/types"; import { ProjectsState, store } from "./store"; import { load_all_projects, switch_to_project } from "./table"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; import type { CourseInfo, @@ -417,7 +417,7 @@ export class ProjectsActions extends Actions { } // Open the given project - public async open_project(opts: { + open_project = async (opts: { project_id: string; // id of the project to open target?: string; // The file path to open fragmentId?: FragmentId; // if given, an uri fragment in the editor that is opened. @@ -425,7 +425,7 @@ export class ProjectsActions extends Actions { ignore_kiosk?: boolean; // Ignore ?fullscreen=kiosk change_history?: boolean; // (default: true) Whether or not to alter browser history restore_session?: boolean; // (default: true) Opens up previously closed editor tabs - }) { + }) => { opts = defaults(opts, { project_id: undefined, target: undefined, @@ -464,7 +464,6 @@ export class ProjectsActions extends Actions { if (relation == null || ["public", "admin"].includes(relation)) { this.fetch_public_project_title(opts.project_id); } - project_actions.fetch_directory_listing(); if (opts.switch_to) { redux .getActions("page") @@ -487,7 +486,7 @@ export class ProjectsActions extends Actions { } // initialize project project_actions.init(); - } + }; // tab at old_index taken out and then inserted into the resulting array's new index public move_project_tab({ @@ -927,102 +926,86 @@ export class ProjectsActions extends Actions { } // return true, if it actually started the project - start_project = async ( - project_id: string, - options: { disablePayAsYouGo?: boolean } = {}, - ): Promise => { - if ( - !(await allow_project_to_run(project_id)) || - !store.getIn(["project_map", project_id]) - ) { - return false; - } - if (!options.disablePayAsYouGo) { - const quota = store - .getIn([ - "project_map", - project_id, - "pay_as_you_go_quotas", - webapp_client.account_id ?? "", - ]) - ?.toJS(); - if (quota?.enabled) { - return await startProjectPayg({ project_id, quota }); + start_project = reuseInFlight( + async ( + project_id: string, + options: { disablePayAsYouGo?: boolean } = {}, + ): Promise => { + if ( + !(await allow_project_to_run(project_id)) || + !store.getIn(["project_map", project_id]) + ) { + return false; + } + if (!options.disablePayAsYouGo) { + const quota = store + .getIn([ + "project_map", + project_id, + "pay_as_you_go_quotas", + webapp_client.account_id ?? "", + ]) + ?.toJS(); + if (quota?.enabled) { + return await startProjectPayg({ project_id, quota }); + } } - } - const t0 = webapp_client.server_time().getTime(); - // make an action request: - this.project_log(project_id, { - event: "project_start_requested", - }); - await this.projects_table_set({ - project_id, - action_request: { - action: "start", - time: webapp_client.server_time(), - }, - }); + const t0 = webapp_client.server_time().getTime(); + // make an action request: + this.project_log(project_id, { + event: "project_start_requested", + }); + const runner = webapp_client.conat_client.projectRunner(project_id); + await runner.start({ project_id }); - // Wait until it is running - await store.async_wait({ - timeout: 120, - until(store) { - return store.get_state(project_id) == "running"; - }, - }); - this.project_log(project_id, { - event: "project_started", - duration_ms: webapp_client.server_time().getTime() - t0, - ...store.classify_project(project_id), - }); + this.project_log(project_id, { + event: "project_started", + duration_ms: webapp_client.server_time().getTime() - t0, + ...store.classify_project(project_id), + }); - return true; - }; + return true; + }, + ); // returns true, if it actually stopped the project - stop_project = async (project_id: string): Promise => { + stop_project = reuseInFlight(async (project_id: string): Promise => { const t0 = webapp_client.server_time().getTime(); this.project_log(project_id, { event: "project_stop_requested", }); - await this.projects_table_set({ - project_id, - action_request: { action: "stop", time: webapp_client.server_time() }, - }); - - // Wait until it is no longer running or stopping. We don't - // wait for "opened" because something or somebody else could - // have started the project and we missed that, and don't - // want to get stuck. - await store.async_wait({ - timeout: 60, - until(store) { - const state = store.get_state(project_id); - return state != "running" && state != "stopping"; - }, - }); + const runner = webapp_client.conat_client.projectRunner(project_id); + await runner.stop({ project_id }); this.project_log(project_id, { event: "project_stopped", duration_ms: webapp_client.server_time().getTime() - t0, ...store.classify_project(project_id), }); return true; - }; + }); - restart_project = async (project_id: string, options?): Promise => { - if (!(await allow_project_to_run(project_id))) { - return; - } - this.project_log(project_id, { - event: "project_restart_requested", - }); - const state = store.get_state(project_id); - if (state == "running") { - await this.stop_project(project_id); - } - await this.start_project(project_id, options); - }; + restart_project = reuseInFlight( + async (project_id: string, options?): Promise => { + if (!(await allow_project_to_run(project_id))) { + return; + } + this.project_log(project_id, { + event: "project_restart_requested", + }); + const state = store.get_state(project_id); + if (state == "running") { + await this.stop_project(project_id); + } + await this.start_project(project_id, options); + }, + ); + + updateProjectState = reuseInFlight(async (project_id: string) => { + const runner = webapp_client.conat_client.projectRunner(project_id); + // this causes an update + return await runner.status({ project_id }); + }); // Explcitly set whether or not project is hidden for the given account // (hide=true means hidden) diff --git a/src/packages/frontend/projects/project-row.tsx b/src/packages/frontend/projects/project-row.tsx index 5631972a8d0..08c03de7766 100644 --- a/src/packages/frontend/projects/project-row.tsx +++ b/src/packages/frontend/projects/project-row.tsx @@ -34,7 +34,7 @@ import track from "@cocalc/frontend/user-tracking"; import { DEFAULT_COMPUTE_IMAGE } from "@cocalc/util/db-schema"; import { KUCALC_COCALC_COM } from "@cocalc/util/db-schema/site-defaults"; import { COLORS } from "@cocalc/util/theme"; -import { Avatar } from "antd"; +import { Avatar, Button, Tooltip } from "antd"; import { CSSProperties, useEffect } from "react"; import { ProjectUsers } from "./project-users"; @@ -215,7 +215,7 @@ export const ProjectRow: React.FC = ({ project_id, index }: Props) => { = ({ project_id, index }: Props) => { /> )} + + {!is_anonymous && ( + + + + )} + ); diff --git a/src/packages/frontend/projects/store.ts b/src/packages/frontend/projects/store.ts index 066befc4a11..1ed42338fae 100644 --- a/src/packages/frontend/projects/store.ts +++ b/src/packages/frontend/projects/store.ts @@ -299,17 +299,15 @@ export class ProjectsStore extends Store { return this.get("open_projects").includes(project_id); } - public wait_until_project_is_open( + waitUntilProjectIsOpen = async ( project_id: string, timeout: number, // timeout in seconds (NOT milliseconds!) - cb: (err?) => void, - ): void { - this.wait({ + ) => { + await this.async_wait({ until: () => this.is_project_open(project_id), timeout, - cb, }); - } + }; public wait_until_project_exists( project_id: string, @@ -539,7 +537,7 @@ export class ProjectsStore extends Store { if (quotas == null) { return undefined; } - const kind = quotas.member_host ?? true ? "member" : "free"; + const kind = (quotas.member_host ?? true) ? "member" : "free"; // if any quota regarding cpu or memory is upgraded, we treat it better than purely free projects const upgraded = (quotas.memory != null && quotas.memory > DEFAULT_QUOTAS.memory) || diff --git a/src/packages/frontend/share/config.tsx b/src/packages/frontend/share/config.tsx index ab88b1c97c1..48b0a407a8a 100644 --- a/src/packages/frontend/share/config.tsx +++ b/src/packages/frontend/share/config.tsx @@ -33,8 +33,7 @@ import { Row, Space, } from "antd"; -import { useEffect, useState } from "react"; - +import { useEffect, useMemo, useState } from "react"; import { CSS, redux, useTypedRedux } from "@cocalc/frontend/app-framework"; import { A, @@ -60,46 +59,19 @@ import { COLORS } from "@cocalc/util/theme"; import { ConfigureName } from "./configure-name"; import { License } from "./license"; import { publicShareUrl, shareServerUrl } from "./util"; +import { containing_public_path } from "@cocalc/util/misc"; +import { type PublicPath } from "@cocalc/util/db-schema/public-paths"; +import { type ProjectActions } from "@cocalc/frontend/project_store"; // https://ant.design/components/grid/ const GUTTER: [number, number] = [20, 30]; -interface PublicInfo { - created: Date; - description: string; - disabled: boolean; - last_edited: Date; - path: string; - unlisted: boolean; - authenticated?: boolean; - license?: string; - name?: string; - site_license_id?: string; - redirect?: string; - jupyter_api?: boolean; -} - interface Props { project_id: string; path: string; - size: number; - mtime: number; - isdir?: boolean; - is_public?: boolean; - public?: PublicInfo; close: (event: any) => void; - action_key: (event: any) => void; - site_license_id?: string; - set_public_path: (options: { - description?: string; - unlisted?: boolean; - license?: string; - disabled?: boolean; - authenticated?: boolean; - site_license_id?: string | null; - redirect?: string; - jupyter_api?: boolean; - }) => void; + onKeyUp?: (event: any) => void; + actions: ProjectActions; has_network_access?: boolean; compute_server_id?: number; } @@ -121,31 +93,55 @@ function SC({ children }) { return {children}; } -export default function Configure(props: Props) { - const student = useStudentProjectFunctionality(props.project_id); +export default function Configure({ + project_id, + path, + close, + onKeyUp, + actions, + has_network_access, + compute_server_id, +}: Props) { + const publicPaths = useTypedRedux({ project_id }, "public_paths"); + const publicInfo: null | PublicPath = useMemo(() => { + for (const x of publicPaths?.valueSeq() ?? []) { + if ( + !x.get("disabled") && + containing_public_path(path, [x.get("path")]) != null + ) { + return x.toJS(); + } + } + return null; + }, [publicPaths]); + + const student = useStudentProjectFunctionality(project_id); const [description, setDescription] = useState( - props.public?.description ?? "", + publicInfo?.description ?? "", ); const [sharingOptionsState, setSharingOptionsState] = useState(() => { - if (props.is_public && props.public?.unlisted) { + if (publicInfo == null) { + return "private"; + } + if (publicInfo?.unlisted) { return "public_unlisted"; } - if (props.is_public && props.public?.authenticated) { + if (publicInfo?.authenticated) { return "authenticated"; } - if (props.is_public && !props.public?.unlisted) { + if (!publicInfo?.unlisted) { return "public_listed"; } return "private"; }); const [licenseId, setLicenseId] = useState( - props.public?.site_license_id, + publicInfo?.site_license_id, ); const kucalc = useTypedRedux("customize", "kucalc"); const shareServer = useTypedRedux("customize", "share_server"); - if (props.compute_server_id) { + if (compute_server_id) { return ( + ); @@ -253,12 +246,10 @@ export default function Configure(props: Props) { <a onClick={() => { - redux - .getProjectActions(props.project_id) - ?.load_target("files/" + props.path); + redux.getProjectActions(project_id)?.load_target("files/" + path); }} > - {trunc_middle(props.path, 128)} + {trunc_middle(path, 128)} </a> <span style={{ float: "right" }}>{renderFinishedButton()}</span> @@ -282,7 +273,7 @@ export default function Configure(props: Props) { - {!parent_is_public && ( + {!parentIsPublic && ( <> {STATES.public_listed} - on the{" "} public search engine indexed server.{" "} - {!props.has_network_access && ( + {!has_network_access && ( (This project must be upgraded to have Internet access.) @@ -349,16 +340,16 @@ export default function Configure(props: Props) { )} - {parent_is_public && props.public != null && ( + {parentIsPublic && publicInfo != null && ( - This {props.isdir ? "directory" : "file"} is public because it - is in the public folder "{props.public.path}". Adjust the - sharing configuration of that folder instead. + This is public because it is in the public folder " + {publicInfo.path}". Adjust the sharing configuration of that + folder instead. } /> @@ -393,11 +384,11 @@ export default function Configure(props: Props) { style={{ paddingTop: "5px", margin: "15px 0" }} value={description} onChange={(e) => setDescription(e.target.value)} - disabled={parent_is_public} + disabled={parentIsPublic} placeholder="Describe what you are sharing. You can change this at any time." - onKeyUp={props.action_key} + onKeyUp={onKeyUp} onBlur={() => { - props.set_public_path({ description }); + actions.set_public_path(path, { description }); }} />
@@ -415,10 +406,10 @@ export default function Configure(props: Props) { - props.set_public_path({ license }) + actions.set_public_path(path, { license }) } /> @@ -431,7 +422,9 @@ export default function Configure(props: Props) { licenseId={licenseId} setLicenseId={(licenseId) => { setLicenseId(licenseId); - props.set_public_path({ site_license_id: licenseId }); + actions.set_public_path(path, { + site_license_id: licenseId, + }); }} /> @@ -446,10 +439,10 @@ export default function Configure(props: Props) {
{ - props.set_public_path({ jupyter_api }); + actions.set_public_path(path, { jupyter_api }); }} /> @@ -477,12 +470,12 @@ export default function Configure(props: Props) {
{ - props.set_public_path({ redirect }); + actions.set_public_path(path, { redirect }); }} - disabled={parent_is_public} + disabled={parentIsPublic} /> diff --git a/src/packages/frontend/share/license.tsx b/src/packages/frontend/share/license.tsx index 784e08d6b8e..819c9f1e967 100644 --- a/src/packages/frontend/share/license.tsx +++ b/src/packages/frontend/share/license.tsx @@ -12,8 +12,7 @@ between them. I think this is acceptable, since it is unlikely for people to do that. */ -import { FC, memo, useMemo, useState } from "react"; - +import { useMemo, useState } from "react"; import { DropdownMenu } from "@cocalc/frontend/components"; import { MenuItems } from "../components/dropdown-menu"; import { LICENSES } from "./licenses"; @@ -24,9 +23,7 @@ interface Props { disabled?: boolean; } -export const License: FC = memo((props: Props) => { - const { license, set_license, disabled = false } = props; - +export function License({ license, set_license, disabled = false }: Props) { const [sel_license, set_sel_license] = useState(license); function select(license: string): void { @@ -65,4 +62,4 @@ export const License: FC = memo((props: Props) => { items={items} /> ); -}); +} diff --git a/src/packages/frontend/syncdoc.coffee b/src/packages/frontend/syncdoc.coffee index ae8ab1995cb..535a38c5c67 100644 --- a/src/packages/frontend/syncdoc.coffee +++ b/src/packages/frontend/syncdoc.coffee @@ -66,7 +66,7 @@ class SynchronizedString extends AbstractSynchronizedDoc @project_id = @opts.project_id @filename = @opts.filename @connect = @_connect - @_syncstring = webapp_client.sync_client.sync_string + @_syncstring = webapp_client.conat_client.conat().sync.string project_id : @project_id path : @filename cursors : opts.cursors @@ -209,7 +209,7 @@ class SynchronizedDocument2 extends SynchronizedDocument @filename = '.smc/root' + @filename id = require('@cocalc/util/schema').client_db.sha1(@project_id, @filename) - @_syncstring = webapp_client.sync_client.sync_string + @_syncstring = webapp_client.conat_client.conat().sync.string id : id project_id : @project_id path : @filename diff --git a/src/packages/hub/copy-path.ts b/src/packages/hub/copy-path.ts index 67c0f3b7a01..fe92300bebf 100644 --- a/src/packages/hub/copy-path.ts +++ b/src/packages/hub/copy-path.ts @@ -3,6 +3,7 @@ * License: MS-RSL – see LICENSE.md for details */ +// DEPRECATED // Copy Operations Provider // Used in the "Client" @@ -148,6 +149,8 @@ export class CopyPath { const project = projectControl(mesg.src_project_id); // do the copy + throw Error("DEPRECATED"); + // @ts-ignore const copy_id = await project.copyPath({ path: mesg.src_path, target_project_id: mesg.target_project_id, diff --git a/src/packages/hub/hub.ts b/src/packages/hub/hub.ts index 5e5b84d4f1f..ff8e38c5b13 100644 --- a/src/packages/hub/hub.ts +++ b/src/packages/hub/hub.ts @@ -10,7 +10,7 @@ import { callback } from "awaiting"; import blocked from "blocked"; import { spawn } from "child_process"; -import { program as commander, Option } from "commander"; +import { program as commander } from "commander"; import basePath from "@cocalc/backend/base-path"; import { pghost as DEFAULT_DB_HOST, @@ -25,9 +25,7 @@ import { init_passport } from "@cocalc/server/hub/auth"; import { initialOnPremSetup } from "@cocalc/server/initial-onprem-setup"; import initHandleMentions from "@cocalc/server/mentions/handle"; import initMessageMaintenance from "@cocalc/server/messages/maintenance"; -import initProjectControl, { - COCALC_MODES, -} from "@cocalc/server/projects/control"; +import initProjectControl from "@cocalc/server/projects/control"; import initIdleTimeout from "@cocalc/server/projects/control/stop-idle-projects"; import initNewProjectPoolMaintenanceLoop from "@cocalc/server/projects/pool/maintain"; import initPurchasesMaintenanceLoop from "@cocalc/server/purchases/maintenance"; @@ -44,6 +42,7 @@ import { initConatChangefeedServer, initConatApi, initConatPersist, + initConatFileserver, } from "@cocalc/server/conat"; import { initConatServer } from "@cocalc/server/conat/socketio"; @@ -162,7 +161,7 @@ async function startServer(): Promise { // Project control logger.info("initializing project control..."); - const projectControl = initProjectControl(program.mode); + const projectControl = initProjectControl(); // used for nextjs hot module reloading dev server process.env["COCALC_MODE"] = program.mode; @@ -182,6 +181,10 @@ async function startServer(): Promise { await initConatServer({ kucalc: program.mode == "kucalc" }); } + if (program.conatFileserver || program.conatServer) { + await initConatFileserver(); + } + if (program.conatApi || program.conatServer) { await initConatApi(); await initConatChangefeedServer(); @@ -304,13 +307,10 @@ async function main(): Promise { commander .name("cocalc-hub-server") .usage("options") - .addOption( - new Option( - "--mode [string]", - `REQUIRED mode in which to run CoCalc (${COCALC_MODES.join( - ", ", - )}) - or set COCALC_MODE env var`, - ).choices(COCALC_MODES as any as string[]), + .option( + "--mode ", + `REQUIRED mode in which to run CoCalc or set COCALC_MODE env var`, + "", ) .option( "--all", @@ -318,12 +318,16 @@ async function main(): Promise { ) .option( "--conat-server", - "run a hub that provides a single-core conat server (i.e., conat-router but integrated with the http server), api, and persistence, along with an http server. This is for dev and small deployments of cocalc (and if given, do not bother with --conat-[core|api|persist] below.)", + "run a hub that provides a single-core conat server (i.e., conat-router but integrated with the http server), api, and persistence, fileserver, along with an http server. This is for dev and small deployments of cocalc (and if given, do not bother with --conat-[core|api|persist] below.)", ) .option( "--conat-router", "run a hub that provides the core conat communication layer server over a websocket (but not http server).", ) + .option( + "--conat-fileserver", + "run a hub that provides a fileserver conat service", + ) .option( "--conat-api", "run a hub that connect to conat-router and provides the standard conat API services, e.g., basic api, LLM's, changefeeds, http file upload/download, etc. There must be at least one of these. You can increase or decrease the number of these servers with no coordination needed.", @@ -421,9 +425,7 @@ async function main(): Promise { program.mode = process.env.COCALC_MODE; if (!program.mode) { throw Error( - `the --mode option must be specified or the COCALC_MODE env var set to one of ${COCALC_MODES.join( - ", ", - )}`, + `the --mode option must be specified or the COCALC_MODE env var`, ); process.exit(1); } diff --git a/src/packages/hub/package.json b/src/packages/hub/package.json index f7247e7597d..24f258311fa 100644 --- a/src/packages/hub/package.json +++ b/src/packages/hub/package.json @@ -60,9 +60,9 @@ "hub-project-dev": "pnpm build && NODE_OPTIONS='--inspect' pnpm hub-project-dev-nobuild", "hub-project-prod-nobuild": "unset DATA COCALC_ROOT && export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && export PGHOST=${PGHOST:=`realpath $INIT_CWD/../../data/postgres/socket`} && PGUSER='smc' NODE_ENV=production NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=single-user --all --hostname=0.0.0.0", "hub-project-prod-ssl": "unset DATA COCALC_ROOT && export CONAT_SERVER=https://localhost:$PORT && export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && export PGHOST=${PGHOST:=`realpath $INIT_CWD/../../data/postgres/socket`} && PGUSER='smc' NODE_ENV=production NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=single-user --all --hostname=0.0.0.0 --https-key=$INIT_CWD/../../data/secrets/cert/key.pem --https-cert=$INIT_CWD/../../data/secrets/cert/cert.pem", - "hub-docker-dev": "export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && COCALC_DOCKER=true NODE_ENV=development PROJECTS=/projects/[project_id] PORT=443 NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps --trace-warnings --inspect' cocalc-hub-server --mode=multi-user --all --hostname=0.0.0.0 --https-key=/projects/conf/cert/key.pem --https-cert=/projects/conf/cert/cert.pem", - "hub-docker-prod": "export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && COCALC_DOCKER=true NODE_ENV=production PROJECTS=/projects/[project_id] PORT=443 NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=multi-user --all --hostname=0.0.0.0 --https-key=/projects/conf/cert/key.pem --https-cert=/projects/conf/cert/cert.pem", - "hub-docker-prod-nossl": "export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && COCALC_DOCKER=true NODE_ENV=production PROJECTS=/projects/[project_id] PORT=80 NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=multi-user --all --hostname=0.0.0.0", + "hub-docker-dev": "export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && NODE_ENV=development NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps --trace-warnings --inspect' cocalc-hub-server --mode=multi-user --all --hostname=0.0.0.0 --https-key=/data/conf/cert/key.pem --https-cert=/data/conf/cert/cert.pem", + "hub-docker-prod": "export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && NODE_ENV=production NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=multi-user --all --hostname=0.0.0.0 --https-key=/data/conf/cert/key.pem --https-cert=/data/conf/cert/cert.pem", + "hub-docker-prod-nossl": "export DEBUG=${DEBUG:='cocalc:*,-cocalc:silly:*'} && NODE_ENV=production NODE_OPTIONS='--max_old_space_size=8000 --enable-source-maps' cocalc-hub-server --mode=multi-user --all --hostname=0.0.0.0", "tsc": "tsc --watch --pretty --preserveWatchOutput", "test": "jest dist/", "depcheck": "pnpx depcheck | grep -Ev '\\.coffee|coffee$'", diff --git a/src/packages/jupyter/control.ts b/src/packages/jupyter/control.ts new file mode 100644 index 00000000000..1ae9e3533fc --- /dev/null +++ b/src/packages/jupyter/control.ts @@ -0,0 +1,235 @@ +import { SyncDB } from "@cocalc/sync/editor/db/sync"; +import { SYNCDB_OPTIONS } from "@cocalc/jupyter/redux/sync"; +import { type Filesystem } from "@cocalc/conat/files/fs"; +import { initJupyterRedux, removeJupyterRedux } from "@cocalc/jupyter/kernel"; +import { syncdbPath, ipynbPath } from "@cocalc/util/jupyter/names"; +import { once } from "@cocalc/util/async-utils"; +import { OutputHandler } from "@cocalc/jupyter/execute/output-handler"; +import { throttle } from "lodash"; +import { type RunOptions } from "@cocalc/conat/project/jupyter/run-code"; +import { type JupyterActions } from "@cocalc/jupyter/redux/project-actions"; +import { getLogger } from "@cocalc/backend/logger"; + +const logger = getLogger("jupyter:control"); + +const jupyterActions: { [ipynbPath: string]: JupyterActions } = {}; + +export function isRunning(path): boolean { + return jupyterActions[ipynbPath(path)] != null; +} + +let project_id: string = ""; + +export function start({ + path, + project_id: project_id0, + client, + fs, +}: { + path: string; + client; + project_id: string; + fs: Filesystem; +}) { + if (isRunning(path)) { + return; + } + project_id = project_id0; + logger.debug("start: ", path, " - starting it"); + const syncdb = new SyncDB({ + ...SYNCDB_OPTIONS, + project_id, + path: syncdbPath(path), + client, + fs, + }); + syncdb.on("error", (err) => { + // [ ] TODO: some way to convey this to clients (?) + logger.debug(`syncdb error -- ${err}`, path); + stop({ path }); + }); + syncdb.once("closed", () => { + stop({ path }); + }); + const { actions } = initJupyterRedux(syncdb, client); + jupyterActions[ipynbPath(path)] = actions; +} + +export function stop({ path }: { path: string }) { + const actions = jupyterActions[ipynbPath(path)]; + if (actions == null) { + logger.debug("stop: ", path, " - not running"); + } else { + delete jupyterActions[ipynbPath(path)]; + const { syncdb } = actions; + logger.debug("stop: ", path, " - stopping it"); + syncdb.close(); + removeJupyterRedux(ipynbPath(path), project_id); + } +} + +// Returns async iterator over outputs +export async function run({ path, cells, noHalt, socket }: RunOptions) { + logger.debug("run:", { path, noHalt }); + + const actions = jupyterActions[ipynbPath(path)]; + if (actions == null) { + throw Error(`${ipynbPath(path)} not running`); + } + if (actions.syncdb.isClosed()) { + // shouldn't be possible + throw Error("syncdb is closed"); + } + if (!actions.syncdb.isReady()) { + logger.debug("jupyterRun: waiting until ready"); + await once(actions.syncdb, "ready"); + } + logger.debug("jupyterRun: running"); + async function* runCells() { + for (const cell of cells) { + actions.ensureKernelIsReady(); + const kernel = actions.jupyter_kernel!; + const output = kernel.execute_code({ + halt_on_error: !noHalt, + code: cell.input, + stdin: async (prompt: string, password: boolean) => { + try { + const resp = await socket.request( + { + type: "stdin", + id: cell.id, + prompt, + password, + }, + // timeout + { timeout: 1000 * 60 * 15 }, + ); + return resp.data; + } catch (err) { + return `${err}`; + } + }, + }); + for await (const mesg0 of output.iter()) { + const content = mesg0?.content; + if (content != null) { + // this mutates content, removing large messages + await kernel.process_output(content); + } + const mesg = { ...mesg0, id: cell.id }; + yield mesg; + if (!noHalt && mesg.msg_type == "error") { + // done running code because there was an error. + return; + } + } + if (kernel.failedError) { + // kernel failed during call + throw Error(kernel.failedError); + } + } + } + return await runCells(); +} + +const BACKEND_OUTPUT_FPS = 8; + +class MulticellOutputHandler { + private id: string | null = null; + private handler: OutputHandler | null = null; + + constructor( + private cells: RunOptions["cells"], + private actions, + ) {} + + process = (mesg) => { + if (mesg.id !== this.id || this.handler == null) { + this.id = mesg.id; + let cell = this.cells[mesg.id] ?? { id: mesg.id }; + this.handler?.done(); + this.handler = new OutputHandler({ cell }); + const f = throttle( + () => { + const { id, state, output, start, end, exec_count } = cell; + this.actions._set( + { type: "cell", id, state, output, start, end, exec_count }, + true, + ); + }, + 1000 / BACKEND_OUTPUT_FPS, + { + leading: true, + trailing: true, + }, + ); + this.handler.on("change", f); + + this.handler.on("process", async (mesg) => { + const kernel = this.actions.jupyter_kernel; + if ((kernel?.get_state() ?? "closed") == "closed") { + return; + } + await kernel.process_output(mesg); + }); + } + this.handler!.process(mesg); + }; + + done = () => { + this.handler?.done(); + this.handler = null; + }; +} + + +export function outputHandler({ path, cells }: RunOptions) { + if (jupyterActions[ipynbPath(path)] == null) { + throw Error(`session '${ipynbPath(path)}' not available`); + } + const actions = jupyterActions[ipynbPath(path)]; + return new MulticellOutputHandler(cells, actions); +} + +function getKernel(path: string) { + const actions = jupyterActions[ipynbPath(path)]; + if (actions == null) { + throw Error(`${ipynbPath(path)} not running`); + } + actions.ensureKernelIsReady(); + return actions.jupyter_kernel!; +} + +export async function introspect(opts: { + path: string; + code: string; + cursor_pos: number; + detail_level: 0 | 1; +}) { + const kernel = getKernel(opts.path); + return await kernel.introspect(opts); +} + +export async function complete(opts: { + path: string; + code: string; + cursor_pos: number; +}) { + const kernel = getKernel(opts.path); + return await kernel.complete(opts); +} + +export async function getConnectionFile(opts: { path }) { + const kernel = getKernel(opts.path); + await kernel.ensure_running(); + const c = kernel.getConnectionFile(); + if (c == null) { + throw Error("unable to start kernel"); + } + return c; +} + +export async function signal(opts: { path: string; signal: string }) { + const kernel = getKernel(opts.path); + await kernel.signal(opts.signal); +} diff --git a/src/packages/jupyter/execute/execute-code.ts b/src/packages/jupyter/execute/execute-code.ts index 73744b34a7e..df396ebbe84 100644 --- a/src/packages/jupyter/execute/execute-code.ts +++ b/src/packages/jupyter/execute/execute-code.ts @@ -150,7 +150,13 @@ export class CodeExecutionEmitter }; throw_error = (err): void => { - this.emit("error", err); + if (this._iter != null) { + // using the iter, so we can use that to report the error + this._iter.throw(err); + } else { + // no iter so make error known via error event + this.emit("error", err); + } this.close(); }; diff --git a/src/packages/jupyter/execute/output-handler.ts b/src/packages/jupyter/execute/output-handler.ts index 80312d1e6b7..94d5ae0cd0f 100644 --- a/src/packages/jupyter/execute/output-handler.ts +++ b/src/packages/jupyter/execute/output-handler.ts @@ -28,52 +28,79 @@ OutputHandler emits these events: import { callback } from "awaiting"; import { EventEmitter } from "events"; -import { - close, - defaults, - required, - server_time, - len, - to_json, - is_object, -} from "@cocalc/util/misc"; - -const now = () => server_time().valueOf() - 0; +import { close, len, is_object } from "@cocalc/util/misc"; +import { type TypedMap } from "@cocalc/util/types/typed-map"; + +const now = () => Date.now(); const MIN_SAVE_INTERVAL_MS = 500; const MAX_SAVE_INTERVAL_MS = 45000; +import { type Cell } from "@cocalc/jupyter/ipynb/export-to-ipynb"; + +export { type Cell }; + +interface Message { + execution_state?; + execution_count?: number; + exec_count?: number | null; + code?: string; + status?; + source?; + name?: string; + opts?; + more_output?: boolean; + text?: string; + data?: { [mimeType: string]: any }; +} + +interface JupyterMessage { + metadata?; + content?; + buffers?; + msg_type?: string; + done?: boolean; +} + +interface Options { + // object; the cell whose output (etc.) will get mutated + cell: Cell; + // If given, used to truncate, discard output messages; extra + // messages are saved and made available. + max_output_length?: number; + max_output_messages?: number; + // If no messages for this many ms, then we update via set to indicate + // that cell is being run. + report_started_ms?: number; +} + +type State = "ready" | "closed"; + export class OutputHandler extends EventEmitter { - private _opts: any; + private _opts: Options; private _n: number; private _clear_before_next_output: boolean; private _output_length: number; - private _in_more_output_mode: any; - private _state: any; - private _stdin_cb: any; + private _in_more_output_mode: boolean; + private _state: State; + private _stdin_cb?: Function; - // Never commit output to send to the frontend more frequently than this.saveIntervalMs + // Never commit output to send to the frontend more frequently + // than this.saveIntervalMs // Otherwise, we'll end up with a large number of patches. // We start out with MIN_SAVE_INTERVAL_MS and exponentially back it off to // MAX_SAVE_INTERVAL_MS. private lastSave: number = 0; private saveIntervalMs = MIN_SAVE_INTERVAL_MS; - constructor(opts: any) { + constructor(opts: Options) { super(); - this._opts = defaults(opts, { - cell: required, // object; the cell whose output (etc.) will get mutated - // If given, used to truncate, discard output messages; extra - // messages are saved and made available. - max_output_length: undefined, - max_output_messages: undefined, - report_started_ms: undefined, // If no messages for this many ms, then we update via set to indicate - // that cell is being run. - dbg: undefined, - }); + this._opts = opts; const { cell } = this._opts; cell.output = null; cell.exec_count = null; + // running a cell always de-collapses it: + cell.collapsed = false; cell.state = "run"; cell.start = null; cell.end = null; @@ -91,6 +118,57 @@ export class OutputHandler extends EventEmitter { this.stdin = this.stdin.bind(this); } + // mesg = from the kernel + process = (mesg: JupyterMessage) => { + if (mesg == null) { + // can't possibly happen, + return; + } + if (mesg.done) { + // done is a special internal cocalc message. + this.done(); + return; + } + if (mesg.content?.transient?.display_id != null) { + //this.handleTransientUpdate(mesg); + if (mesg.msg_type == "update_display_data") { + // don't also create a new output + return; + } + } + + if (mesg.msg_type === "clear_output") { + this.clear(mesg.content.wait); + return; + } + + if (mesg.content.comm_id != null) { + // ignore any comm/widget related messages here + return; + } + + if (mesg.content.execution_state === "busy") { + this.start(); + } + + if (mesg.content.payload != null) { + if (mesg.content.payload.length > 0) { + // payload shell message: + // Despite https://ipython.org/ipython-doc/3/development/messaging.html#payloads saying + // ""Payloads are considered deprecated, though their replacement is not yet implemented." + // we fully have to implement them, since they are used to implement (crazy, IMHO) + // things like %load in the python2 kernel! + for (const p of mesg.content.payload) { + this.payload(p); + } + return; + } + } else { + // Normal iopub output message + this.message(mesg.content); + } + }; + close = (): void => { if (this._state == "closed") return; this._state = "closed"; @@ -177,7 +255,7 @@ export class OutputHandler extends EventEmitter { this._clear_output(); }; - _clean_mesg = (mesg: any): void => { + _clean_mesg = (mesg: Message): void => { delete mesg.execution_state; delete mesg.code; delete mesg.status; @@ -190,7 +268,7 @@ export class OutputHandler extends EventEmitter { } }; - private _push_mesg = (mesg: any, save?: boolean): void => { + private _push_mesg = (mesg: Message, save?: boolean): void => { if (this._state === "closed") { return; } @@ -209,7 +287,7 @@ export class OutputHandler extends EventEmitter { this.lastSave = now(); } - if (this._opts.cell.output === null) { + if (this._opts.cell.output == null) { this._opts.cell.output = {}; } this._opts.cell.output[`${this._n}`] = mesg; @@ -217,7 +295,7 @@ export class OutputHandler extends EventEmitter { this.emit("change", save); }; - set_input = (input: any, save = true): void => { + set_input = (input: string, save = true): void => { if (this._state === "closed") { return; } @@ -225,9 +303,10 @@ export class OutputHandler extends EventEmitter { this.emit("change", save); }; - // Process incoming messages. This may mutate mesg. - message = (mesg: any): void => { - let has_exec_count: any; + // Process incoming messages. **This may mutate mesg** and + // definitely mutates this.cell. + message = (mesg: Message): void => { + let has_exec_count: boolean; if (this._state === "closed") { return; } @@ -299,7 +378,7 @@ export class OutputHandler extends EventEmitter { }; async stdin(prompt: string, password: boolean): Promise { - // See docs for stdin option to execute_code in backend jupyter.coffee + // See docs for stdin option to execute_code in backend. this._push_mesg({ name: "input", opts: { prompt, password } }); // Now we wait until the output message we just included has its // value set. Then we call cb with that value. @@ -310,14 +389,14 @@ export class OutputHandler extends EventEmitter { } // Call this when the cell changes; only used for stdin right now. - cell_changed = (cell: any, get_password: any): void => { + cell_changed = (cell: TypedMap, get_password: () => string): void => { if (this._state === "closed") { return; } if (this._stdin_cb == null) { return; } - const output = cell != null ? cell.get("output") : undefined; + const output = cell?.get("output"); if (output == null) { return; } @@ -346,7 +425,7 @@ export class OutputHandler extends EventEmitter { } }; - payload = (payload: any): void => { + payload = (payload: { source?; text: string }): void => { if (this._state === "closed") { return; } @@ -359,10 +438,7 @@ export class OutputHandler extends EventEmitter { // https://github.com/sagemathinc/cocalc/issues/1933 this.message(payload); } else { - // No idea what to do with this... - if (typeof this._opts.dbg === "function") { - this._opts.dbg(`Unknown PAYLOAD: ${to_json(payload)}`); - } + // TODO: No idea what to do with this... } }; } diff --git a/src/packages/jupyter/ipynb/export-to-ipynb.ts b/src/packages/jupyter/ipynb/export-to-ipynb.ts index 2e273a69b4b..97aa2a98066 100644 --- a/src/packages/jupyter/ipynb/export-to-ipynb.ts +++ b/src/packages/jupyter/ipynb/export-to-ipynb.ts @@ -9,10 +9,13 @@ Exporting from our in-memory sync-friendly format to ipynb import { CellType } from "@cocalc/util/jupyter/types"; import { deep_copy, filename_extension, keys } from "@cocalc/util/misc"; +import { isSha1 } from "@cocalc/util/misc"; type Tags = { [key: string]: boolean }; -interface Cell { +export interface Cell { + type?: "cell"; + id?: string; cell_type?: CellType; input?: string; collapsed?: boolean; @@ -20,9 +23,13 @@ interface Cell { slide?; attachments?; tags?: Tags; - output?: { [n: string]: OutputMessage }; + output?: { [n: string]: OutputMessage } | null; metadata?: Metadata; - exec_count?: number; + exec_count?: number | null; + + start?: number | null; + end?: number | null; + state?: "done" | "busy" | "run"; } type OutputMessage = any; @@ -277,7 +284,10 @@ function processOutputN( if (k.slice(0, 5) === "text/") { output_n.data[k] = diff_friendly(output_n.data[k]); } - if (k.startsWith("image/") || k === "application/pdf" || k === "iframe") { + if ( + isSha1(v) && + (k.startsWith("image/") || k === "application/pdf" || k === "iframe") + ) { if (blob_store != null) { let value; if (k === "iframe") { diff --git a/src/packages/jupyter/kernel/kernel.ts b/src/packages/jupyter/kernel/kernel.ts index 9f12677ccd1..e37b30d501e 100644 --- a/src/packages/jupyter/kernel/kernel.ts +++ b/src/packages/jupyter/kernel/kernel.ts @@ -36,7 +36,6 @@ import { jupyterSockets, type JupyterSockets } from "@cocalc/jupyter/zmq"; import { EventEmitter } from "node:events"; import { unlink } from "@cocalc/backend/misc/async-utils-node"; import { remove_redundant_reps } from "@cocalc/jupyter/ipynb/import-from-ipynb"; -import { JupyterActions } from "@cocalc/jupyter/redux/project-actions"; import { type BlobStoreInterface, CodeExecutionEmitterInterface, @@ -44,8 +43,10 @@ import { JupyterKernelInterface, KernelInfo, } from "@cocalc/jupyter/types/project-interface"; +import { JupyterActions } from "@cocalc/jupyter/redux/project-actions"; import { JupyterStore } from "@cocalc/jupyter/redux/store"; import { JUPYTER_MIMETYPES } from "@cocalc/jupyter/util/misc"; +import { isSha1 } from "@cocalc/util/misc"; import type { SyncDB } from "@cocalc/sync/editor/db/sync"; import { retry_until_success, until } from "@cocalc/util/async-utils"; import createChdirCommand from "@cocalc/util/jupyter-api/chdir-commands"; @@ -121,7 +122,7 @@ const SAGE_JUPYTER_ENV = merge(copy(process.env), { // the ipynb file, and this function creates the corresponding // actions and store, which make it possible to work with this // notebook. -export async function initJupyterRedux(syncdb: SyncDB, client: Client) { +export function initJupyterRedux(syncdb: SyncDB, client: Client) { const project_id = syncdb.project_id; if (project_id == null) { throw Error("project_id must be defined"); @@ -147,11 +148,10 @@ export async function initJupyterRedux(syncdb: SyncDB, client: Client) { // Having two at once basically results in things feeling hung. // This should never happen, but we ensure it // See https://github.com/sagemathinc/cocalc/issues/4331 - await removeJupyterRedux(path, project_id); + removeJupyterRedux(path, project_id); } const store = redux.createStore(name, JupyterStore); const actions = redux.createActions(name, JupyterActions); - actions._init(project_id, path, syncdb, store, client); syncdb.once("error", (err) => @@ -160,6 +160,8 @@ export async function initJupyterRedux(syncdb: SyncDB, client: Client) { syncdb.once("ready", () => logger.debug("initJupyterRedux", path, "syncdb ready"), ); + + return { actions, store }; } export async function getJupyterRedux(syncdb: SyncDB) { @@ -171,14 +173,11 @@ export async function getJupyterRedux(syncdb: SyncDB) { // Remove the store/actions for a given Jupyter notebook, // and also close the kernel if it is running. -export async function removeJupyterRedux( - path: string, - project_id: string, -): Promise { +export function removeJupyterRedux(path: string, project_id: string): void { logger.debug("removeJupyterRedux", path); // if there is a kernel, close it try { - await kernels.get(path)?.close(); + kernels.get(path)?.close(); } catch (_err) { // ignore } @@ -186,7 +185,7 @@ export async function removeJupyterRedux( const actions = redux.getActions(name); if (actions != null) { try { - await actions.close(); + actions.close(); } catch (err) { logger.debug( "removeJupyterRedux", @@ -252,7 +251,7 @@ export class JupyterKernel public _execute_code_queue: CodeExecutionEmitter[] = []; public sockets?: JupyterSockets; private has_ensured_running: boolean = false; - private failedError: string = ""; + public failedError: string = ""; constructor( name: string | undefined, @@ -286,6 +285,8 @@ export class JupyterKernel dbg("done"); } + isClosed = () => this._state == "closed"; + get_path = () => { return this._path; }; @@ -390,7 +391,7 @@ export class JupyterKernel } catch (err) { dbg(`ERROR spawning kernel - ${err}, ${err.stack}`); // @ts-ignore - if (this._state == "closed") { + if (this.isClosed()) { throw Error("closed"); } // console.trace(err); @@ -404,7 +405,7 @@ export class JupyterKernel return this._kernel; }; - get_connection_file = (): string | undefined => { + getConnectionFile = (): string | undefined => { return this._kernel?.connectionFile; }; @@ -556,6 +557,7 @@ export class JupyterKernel // Signal should be a string like "SIGINT", "SIGKILL". // See https://nodejs.org/api/process.html#process_process_kill_pid_signal + // this does NOT raise an error. signal = (signal: string): void => { const dbg = this.dbg("signal"); const pid = this.pid(); @@ -566,9 +568,7 @@ export class JupyterKernel try { process.kill(-pid, signal); // negative to signal the process group this.clear_execute_code_queue(); - } catch (err) { - dbg(`error: ${err}`); - } + } catch {} }; close = (): void => { @@ -630,7 +630,7 @@ export class JupyterKernel ensure_running = reuseInFlight(async (): Promise => { const dbg = this.dbg("ensure_running"); dbg(this._state); - if (this._state == "closed") { + if (this.isClosed()) { throw Error("closed so not possible to ensure running"); } if (this._state == "running") { @@ -661,7 +661,7 @@ export class JupyterKernel opts.halt_on_error = true; } if (this._state === "closed") { - throw Error("closed -- kernel -- execute_code"); + throw Error("execute_code: jupyter kernel is closed"); } const code = new CodeExecutionEmitter(this, opts); if (skipToFront) { @@ -742,7 +742,7 @@ export class JupyterKernel const dbg = this.dbg("_clear_execute_code_queue"); // ensure no future queued up evaluation occurs (currently running // one will complete and new executions could happen) - if (this._state === "closed") { + if (this.isClosed()) { dbg("no op since state is closed"); return; } @@ -764,7 +764,7 @@ export class JupyterKernel // the terminal and nbgrader and the stateless api. execute_code_now = async (opts: ExecOpts): Promise => { this.dbg("execute_code_now")(); - if (this._state == "closed") { + if (this.isClosed()) { throw Error("closed"); } if (this.failedError) { @@ -782,9 +782,8 @@ export class JupyterKernel return v; }; - private saveBlob = (data: string, type: string) => { - const blobs = this._actions?.blobs; - if (blobs == null) { + private saveBlob = async (data: string, type: string) => { + if (this._actions == null) { throw Error("blob store not available"); } const buf: Buffer = !type.startsWith("text/") @@ -792,15 +791,16 @@ export class JupyterKernel : Buffer.from(data); const sha1: string = misc_node_sha1(buf); - blobs.set(sha1, buf); + await this._actions.asyncBlobStore.set(sha1, buf); return sha1; }; - process_output = (content: any): void => { + process_output = async (content: any) => { if (this._state === "closed") { return; } const dbg = this.dbg("process_output"); + dbg(); if (content.data == null) { // No data -- https://github.com/sagemathinc/cocalc/issues/6665 // NO do not do this sort of thing. This is exactly the sort of situation where @@ -814,11 +814,11 @@ export class JupyterKernel remove_redundant_reps(content.data); - const saveBlob = (data, type) => { + const saveBlob = async (data, type) => { try { - return this.saveBlob(data, type); + return await this.saveBlob(data, type); } catch (err) { - dbg(`WARNING: Jupyter blob store not working -- ${err}`); + dbg("WARNING: Jupyter blob store not working -- skipping use", err); // i think it'll just send the large data on in the usual way instead // via the output, instead of using the blob store. It's probably just // less efficient. @@ -835,16 +835,26 @@ export class JupyterKernel type === "application/pdf" || type === "text/html" ) { + dbg("removing ", type); // Store all images and PDF and text/html in a binary blob store, so we don't have // to involve it in realtime sync. It tends to be large, etc. - const sha1 = saveBlob(content.data[type], type); - if (type == "text/html") { - // NOTE: in general, this may or may not get rendered as an iframe -- - // we use iframe for backward compatibility. - content.data["iframe"] = sha1; - delete content.data["text/html"]; - } else { - content.data[type] = sha1; + if (isSha1(content.data[type])) { + // it was already processed, e.g., this happens when a browser that was + // processing output closes and we cutoff to the project processing output. + continue; + } + const sha1 = await saveBlob(content.data[type], type); + if (sha1) { + // only remove if the save actually worked -- we don't want to break output + // for the user for a little optimization! + if (type == "text/html") { + // NOTE: in general, this may or may not get rendered as an iframe -- + // we use iframe for backward compatibility. + content.data["iframe"] = sha1; + delete content.data["text/html"]; + } else { + content.data[type] = sha1; + } } } } @@ -856,7 +866,7 @@ export class JupyterKernel await this.ensure_running(); } // Do a paranoid double check anyways... - if (this.sockets == null || this._state == "closed") { + if (this.sockets == null || this.isClosed()) { throw Error("not running, so can't call"); } diff --git a/src/packages/jupyter/package.json b/src/packages/jupyter/package.json index d1e3f3ab14e..24c0d73b8fc 100644 --- a/src/packages/jupyter/package.json +++ b/src/packages/jupyter/package.json @@ -24,7 +24,7 @@ "build": "../node_modules/.bin/tsc --build", "clean": "rm -rf node_modules dist", "test": "pnpm exec jest --forceExit --maxWorkers=1", - "depcheck": "pnpx depcheck", + "depcheck": "pnpx depcheck --ignores events", "tsc": "../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput" }, "files": [ @@ -47,6 +47,7 @@ "@cocalc/util": "workspace:*", "awaiting": "^3.0.0", "debug": "^4.4.0", + "events": "3.3.0", "expect": "^26.6.2", "he": "^1.2.0", "immutable": "^4.3.0", @@ -63,7 +64,6 @@ "zeromq": "^6.4.2" }, "devDependencies": { - "@types/json-stable-stringify": "^1.0.32", "@types/node": "^18.16.14", "@types/node-cleanup": "^2.1.2" }, diff --git a/src/packages/jupyter/pool/pool.ts b/src/packages/jupyter/pool/pool.ts index 8e74263a2cf..da5fc3091c1 100644 --- a/src/packages/jupyter/pool/pool.ts +++ b/src/packages/jupyter/pool/pool.ts @@ -267,8 +267,8 @@ export async function killKernel(kernel: SpawnedKernel) { log.debug("killKernel pid=", kernel.spawn.pid); try { process.kill(-kernel.spawn.pid, "SIGTERM"); - } catch (error) { - log.error("Failed to send SIGTERM to Jupyter kernel", error); + } catch { + //log.error("Failed to send SIGTERM to Jupyter kernel", error); } } kernel.spawn?.close?.(); diff --git a/src/packages/jupyter/redux/actions.ts b/src/packages/jupyter/redux/actions.ts index be7a437acf8..b66bddb166c 100644 --- a/src/packages/jupyter/redux/actions.ts +++ b/src/packages/jupyter/redux/actions.ts @@ -39,11 +39,6 @@ import { JupyterStore, JupyterStoreState } from "@cocalc/jupyter/redux/store"; import { Cell, KernelInfo } from "@cocalc/jupyter/types"; import { IPynbImporter } from "@cocalc/jupyter/ipynb/import-from-ipynb"; import type { JupyterKernelInterface } from "@cocalc/jupyter/types/project-interface"; -import { - char_idx_to_js_idx, - codemirror_to_jupyter_pos, - js_idx_to_char_idx, -} from "@cocalc/jupyter/util/misc"; import { SyncDB } from "@cocalc/sync/editor/db/sync"; import type { Client } from "@cocalc/sync/client/types"; import latexEnvs from "@cocalc/util/latex-envs"; @@ -64,16 +59,13 @@ const CellDeleteProtectedException = new Error("CellDeleteProtectedException"); type State = "init" | "load" | "ready" | "closed"; -export abstract class JupyterActions extends Actions { +export class JupyterActions extends Actions { public is_project: boolean; public is_compute_server?: boolean; readonly path: string; readonly project_id: string; - private _last_start?: number; public jupyter_kernel?: JupyterKernelInterface; - private last_cursor_move_time: Date = new Date(0); - private _cursor_locs?: any; - private _introspect_request?: any; + public _cursor_locs?: any; protected set_save_status: any; protected _client: Client; protected _file_watcher: any; @@ -81,7 +73,6 @@ export abstract class JupyterActions extends Actions { protected restartKernelOnClose?: (...args: any[]) => void; public asyncBlobStore: AKV; - public _complete_request?: number; public store: JupyterStore; public syncdb: SyncDB; private labels?: { @@ -114,20 +105,17 @@ export abstract class JupyterActions extends Actions { this.path = path; store.syncdb = syncdb; this.syncdb = syncdb; - // the project client is designated to manage execution/conflict, etc. this.is_project = client.is_project(); - if (this.is_project) { - this.syncdb.on("first-load", () => { - dbg("handling first load of syncdb in project"); - // Clear settings the first time the syncdb is ever - // loaded, since it has settings like "ipynb last save" - // and trust, which shouldn't be initialized to - // what they were before. Not doing this caused - // https://github.com/sagemathinc/cocalc/issues/7074 - this.syncdb.delete({ type: "settings" }); - this.syncdb.commit(); - }); - } + this.syncdb.on("first-load", () => { + dbg("handling first load of syncdb"); + // Clear settings the first time the syncdb is ever + // loaded, since it has settings like "ipynb last save" + // and trust, which shouldn't be initialized to + // what they were before. Not doing this caused + // https://github.com/sagemathinc/cocalc/issues/7074 + this.syncdb.delete({ type: "settings" }); + this.syncdb.commit(); + }); this.is_compute_server = client.is_compute_server(); let directory: any; @@ -212,31 +200,15 @@ export abstract class JupyterActions extends Actions { // an account_change listener. } - public is_closed(): boolean { - return this._state === "closed" || this._state === undefined; - } + isClosed = () => (this._state ?? "closed") == "closed"; + is_closed = () => (this._state ?? "closed") == "closed"; - public async close({ noSave }: { noSave?: boolean } = {}): Promise { + close() { if (this.is_closed()) { return; } - // ensure save to disk happens: - // - it will automatically happen for the sync-doc file, but - // we also need it for the ipynb file... as ipynb is unique - // in having two formats. - if (!noSave) { - await this.save(); - } - if (this.is_closed()) { - return; - } - - if (this.syncdb != null) { - this.syncdb.close(); - } - if (this._file_watcher != null) { - this._file_watcher.close(); - } + this.syncdb?.close(); + this._file_watcher?.close(); if (this.is_project || this.is_compute_server) { this.close_project_only(); } else { @@ -246,7 +218,7 @@ export abstract class JupyterActions extends Actions { // since otherwise this.redux and this.name are gone, // which makes destroying the actions properly impossible. this.destroy(); - this.store.destroy(); + this.store?.destroy(); close(this); this._state = "closed"; } @@ -541,7 +513,6 @@ export abstract class JupyterActions extends Actions { } } - this.onCellChange(id, new_cell, old_cell); this.store.emit("cell_change", id, new_cell, old_cell); return cell_list_needs_recompute; @@ -670,7 +641,6 @@ export abstract class JupyterActions extends Actions { kernel_state: record.get("kernel_state"), kernel_error: record.get("kernel_error"), metadata: record.get("metadata"), // extra custom user-specified metadata - connection_file: record.get("connection_file") ?? "", max_output_length: bounded_integer( record.get("max_output_length"), 100, @@ -705,6 +675,7 @@ export abstract class JupyterActions extends Actions { this.set_cell_list(); } + this.ensureThereIsACell(); this.__syncdb_change_post_hook(doInit); }; @@ -713,11 +684,6 @@ export abstract class JupyterActions extends Actions { // things in project, browser, etc. } - protected onCellChange(_id: string, _new_cell: any, _old_cell: any) { - // no-op in base class. This is a hook though - // for potentially doing things when any cell changes. - } - ensure_backend_kernel_setup() { // nontrivial in the project, but not in client or here. } @@ -756,7 +722,6 @@ export abstract class JupyterActions extends Actions { } } } - //@dbg("_set")("obj=#{misc.to_json(obj)}") this.syncdb.set(obj); if (save) { this.syncdb.commit(); @@ -972,61 +937,11 @@ export abstract class JupyterActions extends Actions { } public run_code_cell( - id: string, - save: boolean = true, - no_halt: boolean = false, + _id: string, + _save: boolean = true, + _no_halt: boolean = false, ): void { - const cell = this.store.getIn(["cells", id]); - if (cell == null) { - // it is trivial to run a cell that does not exist -- nothing needs to be done. - return; - } - const kernel = this.store.get("kernel"); - if (kernel == null || kernel === "") { - // just in case, we clear any "running" indicators - this._set({ type: "cell", id, state: "done" }); - // don't attempt to run a code-cell if there is no kernel defined - this.set_error( - "No kernel set for running cells. Therefore it is not possible to run a code cell. You have to select a kernel!", - ); - return; - } - - if (cell.get("state", "done") != "done") { - // already running -- stop it first somehow if you want to run it again... - return; - } - - // We mark the start timestamp uniquely, so that the backend can sort - // multiple cells with a simultaneous time to start request. - - let start: number = this._client.server_time().valueOf(); - if (this._last_start != null && start <= this._last_start) { - start = this._last_start + 1; - } - this._last_start = start; - this.set_jupyter_metadata(id, "outputs_hidden", undefined, false); - - this._set( - { - type: "cell", - id, - state: "start", - start, - end: null, - // time last evaluation took - last: - cell.get("start") != null && cell.get("end") != null - ? cell.get("end") - cell.get("start") - : cell.get("last"), - output: null, - exec_count: null, - collapsed: null, - no_halt: no_halt ? no_halt : null, - }, - save, - ); - this.set_trust_notebook(true, save); + console.log("run_code_cell: deprecated"); } clear_cell = (id: string, save = true) => { @@ -1055,16 +970,21 @@ export abstract class JupyterActions extends Actions { this.deprecated("run_selected_cells"); }; - public abstract run_cell(id: string, save?: boolean, no_halt?: boolean): void; + runCells(_ids: string[], _opts?: { noHalt?: boolean }): Promise { + // defined in derived class (e.g., frontend browser). + throw Error("DEPRECATED"); + } run_all_cells = (no_halt: boolean = false): void => { - this.store.get_cell_list().forEach((id) => { - this.run_cell(id, false, no_halt); - }); - this.save_asap(); + this.runCells(this.store.get_cell_list().toJS(), { noHalt: no_halt }); }; + protected clearRunQueue() { + // implemented in frontend browser actions + } + clear_all_cell_run_state = (): void => { + this.clearRunQueue(); const { store } = this; if (!store) { return; @@ -1090,34 +1010,14 @@ export abstract class JupyterActions extends Actions { run_all_above_cell(id: string): void { const i: number = this.store.get_cell_index(id); const v: string[] = this.store.get_cell_list().toJS(); - for (const id of v.slice(0, i)) { - this.run_cell(id, false); - } - this.save_asap(); + this.runCells(v.slice(0, i)); } // Run all cells below (and *including*) the specified cell. public run_all_below_cell(id: string): void { const i: number = this.store.get_cell_index(id); const v: string[] = this.store.get_cell_list().toJS(); - for (const id of v.slice(i)) { - this.run_cell(id, false); - } - this.save_asap(); - } - - public set_cursor_locs(locs: any[] = [], side_effect: boolean = false): void { - this.last_cursor_move_time = new Date(); - if (this.syncdb == null) { - // syncdb not always set -- https://github.com/sagemathinc/cocalc/issues/2107 - return; - } - if (locs.length === 0) { - // don't remove on blur -- cursor will fade out just fine - return; - } - this._cursor_locs = locs; // remember our own cursors for splitting cell - this.syncdb.set_cursor_locs(locs, side_effect); + this.runCells(v.slice(i)); } public split_cell(id: string, cursor: { line: number; ch: number }): void { @@ -1400,7 +1300,7 @@ export abstract class JupyterActions extends Actions { cell = cell.set("pos", i); this._set(cell, false); }); - this.ensure_positions_are_unique(); + this.ensurePositionsAreUnique(); this._sync(); return; } @@ -1485,155 +1385,6 @@ export abstract class JupyterActions extends Actions { return this.store.getIn(["cells", id, "input"], ""); } - // Attempt to fetch completions for give code and cursor_pos - // If successful, the completions are put in store.get('completions') and looks like - // this (as an immutable map): - // cursor_end : 2 - // cursor_start : 0 - // matches : ['the', 'completions', ...] - // status : "ok" - // code : code - // cursor_pos : cursor_pos - // - // If not successful, result is: - // status : "error" - // code : code - // cursor_pos : cursor_pos - // error : 'an error message' - // - // Only the most recent fetch has any impact, and calling - // clear_complete() ensures any fetch made before that - // is ignored. - - // Returns true if a dialog with options appears, and false otherwise. - public async complete( - code: string, - pos?: { line: number; ch: number } | number, - id?: string, - offset?: any, - ): Promise { - let cursor_pos; - const req = (this._complete_request = - (this._complete_request != null ? this._complete_request : 0) + 1); - - this.setState({ complete: undefined }); - - // pos can be either a {line:?, ch:?} object as in codemirror, - // or a number. - if (pos == null || typeof pos == "number") { - cursor_pos = pos; - } else { - cursor_pos = codemirror_to_jupyter_pos(code, pos); - } - cursor_pos = js_idx_to_char_idx(cursor_pos, code); - - const start = new Date(); - let complete; - try { - complete = await this.api().complete({ - code, - cursor_pos, - }); - } catch (err) { - if (this._complete_request > req) return false; - this.setState({ complete: { error: err } }); - // no op for now... - throw Error(`ignore -- ${err}`); - //return false; - } - - if (this.last_cursor_move_time >= start) { - // see https://github.com/sagemathinc/cocalc/issues/3611 - throw Error("ignore"); - //return false; - } - if (this._complete_request > req) { - // future completion or clear happened; so ignore this result. - throw Error("ignore"); - //return false; - } - - if (complete.status !== "ok") { - this.setState({ - complete: { - error: complete.error ? complete.error : "completion failed", - }, - }); - return false; - } - - if (complete.matches == 0) { - return false; - } - - delete complete.status; - complete.base = code; - complete.code = code; - complete.pos = char_idx_to_js_idx(cursor_pos, code); - complete.cursor_start = char_idx_to_js_idx(complete.cursor_start, code); - complete.cursor_end = char_idx_to_js_idx(complete.cursor_end, code); - complete.id = id; - // Set the result so the UI can then react to the change. - if (offset != null) { - complete.offset = offset; - } - // For some reason, sometimes complete.matches are not unique, which is annoying/confusing, - // and breaks an assumption in our react code too. - // I think the reason is e.g., a filename and a variable could be the same. We're not - // worrying about that now. - complete.matches = Array.from(new Set(complete.matches)); - // sort in a way that matches how JupyterLab sorts completions, which - // is case insensitive with % magics at the bottom - complete.matches.sort((x, y) => { - const c = misc.cmp(getCompletionGroup(x), getCompletionGroup(y)); - if (c) { - return c; - } - return misc.cmp(x.toLowerCase(), y.toLowerCase()); - }); - const i_complete = immutable.fromJS(complete); - if (complete.matches && complete.matches.length === 1 && id != null) { - // special case -- a unique completion and we know id of cell in which completing is given. - this.select_complete(id, complete.matches[0], i_complete); - return false; - } else { - this.setState({ complete: i_complete }); - return true; - } - } - - clear_complete = (): void => { - this._complete_request = - (this._complete_request != null ? this._complete_request : 0) + 1; - this.setState({ complete: undefined }); - }; - - public select_complete( - id: string, - item: string, - complete?: immutable.Map, - ): void { - if (complete == null) { - complete = this.store.get("complete"); - } - this.clear_complete(); - if (complete == null) { - return; - } - const input = complete.get("code"); - if (input != null && complete.get("error") == null) { - const starting = input.slice(0, complete.get("cursor_start")); - const ending = input.slice(complete.get("cursor_end")); - const new_input = starting + item + ending; - const base = complete.get("base"); - this.complete_cell(id, base, new_input); - } - } - - complete_cell(id: string, base: string, new_input: string): void { - this.merge_cell_input(id, base, new_input); - } - merge_cell_input( id: string, base: string, @@ -1652,126 +1403,6 @@ export abstract class JupyterActions extends Actions { this.set_cell_input(id, new_input, save); } - is_introspecting(): boolean { - const actions = this.getFrameActions() as any; - return actions?.store?.get("introspect") != null; - } - - introspect_close = () => { - if (this.is_introspecting()) { - this.getFrameActions()?.setState({ introspect: undefined }); - } - }; - - introspect_at_pos = async ( - code: string, - level: 0 | 1 = 0, - pos: { ch: number; line: number }, - ): Promise => { - if (code === "") return; // no-op if there is no code (should never happen) - await this.introspect(code, level, codemirror_to_jupyter_pos(code, pos)); - }; - - introspect = async ( - code: string, - level: 0 | 1, - cursor_pos?: number, - ): Promise | undefined> => { - const req = (this._introspect_request = - (this._introspect_request != null ? this._introspect_request : 0) + 1); - - if (cursor_pos == null) { - cursor_pos = code.length; - } - cursor_pos = js_idx_to_char_idx(cursor_pos, code); - - let introspect; - try { - introspect = await this.api().introspect({ - code, - cursor_pos, - level, - }); - if (introspect.status !== "ok") { - introspect = { error: "completion failed" }; - } - delete introspect.status; - } catch (err) { - introspect = { error: err }; - } - if (this._introspect_request > req) return; - const i = immutable.fromJS(introspect); - this.getFrameActions()?.setState({ - introspect: i, - }); - return i; // convenient / useful, e.g., for use by whiteboard. - }; - - clear_introspect = (): void => { - this._introspect_request = - (this._introspect_request != null ? this._introspect_request : 0) + 1; - this.getFrameActions()?.setState({ introspect: undefined }); - }; - - public async signal(signal = "SIGINT"): Promise { - const api = this.api({ timeout: 5000 }); - try { - await api.signal(signal); - } catch (err) { - this.set_error(err); - } - } - - // Kill the running kernel and does NOT start it up again. - halt = reuseInFlight(async (): Promise => { - if (this.restartKernelOnClose != null && this.jupyter_kernel != null) { - this.jupyter_kernel.removeListener("closed", this.restartKernelOnClose); - delete this.restartKernelOnClose; - } - this.clear_all_cell_run_state(); - await this.signal("SIGKILL"); - // Wait a little, since SIGKILL has to really happen on backend, - // and server has to respond and change state. - const not_running = (s): boolean => { - if (this._state === "closed") return true; - const t = s.get_one({ type: "settings" }); - return t != null && t.get("backend_state") != "running"; - }; - try { - await this.syncdb.wait(not_running, 30); - // worked -- and also no need to show "kernel got killed" message since this was intentional. - this.set_error(""); - } catch (err) { - // failed - this.set_error(err); - } - }); - - restart = reuseInFlight(async (): Promise => { - await this.halt(); - if (this._state === "closed") return; - this.clear_all_cell_run_state(); - // Actually start it running again (rather than waiting for - // user to do something), since this is called "restart". - try { - await this.set_backend_kernel_info(); // causes kernel to start - } catch (err) { - this.set_error(err); - } - }); - - public shutdown = reuseInFlight(async (): Promise => { - if (this._state === ("closed" as State)) { - return; - } - await this.signal("SIGKILL"); - if (this._state === ("closed" as State)) { - return; - } - this.clear_all_cell_run_state(); - await this.save_asap(); - }); - set_backend_kernel_info = async (): Promise => { if (this._state === "closed" || this.syncdb.is_read_only()) { return; @@ -2100,7 +1731,7 @@ export abstract class JupyterActions extends Actions { this._state = "ready"; }; - public set_cell_slide(id: string, value: any): void { + set_cell_slide = (id: string, value: any) => { if (!value) { value = null; // delete } @@ -2112,15 +1743,11 @@ export abstract class JupyterActions extends Actions { id, slide: value, }); - } + }; - public ensure_positions_are_unique(): void { - if (this._state != "ready" || this.store == null) { - // because of debouncing, this ensure_positions_are_unique can - // be called after jupyter actions are closed. - return; - } - const changes = cell_utils.ensure_positions_are_unique( + ensurePositionsAreUnique = () => { + if (this.isClosed()) return; + const changes = cell_utils.ensurePositionsAreUnique( this.store.get("cells"), ); if (changes != null) { @@ -2130,9 +1757,9 @@ export abstract class JupyterActions extends Actions { } } this._sync(); - } + }; - public set_default_kernel(kernel?: string): void { + set_default_kernel = (kernel?: string) => { if (kernel == null || kernel === "") return; // doesn't make sense for project (right now at least) if (this.is_project || this.is_compute_server) return; @@ -2149,7 +1776,7 @@ export abstract class JupyterActions extends Actions { (this.redux.getTable("account") as any).set({ editor_settings: { jupyter: cur }, }); - } + }; edit_attachments = (id: string): void => { this.setState({ edit_attachments: id }); @@ -2582,6 +2209,25 @@ export abstract class JupyterActions extends Actions { } this.setState({ runProgress: total > 0 ? (100 * ran) / total : 100 }); }; + + ensureThereIsACell = () => { + if (this._state !== "ready") { + return; + } + const cells = this.store.get("cells"); + if (cells == null || cells.size === 0) { + this._set({ + type: "cell", + // by using the same id across clients we solve the problem of multiple + // clients creating a cell at the same time. + id: "alpha", + pos: 0, + input: "", + }); + // We are obviously contributing content to this (empty!) notebook. + return this.set_trust_notebook(true); + } + }; } function extractLabel(content: string): string { @@ -2606,14 +2252,3 @@ function bounded_integer(n: any, min: any, max: any, def: any) { } return n; } - -function getCompletionGroup(x: string): number { - switch (x[0]) { - case "_": - return 1; - case "%": - return 2; - default: - return 0; - } -} diff --git a/src/packages/jupyter/redux/project-actions.ts b/src/packages/jupyter/redux/project-actions.ts index ee1300deec9..8c5e807d562 100644 --- a/src/packages/jupyter/redux/project-actions.ts +++ b/src/packages/jupyter/redux/project-actions.ts @@ -14,1596 +14,41 @@ fully unit test it via mocking of components. NOTE: this is also now the actions used by remote compute servers as well. */ -import { get_kernel_data } from "@cocalc/jupyter/kernel/kernel-data"; -import * as immutable from "immutable"; -import json_stable from "json-stable-stringify"; -import { debounce } from "lodash"; -import { - JupyterActions as JupyterActions0, - MAX_OUTPUT_MESSAGES, -} from "@cocalc/jupyter/redux/actions"; -import { callback2, once } from "@cocalc/util/async-utils"; -import * as misc from "@cocalc/util/misc"; -import { OutputHandler } from "@cocalc/jupyter/execute/output-handler"; -import { RunAllLoop } from "./run-all-loop"; -import nbconvertChange from "./handle-nbconvert-change"; -import type { ClientFs } from "@cocalc/sync/client/types"; +import { JupyterActions as JupyterActions0 } from "@cocalc/jupyter/redux/actions"; import { kernel as createJupyterKernel } from "@cocalc/jupyter/kernel"; -import { removeJupyterRedux } from "@cocalc/jupyter/kernel"; -import { initConatService } from "@cocalc/jupyter/kernel/conat-service"; -import { type DKV, dkv } from "@cocalc/conat/sync/dkv"; -import { computeServerManager } from "@cocalc/conat/compute/manager"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import { getLogger } from "@cocalc/backend/logger"; -// see https://github.com/sagemathinc/cocalc/issues/8060 -const MAX_OUTPUT_SAVE_DELAY = 30000; - -// refuse to open an ipynb that is bigger than this: -const MAX_SIZE_IPYNB_MB = 150; - -type BackendState = "init" | "ready" | "spawning" | "starting" | "running"; +const logger = getLogger("jupyter:project-actions"); export class JupyterActions extends JupyterActions0 { - private _backend_state: BackendState = "init"; - private lastSavedBackendState?: BackendState; - private _initialize_manager_already_done: any; - private _kernel_state: any; - private _manager_run_cell_queue: any; - private _running_cells: { [id: string]: string }; - private _throttled_ensure_positions_are_unique: any; - private run_all_loop?: RunAllLoop; - private clear_kernel_error?: any; - private running_manager_run_cell_process_queue: boolean = false; - private last_ipynb_save: number = 0; - protected _client: ClientFs; // this has filesystem access, etc. - public blobs: DKV; - private computeServers?; - - private initBlobStore = async () => { - this.blobs = await dkv(this.blobStoreOptions()); + public blobs = { + set: (_k, _v) => {}, + get: (_k): any => {}, }; + save_ipynb_file = async (_opts?) => {}; + capture_output_message = (_opts) => {}; + process_comm_message_from_kernel = (_mesg) => {}; - // uncomment for verbose logging of everything here to the console. - // dbg(f: string) { - // return (...args) => console.log(f, args); - // } - - public run_cell( - id: string, - save: boolean = true, - no_halt: boolean = false, - ): void { - if (this.store.get("read_only")) { - return; - } - const cell = this.store.getIn(["cells", id]); - if (cell == null) { - // it is trivial to run a cell that does not exist -- nothing needs to be done. - return; - } - const cell_type = cell.get("cell_type", "code"); - if (cell_type == "code") { - // when the backend is running code, just don't worry about - // trying to parse things like "foo?" out. We can't do - // it without CodeMirror, and it isn't worth it for that - // application. - this.run_code_cell(id, save, no_halt); - } - if (save) { - this.save_asap(); - } - } - - private set_backend_state(backend_state: BackendState): void { - this.dbg("set_backend_state")(backend_state); - - /* - The backend states, which are put in the syncdb so clients - can display this: - - - 'init' -- the backend is checking the file on disk, etc. - - 'ready' -- the backend is setup and ready to use; kernel isn't running though - - 'starting' -- the kernel itself is actived and currently starting up (e.g., Sage is starting up) - - 'running' -- the kernel is running and ready to evaluate code - - - 'init' --> 'ready' --> 'spawning' --> 'starting' --> 'running' - /|\ | - |-----------------------------------------| - - Going from ready to starting happens first when a code execution is requested. - */ - - // Check just in case Typescript doesn't catch something: - if ( - ["init", "ready", "spawning", "starting", "running"].indexOf( - backend_state, - ) === -1 - ) { - throw Error(`invalid backend state '${backend_state}'`); - } - if (backend_state == "init" && this._backend_state != "init") { - // Do NOT allow changing the state to init from any other state. - throw Error( - `illegal state change '${this._backend_state}' --> '${backend_state}'`, - ); - } - this._backend_state = backend_state; - - if (this.lastSavedBackendState != backend_state) { - this._set({ - type: "settings", - backend_state, - last_backend_state: Date.now(), - }); - this.save_asap(); - this.lastSavedBackendState = backend_state; - } - - // The following is to clear kernel_error if things are working only. - if (backend_state == "running") { - // clear kernel error if kernel successfully starts and stays - // in running state for a while. - this.clear_kernel_error = setTimeout(() => { - this._set({ - type: "settings", - kernel_error: "", - }); - }, 3000); - } else { - // change to a different state; cancel attempt to clear kernel error - if (this.clear_kernel_error) { - clearTimeout(this.clear_kernel_error); - delete this.clear_kernel_error; - } - } - } - - set_kernel_state = (state: any, save = false) => { - this._kernel_state = state; - this._set({ type: "settings", kernel_state: state }, save); - }; - - // Called exactly once when the manager first starts up after the store is initialized. - // Here we ensure everything is in a consistent state so that we can react - // to changes later. - async initialize_manager() { - if (this._initialize_manager_already_done) { - return; - } - const dbg = this.dbg("initialize_manager"); - dbg(); - this._initialize_manager_already_done = true; - - dbg("initialize Jupyter Conat api handler"); - await this.initConatApi(); - - dbg("initializing blob store"); - await this.initBlobStore(); - - this.sync_exec_state = debounce(this.sync_exec_state, 2000); - this._throttled_ensure_positions_are_unique = debounce( - this.ensure_positions_are_unique, - 5000, - ); - // Listen for changes... - this.syncdb.on("change", this.backendSyncdbChange); - - this.setState({ - // used by the kernel_info function of this.jupyter_kernel - start_time: this._client.server_time().valueOf(), - }); - - // clear nbconvert start on init, since no nbconvert can be running yet - this.syncdb.delete({ type: "nbconvert" }); - - // Initialize info about available kernels, which is used e.g., for - // saving to ipynb format. - this.init_kernel_info(); - - // We try once to load from disk. If it fails, then - // a record with type:'fatal' - // is created in the database; if it succeeds, that record is deleted. - // Try again only when the file changes. - await this._first_load(); - - // Listen for model state changes... - if (this.syncdb.ipywidgets_state == null) { - throw Error("syncdb's ipywidgets_state must be defined!"); - } - this.syncdb.ipywidgets_state.on( - "change", - this.handle_ipywidgets_state_change, - ); - } - - private conatService?; - private initConatApi = reuseInFlight(async () => { - if (this.conatService != null) { - this.conatService.close(); - this.conatService = null; - } - const service = (this.conatService = await initConatService({ - project_id: this.project_id, - path: this.path, - })); - this.syncdb.on("closed", () => { - service.close(); - }); - }); - - private _first_load = async () => { - const dbg = this.dbg("_first_load"); - dbg("doing load"); - if (this.is_closed()) { - throw Error("actions must not be closed"); - } - try { - await this.loadFromDiskIfNewer(); - } catch (err) { - dbg(`load failed -- ${err}; wait for file change and try again`); - const path = this.store.get("path"); - const watcher = this._client.watch_file({ path }); - await once(watcher, "change"); - dbg("file changed"); - watcher.close(); - await this._first_load(); - return; - } - dbg("loading worked"); - this._init_after_first_load(); - }; - - private _init_after_first_load = () => { - const dbg = this.dbg("_init_after_first_load"); - - dbg("initializing"); - // this may change the syncdb. - this.ensure_backend_kernel_setup(); - - this.init_file_watcher(); - - this._state = "ready"; - }; - - private backendSyncdbChange = (changes: any) => { - if (this.is_closed()) { - return; - } - const dbg = this.dbg("backendSyncdbChange"); - if (changes != null) { - changes.forEach((key) => { - switch (key.get("type")) { - case "settings": - dbg("settings change"); - var record = this.syncdb.get_one(key); - if (record != null) { - // ensure kernel is properly configured - this.ensure_backend_kernel_setup(); - // only the backend should change kernel and backend state; - // however, our security model allows otherwise (e.g., via TimeTravel). - if ( - record.get("kernel_state") !== this._kernel_state && - this._kernel_state != null - ) { - this.set_kernel_state(this._kernel_state, true); - } - if (record.get("backend_state") !== this._backend_state) { - this.set_backend_state(this._backend_state); - } - - if (record.get("run_all_loop_s")) { - if (this.run_all_loop == null) { - this.run_all_loop = new RunAllLoop( - this, - record.get("run_all_loop_s"), - ); - } else { - // ensure interval is correct - this.run_all_loop.set_interval(record.get("run_all_loop_s")); - } - } else if ( - !record.get("run_all_loop_s") && - this.run_all_loop != null - ) { - // stop it. - this.run_all_loop.close(); - delete this.run_all_loop; - } - } - break; - } - }); - } - - this.ensure_there_is_a_cell(); - this._throttled_ensure_positions_are_unique(); - this.sync_exec_state(); - }; - - // ensure_backend_kernel_setup ensures that we have a connection - // to the selected Jupyter kernel, if any. - ensure_backend_kernel_setup = () => { - const dbg = this.dbg("ensure_backend_kernel_setup"); - if (this.isDeleted()) { - dbg("file is deleted"); - return; - } - - const kernel = this.store.get("kernel"); - dbg("ensure_backend_kernel_setup", { kernel }); - - let current: string | undefined = undefined; + ensureKernelIsReady = () => { if (this.jupyter_kernel != null) { - current = this.jupyter_kernel.name; - if (current == kernel) { - const state = this.jupyter_kernel.get_state(); - if (state == "error") { - dbg("kernel is broken"); - // nothing to do -- let user ponder the error they should see. - return; - } - if (state != "closed") { - dbg("everything is properly setup and working"); - return; - } - } - } - - dbg(`kernel='${kernel}', current='${current}'`); - if ( - this.jupyter_kernel != null && - this.jupyter_kernel.get_state() != "closed" - ) { - if (current != kernel) { - dbg("kernel changed -- kill running kernel to trigger switch"); - this.jupyter_kernel.close(); - return; + if (this.jupyter_kernel.isClosed()) { + delete this.jupyter_kernel; } else { - dbg("nothing to do"); return; } } - - dbg("make a new kernel"); - + const kernel = this.store.get("kernel"); + logger.debug("initKernel", { kernel, path: this.path }); // No kernel wrapper object setup at all. Make one. this.jupyter_kernel = createJupyterKernel({ name: kernel, - path: this.store.get("path"), + path: this.path, actions: this, }); - - if (this.syncdb.ipywidgets_state == null) { - throw Error("syncdb's ipywidgets_state must be defined!"); - } - this.syncdb.ipywidgets_state.clear(); - - if (this.jupyter_kernel == null) { - // to satisfy typescript. - throw Error("jupyter_kernel must be defined"); - } - dbg("kernel created -- installing handlers"); - - // save so gets reported to frontend, and surfaced to user: - // https://github.com/sagemathinc/cocalc/issues/4847 - this.jupyter_kernel.on("kernel_error", (error) => { - this.set_kernel_error(error); - }); - - // Since we just made a new kernel, clearly no cells are running on the backend: - this._running_cells = {}; - - const toStart: string[] = []; - this.store?.get_cell_list().forEach((id) => { - if (this.store.getIn(["cells", id, "state"]) == "start") { - toStart.push(id); - } - }); - - dbg("clear cell run state"); - this.clear_all_cell_run_state(); - - this.restartKernelOnClose = () => { - // When the kernel closes, make sure a new kernel gets setup. - if (this.store == null || this._state !== "ready") { - // This event can also happen when this actions is being closed, - // in which case obviously we shouldn't make a new kernel. - return; - } - dbg("kernel closed -- make new one."); - this.ensure_backend_kernel_setup(); - }; - - this.jupyter_kernel.once("closed", this.restartKernelOnClose); - - // Track backend state changes other than closing, so they - // are visible to user etc. - // TODO: Maybe all these need to move to ephemeral table? - // There's a good argument that recording these is useful though, so when - // looking at time travel or debugging, you know what was going on. - this.jupyter_kernel.on("state", (state) => { - dbg("jupyter_kernel state --> ", state); - switch (state) { - case "off": - case "closed": - // things went wrong. - this._running_cells = {}; - this.clear_all_cell_run_state(); - this.set_backend_state("ready"); - this.jupyter_kernel?.close(); - this.running_manager_run_cell_process_queue = false; - delete this.jupyter_kernel; - return; - case "spawning": - case "starting": - this.set_connection_file(); // yes, fall through - case "running": - this.set_backend_state(state); - } - }); - - this.jupyter_kernel.on("execution_state", this.set_kernel_state); - - this.handle_all_cell_attachments(); - dbg("ready"); - this.set_backend_state("ready"); - - // Run cells that the user explicitly set to be running before the - // kernel actually had finished starting up. - // This must be done after the state is ready. - if (toStart.length > 0) { - for (const id of toStart) { - this.run_cell(id); - } - } - }; - - set_connection_file = () => { - const connection_file = this.jupyter_kernel?.get_connection_file() ?? ""; - this._set({ - type: "settings", - connection_file, - }); - }; - - init_kernel_info = async () => { - let kernels0 = this.store.get("kernels"); - if (kernels0 != null) { - return; - } - const dbg = this.dbg("init_kernel_info"); - dbg("getting"); - let kernels; - try { - kernels = await get_kernel_data(); - dbg("success"); - } catch (err) { - dbg(`FAILED to get kernel info: ${err}`); - // TODO: what to do?? Saving will be broken... - return; - } - this.setState({ - kernels: immutable.fromJS(kernels), - }); - }; - - async ensure_backend_kernel_is_running() { - const dbg = this.dbg("ensure_backend_kernel_is_running"); - if (this._backend_state == "ready") { - dbg("in state 'ready', so kick it into gear"); - await this.set_backend_kernel_info(); - dbg("done getting kernel info"); - } - const is_running = (s): boolean => { - if (this._state === "closed") { - return true; - } - const t = s.get_one({ type: "settings" }); - if (t == null) { - dbg("no settings"); - return false; - } else { - const state = t.get("backend_state"); - dbg(`state = ${state}`); - return state == "running"; - } - }; - await this.syncdb.wait(is_running, 60); - } - - // onCellChange is called after a cell change has been - // incorporated into the store after the syncdb change event. - // - If we are responsible for running cells, then it ensures - // that cell gets computed. - // - We also handle attachments for markdown cells. - protected onCellChange(id: string, new_cell: any, old_cell: any) { - const dbg = this.dbg(`onCellChange(id='${id}')`); - dbg(); - // this logging could be expensive due to toJS, so only uncomment - // if really needed - // dbg("new_cell=", new_cell?.toJS(), "old_cell", old_cell?.toJS()); - - if ( - new_cell?.get("state") === "start" && - old_cell?.get("state") !== "start" - ) { - this.manager_run_cell_enqueue(id); - // attachments below only happen for markdown cells, which don't get run, - // we can return here: - return; - } - - const attachments = new_cell?.get("attachments"); - if (attachments != null && attachments !== old_cell?.get("attachments")) { - this.handle_cell_attachments(new_cell); - } - } - - protected __syncdb_change_post_hook(doInit: boolean) { - if (doInit) { - // Since just opening the actions in the project, definitely the kernel - // isn't running so set this fact in the shared database. It will make - // things always be in the right initial state. - this.syncdb.set({ - type: "settings", - backend_state: "init", - kernel_state: "idle", - kernel_usage: { memory: 0, cpu: 0 }, - }); - this.syncdb.commit(); - - // Also initialize the execution manager, which runs cells that have been - // requested to run. - this.initialize_manager(); - } - if (this.store.get("kernel")) { - this.manager_run_cell_process_queue(); - } - } - - // Ensure that the cells listed as running *are* exactly the - // ones actually running or queued up to run. - sync_exec_state = () => { - // sync_exec_state is debounced, so it is *expected* to get called - // after actions have been closed. - if (this.store == null || this._state !== "ready") { - // not initialized, so we better not - // mess with cell state (that is somebody else's responsibility). - return; - } - - const dbg = this.dbg("sync_exec_state"); - let change = false; - const cells = this.store.get("cells"); - // First verify that all actual cells that are said to be running - // (according to the store) are in fact running. - if (cells != null) { - cells.forEach((cell, id) => { - const state = cell.get("state"); - if ( - state != null && - state != "done" && - state != "start" && // regarding "start", see https://github.com/sagemathinc/cocalc/issues/5467 - !this._running_cells?.[id] - ) { - dbg(`set cell ${id} with state "${state}" to done`); - this._set({ type: "cell", id, state: "done" }, false); - change = true; - } - }); - } - if (this._running_cells != null) { - const cells = this.store.get("cells"); - // Next verify that every cell actually running is still in the document - // and listed as running. TimeTravel, deleting cells, etc., can - // certainly lead to this being necessary. - for (const id in this._running_cells) { - const state = cells.getIn([id, "state"]); - if (state == null || state === "done") { - // cell no longer exists or isn't in a running state - dbg(`tell kernel to not run ${id}`); - this._cancel_run(id); - } - } - } - if (change) { - return this._sync(); - } - }; - - _cancel_run = (id: any) => { - const dbg = this.dbg(`_cancel_run ${id}`); - // All these checks are so we only cancel if it is actually running - // with the current kernel... - if (this._running_cells == null || this.jupyter_kernel == null) return; - const identity = this._running_cells[id]; - if (identity == null) return; - if (this.jupyter_kernel.identity == identity) { - dbg("canceling"); - this.jupyter_kernel.cancel_execute(id); - } else { - dbg("not canceling since wrong identity"); - } - }; - - // Note that there is a request to run a given cell. - // You must call manager_run_cell_process_queue for them to actually start running. - protected manager_run_cell_enqueue(id: string) { - if (this._running_cells?.[id]) { - return; - } - if (this._manager_run_cell_queue == null) { - this._manager_run_cell_queue = {}; - } - this._manager_run_cell_queue[id] = true; - } - - // properly start running -- in order -- the cells that have been requested to run - protected async manager_run_cell_process_queue() { - if (this.running_manager_run_cell_process_queue) { - return; - } - this.running_manager_run_cell_process_queue = true; - try { - const dbg = this.dbg("manager_run_cell_process_queue"); - const queue = this._manager_run_cell_queue; - if (queue == null) { - //dbg("queue is null"); - return; - } - delete this._manager_run_cell_queue; - const v: any[] = []; - for (const id in queue) { - if (!this._running_cells?.[id]) { - v.push(this.store.getIn(["cells", id])); - } - } - - if (v.length == 0) { - dbg("no non-running cells"); - return; // nothing to do - } - - v.sort((a, b) => - misc.cmp( - a != null ? a.get("start") : undefined, - b != null ? b.get("start") : undefined, - ), - ); - - dbg( - `found ${v.length} non-running cell that should be running, so ensuring kernel is running...`, - ); - this.ensure_backend_kernel_setup(); - try { - await this.ensure_backend_kernel_is_running(); - if (this._state == "closed") return; - } catch (err) { - // if this fails, give up on evaluation. - return; - } - - dbg( - `kernel is now running; requesting that each ${v.length} cell gets executed`, - ); - for (const cell of v) { - if (cell != null) { - this.manager_run_cell(cell.get("id")); - } - } - - if (this._manager_run_cell_queue != null) { - // run it again to process additional entries. - setTimeout(this.manager_run_cell_process_queue, 1); - } - } finally { - this.running_manager_run_cell_process_queue = false; - } - } - - // returns new output handler for this cell. - protected _output_handler(cell) { - const dbg = this.dbg(`_output_handler(id='${cell.id}')`); - if ( - this.jupyter_kernel == null || - this.jupyter_kernel.get_state() == "closed" - ) { - throw Error("jupyter kernel must exist and not be closed"); - } - this.reset_more_output(cell.id); - - const handler = new OutputHandler({ - cell, - max_output_length: this.store.get("max_output_length"), - max_output_messages: MAX_OUTPUT_MESSAGES, - report_started_ms: 250, - dbg, - }); - - dbg("setting up jupyter_kernel.once('closed', ...) handler"); - const handleKernelClose = () => { - dbg("output handler -- closing due to jupyter kernel closed"); - handler.close(); - }; - this.jupyter_kernel.once("closed", handleKernelClose); - // remove the "closed" handler we just defined above once - // we are done waiting for output from this cell. - // The output handler removes all listeners whenever it is - // finished, so we don't have to remove this listener for done. - handler.once("done", () => - this.jupyter_kernel?.removeListener("closed", handleKernelClose), - ); - - handler.on("more_output", (mesg, mesg_length) => { - this.set_more_output(cell.id, mesg, mesg_length); - }); - - handler.on("process", (mesg) => { - // Do not enable -- mesg often very large! - // dbg("handler.on('process')", mesg); - if ( - this.jupyter_kernel == null || - this.jupyter_kernel.get_state() == "closed" - ) { - return; - } - this.jupyter_kernel.process_output(mesg); - // dbg("handler -- after processing ", mesg); - }); - - return handler; - } - - manager_run_cell = (id: string) => { - const dbg = this.dbg(`manager_run_cell(id='${id}')`); - dbg(JSON.stringify(misc.keys(this._running_cells))); - - if (this._running_cells == null) { - this._running_cells = {}; - } - - if (this._running_cells[id]) { - dbg("cell already queued to run in kernel"); - return; - } - - // It's important to set this._running_cells[id] to be true so that - // sync_exec_state doesn't declare this cell done. The kernel identity - // will get set properly below in case it changes. - this._running_cells[id] = this.jupyter_kernel?.identity ?? "none"; - - const orig_cell = this.store.get("cells").get(id); - if (orig_cell == null) { - // nothing to do -- cell deleted - return; - } - - let input: string | undefined = orig_cell.get("input", ""); - if (input == null) { - input = ""; - } else { - input = input.trim(); - } - - const halt_on_error: boolean = !orig_cell.get("no_halt", false); - - if (this.jupyter_kernel == null) { - throw Error("bug -- this is guaranteed by the above"); - } - this._running_cells[id] = this.jupyter_kernel.identity; - - const cell: any = { - id, - type: "cell", - kernel: this.store.get("kernel"), - }; - - dbg(`using max_output_length=${this.store.get("max_output_length")}`); - const handler = this._output_handler(cell); - - // exponentiallyThrottledSaved calls this.syncdb?.save, but - // it throttles the calls, and does so using exponential backoff - // up to MAX_OUTPUT_SAVE_DELAY milliseconds. Basically every - // time exponentiallyThrottledSaved is called it increases the - // interval used for throttling by multiplying saveThrottleMs by 1.3 - // until saveThrottleMs gets to MAX_OUTPUT_SAVE_DELAY. There is no - // need at all to do a trailing call, since other code handles that. - let saveThrottleMs = 1; - let lastCall = 0; - const exponentiallyThrottledSaved = () => { - const now = Date.now(); - if (now - lastCall < saveThrottleMs) { - return; - } - lastCall = now; - saveThrottleMs = Math.min(1.3 * saveThrottleMs, MAX_OUTPUT_SAVE_DELAY); - this.syncdb?.save(); - }; - - handler.on("change", (save) => { - if (!this.store.getIn(["cells", id])) { - // The cell was deleted, but we just got some output - // NOTE: client shouldn't allow deleting running or queued - // cells, but we still want to do something useful/sensible. - // We put cell back where it was with same input. - cell.input = orig_cell.get("input"); - cell.pos = orig_cell.get("pos"); - } - this.syncdb.set(cell); - // This is potentially very verbose -- don't due it unless - // doing low level debugging: - //dbg(`change (save=${save}): cell='${JSON.stringify(cell)}'`); - if (save) { - exponentiallyThrottledSaved(); - } - }); - - handler.once("done", () => { - dbg("handler is done"); - this.store.removeListener("cell_change", cell_change); - exec.close(); - if (this._running_cells != null) { - delete this._running_cells[id]; - } - this.syncdb?.save(); - setTimeout(() => this.syncdb?.save(), 100); - }); - - if (this.jupyter_kernel == null) { - handler.error("Unable to start Jupyter"); - return; - } - - const get_password = (): string => { - if (this.jupyter_kernel == null) { - dbg("get_password", id, "no kernel"); - return ""; - } - const password = this.jupyter_kernel.store.get(id); - dbg("get_password", id, password); - this.jupyter_kernel.store.delete(id); - return password; - }; - - // This is used only for stdin right now. - const cell_change = (cell_id, new_cell) => { - if (id === cell_id) { - dbg("cell_change"); - handler.cell_changed(new_cell, get_password); - } - }; - this.store.on("cell_change", cell_change); - - const exec = this.jupyter_kernel.execute_code({ - code: input, - id, - stdin: handler.stdin, - halt_on_error, - }); - - exec.on("output", (mesg) => { - // uncomment only for specific low level debugging -- see https://github.com/sagemathinc/cocalc/issues/7022 - // dbg(`got mesg='${JSON.stringify(mesg)}'`); // !!!☡ ☡ ☡ -- EXTREME DANGER ☡ ☡ ☡ !!!! - - if (mesg == null) { - // can't possibly happen, of course. - const err = "empty mesg"; - dbg(`got error='${err}'`); - handler.error(err); - return; - } - if (mesg.done) { - // done is a special internal cocalc message. - handler.done(); - return; - } - if (mesg.content?.transient?.display_id != null) { - // See https://github.com/sagemathinc/cocalc/issues/2132 - // We find any other outputs in the document with - // the same transient.display_id, and set their output to - // this mesg's output. - this.handleTransientUpdate(mesg); - if (mesg.msg_type == "update_display_data") { - // don't also create a new output - return; - } - } - - if (mesg.msg_type === "clear_output") { - handler.clear(mesg.content.wait); - return; - } - - if (mesg.content.comm_id != null) { - // ignore any comm/widget related messages - return; - } - - if (mesg.content.execution_state === "idle") { - this.store.removeListener("cell_change", cell_change); - return; - } - if (mesg.content.execution_state === "busy") { - handler.start(); - } - if (mesg.content.payload != null) { - if (mesg.content.payload.length > 0) { - // payload shell message: - // Despite https://ipython.org/ipython-doc/3/development/messaging.html#payloads saying - // ""Payloads are considered deprecated, though their replacement is not yet implemented." - // we fully have to implement them, since they are used to implement (crazy, IMHO) - // things like %load in the python2 kernel! - mesg.content.payload.map((p) => handler.payload(p)); - return; - } - } else { - // Normal iopub output message - handler.message(mesg.content); - return; - } - }); - - exec.on("error", (err) => { - dbg(`got error='${err}'`); - handler.error(err); - }); - }; - - reset_more_output = (id: string) => { - if (id == null) { - this.store._more_output = {}; - } - if (this.store._more_output[id] != null) { - delete this.store._more_output[id]; - } - }; - - set_more_output = (id: string, mesg: object, length: number): void => { - if (this.store._more_output[id] == null) { - this.store._more_output[id] = { - length: 0, - messages: [], - lengths: [], - discarded: 0, - truncated: 0, - }; - } - const output = this.store._more_output[id]; - - output.length += length; - output.lengths.push(length); - output.messages.push(mesg); - - const goal_length = 10 * this.store.get("max_output_length"); - while (output.length > goal_length) { - let need: any; - let did_truncate = false; - - // check if there is a text field, which we can truncate - let len = output.messages[0].text?.length; - if (len != null) { - need = output.length - goal_length + 50; - if (len > need) { - // Instead of throwing this message away, let's truncate its text part. After - // doing this, the message is at least shorter than it was before. - output.messages[0].text = misc.trunc( - output.messages[0].text, - len - need, - ); - did_truncate = true; - } - } - - // check if there is a text/plain field, which we can thus also safely truncate - if (!did_truncate && output.messages[0].data != null) { - for (const field in output.messages[0].data) { - if (field === "text/plain") { - const val = output.messages[0].data[field]; - len = val.length; - if (len != null) { - need = output.length - goal_length + 50; - if (len > need) { - // Instead of throwing this message away, let's truncate its text part. After - // doing this, the message is at least need shorter than it was before. - output.messages[0].data[field] = misc.trunc(val, len - need); - did_truncate = true; - } - } - } - } - } - - if (did_truncate) { - const new_len = JSON.stringify(output.messages[0]).length; - output.length -= output.lengths[0] - new_len; // how much we saved - output.lengths[0] = new_len; - output.truncated += 1; - break; - } - - const n = output.lengths.shift(); - output.messages.shift(); - output.length -= n; - output.discarded += 1; - } - }; - - private init_file_watcher = () => { - const dbg = this.dbg("file_watcher"); - dbg(); - this._file_watcher = this._client.watch_file({ - path: this.store.get("path"), - debounce: 1000, - }); - - this._file_watcher.on("change", async () => { - dbg("change"); - try { - await this.loadFromDiskIfNewer(); - } catch (err) { - dbg("failed to load on change", err); - } - }); }; - /* - * Unfortunately, though I spent two hours on this approach... it just doesn't work, - * since, e.g., if the sync file doesn't already exist, it can't be created, - * which breaks everything. So disabling for now and re-opening the issue. - _sync_file_mode: => - dbg = @dbg("_sync_file_mode"); dbg() - * Make the mode of the syncdb file the same as the mode of the .ipynb file. - * This is used for read-only status. - ipynb_file = @store.get('path') - locals = - ipynb_file_ro : undefined - syncdb_file_ro : undefined - syncdb_file = @syncdb.get_path() - async.parallel([ - (cb) -> - fs.access ipynb_file, fs.constants.W_OK, (err) -> - * Also store in @_ipynb_file_ro to prevent starting kernel in this case. - @_ipynb_file_ro = locals.ipynb_file_ro = !!err - cb() - (cb) -> - fs.access syncdb_file, fs.constants.W_OK, (err) -> - locals.syncdb_file_ro = !!err - cb() - ], -> - if locals.ipynb_file_ro == locals.syncdb_file_ro - return - dbg("mode change") - async.parallel([ - (cb) -> - fs.stat ipynb_file, (err, stats) -> - locals.ipynb_stats = stats - cb(err) - (cb) -> - * error if syncdb_file doesn't exist, which is GOOD, since - * in that case we do not want to chmod which would create - * that file as empty and blank it. - fs.stat(syncdb_file, cb) - ], (err) -> - if not err - dbg("changing syncb mode to match ipynb mode") - fs.chmod(syncdb_file, locals.ipynb_stats.mode) - else - dbg("error stating ipynb", err) - ) - ) - */ - - // Load file from disk if it is newer than - // the last we saved to disk. - private loadFromDiskIfNewer = async () => { - const dbg = this.dbg("loadFromDiskIfNewer"); - // Get mtime of last .ipynb file that we explicitly saved. - - // TODO: breaking the syncdb typescript data hiding. The - // right fix will be to move - // this info to a new ephemeral state table. - const last_ipynb_save = await this.get_last_ipynb_save(); - dbg(`syncdb last_ipynb_save=${last_ipynb_save}`); - let file_changed; - if (last_ipynb_save == 0) { - // we MUST load from file the first time, of course. - file_changed = true; - dbg("file changed because FIRST TIME"); - } else { - const path = this.store.get("path"); - let stats; - try { - stats = await callback2(this._client.path_stat, { path }); - dbg(`stats.mtime = ${stats.mtime}`); - } catch (err) { - // This err just means the file doesn't exist. - // We set the 'last load' to now in this case, since - // the frontend clients need to know that we - // have already scanned the disk. - this.set_last_load(); - return; - } - const mtime = stats.mtime.getTime(); - file_changed = mtime > last_ipynb_save; - dbg({ mtime, last_ipynb_save }); - } - if (file_changed) { - dbg(".ipynb disk file changed ==> loading state from disk"); - try { - await this.load_ipynb_file(); - } catch (err) { - dbg("failed to load on change", err); - } - } else { - dbg("disk file NOT changed: NOT loading"); - } - }; - - // if also set load is true, we also set the "last_ipynb_save" time. - set_last_load = (alsoSetLoad: boolean = false) => { - const last_load = new Date().getTime(); - this.syncdb.set({ - type: "file", - last_load, - }); - if (alsoSetLoad) { - // yes, load v save is inconsistent! - this.syncdb.set({ type: "settings", last_ipynb_save: last_load }); - } - this.syncdb.commit(); - }; - - /* Determine timestamp of aux .ipynb file, and record it here, - so we know that we do not have to load exactly that file - back from disk. */ - private set_last_ipynb_save = async () => { - let stats; - try { - stats = await callback2(this._client.path_stat, { - path: this.store.get("path"), - }); - } catch (err) { - // no-op -- nothing to do. - this.dbg("set_last_ipynb_save")(`WARNING -- issue in path_stat ${err}`); - return; - } - - // This is ugly (i.e., how we get access), but I need to get this done. - // This is the RIGHT place to save the info though. - // TODO: move this state info to new ephemeral table. - try { - const last_ipynb_save = stats.mtime.getTime(); - this.last_ipynb_save = last_ipynb_save; - this._set({ - type: "settings", - last_ipynb_save, - }); - this.dbg("stats.mtime.getTime()")( - `set_last_ipynb_save = ${last_ipynb_save}`, - ); - } catch (err) { - this.dbg("set_last_ipynb_save")( - `WARNING -- issue in set_last_ipynb_save ${err}`, - ); - return; - } - }; - - private get_last_ipynb_save = async () => { - const x = - this.syncdb.get_one({ type: "settings" })?.get("last_ipynb_save") ?? 0; - return Math.max(x, this.last_ipynb_save); - }; - - load_ipynb_file = async () => { - /* - Read the ipynb file from disk. Fully use the ipynb file to - set the syncdb's state. We do this when opening a new file, or when - the file changes on disk (e.g., a git checkout or something). - */ - const dbg = this.dbg(`load_ipynb_file`); - dbg("reading file"); - const path = this.store.get("path"); - let content: string; - try { - content = await callback2(this._client.path_read, { - path, - maxsize_MB: MAX_SIZE_IPYNB_MB, - }); - } catch (err) { - // possibly file doesn't exist -- set notebook to empty. - const exists = await callback2(this._client.path_exists, { - path, - }); - if (!exists) { - content = ""; - } else { - // It would be better to have a button to push instead of - // suggesting running a command in the terminal, but - // adding that took 1 second. Better than both would be - // making it possible to edit huge files :-). - const error = `Error reading ipynb file '${path}': ${err.toString()}. Fix this to continue. You can delete all output by typing cc-jupyter-no-output [filename].ipynb in a terminal.`; - this.syncdb.set({ type: "fatal", error }); - throw Error(error); - } - } - if (content.length === 0) { - // Blank file, e.g., when creating in CoCalc. - // This is good, works, etc. -- just clear state, including error. - this.syncdb.delete(); - this.set_last_load(true); - return; - } - - // File is nontrivial -- parse and load. - let parsed_content; - try { - parsed_content = JSON.parse(content); - } catch (err) { - const error = `Error parsing the ipynb file '${path}': ${err}. You must fix the ipynb file somehow before continuing, or use TimeTravel to revert to a recent version.`; - dbg(error); - this.syncdb.set({ type: "fatal", error }); - throw Error(error); - } - this.syncdb.delete({ type: "fatal" }); - await this.set_to_ipynb(parsed_content); - this.set_last_load(true); - }; - - private fetch_jupyter_kernels = async () => { - const data = await get_kernel_data(); - const kernels = immutable.fromJS(data as any); - this.setState({ kernels }); - }; - - save_ipynb_file = async ({ - version = 0, - timeout = 15000, - }: { - // if version is given, waits (up to timeout ms) for syncdb to - // contain that exact version before writing the ipynb to disk. - // This may be needed to ensure that ipynb saved to disk - // reflects given frontend state. This comes up, e.g., in - // generating the nbgrader version of a document. - version?: number; - timeout?: number; - } = {}) => { - const dbg = this.dbg("save_ipynb_file"); - if (version && !this.syncdb.hasVersion(version)) { - dbg(`frontend needs ${version}, which we do not yet have`); - const start = Date.now(); - while (true) { - if (this.is_closed()) { - return; - } - if (Date.now() - start >= timeout) { - dbg("timed out waiting"); - break; - } - try { - dbg(`waiting for version ${version}`); - await once(this.syncdb, "change", timeout - (Date.now() - start)); - } catch { - dbg("timed out waiting"); - break; - } - if (this.syncdb.hasVersion(version)) { - dbg("now have the version"); - break; - } - } - } - if (this.is_closed()) { - return; - } - dbg("saving to file"); - - // Check first if file was deleted, in which case instead of saving to disk, - // we should terminate and clean up everything. - if (this.isDeleted()) { - dbg("ipynb file is deleted, so NOT saving to disk and closing"); - this.close({ noSave: true }); - return; - } - - if (this.jupyter_kernel == null) { - // The kernel is needed to get access to the blob store, which - // may be needed to save to disk. - this.ensure_backend_kernel_setup(); - if (this.jupyter_kernel == null) { - // still not null? This would happen if no kernel is set at all, - // in which case it's OK that saving isn't possible. - throw Error("no kernel so cannot save"); - } - } - if (this.store.get("kernels") == null) { - await this.init_kernel_info(); - if (this.store.get("kernels") == null) { - // This should never happen, but maybe could in case of a very - // messed up compute environment where the kernelspecs can't be listed. - throw Error( - "kernel info not known and can't be determined, so can't save", - ); - } - } - dbg("going to try to save: getting ipynb object..."); - const blob_store = this.jupyter_kernel.get_blob_store(); - let ipynb = this.store.get_ipynb(blob_store); - if (this.store.get("kernel")) { - // if a kernel is set, check that it was sufficiently known that - // we can fill in data about it -- - // see https://github.com/sagemathinc/cocalc/issues/7286 - if (ipynb?.metadata?.kernelspec?.name == null) { - dbg("kernelspec not known -- try loading kernels again"); - await this.fetch_jupyter_kernels(); - // and again grab the ipynb - ipynb = this.store.get_ipynb(blob_store); - if (ipynb?.metadata?.kernelspec?.name == null) { - dbg("kernelspec STILL not known: metadata will be incomplete"); - } - } - } - dbg("got ipynb object"); - // We use json_stable (and indent 1) to be more diff friendly to user, - // and more consistent with official Jupyter. - const data = json_stable(ipynb, { space: 1 }); - if (data == null) { - dbg("failed -- ipynb not defined yet"); - throw Error("ipynb not defined yet; can't save"); - } - dbg("converted ipynb to stable JSON string", data?.length); - //dbg(`got string version '${data}'`) - try { - dbg("writing to disk..."); - await callback2(this._client.write_file, { - path: this.store.get("path"), - data, - }); - dbg("succeeded at saving"); - await this.set_last_ipynb_save(); - } catch (err) { - const e = `error writing file: ${err}`; - dbg(e); - throw Error(e); - } - }; - - ensure_there_is_a_cell = () => { - if (this._state !== "ready") { - return; - } - const cells = this.store.get("cells"); - if (cells == null || cells.size === 0) { - this._set({ - type: "cell", - id: this.new_id(), - pos: 0, - input: "", - }); - // We are obviously contributing content to this (empty!) notebook. - return this.set_trust_notebook(true); - } - }; - - private handle_all_cell_attachments() { - // Check if any cell attachments need to be loaded. - const cells = this.store.get("cells"); - cells?.forEach((cell) => { - this.handle_cell_attachments(cell); - }); - } - - private handle_cell_attachments(cell) { - if (this.jupyter_kernel == null) { - // can't do anything - return; - } - const dbg = this.dbg(`handle_cell_attachments(id=${cell.get("id")})`); - dbg(); - - const attachments = cell.get("attachments"); - if (attachments == null) return; // nothing to do - attachments.forEach(async (x, name) => { - if (x == null) return; - if (x.get("type") === "load") { - if (this.jupyter_kernel == null) return; // try later - // need to load from disk - this.set_cell_attachment(cell.get("id"), name, { - type: "loading", - value: null, - }); - let sha1: string; - try { - sha1 = await this.jupyter_kernel.load_attachment(x.get("value")); - } catch (err) { - this.set_cell_attachment(cell.get("id"), name, { - type: "error", - value: `${err}`, - }); - return; - } - this.set_cell_attachment(cell.get("id"), name, { - type: "sha1", - value: sha1, - }); - } - }); - } - - // handle_ipywidgets_state_change is called when the project ipywidgets_state - // object changes, e.g., in response to a user moving a slider in the browser. - // It crafts a comm message that is sent to the running Jupyter kernel telling - // it about this change by calling send_comm_message_to_kernel. - private handle_ipywidgets_state_change = (keys): void => { - if (this.is_closed()) { - return; - } - const dbg = this.dbg("handle_ipywidgets_state_change"); - dbg(keys); - if (this.jupyter_kernel == null) { - dbg("no kernel, so ignoring changes to ipywidgets"); - return; - } - if (this.syncdb.ipywidgets_state == null) { - throw Error("syncdb's ipywidgets_state must be defined!"); - } - for (const key of keys) { - const [, model_id, type] = JSON.parse(key); - dbg({ key, model_id, type }); - let data: any; - if (type === "value") { - const state = this.syncdb.ipywidgets_state.get_model_value(model_id); - // Saving the buffers on change is critical since otherwise this breaks: - // https://ipywidgets.readthedocs.io/en/latest/examples/Widget%20List.html#file-upload - // Note that stupidly the buffer (e.g., image upload) gets sent to the kernel twice. - // But it does work robustly, and the kernel and nodejs server processes next to each - // other so this isn't so bad. - const { buffer_paths, buffers } = - this.syncdb.ipywidgets_state.getKnownBuffers(model_id); - data = { method: "update", state, buffer_paths }; - this.jupyter_kernel.send_comm_message_to_kernel({ - msg_id: misc.uuid(), - target_name: "jupyter.widget", - comm_id: model_id, - data, - buffers, - }); - } else if (type === "buffers") { - // TODO: we MIGHT need implement this... but MAYBE NOT. An example where this seems like it might be - // required is by the file upload widget, but actually that just uses the value type above, since - // we explicitly fill in the widgets there; also there is an explicit comm upload message that - // the widget sends out that updates the buffer, and in send_comm_message_to_kernel in jupyter/kernel/kernel.ts - // when processing that message, we saves those buffers and make sure they are set in the - // value case above (otherwise they would get removed). - // https://ipywidgets.readthedocs.io/en/latest/examples/Widget%20List.html#file-upload - // which creates a buffer from the content of the file, then sends it to the backend, - // which sees a change and has to write that buffer to the kernel (here) so that - // the running python process can actually do something with the file contents (e.g., - // process data, save file to disk, etc). - // We need to be careful though to not send buffers to the kernel that the kernel sent us, - // since that would be a waste. - } else if (type === "state") { - // TODO: currently ignoring this, since it seems chatty and pointless, - // and could lead to race conditions probably with multiple users, etc. - // It happens right when the widget is created. - /* - const state = this.syncdb.ipywidgets_state.getModelSerializedState(model_id); - data = { method: "update", state }; - this.jupyter_kernel.send_comm_message_to_kernel( - misc.uuid(), - model_id, - data - ); - */ - } else { - const m = `Jupyter: unknown type '${type}'`; - console.warn(m); - dbg(m); - } - } - }; - - async process_comm_message_from_kernel(mesg: any): Promise { - const dbg = this.dbg("process_comm_message_from_kernel"); - // serializing the full message could cause enormous load on the server, since - // the mesg may contain large buffers. Only do for low level debugging! - // dbg(mesg); // EXTREME DANGER! - // This should be safe: - dbg(JSON.stringify(mesg.header)); - if (this.syncdb.ipywidgets_state == null) { - throw Error("syncdb's ipywidgets_state must be defined!"); - } - await this.syncdb.ipywidgets_state.process_comm_message_from_kernel(mesg); - } - - capture_output_message(mesg: any): boolean { - if (this.syncdb.ipywidgets_state == null) { - throw Error("syncdb's ipywidgets_state must be defined!"); - } - return this.syncdb.ipywidgets_state.capture_output_message(mesg); - } - - close_project_only() { - const dbg = this.dbg("close_project_only"); - dbg(); - if (this.run_all_loop) { - this.run_all_loop.close(); - delete this.run_all_loop; - } - // this stops the kernel and cleans everything up - // so no resources are wasted and next time starting - // is clean - (async () => { - try { - await removeJupyterRedux(this.store.get("path"), this.project_id); - } catch (err) { - dbg("WARNING -- issue removing jupyter redux", err); - } - })(); - - this.blobs?.close(); - } - // not actually async... - async signal(signal = "SIGINT"): Promise { + signal = async (signal = "SIGINT"): Promise => { this.jupyter_kernel?.signal(signal); - } - - handle_nbconvert_change(oldVal, newVal): void { - nbconvertChange(this, oldVal?.toJS(), newVal?.toJS()); - } - - // Handle transient cell messages. - handleTransientUpdate = (mesg) => { - const display_id = mesg.content?.transient?.display_id; - if (!display_id) { - return false; - } - - let matched = false; - // are there any transient outputs in the entire document that - // have this display_id? search to find them. - // TODO: we could use a clever data structure to make - // this faster and more likely to have bugs. - const cells = this.syncdb.get({ type: "cell" }); - for (let cell of cells) { - let output = cell.get("output"); - if (output != null) { - for (const [n, val] of output) { - if (val.getIn(["transient", "display_id"]) == display_id) { - // found a match -- replace it - output = output.set(n, immutable.fromJS(mesg.content)); - this.syncdb.set({ type: "cell", id: cell.get("id"), output }); - matched = true; - } - } - } - } - if (matched) { - this.syncdb.commit(); - } - }; - - getComputeServers = () => { - // we don't bother worrying about freeing this since it is only - // run in the project or compute server, which needs the underlying - // dkv for its entire lifetime anyways. - if (this.computeServers == null) { - this.computeServers = computeServerManager({ - project_id: this.project_id, - }); - } - return this.computeServers; - }; - - getComputeServerIdSync = (): number => { - const c = this.getComputeServers(); - return c.get(this.syncdb.path) ?? 0; - }; - - getComputeServerId = async (): Promise => { - const c = this.getComputeServers(); - return (await c.getServerIdForPath(this.syncdb.path)) ?? 0; }; } diff --git a/src/packages/jupyter/redux/run-all-loop.ts b/src/packages/jupyter/redux/run-all-loop.ts deleted file mode 100644 index 472aa42ab36..00000000000 --- a/src/packages/jupyter/redux/run-all-loop.ts +++ /dev/null @@ -1,63 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -import { delay } from "awaiting"; - -import { close } from "@cocalc/util/misc"; -import { JupyterActions } from "./project-actions"; - -export class RunAllLoop { - private actions: JupyterActions; - public interval_s: number; - private closed: boolean = false; - private dbg: Function; - - constructor(actions, interval_s) { - this.actions = actions; - this.interval_s = interval_s; - this.dbg = actions.dbg("RunAllLoop"); - this.dbg(`interval_s=${interval_s}`); - this.loop(); - } - - public set_interval(interval_s: number): void { - if (this.closed) { - throw Error("should not call set_interval if RunAllLoop is closed"); - } - if (this.interval_s == interval_s) return; - this.dbg(`.set_interval: interval_s=${interval_s}`); - this.interval_s = interval_s; - } - - private async loop(): Promise { - this.dbg("starting loop..."); - while (true) { - if (this.closed) break; - try { - this.dbg("loop: restart"); - await this.actions.restart(); - } catch (err) { - this.dbg(`restart failed (will try run-all anyways) - ${err}`); - } - if (this.closed) break; - try { - this.dbg("loop: run_all_cells"); - await this.actions.run_all_cells(true); - } catch (err) { - this.dbg(`run_all_cells failed - ${err}`); - } - if (this.closed) break; - this.dbg(`loop: waiting ${this.interval_s} seconds`); - await delay(this.interval_s * 1000); - } - this.dbg("terminating loop..."); - } - - public close() { - this.dbg("close"); - close(this); - this.closed = true; - } -} diff --git a/src/packages/jupyter/redux/store.ts b/src/packages/jupyter/redux/store.ts index 1d62316bdfd..c257d076410 100644 --- a/src/packages/jupyter/redux/store.ts +++ b/src/packages/jupyter/redux/store.ts @@ -60,7 +60,6 @@ export interface JupyterStoreState { cm_options: any; complete: any; confirm_dialog: any; - connection_file?: string; contents?: List>; // optional global contents info (about sections, problems, etc.) default_kernel?: string; directory: string; @@ -103,6 +102,13 @@ export interface JupyterStoreState { // run progress = Percent (0-100) of runnable cells that have been run since the last // kernel restart. (Thus markdown and empty cells are excluded.) runProgress?: number; + + // cells that this particular client has queued up to run. This is + // only known to this client, goes away on browser refresh, and is used + // only visually for the user to see. + pendingCells: Set; + + stdin?: { id: string; prompt: string; password?: boolean }; } export const initial_jupyter_store_state: { @@ -446,10 +452,10 @@ export class JupyterStore extends Store { // (??) return `${project_id}-${computeServerId}-default`; } - const dflt_img = await customize.getDefaultComputeImage(); + const defaultImage = await customize.getDefaultComputeImage(); const compute_image = projects_store.getIn( ["project_map", project_id, "compute_image"], - dflt_img, + defaultImage, ); const key = [project_id, `${computeServerId}`, compute_image].join("::"); // console.log("jupyter store / jupyter_kernel_key", key); diff --git a/src/packages/jupyter/redux/sync.ts b/src/packages/jupyter/redux/sync.ts index d36ef1a9710..b94935c1960 100644 --- a/src/packages/jupyter/redux/sync.ts +++ b/src/packages/jupyter/redux/sync.ts @@ -1,6 +1,6 @@ export const SYNCDB_OPTIONS = { - change_throttle: 50, // our UI/React can handle more rapid updates; plus we want output FAST. - patch_interval: 50, + change_throttle: 25, + patch_interval: 25, primary_keys: ["type", "id"], string_cols: ["input"], cursors: true, diff --git a/src/packages/jupyter/types/project-interface.ts b/src/packages/jupyter/types/project-interface.ts index 009f4b5226c..986d16cd284 100644 --- a/src/packages/jupyter/types/project-interface.ts +++ b/src/packages/jupyter/types/project-interface.ts @@ -76,7 +76,7 @@ export interface ExecOpts { timeout_ms?: number; } -export type OutputMessage = object; // todo +export type OutputMessage = any; // todo export interface CodeExecutionEmitterInterface extends EventEmitterInterface { emit_output(result: OutputMessage): void; @@ -97,7 +97,9 @@ export interface JupyterKernelInterface extends EventEmitterInterface { name: string | undefined; // name = undefined implies it is not spawnable. It's a notebook with no actual jupyter kernel process. store: any; readonly identity: string; + failedError: string; + isClosed(): boolean; get_state(): string; signal(signal: string): void; close(): void; @@ -125,7 +127,7 @@ export interface JupyterKernelInterface extends EventEmitterInterface { buffers?: any[]; buffers64?: any[]; }): void; - get_connection_file(): string | undefined; + getConnectionFile(): string | undefined; _execute_code_queue: CodeExecutionEmitterInterface[]; clear_execute_code_queue(): void; diff --git a/src/packages/jupyter/util/cell-utils.ts b/src/packages/jupyter/util/cell-utils.ts index ecc59eef87a..e6d68ac9b3b 100644 --- a/src/packages/jupyter/util/cell-utils.ts +++ b/src/packages/jupyter/util/cell-utils.ts @@ -13,7 +13,7 @@ import { field_cmp, len } from "@cocalc/util/misc"; export function positions_between( before_pos: number | undefined, after_pos: number | undefined, - num: number + num: number, ) { // Return an array of num equally spaced positions starting after // before_pos and ending before after_pos, so @@ -66,22 +66,22 @@ export function sorted_cell_list(cells: Map): List { .toList(); } -export function ensure_positions_are_unique(cells?: Map) { +export function ensurePositionsAreUnique(cells?: Map) { // Verify that pos's of cells are distinct. If not // return map from id's to new unique positions. if (cells == null) { return; } - const v: any = {}; + const v = new Set(); let all_unique = true; cells.forEach((cell) => { const pos = cell.get("pos"); - if (pos == null || v[pos]) { + if (pos == null || v.has(pos)) { // dup! (or not defined) all_unique = false; return false; } - v[pos] = true; + v.add(pos); }); if (all_unique) { return; @@ -99,7 +99,7 @@ export function new_cell_pos( cells: Map, cell_list: List, cur_id: string, - delta: -1 | 1 + delta: -1 | 1, ): number { /* Returns pos for a new cell whose position @@ -145,7 +145,7 @@ export function new_cell_pos( export function move_selected_cells( v?: string[], selected?: { [id: string]: true }, - delta?: number + delta?: number, ) { /* - v = ordered js array of all cell id's diff --git a/src/packages/jupyter/zmq/index.ts b/src/packages/jupyter/zmq/index.ts index 3a18b162ba5..632a3e5ff0f 100644 --- a/src/packages/jupyter/zmq/index.ts +++ b/src/packages/jupyter/zmq/index.ts @@ -1,10 +1,10 @@ import { EventEmitter } from "events"; import { Dealer, Subscriber } from "zeromq"; import { Message } from "./message"; -import { getLogger } from "@cocalc/backend/logger"; import type { JupyterMessage } from "./types"; -const logger = getLogger("jupyter:zmq"); +//import { getLogger } from "@cocalc/backend/logger"; +//const logger = getLogger("jupyter:zmq"); type JupyterSocketName = "iopub" | "shell" | "stdin" | "control"; @@ -76,7 +76,7 @@ export class JupyterSockets extends EventEmitter { throw Error(`invalid socket name '${name}'`); } - logger.debug("send message", message); + //logger.debug("send message", message); const jMessage = new Message(message); socket.send( jMessage._encode( @@ -119,9 +119,9 @@ export class JupyterSockets extends EventEmitter { private listen = async (name: JupyterSocketName, socket) => { if (ZMQ_TYPE[name] == "sub") { - // subscribe to everything -- + // subscribe to everything -- // https://zeromq.github.io/zeromq.js/classes/Subscriber.html#subscribe - socket.subscribe(); + socket.subscribe(); } for await (const data of socket) { const mesg = Message._decode( diff --git a/src/packages/next/components/store/quota-config.tsx b/src/packages/next/components/store/quota-config.tsx index 005a21cec61..13845ec3e06 100644 --- a/src/packages/next/components/store/quota-config.tsx +++ b/src/packages/next/components/store/quota-config.tsx @@ -17,7 +17,6 @@ import { Typography, } from "antd"; import { useEffect, useRef, useState, type JSX } from "react"; - import { HelpIcon } from "@cocalc/frontend/components/help-icon"; import { Icon } from "@cocalc/frontend/components/icon"; import { displaySiteLicense } from "@cocalc/util/consts/site-license"; @@ -199,7 +198,7 @@ export const QuotaConfig: React.FC = (props: Props) => { = (props: Props) => { = (props: Props) => { diff --git a/src/packages/next/components/store/quota-query-params.ts b/src/packages/next/components/store/quota-query-params.ts index 92e85f30aa9..c492089e6f8 100644 --- a/src/packages/next/components/store/quota-query-params.ts +++ b/src/packages/next/components/store/quota-query-params.ts @@ -138,10 +138,10 @@ function decodeValue(val): boolean | number | string | DateRange { function fixNumVal( val: any, - param: { min: number; max: number; dflt: number }, + param: { min: number; max: number; default: number }, ): number { if (typeof val !== "number") { - return param.dflt; + return param.default; } else { return clamp(val, param.min, param.max); } diff --git a/src/packages/next/components/store/site-license.tsx b/src/packages/next/components/store/site-license.tsx index 1b1f99f858d..4f46d568d82 100644 --- a/src/packages/next/components/store/site-license.tsx +++ b/src/packages/next/components/store/site-license.tsx @@ -292,14 +292,14 @@ function CreateSiteLicense({ })(); } else { const vals = decodeFormValues(router, "regular"); - const dflt = presets[DEFAULT_PRESET]; + const defaultPreset = presets[DEFAULT_PRESET]; // Only use the configuration fields from the default preset, not the entire object const defaultConfig = { - cpu: dflt.cpu, - ram: dflt.ram, - disk: dflt.disk, - uptime: dflt.uptime, - member: dflt.member, + cpu: defaultPreset.cpu, + ram: defaultPreset.ram, + disk: defaultPreset.disk, + uptime: defaultPreset.uptime, + member: defaultPreset.member, // Add other form fields that might be needed period: source === "course" ? "range" : "monthly", user: source === "course" ? "academic" : "business", diff --git a/src/packages/next/pages/api/v2/projects/copy-path.ts b/src/packages/next/pages/api/v2/projects/copy-path.ts index 6d9390e1bb9..d38a53b6550 100644 --- a/src/packages/next/pages/api/v2/projects/copy-path.ts +++ b/src/packages/next/pages/api/v2/projects/copy-path.ts @@ -60,8 +60,10 @@ export default async function handle(req, res) { throw Error("must be a collaborator on source project"); } } + throw Error("TODO: reimplement copyPath"); const project = getProject(src_project_id); - await project.copyPath({ + console.log({ + project, path, target_project_id, target_path, diff --git a/src/packages/package.json b/src/packages/package.json index a266b29591a..4c1e285ecf7 100644 --- a/src/packages/package.json +++ b/src/packages/package.json @@ -34,10 +34,10 @@ "axios@1.10.0": "^1.11.0" }, "onlyBuiltDependencies": [ + "@vscode/ripgrep", + "better-sqlite3", "websocket-sftp", "websocketfs", - "zeromq", - "better-sqlite3", "zstd-napi" ] } diff --git a/src/packages/pnpm-lock.yaml b/src/packages/pnpm-lock.yaml index bb1d8582893..3d7a0970fcf 100644 --- a/src/packages/pnpm-lock.yaml +++ b/src/packages/pnpm-lock.yaml @@ -36,13 +36,13 @@ importers: version: 5.0.1 jest: specifier: ^29.7.0 - version: 29.7.0(@types/node@24.2.1) + version: 29.7.0(@types/node@18.19.122) jest-junit: specifier: ^16.0.0 version: 16.0.0 ts-jest: specifier: ^29.2.3 - version: 29.4.1(@babel/core@7.28.0)(@jest/transform@29.7.0)(@jest/types@30.0.5)(babel-jest@29.7.0(@babel/core@7.28.0))(jest-util@30.0.5)(jest@29.7.0(@types/node@24.2.1))(typescript@5.9.2) + version: 29.4.1(@babel/core@7.28.0)(@jest/transform@29.7.0)(@jest/types@30.0.5)(babel-jest@29.7.0(@babel/core@7.28.0))(jest-util@30.0.5)(jest@29.7.0(@types/node@18.19.122))(typescript@5.9.2) typescript: specifier: ^5.7.3 version: 5.9.2 @@ -90,18 +90,12 @@ importers: '@cocalc/util': specifier: workspace:* version: link:../util - '@types/debug': - specifier: ^4.1.12 - version: 4.1.12 - '@types/jest': - specifier: ^29.5.14 - version: 29.5.14 awaiting: specifier: ^3.0.0 version: 3.0.0 better-sqlite3: - specifier: ^11.10.0 - version: 11.10.0 + specifier: ^12.2.0 + version: 12.2.0 chokidar: specifier: ^3.6.0 version: 3.6.0 @@ -109,7 +103,7 @@ importers: specifier: ^4.4.0 version: 4.4.1 fs-extra: - specifier: ^11.2.0 + specifier: ^11.3.1 version: 11.3.1 lodash: specifier: ^4.17.21 @@ -136,6 +130,12 @@ importers: specifier: ^0.0.10 version: 0.0.10 devDependencies: + '@types/debug': + specifier: ^4.1.12 + version: 4.1.12 + '@types/jest': + specifier: ^29.5.14 + version: 29.5.14 '@types/node': specifier: ^18.16.14 version: 18.19.122 @@ -173,6 +173,9 @@ importers: '@cocalc/conat': specifier: workspace:* version: 'link:' + '@cocalc/sync': + specifier: workspace:* + version: link:../sync '@cocalc/util': specifier: workspace:* version: link:../util @@ -211,17 +214,14 @@ importers: version: 4.17.21 socket.io: specifier: ^4.8.1 - version: 4.8.1 + version: 4.8.1(utf-8-validate@6.0.5) socket.io-client: specifier: ^4.8.1 - version: 4.8.1 + version: 4.8.1(utf-8-validate@6.0.5) devDependencies: '@types/better-sqlite3': specifier: ^7.6.13 version: 7.6.13 - '@types/json-stable-stringify': - specifier: ^1.0.32 - version: 1.2.0 '@types/lodash': specifier: ^4.14.202 version: 4.17.20 @@ -373,13 +373,13 @@ importers: version: 1.4.1 '@jupyter-widgets/base': specifier: ^4.1.1 - version: 4.1.7(react@19.1.1) + version: 4.1.7(react@19.1.1)(utf-8-validate@6.0.5) '@jupyter-widgets/controls': specifier: 5.0.0-rc.2 - version: 5.0.0-rc.2(crypto@1.0.1)(react@19.1.1) + version: 5.0.0-rc.2(crypto@1.0.1)(react@19.1.1)(utf-8-validate@6.0.5) '@jupyter-widgets/output': specifier: ^4.1.0 - version: 4.1.7(react@19.1.1) + version: 4.1.7(react@19.1.1)(utf-8-validate@6.0.5) '@microlink/react-json-view': specifier: ^1.23.3 version: 1.27.0(@types/react@19.1.10)(react-dom@19.1.1(react@19.1.1))(react@19.1.1) @@ -511,7 +511,7 @@ importers: version: 0.2.0 jest-environment-jsdom: specifier: ^30.0.2 - version: 30.0.5 + version: 30.0.5(utf-8-validate@6.0.5) jquery: specifier: ^3.6.0 version: 3.7.1 @@ -853,6 +853,9 @@ importers: debug: specifier: ^4.4.0 version: 4.4.1 + events: + specifier: 3.3.0 + version: 3.3.0 expect: specifier: ^26.6.2 version: 26.6.2 @@ -896,9 +899,6 @@ importers: specifier: ^6.4.2 version: 6.5.0 devDependencies: - '@types/json-stable-stringify': - specifier: ^1.0.32 - version: 1.2.0 '@types/node': specifier: ^18.16.14 version: 18.19.122 @@ -1139,13 +1139,13 @@ importers: version: 8.3.2 websocket-sftp: specifier: ^0.8.4 - version: 0.8.4 + version: 0.8.4(utf-8-validate@6.0.5) which: specifier: ^2.0.2 version: 2.0.2 ws: specifier: ^8.18.0 - version: 8.18.3 + version: 8.18.3(utf-8-validate@6.0.5) devDependencies: '@types/body-parser': specifier: ^1.19.5 @@ -1177,6 +1177,9 @@ importers: '@cocalc/database': specifier: workspace:* version: link:../database + '@cocalc/file-server': + specifier: workspace:* + version: link:../file-server '@cocalc/gcloud-pricing-calculator': specifier: ^1.17.0 version: 1.17.0 @@ -1209,22 +1212,22 @@ importers: version: 1.4.1 '@langchain/anthropic': specifier: ^0.3.26 - version: 0.3.26(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3)(zod@3.25.76))) + version: 0.3.26(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76))) '@langchain/core': specifier: ^0.3.68 - version: 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3)(zod@3.25.76)) + version: 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)) '@langchain/google-genai': specifier: ^0.2.16 - version: 0.2.16(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3)(zod@3.25.76))) + version: 0.2.16(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76))) '@langchain/mistralai': specifier: ^0.2.1 - version: 0.2.1(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3)(zod@3.25.76)))(zod@3.25.76) + version: 0.2.1(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)))(zod@3.25.76) '@langchain/ollama': specifier: ^0.2.3 - version: 0.2.3(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3)(zod@3.25.76))) + version: 0.2.3(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76))) '@langchain/openai': specifier: ^0.6.6 - version: 0.6.7(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3)(zod@3.25.76)))(ws@8.18.3) + version: 0.6.7(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)))(ws@8.18.3(utf-8-validate@6.0.5)) '@node-saml/passport-saml': specifier: ^5.1.0 version: 5.1.0 @@ -1255,9 +1258,6 @@ importers: async: specifier: ^1.5.2 version: 1.5.2 - await-spawn: - specifier: ^4.0.2 - version: 4.0.2 awaiting: specifier: ^3.0.0 version: 3.0.0 @@ -1335,7 +1335,7 @@ importers: version: 6.10.1 openai: specifier: ^5.12.1 - version: 5.12.2(ws@8.18.3)(zod@3.25.76) + version: 5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76) parse-domain: specifier: ^5.0.0 version: 5.0.0(encoding@0.1.13) @@ -1716,7 +1716,7 @@ importers: version: 8.0.9 ws: specifier: ^8.18.0 - version: 8.18.3 + version: 8.18.3(utf-8-validate@6.0.5) devDependencies: '@types/cookie': specifier: ^0.6.0 @@ -1768,6 +1768,40 @@ importers: specifier: ^18.16.14 version: 18.19.122 + test: + dependencies: + '@cocalc/backend': + specifier: workspace:* + version: link:../backend + '@cocalc/conat': + specifier: workspace:* + version: link:../conat + '@cocalc/frontend': + specifier: workspace:* + version: link:../frontend + '@cocalc/util': + specifier: workspace:* + version: link:../util + devDependencies: + '@testing-library/jest-dom': + specifier: ^6.6.3 + version: 6.6.4 + '@testing-library/react': + specifier: ^16.3.0 + version: 16.3.0(@testing-library/dom@10.4.0)(@types/react-dom@19.1.7(@types/react@19.1.10))(@types/react@19.1.10)(react-dom@19.1.1(react@19.1.1))(react@19.1.1) + '@types/debug': + specifier: ^4.1.12 + version: 4.1.12 + '@types/jest': + specifier: ^29.5.14 + version: 29.5.14 + '@types/node': + specifier: ^18.16.14 + version: 18.19.122 + jest-environment-jsdom: + specifier: ^30.0.2 + version: 30.0.4(utf-8-validate@6.0.5) + util: dependencies: '@ant-design/colors': @@ -1846,9 +1880,6 @@ importers: specifier: ^1.3.0 version: 1.3.0 devDependencies: - '@types/json-stable-stringify': - specifier: ^1.0.32 - version: 1.2.0 '@types/lodash': specifier: ^4.14.202 version: 4.17.20 @@ -2874,6 +2905,16 @@ packages: node-notifier: optional: true + '@jest/environment-jsdom-abstract@30.0.4': + resolution: {integrity: sha512-pUKfqgr5Nki9kZ/3iV+ubDsvtPq0a0oNL6zqkKLM1tPQI8FBJeuWskvW1kzc5pOvqlgpzumYZveJ4bxhANY0hg==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + peerDependencies: + canvas: ^3.0.0 + jsdom: '*' + peerDependenciesMeta: + canvas: + optional: true + '@jest/environment-jsdom-abstract@30.0.5': resolution: {integrity: sha512-gpWwiVxZunkoglP8DCnT3As9x5O8H6gveAOpvaJd2ATAoSh7ZSSCWbr9LQtUMvr8WD3VjG9YnDhsmkCK5WN1rQ==} engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} @@ -2888,6 +2929,10 @@ packages: resolution: {integrity: sha512-aQIfHDq33ExsN4jP1NWGXhxgQ/wixs60gDiKO+XVMd8Mn0NWPWgc34ZQDTb2jKaUWQ7MuwoitXAsN2XVXNMpAw==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + '@jest/environment@30.0.4': + resolution: {integrity: sha512-5NT+sr7ZOb8wW7C4r7wOKnRQ8zmRWQT2gW4j73IXAKp5/PX1Z8MCStBLQDYfIG3n1Sw0NRfYGdp0iIPVooBAFQ==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + '@jest/environment@30.0.5': resolution: {integrity: sha512-aRX7WoaWx1oaOkDQvCWImVQ8XNtdv5sEWgk4gxR6NXb7WBUnL5sRak4WRzIQRZ1VTWPvV4VI4mgGjNL9TeKMYA==} engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} @@ -2904,6 +2949,10 @@ packages: resolution: {integrity: sha512-q4DH1Ha4TTFPdxLsqDXK1d3+ioSL7yL5oCMJZgDYm6i+6CygW5E5xVr/D1HdsGxjt1ZWSfUAs9OxSB/BNelWrQ==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + '@jest/fake-timers@30.0.4': + resolution: {integrity: sha512-qZ7nxOcL5+gwBO6LErvwVy5k06VsX/deqo2XnVUSTV0TNC9lrg8FC3dARbi+5lmrr5VyX5drragK+xLcOjvjYw==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + '@jest/fake-timers@30.0.5': resolution: {integrity: sha512-ZO5DHfNV+kgEAeP3gK3XlpJLL4U3Sz6ebl/n68Uwt64qFFs5bv4bfEEjyRGK5uM0C90ewooNgFuKMdkbEoMEXw==} engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} @@ -2929,6 +2978,10 @@ packages: resolution: {integrity: sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + '@jest/schemas@30.0.1': + resolution: {integrity: sha512-+g/1TKjFuGrf1Hh0QPCv0gISwBxJ+MQSNXmG9zjHy7BmFhtoJ9fdNhWJp3qUKRi93AOZHXtdxZgJ1vAtz6z65w==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + '@jest/schemas@30.0.5': resolution: {integrity: sha512-DmdYgtezMkh3cpU8/1uyXakv3tJRcmcXxBOcO0tbaozPwpmh4YMsnWrQm9ZmZMfa5ocbxzbFk6O4bDPEc/iAnA==} engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} @@ -2957,6 +3010,10 @@ packages: resolution: {integrity: sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + '@jest/types@30.0.1': + resolution: {integrity: sha512-HGwoYRVF0QSKJu1ZQX0o5ZrUrrhj0aOOFA8hXrumD7SIzjouevhawbTjmXdwOmURdGluU9DM/XvGm3NyFoiQjw==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + '@jest/types@30.0.5': resolution: {integrity: sha512-aREYa3aku9SSnea4aX6bhKn4bgv3AXkgijoQgbYV3yvbiGt6z+MQ85+6mIhx9DsKW2BuB/cLR/A+tcMThx+KLQ==} engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} @@ -4289,10 +4346,6 @@ packages: '@types/json-schema@7.0.15': resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} - '@types/json-stable-stringify@1.2.0': - resolution: {integrity: sha512-PEHY3ohqolHqAzDyB1+31tFaAMnoLN7x/JgdcGmNZ2uvtEJ6rlFCUYNQc0Xe754xxCYLNGZbLUGydSE6tS4S9A==} - deprecated: This is a stub types definition. json-stable-stringify provides its own type definitions, so you do not need this installed. - '@types/katex@0.16.7': resolution: {integrity: sha512-HMwFiRujE5PjrgwHQ25+bsLJgowjGjm5Z8FVSf0N6PwgJrwxH0QxzHYDcKsTfV3wva0vzrpqMTJS2jXPr5BMEQ==} @@ -4951,10 +5004,6 @@ packages: resolution: {integrity: sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ==} engines: {node: '>= 0.4'} - await-spawn@4.0.2: - resolution: {integrity: sha512-GdADmeLJiMvGKJD3xWBcX40DMn07JNH1sqJYgYJZH7NTGJ3B1qDjKBKzxhhyR1hjIcnUGFUmE/+4D1HcHAJBAA==} - engines: {node: '>=10'} - awaiting@3.0.0: resolution: {integrity: sha512-19i4G7Hjxj9idgMlAM0BTRII8HfvsOdlr4D9cf3Dm1MZhvcKjBpzY8AMNEyIKyi+L9TIK15xZatmdcPG003yww==} engines: {node: '>=7.6.x'} @@ -5042,8 +5091,9 @@ packages: batch@0.6.1: resolution: {integrity: sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==} - better-sqlite3@11.10.0: - resolution: {integrity: sha512-EwhOpyXiOEL/lKzHz9AW1msWFNzGc/z+LzeB3/jnFJpxu+th2yqvzsSWas1v9jgs9+xiXJcD5A8CJxAG2TaghQ==} + better-sqlite3@12.2.0: + resolution: {integrity: sha512-eGbYq2CT+tos1fBwLQ/tkBt9J5M3JEHjku4hbvQUePCckkvVf14xWj+1m7dGoK81M/fOjFT7yM9UMeKT/+vFLQ==} + engines: {node: 20.x || 22.x || 23.x || 24.x} big.js@3.2.0: resolution: {integrity: sha512-+hN/Zh2D08Mx65pZ/4g5bsmNiZUuChDiQfTUQ7qJr4/kuopCr88xZsAXv6mBoZEsUI4OuGHlX59qE94K2mMW8Q==} @@ -7844,6 +7894,15 @@ packages: resolution: {integrity: sha512-gns+Er14+ZrEoC5fhOfYCY1LOHHr0TI+rQUHZS8Ttw2l7gl+80eHc/gFf2Ktkw0+SIACDTeWvpFcv3B04VembQ==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + jest-environment-jsdom@30.0.4: + resolution: {integrity: sha512-9WmS3oyCLFgs6DUJSoMpVb+AbH62Y2Xecw3XClbRgj6/Z+VjNeSLjrhBgVvTZ40njZTWeDHv8unp+6M/z8ADDg==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + peerDependencies: + canvas: ^3.0.0 + peerDependenciesMeta: + canvas: + optional: true + jest-environment-jsdom@30.0.5: resolution: {integrity: sha512-BmnDEoAH+jEjkPrvE9DTKS2r3jYSJWlN/r46h0/DBUxKrkgt2jAZ5Nj4wXLAcV1KWkRpcFqA5zri9SWzJZ1cCg==} engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} @@ -7893,6 +7952,10 @@ packages: resolution: {integrity: sha512-GBEV4GRADeP+qtB2+6u61stea8mGcOT4mCtrYISZwfu9/ISHFJ/5zOMXYbpBE9RsS5+Gb63DW4FgmnKJ79Kf6w==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + jest-message-util@30.0.2: + resolution: {integrity: sha512-vXywcxmr0SsKXF/bAD7t7nMamRvPuJkras00gqYeB1V0WllxZrbZ0paRr3XqpFU2sYYjD0qAaG2fRyn/CGZ0aw==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + jest-message-util@30.0.5: resolution: {integrity: sha512-NAiDOhsK3V7RU0Aa/HnrQo+E4JlbarbmI3q6Pi4KcxicdtjV82gcIUrejOtczChtVQR4kddu1E1EJlW6EN9IyA==} engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} @@ -7901,6 +7964,10 @@ packages: resolution: {integrity: sha512-ITOMZn+UkYS4ZFh83xYAOzWStloNzJFO2s8DWrE4lhtGD+AorgnbkiKERe4wQVBydIGPx059g6riW5Btp6Llnw==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + jest-mock@30.0.2: + resolution: {integrity: sha512-PnZOHmqup/9cT/y+pXIVbbi8ID6U1XHRmbvR7MvUy4SLqhCbwpkmXhLbsWbGewHrV5x/1bF7YDjs+x24/QSvFA==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + jest-mock@30.0.5: resolution: {integrity: sha512-Od7TyasAAQX/6S+QCbN6vZoWOMwlTtzzGuxJku1GhGanAjz9y+QsQkpScDmETvdc9aSXyJ/Op4rhpMYBWW91wQ==} engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} @@ -7950,6 +8017,10 @@ packages: resolution: {integrity: sha512-z6EbKajIpqGKU56y5KBUgy1dt1ihhQJgWzUlZHArA/+X2ad7Cb5iF+AK1EWVL/Bo7Rz9uurpqw6SiBCefUbCGA==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + jest-util@30.0.2: + resolution: {integrity: sha512-8IyqfKS4MqprBuUpZNlFB5l+WFehc8bfCe1HSZFHzft2mOuND8Cvi9r1musli+u6F3TqanCZ/Ik4H4pXUolZIg==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + jest-util@30.0.5: resolution: {integrity: sha512-pvyPWssDZR0FlfMxCBoc0tvM8iUEskaRFALUtGQYzVEAqisAztmy+R8LnU14KT4XA0H/a5HMVTXat1jLne010g==} engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} @@ -8797,6 +8868,10 @@ packages: resolution: {integrity: sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==} engines: {node: '>= 6.13.0'} + node-gyp-build@4.8.4: + resolution: {integrity: sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==} + hasBin: true + node-int64@0.4.0: resolution: {integrity: sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw==} @@ -9439,6 +9514,10 @@ packages: resolution: {integrity: sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==} engines: {node: ^14.15.0 || ^16.10.0 || >=18.0.0} + pretty-format@30.0.2: + resolution: {integrity: sha512-yC5/EBSOrTtqhCKfLHqoUIAXVRZnukHPwWBJWR7h84Q3Be1DRQZLncwcfLoPA5RPQ65qfiCMqgYwdUuQ//eVpg==} + engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} + pretty-format@30.0.5: resolution: {integrity: sha512-D1tKtYvByrBkFLe2wHJl2bwMJIiT8rW+XA+TiataH79/FszLQMrpGEvzUVkzPau7OCO0Qnrhpe87PqtOAIB8Yw==} engines: {node: ^18.14.0 || ^20.0.0 || ^22.0.0 || >=24.0.0} @@ -11252,6 +11331,10 @@ packages: peerDependencies: react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + utf-8-validate@6.0.5: + resolution: {integrity: sha512-EYZR+OpIXp9Y1eG1iueg8KRsY8TuT8VNgnanZ0uA3STqhHQTLwbl+WX76/9X5OY12yQubymBpaBSmMPkSTQcKA==} + engines: {node: '>=6.14.2'} + util-deprecate@1.0.2: resolution: {integrity: sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==} @@ -12732,7 +12815,7 @@ snapshots: '@jest/console@29.7.0': dependencies: '@jest/types': 29.6.3 - '@types/node': 24.2.1 + '@types/node': 18.19.122 chalk: 4.1.2 jest-message-util: 29.7.0 jest-util: 29.7.0 @@ -12745,14 +12828,14 @@ snapshots: '@jest/test-result': 29.7.0 '@jest/transform': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 24.2.1 + '@types/node': 18.19.122 ansi-escapes: 4.3.2 chalk: 4.1.2 ci-info: 3.9.0 exit: 0.1.2 graceful-fs: 4.2.11 jest-changed-files: 29.7.0 - jest-config: 29.7.0(@types/node@24.2.1) + jest-config: 29.7.0(@types/node@18.19.122) jest-haste-map: 29.7.0 jest-message-util: 29.7.0 jest-regex-util: 29.6.3 @@ -12773,29 +12856,47 @@ snapshots: - supports-color - ts-node - '@jest/environment-jsdom-abstract@30.0.5(jsdom@26.1.0)': + '@jest/environment-jsdom-abstract@30.0.4(jsdom@26.1.0(utf-8-validate@6.0.5))': + dependencies: + '@jest/environment': 30.0.4 + '@jest/fake-timers': 30.0.4 + '@jest/types': 30.0.1 + '@types/jsdom': 21.1.7 + '@types/node': 18.19.122 + jest-mock: 30.0.2 + jest-util: 30.0.2 + jsdom: 26.1.0(utf-8-validate@6.0.5) + + '@jest/environment-jsdom-abstract@30.0.5(jsdom@26.1.0(utf-8-validate@6.0.5))': dependencies: '@jest/environment': 30.0.5 '@jest/fake-timers': 30.0.5 '@jest/types': 30.0.5 '@types/jsdom': 21.1.7 - '@types/node': 24.2.1 + '@types/node': 18.19.122 jest-mock: 30.0.5 jest-util: 30.0.5 - jsdom: 26.1.0 + jsdom: 26.1.0(utf-8-validate@6.0.5) '@jest/environment@29.7.0': dependencies: '@jest/fake-timers': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 24.2.1 + '@types/node': 18.19.122 jest-mock: 29.7.0 + '@jest/environment@30.0.4': + dependencies: + '@jest/fake-timers': 30.0.4 + '@jest/types': 30.0.1 + '@types/node': 18.19.122 + jest-mock: 30.0.2 + '@jest/environment@30.0.5': dependencies: '@jest/fake-timers': 30.0.5 '@jest/types': 30.0.5 - '@types/node': 24.2.1 + '@types/node': 18.19.122 jest-mock: 30.0.5 '@jest/expect-utils@29.7.0': @@ -12813,16 +12914,25 @@ snapshots: dependencies: '@jest/types': 29.6.3 '@sinonjs/fake-timers': 10.3.0 - '@types/node': 24.2.1 + '@types/node': 18.19.122 jest-message-util: 29.7.0 jest-mock: 29.7.0 jest-util: 29.7.0 + '@jest/fake-timers@30.0.4': + dependencies: + '@jest/types': 30.0.1 + '@sinonjs/fake-timers': 13.0.5 + '@types/node': 18.19.122 + jest-message-util: 30.0.2 + jest-mock: 30.0.2 + jest-util: 30.0.2 + '@jest/fake-timers@30.0.5': dependencies: '@jest/types': 30.0.5 '@sinonjs/fake-timers': 13.0.5 - '@types/node': 24.2.1 + '@types/node': 18.19.122 jest-message-util: 30.0.5 jest-mock: 30.0.5 jest-util: 30.0.5 @@ -12838,7 +12948,7 @@ snapshots: '@jest/pattern@30.0.1': dependencies: - '@types/node': 24.2.1 + '@types/node': 18.19.122 jest-regex-util: 30.0.1 '@jest/reporters@29.7.0': @@ -12849,7 +12959,7 @@ snapshots: '@jest/transform': 29.7.0 '@jest/types': 29.6.3 '@jridgewell/trace-mapping': 0.3.29 - '@types/node': 24.2.1 + '@types/node': 18.19.122 chalk: 4.1.2 collect-v8-coverage: 1.0.2 exit: 0.1.2 @@ -12874,6 +12984,10 @@ snapshots: dependencies: '@sinclair/typebox': 0.27.8 + '@jest/schemas@30.0.1': + dependencies: + '@sinclair/typebox': 0.34.38 + '@jest/schemas@30.0.5': dependencies: '@sinclair/typebox': 0.34.38 @@ -12931,7 +13045,17 @@ snapshots: '@jest/schemas': 29.6.3 '@types/istanbul-lib-coverage': 2.0.6 '@types/istanbul-reports': 3.0.4 - '@types/node': 24.2.1 + '@types/node': 18.19.122 + '@types/yargs': 17.0.33 + chalk: 4.1.2 + + '@jest/types@30.0.1': + dependencies: + '@jest/pattern': 30.0.1 + '@jest/schemas': 30.0.1 + '@types/istanbul-lib-coverage': 2.0.6 + '@types/istanbul-reports': 3.0.4 + '@types/node': 18.19.122 '@types/yargs': 17.0.33 chalk: 4.1.2 @@ -12941,7 +13065,7 @@ snapshots: '@jest/schemas': 30.0.5 '@types/istanbul-lib-coverage': 2.0.6 '@types/istanbul-reports': 3.0.4 - '@types/node': 24.2.1 + '@types/node': 18.19.122 '@types/yargs': 17.0.33 chalk: 4.1.2 @@ -12991,9 +13115,9 @@ snapshots: '@juggle/resize-observer@3.4.0': {} - '@jupyter-widgets/base@4.1.7(react@19.1.1)': + '@jupyter-widgets/base@4.1.7(react@19.1.1)(utf-8-validate@6.0.5)': dependencies: - '@jupyterlab/services': 7.4.4(react@19.1.1) + '@jupyterlab/services': 7.4.4(react@19.1.1)(utf-8-validate@6.0.5) '@lumino/coreutils': 2.2.1 '@lumino/messaging': 2.0.3 '@lumino/widgets': 2.7.1 @@ -13008,9 +13132,9 @@ snapshots: - react - utf-8-validate - '@jupyter-widgets/base@6.0.11(crypto@1.0.1)(react@19.1.1)': + '@jupyter-widgets/base@6.0.11(crypto@1.0.1)(react@19.1.1)(utf-8-validate@6.0.5)': dependencies: - '@jupyterlab/services': 7.4.4(react@19.1.1) + '@jupyterlab/services': 7.4.4(react@19.1.1)(utf-8-validate@6.0.5) '@lumino/coreutils': 2.2.1 '@lumino/messaging': 1.10.3 '@lumino/widgets': 1.37.2(crypto@1.0.1) @@ -13025,9 +13149,9 @@ snapshots: - react - utf-8-validate - '@jupyter-widgets/controls@5.0.0-rc.2(crypto@1.0.1)(react@19.1.1)': + '@jupyter-widgets/controls@5.0.0-rc.2(crypto@1.0.1)(react@19.1.1)(utf-8-validate@6.0.5)': dependencies: - '@jupyter-widgets/base': 6.0.11(crypto@1.0.1)(react@19.1.1) + '@jupyter-widgets/base': 6.0.11(crypto@1.0.1)(react@19.1.1)(utf-8-validate@6.0.5) '@lumino/algorithm': 1.9.2 '@lumino/domutils': 1.8.2 '@lumino/messaging': 1.10.3 @@ -13043,9 +13167,9 @@ snapshots: - react - utf-8-validate - '@jupyter-widgets/output@4.1.7(react@19.1.1)': + '@jupyter-widgets/output@4.1.7(react@19.1.1)(utf-8-validate@6.0.5)': dependencies: - '@jupyter-widgets/base': 4.1.7(react@19.1.1) + '@jupyter-widgets/base': 4.1.7(react@19.1.1)(utf-8-validate@6.0.5) transitivePeerDependencies: - bufferutil - react @@ -13073,7 +13197,7 @@ snapshots: dependencies: '@lumino/coreutils': 2.2.1 - '@jupyterlab/services@7.4.4(react@19.1.1)': + '@jupyterlab/services@7.4.4(react@19.1.1)(utf-8-validate@6.0.5)': dependencies: '@jupyter/ydoc': 3.1.0 '@jupyterlab/coreutils': 6.4.4 @@ -13085,7 +13209,7 @@ snapshots: '@lumino/polling': 2.1.4 '@lumino/properties': 2.0.3 '@lumino/signaling': 2.1.4 - ws: 8.18.3 + ws: 8.18.3(utf-8-validate@6.0.5) transitivePeerDependencies: - bufferutil - react @@ -13112,20 +13236,20 @@ snapshots: '@lumino/properties': 2.0.3 '@lumino/signaling': 2.1.4 - '@langchain/anthropic@0.3.26(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3)(zod@3.25.76)))': + '@langchain/anthropic@0.3.26(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)))': dependencies: '@anthropic-ai/sdk': 0.56.0 - '@langchain/core': 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3)(zod@3.25.76)) + '@langchain/core': 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)) fast-xml-parser: 4.5.3 - '@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3)(zod@3.25.76))': + '@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76))': dependencies: '@cfworker/json-schema': 4.1.1 ansi-styles: 5.2.0 camelcase: 6.3.0 decamelize: 1.2.0 js-tiktoken: 1.0.20 - langsmith: 0.3.46(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3)(zod@3.25.76)) + langsmith: 0.3.46(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)) mustache: 4.2.0 p-queue: 6.6.2 p-retry: 4.6.2 @@ -13138,31 +13262,31 @@ snapshots: - '@opentelemetry/sdk-trace-base' - openai - '@langchain/google-genai@0.2.16(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3)(zod@3.25.76)))': + '@langchain/google-genai@0.2.16(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)))': dependencies: '@google/generative-ai': 0.24.1 - '@langchain/core': 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3)(zod@3.25.76)) + '@langchain/core': 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)) uuid: 11.1.0 - '@langchain/mistralai@0.2.1(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3)(zod@3.25.76)))(zod@3.25.76)': + '@langchain/mistralai@0.2.1(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)))(zod@3.25.76)': dependencies: - '@langchain/core': 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3)(zod@3.25.76)) + '@langchain/core': 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)) '@mistralai/mistralai': 1.7.4(zod@3.25.76) uuid: 10.0.0 transitivePeerDependencies: - zod - '@langchain/ollama@0.2.3(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3)(zod@3.25.76)))': + '@langchain/ollama@0.2.3(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)))': dependencies: - '@langchain/core': 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3)(zod@3.25.76)) + '@langchain/core': 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)) ollama: 0.5.16 uuid: 10.0.0 - '@langchain/openai@0.6.7(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3)(zod@3.25.76)))(ws@8.18.3)': + '@langchain/openai@0.6.7(@langchain/core@0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)))(ws@8.18.3(utf-8-validate@6.0.5))': dependencies: - '@langchain/core': 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3)(zod@3.25.76)) + '@langchain/core': 0.3.69(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)) js-tiktoken: 1.0.20 - openai: 5.12.2(ws@8.18.3)(zod@3.25.76) + openai: 5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76) zod: 3.25.76 transitivePeerDependencies: - ws @@ -14103,7 +14227,7 @@ snapshots: http-proxy-middleware: 2.0.9(@types/express@4.17.23) p-retry: 6.2.1 webpack-dev-server: 5.2.2(webpack@5.100.1) - ws: 8.18.3 + ws: 8.18.3(utf-8-validate@6.0.5) transitivePeerDependencies: - '@types/express' - bufferutil @@ -14517,7 +14641,7 @@ snapshots: '@types/graceful-fs@4.1.9': dependencies: - '@types/node': 24.2.1 + '@types/node': 18.19.122 '@types/hast@2.3.10': dependencies: @@ -14563,16 +14687,12 @@ snapshots: '@types/jsdom@21.1.7': dependencies: - '@types/node': 24.2.1 + '@types/node': 18.19.122 '@types/tough-cookie': 4.0.5 parse5: 7.3.0 '@types/json-schema@7.0.15': {} - '@types/json-stable-stringify@1.2.0': - dependencies: - json-stable-stringify: 1.3.0 - '@types/katex@0.16.7': {} '@types/keyv@3.1.4': @@ -15337,10 +15457,6 @@ snapshots: dependencies: possible-typed-array-names: 1.1.0 - await-spawn@4.0.2: - dependencies: - bl: 4.1.0 - awaiting@3.0.0: {} axios@1.11.0: @@ -15449,7 +15565,7 @@ snapshots: batch@0.6.1: {} - better-sqlite3@11.10.0: + better-sqlite3@12.2.0: dependencies: bindings: 1.5.0 prebuild-install: 7.1.3 @@ -16090,21 +16206,6 @@ snapshots: - supports-color - ts-node - create-jest@29.7.0(@types/node@24.2.1): - dependencies: - '@jest/types': 29.6.3 - chalk: 4.1.2 - exit: 0.1.2 - graceful-fs: 4.2.11 - jest-config: 29.7.0(@types/node@24.2.1) - jest-util: 29.7.0 - prompts: 2.4.2 - transitivePeerDependencies: - - '@types/node' - - babel-plugin-macros - - supports-color - - ts-node - create-react-class@15.7.0: dependencies: loose-envify: 1.4.0 @@ -16891,12 +16992,12 @@ snapshots: dependencies: once: 1.4.0 - engine.io-client@6.6.3: + engine.io-client@6.6.3(utf-8-validate@6.0.5): dependencies: '@socket.io/component-emitter': 3.1.2 debug: 4.3.7 engine.io-parser: 5.2.3 - ws: 8.17.1 + ws: 8.17.1(utf-8-validate@6.0.5) xmlhttprequest-ssl: 2.1.2 transitivePeerDependencies: - bufferutil @@ -16905,7 +17006,7 @@ snapshots: engine.io-parser@5.2.3: {} - engine.io@6.6.4: + engine.io@6.6.4(utf-8-validate@6.0.5): dependencies: '@types/cors': 2.8.19 '@types/node': 18.19.122 @@ -16915,7 +17016,7 @@ snapshots: cors: 2.8.5 debug: 4.3.7 engine.io-parser: 5.2.3 - ws: 8.17.1 + ws: 8.17.1(utf-8-validate@6.0.5) transitivePeerDependencies: - bufferutil - supports-color @@ -18704,7 +18805,7 @@ snapshots: '@jest/expect': 29.7.0 '@jest/test-result': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 24.2.1 + '@types/node': 18.19.122 chalk: 4.1.2 co: 4.6.0 dedent: 1.6.0 @@ -18743,25 +18844,6 @@ snapshots: - supports-color - ts-node - jest-cli@29.7.0(@types/node@24.2.1): - dependencies: - '@jest/core': 29.7.0 - '@jest/test-result': 29.7.0 - '@jest/types': 29.6.3 - chalk: 4.1.2 - create-jest: 29.7.0(@types/node@24.2.1) - exit: 0.1.2 - import-local: 3.2.0 - jest-config: 29.7.0(@types/node@24.2.1) - jest-util: 29.7.0 - jest-validate: 29.7.0 - yargs: 17.7.2 - transitivePeerDependencies: - - '@types/node' - - babel-plugin-macros - - supports-color - - ts-node - jest-config@29.7.0(@types/node@18.19.122): dependencies: '@babel/core': 7.28.0 @@ -18792,36 +18874,6 @@ snapshots: - babel-plugin-macros - supports-color - jest-config@29.7.0(@types/node@24.2.1): - dependencies: - '@babel/core': 7.28.0 - '@jest/test-sequencer': 29.7.0 - '@jest/types': 29.6.3 - babel-jest: 29.7.0(@babel/core@7.28.0) - chalk: 4.1.2 - ci-info: 3.9.0 - deepmerge: 4.3.1 - glob: 7.2.3 - graceful-fs: 4.2.11 - jest-circus: 29.7.0 - jest-environment-node: 29.7.0 - jest-get-type: 29.6.3 - jest-regex-util: 29.6.3 - jest-resolve: 29.7.0 - jest-runner: 29.7.0 - jest-util: 29.7.0 - jest-validate: 29.7.0 - micromatch: 4.0.8 - parse-json: 5.2.0 - pretty-format: 29.7.0 - slash: 3.0.0 - strip-json-comments: 3.1.1 - optionalDependencies: - '@types/node': 24.2.1 - transitivePeerDependencies: - - babel-plugin-macros - - supports-color - jest-diff@26.6.2: dependencies: chalk: 4.1.2 @@ -18848,13 +18900,25 @@ snapshots: jest-util: 29.7.0 pretty-format: 29.7.0 - jest-environment-jsdom@30.0.5: + jest-environment-jsdom@30.0.4(utf-8-validate@6.0.5): + dependencies: + '@jest/environment': 30.0.4 + '@jest/environment-jsdom-abstract': 30.0.4(jsdom@26.1.0(utf-8-validate@6.0.5)) + '@types/jsdom': 21.1.7 + '@types/node': 18.19.122 + jsdom: 26.1.0(utf-8-validate@6.0.5) + transitivePeerDependencies: + - bufferutil + - supports-color + - utf-8-validate + + jest-environment-jsdom@30.0.5(utf-8-validate@6.0.5): dependencies: '@jest/environment': 30.0.5 - '@jest/environment-jsdom-abstract': 30.0.5(jsdom@26.1.0) + '@jest/environment-jsdom-abstract': 30.0.5(jsdom@26.1.0(utf-8-validate@6.0.5)) '@types/jsdom': 21.1.7 '@types/node': 24.2.1 - jsdom: 26.1.0 + jsdom: 26.1.0(utf-8-validate@6.0.5) transitivePeerDependencies: - bufferutil - supports-color @@ -18865,7 +18929,7 @@ snapshots: '@jest/environment': 29.7.0 '@jest/fake-timers': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 24.2.1 + '@types/node': 18.19.122 jest-mock: 29.7.0 jest-util: 29.7.0 @@ -18877,7 +18941,7 @@ snapshots: dependencies: '@jest/types': 29.6.3 '@types/graceful-fs': 4.1.9 - '@types/node': 24.2.1 + '@types/node': 18.19.122 anymatch: 3.1.3 fb-watchman: 2.0.2 graceful-fs: 4.2.11 @@ -18939,6 +19003,18 @@ snapshots: slash: 3.0.0 stack-utils: 2.0.6 + jest-message-util@30.0.2: + dependencies: + '@babel/code-frame': 7.27.1 + '@jest/types': 30.0.1 + '@types/stack-utils': 2.0.3 + chalk: 4.1.2 + graceful-fs: 4.2.11 + micromatch: 4.0.8 + pretty-format: 30.0.2 + slash: 3.0.0 + stack-utils: 2.0.6 + jest-message-util@30.0.5: dependencies: '@babel/code-frame': 7.27.1 @@ -18954,13 +19030,19 @@ snapshots: jest-mock@29.7.0: dependencies: '@jest/types': 29.6.3 - '@types/node': 24.2.1 + '@types/node': 18.19.122 jest-util: 29.7.0 + jest-mock@30.0.2: + dependencies: + '@jest/types': 30.0.1 + '@types/node': 18.19.122 + jest-util: 30.0.2 + jest-mock@30.0.5: dependencies: '@jest/types': 30.0.5 - '@types/node': 24.2.1 + '@types/node': 18.19.122 jest-util: 30.0.5 jest-pnp-resolver@1.2.3(jest-resolve@29.7.0): @@ -18999,7 +19081,7 @@ snapshots: '@jest/test-result': 29.7.0 '@jest/transform': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 24.2.1 + '@types/node': 18.19.122 chalk: 4.1.2 emittery: 0.13.1 graceful-fs: 4.2.11 @@ -19027,7 +19109,7 @@ snapshots: '@jest/test-result': 29.7.0 '@jest/transform': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 24.2.1 + '@types/node': 18.19.122 chalk: 4.1.2 cjs-module-lexer: 1.4.3 collect-v8-coverage: 1.0.2 @@ -19073,16 +19155,25 @@ snapshots: jest-util@29.7.0: dependencies: '@jest/types': 29.6.3 - '@types/node': 24.2.1 + '@types/node': 18.19.122 chalk: 4.1.2 ci-info: 3.9.0 graceful-fs: 4.2.11 picomatch: 2.3.1 + jest-util@30.0.2: + dependencies: + '@jest/types': 30.0.1 + '@types/node': 18.19.122 + chalk: 4.1.2 + ci-info: 4.3.0 + graceful-fs: 4.2.11 + picomatch: 4.0.3 + jest-util@30.0.5: dependencies: '@jest/types': 30.0.5 - '@types/node': 24.2.1 + '@types/node': 18.19.122 chalk: 4.1.2 ci-info: 4.3.0 graceful-fs: 4.2.11 @@ -19101,7 +19192,7 @@ snapshots: dependencies: '@jest/test-result': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 24.2.1 + '@types/node': 18.19.122 ansi-escapes: 4.3.2 chalk: 4.1.2 emittery: 0.13.1 @@ -19116,7 +19207,7 @@ snapshots: jest-worker@29.7.0: dependencies: - '@types/node': 24.2.1 + '@types/node': 18.19.122 jest-util: 29.7.0 merge-stream: 2.0.0 supports-color: 8.1.1 @@ -19133,18 +19224,6 @@ snapshots: - supports-color - ts-node - jest@29.7.0(@types/node@24.2.1): - dependencies: - '@jest/core': 29.7.0 - '@jest/types': 29.6.3 - import-local: 3.2.0 - jest-cli: 29.7.0(@types/node@24.2.1) - transitivePeerDependencies: - - '@types/node' - - babel-plugin-macros - - supports-color - - ts-node - jquery-focus-exit@1.0.1(jquery@3.7.1): dependencies: jquery: 3.7.1 @@ -19208,7 +19287,7 @@ snapshots: jsbn@1.1.0: {} - jsdom@26.1.0: + jsdom@26.1.0(utf-8-validate@6.0.5): dependencies: cssstyle: 4.6.0 data-urls: 5.0.0 @@ -19228,7 +19307,7 @@ snapshots: whatwg-encoding: 3.1.1 whatwg-mimetype: 4.0.0 whatwg-url: 14.2.0 - ws: 8.18.3 + ws: 8.18.3(utf-8-validate@6.0.5) xml-name-validator: 5.0.0 transitivePeerDependencies: - bufferutil @@ -19403,7 +19482,7 @@ snapshots: langs@2.0.0: {} - langsmith@0.3.46(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3)(zod@3.25.76)): + langsmith@0.3.46(@opentelemetry/api@1.9.0)(openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76)): dependencies: '@types/uuid': 10.0.0 chalk: 4.1.2 @@ -19414,7 +19493,7 @@ snapshots: uuid: 10.0.0 optionalDependencies: '@opentelemetry/api': 1.9.0 - openai: 5.12.2(ws@8.18.3)(zod@3.25.76) + openai: 5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76) launch-editor@2.11.1: dependencies: @@ -20058,6 +20137,9 @@ snapshots: node-forge@1.3.1: {} + node-gyp-build@4.8.4: + optional: true + node-int64@0.4.0: {} node-jose@2.2.0: @@ -20220,9 +20302,9 @@ snapshots: is-inside-container: 1.0.0 wsl-utils: 0.1.0 - openai@5.12.2(ws@8.18.3)(zod@3.25.76): + openai@5.12.2(ws@8.18.3(utf-8-validate@6.0.5))(zod@3.25.76): optionalDependencies: - ws: 8.18.3 + ws: 8.18.3(utf-8-validate@6.0.5) zod: 3.25.76 opener@1.5.2: {} @@ -20792,6 +20874,12 @@ snapshots: ansi-styles: 5.2.0 react-is: 18.3.1 + pretty-format@30.0.2: + dependencies: + '@jest/schemas': 30.0.1 + ansi-styles: 5.2.0 + react-is: 18.3.1 + pretty-format@30.0.5: dependencies: '@jest/schemas': 30.0.5 @@ -22073,20 +22161,20 @@ snapshots: smart-buffer@4.2.0: {} - socket.io-adapter@2.5.5: + socket.io-adapter@2.5.5(utf-8-validate@6.0.5): dependencies: debug: 4.3.7 - ws: 8.17.1 + ws: 8.17.1(utf-8-validate@6.0.5) transitivePeerDependencies: - bufferutil - supports-color - utf-8-validate - socket.io-client@4.8.1: + socket.io-client@4.8.1(utf-8-validate@6.0.5): dependencies: '@socket.io/component-emitter': 3.1.2 debug: 4.3.7 - engine.io-client: 6.6.3 + engine.io-client: 6.6.3(utf-8-validate@6.0.5) socket.io-parser: 4.2.4 transitivePeerDependencies: - bufferutil @@ -22100,14 +22188,14 @@ snapshots: transitivePeerDependencies: - supports-color - socket.io@4.8.1: + socket.io@4.8.1(utf-8-validate@6.0.5): dependencies: accepts: 1.3.8 base64id: 2.0.0 cors: 2.8.5 debug: 4.3.7 - engine.io: 6.6.4 - socket.io-adapter: 2.5.5 + engine.io: 6.6.4(utf-8-validate@6.0.5) + socket.io-adapter: 2.5.5(utf-8-validate@6.0.5) socket.io-parser: 4.2.4 transitivePeerDependencies: - bufferutil @@ -22670,26 +22758,6 @@ snapshots: babel-jest: 29.7.0(@babel/core@7.28.0) jest-util: 30.0.5 - ts-jest@29.4.1(@babel/core@7.28.0)(@jest/transform@29.7.0)(@jest/types@30.0.5)(babel-jest@29.7.0(@babel/core@7.28.0))(jest-util@30.0.5)(jest@29.7.0(@types/node@24.2.1))(typescript@5.9.2): - dependencies: - bs-logger: 0.2.6 - fast-json-stable-stringify: 2.1.0 - handlebars: 4.7.8 - jest: 29.7.0(@types/node@24.2.1) - json5: 2.2.3 - lodash.memoize: 4.1.2 - make-error: 1.3.6 - semver: 7.7.2 - type-fest: 4.41.0 - typescript: 5.9.2 - yargs-parser: 21.1.1 - optionalDependencies: - '@babel/core': 7.28.0 - '@jest/transform': 29.7.0 - '@jest/types': 30.0.5 - babel-jest: 29.7.0(@babel/core@7.28.0) - jest-util: 30.0.5 - tsd@0.22.0: dependencies: '@tsd/typescript': 4.7.4 @@ -22955,6 +23023,11 @@ snapshots: dependencies: react: 19.1.1 + utf-8-validate@6.0.5: + dependencies: + node-gyp-build: 4.8.4 + optional: true + util-deprecate@1.0.2: {} util@0.12.5: @@ -23181,7 +23254,7 @@ snapshots: sockjs: 0.3.24 spdy: 4.0.2 webpack-dev-middleware: 7.4.2(webpack@5.100.1) - ws: 8.18.3 + ws: 8.18.3(utf-8-validate@6.0.5) optionalDependencies: webpack: 5.100.1 transitivePeerDependencies: @@ -23274,12 +23347,12 @@ snapshots: websocket-extensions@0.1.4: {} - websocket-sftp@0.8.4: + websocket-sftp@0.8.4(utf-8-validate@6.0.5): dependencies: awaiting: 3.0.0 debug: 4.4.1 port-get: 1.0.4 - ws: 8.18.3 + ws: 8.18.3(utf-8-validate@6.0.5) transitivePeerDependencies: - bufferutil - supports-color @@ -23387,9 +23460,13 @@ snapshots: ws@7.5.10: {} - ws@8.17.1: {} + ws@8.17.1(utf-8-validate@6.0.5): + optionalDependencies: + utf-8-validate: 6.0.5 - ws@8.18.3: {} + ws@8.18.3(utf-8-validate@6.0.5): + optionalDependencies: + utf-8-validate: 6.0.5 wsl-utils@0.1.0: dependencies: diff --git a/src/packages/project/browser-websocket/api.ts b/src/packages/project/browser-websocket/api.ts index 8ca6e9387bb..edccfdb87f9 100644 --- a/src/packages/project/browser-websocket/api.ts +++ b/src/packages/project/browser-websocket/api.ts @@ -16,7 +16,7 @@ import { getClient } from "@cocalc/project/client"; import { get_configuration } from "../configuration"; -import { run_formatter, run_formatter_string } from "../formatters"; +import { formatString } from "../formatters"; import { nbconvert as jupyter_nbconvert } from "../jupyter/convert"; import { jupyter_strip_notebook } from "@cocalc/jupyter/nbgrader/jupyter-parse"; import { jupyter_run_notebook } from "@cocalc/jupyter/nbgrader/jupyter-run"; @@ -131,12 +131,8 @@ export async function handleApiCall({ return await canonical_paths(data.paths); case "configuration": return await get_configuration(data.aspect, data.no_cache); - case "prettier": // deprecated - case "formatter": - return await run_formatter(data); - case "prettier_string": // deprecated case "formatter_string": - return await run_formatter_string(data); + return await formatString(data); case "exec": if (data.opts == null) { throw Error("opts must not be null"); diff --git a/src/packages/project/client.ts b/src/packages/project/client.ts index ea631ddc47e..9b1fa744e7f 100644 --- a/src/packages/project/client.ts +++ b/src/packages/project/client.ts @@ -27,7 +27,6 @@ import { join } from "node:path"; import { FileSystemClient } from "@cocalc/sync-client/lib/client-fs"; import { execute_code, uuidsha1 } from "@cocalc/backend/misc_node"; import { CoCalcSocket } from "@cocalc/backend/tcp/enable-messaging-protocol"; -import type { SyncDoc } from "@cocalc/sync/editor/generic/sync-doc"; import type { ProjectClient as ProjectClientInterface } from "@cocalc/sync/editor/generic/types"; import { SyncString } from "@cocalc/sync/editor/string/sync"; import * as synctable2 from "@cocalc/sync/table"; @@ -54,7 +53,6 @@ import { type CreateConatServiceFunction, } from "@cocalc/conat/service"; import { connectToConat } from "./conat/connection"; -import { getSyncDoc } from "@cocalc/project/conat/open-files"; import { isDeleted } from "@cocalc/project/conat/listings"; const winston = getLogger("client"); @@ -520,15 +518,6 @@ export class Client extends EventEmitter implements ProjectClientInterface { }); }; - // WARNING: making two of the exact same sync_string or sync_db will definitely - // lead to corruption! - - // Get the synchronized doc with the given path. Returns undefined - // if currently no such sync-doc. - syncdoc = ({ path }: { path: string }): SyncDoc | undefined => { - return getSyncDoc(path); - }; - public path_access(opts: { path: string; mode: string; cb: CB }): void { // mode: sub-sequence of 'rwxf' -- see https://nodejs.org/api/fs.html#fs_class_fs_stats // cb(err); err = if any access fails; err=undefined if all access is OK diff --git a/src/packages/project/conat/api/editor.ts b/src/packages/project/conat/api/editor.ts index 32358e614da..b03f0c4ea1f 100644 --- a/src/packages/project/conat/api/editor.ts +++ b/src/packages/project/conat/api/editor.ts @@ -1,12 +1,9 @@ -export { jupyter_strip_notebook as jupyterStripNotebook } from "@cocalc/jupyter/nbgrader/jupyter-parse"; -export { jupyter_run_notebook as jupyterRunNotebook } from "@cocalc/jupyter/nbgrader/jupyter-run"; -export { nbconvert as jupyterNbconvert } from "../../jupyter/convert"; -export { run_formatter_string as formatterString } from "../../formatters"; -export { logo as jupyterKernelLogo } from "@cocalc/jupyter/kernel/logo"; -export { get_kernel_data as jupyterKernels } from "@cocalc/jupyter/kernel/kernel-data"; +export { formatString } from "../../formatters"; export { newFile } from "@cocalc/backend/misc/new-file"; import { printSageWS as printSageWS0 } from "@cocalc/project/print_to_pdf"; +export { sagewsStart, sagewsStop } from "@cocalc/project/sagews/control"; + import { filename_extension } from "@cocalc/util/misc"; export async function printSageWS(opts): Promise { let pdf; diff --git a/src/packages/project/conat/api/index.ts b/src/packages/project/conat/api/index.ts index 7ce9561c646..a6b32a86ce8 100644 --- a/src/packages/project/conat/api/index.ts +++ b/src/packages/project/conat/api/index.ts @@ -53,11 +53,11 @@ Remember, if you don't set API_KEY, then the project MUST be running so that the import { type ProjectApi } from "@cocalc/conat/project/api"; import { connectToConat } from "@cocalc/project/conat/connection"; import { getSubject } from "../names"; -import { terminate as terminateOpenFiles } from "@cocalc/project/conat/open-files"; import { close as closeListings } from "@cocalc/project/conat/listings"; import { project_id } from "@cocalc/project/data"; import { close as closeFilesRead } from "@cocalc/project/conat/files/read"; import { close as closeFilesWrite } from "@cocalc/project/conat/files/write"; +import { close as closeJupyter } from "@cocalc/project/conat/jupyter"; import { getLogger } from "@cocalc/project/logger"; const logger = getLogger("conat:api"); @@ -101,12 +101,12 @@ async function handleMessage(api, subject, mesg) { // TODO: should be part of handleApiRequest below, but done differently because // one case halts this loop const { service } = request.args[0] ?? {}; - if (service == "open-files") { - terminateOpenFiles(); + if (service == "listings") { + closeListings(); await mesg.respond({ status: "terminated", service }); return; - } else if (service == "listings") { - closeListings(); + } else if (service == "jupyter") { + closeJupyter(); await mesg.respond({ status: "terminated", service }); return; } else if (service == "files:read") { @@ -152,11 +152,13 @@ async function handleApiRequest(request, mesg) { import * as system from "./system"; import * as editor from "./editor"; +import * as jupyter from "./jupyter"; import * as sync from "./sync"; export const projectApi: ProjectApi = { system, editor, + jupyter, sync, }; @@ -164,7 +166,9 @@ async function getResponse({ name, args }) { const [group, functionName] = name.split("."); const f = projectApi[group]?.[functionName]; if (f == null) { - throw Error(`unknown function '${name}'`); + throw Error( + `unknown function '${name}' -- available functions are ${JSON.stringify(Object.keys(projectApi[group]))}`, + ); } return await f(...args); } diff --git a/src/packages/project/conat/api/jupyter.ts b/src/packages/project/conat/api/jupyter.ts new file mode 100644 index 00000000000..a65076eb695 --- /dev/null +++ b/src/packages/project/conat/api/jupyter.ts @@ -0,0 +1,62 @@ +export { jupyter_strip_notebook as stripNotebook } from "@cocalc/jupyter/nbgrader/jupyter-parse"; +export { jupyter_run_notebook as runNotebook } from "@cocalc/jupyter/nbgrader/jupyter-run"; +export { nbconvert } from "../../jupyter/convert"; +export { formatString } from "../../formatters"; +export { logo as kernelLogo } from "@cocalc/jupyter/kernel/logo"; +export { get_kernel_data as kernels } from "@cocalc/jupyter/kernel/kernel-data"; +export { newFile } from "@cocalc/backend/misc/new-file"; +import { getClient } from "@cocalc/project/client"; +import { project_id } from "@cocalc/project/data"; +import * as control from "@cocalc/jupyter/control"; +import { SandboxedFilesystem } from "@cocalc/backend/sandbox"; +import { type ServerSocket } from "@cocalc/conat/socket"; + +let fs: SandboxedFilesystem | null = null; +export async function start(path: string) { + if (control.isRunning(path)) { + return; + } + fs ??= new SandboxedFilesystem(process.env.HOME ?? "/tmp", { + unsafeMode: true, + }); + await control.start({ project_id, path, client: getClient(), fs }); +} + +// IMPORTANT: run is NOT used directly by the API, but instead by packages/project/conat/jupyter.ts +// It is convenient to have it here so it can call start above, etc. The reason is because +// this returns an async iterator managed using a dedicated socket, and the api is request/response, +// so it can't just be part of the normal api. +export async function run(opts: { + path: string; + cells: { id: string; input: string }[]; + socket: ServerSocket; +}) { + await start(opts.path); + return await control.run(opts); +} + +export async function stop(path: string) { + await control.stop({ path }); +} + +export async function introspect(opts) { + await start(opts.path); + return await control.introspect(opts); +} + +export async function complete(opts) { + await start(opts.path); + return await control.complete(opts); +} + +export async function getConnectionFile(opts) { + await start(opts.path); + return await control.getConnectionFile(opts); +} + +export async function signal(opts) { + if (!control.isRunning(opts.path)) { + return; + } + await control.signal(opts); +} diff --git a/src/packages/project/conat/api/system.ts b/src/packages/project/conat/api/system.ts index 60bedcc537e..3750ac32ee8 100644 --- a/src/packages/project/conat/api/system.ts +++ b/src/packages/project/conat/api/system.ts @@ -19,12 +19,6 @@ export async function listing({ path, hidden }) { return await getListing(path, hidden); } -import { delete_files } from "@cocalc/backend/files/delete-files"; - -export async function deleteFiles({ paths }: { paths: string[] }) { - return await delete_files(paths); -} - import { getClient } from "@cocalc/project/client"; async function setDeleted(path) { const client = getClient(); diff --git a/src/packages/project/conat/files/fs.ts b/src/packages/project/conat/files/fs.ts new file mode 100644 index 00000000000..718d011acef --- /dev/null +++ b/src/packages/project/conat/files/fs.ts @@ -0,0 +1,30 @@ +/* +Fileserver with all safety off for the project. This is run inside the project by the project, +so the security is off. +*/ + +import { localPathFileserver } from "@cocalc/backend/conat/files/local-path"; +import { getService } from "@cocalc/conat/files/fs"; +import { compute_server_id, project_id } from "@cocalc/project/data"; +import { connectToConat } from "@cocalc/project/conat/connection"; + +let server: any = undefined; +export async function init() { + if (server) { + return; + } + const client = connectToConat(); + const service = getService({ compute_server_id }); + server = await localPathFileserver({ + client, + service, + path: process.env.HOME ?? "/tmp", + unsafeMode: true, + project_id, + }); +} + +export function close() { + server?.close(); + server = undefined; +} diff --git a/src/packages/project/conat/formatter.ts b/src/packages/project/conat/formatter.ts deleted file mode 100644 index aa593e89bc2..00000000000 --- a/src/packages/project/conat/formatter.ts +++ /dev/null @@ -1,25 +0,0 @@ -/* -File formatting service. -*/ - -import { run_formatter, type Options } from "../formatters"; -import { createFormatterService as create } from "@cocalc/conat/service/formatter"; -import { compute_server_id, project_id } from "@cocalc/project/data"; - -interface Message { - path: string; - options: Options; -} - -export async function createFormatterService({ openSyncDocs }) { - const impl = { - formatter: async (opts: Message) => { - const syncstring = openSyncDocs[opts.path]; - if (syncstring == null) { - throw Error(`"${opts.path}" is not opened`); - } - return await run_formatter({ ...opts, syncstring }); - }, - }; - return await create({ compute_server_id, project_id, impl }); -} diff --git a/src/packages/project/conat/index.ts b/src/packages/project/conat/index.ts index 056c9c8c7bb..2c2d9121eb7 100644 --- a/src/packages/project/conat/index.ts +++ b/src/packages/project/conat/index.ts @@ -9,7 +9,7 @@ Start the NATS servers: import "./connection"; import { getLogger } from "@cocalc/project/logger"; import { init as initAPI } from "./api"; -import { init as initOpenFiles } from "./open-files"; +// import { init as initOpenFiles } from "./open-files"; // TODO: initWebsocketApi is temporary import { init as initWebsocketApi } from "./browser-websocket-api"; import { init as initListings } from "./listings"; @@ -17,13 +17,15 @@ import { init as initRead } from "./files/read"; import { init as initWrite } from "./files/write"; import { init as initProjectStatus } from "@cocalc/project/project-status/server"; import { init as initUsageInfo } from "@cocalc/project/usage-info"; +import { init as initJupyter } from "./jupyter"; const logger = getLogger("project:conat:index"); export default async function init() { logger.debug("starting Conat project services"); await initAPI(); - await initOpenFiles(); + await initJupyter(); + // await initOpenFiles(); initWebsocketApi(); await initListings(); await initRead(); diff --git a/src/packages/project/conat/jupyter.ts b/src/packages/project/conat/jupyter.ts new file mode 100644 index 00000000000..82a04312d3d --- /dev/null +++ b/src/packages/project/conat/jupyter.ts @@ -0,0 +1,45 @@ +/* + +To run just this for a project in a console, from the browser, terminate the jupyter server by running this +in your browser with the project open: + + await cc.client.conat_client.projectApi(cc.current()).system.terminate({service:'jupyter'}) + +As explained in packages/project/conat/api/index.ts setup your environment as for the project. + +Then run this code in nodejs: + + require("@cocalc/project/conat/jupyter").init() + + + + +*/ + +import { run } from "@cocalc/project/conat/api/jupyter"; +import { outputHandler } from "@cocalc/jupyter/control"; +import { jupyterServer } from "@cocalc/conat/project/jupyter/run-code"; +import { connectToConat } from "@cocalc/project/conat/connection"; +import { compute_server_id, project_id } from "@cocalc/project/data"; +import { getLogger } from "@cocalc/project/logger"; + +const logger = getLogger("project:conat:jupyter"); + +let server: any = null; +export function init() { + logger.debug("initializing jupyter run server"); + const client = connectToConat(); + server = jupyterServer({ + client, + project_id, + compute_server_id, + run, + outputHandler, + }); +} + +export function close() { + logger.debug("closing jupyter run server"); + server?.close(); + server = null; +} diff --git a/src/packages/project/conat/open-files.ts b/src/packages/project/conat/open-files.ts deleted file mode 100644 index 10c96d1edec..00000000000 --- a/src/packages/project/conat/open-files.ts +++ /dev/null @@ -1,512 +0,0 @@ -/* -Handle opening files in a project to save/load from disk and also enable compute capabilities. - -DEVELOPMENT: - -0. From the browser with the project opened, terminate the open-files api service: - - - await cc.client.conat_client.projectApi(cc.current()).system.terminate({service:'open-files'}) - - - -Set env variables as in a project (see api/index.ts ), then in nodejs: - -DEBUG_CONSOLE=yes DEBUG=cocalc:debug:project:conat:* node - - x = await require("@cocalc/project/conat/open-files").init(); Object.keys(x) - - -[ 'openFiles', 'openDocs', 'formatter', 'terminate', 'computeServers', 'cc' ] - -> x.openFiles.getAll(); - -> Object.keys(x.openDocs) - -> s = x.openDocs['z4.tasks'] -// now you can directly work with the syncdoc for a given file, -// but from the perspective of the project, not the browser! -// -// - -OR: - - echo "require('@cocalc/project/conat/open-files').init(); require('@cocalc/project/bug-counter').init()" | node - -COMPUTE SERVER: - -To simulate a compute server, do exactly as above, but also set the environment -variable COMPUTE_SERVER_ID to the *global* (not project specific) id of the compute -server: - - COMPUTE_SERVER_ID=84 node - -In this case, you aso don't need to use the terminate command if the compute -server isn't actually running. To terminate a compute server open files service though: - - (TODO) - - -EDITOR ACTIONS: - -Stop the open-files server and define x as above in a terminal. You can -then get the actions or store in a nodejs terminal for a particular document -as follows: - -project_id = '00847397-d6a8-4cb0-96a8-6ef64ac3e6cf'; path = '2025-03-21-100921.ipynb'; -redux = require("@cocalc/jupyter/redux/app").redux; a = redux.getEditorActions(project_id, path); s = redux.getEditorStore(project_id, path); 0; - - -IN A LIVE RUNNING PROJECT IN KUCALC: - -Ssh in to the project itself. You can use a terminal because that very terminal will be broken by -doing this! Then: - -/cocalc/github/src/packages/project$ . /cocalc/nvm/nvm.sh -/cocalc/github/src/packages/project$ COCALC_PROJECT_ID=... COCALC_SECRET_TOKEN="/secrets/secret-token/token" CONAT_SERVER=hub-conat node # not sure about CONAT_SERVER -Welcome to Node.js v20.19.0. -Type ".help" for more information. -> x = await require("@cocalc/project/conat/open-files").init(); Object.keys(x) -[ 'openFiles', 'openDocs', 'formatter', 'terminate', 'computeServers' ] -> - - -*/ - -import { - openFiles as createOpenFiles, - type OpenFiles, - type OpenFileEntry, -} from "@cocalc/project/conat/sync"; -import { CONAT_OPEN_FILE_TOUCH_INTERVAL } from "@cocalc/util/conat"; -import { compute_server_id, project_id } from "@cocalc/project/data"; -import type { SyncDoc } from "@cocalc/sync/editor/generic/sync-doc"; -import { getClient } from "@cocalc/project/client"; -import { SyncString } from "@cocalc/sync/editor/string/sync"; -import { SyncDB } from "@cocalc/sync/editor/db/sync"; -import getLogger from "@cocalc/backend/logger"; -import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; -import { delay } from "awaiting"; -import { initJupyterRedux, removeJupyterRedux } from "@cocalc/jupyter/kernel"; -import { filename_extension, original_path } from "@cocalc/util/misc"; -import { createFormatterService } from "./formatter"; -import { type ConatService } from "@cocalc/conat/service/service"; -import { exists } from "@cocalc/backend/misc/async-utils-node"; -import { map as awaitMap } from "awaiting"; -import { unlink } from "fs/promises"; -import { join } from "path"; -import { - computeServerManager, - ComputeServerManager, -} from "@cocalc/conat/compute/manager"; -import { JUPYTER_SYNCDB_EXTENSIONS } from "@cocalc/util/jupyter/names"; -import { connectToConat } from "@cocalc/project/conat/connection"; - -// ensure conat connection stuff is initialized -import "@cocalc/project/conat/env"; -import { chdir } from "node:process"; - -const logger = getLogger("project:conat:open-files"); - -// we check all files we are currently managing this frequently to -// see if they exist on the filesystem: -const FILE_DELETION_CHECK_INTERVAL = 5000; - -// once we determine that a file does not exist for some reason, we -// wait this long and check *again* just to be sure. If it is still missing, -// then we close the file in memory and set the file as deleted in the -// shared openfile state. -const FILE_DELETION_GRACE_PERIOD = 2000; - -// We NEVER check a file for deletion for this long after first opening it. -// This is VERY important, since some documents, e.g., jupyter notebooks, -// can take a while to get created on disk the first time. -const FILE_DELETION_INITIAL_DELAY = 15000; - -let openFiles: OpenFiles | null = null; -let formatter: any = null; -const openDocs: { [path: string]: SyncDoc | ConatService } = {}; -let computeServers: ComputeServerManager | null = null; -const openTimes: { [path: string]: number } = {}; - -export function getSyncDoc(path: string): SyncDoc | undefined { - const doc = openDocs[path]; - if (doc instanceof SyncString || doc instanceof SyncDB) { - return doc; - } - return undefined; -} - -export async function init() { - logger.debug("init"); - - if (process.env.HOME) { - chdir(process.env.HOME); - } - - openFiles = await createOpenFiles(); - - computeServers = computeServerManager({ project_id }); - await computeServers.waitUntilReady(); - computeServers.on("change", async ({ path, id }) => { - if (openFiles == null) { - return; - } - const entry = openFiles?.get(path); - if (entry != null) { - await handleChange({ ...entry, id }); - } else { - await closeDoc(path); - } - }); - - // initialize - for (const entry of openFiles.getAll()) { - handleChange(entry); - } - - // start loop to watch for and close files that aren't touched frequently: - closeIgnoredFilesLoop(); - - // periodically update timestamp on backend for files we have open - touchOpenFilesLoop(); - // watch if any file that is currently opened on this host gets deleted, - // and if so, mark it as such, and set it to closed. - watchForFileDeletionLoop(); - - // handle changes - openFiles.on("change", (entry) => { - // we ONLY actually try to open the file here if there - // is a doctype set. When it is first being created, - // the doctype won't be the first field set, and we don't - // want to launch this until it is set. - if (entry.doctype) { - handleChange(entry); - } - }); - - formatter = await createFormatterService({ openSyncDocs: openDocs }); - - // useful for development - return { - openFiles, - openDocs, - formatter, - terminate, - computeServers, - cc: connectToConat(), - }; -} - -export function terminate() { - logger.debug("terminating open-files service"); - for (const path in openDocs) { - closeDoc(path); - } - openFiles?.close(); - openFiles = null; - - formatter?.close(); - formatter = null; - - computeServers?.close(); - computeServers = null; -} - -function getCutoff(): number { - return Date.now() - 2.5 * CONAT_OPEN_FILE_TOUCH_INTERVAL; -} - -function computeServerId(path: string): number { - return computeServers?.get(path) ?? 0; -} - -async function handleChange({ - path, - time, - deleted, - backend, - doctype, - id, -}: OpenFileEntry & { id?: number }) { - try { - if (id == null) { - id = computeServerId(path); - } - logger.debug("handleChange", { path, time, deleted, backend, doctype, id }); - const syncDoc = openDocs[path]; - const isOpenHere = syncDoc != null; - - if (id != compute_server_id) { - if (backend?.id == compute_server_id) { - // we are definitely not the backend right now. - openFiles?.setNotBackend(path, compute_server_id); - } - // only thing we should do is close it if it is open. - if (isOpenHere) { - await closeDoc(path); - } - return; - } - - if (deleted?.deleted) { - if (await exists(path)) { - // it's back - openFiles?.setNotDeleted(path); - } else { - if (isOpenHere) { - await closeDoc(path); - } - return; - } - } - - if (time != null && time >= getCutoff()) { - if (!isOpenHere) { - logger.debug("handleChange: opening", { path }); - // users actively care about this file being opened HERE, but it isn't - await openDoc(path); - } - return; - } - } catch (err) { - console.trace(err); - logger.debug(`handleChange: WARNING - error opening ${path} -- ${err}`); - } -} - -function supportAutoclose(path: string): boolean { - // this feels way too "hard coded"; alternatively, maybe we make the kernel or whatever - // actually update the interest? or something else... - if ( - path.endsWith("." + JUPYTER_SYNCDB_EXTENSIONS) || - path.endsWith(".sagews") || - path.endsWith(".term") - ) { - return false; - } - return true; -} - -async function closeIgnoredFilesLoop() { - while (openFiles?.state == "connected") { - await delay(CONAT_OPEN_FILE_TOUCH_INTERVAL); - if (openFiles?.state != "connected") { - return; - } - const paths = Object.keys(openDocs); - if (paths.length == 0) { - logger.debug("closeIgnoredFiles: no paths currently open"); - continue; - } - logger.debug( - "closeIgnoredFiles: checking", - paths.length, - "currently open paths...", - ); - const cutoff = getCutoff(); - for (const entry of openFiles.getAll()) { - if ( - entry != null && - entry.time != null && - openDocs[entry.path] != null && - entry.time <= cutoff && - supportAutoclose(entry.path) - ) { - logger.debug("closeIgnoredFiles: closing due to inactivity", entry); - closeDoc(entry.path); - } - } - } -} - -async function touchOpenFilesLoop() { - while (openFiles?.state == "connected" && openDocs != null) { - for (const path in openDocs) { - openFiles.setBackend(path, compute_server_id); - } - await delay(CONAT_OPEN_FILE_TOUCH_INTERVAL); - } -} - -async function checkForFileDeletion(path: string) { - if (openFiles == null) { - return; - } - if (Date.now() - (openTimes[path] ?? 0) <= FILE_DELETION_INITIAL_DELAY) { - return; - } - const id = computeServerId(path); - if (id != compute_server_id) { - // not our concern - return; - } - - if (path.endsWith(".term")) { - // term files are exempt -- we don't save data in them and often - // don't actually make the hidden ones for each frame in the - // filesystem at all. - return; - } - const entry = openFiles.get(path); - if (entry == null) { - return; - } - if (entry.deleted?.deleted) { - // already set as deleted -- shouldn't still be opened - await closeDoc(entry.path); - } else { - if (!process.env.HOME) { - // too dangerous - return; - } - const fullPath = join(process.env.HOME, entry.path); - // if file doesn't exist and still doesn't exist in a while, - // mark deleted, which also causes a close. - if (await exists(fullPath)) { - return; - } - // still doesn't exist? - // We must give things a reasonable amount of time, e.g., otherwise - // creating a file (e.g., jupyter notebook) might take too long and - // we randomly think it is deleted before we even make it! - await delay(FILE_DELETION_GRACE_PERIOD); - if (await exists(fullPath)) { - return; - } - // still doesn't exist - if (openFiles != null) { - logger.debug("checkForFileDeletion: marking as deleted -- ", entry); - openFiles.setDeleted(entry.path); - await closeDoc(fullPath); - // closing a file may cause it to try to save to disk the last version, - // so we delete it if that happens. - // TODO: add an option to close everywhere to not do this, and/or make - // it not save on close if the file doesn't exist. - try { - if (await exists(fullPath)) { - await unlink(fullPath); - } - } catch {} - } - } -} - -async function watchForFileDeletionLoop() { - while (openFiles != null && openFiles.state == "connected") { - await delay(FILE_DELETION_CHECK_INTERVAL); - if (openFiles?.state != "connected") { - return; - } - const paths = Object.keys(openDocs); - if (paths.length == 0) { - // logger.debug("watchForFileDeletionLoop: no paths currently open"); - continue; - } - // logger.debug( - // "watchForFileDeletionLoop: checking", - // paths.length, - // "currently open paths to see if any were deleted", - // ); - await awaitMap(paths, 20, checkForFileDeletion); - } -} - -const closeDoc = reuseInFlight(async (path: string) => { - logger.debug("close", { path }); - try { - const doc = openDocs[path]; - if (doc == null) { - return; - } - delete openDocs[path]; - delete openTimes[path]; - try { - await doc.close(); - } catch (err) { - logger.debug(`WARNING -- issue closing doc -- ${err}`); - openFiles?.setError(path, err); - } - } finally { - if (openDocs[path] == null) { - openFiles?.setNotBackend(path, compute_server_id); - } - } -}); - -const openDoc = reuseInFlight(async (path: string) => { - logger.debug("openDoc", { path }); - try { - const doc = openDocs[path]; - if (doc != null) { - return; - } - openTimes[path] = Date.now(); - - if (path.endsWith(".term")) { - // terminals are handled directly by the project api -- also since - // doctype probably not set for them, they won't end up here. - // (this could change though, e.g., we might use doctype to - // set the terminal command). - return; - } - - const client = getClient(); - let doctype: any = openFiles?.get(path)?.doctype; - logger.debug("openDoc: open files table knows ", openFiles?.get(path), { - path, - }); - if (doctype == null) { - logger.debug("openDoc: doctype must be set but isn't, so bailing", { - path, - }); - } else { - logger.debug("openDoc: got doctype from openFiles table", { - path, - doctype, - }); - } - - let syncdoc; - if (doctype.type == "string") { - syncdoc = new SyncString({ - ...doctype.opts, - project_id, - path, - client, - }); - } else { - syncdoc = new SyncDB({ - ...doctype.opts, - project_id, - path, - client, - }); - } - openDocs[path] = syncdoc; - - syncdoc.on("error", (err) => { - closeDoc(path); - openFiles?.setError(path, err); - logger.debug(`syncdoc error -- ${err}`, path); - }); - - // Extra backend support in some cases, e.g., Jupyter, Sage, etc. - const ext = filename_extension(path); - switch (ext) { - case JUPYTER_SYNCDB_EXTENSIONS: - logger.debug("initializing Jupyter backend for ", path); - await initJupyterRedux(syncdoc, client); - const path1 = original_path(syncdoc.get_path()); - syncdoc.on("closed", async () => { - logger.debug("removing Jupyter backend for ", path1); - await removeJupyterRedux(path1, project_id); - }); - break; - } - } finally { - if (openDocs[path] != null) { - openFiles?.setBackend(path, compute_server_id); - } - } -}); diff --git a/src/packages/project/conat/sync.ts b/src/packages/project/conat/sync.ts index ca2c3435017..b705b4b3eea 100644 --- a/src/packages/project/conat/sync.ts +++ b/src/packages/project/conat/sync.ts @@ -10,11 +10,6 @@ import { } from "@cocalc/conat/sync/dkv"; import { dko as createDKO, type DKO } from "@cocalc/conat/sync/dko"; import { project_id } from "@cocalc/project/data"; -import { - createOpenFiles, - type OpenFiles, - Entry as OpenFileEntry, -} from "@cocalc/conat/sync/open-files"; import { inventory as createInventory, type Inventory, @@ -26,7 +21,7 @@ import { type AStream, } from "@cocalc/conat/sync/astream"; -export type { DStream, DKV, OpenFiles, OpenFileEntry }; +export type { DStream, DKV }; export async function dstream( opts: DStreamOptions, @@ -50,10 +45,6 @@ export async function dko(opts: DKVOptions): Promise> { return await createDKO({ project_id, ...opts }); } -export async function openFiles(): Promise { - return await createOpenFiles({ project_id }); -} - export async function inventory(): Promise { return await createInventory({ project_id }); } diff --git a/src/packages/project/conat/terminal/session.ts b/src/packages/project/conat/terminal/session.ts index 63a0097857a..be6c059704d 100644 --- a/src/packages/project/conat/terminal/session.ts +++ b/src/packages/project/conat/terminal/session.ts @@ -1,6 +1,6 @@ import { spawn } from "@lydell/node-pty"; import { envForSpawn } from "@cocalc/backend/misc"; -import { path_split } from "@cocalc/util/misc"; +import { path_split, split } from "@cocalc/util/misc"; import { console_init_filename, len } from "@cocalc/util/misc"; import { exists } from "@cocalc/backend/misc/async-utils-node"; import { getLogger } from "@cocalc/project/logger"; @@ -12,7 +12,7 @@ import { } from "@cocalc/conat/service/terminal"; import { project_id, compute_server_id } from "@cocalc/project/data"; import { throttle } from "lodash"; -import { ThrottleString as Throttle } from "@cocalc/util/throttle"; +import { ThrottleString } from "@cocalc/util/throttle"; import { join } from "path"; import type { CreateTerminalOptions } from "@cocalc/conat/project/api/editor"; import { delay } from "awaiting"; @@ -28,14 +28,13 @@ const INPUT_CHUNK_SIZE = 50; const EXIT_MESSAGE = "\r\n\r\n[Process completed - press any key]\r\n\r\n"; -const SOFT_RESET = - "tput rmcup; printf '\e[?1000l\e[?1002l\e[?1003l\e[?1006l\e[?1l'; clear -x; sleep 0.1; clear -x; sleep 0.1; clear -x"; +const HARD_RESET = "reset"; -const COMPUTE_SERVER_INIT = `PS1="(\\h) \\w$ "; ${SOFT_RESET}; history -d $(history 1);\n`; +const COMPUTE_SERVER_INIT = `PS1="(\\h) \\w$ "; ${HARD_RESET}; history -d $(history 1);\n`; -const PROJECT_INIT = `${SOFT_RESET}; history -d $(history 1);\n`; +const PROJECT_INIT = `${HARD_RESET}; history -d $(history 1);\n`; -const DEFAULT_COMMAND = "/bin/bash"; +const DEFAULT_COMMAND = "/usr/bin/bash"; const INFINITY = 999999; const HISTORY_LIMIT_BYTES = parseInt( @@ -55,7 +54,7 @@ const MAX_BYTES_PER_SECOND = parseInt( // having to discard writes. This is basically the "frame rate" // we are supporting for users. const MAX_MSGS_PER_SECOND = parseInt( - process.env.COCALC_TERMINAL_MAX_MSGS_PER_SECOND ?? "24", + process.env.COCALC_TERMINAL_MAX_MSGS_PER_SECOND ?? "20", ); type State = "running" | "off" | "closed"; @@ -73,10 +72,21 @@ export class Session { [browser_id: string]: { rows: number; cols: number; time: number }; } = {}; public pid: number; + private nsjail?: boolean; - constructor({ termPath, options }) { + constructor({ + termPath, + options, + nsjail = false, + }: { + termPath: string; + options: CreateTerminalOptions; + // nsjail -- just a proof of concept for experimentation + nsjail?: boolean; + }) { logger.debug("create session ", { termPath, options }); this.termPath = termPath; + this.nsjail = nsjail; this.browserApi = createBrowserClient({ project_id, termPath }); this.options = options; this.streamName = `terminal-${termPath}`; @@ -171,7 +181,7 @@ export class Session { max_msgs_per_second: 5 * MAX_MSGS_PER_SECOND, }, }); - this.stream.publish("\r\n".repeat((this.size?.rows ?? 40) + 40)); + // this.stream.publish("\r\n".repeat((this.size?.rows ?? 40) + 40)); this.stream.on("reject", () => { this.throttledEllipses(); }); @@ -196,8 +206,8 @@ export class Session { COCALC_TERMINAL_FILENAME: tail, TMUX: undefined, // ensure not set }; - const command = this.options.command ?? DEFAULT_COMMAND; - const args = this.options.args ?? []; + let command = this.options.command ?? DEFAULT_COMMAND; + let args = this.options.args ?? []; const initFilename: string = console_init_filename(this.termPath); if (await exists(initFilename)) { args.push("--init-file"); @@ -208,6 +218,19 @@ export class Session { } const cwd = getCWD(head, this.options.cwd); logger.debug("creating pty"); + if (this.nsjail) { + // just a proof of concept to see what it is like! + const lib64 = await exists("/lib64"); + args = [ + ...split( + `-q -B /dev -R /var --disable_clone_newnet -E TERM=screen -E HOME=/home/user --cwd=/home/user -Mo -m none:/tmp:tmpfs:size=100000000 -R /etc -R /bin ${lib64 ? "-R /lib64" : ""} -R /lib -R /dev/urandom -R /usr --keep_caps --skip_setsid --disable_rlimits -B ${process.env.HOME}:/home/user `, + ), + "--", + command, + ...args, + ]; + command = "nsjail"; + } this.pty = spawn(command, args, { cwd, env, @@ -236,7 +259,7 @@ export class Session { // use slighlty less than MAX_MSGS_PER_SECOND to avoid reject // due to being *slightly* off. - const throttle = new Throttle(1000 / (MAX_MSGS_PER_SECOND - 3)); + const throttle = new ThrottleString(MAX_MSGS_PER_SECOND - 3); throttle.on("data", (data: string) => { // logger.debug("got data out of pty"); this.handleBackendMessages(data); diff --git a/src/packages/project/formatters/format.test.ts b/src/packages/project/formatters/format.test.ts new file mode 100644 index 00000000000..34ce38cca8e --- /dev/null +++ b/src/packages/project/formatters/format.test.ts @@ -0,0 +1,36 @@ +import { formatString } from "./index"; + +describe("format some strings", () => { + it("formats markdown with math", async () => { + const s = await formatString({ + str: "# foo\n\n- $\\int x^2$\n- blah", + options: { parser: "markdown" }, + }); + expect(s).toEqual("# foo\n\n- $\\int x^2$\n- blah\n"); + }); + + it("formats some python", async () => { + const s = await formatString({ + str: "def f( n = 0):\n print( n )", + options: { parser: "python" }, + }); + expect(s).toEqual("def f(n=0):\n print(n)\n"); + }); + + it("format some typescript", async () => { + const s = await formatString({ + str: "function f( n = 0) { console.log( n ) }", + options: { parser: "typescript" }, + }); + expect(s).toEqual("function f(n = 0) {\n console.log(n);\n}\n"); + }); + + it("formatting invalid typescript throws an error", async () => { + await expect(async () => { + await formatString({ + str: "function f( n = 0) { console.log( n ) ", + options: { parser: "typescript" }, + }); + }).rejects.toThrow("'}' expected"); + }); +}); diff --git a/src/packages/project/formatters/index.ts b/src/packages/project/formatters/index.ts index e9bff1e08c5..62fdcaa16a4 100644 --- a/src/packages/project/formatters/index.ts +++ b/src/packages/project/formatters/index.ts @@ -4,18 +4,9 @@ */ /* -Use a formatter like prettier to reformat a syncstring. - -This very nicely use the in-memory node module to prettyify code, by simply modifying the syncstring -on the backend. This avoids having to send the whole file back and forth, worrying about multiple users -and their cursors, file state etc. -- it just merges in the prettification at a point in time. -Also, by doing this on the backend we don't add 5MB (!) to the webpack frontend bundle, to install -something that is not supported on the frontend anyway. +Use a formatter like prettier to format a string of code. */ -declare let require: any; - -import { make_patch } from "@cocalc/sync/editor/generic/util"; import { math_escape, math_unescape } from "@cocalc/util/markdown-utils"; import { filename_extension } from "@cocalc/util/misc"; import { bib_format } from "./bib-format"; @@ -28,102 +19,33 @@ import { r_format } from "./r-format"; import { rust_format } from "./rust-format"; import { xml_format } from "./xml-format"; // mathjax-utils is from upstream project Jupyter -import { once } from "@cocalc/util/async-utils"; import { remove_math, replace_math } from "@cocalc/util/mathjax-utils"; -import { get_prettier } from "./prettier-lib"; import type { Syntax as FormatterSyntax, Config, Options, - FormatResult, } from "@cocalc/util/code-formatter"; export type { Config, Options, FormatterSyntax }; import { getLogger } from "@cocalc/backend/logger"; -import { getClient } from "@cocalc/project/client"; - -// don't wait too long, since the entire api call likely times out after 5s. -const MAX_WAIT_FOR_SYNC = 3000; const logger = getLogger("project:formatters"); -export async function run_formatter({ - path, - options, - syncstring, -}: { - path: string; - options: Options; - syncstring?; -}): Promise { - const client = getClient(); - // What we do is edit the syncstring with the given path to be "prettier" if possible... - if (syncstring == null) { - syncstring = client.syncdoc({ path }); - } - if (syncstring == null || syncstring.get_state() == "closed") { - return { - status: "error", - error: "document not fully opened", - phase: "format", - }; - } - if (syncstring.get_state() != "ready") { - await once(syncstring, "ready"); - } - if (options.lastChanged) { - // wait within reason until syncstring's last change is this new. - // (It's not a huge problem if this fails for some reason.) - const start = Date.now(); - const waitUntil = new Date(options.lastChanged); - while ( - Date.now() - start < MAX_WAIT_FOR_SYNC && - syncstring.last_changed() < waitUntil - ) { - try { - await once( - syncstring, - "change", - MAX_WAIT_FOR_SYNC - (Date.now() - start), - ); - } catch { - break; - } - } - } - const doc = syncstring.get_doc(); - let formatted, math, input0; - let input = (input0 = doc.to_str()); - if (options.parser === "markdown") { - [input, math] = remove_math(math_escape(input)); - } - try { - formatted = await run_formatter_string({ path, str: input, options }); - } catch (err) { - logger.debug(`run_formatter error: ${err.message}`); - return { status: "error", phase: "format", error: err.message }; - } - if (options.parser === "markdown") { - formatted = math_unescape(replace_math(formatted, math)); - } - // NOTE: the code used to make the change here on the backend. - // See https://github.com/sagemathinc/cocalc/issues/4335 for why - // that leads to confusion. - const patch = make_patch(input0, formatted); - return { status: "ok", patch }; -} - -export async function run_formatter_string({ +export async function formatString({ options, str, path, }: { str: string; - options: Options; + options: Options; // e.g., {parser:'python'} path?: string; // only used for CLANG }): Promise { - let formatted; - const input = str; + let formatted, math; + let input = str; logger.debug(`run_formatter options.parser: "${options.parser}"`); + if (options.parser === "markdown") { + [input, math] = remove_math(math_escape(input)); + } + switch (options.parser) { case "latex": case "latexindent": @@ -163,12 +85,12 @@ export async function run_formatter_string({ formatted = await rust_format(input, options, logger); break; default: - const prettier = get_prettier(); - if (prettier != null) { - formatted = prettier.format(input, options); - } else { - throw Error("Could not load 'prettier'"); - } + const prettier = await import("prettier"); + formatted = await prettier.format(input, options as any); + } + + if (options.parser === "markdown") { + formatted = math_unescape(replace_math(formatted, math)); } return formatted; } diff --git a/src/packages/project/formatters/prettier-lib.ts b/src/packages/project/formatters/prettier-lib.ts deleted file mode 100644 index 413569cb427..00000000000 --- a/src/packages/project/formatters/prettier-lib.ts +++ /dev/null @@ -1,15 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2020 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -// The whole purpose of this is to only load prettier if we really need it – this saves a few MB of project memory usage - -let instance: { format: Function } | null = null; - -export function get_prettier() { - if (instance == null) { - instance = require("prettier"); - } - return instance; -} diff --git a/src/packages/project/package.json b/src/packages/project/package.json index bfc3ada64da..7ae58a436d3 100644 --- a/src/packages/project/package.json +++ b/src/packages/project/package.json @@ -68,7 +68,7 @@ "start": "NODE_OPTIONS='--trace-warnings --unhandled-rejections=strict --enable-source-maps' pnpm cocalc-project", "build": "../node_modules/.bin/tsc --build", "tsc": "../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput", - "test": "COCALC_PROJECT_ID=812abe34-a382-4bd1-9071-29b6f4334f03 COCALC_USERNAME=user pnpm exec jest", + "test": "COCALC_PROJECT_ID=812abe34-a382-4bd1-9071-29b6f4334f03 COCALC_USERNAME=user NODE_OPTIONS='--experimental-vm-modules' pnpm exec jest", "depcheck": "pnpx depcheck", "prepublishOnly": "pnpm test", "clean": "rm -rf dist" diff --git a/src/packages/project/sagews/control.ts b/src/packages/project/sagews/control.ts new file mode 100644 index 00000000000..fa31dd09f82 --- /dev/null +++ b/src/packages/project/sagews/control.ts @@ -0,0 +1,10 @@ +import { getLogger } from "@cocalc/backend/logger"; +const logger = getLogger("project:sagews:control"); + +export async function sagewsStart(path_ipynb: string) { + logger.debug("sagewsStart: ", path_ipynb); +} + +export async function sagewsStop(path_ipynb: string) { + logger.debug("sagewsStop: ", path_ipynb); +} diff --git a/src/packages/server/compute/control.test.ts b/src/packages/server/compute/control.test.ts index 82449e24e06..08291f266fb 100644 --- a/src/packages/server/compute/control.test.ts +++ b/src/packages/server/compute/control.test.ts @@ -1,4 +1,3 @@ -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import { uuid } from "@cocalc/util/misc"; import createAccount from "@cocalc/server/accounts/create-account"; import createProject from "@cocalc/server/projects/create"; @@ -6,13 +5,9 @@ import createServer from "./create-server"; import * as control from "./control"; import { delay } from "awaiting"; -beforeAll(async () => { - await initEphemeralDatabase(); -}, 15000); - -afterAll(async () => { - await getPool().end(); -}); +import { before, after } from "@cocalc/server/test"; +beforeAll(before, 15000); +afterAll(after); describe("creates account, project and a test compute server, then control it", () => { const account_id = uuid(); @@ -26,6 +21,9 @@ describe("creates account, project and a test compute server, then control it", lastName: "One", account_id, }); + }); + + it("create project", async () => { // Only User One: project_id = await createProject({ account_id, diff --git a/src/packages/server/compute/create-server.test.ts b/src/packages/server/compute/create-server.test.ts index 86870f86059..6f7acf44fc7 100644 --- a/src/packages/server/compute/create-server.test.ts +++ b/src/packages/server/compute/create-server.test.ts @@ -1,18 +1,13 @@ -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import getServers from "./get-servers"; import { uuid } from "@cocalc/util/misc"; import createAccount from "@cocalc/server/accounts/create-account"; import createProject from "@cocalc/server/projects/create"; import createServer from "./create-server"; import { CLOUDS_BY_NAME } from "@cocalc/util/db-schema/compute-servers"; +import { before, after } from "@cocalc/server/test"; -beforeAll(async () => { - await initEphemeralDatabase(); -}, 15000); - -afterAll(async () => { - await getPool().end(); -}); +beforeAll(before, 15000); +afterAll(after); describe("creates account, project and then compute servers in various ways", () => { const account_id = uuid(); diff --git a/src/packages/server/compute/database-cache.test.ts b/src/packages/server/compute/database-cache.test.ts index 6202b04a2d0..bf67c6540d0 100644 --- a/src/packages/server/compute/database-cache.test.ts +++ b/src/packages/server/compute/database-cache.test.ts @@ -1,14 +1,11 @@ -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import { createDatabaseCachedResource, createTTLCache } from "./database-cache"; import { delay } from "awaiting"; +import { before, after } from "@cocalc/server/test"; beforeAll(async () => { - await initEphemeralDatabase(); + await before({ noConat: true }); }, 15000); - -afterAll(async () => { - await getPool().end(); -}); +afterAll(after); // keep short so that unit testing is fast... but long enough // that things don't break on github actions. diff --git a/src/packages/server/compute/get-servers.test.ts b/src/packages/server/compute/get-servers.test.ts index a4b60a217c9..2bfcbdf78c6 100644 --- a/src/packages/server/compute/get-servers.test.ts +++ b/src/packages/server/compute/get-servers.test.ts @@ -1,18 +1,13 @@ -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import getServers, { getServer } from "./get-servers"; import { uuid } from "@cocalc/util/misc"; import createAccount from "@cocalc/server/accounts/create-account"; import createProject from "@cocalc/server/projects/create"; import addUserToProject from "@cocalc/server/projects/add-user-to-project"; import createServer from "./create-server"; +import { before, after } from "@cocalc/server/test"; -beforeAll(async () => { - await initEphemeralDatabase(); -}, 15000); - -afterAll(async () => { - await getPool().end(); -}); +beforeAll(before, 15000); +afterAll(after); describe("calls get compute servers with various inputs with a new account with no data (everything should return [])", () => { it("throws 'account_id is not a valid uuid' if account_id not specified", async () => { diff --git a/src/packages/server/compute/maintenance/purchases/close.test.ts b/src/packages/server/compute/maintenance/purchases/close.test.ts index b413b1b852f..98f810f4d7b 100644 --- a/src/packages/server/compute/maintenance/purchases/close.test.ts +++ b/src/packages/server/compute/maintenance/purchases/close.test.ts @@ -2,7 +2,6 @@ Test functions for closing purchases in various ways. */ -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import createAccount from "@cocalc/server/accounts/create-account"; import { setTestNetworkUsage } from "@cocalc/server/compute/control"; import createServer from "@cocalc/server/compute/create-server"; @@ -17,14 +16,10 @@ import { closePurchase, } from "./close"; import { getPurchase } from "./util"; +import { before, after } from "@cocalc/server/test"; -beforeAll(async () => { - await initEphemeralDatabase(); -}, 15000); - -afterAll(async () => { - await getPool().end(); -}); +beforeAll(before, 15000); +afterAll(after); describe("creates account, project, test compute server, and purchase, then close the purchase, and confirm it worked properly", () => { const account_id = uuid(); diff --git a/src/packages/server/compute/maintenance/purchases/manage-purchases.test.ts b/src/packages/server/compute/maintenance/purchases/manage-purchases.test.ts index 5259ea5bc80..1f3589c78da 100644 --- a/src/packages/server/compute/maintenance/purchases/manage-purchases.test.ts +++ b/src/packages/server/compute/maintenance/purchases/manage-purchases.test.ts @@ -2,7 +2,6 @@ Test managing purchases */ -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import createAccount from "@cocalc/server/accounts/create-account"; import { setTestNetworkUsage } from "@cocalc/server/compute/control"; import createServer from "@cocalc/server/compute/create-server"; @@ -21,18 +20,15 @@ import managePurchases, { outstandingPurchases, } from "./manage-purchases"; import { getPurchase } from "./util"; +import { getPool, before, after, initEphemeralDatabase } from "@cocalc/server/test"; + +beforeAll(before, 15000); +afterAll(after); + // we put a small delay in some cases due to using a database query pool. -// This might need to be adjusted for CI infrastructure. const DELAY = 250; -beforeAll(async () => { - await initEphemeralDatabase(); -}, 15000); - -afterAll(async () => { - await getPool().end(); -}); describe("confirm managing of purchases works", () => { const account_id = uuid(); diff --git a/src/packages/server/compute/maintenance/purchases/ongoing-purchases.test.ts b/src/packages/server/compute/maintenance/purchases/ongoing-purchases.test.ts index 360cb22a0e3..628fc9ad73f 100644 --- a/src/packages/server/compute/maintenance/purchases/ongoing-purchases.test.ts +++ b/src/packages/server/compute/maintenance/purchases/ongoing-purchases.test.ts @@ -8,7 +8,6 @@ in the database, in order to run the test. */ import ongoingPurchases from "./ongoing-purchases"; -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import { uuid } from "@cocalc/util/misc"; import createAccount from "@cocalc/server/accounts/create-account"; import createProject from "@cocalc/server/projects/create"; @@ -20,14 +19,11 @@ import { MAX_PURCHASE_LENGTH_MS, } from "./manage-purchases"; import createPurchase from "@cocalc/server/purchases/create-purchase"; +import { getPool, before, after } from "@cocalc/server/test"; -beforeAll(async () => { - await initEphemeralDatabase(); -}, 15000); +beforeAll(before, 15000); +afterAll(after); -afterAll(async () => { - await getPool().end(); -}); async function getUpdatePurchase(id: number) { const pool = getPool(); diff --git a/src/packages/server/compute/maintenance/purchases/util.test.ts b/src/packages/server/compute/maintenance/purchases/util.test.ts index f0545790350..7c3477a2d3c 100644 --- a/src/packages/server/compute/maintenance/purchases/util.test.ts +++ b/src/packages/server/compute/maintenance/purchases/util.test.ts @@ -1,18 +1,15 @@ -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import { uuid } from "@cocalc/util/misc"; import createAccount from "@cocalc/server/accounts/create-account"; import createProject from "@cocalc/server/projects/create"; import createServer from "@cocalc/server/compute/create-server"; import { getServer } from "@cocalc/server/compute/get-servers"; import { setPurchaseId } from "./util"; +import { before, after } from "@cocalc/server/test"; beforeAll(async () => { - await initEphemeralDatabase(); + await before({ noConat: true }); }, 15000); - -afterAll(async () => { - await getPool().end(); -}); +afterAll(after); describe("creates compute server then sets the purchase id and confirms it", () => { const account_id = uuid(); diff --git a/src/packages/server/conat/api/index.ts b/src/packages/server/conat/api/index.ts index 10840c2537c..998163933c9 100644 --- a/src/packages/server/conat/api/index.ts +++ b/src/packages/server/conat/api/index.ts @@ -46,6 +46,8 @@ import { conat } from "@cocalc/backend/conat"; import userIsInGroup from "@cocalc/server/accounts/is-in-group"; import { close as terminateChangefeedServer } from "@cocalc/database/conat/changefeed-api"; import { close as terminatePersistServer } from "@cocalc/backend/conat/persist"; +import { close as terminateProjectRunner } from "@cocalc/server/conat/project/run"; +import { close as terminateProjectRunnerLoadBalancer } from "@cocalc/server/conat/project/load-balancer"; import { delay } from "awaiting"; const logger = getLogger("server:conat:api"); @@ -117,6 +119,14 @@ async function handleMessage({ api, subject, mesg }) { terminatePersistServer(); mesg.respond({ status: "terminated", service }, { noThrow: true }); return; + } else if (service == "project-runner") { + terminateProjectRunner(); + mesg.respond({ status: "terminated", service }, { noThrow: true }); + return; + } else if (service == "project-runner-load-balancer") { + terminateProjectRunnerLoadBalancer(); + mesg.respond({ status: "terminated", service }, { noThrow: true }); + return; } else if (service == "api") { // special hook so admin can terminate handling. This is useful for development. console.warn("TERMINATING listening on ", subject); diff --git a/src/packages/server/conat/api/projects.ts b/src/packages/server/conat/api/projects.ts index 7e43f5a6f77..86070a41061 100644 --- a/src/packages/server/conat/api/projects.ts +++ b/src/packages/server/conat/api/projects.ts @@ -1,50 +1,37 @@ -import { delay } from "awaiting"; - import createProject from "@cocalc/server/projects/create"; export { createProject }; - - import isAdmin from "@cocalc/server/accounts/is-admin"; - import { getProject } from "@cocalc/server/projects/control"; - import isCollaborator from "@cocalc/server/projects/is-collaborator"; - import { type UserCopyOptions } from "@cocalc/util/db-schema/projects"; +import isAdmin from "@cocalc/server/accounts/is-admin"; +import isCollaborator from "@cocalc/server/projects/is-collaborator"; export * from "@cocalc/server/projects/collaborators"; +import { type CopyOptions } from "@cocalc/conat/files/fs"; +import { client as filesystemClient } from "@cocalc/conat/files/file-server"; -export async function copyPathBetweenProjects( - opts: UserCopyOptions, -): Promise { - const { account_id, src_project_id, target_project_id } = opts; +export async function copyPathBetweenProjects({ + src, + dest, + options, + account_id, +}: { + src: { project_id: string; path: string | string[] }; + dest: { project_id: string; path: string }; + options?: CopyOptions; + account_id?: string; +}): Promise { if (!account_id) { throw Error("user must be signed in"); } - if (opts.target_path == null) { - opts.target_path = opts.src_path; - } - if (!(await isCollaborator({ account_id, project_id: src_project_id }))) { + if (!(await isCollaborator({ account_id, project_id: src.project_id }))) { throw Error("user must be collaborator on source project"); } if ( - !!target_project_id && - target_project_id != src_project_id && - !(await isCollaborator({ account_id, project_id: target_project_id })) + dest.project_id != src.project_id && + !(await isCollaborator({ account_id, project_id: dest.project_id })) ) { - throw Error("user must be collaborator on target project"); + throw Error("user must be collaborator on dest project"); } - await doCopyPathBetweenProjects(opts); -} - -// do the actual copy, awaiting as long as it takes to finish, -// with no security checks. -async function doCopyPathBetweenProjects(opts: UserCopyOptions) { - const project = await getProject(opts.src_project_id); - await project.copyPath({ - ...opts, - path: opts.src_path, - wait_until_done: true, - }); - if (opts.debug_delay_ms) { - await delay(opts.debug_delay_ms); - } + const client = filesystemClient(); + await client.cp({ src, dest, options }); } import { db } from "@cocalc/database"; @@ -75,3 +62,17 @@ export async function setQuotas(opts: { // @ts-ignore await project?.setAllQuotas(); } + +export async function getDiskQuota({ + account_id, + project_id, +}: { + account_id: string; + project_id: string; +}): Promise<{ used: number; size: number }> { + if (!(await isCollaborator({ account_id, project_id }))) { + throw Error("user must be a collaborator on project to get quota"); + } + const client = filesystemClient(); + return await client.getQuota({ project_id }); +} diff --git a/src/packages/server/conat/file-server/index.ts b/src/packages/server/conat/file-server/index.ts new file mode 100644 index 00000000000..58d9a556691 --- /dev/null +++ b/src/packages/server/conat/file-server/index.ts @@ -0,0 +1,187 @@ +/* + + +DEVELOPMENT: + +~/cocalc/src/packages/server/conat/file-server$ node +Welcome to Node.js v20.19.1. +Type ".help" for more information. +> require('@cocalc/backend/conat'); c = require('@cocalc/conat/files/file-server').client() + +*/ + +import { conat } from "@cocalc/backend/conat"; +import { + server as createFileServer, + client as createFileClient, + type Fileserver, + type CopyOptions, +} from "@cocalc/conat/files/file-server"; +export type { Fileserver }; +import { loadConatConfiguration } from "../configuration"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import getLogger from "@cocalc/backend/logger"; +import { data, rusticRepo } from "@cocalc/backend/data"; +import { join } from "node:path"; +import { mkdir } from "fs/promises"; +import { filesystem, type Filesystem } from "@cocalc/file-server/btrfs"; +import { exists } from "@cocalc/backend/misc/async-utils-node"; +import * as rustic from "./rustic"; + +const logger = getLogger("server:conat:file-server"); + +function name(project_id: string) { + return `project-${project_id}`; +} + +export async function getVolume(project_id) { + if (fs == null) { + throw Error("file server not initialized"); + } + return await fs.subvolumes.get(name(project_id)); +} + +async function mount({ + project_id, +}: { + project_id: string; +}): Promise<{ path: string }> { + logger.debug("mount", { project_id }); + const { path } = await getVolume(project_id); + return { path }; +} + +async function clone({ + project_id, + src_project_id, +}: { + project_id: string; + src_project_id: string; +}): Promise { + logger.debug("clone", { project_id }); + + if (fs == null) { + throw Error("file server not initialized"); + } + await fs.subvolumes.clone(name(src_project_id), name(project_id)); +} + +async function getUsage({ project_id }: { project_id: string }): Promise<{ + size: number; + used: number; + free: number; +}> { + logger.debug("getUsage", { project_id }); + const vol = await getVolume(project_id); + return await vol.quota.usage(); +} + +async function getQuota({ project_id }: { project_id: string }): Promise<{ + size: number; + used: number; +}> { + logger.debug("getQuota", { project_id }); + const vol = await getVolume(project_id); + return await vol.quota.get(); +} + +async function setQuota({ + project_id, + size, +}: { + project_id: string; + size: number | string; +}): Promise { + logger.debug("setQuota", { project_id }); + const vol = await getVolume(project_id); + await vol.quota.set(size); +} + +async function cp({ + src, + dest, + options, +}: { + // the src paths are relative to the src volume + src: { project_id: string; path: string | string[] }; + // the dest path is relative to the dest volume + dest: { project_id: string; path: string }; + // because our fs is btrfs, we default the options to reflink=true, + // which uses almost no disk and is super fast + options?: CopyOptions; +}): Promise { + if (fs == null) { + throw Error("file server not initialized"); + } + const srcVolume = await getVolume(src.project_id); + const destVolume = await getVolume(dest.project_id); + let srcPaths = await srcVolume.fs.safeAbsPaths(src.path); + let destPath = await destVolume.fs.safeAbsPath(dest.path); + + const toRelative = (path) => { + if (!path.startsWith(fs!.subvolumes.fs.path)) { + throw Error("bug"); + } + return path.slice(fs!.subvolumes.fs.path.length + 1); + }; + srcPaths = srcPaths.map(toRelative); + destPath = toRelative(destPath); + // NOTE: we *always* make reflink true because the filesystem is btrfs! + await fs.subvolumes.fs.cp( + typeof src.path == "string" ? srcPaths[0] : srcPaths, // careful to preserve string versus string[] + destPath, + { ...options, reflink: true }, + ); +} + +let fs: Filesystem | null = null; +let server: any = null; +export async function init(_fs?) { + if (server != null) { + return; + } + await loadConatConfiguration(); + const image = join(data, "btrfs", "image"); + if (!(await exists(image))) { + await mkdir(image, { recursive: true }); + } + const mountPoint = join(data, "btrfs", "mnt"); + if (!(await exists(mountPoint))) { + await mkdir(mountPoint, { recursive: true }); + } + + fs = + _fs ?? + (await filesystem({ + image: join(image, "btrfs.img"), + size: "25G", + mount: mountPoint, + rustic: rusticRepo, + })); + + server = await createFileServer({ + client: conat(), + mount: reuseInFlight(mount), + clone, + getUsage: reuseInFlight(getUsage), + getQuota: reuseInFlight(getQuota), + setQuota, + cp, + backup: reuseInFlight(rustic.backup), + restore: rustic.restore, + deleteBackup: rustic.deleteBackup, + getBackups: reuseInFlight(rustic.getBackups), + getBackupFiles: reuseInFlight(rustic.getBackupFiles), + }); +} + +export function close() { + server?.close(); + server = null; +} + +let cachedClient: null | Fileserver = null; +export function client(): Fileserver { + cachedClient ??= createFileClient({ client: conat() }); + return cachedClient!; +} diff --git a/src/packages/server/conat/file-server/rustic.ts b/src/packages/server/conat/file-server/rustic.ts new file mode 100644 index 00000000000..b89bcff4eb9 --- /dev/null +++ b/src/packages/server/conat/file-server/rustic.ts @@ -0,0 +1,76 @@ +import { getVolume } from "./index"; +import getLogger from "@cocalc/backend/logger"; + +const logger = getLogger("server:conat:file-server:rustic"); + +// create new complete backup of the project; this first snapshots the +// project, makes a backup of the snapshot, then deletes the snapshot, so the +// backup is guranteed to be consistent. +export async function backup({ + project_id, +}: { + project_id: string; +}): Promise<{ time: Date; id: string }> { + logger.debug("backup", { project_id }); + const vol = await getVolume(project_id); + return await vol.rustic.backup(); +} + +// restore the given path in the backup to the given dest. The default +// path is '' (the whole project) and the default destination is the +// same as the path. +export async function restore({ + project_id, + id, + path, + dest, +}: { + project_id: string; + id: string; + path?: string; + dest?: string; +}): Promise { + logger.debug("restore", { project_id, id, path, dest }); + const vol = await getVolume(project_id); + await vol.rustic.restore({ id, path, dest }); +} + +export async function deleteBackup({ + project_id, + id, +}: { + project_id: string; + id: string; +}): Promise { + logger.debug("deleteBackup", { project_id, id }); + const vol = await getVolume(project_id); + await vol.rustic.forget({ id }); +} + +// Return list of id's and timestamps of all backups of this project. +export async function getBackups({ + project_id, +}: { + project_id: string; +}): Promise< + { + id: string; + time: Date; + }[] +> { + logger.debug("getBackups", { project_id }); + const vol = await getVolume(project_id); + return await vol.rustic.snapshots(); +} +// Return list of all files in the given backup. +export async function getBackupFiles({ + project_id, + id, +}: { + project_id: string; + id: string; +}): Promise { + logger.debug("getBackupFiles", { project_id, id }); + const vol = await getVolume(project_id); + return await vol.rustic.ls({ id }); +} diff --git a/src/packages/server/conat/index.ts b/src/packages/server/conat/index.ts index 56e08e45b5a..a5207d149b9 100644 --- a/src/packages/server/conat/index.ts +++ b/src/packages/server/conat/index.ts @@ -5,7 +5,11 @@ import { init as initLLM } from "./llm"; import { loadConatConfiguration } from "./configuration"; import { createTimeService } from "@cocalc/conat/service/time"; export { initConatPersist } from "./persist"; -import { conatApiCount } from "@cocalc/backend/data"; +import { conatApiCount, conatProjectRunnerCount } from "@cocalc/backend/data"; +import { localPathFileserver } from "@cocalc/backend/conat/files/local-path"; +import { init as initProjectRunner } from "./project/run"; +import { init as initProjectRunnerLoadBalancer } from "./project/load-balancer"; +import { init as initFileserver } from "@cocalc/server/conat/file-server"; export { loadConatConfiguration }; @@ -28,5 +32,16 @@ export async function initConatApi() { initAPI(); } initLLM(); + for (let i = 0; i < conatProjectRunnerCount; i++) { + initProjectRunner(); + } + initProjectRunnerLoadBalancer(); createTimeService(); } + +export async function initConatFileserver() { + await loadConatConfiguration(); + logger.debug("initFileserver"); + localPathFileserver(); + initFileserver(); +} diff --git a/src/packages/server/conat/project/limits.ts b/src/packages/server/conat/project/limits.ts new file mode 100644 index 00000000000..ea87b7bde70 --- /dev/null +++ b/src/packages/server/conat/project/limits.ts @@ -0,0 +1,78 @@ +import { k8sCpuParser, k8sMemoryParser } from "@cocalc/util/misc"; +import { type Configuration } from "./types"; + +// I have not figured out how to use cgroups yet, or which cgroups to use. +// See discussion here: https://github.com/google/nsjail/issues/196 +// TODO: cgroups are of course much better. +const USE_CGROUPS = false; + +export function limits(config?: Configuration): string[] { + const args: string[] = []; + if (config == null) { + args.push("--disable_rlimits"); + return args; + } + // rlimits we don't change below + args.push("--rlimit_cpu", "max"); + args.push("--rlimit_fsize", "max"); + args.push("--rlimit_nofile", "max"); + + // need '--detect_cgroupv2' or it won't work at all since it'll try to use ancient cgroups v1 + if (USE_CGROUPS) { + args.push("--detect_cgroupv2"); + } + + if (config.cpu != null) { + const cpu = k8sCpuParser(config.cpu); + if (!isFinite(cpu) || cpu <= 0) { + throw Error(`invalid cpu limit: '${cpu}'`); + } + if (USE_CGROUPS) { + // "Number of milliseconds of CPU time per second" + args.push("--cgroup_cpu_ms_per_sec", `${Math.ceil(cpu * 1000)}`); + } else { + // --max_cpus only takes an INTEGER as input, hence ceil + args.push("--max_cpus", `${Math.ceil(cpu)}`); + } + } + + if (config.memory != null) { + const memory = k8sMemoryParser(config.memory); + if (!isFinite(memory) || memory <= 0) { + throw Error(`invalid memory limit: '${memory}'`); + } + if (USE_CGROUPS) { + args.push("--cgroup_mem_max", `${memory}`); + args.push("--rlimit_as", "hard"); + } else { + args.push("--rlimit_as", `${Math.ceil(memory / 1000000)}`); + } + } else { + args.push("--rlimit_as", "hard"); + } + + if (config.swap != null) { + if (USE_CGROUPS) { + const swap = k8sMemoryParser(config.swap); + if (!isFinite(swap) || swap <= 0) { + throw Error(`invalid swap limit: '${swap}'`); + } + args.push("--cgroup_mem_swap_max", `${swap}`); + } + } + + if (config.pids != null) { + const pids = parseInt(`${config.pids}`); + if (!isFinite(pids) || pids <= 0) { + throw Error(`invalid pids limit: '${pids}'`); + } + if (USE_CGROUPS) { + args.push("--cgroup_pids_max", `${pids}`); + } else { + // nproc is maybe a bit tighter than limiting pids, due to threads + args.push("--rlimit_nproc", `${pids}`); + } + } + + return args; +} diff --git a/src/packages/server/conat/project/load-balancer.ts b/src/packages/server/conat/project/load-balancer.ts new file mode 100644 index 00000000000..b6ea6126c0d --- /dev/null +++ b/src/packages/server/conat/project/load-balancer.ts @@ -0,0 +1,49 @@ +/* +Project run server load balancer +*/ + +import { conat } from "@cocalc/backend/conat"; +import { server as loadBalancer } from "@cocalc/conat/project/runner/load-balancer"; +import { loadConatConfiguration } from "../configuration"; +import getLogger from "@cocalc/backend/logger"; +import { setProjectState } from "./run"; +import getPool from "@cocalc/database/pool"; + +const logger = getLogger("server:conat:project:load-balancer"); + +let server; +export async function init() { + logger.debug("init"); + await loadConatConfiguration(); + server = await loadBalancer({ + client: conat(), + setState: setProjectState, + getConfig, + }); + logger.debug("running"); +} + +export function close() { + logger.debug("close"); + server?.close(); +} + +async function getConfig({ project_id }) { + const pool = getPool("medium"); + const { rows } = await pool.query( + "SELECT settings FROM projects WHERE project_id=$1", + [project_id], + ); + if (rows[0]?.settings?.admin) { + return { admin: true, disk: "25G" }; + } else { + // some defaults, mainly for testing + return { + cpu: "1000m", + memory: "8Gi", + pids: 10000, + swap: "5000Gi", + disk: "1G", + }; + } +} diff --git a/src/packages/server/conat/project/run.ts b/src/packages/server/conat/project/run.ts new file mode 100644 index 00000000000..0c9abdfa4cc --- /dev/null +++ b/src/packages/server/conat/project/run.ts @@ -0,0 +1,297 @@ +/* +Project run server. + +It may be necessary to do this to enable the user running this +code to use nsjail: + + sudo sysctl -w kernel.apparmor_restrict_unprivileged_unconfined=0 && sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + +See https://github.com/google/nsjail/issues/236#issuecomment-2267096267 + + + +--- + +DEV + + Turn off in the hub by sending this message from a browser as an admin: + + await cc.client.conat_client.hub.system.terminate({service:'project-runner'}) + +Then start this in nodejs + + require('@cocalc/server/conat/project/run').init() +*/ + +import { conat } from "@cocalc/backend/conat"; +import { server as projectRunnerServer } from "@cocalc/conat/project/runner/run"; +import { isValidUUID } from "@cocalc/util/misc"; +import { loadConatConfiguration } from "../configuration"; +import { getProject } from "@cocalc/server/projects/control"; +import { reuseInFlight } from "@cocalc/util/reuse-in-flight"; +import getLogger from "@cocalc/backend/logger"; +import { root } from "@cocalc/backend/data"; +import { dirname, join } from "node:path"; +import { userInfo } from "node:os"; +import { + chown, + ensureConfFilesExists, + getEnvironment, + homePath, + setupDataPath, + writeSecretToken, +} from "@cocalc/server/projects/control/util"; +import { mkdir } from "fs/promises"; +import { getProjectSecretToken } from "@cocalc/server/projects/control/secret-token"; +import { exists } from "@cocalc/backend/misc/async-utils-node"; +import { spawn } from "node:child_process"; +import { type Configuration } from "./types"; +import { limits } from "./limits"; +import { + client as fileserverClient, + type Fileserver, +} from "@cocalc/server/conat/file-server"; +import { nsjail } from "@cocalc/backend/sandbox/install"; + +// for development it may be useful to just disabling using nsjail namespaces +// entirely -- change this to true to do so. +const DISABLE_NSJAIL = false; + +const DEFAULT_UID = 2001; + +const logger = getLogger("server:conat:project:run"); + +let servers: any[] = []; + +const children: { [project_id: string]: any } = {}; + +export async function setProjectState({ project_id, state }) { + try { + const p = await getProject(project_id); + await p.saveStateToDatabase({ state }); + } catch {} +} + +async function touch(project_id) { + try { + const p = await getProject(project_id); + await p.touch(undefined, { noStart: true }); + } catch {} +} + +const MOUNTS = { + "-R": ["/etc", "/var", "/bin", "/lib", "/usr", "/lib64"], + "-B": ["/dev"], +}; + +async function initMounts() { + for (const type in MOUNTS) { + const v: string[] = []; + for (const path of MOUNTS[type]) { + if (await exists(path)) { + v.push(path); + } + } + MOUNTS[type] = v; + } + MOUNTS["-R"].push(`${dirname(root)}:/cocalc`); + + // also if this node is install via nvm, we make exactly this + // version of node's install available + if (!process.execPath.startsWith("/usr/")) { + // not already in an obvious system-wide place we included above + if (process.execPath.includes("nvm/versions/node/")) { + const j = process.execPath.lastIndexOf("/bin/node"); + if (j != -1) { + MOUNTS["-R"].push(process.execPath.slice(0, j)); + } + } + } +} + +let fsclient: Fileserver | null = null; +async function setQuota(project_id: string, size: number | string) { + fsclient ??= fileserverClient(); + await fsclient.setQuota({ project_id, size }); +} + +async function start({ + project_id, + config, +}: { + project_id: string; + config?: Configuration; +}) { + if (!isValidUUID(project_id)) { + throw Error("start: project_id must be valid"); + } + logger.debug("start", { project_id, config }); + setProjectState({ project_id, state: "starting" }); + if (children[project_id] != null && children[project_id].exitCode == null) { + logger.debug("start -- already running"); + return; + } + let uid, gid; + if (userInfo().uid) { + // server running as non-root user -- single user mode + uid = gid = userInfo().uid; + } else { + // server is running as root -- multiuser mode + uid = gid = DEFAULT_UID; + } + + const home = await homePath(project_id); + await mkdir(home, { recursive: true }); + await chown(home, uid); + await ensureConfFilesExists(home, uid); + const env = await getEnvironment( + project_id, + // for admin HOME stays with original source, to avoid complexity of bind mounting /home/user + // via the unshare system call... + config?.admin ? { HOME: home } : undefined, + ); + await setupDataPath(home, uid); + await writeSecretToken(home, await getProjectSecretToken(project_id), uid); + + if (config?.disk) { + // TODO: maybe this should be done in parallel with other things + // to make startup time slightly faster (?) -- could also be incorporated + // into mount. + await setQuota(project_id, config.disk); + } + + let script: string, + cmd: string, + args: string[] = []; + if (config?.admin || DISABLE_NSJAIL) { + // DANGEROUS: no safety at all here! + // This is needed to develop cocalc, since we want to run btrfs and nsjail + // from within a cocalc project. + cmd = process.execPath; + script = join(root, "packages/project/bin/cocalc-project.js"); + } else { + script = join("/cocalc/src/packages/project/bin/cocalc-project.js"); + args.push( + "-q", // not too verbose + "-Mo", // run a command once + "--disable_clone_newnet", // [ ] TODO: for now we have the full host network + "--keep_env", // this just keeps env + "--keep_caps", // [ ] TODO: maybe NOT needed! + "--skip_setsid", // evidently needed for terminal signals (e.g., ctrl+z); dangerous. [ ] TODO -- really needed? + ); + + args.push("--hostname", `project-${env.COCALC_PROJECT_ID}`); + + if (uid != null && gid != null) { + args.push("-u", `${uid}`, "-g", `${gid}`); + } + + for (const type in MOUNTS) { + for (const path of MOUNTS[type]) { + args.push(type, path); + } + } + // need a /tmp directory + args.push("-m", "none:/tmp:tmpfs:size=500000000"); + + args.push("-B", `${home}:${env.HOME}`); + args.push(...limits(config)); + args.push("--"); + args.push(process.execPath); + cmd = nsjail; + } + + args.push(script, "--init", "project_init.sh"); + + //logEnv(env); + // console.log(`${cmd} ${args.join(" ")}`); + logger.debug(`${cmd} ${args.join(" ")}`); + const child = spawn(cmd, args, { + env, + uid, + gid: uid, + }); + children[project_id] = child; + + child.stdout.on("data", (chunk: Buffer) => { + logger.debug(`project_id=${project_id}.stdout: `, chunk.toString()); + }); + child.stderr.on("data", (chunk: Buffer) => { + logger.debug(`project_id=${project_id}.stderr: `, chunk.toString()); + }); + + touch(project_id); + setProjectState({ project_id, state: "running" }); +} + +async function stop({ project_id }) { + if (!isValidUUID(project_id)) { + throw Error("stop: project_id must be valid"); + } + logger.debug("stop", { project_id }); + if (children[project_id] != null && children[project_id].exitCode == null) { + setProjectState({ project_id, state: "stopping" }); + children[project_id]?.kill("SIGKILL"); + delete children[project_id]; + } + setProjectState({ project_id, state: "opened" }); +} + +async function status({ project_id }) { + if (!isValidUUID(project_id)) { + throw Error("status: project_id must be valid"); + } + logger.debug("status", { project_id }); + let state; + if (children[project_id] == null || children[project_id].exitCode) { + state = "opened"; + } else { + state = "running"; + } + setProjectState({ project_id, state }); + // [ ] TODO: ip -- need to figure out the networking story for running projects + // The following will only work on a single machine with global network address space + return { state, ip: "127.0.0.1" }; +} + +export async function init(count: number = 1) { + await initMounts(); + await loadConatConfiguration(); + for (let i = 0; i < count; i++) { + const server = await projectRunnerServer({ + client: conat(), + start: reuseInFlight(start), + stop: reuseInFlight(stop), + status: reuseInFlight(status), + }); + servers.push(server); + } +} + +export function close() { + for (const project_id in children) { + logger.debug(`killing project_id=${project_id}`); + children[project_id]?.kill("SIGKILL"); + delete children[project_id]; + } + for (const server of servers) { + server.close(); + } + servers.length = 0; +} + +// important to close, because it kills all the processes that were spawned +process.once("exit", close); +["SIGINT", "SIGTERM", "SIGQUIT"].forEach((sig) => { + process.once(sig, () => { + process.exit(); + }); +}); + +// function logEnv(env) { +// let s = "export "; +// for (const key in env) { +// s += `${key}="${env[key]}" `; +// } +// console.log(s); +// } diff --git a/src/packages/server/conat/project/test/project.test.ts b/src/packages/server/conat/project/test/project.test.ts new file mode 100644 index 00000000000..3f8f59966af --- /dev/null +++ b/src/packages/server/conat/project/test/project.test.ts @@ -0,0 +1,82 @@ +import { uuid } from "@cocalc/util/misc"; +import createAccount from "@cocalc/server/accounts/create-account"; +import createProject from "@cocalc/server/projects/create"; +import { projectApiClient } from "@cocalc/conat/project/api"; +import { getProject } from "@cocalc/server/projects/control"; +import { restartProjectIfRunning } from "@cocalc/server/projects/control/util"; +import { before, after } from "@cocalc/server/test"; + +beforeAll(before); +afterAll(after); + +describe("create account, project, then start and stop project", () => { + const account_id = uuid(); + let project_id; + + it("create an account and a project so we can control it", async () => { + await createAccount({ + email: "", + password: "xyz", + firstName: "Test", + lastName: "User", + account_id, + }); + }); + + it("creates a project", async () => { + project_id = await createProject({ + account_id, + title: "My First Project", + noPool: true, + start: false, + }); + }); + + it("restart if running (it is not)", async () => { + await restartProjectIfRunning(project_id); + }); + + let project; + it("get state of project", async () => { + project = getProject(project_id); + const { state } = await project.state(); + expect(state).toEqual("opened"); + + // cached + expect(getProject(project_id)).toBe(project); + }); + + let projectStartTime; + it("start the project", async () => { + projectStartTime = Date.now(); + await project.start(); + const { state } = await project.state(); + expect(state).toEqual("running"); + const startupTime = Date.now() - projectStartTime; + // this better be fast (on unloaded system it is about 100ms) + expect(startupTime).toBeLessThan(2000); + }); + + it("run a command in the project to confirm everything is properly working, available and the project started and connected to conat", async () => { + const api = projectApiClient({ project_id }); + const { stdout, stderr, exit_code } = await api.system.exec({ + command: "bash", + args: ["-c", "echo $((2+3))"], + }); + expect({ stdout, stderr, exit_code }).toEqual({ + stdout: "5\n", + stderr: "", + exit_code: 0, + }); + + const firstOutputTime = Date.now() - projectStartTime; + // this better be fast (on unloaded system is less than 1 second) + expect(firstOutputTime).toBeLessThan(5000); + }); + + it("stop the project", async () => { + await project.stop(); + const { state } = await project.state(); + expect(state).toEqual("opened"); + }); +}); diff --git a/src/packages/server/conat/project/test/sync.test.ts b/src/packages/server/conat/project/test/sync.test.ts new file mode 100644 index 00000000000..b9a7f2d0254 --- /dev/null +++ b/src/packages/server/conat/project/test/sync.test.ts @@ -0,0 +1,149 @@ +import { uuid } from "@cocalc/util/misc"; +import createAccount from "@cocalc/server/accounts/create-account"; +import createProject from "@cocalc/server/projects/create"; +import { getProject } from "@cocalc/server/projects/control"; +import { before, after, client, connect } from "@cocalc/server/test"; +import { addCollaborator } from "@cocalc/server/projects/collaborators"; +import { once } from "@cocalc/util/async-utils"; +import { delay } from "awaiting"; + +beforeAll(before); +afterAll(after); + +describe("basic collab editing of a file *on disk* in a project -- verifying interaction between filesystem and editor", () => { + const account_id1 = uuid(), + account_id2 = uuid(); + let project_id; + let project; + let fs; + + it("create accounts and a project", async () => { + await createAccount({ + email: "", + password: "xyz", + firstName: "User", + lastName: "One", + account_id: account_id1, + }); + + await createAccount({ + email: "", + password: "xyz", + firstName: "User", + lastName: "Two", + account_id: account_id2, + }); + + project_id = await createProject({ + account_id: account_id1, + title: "Collab Project", + start: false, + noPool: true, + }); + project = getProject(project_id); + + fs = client.fs({ project_id }); + + await addCollaborator({ + account_id: account_id1, + opts: { account_id: account_id2, project_id }, + }); + }); + + it("write a file that we will then open and edit (we have not started the project yet)", async () => { + await fs.writeFile("a.txt", "hello"); + await fs.writeFile("b.txt", "hello"); + expect((await project.state()).state).toBe("opened"); + }); + + let syncstring, syncstring2; + it("open 'a.txt' for sync editing", async () => { + const opts = { + project_id, + path: "a.txt", + // we use a much shorter "ignoreOnSaveInterval" so testing is fast. + ignoreOnSaveInterval: 100, + watchDebounce: 1, + deletedThreshold: 100, + watchRecreateWait: 100, + deletedCheckInterval: 50, + }; + syncstring = client.sync.string(opts); + // a second completely separate client: + syncstring2 = connect().sync.string(opts); + await Promise.all([syncstring.init(), syncstring2.init()]); + expect(syncstring).not.toBe(syncstring2); + + expect(syncstring.to_str()).toEqual("hello"); + // the first version of the document should NOT be blank + expect(syncstring.versions().length).toEqual(1); + + expect(syncstring2.to_str()).toEqual("hello"); + expect(syncstring2.versions().length).toEqual(1); + }); + + it("change the file and save to disk, then read from filesystem", async () => { + syncstring.from_str("hello world"); + await syncstring.save_to_disk(); + expect((await fs.readFile("a.txt")).toString()).toEqual("hello world"); + }); + + it("change the file on disk and observe s updates", async () => { + const change = once(syncstring, "change"); + // wait so changes to the file on disk won't be ignored: + await delay(syncstring.opts.ignoreOnSaveInterval + 50); + await fs.writeFile("a.txt", "Hello World!"); + await change; + expect(syncstring.to_str()).toEqual("Hello World!"); + }); + + it("overwrite a.txt with the older b.txt and see that this update also triggers a change even though b.txt is older -- the point is that the time is *different*", async () => { + await delay(syncstring.opts.ignoreOnSaveInterval + 50); + const change = once(syncstring, "change"); + await fs.cp("b.txt", "a.txt", { preserveTimestamps: true }); + await change; + expect(syncstring.to_str()).toEqual("hello"); + const a_stat = await fs.stat("a.txt"); + const b_stat = await fs.stat("b.txt"); + expect(a_stat.atime).toEqual(b_stat.atime); + expect(a_stat.mtime).toEqual(b_stat.mtime); + }); + + it("delete 'a.txt' from disk and observe a 'deleted' event is emitted", async () => { + await delay(250); // TODO: not good! + const deleted = once(syncstring, "deleted"); + const deleted2 = once(syncstring2, "deleted"); + await fs.unlink("a.txt"); + await deleted; + await deleted2; + // good we got the event -- we can ignore it; but doc is now blank + expect(syncstring.to_str()).toEqual(""); + expect(syncstring.isDeleted).toEqual(true); + expect(syncstring2.to_str()).toEqual(""); + expect(syncstring2.isDeleted).toEqual(true); + }); + + // this fails! + it("put a really old file at a.txt and it comes back from being deleted", async () => { + const change = once(syncstring, "change"); + await fs.writeFile("old.txt", "i am old"); + await fs.utimes( + "old.txt", + (Date.now() - 100_000) / 1000, + (Date.now() - 100_000) / 1000, + ); + await fs.cp("old.txt", "a.txt", { preserveTimestamps: true }); + await change; + expect(syncstring.to_str()).toEqual("i am old"); + // [ ] TODO: it's very disconceting that isDeleted stays true for + // one of these! + // await wait({ + // until: () => { + // console.log([syncstring.isDeleted, syncstring2.isDeleted]); + // return !syncstring.isDeleted && !syncstring2.isDeleted; + // }, + // }); + // expect(syncstring.isDeleted).toEqual(false); + // expect(syncstring2.isDeleted).toEqual(false); + }); +}); diff --git a/src/packages/server/conat/project/types.ts b/src/packages/server/conat/project/types.ts new file mode 100644 index 00000000000..15ae892e852 --- /dev/null +++ b/src/packages/server/conat/project/types.ts @@ -0,0 +1,13 @@ +export interface Configuration { + admin?: boolean; + // cpu limit: sames as k8s format + cpu?: number | string; + // memory limit: sames as k8s format + memory?: number | string; + // swap limit + swap?: number | string; + // pid limit + pids?: number | string; + // disk size + disk?: number | string; +} diff --git a/src/packages/server/hub/email.ts b/src/packages/server/hub/email.ts index b1083c35bbb..833287e0b70 100644 --- a/src/packages/server/hub/email.ts +++ b/src/packages/server/hub/email.ts @@ -41,7 +41,7 @@ import { const fs_readFile_prom = promisify(fs.readFile); -const winston = getLogger("server:hub:email"); +const logger = getLogger("server:hub:email"); export function escape_email_body(body: string, allow_urls: boolean): string { // in particular, no img and no anchor a @@ -392,7 +392,7 @@ export function is_banned(address): boolean { function make_dbg(opts) { if (opts.verbose) { - return (m) => winston.debug(`send_email(to:${opts.to}) -- ${m}`); + return (m) => logger.debug(`send_email(to:${opts.to}) -- ${m}`); } else { return function (_) {}; } @@ -612,7 +612,7 @@ export function mass_email(opts): void { cb: undefined, }); // cb(err, list of recipients that we succeeded in sending email to) - const dbg = (m) => winston.debug(`mass_email: ${m}`); + const dbg = (m) => logger.debug(`mass_email: ${m}`); dbg(opts.filename); dbg(opts.subject); dbg(opts.body); diff --git a/src/packages/server/licenses/add-to-project.test.ts b/src/packages/server/licenses/add-to-project.test.ts index 1c19b76764b..756c3f2945e 100644 --- a/src/packages/server/licenses/add-to-project.test.ts +++ b/src/packages/server/licenses/add-to-project.test.ts @@ -1,15 +1,10 @@ import { uuid } from "@cocalc/util/misc"; -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import createProject from "@cocalc/server/projects/create"; import addLicenseToProject from "./add-to-project"; +import { before, after, getPool } from "@cocalc/server/test"; -beforeAll(async () => { - await initEphemeralDatabase({}); -}, 15000); - -afterAll(async () => { - await getPool().end(); -}); +beforeAll(before, 15000); +afterAll(after); describe("test various cases of adding a license to a project", () => { let project_id = uuid(); @@ -29,7 +24,7 @@ describe("test various cases of adding a license to a project", () => { const pool = getPool(); const { rows } = await pool.query( "SELECT site_license FROM projects WHERE project_id=$1", - [project_id] + [project_id], ); expect(rows[0].site_license).toEqual({ [license_id]: {} }); }); @@ -39,7 +34,7 @@ describe("test various cases of adding a license to a project", () => { const pool = getPool(); const { rows } = await pool.query( "SELECT site_license FROM projects WHERE project_id=$1", - [project_id] + [project_id], ); expect(rows[0].site_license).toEqual({ [license_id]: {} }); }); @@ -50,7 +45,7 @@ describe("test various cases of adding a license to a project", () => { const pool = getPool(); const { rows } = await pool.query( "SELECT site_license FROM projects WHERE project_id=$1", - [project_id] + [project_id], ); expect(rows[0].site_license).toEqual({ [license_id]: {}, @@ -62,12 +57,12 @@ describe("test various cases of adding a license to a project", () => { const pool = getPool(); await pool.query( "UPDATE projects SET site_license='{}' WHERE project_id=$1", - [project_id] + [project_id], ); await addLicenseToProject({ project_id, license_id }); const { rows } = await pool.query( "SELECT site_license FROM projects WHERE project_id=$1", - [project_id] + [project_id], ); expect(rows[0].site_license).toEqual({ [license_id]: {} }); }); diff --git a/src/packages/server/llm/test/models.test.ts b/src/packages/server/llm/test/models.test.ts index 21285341427..863e8fea785 100644 --- a/src/packages/server/llm/test/models.test.ts +++ b/src/packages/server/llm/test/models.test.ts @@ -1,6 +1,5 @@ // import { log } from "console"; -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import { AnthropicModel, LanguageModelCore, @@ -28,19 +27,18 @@ import { evaluateMistral } from "../mistral"; import { evaluateOpenAILC } from "../openai-lc"; import { evaluateUserDefinedLLM } from "../user-defined"; import { enableModels, setupAPIKeys, test_llm } from "./shared"; +import { before, after, getPool } from "@cocalc/server/test"; // sometimes (flaky case) they take more than 10s to even start a response const LLM_TIMEOUT = 15_000; beforeAll(async () => { - await initEphemeralDatabase(); + await before({ noConat: true }); await setupAPIKeys(); await enableModels(); }, 15000); -afterAll(async () => { - await getPool().end(); -}); +afterAll(after); const QUERY = { input: "What's 99 + 1?", diff --git a/src/packages/server/package.json b/src/packages/server/package.json index 31cb638febe..b05a37be93c 100644 --- a/src/packages/server/package.json +++ b/src/packages/server/package.json @@ -15,7 +15,9 @@ "./conat": "./dist/conat/index.js", "./conat/api": "./dist/conat/api/index.js", "./conat/auth": "./dist/conat/auth/index.js", + "./conat/file-server": "./dist/conat/file-server/index.js", "./conat/socketio": "./dist/conat/socketio/index.js", + "./conat/project/*": "./dist/conat/project/*.js", "./purchases/*": "./dist/purchases/*.js", "./stripe/*": "./dist/stripe/*.js", "./licenses/purchase": "./dist/licenses/purchase/index.js", @@ -24,18 +26,17 @@ "./projects/connection": "./dist/projects/connection/index.js", "./projects/*": "./dist/projects/*.js", "./settings": "./dist/settings/index.js", - "./settings/*": "./dist/settings/*.js" + "./settings/*": "./dist/settings/*.js", + "./test": "./dist/test/index.js", + "./test/*": "./dist/test/*.js" }, - "keywords": [ - "utilities", - "cocalc" - ], + "keywords": ["utilities", "cocalc"], "scripts": { "preinstall": "npx only-allow pnpm", "clean": "rm -rf node_modules dist", "build": "../node_modules/.bin/tsc --build", "tsc": "../node_modules/.bin/tsc --watch --pretty --preserveWatchOutput ", - "test": "TZ=UTC jest --forceExit --runInBand", + "test": "TZ=UTC jest --forceExit --maxWorkers=1", "depcheck": "pnpx depcheck", "prepublishOnly": "test" }, @@ -45,6 +46,7 @@ "@cocalc/backend": "workspace:*", "@cocalc/conat": "workspace:*", "@cocalc/database": "workspace:*", + "@cocalc/file-server": "workspace:*", "@cocalc/gcloud-pricing-calculator": "^1.17.0", "@cocalc/server": "workspace:*", "@cocalc/util": "workspace:*", @@ -71,7 +73,6 @@ "@zxcvbn-ts/language-common": "^3.0.4", "@zxcvbn-ts/language-en": "^3.0.2", "async": "^1.5.2", - "await-spawn": "^4.0.2", "awaiting": "^3.0.0", "axios": "^1.11.0", "base62": "^2.0.1", diff --git a/src/packages/server/projects/control/base.ts b/src/packages/server/projects/control/base.ts index c36a7c3a61f..b621f18d63d 100644 --- a/src/packages/server/projects/control/base.ts +++ b/src/packages/server/projects/control/base.ts @@ -4,7 +4,7 @@ */ /* -Project control abstract base class. +Project control class. The hub uses this to get information about a project and do some basic tasks. There are different implementations for different ways in which cocalc @@ -25,11 +25,7 @@ import { callback2, until } from "@cocalc/util/async-utils"; import { db } from "@cocalc/database"; import { EventEmitter } from "events"; import { isEqual } from "lodash"; -import { - type CopyOptions, - ProjectState, - ProjectStatus, -} from "@cocalc/util/db-schema/projects"; +import { ProjectState, ProjectStatus } from "@cocalc/util/db-schema/projects"; import { Quota, quota } from "@cocalc/util/upgrades/quota"; import getLogger from "@cocalc/backend/logger"; import { site_license_hook } from "@cocalc/database/postgres/site-license/hook"; @@ -39,8 +35,9 @@ import { closePayAsYouGoPurchases } from "@cocalc/server/purchases/project-quota import { handlePayAsYouGoQuotas } from "./pay-as-you-go"; import { query } from "@cocalc/database/postgres/query"; import { getProjectSecretToken } from "./secret-token"; +import { client as projectRunnerClient } from "@cocalc/conat/project/runner/run"; +import { conat } from "@cocalc/backend/conat"; -export type { CopyOptions }; export type { ProjectState, ProjectStatus }; const logger = getLogger("project-control"); @@ -55,11 +52,16 @@ export type Action = "open" | "start" | "stop" | "restart"; // collected. These objects don't use much memory, but blocking garbage collection // would be bad. const projectCache: { [project_id: string]: WeakRef } = {}; -export function getProject(project_id: string): BaseProject | undefined { - return projectCache[project_id]?.deref(); +export function getProject(project_id: string): BaseProject { + let project = projectCache[project_id]?.deref(); + if (project == null) { + project = new BaseProject(project_id); + projectCache[project_id] = new WeakRef(project); + } + return project!; } -export abstract class BaseProject extends EventEmitter { +export class BaseProject extends EventEmitter { public readonly project_id: string; public is_ready: boolean = false; public is_freed: boolean = false; @@ -108,7 +110,7 @@ export abstract class BaseProject extends EventEmitter { } } - protected async saveStateToDatabase(state: ProjectState): Promise { + async saveStateToDatabase(state: ProjectState): Promise { await callback2(db().set_project_state, { ...state, project_id: this.project_id, @@ -131,17 +133,36 @@ export abstract class BaseProject extends EventEmitter { }; } + private projectRunner = () => { + return projectRunnerClient({ + subject: `project.${this.project_id}.run`, + client: conat(), + }); + }; + // Get the state of the project -- state is just whether or not // it is runnig, stopping, starting. It's not much info. - abstract state(): Promise; - - // Get the status of the project -- status is MUCH more information - // about the project, including ports of various services. - abstract status(): Promise; - - abstract start(): Promise; - - abstract stop(): Promise; + state = async (): Promise => { + // rename everywhere to status? state is a field, and status + // is the whole object + const runner = this.projectRunner(); + return await runner.status({ project_id: this.project_id }); + }; + + status = async (): Promise => { + // deprecated? + return {} as ProjectStatus; + }; + + start = async (): Promise => { + const runner = this.projectRunner(); + await runner.start({ project_id: this.project_id }); + }; + + stop = async (): Promise => { + const runner = this.projectRunner(); + await runner.stop({ project_id: this.project_id }); + }; async restart(): Promise { this.dbg("restart")(); @@ -198,12 +219,6 @@ export abstract class BaseProject extends EventEmitter { }; } - // Copy files from one project and path to another. This doesn't - // worry at all about permissions; it's assumed the caller has already - // figured out whether the copy is allowed. - // Returns a copy_id string if scheduled is true (otherwise return ''). - abstract copyPath(opts: CopyOptions): Promise; - /* set_all_quotas ensures that if the project is running and the quotas (except idle_timeout) have changed, then the project is restarted. diff --git a/src/packages/server/projects/control/index.ts b/src/packages/server/projects/control/index.ts index c7558654aaf..88ec83fcb70 100644 --- a/src/packages/server/projects/control/index.ts +++ b/src/packages/server/projects/control/index.ts @@ -1,62 +1,13 @@ import getLogger from "@cocalc/backend/logger"; import { db } from "@cocalc/database"; -import connectToProject from "@cocalc/server/projects/connection"; -import { BaseProject } from "./base"; -import kubernetes from "./kubernetes"; -import kucalc from "./kucalc"; -import multiUser from "./multi-user"; -import singleUser from "./single-user"; -import getPool from "@cocalc/database/pool"; - -export const COCALC_MODES = [ - "single-user", - "multi-user", - "kucalc", - "kubernetes", -] as const; - -export type CocalcMode = typeof COCALC_MODES[number]; +import { BaseProject, getProject } from "./base"; +export { getProject }; export type ProjectControlFunction = (project_id: string) => BaseProject; -// NOTE: you can't *change* the mode -- caching just caches what you first set. -let cached: ProjectControlFunction | undefined = undefined; - -export default function init(mode?: CocalcMode): ProjectControlFunction { - const winston = getLogger("project-control"); - winston.debug("init", mode); - if (cached !== undefined) { - winston.info("using cached project control client"); - return cached; - } - if (!mode) { - mode = process.env.COCALC_MODE as CocalcMode; - } - if (!mode) { - throw Error( - "you can only call projects/control with no mode argument AFTER it has been initialized by the hub or if you set the COCALC_MODE env var" - ); - } - winston.info("creating project control client"); - - let getProject; - switch (mode) { - case "single-user": - getProject = singleUser; - break; - case "multi-user": - getProject = multiUser; - break; - case "kucalc": - getProject = kucalc; - break; - case "kubernetes": - getProject = kubernetes; - break; - default: - throw Error(`invalid mode "${mode}"`); - } - winston.info(`project controller created with mode ${mode}`); +export default function init(): ProjectControlFunction { + const logger = getLogger("project-control"); + logger.debug("init"); const database = db(); database.projectControl = getProject; @@ -64,46 +15,12 @@ export default function init(mode?: CocalcMode): ProjectControlFunction { // that the there is a connection to the corresponding project, so that // the project can respond. database.ensure_connection_to_project = async ( - project_id: string, - cb?: Function + _project_id: string, + cb?: Function, ): Promise => { - const dbg = (...args) => { - winston.debug("ensure_connection_to_project: ", project_id, ...args); - }; - const pool = getPool(); - const { rows } = await pool.query( - "SELECT state->'state' AS state FROM projects WHERE project_id=$1", - [project_id] - ); - const state = rows[0]?.state; - if (state != "running") { - dbg("NOT connecting because state is not 'running', state=", state); - return; - } - dbg("connecting"); - try { - await connectToProject(project_id); - cb?.(); - } catch (err) { - dbg("WARNING: unable to make a connection", err); - cb?.(err); - } + console.log("database.ensure_connection_to_project -- DEPRECATED"); + cb?.(); }; - cached = getProject; return getProject; } - -export const getProject: ProjectControlFunction = (project_id: string) => { - if (cached == null) { - if (process.env["COCALC_MODE"]) { - return init(process.env["COCALC_MODE"] as CocalcMode)(project_id); - } - throw Error( - `must call init first or set the environment variable COCALC_MODE to one of ${COCALC_MODES.join( - ", " - )}` - ); - } - return cached(project_id); -}; diff --git a/src/packages/server/projects/control/kubernetes.ts b/src/packages/server/projects/control/kubernetes.ts deleted file mode 100644 index 8375e71e241..00000000000 --- a/src/packages/server/projects/control/kubernetes.ts +++ /dev/null @@ -1,61 +0,0 @@ -/* -cocalc-kubernetes support. - -TODO/CRITICAL: I deleted this from target.ts, so be sure to make this.host be actually right! - - if (project._kubernetes) { - // this is ugly -- need to determine host in case of kubernetes, since - // host as set in the project object is old/wrong. - const status = await callback2(project.status); - if (!status.ip) { - throw Error("must wait for project to start"); - } - host = status.ip; - } - - - -*/ - -import { - BaseProject, - CopyOptions, - ProjectStatus, - ProjectState, - getProject, -} from "./base"; - -import getLogger from "@cocalc/backend/logger"; -const winston = getLogger("project-control-kubernetes"); - -class Project extends BaseProject { - async state(): Promise { - winston.debug("state ", this.project_id); - throw Error("implement me"); - } - - async status(): Promise { - winston.debug("status ", this.project_id); - throw Error("implement me"); - } - - async start(): Promise { - winston.debug("start ", this.project_id); - //await this.touch(undefined, { noStart: true }); - throw Error("implement me"); - } - - async stop(): Promise { - winston.debug("stop ", this.project_id); - throw Error("implement me"); - } - - async copyPath(opts: CopyOptions): Promise { - winston.debug("doCopyPath ", this.project_id, opts); - throw Error("implement me"); - } -} - -export default function get(project_id: string): Project { - return (getProject(project_id) as Project) ?? new Project(project_id); -} diff --git a/src/packages/server/projects/control/kucalc.test.ts b/src/packages/server/projects/control/kucalc.test.ts deleted file mode 100644 index 0519f33c1fe..00000000000 --- a/src/packages/server/projects/control/kucalc.test.ts +++ /dev/null @@ -1,113 +0,0 @@ -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; -import init, { getProject } from "@cocalc/server/projects/control"; -import { uuid } from "@cocalc/util/misc"; - -beforeAll(async () => { - await initEphemeralDatabase(); -}, 15000); - -afterAll(async () => { - await getPool().end(); -}); - -// hardcoded in projects/control/kucalc.ts -const EXPECTED_EXPIRATION_MS = 30 * 24 * 60 * 60 * 1000; - -describe("kucalc", () => { - init("kucalc"); - - const id = uuid(); - const project = getProject(id); - - test("scheduled copy/date", async () => { - const in1minute = new Date(Date.now() + 60 * 1000); - const id2 = uuid(); - const copyID = await project.copyPath({ - path: "file.md", - target_path: "file2.md", - target_project_id: id2, - scheduled: in1minute, - }); - - const pool = getPool(); - const data = ( - await pool.query("SELECT * from copy_paths WHERE id=$1", [copyID]) - ).rows[0]; - - expect(data.id).toEqual(copyID); - expect(data.expire.getTime() / 1000).toBeCloseTo( - (in1minute.getTime() + EXPECTED_EXPIRATION_MS) / 1000, - 0, - ); - expect(data.scheduled.getTime() / 1000).toBeCloseTo( - in1minute.getTime() / 1000, - 1, - ); - expect(data.source_path).toEqual("file.md"); - expect(data.target_path).toEqual("file2.md"); - expect(data.target_project_id).toEqual(id2); - expect(data.time.getTime()).toBeLessThan(data.scheduled.getTime()); - }); - - // in this test, we also don't specify a sepearte project - test("scheduled copy/string", async () => { - const in1minute: string = new Date(Date.now() + 60 * 1000).toISOString(); - const copyID = await project.copyPath({ - path: "file.md", - scheduled: in1minute, - }); - - // this mimics database::copy_path.status(...) - const pool = getPool(); - const data = ( - await pool.query("SELECT * from copy_paths WHERE id=$1", [copyID]) - ).rows[0]; - - const in1minuteDate = new Date(in1minute); - expect(data.id).toEqual(copyID); - expect(data.expire.getTime() / 1000).toBeCloseTo( - (in1minuteDate.getTime() + EXPECTED_EXPIRATION_MS) / 1000, - ); - expect(data.scheduled.getTime() / 1000).toBeCloseTo( - in1minuteDate.getTime() / 1000, - ); - expect(data.source_path).toEqual("file.md"); - expect(data.target_path).toEqual("file.md"); - expect(data.target_project_id).toEqual(id); - expect(data.time.getTime()).toBeLessThan(data.scheduled.getTime()); - }); - - test("immediate copy_path", async () => { - const id2 = uuid(); - const copyID = await project.copyPath({ - path: "file.md", - target_path: "file2.md", - target_project_id: id2, - overwrite_newer: true, - delete_missing: true, - backup: true, - wait_until_done: false, - }); - - // this mimics database::copy_path.status(...) - const pool = getPool(); - const data = ( - await pool.query("SELECT * from copy_paths WHERE id=$1", [copyID]) - ).rows[0]; - - expect(data.id).toEqual(copyID); - expect(data.expire.getTime() / 1000).toBeCloseTo( - (Date.now() + EXPECTED_EXPIRATION_MS) / 1000, - 1, - ); - expect(data.scheduled).toBeNull(); - expect(data.source_path).toEqual("file.md"); - expect(data.target_path).toEqual("file2.md"); - expect(data.target_project_id).toEqual(id2); - expect(data.time.getTime()).toBeLessThanOrEqual(Date.now()); - expect(data.backup).toBe(true); - expect(data.delete_missing).toBe(true); - expect(data.overwrite_newer).toBe(true); - expect(data.public).toBeNull(); - }); -}); diff --git a/src/packages/server/projects/control/kucalc.ts b/src/packages/server/projects/control/kucalc.ts deleted file mode 100644 index a20151b2e7a..00000000000 --- a/src/packages/server/projects/control/kucalc.ts +++ /dev/null @@ -1,265 +0,0 @@ -/* -Compute client for use in Kubernetes cluster by the hub. - -This **modifies the database** to get "something out there (manage-actions) to" -start and stop the project, copy files between projects, etc. -*/ - -import { - BaseProject, - CopyOptions, - ProjectStatus, - ProjectState, - getProject, -} from "./base"; -import { db } from "@cocalc/database"; -import { callback2 } from "@cocalc/util/async-utils"; -import { expire_time, is_valid_uuid_string, uuid } from "@cocalc/util/misc"; - -import getLogger from "@cocalc/backend/logger"; -const winston = getLogger("project-control-kucalc"); - -class Project extends BaseProject { - constructor(project_id: string) { - super(project_id); - } - - private async get(columns: string[]): Promise<{ [field: string]: any }> { - return await callback2(db().get_project, { - project_id: this.project_id, - columns, - }); - } - - async state(): Promise { - return (await this.get(["state"]))?.state ?? {}; - } - - async status(): Promise { - const status = (await this.get(["status"]))?.status ?? {}; - // In KuCalc the ports for various services are hardcoded constants, - // and not actually storted in the database, so we put them here. - // This is also hardcoded in kucalc's addons/project/image/init/init.sh (!) - status["hub-server.port"] = 6000; - status["browser-server.port"] = 6001; - return status; - } - - async start(): Promise { - if (this.stateChanging != null) return; - winston.info(`start ${this.project_id}`); - - if ((await this.state()).state == "running") { - winston.debug("start -- already running"); - return; - } - try { - this.stateChanging = { state: "starting" }; - - // TODO: once "manage" processes site licenses, we can remove this line! - // Manage has to do the equivalent of this.computeQuota() - await this.siteLicenseHook(false); - await this.actionRequest("start"); - await this.touch(undefined, { noStart: true }); - await this.waitUntilProject( - (project) => - project.state?.state == "running" || project.action_request?.finished, - 120, - ); - } finally { - this.stateChanging = undefined; - } - } - - // TODO/ATTN: I don't think this is actualy used by kucalc yet. - // In kucalc there's cluster2/addons/manage/image/src/k8s-control.coffee - // which does a lot of this stuff *directly* via the database and - // kubernetes. It needs to be rewritten... - async stop(): Promise { - if (this.stateChanging != null) return; - winston.info("stop ", this.project_id); - await this.closePayAsYouGoPurchases(); - if ((await this.state()).state != "running") { - return; - } - try { - this.stateChanging = { state: "stopping" }; - await this.actionRequest("stop"); - await this.waitUntilProject( - (project) => - (project.state != null && - project.state != "running" && - project.state != "stopping") || - project.action_request?.finished, - 60, - ); - } finally { - this.stateChanging = undefined; - } - } - - async copyPath(opts: CopyOptions): Promise { - const dbg = this.dbg("copyPath"); - dbg(JSON.stringify(opts)); - if (opts.path == null) { - throw Error("path must be specified"); - } - opts.target_project_id = opts.target_project_id - ? opts.target_project_id - : this.project_id; - opts.target_path = opts.target_path ? opts.target_path : opts.path; - - // check UUID are valid - if (!is_valid_uuid_string(opts.target_project_id)) { - throw Error(`target_project_id=${opts.target_project_id} is invalid`); - } - - const copyID = uuid(); - dbg(`copyID=${copyID}`); - - // expire in 1 month - const oneMonthSecs = 60 * 60 * 24 * 30; - let expire: Date = expire_time(oneMonthSecs); - - if (opts.scheduled) { - // we parse it if it is a string - if (typeof opts.scheduled === "string") { - const scheduledTS: number = Date.parse(opts.scheduled); - - if (isNaN(scheduledTS)) { - throw new Error( - `opts.scheduled = ${opts.scheduled} is not a valid date. Can't be parsed by Date.parse()`, - ); - } - - opts.scheduled = new Date(scheduledTS); - } - - if (opts.scheduled instanceof Date) { - // We have to remove the timezone info, b/c the PostgreSQL field is without timezone. - // Ideally though, this is always UTC, e.g. "2019-08-08T18:34:49". - const d = new Date(opts.scheduled); - const offset = d.getTimezoneOffset() / 60; - opts.scheduled = new Date(d.getTime() - offset); - opts.wait_until_done = false; - dbg(`opts.scheduled = ${opts.scheduled}`); - // since scheduled could be in the future, we want to expire it 1 month after that - expire = new Date( - Math.max(opts.scheduled.getTime(), Date.now()) + oneMonthSecs * 1000, - ); - } else { - throw new Error( - `opts.scheduled = ${opts.scheduled} is not a valid date.`, - ); - } - } - - dbg("write query requesting the copy to happen to the database"); - - await callback2(db()._query, { - query: "INSERT INTO copy_paths", - values: { - "id ::UUID": copyID, - "time ::TIMESTAMP": new Date(), - "source_project_id ::UUID": this.project_id, - "source_path ::TEXT": opts.path, - "target_project_id ::UUID": opts.target_project_id, - "target_path ::TEXT": opts.target_path, - "overwrite_newer ::BOOLEAN": opts.overwrite_newer, - "public ::BOOLEAN": opts.public, - "delete_missing ::BOOLEAN": opts.delete_missing, - "backup ::BOOLEAN": opts.backup, - "bwlimit ::TEXT": opts.bwlimit, - "timeout ::NUMERIC": opts.timeout, - "scheduled ::TIMESTAMP": opts.scheduled, - "exclude ::TEXT[]": opts.exclude, - "expire ::TIMESTAMP": expire, - }, - }); - - if (opts.wait_until_done == true) { - dbg("waiting for the copy request to complete..."); - await this.waitUntilCopyFinished(copyID, 60 * 4); - dbg("finished"); - return ""; - } else { - dbg("NOT waiting for copy to complete"); - return copyID; - } - } - - private getProjectSynctable() { - // this is all in coffeescript, hence the any type above. - return db().synctable({ - table: "projects", - columns: ["state", "action_request"], - where: { "project_id = $::UUID": this.project_id }, - // where_function is a fast easy test for matching: - where_function: (project_id) => project_id == this.project_id, - }); - } - - private async actionRequest(action: "start" | "stop"): Promise { - await callback2(db()._query, { - query: "UPDATE projects", - where: { "project_id = $::UUID": this.project_id }, - jsonb_set: { - action_request: { - action, - time: new Date(), - started: undefined, - finished: undefined, - }, - }, - }); - } - - private async waitUntilProject( - until: (obj) => boolean, - timeout: number, // in seconds - ): Promise { - let synctable: any = undefined; - try { - synctable = this.getProjectSynctable(); - await callback2(synctable.wait, { - until: () => until(synctable.get(this.project_id)?.toJS() ?? {}), - timeout, - }); - } finally { - synctable?.close(); - } - } - - private getCopySynctable(copyID: string): any { - return db().synctable({ - table: "copy_paths", - columns: ["started", "error", "finished"], - where: { "id = $::UUID": copyID }, - where_function: (id) => id == copyID, - }); - } - - private async waitUntilCopyFinished( - copyID: string, - timeout: number, // in seconds - ): Promise { - let synctable: any = undefined; - try { - synctable = this.getCopySynctable(copyID); - await callback2(synctable.wait, { - until: () => synctable.getIn([copyID, "finished"]), - timeout, - }); - const err = synctable.getIn([copyID, "error"]); - if (err) { - throw Error(err); - } - } finally { - synctable?.close(); - } - } -} - -export default function get(project_id: string): Project { - return (getProject(project_id) as Project) ?? new Project(project_id); -} diff --git a/src/packages/server/projects/control/multi-user.ts b/src/packages/server/projects/control/multi-user.ts deleted file mode 100644 index 78b6592eeb8..00000000000 --- a/src/packages/server/projects/control/multi-user.ts +++ /dev/null @@ -1,180 +0,0 @@ -/* -multi-user: a multi-user Linux system where the hub runs as root, -so can create and delete user accounts, etc. - -There is some security and isolation between projects, coming from -different operating system users. - -This is mainly used for cocalc-docker, which is a deployment of -CoCalc running in a single docker container, with one hub running -as root. - -This **executes some basic shell commands** (e.g., useradd, rsync) -to start and stop the project, copy files between projects, etc. - -This code is very similar to single-user.ts, except with some -small modifications due to having to create and delete Linux users. -*/ - -import { - chown, - copyPath, - createUser, - // deleteUser, - ensureConfFilesExists, - getEnvironment, - getState, - getStatus, - homePath, - isProjectRunning, - launchProjectDaemon, - mkdir, - setupDataPath, - stopProjectProcesses, - writeSecretToken, -} from "./util"; -import { - BaseProject, - CopyOptions, - getProject, - ProjectStatus, - ProjectState, -} from "./base"; -import getLogger from "@cocalc/backend/logger"; -import { getUid } from "@cocalc/backend/misc"; -import { - deleteProjectSecretToken, - getProjectSecretToken, -} from "./secret-token"; - -const winston = getLogger("project-control:multi-user"); - -const MAX_START_TIME_MS = 30000; -const MAX_STOP_TIME_MS = 20000; - -class Project extends BaseProject { - private HOME: string; - private uid: number; - - constructor(project_id: string) { - super(project_id); - this.HOME = homePath(this.project_id); - this.uid = getUid(this.project_id); - } - - async state(): Promise { - if (this.stateChanging != null) { - return this.stateChanging; - } - const state = await getState(this.HOME); - winston.debug(`got state of ${this.project_id} = ${JSON.stringify(state)}`); - this.saveStateToDatabase(state); - return state; - } - - async status(): Promise { - const status = await getStatus(this.HOME); - // TODO: don't include secret token in log message. - winston.debug( - `got status of ${this.project_id} = ${JSON.stringify(status)}`, - ); - this.saveStatusToDatabase(status); - return status; - } - - async start(): Promise { - if (this.stateChanging != null) return; - winston.info(`start ${this.project_id}`); - - // Home directory - const HOME = this.HOME; - - if (await isProjectRunning(HOME)) { - winston.debug("start -- already running"); - await this.saveStateToDatabase({ state: "running" }); - return; - } - - try { - this.stateChanging = { state: "starting" }; - await this.saveStateToDatabase(this.stateChanging); - await this.computeQuota(); - await mkdir(HOME, { recursive: true }); - await createUser(this.project_id); - await chown(HOME, this.uid); - await ensureConfFilesExists(HOME, this.uid); - - // this.get('env') = extra env vars for project (from synctable): - const env = await getEnvironment(this.project_id); - - winston.debug(`start ${this.project_id}: env = ${JSON.stringify(env)}`); - - // Setup files - await setupDataPath(HOME, this.uid); - - await writeSecretToken( - HOME, - await getProjectSecretToken(this.project_id), - this.uid, - ); - - await this.touch(undefined, { noStart: true }); - - // Fork and launch project server daemon - await launchProjectDaemon(env, this.uid); - - await this.wait({ - until: async () => { - if (!(await isProjectRunning(this.HOME))) { - return false; - } - const status = await this.status(); - return !!status["hub-server.port"]; - }, - maxTime: MAX_START_TIME_MS, - }); - } finally { - this.stateChanging = undefined; - // ensure state valid - await this.state(); - } - } - - async stop(): Promise { - if (this.stateChanging != null) return; - winston.info("stop ", this.project_id); - if (!(await isProjectRunning(this.HOME))) { - await this.saveStateToDatabase({ state: "opened" }); - return; - } - try { - this.stateChanging = { state: "stopping" }; - await this.saveStateToDatabase(this.stateChanging); - await stopProjectProcesses(this.project_id); - // await deleteUser(this.project_id); - await this.wait({ - until: async () => !(await isProjectRunning(this.HOME)), - maxTime: MAX_STOP_TIME_MS, - }); - await deleteProjectSecretToken(this.project_id); - } finally { - this.stateChanging = undefined; - // ensure state valid in database - await this.state(); - } - } - - async copyPath(opts: CopyOptions): Promise { - winston.debug("copyPath ", this.project_id, opts); - await copyPath( - opts, - this.project_id, - opts.target_project_id ? getUid(opts.target_project_id) : undefined, - ); - return ""; - } -} - -export default function get(project_id: string): Project { - return (getProject(project_id) as Project) ?? new Project(project_id); -} diff --git a/src/packages/server/projects/control/single-user.ts b/src/packages/server/projects/control/single-user.ts deleted file mode 100644 index 64103e9e909..00000000000 --- a/src/packages/server/projects/control/single-user.ts +++ /dev/null @@ -1,195 +0,0 @@ -/* - * This file is part of CoCalc: Copyright © 2022 Sagemath, Inc. - * License: MS-RSL – see LICENSE.md for details - */ - -/* -This is meant to run on a multi-user system, but where the hub -runs as a single user and all projects also run as that same -user, but with there own HOME directories. There is thus no -security or isolation at all between projects. There is still -a notion of multiple cocalc projects and cocalc users. - -This is useful for: - - development of cocalc from inside of a CoCalc project - - non-collaborative use of cocalc on your own - laptop, e.g., when you're on an airplane. - - -DEVELOPMENT: - - -~/cocalc/src/packages/server/projects/control$ COCALC_MODE='single-user' node -Welcome to Node.js v20.19.1. -Type ".help" for more information. -> a = require('@cocalc/server/projects/control'); -> p = a.getProject('8a840733-93b6-415c-83d4-7e5712a6266b') -> await p.start() -*/ - -import { kill } from "node:process"; -import getLogger from "@cocalc/backend/logger"; -import { - BaseProject, - CopyOptions, - ProjectState, - ProjectStatus, - getProject, -} from "./base"; -import { - copyPath, - ensureConfFilesExists, - getEnvironment, - getProjectPID, - getState, - getStatus, - homePath, - isProjectRunning, - launchProjectDaemon, - mkdir, - setupDataPath, - writeSecretToken, -} from "./util"; -import { - getProjectSecretToken, - deleteProjectSecretToken, -} from "./secret-token"; - -const logger = getLogger("project-control:single-user"); - -// Usually should fully start in about 5 seconds, but we give it 20s. -const MAX_START_TIME_MS = 20000; -const MAX_STOP_TIME_MS = 10000; - -class Project extends BaseProject { - private HOME: string; - - constructor(project_id: string) { - super(project_id); - this.HOME = homePath(this.project_id); - } - - async state(): Promise { - if (this.stateChanging != null) { - return this.stateChanging; - } - const state = await getState(this.HOME); - this.saveStateToDatabase(state); - return state; - } - - async status(): Promise { - const status = await getStatus(this.HOME); - // TODO: don't include secret token in log message. - logger.debug( - `got status of ${this.project_id} = ${JSON.stringify(status)}`, - ); - await this.saveStatusToDatabase(status); - return status; - } - - async start(): Promise { - logger.debug("start", this.project_id); - if (this.stateChanging != null) return; - - // Home directory - const HOME = this.HOME; - - if (await isProjectRunning(HOME)) { - logger.debug("start -- already running"); - await this.saveStateToDatabase({ state: "running" }); - return; - } - - try { - this.stateChanging = { state: "starting" }; - await this.saveStateToDatabase(this.stateChanging); - await this.computeQuota(); - await mkdir(HOME, { recursive: true }); - await ensureConfFilesExists(HOME); - - // this.get('env') = extra env vars for project (from synctable): - const env = await getEnvironment(this.project_id); - logger.debug(`start ${this.project_id}: env = ${JSON.stringify(env)}`); - - // Setup files - await setupDataPath(HOME); - - await writeSecretToken( - HOME, - await getProjectSecretToken(this.project_id), - ); - - // Fork and launch project server - await launchProjectDaemon(env); - await this.touch(undefined, { noStart: true }); - - await this.wait({ - until: async () => { - if (!(await isProjectRunning(this.HOME))) { - return false; - } - const status = await this.status(); - return !!status["hub-server.port"]; - }, - maxTime: MAX_START_TIME_MS, - }); - } finally { - this.stateChanging = undefined; - // ensure state valid in database - await this.state(); - } - } - - async stop(): Promise { - if (this.stateChanging != null) return; - logger.debug("stop: ", this.project_id); - if (!(await isProjectRunning(this.HOME))) { - logger.debug("stop: project not running so nothing to kill"); - await this.saveStateToDatabase({ state: "opened" }); - return; - } - try { - this.stateChanging = { state: "stopping" }; - await this.saveStateToDatabase(this.stateChanging); - const pid = await getProjectPID(this.HOME); - const killProject = () => { - try { - logger.debug(`stop: sending kill -${pid}`); - kill(-pid, "SIGKILL"); - } catch (err) { - // expected exception if no pid - logger.debug(`stop: kill err ${err}`); - } - }; - killProject(); - await this.wait({ - until: async () => { - if (await isProjectRunning(this.HOME)) { - killProject(); - return false; - } else { - return true; - } - }, - maxTime: MAX_STOP_TIME_MS, - }); - await deleteProjectSecretToken(this.project_id); - logger.debug("stop: project is not running"); - } finally { - this.stateChanging = undefined; - // ensure state valid. - await this.state(); - } - } - - async copyPath(opts: CopyOptions): Promise { - logger.debug("copyPath ", this.project_id, opts); - await copyPath(opts, this.project_id); - return ""; - } -} - -export default function get(project_id: string): Project { - return (getProject(project_id) as Project) ?? new Project(project_id); -} diff --git a/src/packages/server/projects/control/stop-idle-projects.test.ts b/src/packages/server/projects/control/stop-idle-projects.test.ts index 69926aa51fe..26f598d0cbf 100644 --- a/src/packages/server/projects/control/stop-idle-projects.test.ts +++ b/src/packages/server/projects/control/stop-idle-projects.test.ts @@ -4,19 +4,15 @@ */ import createProject from "@cocalc/server/projects/create"; -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; +import getPool from "@cocalc/database/pool"; import { isValidUUID } from "@cocalc/util/misc"; import { test } from "./stop-idle-projects"; const { stopIdleProjects } = test; import { delay } from "awaiting"; +import { before, after } from "@cocalc/server/test"; -beforeAll(async () => { - await initEphemeralDatabase(); -}, 15000); - -afterAll(async () => { - await getPool().end(); -}); +beforeAll(before, 15000); +afterAll(after); describe("creates a project, set various parameters, and runs idle project function, it and confirm that things work as intended", () => { let project_id; diff --git a/src/packages/server/projects/control/util.ts b/src/packages/server/projects/control/util.ts index 80f4fb0ec4f..67cefd4ab14 100644 --- a/src/packages/server/projects/control/util.ts +++ b/src/packages/server/projects/control/util.ts @@ -1,14 +1,12 @@ import { promisify } from "util"; -import { dirname, join, resolve } from "path"; -import { exec as exec0, spawn } from "child_process"; -import spawnAsync from "await-spawn"; +import { join } from "path"; +import { exec as exec0 } from "child_process"; import * as fs from "fs"; import { writeFile } from "fs/promises"; -import { projects, root } from "@cocalc/backend/data"; -import { is_valid_uuid_string } from "@cocalc/util/misc"; +import { root } from "@cocalc/backend/data"; import { callback2 } from "@cocalc/util/async-utils"; import getLogger from "@cocalc/backend/logger"; -import { CopyOptions, ProjectState, ProjectStatus } from "./base"; +import { ProjectState, ProjectStatus } from "./base"; import { getUid } from "@cocalc/backend/misc"; import base_path from "@cocalc/backend/base-path"; import { db } from "@cocalc/database"; @@ -17,7 +15,10 @@ import { conatServer } from "@cocalc/backend/data"; import { pidFilename } from "@cocalc/util/project-info"; import { executeCode } from "@cocalc/backend/execute-code"; import ensureContainingDirectoryExists from "@cocalc/backend/misc/ensure-containing-directory-exists"; - +import { + client as fileserverClient, + type Fileserver, +} from "@cocalc/server/conat/file-server"; const logger = getLogger("project-control:util"); export const mkdir = promisify(fs.mkdir); @@ -34,12 +35,15 @@ export function dataPath(HOME: string): string { return join(HOME, ".smc"); } -export function homePath(project_id: string): string { - return projects.replace("[project_id]", project_id); +let fsclient: Fileserver | null = null; +export async function homePath(project_id: string): Promise { + fsclient ??= fileserverClient(); + const { path } = await fsclient.mount({ project_id }); + return path; } -export function getUsername(project_id: string): string { - return project_id.split("-").join(""); +export function getUsername(_project_id: string): string { + return "user"; } function pidIsRunning(pid: number): boolean { @@ -55,7 +59,7 @@ function pidFile(HOME: string): string { return join(dataPath(HOME), pidFilename); } -function parseDarwinTime(output:string) : number { +function parseDarwinTime(output: string): number { // output = '{ sec = 1747866131, usec = 180679 } Wed May 21 15:22:11 2025'; const match = output.match(/sec\s*=\s*(\d+)/); @@ -72,7 +76,10 @@ export async function bootTime(): Promise { if (!_bootTime) { if (process.platform === "darwin") { // uptime isn't available on macos. - const { stdout } = await executeCode({ command: "sysctl", args: ['-n', 'kern.boottime']}); + const { stdout } = await executeCode({ + command: "sysctl", + args: ["-n", "kern.boottime"], + }); _bootTime = parseDarwinTime(stdout); } else { const { stdout } = await executeCode({ command: "uptime", args: ["-s"] }); @@ -97,18 +104,6 @@ export async function getProjectPID(HOME: string): Promise { return parseInt((await readFile(path)).toString()); } -export async function isProjectRunning(HOME: string): Promise { - try { - const pid = await getProjectPID(HOME); - //logger.debug(`isProjectRunning(HOME="${HOME}") -- pid=${pid}`); - return pidIsRunning(pid); - } catch (err) { - //logger.debug(`isProjectRunning(HOME="${HOME}") -- no pid ${err}`); - // err would happen if file doesn't exist, which means nothing to do. - return false; - } -} - export async function setupDataPath(HOME: string, uid?: number): Promise { const data = dataPath(HOME); logger.debug(`setup "${data}"...`); @@ -138,72 +133,6 @@ export async function writeSecretToken( } } -async function logLaunchParams(params): Promise { - const data = dataPath(params.env.HOME); - const path = join(data, "launch-params.json"); - try { - await writeFile(path, JSON.stringify(params, undefined, 2)); - } catch (err) { - logger.debug( - `WARNING: failed to write ${path}, which is ONLY used for debugging -- ${err}`, - ); - } -} - -export async function launchProjectDaemon(env, uid?: number): Promise { - logger.debug(`launching project daemon at "${env.HOME}"...`); - const cwd = join(root, "packages/project"); - const cmd = "pnpm"; - const args = ["cocalc-project", "--daemon", "--init", "project_init.sh"]; - logger.debug( - `"${cmd} ${args.join(" ")} from "${cwd}" as user with uid=${uid}`, - ); - logLaunchParams({ cwd, env, cmd, args, uid }); - await promisify((cb: Function) => { - const child = spawn(cmd, args, { - env, - cwd, - uid, - gid: uid, - }); - let stdout = ""; - let stderr = ""; - child.stdout.on("data", (data) => { - stdout += data.toString(); - if (stdout.length > 10000) { - stdout = stdout.slice(-5000); - } - }); - child.stderr.on("data", (data) => { - stderr += data.toString(); - if (stderr.length > 10000) { - stderr = stderr.slice(-5000); - } - }); - child.on("error", (err) => { - logger.debug(`project daemon error ${err} -- \n${stdout}\n${stderr}`); - cb(err); - }); - child.on("exit", async (code) => { - logger.debug("project daemon exited", { code, stdout, stderr }); - if (code != 0) { - try { - const s = (await readFile(env.LOGS)).toString(); - logger.debug("project log file ended: ", s.slice(-2000), { - stdout, - stderr, - }); - } catch (err) { - // there's a lot of reasons the log file might not even exist, - // e.g., debugging is not enabled - logger.debug("project log file ended - unable to read log ", err); - } - } - cb(code); - }); - })(); -} - async function exec( command: string, verbose?: boolean, @@ -216,30 +145,6 @@ async function exec( return output; } -export async function createUser(project_id: string): Promise { - const username = getUsername(project_id); - try { - await exec(`/usr/sbin/userdel ${username}`); // this also deletes the group - } catch (_) { - // See https://github.com/sagemathinc/cocalc/issues/6967 for why we try/catch everything and - // that is fine. The user may or may not already exist. - } - const uid = `${getUid(project_id)}`; - logger.debug("createUser: adding group"); - try { - await exec(`/usr/sbin/groupadd -g ${uid} -o ${username}`, true); - } catch (_) {} - logger.debug("createUser: adding user"); - try { - await exec( - `/usr/sbin/useradd -u ${uid} -g ${uid} -o ${username} -m -d ${homePath( - project_id, - )} -s /bin/bash`, - true, - ); - } catch (_) {} -} - export async function stopProjectProcesses(project_id: string): Promise { const uid = `${getUid(project_id)}`; const scmd = `pkill -9 -u ${uid} | true `; // | true since pkill exit 1 if nothing killed. @@ -307,6 +212,7 @@ export function sanitizedEnv(env: { [key: string]: string | undefined }): { export async function getEnvironment( project_id: string, + { HOME = "/home/user" }: { HOME?: string } = {}, ): Promise<{ [key: string]: any }> { const extra: { [key: string]: any } = await callback2( db().get_project_extra_env, @@ -317,7 +223,6 @@ export async function getEnvironment( ); const USER = getUsername(project_id); - const HOME = homePath(project_id); const DATA = dataPath(HOME); return { @@ -341,29 +246,20 @@ export async function getEnvironment( } export async function getState(HOME: string): Promise { - logger.debug(`getState("${HOME}")`); - try { - return { - ip: "127.0.0.1", - state: (await isProjectRunning(HOME)) ? "running" : "opened", - time: new Date(), - }; - } catch (err) { - return { - error: `${err}`, - time: new Date(), - state: "opened", - }; - } + throw Error("getState: deprecated -- redo using conat!"); + // [ ] TODO: deprecate + logger.debug(`getState("${HOME}"): DEPRECATED`); + return { + ip: "127.0.0.1", + state: "running", + time: new Date(), + }; } export async function getStatus(HOME: string): Promise { logger.debug(`getStatus("${HOME}")`); const data = dataPath(HOME); const status: ProjectStatus = {}; - if (!(await isProjectRunning(HOME))) { - return status; - } for (const path of [ "project.pid", "hub-server.port", @@ -424,154 +320,6 @@ export async function ensureConfFilesExists( } } -// Copy a path using rsync and the specified options -// on the local filesystem. -// NOTE: the scheduled CopyOptions -// are not implemented at all here. -export async function copyPath( - opts: CopyOptions, - project_id: string, - target_uid?: number, -): Promise { - logger.info(`copyPath(source="${project_id}"): opts=${JSON.stringify(opts)}`); - const { path, overwrite_newer, delete_missing, backup, timeout, bwlimit } = - opts; - if (path == null) { - // typescript already enforces this... - throw Error("path must be specified"); - } - const target_project_id = opts.target_project_id ?? project_id; - const target_path = opts.target_path ?? path; - - // check that both UUID's are valid - if (!is_valid_uuid_string(project_id)) { - throw Error(`project_id=${project_id} is invalid`); - } - if (!is_valid_uuid_string(target_project_id)) { - throw Error(`target_project_id=${target_project_id} is invalid`); - } - - // determine canonical absolute path to source - const sourceHome = homePath(project_id); - const source_abspath = resolve(join(sourceHome, path)); - if (!source_abspath.startsWith(sourceHome)) { - throw Error(`source path must be contained in project home dir`); - } - // determine canonical absolute path to target - const targetHome = homePath(target_project_id); - const target_abspath = resolve(join(targetHome, target_path)); - if (!target_abspath.startsWith(targetHome)) { - throw Error(`target path must be contained in target project home dir`); - } - - // check for trivial special case. - if (source_abspath == target_abspath) { - return; - } - - // This can throw an exception if path doesn't exist, which is fine. - const stats = await stat(source_abspath); - // We will use this to decide if we need to add / at end in rsync args. - const isDir = stats.isDirectory(); - - // Handle args and options to rsync. - // saxz = compressed, archive mode (so leave symlinks, etc.), don't cross filesystem boundaries - // However, omit-link-times -- see http://forums.whirlpool.net.au/archive/2317650 and - // https://github.com/sagemathinc/cocalc/issues/2713 - const args: string[] = []; - if (process.platform == "darwin") { - // MacOS rsync is pretty cripled, so we omit some very helpful options. - args.push("-zax"); - } else { - args.push(...["-zaxs", "--omit-link-times"]); - } - if (opts.exclude) { - for (const pattern of opts.exclude) { - args.push("--exclude"); - args.push(pattern); - } - } - if (!overwrite_newer) { - args.push("--update"); - } - if (backup) { - args.push("--backup"); - } - if (delete_missing) { - // IMPORTANT: newly created files will be deleted even if overwrite_newer is true - args.push("--delete"); - } - if (bwlimit) { - args.push(`--bwlimit=${bwlimit}`); - } - if (timeout) { - args.push(`--timeout=${timeout}`); - } - if (target_uid && target_project_id != project_id) { - // change target ownership on copy; only do this if explicitly requested and needed. - args.push(`--chown=${target_uid}:${target_uid}`); - } - - args.push(source_abspath + (isDir ? "/" : "")); - args.push(target_abspath + (isDir ? "/" : "")); - - async function make_target_path() { - // note -- uid/gid ignored if target_uid not set. - if (isDir) { - await spawnAsync("mkdir", ["-p", target_abspath], { - uid: target_uid, - gid: target_uid, - }); - } else { - await spawnAsync("mkdir", ["-p", dirname(target_abspath)], { - uid: target_uid, - gid: target_uid, - }); - } - } - - // For making the target directory when target_uid is specified, - // we need to use setuid and be the target user, since otherwise - // the permissions are wrong on the containing directory, - // as explained here: https://github.com/sagemathinc/cocalc-docker/issues/146 - // However, this will fail if the user hasn't been created, hence - // this code is extra complicated. - try { - await make_target_path(); - } catch (_err) { - // The above probably failed due to the uid/gid not existing. - // In that case, we create the user, then try again. - await createUser(target_project_id); - await make_target_path(); - // Assuming the above did work, it's very likely the original - // failing was due to the user not existing, so now we delete - // it again. - await deleteUser(target_project_id); - } - - // do the copy! - logger.info(`doing rsync ${args.join(" ")}`); - if (opts.wait_until_done ?? true) { - try { - const stdout = await spawnAsync("rsync", args, { - timeout: opts.timeout - ? 1000 * opts.timeout - : undefined /* spawnAsync has ms units, but rsync has second units */, - }); - logger.info(`finished rsync ${stdout}`); - } catch (err) { - throw Error( - `WARNING: copy exited with an error -- ${ - err.stderr - } -- "rsync ${args.join(" ")}"`, - ); - } - } else { - // TODO/NOTE: this will silently not report any errors. - spawn("rsync", args, { timeout: opts.timeout }); - } -} - export async function restartProjectIfRunning(project_id: string) { // If necessary, restart project to ensure that license gets applied. // This is not bullet proof in all cases, e.g., for a newly created project, @@ -579,9 +327,6 @@ export async function restartProjectIfRunning(project_id: string) { const project = getProject(project_id); const { state } = await project.state(); if (state == "starting" || state == "running") { - // don't await this -- it could take a long time and isn't necessary to wait for. - (async () => { - await project.restart(); - })(); + await project.restart(); } } diff --git a/src/packages/server/projects/create.ts b/src/packages/server/projects/create.ts index f2a12033750..13fbeecdb3b 100644 --- a/src/packages/server/projects/create.ts +++ b/src/packages/server/projects/create.ts @@ -15,6 +15,8 @@ import { getProject } from "@cocalc/server/projects/control"; import { type CreateProjectOptions } from "@cocalc/util/db-schema/projects"; import { delay } from "awaiting"; import isAdmin from "@cocalc/server/accounts/is-admin"; +import isCollaborator from "@cocalc/server/projects/is-collaborator"; +import { client as filesystemClient } from "@cocalc/conat/files/file-server"; const log = getLogger("server:projects:create"); @@ -34,7 +36,9 @@ export default async function createProject(opts: CreateProjectOptions) { public_path_id, noPool, start, + src_project_id, } = opts; + let license = opts.license; if (public_path_id) { const site_license_id = await associatedLicense(public_path_id); @@ -53,10 +57,10 @@ export default async function createProject(opts: CreateProjectOptions) { } project_id = opts.project_id; } else { - // Try to get from pool if no license and no image specified (so the default), + // Try to get from pool if no license and no image specified (so the default) and not cloning, // and not "noPool". NOTE: we may improve the pool to also provide some // basic licensed projects later, and better support for images. Maybe. - if (!noPool && !license && account_id != null) { + if (!src_project_id && !noPool && !license && account_id != null) { project_id = await getFromPool({ account_id, title, @@ -70,7 +74,19 @@ export default async function createProject(opts: CreateProjectOptions) { project_id = v4(); } - + + if (src_project_id) { + if ( + !account_id || + !(await isCollaborator({ account_id, project_id: src_project_id })) + ) { + throw Error("user must be a collaborator on src_project_id"); + } + // create filesystem for new project as a clone. + const client = filesystemClient(); + await client.clone({ project_id, src_project_id }); + } + const pool = getPool(); const users = account_id == null ? null : { [account_id]: { group: "owner" } }; diff --git a/src/packages/server/purchases/closing-date.test.ts b/src/packages/server/purchases/closing-date.test.ts index cb88f3e42f7..2877b61e440 100644 --- a/src/packages/server/purchases/closing-date.test.ts +++ b/src/packages/server/purchases/closing-date.test.ts @@ -10,16 +10,13 @@ import { setClosingDay, } from "./closing-date"; import { uuid } from "@cocalc/util/misc"; -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import createAccount from "@cocalc/server/accounts/create-account"; +import { before, after } from "@cocalc/server/test"; beforeAll(async () => { - await initEphemeralDatabase(); + await before({ noConat: true }); }, 15000); - -afterAll(async () => { - await getPool().end(); -}); +afterAll(after); describe("basic consistency checks for closing date functions", () => { const account_id = uuid(); diff --git a/src/packages/server/purchases/edit-license.test.ts b/src/packages/server/purchases/edit-license.test.ts index 4e1a9183a7b..4d133311d88 100644 --- a/src/packages/server/purchases/edit-license.test.ts +++ b/src/packages/server/purchases/edit-license.test.ts @@ -5,10 +5,7 @@ import dayjs from "dayjs"; -import getPool, { - getPoolClient, - initEphemeralDatabase, -} from "@cocalc/database/pool"; +import { getPoolClient } from "@cocalc/database/pool"; import createAccount from "@cocalc/server/accounts/create-account"; import getLicense from "@cocalc/server/licenses/get-license"; import createLicense from "@cocalc/server/licenses/purchase/create-license"; @@ -21,14 +18,12 @@ import editLicenseOwner from "./edit-license-owner"; import getSubscriptions from "./get-subscriptions"; import purchaseShoppingCartItem from "./purchase-shopping-cart-item"; import { license0 } from "./test-data"; +import { before, after, getPool } from "@cocalc/server/test"; beforeAll(async () => { - await initEphemeralDatabase(); + await before({ noConat: true }); }, 15000); - -afterAll(async () => { - await getPool().end(); -}); +afterAll(after); describe("create a license and then edit it in various ways", () => { const account_id = uuid(); diff --git a/src/packages/server/purchases/get-balance.test.ts b/src/packages/server/purchases/get-balance.test.ts index a21de53a5d3..2c76adc7f4f 100644 --- a/src/packages/server/purchases/get-balance.test.ts +++ b/src/packages/server/purchases/get-balance.test.ts @@ -6,16 +6,13 @@ import getBalance from "./get-balance"; import createPurchase from "./create-purchase"; import { uuid } from "@cocalc/util/misc"; -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import dayjs from "dayjs"; +import { before, after } from "@cocalc/server/test"; beforeAll(async () => { - await initEphemeralDatabase(); + await before({ noConat: true }); }, 15000); - -afterAll(async () => { - await getPool().end(); -}); +afterAll(after); describe("test computing balance under various conditions", () => { const account_id = uuid(); diff --git a/src/packages/server/purchases/get-min-balance.test.ts b/src/packages/server/purchases/get-min-balance.test.ts index 4c509c11eee..0a804c839fd 100644 --- a/src/packages/server/purchases/get-min-balance.test.ts +++ b/src/packages/server/purchases/get-min-balance.test.ts @@ -1,15 +1,12 @@ import { uuid } from "@cocalc/util/misc"; -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import { createTestAccount } from "./test-data"; import getMinBalance from "./get-min-balance"; +import { before, after, getPool } from "@cocalc/server/test"; beforeAll(async () => { - await initEphemeralDatabase({}); + await before({ noConat: true }); }, 15000); - -afterAll(async () => { - await getPool().end(); -}); +afterAll(after); describe("test that getMinBalance works", () => { const account_id = uuid(); diff --git a/src/packages/server/purchases/get-purchases.test.ts b/src/packages/server/purchases/get-purchases.test.ts index e63317cdd0c..4504037fa43 100644 --- a/src/packages/server/purchases/get-purchases.test.ts +++ b/src/packages/server/purchases/get-purchases.test.ts @@ -7,16 +7,13 @@ import createAccount from "@cocalc/server/accounts/create-account"; import createPurchase from "./create-purchase"; import getPurchases from "./get-purchases"; import { uuid } from "@cocalc/util/misc"; -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import dayjs from "dayjs"; +import { before, after, getPool } from "@cocalc/server/test"; beforeAll(async () => { - await initEphemeralDatabase(); + await before({ noConat: true }); }, 15000); - -afterAll(async () => { - await getPool().end(); -}); +afterAll(after); describe("creates and get purchases using various options", () => { const account_id = uuid(); diff --git a/src/packages/server/purchases/get-service-cost.test.ts b/src/packages/server/purchases/get-service-cost.test.ts index 23039b80ee4..3974b6bbc1b 100644 --- a/src/packages/server/purchases/get-service-cost.test.ts +++ b/src/packages/server/purchases/get-service-cost.test.ts @@ -1,13 +1,10 @@ -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import getServiceCost from "./get-service-cost"; +import { before, after } from "@cocalc/server/test"; beforeAll(async () => { - await initEphemeralDatabase({}); + await before({ noConat: true }); }, 15000); - -afterAll(async () => { - await getPool().end(); -}); +afterAll(after); describe("get some service costs", () => { it("get service cost of openai-gpt-3.5-turbo", async () => { @@ -21,7 +18,7 @@ describe("get some service costs", () => { but it WILL changes over time as we change our rates and openai does as well, so we just test that the keys are there and the values are positive but relatively small. - + NOTE: This is the cost to us, but we don't actually charge users now for this. */ expect(cost.completion_tokens).toBeGreaterThan(0); @@ -74,7 +71,7 @@ describe("get some service costs", () => { it("throws error on invalid service", async () => { await expect( - async () => await getServiceCost("nonsense" as any) + async () => await getServiceCost("nonsense" as any), ).rejects.toThrow(); }); }); diff --git a/src/packages/server/purchases/get-spend-rate.test.ts b/src/packages/server/purchases/get-spend-rate.test.ts index 66aac5ffe51..231d71e9cca 100644 --- a/src/packages/server/purchases/get-spend-rate.test.ts +++ b/src/packages/server/purchases/get-spend-rate.test.ts @@ -1,16 +1,13 @@ -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import getSpendRate from "./get-spend-rate"; import { uuid } from "@cocalc/util/misc"; import createAccount from "@cocalc/server/accounts/create-account"; import createPurchase from "./create-purchase"; +import { before, after, getPool } from "@cocalc/server/test"; beforeAll(async () => { - await initEphemeralDatabase({}); + await before({ noConat: true }); }, 15000); - -afterAll(async () => { - await getPool().end(); -}); +afterAll(after); describe("get the spend rate of a user under various circumstances", () => { const account_id = uuid(); diff --git a/src/packages/server/purchases/is-purchase-allowed.test.ts b/src/packages/server/purchases/is-purchase-allowed.test.ts index cea57fad2af..ab69f5714c4 100644 --- a/src/packages/server/purchases/is-purchase-allowed.test.ts +++ b/src/packages/server/purchases/is-purchase-allowed.test.ts @@ -3,7 +3,6 @@ * License: MS-RSL – see LICENSE.md for details */ -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import { getServerSettings } from "@cocalc/database/settings/server-settings"; import { MAX_COST } from "@cocalc/util/db-schema/purchases"; import { uuid } from "@cocalc/util/misc"; @@ -14,14 +13,12 @@ import { isPurchaseAllowed, } from "./is-purchase-allowed"; import { getPurchaseQuota, setPurchaseQuota } from "./purchase-quotas"; +import { before, after } from "@cocalc/server/test"; beforeAll(async () => { - await initEphemeralDatabase(); + await before({ noConat: true }); }, 15000); - -afterAll(async () => { - await getPool().end(); -}); +afterAll(after); describe("test checking whether or not purchase is allowed under various conditions", () => { const account_id = uuid(); diff --git a/src/packages/server/purchases/maintain-automatic-payments.test.ts b/src/packages/server/purchases/maintain-automatic-payments.test.ts index 63ff071af85..e339541bd09 100644 --- a/src/packages/server/purchases/maintain-automatic-payments.test.ts +++ b/src/packages/server/purchases/maintain-automatic-payments.test.ts @@ -5,7 +5,6 @@ // test the automatic payments maintenance loop -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import maintainAutomaticPayments, { setMockCollectPayment, } from "./maintain-automatic-payments"; @@ -13,18 +12,18 @@ import { uuid } from "@cocalc/util/misc"; import { createTestAccount } from "./test-data"; import createCredit from "./create-credit"; import { getServerSettings } from "@cocalc/database/settings"; +import { before, after, getPool } from "@cocalc/server/test"; -const collect: { account_id: string; amount: number }[] = []; beforeAll(async () => { + await before({ noConat: true }); setMockCollectPayment(async ({ account_id, amount }) => { collect.push({ account_id, amount }); }); - await initEphemeralDatabase({}); }, 15000); -afterAll(async () => { - await getPool().end(); -}); +afterAll(after); + +const collect: { account_id: string; amount: number }[] = []; describe("testing automatic payments in several situations", () => { const account_id1 = uuid(); diff --git a/src/packages/server/purchases/maintain-subscriptions.test.ts b/src/packages/server/purchases/maintain-subscriptions.test.ts index 9f9faa2242d..a8f68d1bbf9 100644 --- a/src/packages/server/purchases/maintain-subscriptions.test.ts +++ b/src/packages/server/purchases/maintain-subscriptions.test.ts @@ -1,13 +1,11 @@ import maintainSubscriptions from "./maintain-subscriptions"; -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; +import { initEphemeralDatabase } from "@cocalc/database/pool"; +import { before, after } from "@cocalc/server/test"; beforeAll(async () => { - await initEphemeralDatabase({}); + await before({ noConat: true }); }, 15000); - -afterAll(async () => { - await getPool().end(); -}); +afterAll(after); describe("test maintainSubscriptions", () => { it("run maintainSubscriptions once and it doesn't crash", async () => { diff --git a/src/packages/server/purchases/purchase-shopping-cart-item.test.ts b/src/packages/server/purchases/purchase-shopping-cart-item.test.ts index 4df853c77ef..7770fe31daa 100644 --- a/src/packages/server/purchases/purchase-shopping-cart-item.test.ts +++ b/src/packages/server/purchases/purchase-shopping-cart-item.test.ts @@ -6,10 +6,7 @@ import createAccount from "@cocalc/server/accounts/create-account"; import getLicense from "@cocalc/server/licenses/get-license"; import { uuid } from "@cocalc/util/misc"; -import getPool, { - initEphemeralDatabase, - getPoolClient, -} from "@cocalc/database/pool"; +import { getPoolClient } from "@cocalc/database/pool"; import purchaseShoppingCartItem from "./purchase-shopping-cart-item"; import { computeCost } from "@cocalc/util/licenses/store/compute-cost"; import { getClosingDay, setClosingDay } from "./closing-date"; @@ -19,14 +16,12 @@ import dayjs from "dayjs"; import cancelSubscription from "./cancel-subscription"; import resumeSubscription from "./resume-subscription"; import createPurchase from "./create-purchase"; +import { before, after, getPool } from "@cocalc/server/test"; beforeAll(async () => { - await initEphemeralDatabase(); + await before({ noConat: true }); }, 15000); - -afterAll(async () => { - await getPool().end(); -}); +afterAll(after); describe("create a subscription license and edit it and confirm the subscription cost changes", () => { // this is a shopping cart item, which I basically copied from the database... diff --git a/src/packages/server/purchases/renew-subscription.test.ts b/src/packages/server/purchases/renew-subscription.test.ts index 232824f1ef9..a76390fd546 100644 --- a/src/packages/server/purchases/renew-subscription.test.ts +++ b/src/packages/server/purchases/renew-subscription.test.ts @@ -6,21 +6,18 @@ // test renew-subscriptions import { test } from "./renew-subscription"; -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import { uuid } from "@cocalc/util/misc"; import renewSubscription, { getSubscription } from "./renew-subscription"; import { createTestAccount, createTestSubscription } from "./test-data"; import dayjs from "dayjs"; import createCredit from "./create-credit"; import getBalance from "./get-balance"; +import { before, after } from "@cocalc/server/test"; beforeAll(async () => { - await initEphemeralDatabase({}); + await before({ noConat: true }); }, 15000); - -afterAll(async () => { - await getPool().end(); -}); +afterAll(after); describe("create a subscription, then renew it", () => { const account_id = uuid(); diff --git a/src/packages/server/purchases/resume-subscription.test.ts b/src/packages/server/purchases/resume-subscription.test.ts index b26800e4c8a..f2cc88526ed 100644 --- a/src/packages/server/purchases/resume-subscription.test.ts +++ b/src/packages/server/purchases/resume-subscription.test.ts @@ -5,7 +5,6 @@ // test resuming a canceled subscription -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import { uuid } from "@cocalc/util/misc"; import { createTestAccount, createTestSubscription } from "./test-data"; //import dayjs from "dayjs"; @@ -14,14 +13,12 @@ import cancelSubscription from "./cancel-subscription"; import { getSubscription } from "./renew-subscription"; import getLicense from "@cocalc/server/licenses/get-license"; import getBalance from "./get-balance"; +import { before, after } from "@cocalc/server/test"; beforeAll(async () => { - await initEphemeralDatabase({}); + await before({ noConat: true }); }, 15000); - -afterAll(async () => { - await getPool().end(); -}); +afterAll(after); describe("create a subscription, cancel it, then resume it", () => { const account_id = uuid(); @@ -70,45 +67,4 @@ describe("create a subscription, cancel it, then resume it", () => { expect(license.expires).toBe(license2.expires); expect(license2.expires).toBe(sub.current_period_end.valueOf()); }); - - /* - - - it("cancels again but delete all of our money, but renew does NOT fail since it doesn't require a payment.", async () => { - await cancelSubscription({ - account_id, - subscription_id, - }); - const pool = getPool(); - await pool.query("DELETE FROM purchases WHERE account_id=$1", [account_id]); - expect.assertions(1); - try { - await resumeSubscription({ account_id, subscription_id }); - } catch (e) { - expect(e.message).toMatch("Please pay"); - } - }); - - it("cancels again then change date so subscription is expired, so renew does fail due to lack of money", async () => { - await cancelSubscription({ - account_id, - subscription_id, - }); - const pool = getPool(); - await pool.query( - "update subscriptions set current_period_end=NOW() - '1 month', current_period_start=NOW()-'2 months' WHERE id=$1", - [subscription_id], - ); - await pool.query( - "update site_licenses set expire=NOW() - '1 month' where id=$1", - [license_id], - ); - expect.assertions(1); - try { - await resumeSubscription({ account_id, subscription_id }); - } catch (e) { - expect(e.message).toMatch("Please pay"); - } - }); - */ }); diff --git a/src/packages/server/purchases/shift-subscription.2.test.ts b/src/packages/server/purchases/shift-subscription.2.test.ts index 02240d2d455..c2815e2fa5a 100644 --- a/src/packages/server/purchases/shift-subscription.2.test.ts +++ b/src/packages/server/purchases/shift-subscription.2.test.ts @@ -3,24 +3,19 @@ * License: MS-RSL – see LICENSE.md for details */ -import getPool, { - getPoolClient, - initEphemeralDatabase, -} from "@cocalc/database/pool"; +import { getPoolClient } from "@cocalc/database/pool"; import { uuid } from "@cocalc/util/misc"; import { createTestAccount, createTestSubscription } from "./test-data"; import { getSubscription } from "./renew-subscription"; import getLicense from "@cocalc/server/licenses/get-license"; import { test } from "./shift-subscriptions"; import { setClosingDay } from "./closing-date"; +import { before, after } from "@cocalc/server/test"; beforeAll(async () => { - await initEphemeralDatabase({}); + await before({ noConat: true }); }, 15000); - -afterAll(async () => { - await getPool().end(); -}); +afterAll(after); describe("test shiftSubscriptionToEndOnDay -- involves actual subscriptions", () => { const account_id = uuid(); @@ -31,9 +26,8 @@ describe("test shiftSubscriptionToEndOnDay -- involves actual subscriptions", () await createTestAccount(account_id); await setClosingDay(account_id, 3); - ({ subscription_id, license_id } = await createTestSubscription( - account_id - )); + ({ subscription_id, license_id } = + await createTestSubscription(account_id)); }); // it("confirms that the newly created subscription has a current period end day of 3", async () => { diff --git a/src/packages/server/purchases/statements/create-statements.test.ts b/src/packages/server/purchases/statements/create-statements.test.ts index ae22dc0dfea..65705e0df22 100644 --- a/src/packages/server/purchases/statements/create-statements.test.ts +++ b/src/packages/server/purchases/statements/create-statements.test.ts @@ -3,7 +3,6 @@ * License: MS-RSL – see LICENSE.md for details */ -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import { createTestAccount } from "@cocalc/server/purchases/test-data"; import createPurchase from "@cocalc/server/purchases/create-purchase"; import { createStatements, _TEST_ } from "./create-statements"; @@ -13,14 +12,12 @@ import getStatements from "./get-statements"; import getPurchases from "../get-purchases"; import dayjs from "dayjs"; import { closeAndContinuePurchase } from "../project-quotas"; +import { before, after, getPool } from "@cocalc/server/test"; beforeAll(async () => { - await initEphemeralDatabase(); + await before({ noConat: true }); }, 15000); - -afterAll(async () => { - await getPool().end(); -}); +afterAll(after); describe("creates an account, then creates purchases and statements", () => { const account_id = uuid(); diff --git a/src/packages/server/purchases/statements/email-statement.test.ts b/src/packages/server/purchases/statements/email-statement.test.ts index 9923cd7da1b..b7dbe9decc4 100644 --- a/src/packages/server/purchases/statements/email-statement.test.ts +++ b/src/packages/server/purchases/statements/email-statement.test.ts @@ -4,21 +4,18 @@ */ import emailStatement from "./email-statement"; -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import { createTestAccount } from "@cocalc/server/purchases/test-data"; import createPurchase from "@cocalc/server/purchases/create-purchase"; import { createStatements } from "./create-statements"; import { uuid } from "@cocalc/util/misc"; import { delay } from "awaiting"; import getStatements from "./get-statements"; +import { before, after, getPool } from "@cocalc/server/test"; beforeAll(async () => { - await initEphemeralDatabase(); + await before({ noConat: true }); }, 15000); - -afterAll(async () => { - await getPool().end(); -}); +afterAll(after); describe("creates an account, then creates statements and corresponding emails and test that everything matches up", () => { const account_id = uuid(); diff --git a/src/packages/server/purchases/student-pay.test.ts b/src/packages/server/purchases/student-pay.test.ts index 729ef2d1238..d5014cee041 100644 --- a/src/packages/server/purchases/student-pay.test.ts +++ b/src/packages/server/purchases/student-pay.test.ts @@ -1,19 +1,14 @@ import { uuid } from "@cocalc/util/misc"; -import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; import { createTestAccount } from "./test-data"; import studentPay from "./student-pay"; import createProject from "@cocalc/server/projects/create"; import createCredit from "./create-credit"; import dayjs from "dayjs"; import { delay } from "awaiting"; +import { before, after, getPool } from "@cocalc/server/test"; -beforeAll(async () => { - await initEphemeralDatabase({}); -}, 15000); - -afterAll(async () => { - await getPool().end(); -}); +beforeAll(before); +afterAll(after); describe("test studentPay behaves at it should in various scenarios", () => { const account_id = uuid(); @@ -90,7 +85,7 @@ describe("test studentPay behaves at it should in various scenarios", () => { } }); - let purchase_id_from_student_pay : undefined | number = 0; + let purchase_id_from_student_pay: undefined | number = 0; it("add a lot of money, so it finally works -- check that the license is applied to the project", async () => { await createCredit({ account_id, amount: 1000 }); const { purchase_id } = await studentPay({ account_id, project_id }); diff --git a/src/packages/server/purchases/student-pay.ts b/src/packages/server/purchases/student-pay.ts index 5cc834088c8..0670d45e888 100644 --- a/src/packages/server/purchases/student-pay.ts +++ b/src/packages/server/purchases/student-pay.ts @@ -108,17 +108,6 @@ export default async function studentPay({ // Add license to the project. await addLicenseToProject({ project_id, license_id, client }); - // nonblocking restart if running - not part of transaction, could take a while, - // and no need to block everything else on this. - (async () => { - try { - await restartProjectIfRunning(project_id); - } catch (err) { - // non-fatal, since it's just a convenience. - logger.debug("WARNING -- issue restarting project ", err); - } - })(); - if (purchaseInfo.start == null || purchaseInfo.end == null) { throw Error("start and end must be set"); } @@ -161,6 +150,17 @@ export default async function studentPay({ // end atomic transaction client.release(); } + + // Do NOT try to restart the project until outside of the transaction! + // Otherwise it deadlocks the database, since this changes the state + // of the project but not as part of the transaction!!!!! That's + // why this code is down here and not up there. + try { + await restartProjectIfRunning(project_id); + } catch (err) { + // non-fatal, since it's just a convenience. + logger.debug("WARNING -- issue restarting project ", err); + } } export function getCost( diff --git a/src/packages/server/test/index.ts b/src/packages/server/test/index.ts new file mode 100644 index 00000000000..b154ebbdf16 --- /dev/null +++ b/src/packages/server/test/index.ts @@ -0,0 +1,77 @@ +/* +Setup an ephemeral environment in process for running tests. This includes a conat socket.io server, +file server, etc. + +TODO: it would be nice to use pglite as an *option* here so there is no need to run a separate database +server. We still need full postgres though, so we can test the ancient versions we use in production, +since pglite is only very recent postgres. +*/ + +import getPool, { initEphemeralDatabase } from "@cocalc/database/pool"; +import { + before as conatTestInit, + after as conatTestClose, + connect, + client, + wait, +} from "@cocalc/backend/conat/test/setup"; +import { localPathFileserver } from "@cocalc/backend/conat/files/local-path"; +import { init as initFileserver } from "@cocalc/server/conat/file-server"; +import { + before as fileserverTestInit, + after as fileserverTestClose, +} from "@cocalc/file-server/btrfs/test/setup"; +import { init as initProjectRunner } from "@cocalc/server/conat/project/run"; +import { init as initProjectRunnerLoadBalancer } from "@cocalc/server/conat/project/load-balancer"; +import { delay } from "awaiting"; + +export { client, connect, getPool, initEphemeralDatabase, wait }; + +let opts: any = {}; +export async function before({ + noConat, + noFileserver, + noDatabase, +}: { noConat?: boolean; noFileserver?: boolean; noDatabase?: boolean } = {}) { + opts = { + noConat, + noFileserver, + noDatabase, + }; + if (!noDatabase) { + await initEphemeralDatabase(); + } + + if (!noConat) { + // run a conat socketio server + await conatTestInit(); + } + + if (!noFileserver && !noConat) { + // run server that can provides an enchanced fs module for files on the local filesystem + await localPathFileserver(); + + const ephemeralFilesystem = await fileserverTestInit(); + // server that provides a btrfs managed filesystem + await initFileserver(ephemeralFilesystem); + + await initProjectRunner(); + await initProjectRunnerLoadBalancer(); + } +} + +export async function after() { + const { noConat, noFileserver, noDatabase } = opts; + if (!noDatabase) { + await getPool().end(); + } + + if (!noFileserver && !noConat) { + await fileserverTestClose(); + await delay(1000); + } + + if (!noConat) { + await conatTestClose(); + } +} diff --git a/src/packages/server/test/setup.js b/src/packages/server/test/setup.js index 68b402d5f5d..a38883f7819 100644 --- a/src/packages/server/test/setup.js +++ b/src/packages/server/test/setup.js @@ -6,4 +6,4 @@ process.env.PGDATABASE = "smc_ephemeral_testing_database"; // checked for in some code to behave differently while running unit tests. process.env.COCALC_TEST_MODE = true; -process.env.COCALC_MODE = "single-user"; +delete process.env.CONAT_SERVER; diff --git a/src/packages/static/src/rspack.config.ts b/src/packages/static/src/rspack.config.ts index c79f2282ba5..dd973ef7d59 100644 --- a/src/packages/static/src/rspack.config.ts +++ b/src/packages/static/src/rspack.config.ts @@ -168,7 +168,10 @@ export default function getConfig({ middleware }: Options = {}): Configuration { const config: Configuration = { // this makes things 10x slower: //cache: RSPACK_DEV_SERVER || PRODMODE ? false : true, - ignoreWarnings: [/Failed to parse source map/], + ignoreWarnings: [ + /Failed to parse source map/, + /formItemNode = ReactDOM.findDOMNode/, + ], devtool: PRODMODE ? undefined : "eval-cheap-module-source-map", mode: PRODMODE ? ("production" as "production") diff --git a/src/packages/sync/client/sync-client.ts b/src/packages/sync/client/sync-client.ts index 56db603ed7d..2edd1eb0080 100644 --- a/src/packages/sync/client/sync-client.ts +++ b/src/packages/sync/client/sync-client.ts @@ -8,9 +8,8 @@ Functionality related to Sync. */ import { once } from "@cocalc/util/async-utils"; -import { defaults, required } from "@cocalc/util/misc"; import { SyncDoc, SyncOpts0 } from "@cocalc/sync/editor/generic/sync-doc"; -import { SyncDB, SyncDBOpts0 } from "@cocalc/sync/editor/db"; +import { SyncDBOpts0 } from "@cocalc/sync/editor/db"; import { SyncString } from "@cocalc/sync/editor/string/sync"; import { synctable, @@ -20,7 +19,6 @@ import { synctable_no_changefeed, } from "@cocalc/sync/table"; import type { AppClient } from "./types"; -import { getSyncDocType } from "@cocalc/conat/sync/syncdoc-info"; interface SyncOpts extends Omit { noCache?: boolean; @@ -71,143 +69,11 @@ export class SyncClient { ); } - // These are not working properly, e.g., if you close and open - // a LARGE jupyter notebook quickly (so save to disk takes a while), - // then it gets broken until browser refresh. The problem is that - // the doc is still closing right when a new one starts being created. - // So for now we just revert to the non-cached-here approach. - // There is other caching elsewhere. - - // public sync_string(opts: SyncOpts): SyncString { - // return syncstringCache({ ...opts, client: this.client }); - // } - - // public sync_db(opts: SyncDBOpts): SyncDB { - // return syncdbCache({ ...opts, client: this.client }); - // } - - public sync_string(opts: SyncOpts): SyncString { - const opts0: SyncOpts0 = defaults(opts, { - id: undefined, - project_id: required, - path: required, - file_use_interval: "default", - cursors: false, - patch_interval: 1000, - save_interval: 2000, - persistent: false, - data_server: undefined, - client: this.client, - ephemeral: false, - }); - return new SyncString(opts0); + public sync_string(_opts: SyncOpts): SyncString { + throw Error("deprecated"); } - public sync_db(opts: SyncDBOpts): SyncDoc { - const opts0: SyncDBOpts0 = defaults(opts, { - id: undefined, - project_id: required, - path: required, - file_use_interval: "default", - cursors: false, - patch_interval: 1000, - save_interval: 2000, - change_throttle: undefined, - persistent: false, - data_server: undefined, - - primary_keys: required, - string_cols: [], - - client: this.client, - - ephemeral: false, - }); - return new SyncDB(opts0); - } - - public async open_existing_sync_document({ - project_id, - path, - data_server, - persistent, - }: { - project_id: string; - path: string; - data_server?: string; - persistent?: boolean; - }): Promise { - const doctype = await getSyncDocType({ - project_id, - path, - client: this.client, - }); - const { type } = doctype; - const f = `sync_${type}`; - return (this as any)[f]({ - project_id, - path, - data_server, - persistent, - ...doctype.opts, - }); + public sync_db(_opts: SyncDBOpts): SyncDoc { + throw Error("deprecated"); } } - -/* -const syncdbCache = refCacheSync({ - name: "syncdb", - - createKey: ({ project_id, path }: SyncDBOpts) => { - return JSON.stringify({ project_id, path }); - }, - - createObject: (opts: SyncDBOpts) => { - const opts0: SyncDBOpts0 = defaults(opts, { - id: undefined, - project_id: required, - path: required, - file_use_interval: "default", - cursors: false, - patch_interval: 1000, - save_interval: 2000, - change_throttle: undefined, - persistent: false, - data_server: undefined, - - primary_keys: required, - string_cols: [], - - client: required, - - ephemeral: false, - }); - return new SyncDB(opts0); - }, -}); - -const syncstringCache = refCacheSync({ - name: "syncstring", - createKey: ({ project_id, path }: SyncOpts) => { - const key = JSON.stringify({ project_id, path }); - return key; - }, - - createObject: (opts: SyncOpts) => { - const opts0: SyncOpts0 = defaults(opts, { - id: undefined, - project_id: required, - path: required, - file_use_interval: "default", - cursors: false, - patch_interval: 1000, - save_interval: 2000, - persistent: false, - data_server: undefined, - client: required, - ephemeral: false, - }); - return new SyncString(opts0); - }, -}); -*/ diff --git a/src/packages/sync/editor/db/sync.ts b/src/packages/sync/editor/db/sync.ts index db970235b20..2ba806511c6 100644 --- a/src/packages/sync/editor/db/sync.ts +++ b/src/packages/sync/editor/db/sync.ts @@ -9,7 +9,10 @@ import { Document, DocType } from "../generic/types"; export interface SyncDBOpts0 extends SyncOpts0 { primary_keys: string[]; - string_cols: string[]; + string_cols?: string[]; + // format = what format to store the underlying file using: json or msgpack + // The default is json unless otherwise specified. + format?: "json" | "msgpack"; } export interface SyncDBOpts extends SyncDBOpts0 { @@ -25,19 +28,19 @@ export class SyncDB extends SyncDoc { throw Error("primary_keys must have length at least 1"); } opts1.from_str = (str) => - from_str(str, opts1.primary_keys, opts1.string_cols); + from_str(str, opts1.primary_keys, opts1.string_cols ?? []); opts1.doctype = { type: "db", patch_format: 1, opts: { primary_keys: opts1.primary_keys, - string_cols: opts1.string_cols, + string_cols: opts1.string_cols ?? [], }, }; super(opts1 as SyncOpts); } - get_one(arg?) : any { + get_one(arg?): any { // I know it is really of type DBDocument. return (this.get_doc() as DBDocument).get_one(arg); } diff --git a/src/packages/sync/editor/generic/sorted-patch-list.ts b/src/packages/sync/editor/generic/sorted-patch-list.ts index 5d35c08bec4..6a2be95f958 100644 --- a/src/packages/sync/editor/generic/sorted-patch-list.ts +++ b/src/packages/sync/editor/generic/sorted-patch-list.ts @@ -102,7 +102,7 @@ export class SortedPatchList extends EventEmitter { }; /* Choose the next available time in ms that is congruent to - m modulo n and is larger than any current times. + m modulo n and is larger than any current times. This is a LOGICAL TIME; it does not have to equal the actual wall clock. The key is that it is increasing. The congruence condition is so that any time @@ -134,9 +134,13 @@ export class SortedPatchList extends EventEmitter { if (n <= 0) { n = 1; } - let a = m - (time % n); + // we add 50 to the modulus so that if a bunch of new users are joining at the exact same moment, + // they don't have to be instantly aware of each other for this to keep working. Basically, we + // give ourself a buffer of 10 + const modulus = n + 10; + let a = m - (time % modulus); if (a < 0) { - a += n; + a += modulus; } time += a; // now time = m (mod n) // There is also no possibility of a conflict with a known time diff --git a/src/packages/sync/editor/generic/sync-doc.ts b/src/packages/sync/editor/generic/sync-doc.ts index 213775e8d3d..d9642da3702 100644 --- a/src/packages/sync/editor/generic/sync-doc.ts +++ b/src/packages/sync/editor/generic/sync-doc.ts @@ -19,57 +19,38 @@ EVENTS: - ... TODO */ -const USE_CONAT = true; - -/* OFFLINE_THRESH_S - If the client becomes disconnected from - the backend for more than this long then---on reconnect---do - extra work to ensure that all snapshots are up to date (in - case snapshots were made when we were offline), and mark the - sent field of patches that weren't saved. I.e., we rebase - all offline changes. */ -// const OFFLINE_THRESH_S = 5 * 60; // 5 minutes. - -/* How often the local hub will autosave this file to disk if - it has it open and there are unsaved changes. This is very - important since it ensures that a user that edits a file but - doesn't click "Save" and closes their browser (right after - their edits have gone to the database), still has their - file saved to disk soon. This is important, e.g., for homework - getting collected and not missing the last few changes. It turns - out this is what people expect. - Set to 0 to disable. (But don't do that.) */ -const FILE_SERVER_AUTOSAVE_S = 45; -// const FILE_SERVER_AUTOSAVE_S = 5; - // How big of files we allow users to open using syncstrings. const MAX_FILE_SIZE_MB = 32; -// How frequently to check if file is or is not read only. -// The filesystem watcher is NOT sufficient for this, because -// it is NOT triggered on permissions changes. Thus we must -// poll for read only status periodically, unfortunately. -const READ_ONLY_CHECK_INTERVAL_MS = 7500; - // This parameter determines throttling when broadcasting cursor position // updates. Make this larger to reduce bandwidth at the expense of making // cursors less responsive. -const CURSOR_THROTTLE_MS = 750; +const CURSOR_THROTTLE_MS = 150; -// NATS is much faster and can handle load, and cursors only uses pub/sub -const CURSOR_THROTTLE_NATS_MS = 150; +// If file does not exist for this long, then syncdoc emits a 'deleted' event. +const DELETED_THRESHOLD = 2000; +const DELETED_CHECK_INTERVAL = 750; +const WATCH_RECREATE_WAIT = 3000; -// Ignore file changes for this long after save to disk. -const RECENT_SAVE_TO_DISK_MS = 2000; +// all clients ignore file changes from when a save starts until this +// amount of time later, so they avoid loading a file just because it was +// saved by themself or another client. This is especially important for +// large files that can take a long time to save. +const IGNORE_ON_SAVE_INTERVAL = 7500; -const PARALLEL_INIT = true; +// reading file when it changes on disk is deboucned this much, e.g., +// if the file keeps changing you won't see those changes until it +// stops changing for this long. +const WATCH_DEBOUNCE = 250; import { COMPUTE_THRESH_MS, COMPUTER_SERVER_CURSOR_TYPE, decodeUUIDtoNum, - SYNCDB_PARAMS as COMPUTE_SERVE_MANAGER_SYNCDB_PARAMS, } from "@cocalc/util/compute/manager"; +const STAT_DEBOUNCE = 10000; + import { DEFAULT_SNAPSHOT_INTERVAL } from "@cocalc/util/db-schema/syncstring-schema"; type XPatch = any; @@ -80,16 +61,13 @@ import { callback2, cancel_scheduled, once, - retry_until_success, - reuse_in_flight_methods, until, + asyncDebounce, } from "@cocalc/util/async-utils"; import { wait } from "@cocalc/util/async-wait"; import { - auxFileToOriginal, assertDefined, close, - endswith, field_cmp, filename_extension, hash_string, @@ -110,14 +88,13 @@ import type { CompressedPatch, DocType, Document, - FileWatcher, Patch, } from "./types"; import { isTestClient, patch_cmp } from "./util"; -import { CONAT_OPEN_FILE_TOUCH_INTERVAL } from "@cocalc/util/conat"; import mergeDeep from "@cocalc/util/immutable-deep-merge"; import { JUPYTER_SYNCDB_EXTENSIONS } from "@cocalc/util/jupyter/names"; import { LegacyHistory } from "./legacy"; +import { type Filesystem, type Stats } from "@cocalc/conat/files/fs"; import { getLogger } from "@cocalc/conat/client"; const DEBUG = false; @@ -151,6 +128,26 @@ export interface SyncOpts0 { // which data/changefeed server to use data_server?: DataServer; + + // filesystem interface. + fs: Filesystem; + + // if true, do not implicitly save on commit. This is very + // useful for unit testing to easily simulate offline state. + noAutosave?: boolean; + + // optional timeout for how long to wait from when a file is + // deleted until emiting a 'deleted' event. + deletedThreshold?: number; + deletedCheckInterval?: number; + // how long to wait before trying to recreate a watch -- this mainly + // matters in cases when the file is deleted and the client ignores + // the 'deleted' event. + watchRecreateWait?: number; + + // instead of the default IGNORE_ON_SAVE_INTERVAL + ignoreOnSaveInterval?: number; + watchDebounce?: number; } export interface SyncOpts extends SyncOpts0 { @@ -172,6 +169,7 @@ const logger = getLogger("sync-doc"); logger.debug("init"); export class SyncDoc extends EventEmitter { + public readonly opts: SyncOpts; public readonly project_id: string; // project_id that contains the doc public readonly path: string; // path of the file corresponding to the doc private string_id: string; @@ -183,12 +181,6 @@ export class SyncDoc extends EventEmitter { // Throttling of incoming upstream patches from project to client. private patch_interval: number = 250; - // This is what's actually output by setInterval -- it's - // not an amount of time. - private fileserver_autosave_timer: number = 0; - - private read_only_timer: number = 0; - // throttling of change events -- e.g., is useful for course // editor where we have hundreds of changes and the UI gets // overloaded unless we throttle and group them. @@ -233,22 +225,14 @@ export class SyncDoc extends EventEmitter { private settings: Map = Map(); - private syncstring_save_state: string = ""; - // patches that this client made during this editing session. private my_patches: { [time: string]: XPatch } = {}; - private watch_path?: string; - private file_watcher?: FileWatcher; - private handle_patch_update_queue_running: boolean; private patch_update_queue: string[] = []; private undo_state: UndoState | undefined; - private save_to_disk_start_ctime: number | undefined; - private save_to_disk_end_ctime: number | undefined; - private persistent: boolean = false; private last_has_unsaved_changes?: boolean = undefined; @@ -258,20 +242,27 @@ export class SyncDoc extends EventEmitter { private sync_is_disabled: boolean = false; private delay_sync_timer: any; - // static because we want exactly one across all docs! - private static computeServerManagerDoc?: SyncDoc; - private useConat: boolean; legacy: LegacyHistory; + public readonly fs: Filesystem; + + private noAutosave?: boolean; + + private readFileDebounced: Function; + public isDeleted: boolean = false; + constructor(opts: SyncOpts) { super(); + this.opts = opts; + if (opts.string_id === undefined) { this.string_id = schema.client_db.sha1(opts.project_id, opts.path); } else { this.string_id = opts.string_id; } + // TODO: it might be better to just use this.opts.field everywhere...? for (const field of [ "project_id", "path", @@ -285,12 +276,31 @@ export class SyncDoc extends EventEmitter { "persistent", "data_server", "ephemeral", + "fs", + "noAutosave", ]) { if (opts[field] != undefined) { this[field] = opts[field]; } } + this.readFileDebounced = asyncDebounce( + async () => { + try { + this.emit("handle-file-change"); + await this.readFile(); + await this.stat(); + } catch {} + }, + this.opts.watchDebounce ?? WATCH_DEBOUNCE, + { + leading: false, + trailing: true, + }, + ); + + this.client.once("closed", this.close); + this.legacy = new LegacyHistory({ project_id: this.project_id, path: this.path, @@ -300,7 +310,7 @@ export class SyncDoc extends EventEmitter { // NOTE: Do not use conat in test mode, since there we use a minimal // "fake" client that does all communication internally and doesn't // use conat. We also use this for the messages composer. - this.useConat = USE_CONAT && !isTestClient(opts.client); + this.useConat = !isTestClient(opts.client); if (this.ephemeral) { // So the doctype written to the database reflects the // ephemeral state. Here ephemeral determines whether @@ -323,13 +333,6 @@ export class SyncDoc extends EventEmitter { // to update this). this.cursor_last_time = this.client?.server_time(); - reuse_in_flight_methods(this, [ - "save", - "save_to_disk", - "load_from_disk", - "handle_patch_update_queue", - ]); - if (this.change_throttle) { this.emit_change = throttle(this.emit_change, this.change_throttle); } @@ -348,7 +351,7 @@ export class SyncDoc extends EventEmitter { this SyncDoc. */ private initialized = false; - private init = async () => { + init = reuseInFlight(async () => { if (this.initialized) { throw Error("init can only be called once"); } @@ -371,7 +374,7 @@ export class SyncDoc extends EventEmitter { } const m = `WARNING: problem initializing ${this.path} -- ${err}`; log(m); - if (DEBUG) { + if (DEBUG || true) { console.trace(err); } // log always @@ -382,171 +385,13 @@ export class SyncDoc extends EventEmitter { }, { start: 3000, max: 15000, decay: 1.3 }, ); + if (this.isClosed()) return; // Success -- everything initialized with no issues. this.set_state("ready"); - this.init_watch(); this.emit_change(); // from nothing to something. - }; - - // True if this client is responsible for managing - // the state of this document with respect to - // the file system. By default, the project is responsible, - // but it could be something else (e.g., a compute server!). It's - // important that whatever algorithm determines this, it is - // a function of state that is eventually consistent. - // IMPORTANT: whether or not we are the file server can - // change over time, so if you call isFileServer and - // set something up (e.g., autosave or a watcher), based - // on the result, you need to clear it when the state - // changes. See the function handleComputeServerManagerChange. - private isFileServer = reuseInFlight(async () => { - if (this.state == "closed") return; - if (this.client == null || this.client.is_browser()) { - // browser is never the file server (yet), and doesn't need to do - // anything related to watching for changes in state. - // Someday via webassembly or browsers making users files availabl, - // etc., we will have this. Not today. - return false; - } - const computeServerManagerDoc = this.getComputeServerManagerDoc(); - const log = this.dbg("isFileServer"); - if (computeServerManagerDoc == null) { - log("not using compute server manager for this doc"); - return this.client.is_project(); - } - - const state = computeServerManagerDoc.get_state(); - log("compute server manager doc state: ", state); - if (state == "closed") { - log("compute server manager is closed"); - // something really messed up - return this.client.is_project(); - } - if (state != "ready") { - try { - log( - "waiting for compute server manager doc to be ready; current state=", - state, - ); - await once(computeServerManagerDoc, "ready", 15000); - log("compute server manager is ready"); - } catch (err) { - log( - "WARNING -- failed to initialize computeServerManagerDoc -- err=", - err, - ); - return this.client.is_project(); - } - } - - // id of who the user *wants* to be the file server. - const path = this.getFileServerPath(); - const fileServerId = - computeServerManagerDoc.get_one({ path })?.get("id") ?? 0; - if (this.client.is_project()) { - log( - "we are project, so we are fileserver if fileServerId=0 and it is ", - fileServerId, - ); - return fileServerId == 0; - } - // at this point we have to be a compute server - const computeServerId = decodeUUIDtoNum(this.client.client_id()); - // this is usually true -- but might not be if we are switching - // directly from one compute server to another. - log("we are compute server and ", { fileServerId, computeServerId }); - return fileServerId == computeServerId; }); - private getFileServerPath = () => { - if (this.path?.endsWith("." + JUPYTER_SYNCDB_EXTENSIONS)) { - // treating jupyter as a weird special case here. - return auxFileToOriginal(this.path); - } - return this.path; - }; - - private getComputeServerManagerDoc = () => { - if (this.path == COMPUTE_SERVE_MANAGER_SYNCDB_PARAMS.path) { - // don't want to recursively explode! - return null; - } - if (SyncDoc.computeServerManagerDoc == null) { - if (this.client.is_project()) { - // @ts-ignore: TODO! - SyncDoc.computeServerManagerDoc = this.client.syncdoc({ - path: COMPUTE_SERVE_MANAGER_SYNCDB_PARAMS.path, - }); - } else { - // @ts-ignore: TODO! - SyncDoc.computeServerManagerDoc = this.client.sync_client.sync_db({ - project_id: this.project_id, - ...COMPUTE_SERVE_MANAGER_SYNCDB_PARAMS, - }); - } - if ( - SyncDoc.computeServerManagerDoc != null && - !this.client.is_browser() - ) { - // start watching for state changes - SyncDoc.computeServerManagerDoc.on( - "change", - this.handleComputeServerManagerChange, - ); - } - } - return SyncDoc.computeServerManagerDoc; - }; - - private handleComputeServerManagerChange = async (keys) => { - if (SyncDoc.computeServerManagerDoc == null) { - return; - } - let relevant = false; - for (const key of keys ?? []) { - if (key.get("path") == this.path) { - relevant = true; - break; - } - } - if (!relevant) { - return; - } - const path = this.getFileServerPath(); - const fileServerId = - SyncDoc.computeServerManagerDoc.get_one({ path })?.get("id") ?? 0; - const ourId = this.client.is_project() - ? 0 - : decodeUUIDtoNum(this.client.client_id()); - // we are considering ourself the file server already if we have - // either a watcher or autosave on. - const thinkWeAreFileServer = - this.file_watcher != null || this.fileserver_autosave_timer; - const weAreFileServer = fileServerId == ourId; - if (thinkWeAreFileServer != weAreFileServer) { - // life has changed! Let's adapt. - if (thinkWeAreFileServer) { - // we were acting as the file server, but now we are not. - await this.save_to_disk_filesystem_owner(); - // Stop doing things we are no longer supposed to do. - clearInterval(this.fileserver_autosave_timer as any); - this.fileserver_autosave_timer = 0; - // stop watching filesystem - await this.update_watch_path(); - } else { - // load our state from the disk - await this.load_from_disk(); - // we were not acting as the file server, but now we need. Let's - // step up to the plate. - // start watching filesystem - await this.update_watch_path(this.path); - // enable autosave - await this.init_file_autosave(); - } - } - }; - // Return id of ACTIVE remote compute server, if one is connected and pinging, or 0 // if none is connected. This is used by Jupyter to determine who // should evaluate code. @@ -600,7 +445,7 @@ export class SyncDoc extends EventEmitter { locs: any, side_effect: boolean = false, ) => { - if (this.state != "ready") { + if (!this.isReady()) { return; } if (this.cursors_table == null) { @@ -653,7 +498,7 @@ export class SyncDoc extends EventEmitter { set_cursor_locs: typeof this.setCursorLocsNoThrottle = throttle( this.setCursorLocsNoThrottle, - USE_CONAT ? CURSOR_THROTTLE_NATS_MS : CURSOR_THROTTLE_MS, + CURSOR_THROTTLE_MS, { leading: true, trailing: true, @@ -702,6 +547,7 @@ export class SyncDoc extends EventEmitter { }; isClosed = () => (this.state ?? "closed") == "closed"; + isReady = () => this.state == "ready"; private set_state = (state: State): void => { this.state = state; @@ -951,42 +797,6 @@ export class SyncDoc extends EventEmitter { return this.undo_state; }; - private save_to_disk_autosave = async (): Promise => { - if (this.state !== "ready") { - return; - } - const dbg = this.dbg("save_to_disk_autosave"); - dbg(); - try { - await this.save_to_disk(); - } catch (err) { - dbg(`failed -- ${err}`); - } - }; - - /* Make it so the local hub project will automatically save - the file to disk periodically. */ - private init_file_autosave = async () => { - // Do not autosave sagews until we resolve - // https://github.com/sagemathinc/cocalc/issues/974 - // Similarly, do not autosave ipynb because of - // https://github.com/sagemathinc/cocalc/issues/5216 - if ( - !FILE_SERVER_AUTOSAVE_S || - !(await this.isFileServer()) || - this.fileserver_autosave_timer || - endswith(this.path, ".sagews") || - endswith(this.path, "." + JUPYTER_SYNCDB_EXTENSIONS) - ) { - return; - } - - // Explicit cast due to node vs browser typings. - this.fileserver_autosave_timer = ( - setInterval(this.save_to_disk_autosave, FILE_SERVER_AUTOSAVE_S * 1000) - ); - }; - // account_id of the user who made the edit at // the given point in time. account_id = (time: number): string => { @@ -1014,26 +824,6 @@ export class SyncDoc extends EventEmitter { return t; }; - /* The project calls set_initialized once it has checked for - the file on disk; this way the frontend knows that the - syncstring has been initialized in the database, and also - if there was an error doing the check. - */ - private set_initialized = async ( - error: string, - read_only: boolean, - size: number, - ): Promise => { - this.assert_table_is_ready("syncstring"); - this.dbg("set_initialized")({ error, read_only, size }); - const init = { time: this.client.server_time(), size, error }; - await this.set_syncstring_table({ - init, - read_only, - last_active: this.client.server_time(), - }); - }; - /* List of logical timestamps of the versions of this string in the sync table that we opened to start editing (so starts with what was the most recent snapshot when we started). The list of timestamps @@ -1112,10 +902,6 @@ export class SyncDoc extends EventEmitter { const dbg = this.dbg("close"); dbg("close"); - SyncDoc.computeServerManagerDoc?.removeListener( - "change", - this.handleComputeServerManagerChange, - ); // // SYNC STUFF // @@ -1145,23 +931,13 @@ export class SyncDoc extends EventEmitter { cancel_scheduled(this.emit_change); } - if (this.fileserver_autosave_timer) { - clearInterval(this.fileserver_autosave_timer as any); - this.fileserver_autosave_timer = 0; - } - - if (this.read_only_timer) { - clearInterval(this.read_only_timer as any); - this.read_only_timer = 0; - } - this.patch_update_queue = []; // Stop watching for file changes. It's important to // do this *before* all the await's below, since // this syncdoc can't do anything in response to a // a file change in its current state. - this.update_watch_path(); // no input = closes it, if open + this.closeFileWatcher(); if (this.patch_list != null) { // not async -- just a data structure in memory @@ -1190,59 +966,6 @@ export class SyncDoc extends EventEmitter { this.ipywidgets_state?.close(); }; - // TODO: We **have** to do this on the client, since the backend - // **security model** for accessing the patches table only - // knows the string_id, but not the project_id/path. Thus - // there is no way currently to know whether or not the client - // has access to the patches, and hence the patches table - // query fails. This costs significant time -- a roundtrip - // and write to the database -- whenever the user opens a file. - // This fix should be to change the patches schema somehow - // to have the user also provide the project_id and path, thus - // proving they have access to the sha1 hash (string_id), but - // don't actually use the project_id and path as columns in - // the table. This requires some new idea I guess of virtual - // fields.... - // Also, this also establishes the correct doctype. - - // Since this MUST succeed before doing anything else. This is critical - // because the patches table can't be opened anywhere if the syncstring - // object doesn't exist, due to how our security works, *AND* that the - // patches table uses the string_id, which is a SHA1 hash. - private ensure_syncstring_exists_in_db = async (): Promise => { - const dbg = this.dbg("ensure_syncstring_exists_in_db"); - if (this.useConat) { - dbg("skipping -- no database"); - return; - } - - if (!this.client.is_connected()) { - dbg("wait until connected...", this.client.is_connected()); - await once(this.client, "connected"); - } - - if (this.client.is_browser() && !this.client.is_signed_in()) { - // the browser has to sign in, unlike the project (and compute servers) - await once(this.client, "signed_in"); - } - - if (this.state == ("closed" as State)) return; - - dbg("do syncstring write query..."); - - await callback2(this.client.query, { - query: { - syncstrings: { - string_id: this.string_id, - project_id: this.project_id, - path: this.path, - doctype: JSON.stringify(this.doctype), - }, - }, - }); - dbg("wrote syncstring to db - done."); - }; - private synctable = async ( query, options: any[], @@ -1277,6 +1000,7 @@ export class SyncDoc extends EventEmitter { desc: { path: this.path }, start_seq: this.last_seq, ephemeral, + noAutosave: this.noAutosave, }); if (this.last_seq) { @@ -1296,6 +1020,7 @@ export class SyncDoc extends EventEmitter { atomic: true, desc: { path: this.path }, ephemeral, + noAutosave: this.noAutosave, }); // also find the correct last_seq: @@ -1343,6 +1068,7 @@ export class SyncDoc extends EventEmitter { immutable: true, desc: { path: this.path }, ephemeral, + noAutosave: this.noAutosave, }); } else if (this.useConat && query.ipywidgets) { synctable = await this.client.synctable_conat(query, { @@ -1358,6 +1084,7 @@ export class SyncDoc extends EventEmitter { config: { max_age: 1000 * 60 * 60 * 24 }, desc: { path: this.path }, ephemeral: true, // ipywidgets state always ephemeral + noAutosave: this.noAutosave, }); } else if (this.useConat && (query.eval_inputs || query.eval_outputs)) { synctable = await this.client.synctable_conat(query, { @@ -1371,6 +1098,7 @@ export class SyncDoc extends EventEmitter { config: { max_age: 5 * 60 * 1000 }, desc: { path: this.path }, ephemeral: true, // eval state (for sagews) is always ephemeral + noAutosave: this.noAutosave, }); } else if (this.useConat) { synctable = await this.client.synctable_conat(query, { @@ -1383,6 +1111,7 @@ export class SyncDoc extends EventEmitter { immutable: true, desc: { path: this.path }, ephemeral, + noAutosave: this.noAutosave, }); } else { // only used for unit tests and the ephemeral messaging composer @@ -1429,15 +1158,10 @@ export class SyncDoc extends EventEmitter { dbg("getting table..."); this.syncstring_table = await this.synctable(query, []); - if (this.ephemeral && this.client.is_project()) { - await this.set_syncstring_table({ - doctype: JSON.stringify(this.doctype), - }); - } else { - dbg("handling the first update..."); - this.handle_syncstring_update(); - } + dbg("handling the first update..."); + this.handle_syncstring_update(); this.syncstring_table.on("change", this.handle_syncstring_update); + this.syncstring_table.on("change", this.update_has_unsaved_changes); }; // Used for internal debug logging @@ -1452,45 +1176,27 @@ export class SyncDoc extends EventEmitter { }; private initAll = async (): Promise => { + //const t0 = Date.now(); if (this.state !== "init") { throw Error("connect can only be called in init state"); } const log = this.dbg("initAll"); - log("update interest"); - this.initInterestLoop(); - - log("ensure syncstring exists"); - this.assert_not_closed("initAll -- before ensuring syncstring exists"); - await this.ensure_syncstring_exists_in_db(); - - await this.init_syncstring_table(); - this.assert_not_closed("initAll -- successful init_syncstring_table"); - log("patch_list, cursors, evaluator, ipywidgets"); this.assert_not_closed( "initAll -- before init patch_list, cursors, evaluator, ipywidgets", ); - if (PARALLEL_INIT) { - await Promise.all([ - this.init_patch_list(), - this.init_cursors(), - this.init_evaluator(), - this.init_ipywidgets(), - ]); - this.assert_not_closed( - "initAll -- successful init patch_list, cursors, evaluator, and ipywidgets", - ); - } else { - await this.init_patch_list(); - this.assert_not_closed("initAll -- successful init_patch_list"); - await this.init_cursors(); - this.assert_not_closed("initAll -- successful init_patch_cursors"); - await this.init_evaluator(); - this.assert_not_closed("initAll -- successful init_evaluator"); - await this.init_ipywidgets(); - this.assert_not_closed("initAll -- successful init_ipywidgets"); - } + await Promise.all([ + this.init_syncstring_table(), + this.init_patch_list(), + this.init_cursors(), + this.init_evaluator(), + this.init_ipywidgets(), + this.initFileWatcherFirstTime(), + ]); + this.assert_not_closed( + "initAll -- successful init patch_list, cursors, evaluator, and ipywidgets", + ); this.init_table_close_handlers(); this.assert_not_closed("initAll -- successful init_table_close_handlers"); @@ -1498,120 +1204,14 @@ export class SyncDoc extends EventEmitter { log("file_use_interval"); this.init_file_use_interval(); - if (await this.isFileServer()) { - log("load_from_disk"); - // This sets initialized, which is needed to be fully ready. - // We keep trying this load from disk until sync-doc is closed - // or it succeeds. It may fail if, e.g., the file is too - // large or is not readable by the user. They are informed to - // fix the problem... and once they do (and wait up to 10s), - // this will finish. - // if (!this.client.is_browser() && !this.client.is_project()) { - // // FAKE DELAY!!! Just to simulate flakiness / slow network!!!! - // await delay(3000); - // } - await retry_until_success({ - f: this.init_load_from_disk, - max_delay: 10000, - desc: "syncdoc -- load_from_disk", - }); - log("done loading from disk"); - } else { - if (this.patch_list!.count() == 0) { - await Promise.race([ - this.waitUntilFullyReady(), - once(this.patch_list!, "change"), - ]); - } - } - this.assert_not_closed("initAll -- load from disk"); - this.emit("init"); + await this.loadFromDiskIfNewer(); + this.emit("init"); this.assert_not_closed("initAll -- after waiting until fully ready"); - if (await this.isFileServer()) { - log("init file autosave"); - this.init_file_autosave(); - } this.update_has_unsaved_changes(); log("done"); - }; - - private init_error = (): string | undefined => { - let x; - try { - x = this.syncstring_table.get_one(); - } catch (_err) { - // if the table hasn't been initialized yet, - // it can't be in error state. - return undefined; - } - return x?.get("init")?.get("error"); - }; - - // wait until the syncstring table is ready to be - // used (so extracted from archive, etc.), - private waitUntilFullyReady = async (): Promise => { - this.assert_not_closed("wait_until_fully_ready"); - const dbg = this.dbg("wait_until_fully_ready"); - dbg(); - - if (this.client.is_browser() && this.init_error()) { - // init is set and is in error state. Give the backend a few seconds - // to try to fix this error before giving up. The browser client - // can close and open the file to retry this (as instructed). - try { - await this.syncstring_table.wait(() => !this.init_error(), 5); - } catch (err) { - // fine -- let the code below deal with this problem... - } - } - - let init; - const is_init = (t: SyncTable) => { - this.assert_not_closed("is_init"); - const tbl = t.get_one(); - if (tbl == null) { - dbg("null"); - return false; - } - init = tbl.get("init")?.toJS(); - return init != null; - }; - dbg("waiting for init..."); - await this.syncstring_table.wait(is_init, 0); - dbg("init done"); - if (init.error) { - throw Error(init.error); - } - assertDefined(this.patch_list); - if (init.size == null) { - // don't crash but warn at least. - console.warn("SYNC BUG -- init.size must be defined", { init }); - } - if ( - !this.client.is_project() && - this.patch_list.count() === 0 && - init.size - ) { - dbg("waiting for patches for nontrivial file"); - // normally this only happens in a later event loop, - // so force it now. - dbg("handling patch update queue since", this.patch_list.count()); - await this.handle_patch_update_queue(); - assertDefined(this.patch_list); - dbg("done handling, now ", this.patch_list.count()); - if (this.patch_list.count() === 0) { - // wait for a change -- i.e., project loading the file from - // disk and making available... Because init.size > 0, we know that - // there must be SOMETHING in the patches table once initialization is done. - // This is the root cause of https://github.com/sagemathinc/cocalc/issues/2382 - await once(this.patches_table, "change"); - dbg("got patches_table change"); - await this.handle_patch_update_queue(); - dbg("handled update queue"); - } - } + //console.log("initAll: done", Date.now() - t0); }; private assert_table_is_ready = (table: string): void => { @@ -1624,7 +1224,7 @@ export class SyncDoc extends EventEmitter { }; assert_is_ready = (desc: string): void => { - if (this.state != "ready") { + if (!this.isReady()) { throw Error(`must be ready -- ${desc}`); } }; @@ -1698,97 +1298,41 @@ export class SyncDoc extends EventEmitter { await Promise.all(v); }; - private pathExistsAndIsReadOnly = async (path): Promise => { + private loadFromDiskIfNewer = async (): Promise => { + const dbg = this.dbg("loadFromDiskIfNewer"); + let stats; try { - await callback2(this.client.path_access, { - path, - mode: "w", - }); - // clearly exists and is NOT read only: - return false; + stats = await this.stat(); } catch (err) { - // either it doesn't exist or it is read only - if (await callback2(this.client.path_exists, { path })) { - // it exists, so is read only and exists - return true; - } - // doesn't exist - return false; - } - }; - - private file_is_read_only = async (): Promise => { - if (await this.pathExistsAndIsReadOnly(this.path)) { - return true; - } - const path = this.getFileServerPath(); - if (path != this.path) { - if (await this.pathExistsAndIsReadOnly(path)) { + this.valueOnDisk = undefined; // nonexistent or don't know + if (err.code == "ENOENT") { + // path does not exist -- nothing further to do + return false; + } else { + // no clue return true; } } - return false; - }; - - private update_if_file_is_read_only = async (): Promise => { - const read_only = await this.file_is_read_only(); - if (this.state == "closed") { - return; - } - this.set_read_only(read_only); - }; - - private init_load_from_disk = async (): Promise => { - if (this.state == "closed") { - // stop trying, no error -- this is assumed - // in a retry_until_success elsewhere. - return; - } - if (await this.load_from_disk_if_newer()) { - throw Error("failed to load from disk"); - } - }; - - private load_from_disk_if_newer = async (): Promise => { - const last_changed = new Date(this.last_changed()); + dbg("path exists"); + const lastChanged = this.last_changed(); const firstLoad = this.versions().length == 0; - const dbg = this.dbg("load_from_disk_if_newer"); - let is_read_only: boolean = false; - let size: number = 0; - let error: string = ""; - try { - dbg("check if path exists"); - if (await callback2(this.client.path_exists, { path: this.path })) { - // the path exists - dbg("path exists -- stat file"); - const stats = await callback2(this.client.path_stat, { - path: this.path, - }); - if (firstLoad || stats.ctime > last_changed) { - dbg( - `disk file changed more recently than edits (or first load), so loading, ${stats.ctime} > ${last_changed}; firstLoad=${firstLoad}`, - ); - size = await this.load_from_disk(); - if (firstLoad) { - dbg("emitting first-load event"); - // this event is emited the first time the document is ever loaded from disk. - this.emit("first-load"); - } - dbg("loaded"); - } else { - dbg("stick with database version"); - } - dbg("checking if read only"); - is_read_only = await this.file_is_read_only(); - dbg("read_only", is_read_only); - } - } catch (err) { - error = `${err}`; + if (firstLoad || stats.mtime.valueOf() > lastChanged) { + dbg( + `disk file changed more recently than edits, so loading ${stats.ctime} > ${lastChanged}; firstLoad=${firstLoad}`, + ); + await this.readFile(); + if (firstLoad) { + dbg("emitting first-load event"); + // this event is emited the first time the document is ever + // loaded from disk. It's used, e.g., for notebook "trust" state, + // so important from a security POV. + this.emit("first-load"); + } + dbg("loaded"); + } else { + dbg("stick with sync version"); } - - await this.set_initialized(error, is_read_only, size); - dbg("done"); - return !!error; + return false; }; private patch_table_query = (cutoff?: number) => { @@ -1857,10 +1401,6 @@ export class SyncDoc extends EventEmitter { update_has_unsaved_changes(); }); - this.syncstring_table.on("change", () => { - update_has_unsaved_changes(); - }); - dbg("adding all known patches"); patch_list.add(this.get_patches()); @@ -2188,12 +1728,12 @@ export class SyncDoc extends EventEmitter { break; } } - if (this.state != "ready") { + if (!this.isReady()) { // above async waits could have resulted in state change. return; } - await this.handle_patch_update_queue(); - if (this.state != "ready") { + await this.handle_patch_update_queue(true); + if (!this.isReady()) { return; } @@ -2354,7 +1894,7 @@ export class SyncDoc extends EventEmitter { this.patch_list.add([obj]); this.patches_table.set(obj); await this.patches_table.save(); - if (this.state != "ready") { + if (!this.isReady()) { return; } @@ -2629,77 +2169,6 @@ export class SyncDoc extends EventEmitter { return this.last_save_to_disk_time; }; - private handle_syncstring_save_state = async ( - state: string, - time: Date, - ): Promise => { - // Called when the save state changes. - - /* this.syncstring_save_state is used to make it possible to emit a - 'save-to-disk' event, whenever the state changes - to indicate a save completed. - - NOTE: it is intentional that this.syncstring_save_state is not defined - the first time this function is called, so that save-to-disk - with last save time gets emitted on initial load (which, e.g., triggers - latex compilation properly in case of a .tex file). - */ - if (state === "done" && this.syncstring_save_state !== "done") { - this.last_save_to_disk_time = time; - this.emit("save-to-disk", time); - } - const dbg = this.dbg("handle_syncstring_save_state"); - dbg( - `state='${state}', this.syncstring_save_state='${this.syncstring_save_state}', this.state='${this.state}'`, - ); - if ( - this.state === "ready" && - (await this.isFileServer()) && - this.syncstring_save_state !== "requested" && - state === "requested" - ) { - this.syncstring_save_state = state; // only used in the if above - dbg("requesting save to disk -- calling save_to_disk"); - // state just changed to requesting a save to disk... - // so we do it (unless of course syncstring is still - // being initialized). - try { - // Uncomment the following to test simulating a - // random failure in save_to_disk: - // if (Math.random() < 0.5) throw Error("CHAOS MONKEY!"); // FOR TESTING ONLY. - await this.save_to_disk(); - } catch (err) { - // CRITICAL: we must unset this.syncstring_save_state (and set the save state); - // otherwise, it stays as "requested" and this if statement would never get - // run again, thus completely breaking saving this doc to disk. - // It is normal behavior that *sometimes* this.save_to_disk might - // throw an exception, e.g., if the file is temporarily deleted - // or save it called before everything is initialized, or file - // is temporarily set readonly, or maybe there is a file system error. - // Of course, the finally below will also take care of this. However, - // it's nice to record the error here. - this.syncstring_save_state = "done"; - await this.set_save({ state: "done", error: `${err}` }); - dbg(`ERROR saving to disk in handle_syncstring_save_state-- ${err}`); - } finally { - // No matter what, after the above code is run, - // the save state in the table better be "done". - // We triple check that here, though of course - // we believe the logic in save_to_disk and above - // should always accomplish this. - dbg("had to set the state to done in finally block"); - if ( - this.state === "ready" && - (this.syncstring_save_state != "done" || - this.syncstring_table_get_one().getIn(["save", "state"]) != "done") - ) { - this.syncstring_save_state = "done"; - await this.set_save({ state: "done", error: "" }); - } - } - } - }; - private handle_syncstring_update = async (): Promise => { if (this.state === "closed") { return; @@ -2710,10 +2179,6 @@ export class SyncDoc extends EventEmitter { const data = this.syncstring_table_get_one(); const x: any = data != null ? data.toJS() : undefined; - if (x != null && x.save != null) { - this.handle_syncstring_save_state(x.save.state, x.save.time); - } - dbg(JSON.stringify(x)); if (x == null || x.users == null) { dbg("new_document"); @@ -2802,218 +2267,86 @@ export class SyncDoc extends EventEmitter { this.emit("metadata-change"); }; - private init_watch = async (): Promise => { - if (!(await this.isFileServer())) { - // ensures we are NOT watching anything - await this.update_watch_path(); - return; - } - - // If path isn't being properly watched, make it so. - if (this.watch_path !== this.path) { - await this.update_watch_path(this.path); - } - - await this.pending_save_to_disk(); - }; - - private pending_save_to_disk = async (): Promise => { - this.assert_table_is_ready("syncstring"); - if (!(await this.isFileServer())) { - return; - } - - const x = this.syncstring_table.get_one(); - // Check if there is a pending save-to-disk that is needed. - if (x != null && x.getIn(["save", "state"]) === "requested") { - try { - await this.save_to_disk(); - } catch (err) { - const dbg = this.dbg("pending_save_to_disk"); - dbg(`ERROR saving to disk in pending_save_to_disk -- ${err}`); - } - } - }; + readFile = reuseInFlight(async (): Promise => { + const dbg = this.dbg("readFile"); - private update_watch_path = async (path?: string): Promise => { - const dbg = this.dbg("update_watch_path"); - if (this.file_watcher != null) { - // clean up - dbg("close"); - this.file_watcher.close(); - delete this.file_watcher; - delete this.watch_path; - } - if (path != null && this.client.is_deleted(path, this.project_id)) { - dbg(`not setting up watching since "${path}" is explicitly deleted`); - return; - } - if (path == null) { - dbg("not opening another watcher since path is null"); - this.watch_path = path; - return; - } - if (this.watch_path != null) { - // this case is impossible since we deleted it above if it is was defined. - dbg("watch_path already defined"); - return; - } - dbg("opening watcher..."); - if (this.state === "closed") { - throw Error("must not be closed"); - } - this.watch_path = path; + let size: number; + let contents; try { - if (!(await callback2(this.client.path_exists, { path }))) { - if (this.client.is_deleted(path, this.project_id)) { - dbg(`not setting up watching since "${path}" is explicitly deleted`); - return; - } - // path does not exist - dbg( - `write '${path}' to disk from syncstring in-memory database version`, - ); - const data = this.to_str(); - await callback2(this.client.write_file, { path, data }); - dbg(`wrote '${path}' to disk`); + contents = await this.fs.readFile(this.path, "utf8"); + // console.log(this.client.client.id, "read from disk --isDeleted = false"); + if (this.isDeleted) { + this.isDeleted = false; } - } catch (err) { - // This can happen, e.g, if path is read only. - dbg(`could NOT write '${path}' to disk -- ${err}`); - await this.update_if_file_is_read_only(); - // In this case, can't really setup a file watcher. - return; - } - - dbg("now requesting to watch file"); - this.file_watcher = this.client.watch_file({ path }); - this.file_watcher.on("change", this.handle_file_watcher_change); - this.file_watcher.on("delete", this.handle_file_watcher_delete); - this.setupReadOnlyTimer(); - }; - - private setupReadOnlyTimer = () => { - if (this.read_only_timer) { - clearInterval(this.read_only_timer as any); - this.read_only_timer = 0; - } - this.read_only_timer = ( - setInterval(this.update_if_file_is_read_only, READ_ONLY_CHECK_INTERVAL_MS) - ); - }; - - private handle_file_watcher_change = async (ctime: Date): Promise => { - const dbg = this.dbg("handle_file_watcher_change"); - const time: number = ctime.valueOf(); - dbg( - `file_watcher: change, ctime=${time}, this.save_to_disk_start_ctime=${this.save_to_disk_start_ctime}, this.save_to_disk_end_ctime=${this.save_to_disk_end_ctime}`, - ); - if ( - this.save_to_disk_start_ctime == null || - (this.save_to_disk_end_ctime != null && - time - this.save_to_disk_end_ctime >= RECENT_SAVE_TO_DISK_MS) - ) { - // Either we never saved to disk, or the last attempt - // to save was at least RECENT_SAVE_TO_DISK_MS ago, and it finished, - // so definitely this change event was not caused by it. - dbg("load_from_disk since no recent save to disk"); - await this.load_from_disk(); - return; - } - }; - - private handle_file_watcher_delete = async (): Promise => { - this.assert_is_ready("handle_file_watcher_delete"); - const dbg = this.dbg("handle_file_watcher_delete"); - dbg("delete: set_deleted and closing"); - await this.client.set_deleted(this.path, this.project_id); - this.close(); - }; - - private load_from_disk = async (): Promise => { - const path = this.path; - const dbg = this.dbg("load_from_disk"); - dbg(); - const exists: boolean = await callback2(this.client.path_exists, { path }); - let size: number; - if (!exists) { - dbg("file no longer exists -- setting to blank"); - size = 0; - this.from_str(""); - } else { + this.valueOnDisk = contents; dbg("file exists"); - await this.update_if_file_is_read_only(); - - const data = await callback2(this.client.path_read, { - path, - maxsize_MB: MAX_FILE_SIZE_MB, - }); - - size = data.length; - dbg(`got it -- length=${size}`); - this.from_str(data); - this.commit(); - // we also know that this is the version on disk, so we update the hash - await this.set_save({ - state: "done", - error: "", - hash: hash_string(data), - }); + size = contents.length; + this.from_str(contents); + } catch (err) { + if (err.code == "ENOENT") { + // console.log(this.client.client.id, "reset doc and set isDeleted=true"); + this.isDeleted = true; + dbg("file no longer exists -- setting to blank"); + size = 0; + this.from_str(""); + this.commit(); + } else { + throw err; + } } - // save new version to database, which we just set via from_str. + // save new version to stream, which we just set via from_str + this.commit(true); await this.save(); + this.emit("after-change"); return size; - }; + }); - private set_save = async (save: { - state: string; - error: string; - hash?: number; - expected_hash?: number; - time?: number; - }): Promise => { - this.assert_table_is_ready("syncstring"); - // set timestamp of when the save happened; this can be useful - // for coordinating running code, etc.... and is just generally useful. - const cur = this.syncstring_table_get_one().toJS()?.save; - if (cur != null) { - if ( - cur.state == save.state && - cur.error == save.error && - cur.hash == (save.hash ?? cur.hash) && - cur.expected_hash == (save.expected_hash ?? cur.expected_hash) && - cur.time == (save.time ?? cur.time) - ) { - // no genuine change, so no point in wasting cycles on updating. - return; - } - } - if (!save.time) { - save.time = Date.now(); + is_read_only = (): boolean => { + if (this.stats) { + return isReadOnlyForOwner(this.stats); + } else { + return false; } - await this.set_syncstring_table({ save }); }; - private set_read_only = async (read_only: boolean): Promise => { - this.assert_table_is_ready("syncstring"); - await this.set_syncstring_table({ read_only }); + private stats?: Stats; + stat = async (): Promise => { + const prevStats = this.stats; + this.stats = (await this.fs.stat(this.path)) as Stats; + if (prevStats?.mode != this.stats.mode) { + // used by clients to track read-only state. + this.emit("metadata-change"); + } + return this.stats; }; - is_read_only = (): boolean => { - this.assert_table_is_ready("syncstring"); - return this.syncstring_table_get_one().get("read_only"); - }; + debouncedStat = debounce( + async () => { + try { + await this.stat(); + } catch {} + }, + STAT_DEBOUNCE, + { leading: true, trailing: true }, + ); wait_until_read_only_known = async (): Promise => { - await this.wait_until_ready(); - function read_only_defined(t: SyncTable): boolean { - const x = t.get_one(); - if (x == null) { + await until( + async () => { + if (this.isClosed()) { + return true; + } + if (this.stats != null) { + return true; + } + try { + await this.stat(); + return true; + } catch {} return false; - } - return x.get("read_only") != null; - } - await this.syncstring_table.wait(read_only_defined, 5 * 60); + }, + { min: 3000 }, + ); }; /* Returns true if the current live version of this document has @@ -3024,32 +2357,21 @@ export class SyncDoc extends EventEmitter { commited to the database yet. Returns *undefined* if initialization not even done yet. */ has_unsaved_changes = (): boolean | undefined => { - if (this.state !== "ready") { - return; - } - const dbg = this.dbg("has_unsaved_changes"); - try { - return this.hash_of_saved_version() !== this.hash_of_live_version(); - } catch (err) { - dbg( - "exception computing hash_of_saved_version and hash_of_live_version", - err, - ); - // This could happen, e.g. when syncstring_table isn't connected - // in some edge case. Better to just say we don't know then crash - // everything. See https://github.com/sagemathinc/cocalc/issues/3577 + if (!this.isReady()) { return; } + return this.hasUnsavedChanges(); }; - // Returns hash of last version saved to disk (as far as we know). + // Returns hash of last version that we saved to disk or undefined + // if we haven't saved yet. + // NOTE: this does not take into account saving by another client + // anymore; it used to, but that made things much more complicated. hash_of_saved_version = (): number | undefined => { - if (this.state !== "ready") { + if (!this.isReady() || this.valueOnDisk == null || this.isDeleted) { return; } - return this.syncstring_table_get_one().getIn(["save", "hash"]) as - | number - | undefined; + return hash_string(this.valueOnDisk); }; /* Return hash of the live version of the document, @@ -3057,7 +2379,7 @@ export class SyncDoc extends EventEmitter { (TODO: write faster version of this for syncdb, which avoids converting to a string, which is a waste of time.) */ hash_of_live_version = (): number | undefined => { - if (this.state !== "ready") { + if (!this.isReady()) { return; } return hash_string(this.doc.to_str()); @@ -3070,7 +2392,7 @@ export class SyncDoc extends EventEmitter { the user to close their browser. */ has_uncommitted_changes = (): boolean => { - if (this.state !== "ready") { + if (!this.isReady()) { return false; } return this.patches_table.has_uncommitted_changes(); @@ -3082,10 +2404,15 @@ export class SyncDoc extends EventEmitter { // fine offline, and does not wait until anything // is saved to the network, etc. commit = (emitChangeImmediately = false): boolean => { - if (this.last == null || this.doc == null || this.last.is_equal(this.doc)) { + if ( + this.last == null || + this.doc == null || + (this.last.is_equal(this.doc) && + (this.patch_list?.getHeads().length ?? 0) <= 1) + ) { return false; } - // console.trace('commit'); + // console.trace("commit"); if (emitChangeImmediately) { // used for local clients. NOTE: don't do this without explicit @@ -3098,85 +2425,91 @@ export class SyncDoc extends EventEmitter { // Now save to backend as a new patch: this.emit("user-change"); - const patch = this.last.make_patch(this.doc); // must be nontrivial + const patch = this.last.make_patch(this.doc); this.last = this.doc; // ... and save that to patches table const time = this.next_patch_time(); this.commit_patch(time, patch); - this.save(); // so eventually also gets sent out. + if (!this.noAutosave) { + this.save(); // so eventually also gets sync'd out to other clients + } this.touchProject(); return true; }; - /* Initiates a save of file to disk, then waits for the - state to change. */ - save_to_disk = async (): Promise => { - if (this.state != "ready") { - // We just make save_to_disk a successful - // no operation, if the document is either - // closed or hasn't finished opening, since - // there's a lot of code that tries to save - // on exit/close or automatically, and it - // is difficult to ensure it all checks state - // properly. - return; - } - const dbg = this.dbg("save_to_disk"); - if (this.client.is_deleted(this.path, this.project_id)) { - dbg("not saving to disk because deleted"); - await this.set_save({ state: "done", error: "" }); - return; - } + // valueOnDisk = value of the file on disk, if known. If there's an + // event indicating what was on disk may have changed, this + // this.valueOnDisk is deleted until the new version is loaded. + private valueOnDisk: string | undefined = undefined; - // Make sure to include changes to the live document. - // A side effect of save if we didn't do this is potentially - // discarding them, which is obviously not good. - this.commit(); + private hasUnsavedChanges = (): boolean => { + return this.valueOnDisk != this.to_str() || this.isDeleted; + }; - dbg("initiating the save"); - if (!this.has_unsaved_changes()) { - dbg("no unsaved changes, so don't save"); - // CRITICAL: this optimization is assumed by - // autosave, etc. - await this.set_save({ state: "done", error: "" }); + writeFile = async () => { + const dbg = this.dbg("writeFile"); + if (this.client.is_deleted(this.path, this.project_id)) { + dbg("not saving to disk because deleted"); return; } - + dbg(); if (this.is_read_only()) { - dbg("read only, so can't save to disk"); - // save should fail if file is read only and there are changes - throw Error("can't save readonly file with changes to disk"); + await this.stat(); + if (this.is_read_only()) { + // it is definitely still read only. + return; + } } - // First make sure any changes are saved to the database. - // One subtle case where this matters is that loading a file - // with \r's into codemirror changes them to \n... - if (!(await this.isFileServer())) { - dbg("browser client -- sending any changes over network"); - await this.save(); - dbg("save done; now do actual save to the *disk*."); - this.assert_is_ready("save_to_disk - after save"); + const value = this.to_str(); + // include {ignore:true} with events for this long, + // so no clients waste resources loading in response to us saving + // to disk. + try { + await this.fileWatcher?.ignore( + this.opts.ignoreOnSaveInterval ?? IGNORE_ON_SAVE_INTERVAL, + ); + } catch { + // not a big problem if we can't ignore (e.g., this happens potentially + // after deleting the file or if file doesn't exist) } - + if (this.isClosed()) return; + this.last_save_to_disk_time = new Date(); try { - await this.save_to_disk_aux(); + await this.fs.writeFile(this.path, value); } catch (err) { - if (this.state != "ready") return; - const error = `save to disk failed -- ${err}`; - dbg(error); - if (await this.isFileServer()) { - this.set_save({ error, state: "done" }); + if (err.code == "EACCES") { + try { + // update read only knowledge -- that may have caused save error. + await this.stat(); + } catch {} } + throw err; } - if (this.state != "ready") return; + const lastChanged = this.last_changed(); + await this.fs.utimes(this.path, lastChanged / 1000, lastChanged / 1000); + this.valueOnDisk = value; + this.emit("save-to-disk"); + }; - if (!(await this.isFileServer())) { - dbg("now wait for the save to disk to finish"); - this.assert_is_ready("save_to_disk - waiting to finish"); - await this.wait_for_save_to_disk_done(); + /* Initiates a save of file to disk, then waits for the + state to change. */ + save_to_disk = reuseInFlight(async (): Promise => { + if (!this.isReady()) { + // We just make save_to_disk a successful + // no operation, if the document is either + // closed or hasn't finished opening, since + // there's a lot of code that tries to save + // on exit/close or automatically, and it + // is difficult to ensure it all checks state + // properly. + return; } + + this.commit(); + await this.writeFile(); this.update_has_unsaved_changes(); - }; + }); /* Export the (currently loaded) history of editing of this document to a simple JSON-able object. */ @@ -3192,7 +2525,7 @@ export class SyncDoc extends EventEmitter { }; private update_has_unsaved_changes = (): void => { - if (this.state != "ready") { + if (!this.isReady()) { // This can happen, since this is called by a debounced function. // Make it a no-op in case we're not ready. // See https://github.com/sagemathinc/cocalc/issues/3577 @@ -3205,174 +2538,6 @@ export class SyncDoc extends EventEmitter { } }; - // wait for save.state to change state. - private wait_for_save_to_disk_done = async (): Promise => { - const dbg = this.dbg("wait_for_save_to_disk_done"); - dbg(); - function until(table): boolean { - const done = table.get_one().getIn(["save", "state"]) === "done"; - dbg("checking... done=", done); - return done; - } - - let last_err: string | undefined = undefined; - const f = async () => { - dbg("f"); - if ( - this.state != "ready" || - this.client.is_deleted(this.path, this.project_id) - ) { - dbg("not ready or deleted - no longer trying to save."); - return; - } - try { - dbg("waiting until done..."); - await this.syncstring_table.wait(until, 15); - } catch (err) { - dbg("timed out after 15s"); - throw Error("timed out"); - } - if ( - this.state != "ready" || - this.client.is_deleted(this.path, this.project_id) - ) { - dbg("not ready or deleted - no longer trying to save."); - return; - } - const err = this.syncstring_table_get_one().getIn(["save", "error"]) as - | string - | undefined; - if (err) { - dbg("error", err); - last_err = err; - throw Error(err); - } - dbg("done, with no error."); - last_err = undefined; - return; - }; - await retry_until_success({ - f, - max_tries: 8, - desc: "wait_for_save_to_disk_done", - }); - if ( - this.state != "ready" || - this.client.is_deleted(this.path, this.project_id) - ) { - return; - } - if (last_err && typeof this.client.log_error != null) { - this.client.log_error?.({ - string_id: this.string_id, - path: this.path, - project_id: this.project_id, - error: `Error saving file -- ${last_err}`, - }); - } - }; - - /* Auxiliary function 2 for saving to disk: - If this is associated with - a project and has a filename. - A user (web browsers) sets the save state to requested. - The project sets the state to saving, does the save - to disk, then sets the state to done. - */ - private save_to_disk_aux = async (): Promise => { - this.assert_is_ready("save_to_disk_aux"); - - if (!(await this.isFileServer())) { - return await this.save_to_disk_non_filesystem_owner(); - } - - try { - return await this.save_to_disk_filesystem_owner(); - } catch (err) { - this.emit("save_to_disk_filesystem_owner", err); - throw err; - } - }; - - private save_to_disk_non_filesystem_owner = async (): Promise => { - this.assert_is_ready("save_to_disk_non_filesystem_owner"); - - if (!this.has_unsaved_changes()) { - /* Browser client has no unsaved changes, - so don't need to save -- - CRITICAL: this optimization is assumed by autosave. - */ - return; - } - const x = this.syncstring_table.get_one(); - if (x != null && x.getIn(["save", "state"]) === "requested") { - // Nothing to do -- save already requested, which is - // all the browser client has to do. - return; - } - - // string version of this doc - const data: string = this.to_str(); - const expected_hash = hash_string(data); - await this.set_save({ state: "requested", error: "", expected_hash }); - }; - - private save_to_disk_filesystem_owner = async (): Promise => { - this.assert_is_ready("save_to_disk_filesystem_owner"); - const dbg = this.dbg("save_to_disk_filesystem_owner"); - - // check if on-disk version is same as in memory, in - // which case no save is needed. - const data = this.to_str(); // string version of this doc - const hash = hash_string(data); - dbg("hash = ", hash); - - /* - // TODO: put this consistency check back in (?). - const expected_hash = this.syncstring_table - .get_one() - .getIn(["save", "expected_hash"]); - */ - - if (hash === this.hash_of_saved_version()) { - // No actual save to disk needed; still we better - // record this fact in table in case it - // isn't already recorded - this.set_save({ state: "done", error: "", hash }); - return; - } - - const path = this.path; - if (!path) { - const err = "cannot save without path"; - this.set_save({ state: "done", error: err }); - throw Error(err); - } - - dbg("project - write to disk file", path); - // set window to slightly earlier to account for clock - // imprecision. - // Over an sshfs mount, all stats info is **rounded down - // to the nearest second**, which this also takes care of. - this.save_to_disk_start_ctime = Date.now() - 1500; - this.save_to_disk_end_ctime = undefined; - try { - await callback2(this.client.write_file, { path, data }); - this.assert_is_ready("save_to_disk_filesystem_owner -- after write_file"); - const stat = await callback2(this.client.path_stat, { path }); - this.assert_is_ready("save_to_disk_filesystem_owner -- after path_state"); - this.save_to_disk_end_ctime = stat.ctime.valueOf() + 1500; - this.set_save({ - state: "done", - error: "", - hash: hash_string(data), - }); - } catch (err) { - this.set_save({ state: "done", error: JSON.stringify(err) }); - throw err; - } - }; - /* When the underlying synctable that defines the state of the document changes due to new remote patches, this @@ -3406,66 +2571,70 @@ export class SyncDoc extends EventEmitter { Whenever new patches are added to this.patches_table, their timestamp gets added to this.patch_update_queue. */ - private handle_patch_update_queue = async (): Promise => { - const dbg = this.dbg("handle_patch_update_queue"); - try { - this.handle_patch_update_queue_running = true; - while (this.state != "closed" && this.patch_update_queue.length > 0) { - dbg("queue size = ", this.patch_update_queue.length); - const v: Patch[] = []; - for (const key of this.patch_update_queue) { - let x = this.patches_table.get(key); - if (x == null) { - continue; + private handle_patch_update_queue = reuseInFlight( + async (save = false): Promise => { + const dbg = this.dbg("handle_patch_update_queue"); + try { + this.handle_patch_update_queue_running = true; + while (this.state != "closed" && this.patch_update_queue.length > 0) { + dbg("queue size = ", this.patch_update_queue.length); + const v: Patch[] = []; + for (const key of this.patch_update_queue) { + let x = this.patches_table.get(key); + if (x == null) { + continue; + } + if (!Map.isMap(x)) { + // TODO: my NATS synctable-stream doesn't convert to immutable on get. + x = fromJS(x); + } + // may be null, e.g., when deleted. + const t = x.get("time"); + // Optimization: only need to process patches that we didn't + // create ourselves during this session. + if (t && !this.my_patches[t.valueOf()]) { + const p = this.processPatch({ x }); + //dbg(`patch=${JSON.stringify(p)}`); + if (p != null) { + v.push(p); + } + } } - if (!Map.isMap(x)) { - // TODO: my NATS synctable-stream doesn't convert to immutable on get. - x = fromJS(x); + this.patch_update_queue = []; + this.emit("patch-update-queue-empty"); + assertDefined(this.patch_list); + this.patch_list.add(v); + + dbg("waiting for remote and doc to sync..."); + this.sync_remote_and_doc(v.length > 0); + if (save || !this.noAutosave) { + await this.patches_table.save(); + if (this.state === ("closed" as State)) return; // closed during await; nothing further to do + dbg("remote and doc now synced"); } - // may be null, e.g., when deleted. - const t = x.get("time"); - // Optimization: only need to process patches that we didn't - // create ourselves during this session. - if (t && !this.my_patches[t.valueOf()]) { - const p = this.processPatch({ x }); - //dbg(`patch=${JSON.stringify(p)}`); - if (p != null) { - v.push(p); - } + + if (this.patch_update_queue.length > 0) { + // It is very important that next loop happen in a later + // event loop to avoid the this.sync_remote_and_doc call + // in this.handle_patch_update_queue above from causing + // sync_remote_and_doc to get called from within itself, + // due to synctable changes being emited on save. + dbg("wait for next event loop"); + await delay(1); } } - this.patch_update_queue = []; - this.emit("patch-update-queue-empty"); - assertDefined(this.patch_list); - this.patch_list.add(v); - - dbg("waiting for remote and doc to sync..."); - this.sync_remote_and_doc(v.length > 0); - await this.patches_table.save(); - if (this.state === ("closed" as State)) return; // closed during await; nothing further to do - dbg("remote and doc now synced"); - - if (this.patch_update_queue.length > 0) { - // It is very important that next loop happen in a later - // event loop to avoid the this.sync_remote_and_doc call - // in this.handle_patch_update_queue above from causing - // sync_remote_and_doc to get called from within itself, - // due to synctable changes being emited on save. - dbg("wait for next event loop"); - await delay(1); - } - } - } finally { - if (this.state == "closed") return; // got closed, so nothing further to do + } finally { + if (this.state == "closed") return; // got closed, so nothing further to do - // OK, done and nothing in the queue - // Notify save() to try again -- it may have - // paused waiting for this to clear. - dbg("done"); - this.handle_patch_update_queue_running = false; - this.emit("handle_patch_update_queue_done"); - } - }; + // OK, done and nothing in the queue + // Notify save() to try again -- it may have + // paused waiting for this to clear. + dbg("done"); + this.handle_patch_update_queue_running = false; + this.emit("handle_patch_update_queue_done"); + } + }, + ); /* Disable and enable sync. When disabled we still collect patches from upstream (but do not apply them @@ -3513,10 +2682,13 @@ export class SyncDoc extends EventEmitter { return; } - // Critical to save what we have now so it doesn't get overwritten during - // before-change or setting this.doc below. This caused - // https://github.com/sagemathinc/cocalc/issues/5871 - this.commit(); + if (!this.last.is_equal(this.doc)) { + // If live versions differs from last commit (or merge of heads), it is + // commit what we have now so it doesn't get overwritten during + // before-change or setting this.doc below. This caused + // https://github.com/sagemathinc/cocalc/issues/5871 + this.commit(); + } if (upstreamPatches && this.state == "ready") { // First save any unsaved changes from the live document, which this @@ -3524,10 +2696,12 @@ export class SyncDoc extends EventEmitter { // rapidly changing live editor with changes not yet saved here. this.emit("before-change"); // As a result of the emit in the previous line, all kinds of - // nontrivial listener code probably just ran, and it should + // nontrivial listener code may have just ran, and it could // have updated this.doc. We commit this.doc, so that the // upstream patches get applied against the correct live this.doc. - this.commit(); + if (!this.last.is_equal(this.doc)) { + this.commit(); + } } // Compute the global current state of the document, @@ -3589,33 +2763,143 @@ export class SyncDoc extends EventEmitter { } }, 60000); - private initInterestLoop = async () => { - if (!this.client.is_browser()) { - // only browser clients -- so actual humans - return; + private initFileWatcherFirstTime = () => { + // set this going, but don't await it. + (async () => { + await until( + async () => { + if (this.isClosed()) return true; + try { + await this.initFileWatcher(); + return true; + } catch { + return false; + } + }, + { min: this.opts?.watchRecreateWait ?? WATCH_RECREATE_WAIT }, + ); + })(); + }; + + private fileWatcher?: any; + private initFileWatcher = async () => { + // use this.fs interface to watch path for changes -- we try once: + try { + this.fileWatcher = await this.fs.watch(this.path, { unique: true }); + if (this.isDeleted) { + await this.readFile(); + } + } catch (err) { + // console.log("error creating watcher", err); + if (err.code == "ENOENT") { + // the file was deleted -- check if this stays deleted long enough to count + await this.signalIfFileDeleted(); + } + // throwing this error just causes initFileWatcher to get + // initialized again soon (a few seconds), again attemping a watch, + // unless it is the first time initializing the document. + throw err; } - const touch = async () => { - if (this.state == "closed" || this.client?.touchOpenFile == null) return; - await this.client.touchOpenFile({ - path: this.path, - project_id: this.project_id, - doctype: this.doctype, - }); - }; - // then every CONAT_OPEN_FILE_TOUCH_INTERVAL (30 seconds). - await until( - async () => { - if (this.state == "closed") { - return true; + if (this.isClosed()) return; + + // not closed -- so if above succeeds we start watching. + // if not, we loop waiting for file to be created so we can watch it + (async () => { + if (this.fileWatcher != null) { + this.emit("watching"); + for await (const { eventType, ignore } of this.fileWatcher) { + if (this.isClosed()) return; + if (!ignore) { + // we don't know what's on disk anymore, + this.valueOnDisk = undefined; + // and we should find out! + this.readFileDebounced(); + } else { + this.debouncedStat(); + } + if (eventType == "rename") { + break; + } } - await touch(); + // check if file was deleted + this.signalIfFileDeleted(); + this.fileWatcher?.close(); + delete this.fileWatcher; + } + if (this.isClosed()) return; + // start a new watcher since file descriptor probably changed or maybe file deleted + await delay(this.opts?.watchRecreateWait ?? WATCH_RECREATE_WAIT); + await until( + async () => { + if (this.isClosed()) return true; + try { + await this.initFileWatcher(); + return true; + } catch { + return false; + } + }, + { min: this.opts?.watchRecreateWait ?? WATCH_RECREATE_WAIT }, + ); + })(); + }; + + private closeFileWatcher = () => { + this.fileWatcher?.close(); + delete this.fileWatcher; + }; + + // returns true if file definitely exists right now, + // false if it definitely does not, and throws exception otherwise, + // e.g., network error. + private fileExists = async (): Promise => { + try { + await this.stat(); + return true; + } catch (err) { + if (err.code == "ENOENT") { + // file not there now. return false; - }, - { - start: CONAT_OPEN_FILE_TOUCH_INTERVAL, - max: CONAT_OPEN_FILE_TOUCH_INTERVAL, - }, - ); + } + throw err; + } + }; + + private signalIfFileDeleted = async (): Promise => { + if (this.isClosed()) return; + const start = Date.now(); + const threshold = this.opts.deletedThreshold ?? DELETED_THRESHOLD; + while (!this.isClosed()) { + try { + if (await this.fileExists()) { + // file definitely exists right now -- NOT deleted. + return; + } + // file definitely does NOT exist right now. + } catch { + // network not working or project off -- no way to know. + return; + } + const elapsed = Date.now() - start; + if (elapsed > threshold) { + // out of time to appear again, and definitely concluded + // it does not exist above + // file still doesn't exist -- consider it deleted -- browsers + // should close the tab and possibly notify user. + this.from_str(""); + this.commit(); + // console.log("emit deleted and set isDeleted=true"); + this.isDeleted = true; + this.emit("deleted"); + return; + } + await delay( + Math.min( + this.opts.deletedCheckInterval ?? DELETED_CHECK_INTERVAL, + threshold - elapsed, + ), + ); + } }; } @@ -3638,3 +2922,8 @@ function isCompletePatchStream(dstream) { } return false; } + +function isReadOnlyForOwner(stats): boolean { + // 0o200 is owner write permission + return (stats.mode & 0o200) === 0; +} diff --git a/src/packages/sync/editor/generic/types.ts b/src/packages/sync/editor/generic/types.ts index 4cf3519e949..f840cec3a18 100644 --- a/src/packages/sync/editor/generic/types.ts +++ b/src/packages/sync/editor/generic/types.ts @@ -102,7 +102,7 @@ export interface ProjectClient extends EventEmitter { // Only required to work on project client. path_access: (opts: { path: string; mode: string; cb: Function }) => void; - path_exists: (opts: { path: string; cb: Function }) => void; + path_exists?: (opts: { path: string; cb: Function }) => void; path_stat: (opts: { path: string; cb: Function }) => void; @@ -169,12 +169,6 @@ export interface Client extends ProjectClient { sage_session: (opts: { path: string }) => any; - touchOpenFile?: (opts: { - project_id: string; - path: string; - doctype?; - }) => Promise; - touch_project?: (path: string) => void; } diff --git a/src/packages/sync/editor/generic/util.ts b/src/packages/sync/editor/generic/util.ts index fd666b09a1d..2c66ee97299 100644 --- a/src/packages/sync/editor/generic/util.ts +++ b/src/packages/sync/editor/generic/util.ts @@ -68,6 +68,7 @@ export function make_patch(s0: string, s1: string): CompressedPatch { } // apply a compressed patch to a string. +// Returns the result *and* whether or not the patch applied cleanly. export function apply_patch( patch: CompressedPatch, s: string, diff --git a/src/packages/sync/editor/string/index.ts b/src/packages/sync/editor/string/index.ts index c1fed4e3a59..5a43e5f755e 100644 --- a/src/packages/sync/editor/string/index.ts +++ b/src/packages/sync/editor/string/index.ts @@ -1 +1,2 @@ export { SyncString } from "./sync"; +export { type SyncStringOpts } from "./sync"; diff --git a/src/packages/sync/editor/string/sync.ts b/src/packages/sync/editor/string/sync.ts index 780c066370a..95494d9f1e8 100644 --- a/src/packages/sync/editor/string/sync.ts +++ b/src/packages/sync/editor/string/sync.ts @@ -6,6 +6,8 @@ import { SyncDoc, SyncOpts0, SyncOpts } from "../generic/sync-doc"; import { StringDocument } from "./doc"; +export type SyncStringOpts = SyncOpts0; + export class SyncString extends SyncDoc { constructor(opts: SyncOpts0) { // TS question -- What is the right way to do this? diff --git a/src/packages/sync/editor/string/test/README.md b/src/packages/sync/editor/string/test/README.md new file mode 100644 index 00000000000..0a064bf05c4 --- /dev/null +++ b/src/packages/sync/editor/string/test/README.md @@ -0,0 +1,5 @@ +There is additional _integration_ testing of the sync code in: + +``` +packages/backend/conat/test/sync-doc +``` \ No newline at end of file diff --git a/src/packages/sync/editor/string/test/client-test.ts b/src/packages/sync/editor/string/test/client-test.ts index b9d43e012ae..0c280276217 100644 --- a/src/packages/sync/editor/string/test/client-test.ts +++ b/src/packages/sync/editor/string/test/client-test.ts @@ -187,3 +187,11 @@ export class Client extends EventEmitter implements Client0 { console.log(`shell: opts=${JSON.stringify(opts)}`); } } + +class Filesystem { + readFile = () => ""; + writeFile = () => {}; + utimes = () => {}; +} + +export const fs = new Filesystem() as any; diff --git a/src/packages/sync/editor/string/test/ephemeral-syncstring.ts b/src/packages/sync/editor/string/test/ephemeral-syncstring.ts index 81cb8001f0d..086fa9a1540 100644 --- a/src/packages/sync/editor/string/test/ephemeral-syncstring.ts +++ b/src/packages/sync/editor/string/test/ephemeral-syncstring.ts @@ -3,7 +3,7 @@ This is useful not just for testing, but also for implementing undo/redo for editing a text document when there is no actual file or project involved. */ -import { Client } from "./client-test"; +import { Client, fs } from "./client-test"; import { SyncString } from "../sync"; import { a_txt } from "./data"; import { once } from "@cocalc/util/async-utils"; @@ -16,6 +16,7 @@ export default async function ephemeralSyncstring() { path, client, ephemeral: true, + fs, }); // replace save to disk, since otherwise unless string is empty, // this will hang forever... and it is called on close. diff --git a/src/packages/sync/editor/string/test/sync.0.test.ts b/src/packages/sync/editor/string/test/sync.0.test.ts index 9d5289dbd14..0dc3607e5fa 100644 --- a/src/packages/sync/editor/string/test/sync.0.test.ts +++ b/src/packages/sync/editor/string/test/sync.0.test.ts @@ -11,7 +11,7 @@ pnpm test sync.0.test.ts */ -import { Client } from "./client-test"; +import { Client, fs } from "./client-test"; import { SyncString } from "../sync"; import { a_txt } from "./data"; import { once } from "@cocalc/util/async-utils"; @@ -23,7 +23,7 @@ describe("create a blank minimal string SyncDoc and call public methods on it", let syncstring: SyncString; it("creates the syncstring and wait for it to be ready", async () => { - syncstring = new SyncString({ project_id, path, client }); + syncstring = new SyncString({ project_id, path, client, fs }); expect(syncstring.get_state()).toBe("init"); await once(syncstring, "ready"); expect(syncstring.get_state()).toBe("ready"); @@ -142,12 +142,11 @@ describe("create a blank minimal string SyncDoc and call public methods on it", }); it("read only checks", async () => { - await syncstring.wait_until_read_only_known(); // no-op expect(syncstring.is_read_only()).toBe(false); }); it("hashes of versions", () => { - expect(syncstring.hash_of_saved_version()).toBe(0); + expect(syncstring.hash_of_saved_version()).toBe(undefined); expect(syncstring.hash_of_live_version()).toBe(0); expect(syncstring.has_uncommitted_changes()).toBe(false); }); diff --git a/src/packages/sync/editor/string/test/sync.1.test.ts b/src/packages/sync/editor/string/test/sync.1.test.ts index a58fa9d3a44..842c85c610e 100644 --- a/src/packages/sync/editor/string/test/sync.1.test.ts +++ b/src/packages/sync/editor/string/test/sync.1.test.ts @@ -12,7 +12,7 @@ pnpm test sync.1.test.ts */ -import { Client } from "./client-test"; +import { Client, fs } from "./client-test"; import { SyncString } from "../sync"; import { once } from "@cocalc/util/async-utils"; import { a_txt } from "./data"; @@ -28,7 +28,13 @@ describe("create syncstring and test doing some edits", () => { ]; it("creates the syncstring and wait until ready", async () => { - syncstring = new SyncString({ project_id, path, client, cursors: true }); + syncstring = new SyncString({ + project_id, + path, + client, + cursors: true, + fs, + }); expect(syncstring.get_state()).toBe("init"); await once(syncstring, "ready"); }); @@ -97,21 +103,6 @@ describe("create syncstring and test doing some edits", () => { expect(syncstring.is_read_only()).toBe(false); }); - it("save to disk", async () => { - expect(syncstring.has_unsaved_changes()).toBe(true); - const promise = syncstring.save_to_disk(); - // Mock: we set save to done in the syncstring - // table, otherwise the promise will never resolve. - (syncstring as any).set_save({ - state: "done", - error: "", - hash: syncstring.hash_of_live_version(), - }); - (syncstring as any).syncstring_table.emit("change-no-throttle"); - await promise; - expect(syncstring.has_unsaved_changes()).toBe(false); - }); - it("close and clean up", async () => { await syncstring.close(); expect(syncstring.get_state()).toBe("closed"); diff --git a/src/packages/sync/table/util.ts b/src/packages/sync/table/util.ts index 6815f3052de..dac8c0befd9 100644 --- a/src/packages/sync/table/util.ts +++ b/src/packages/sync/table/util.ts @@ -46,6 +46,9 @@ export function parseQueryWithOptions(query, options) { query[table][0][k] = obj[k]; } } + if (options?.project_id != null && query[table][0]["project_id"] === null) { + query[table][0]["project_id"] = options.project_id; + } return { query, table }; } diff --git a/src/packages/sync/tsconfig.json b/src/packages/sync/tsconfig.json index 0bdfd8b3a42..6cdb913e19b 100644 --- a/src/packages/sync/tsconfig.json +++ b/src/packages/sync/tsconfig.json @@ -5,5 +5,5 @@ "outDir": "dist" }, "exclude": ["node_modules", "dist", "test"], - "references": [{ "path": "../util" }, { "path": "../conat" }] + "references": [{ "path": "../util" }] } diff --git a/src/packages/test/jest.config.js b/src/packages/test/jest.config.js new file mode 100644 index 00000000000..3e9e290535f --- /dev/null +++ b/src/packages/test/jest.config.js @@ -0,0 +1,13 @@ +/** @type {import('ts-jest').JestConfigWithTsJest} */ +module.exports = { + preset: "ts-jest", + testEnvironment: "jsdom", + testEnvironmentOptions: { + // needed or jest imports the ts directly rather than the compiled + // dist exported from our package.json. Without this imports won't work. + // See https://jestjs.io/docs/configuration#testenvironment-string + customExportConditions: ["node", "node-addons"], + }, + testMatch: ["**/?(*.)+(spec|test).ts?(x)"], + setupFilesAfterEnv: ["./test/setup.js"], +}; diff --git a/src/packages/test/package.json b/src/packages/test/package.json new file mode 100644 index 00000000000..820229b57ed --- /dev/null +++ b/src/packages/test/package.json @@ -0,0 +1,38 @@ +{ + "name": "@cocalc/test", + "version": "1.0.0", + "description": "CoCalc Integration Testing", + "exports": { + "./*": "./dist/*.js" + }, + "keywords": ["test", "cocalc"], + "scripts": { + "preinstall": "npx only-allow pnpm", + "clean": "rm -rf dist node_modules", + "build": "pnpm exec tsc --build", + "tsc": "pnpm exec tsc --watch --pretty --preserveWatchOutput", + "test": "pnpm exec jest --forceExit", + "depcheck": "pnpx depcheck --ignores @types/debug,@types/jest,@types/node,jest-environment-jsdom" + }, + "author": "SageMath, Inc.", + "license": "SEE LICENSE.md", + "dependencies": { + "@cocalc/backend": "workspace:*", + "@cocalc/conat": "workspace:*", + "@cocalc/frontend": "workspace:*", + "@cocalc/util": "workspace:*" + }, + "repository": { + "type": "git", + "url": "https://github.com/sagemathinc/cocalc" + }, + "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/test", + "devDependencies": { + "@testing-library/jest-dom": "^6.6.3", + "@testing-library/react": "^16.3.0", + "@types/debug": "^4.1.12", + "@types/jest": "^29.5.14", + "@types/node": "^18.16.14", + "jest-environment-jsdom": "^30.0.2" + } +} diff --git a/src/packages/test/project/listing/use-files.test.ts b/src/packages/test/project/listing/use-files.test.ts new file mode 100644 index 00000000000..ca113bbbb82 --- /dev/null +++ b/src/packages/test/project/listing/use-files.test.ts @@ -0,0 +1,118 @@ +import { act, renderHook, waitFor } from "@testing-library/react"; +import { fsClient } from "@cocalc/conat/files/fs"; +import { before, after } from "@cocalc/backend/conat/test/setup"; +import { uuid } from "@cocalc/util/misc"; +import { + createPathFileserver, + cleanupFileservers, +} from "@cocalc/backend/conat/files/test/util"; +import useFiles from "@cocalc/frontend/project/listing/use-files"; + +beforeAll(before); + +describe("the useFiles hook", () => { + const project_id = uuid(); + let fs, server; + it("creates fileserver service and fs client", async () => { + server = await createPathFileserver(); + fs = fsClient({ subject: `${server.service}.project-${project_id}` }); + }); + + it("test useFiles and file creation", async () => { + let path = "", + fs2 = fs; + const { result, rerender } = renderHook(() => + useFiles({ fs: fs2, path, throttleUpdate: 0 }), + ); + + expect(result.current).toEqual({ + files: null, + error: null, + refresh: expect.any(Function), + }); + + // eventually it will be initialized to not be null + await waitFor(() => { + expect(result.current.files).not.toBeNull(); + }); + expect(result.current).toEqual({ + files: {}, + error: null, + refresh: expect.any(Function), + }); + + // now create a file + await act(async () => { + await fs.writeFile("hello.txt", "world"); + }); + + await waitFor(() => { + expect(result.current.files?.["hello.txt"]).toBeDefined(); + }); + + expect(result.current).toEqual({ + files: { + "hello.txt": { + size: 5, + mtime: expect.any(Number), + type: "f", + }, + }, + error: null, + refresh: expect.any(Function), + }); + + // change the path to one that does not exist and rerender, + // resulting in an ENOENT error + path = "scratch"; + rerender(); + await waitFor(() => { + expect(result.current.files?.["hello.txt"]).not.toBeDefined(); + }); + await waitFor(() => { + expect(result.current.error).not.toBe(null); + }); + expect(result.current.error?.code).toBe("ENOENT"); + + await act(async () => { + // create the path, a file in there, refresh and it works + await fs.mkdir(path); + await fs.writeFile("scratch/b.txt", "hi"); + result.current.refresh(); + }); + + await waitFor(() => { + expect(result.current).toEqual({ + files: { + "b.txt": { + size: 2, + mtime: expect.any(Number), + type: "f", + }, + }, + error: null, + refresh: expect.any(Function), + }); + }); + + // change fs and see the hook update + const project_id2 = uuid(); + fs2 = fsClient({ + subject: `${server.service}.project-${project_id2}`, + }); + path = ""; + rerender(); + await waitFor(() => { + expect(result.current).toEqual({ + files: {}, + error: null, + refresh: expect.any(Function), + }); + }); + }); +}); + +afterAll(async () => { + await after(); + await cleanupFileservers(); +}); diff --git a/src/packages/test/project/listing/use-listing.test.ts b/src/packages/test/project/listing/use-listing.test.ts new file mode 100644 index 00000000000..37807219dc7 --- /dev/null +++ b/src/packages/test/project/listing/use-listing.test.ts @@ -0,0 +1,213 @@ +import { act, renderHook, waitFor } from "@testing-library/react"; +import { fsClient } from "@cocalc/conat/files/fs"; +import { before, after } from "@cocalc/backend/conat/test/setup"; +import { uuid } from "@cocalc/util/misc"; +import { + createPathFileserver, + cleanupFileservers, +} from "@cocalc/backend/conat/files/test/util"; +import useListing, { + type SortField, + type SortDirection, +} from "@cocalc/frontend/project/listing/use-listing"; +import { type FilesystemClient } from "@cocalc/conat/files/fs"; + +beforeAll(before); + +describe("the useListing hook", () => { + const project_id = uuid(); + let fs, server; + it("creates fileserver service and fs client", async () => { + server = await createPathFileserver(); + fs = fsClient({ subject: `${server.service}.project-${project_id}` }); + }); + + it("test useListing and file creation", async () => { + let path = "", + fs2: FilesystemClient | undefined = undefined; + const { result, rerender } = renderHook(() => + useListing({ fs: fs2, path, throttleUpdate: 0 }), + ); + expect(result.current).toEqual({ + listing: null, + error: null, + refresh: expect.any(Function), + }); + fs2 = fs; + rerender(); + + // now that fs2 is set, eventually it will be initialized to not be null + await waitFor(() => { + expect(result.current.listing).not.toBeNull(); + }); + + expect(result.current).toEqual({ + listing: [], + error: null, + refresh: expect.any(Function), + }); + + // now create a file + await act(async () => { + await fs.writeFile("hello.txt", "world"); + }); + + await waitFor(() => { + expect(result.current.listing?.length).toEqual(1); + }); + + expect(result.current).toEqual({ + listing: [ + { name: "hello.txt", size: 5, mtime: expect.any(Number), type: "f" }, + ], + error: null, + refresh: expect.any(Function), + }); + + // change the path to one that does not exist and rerender, + // resulting in an ENOENT error + path = "scratch"; + rerender(); + await waitFor(() => { + expect(result.current.listing).toBeNull(); + expect(result.current.error?.code).toBe("ENOENT"); + }); + + await act(async () => { + // create the path, a file in there, refresh and it works + await fs.mkdir(path); + await fs.writeFile("scratch/b.txt", "hi"); + result.current.refresh(); + }); + + await waitFor(() => { + expect(result.current).toEqual({ + listing: [ + { + name: "b.txt", + size: 2, + type: "f", + mtime: expect.any(Number), + }, + ], + error: null, + refresh: expect.any(Function), + }); + }); + + // change fs and see the hook update + const project_id2 = uuid(); + fs2 = fsClient({ + subject: `${server.service}.project-${project_id2}`, + }); + path = ""; + rerender(); + await waitFor(() => { + expect(result.current).toEqual({ + listing: [], + error: null, + refresh: expect.any(Function), + }); + }); + }); +}); + +describe("test sorting many files with useListing", () => { + const project_id = uuid(); + let fs, server; + it("creates fileserver service and fs client", async () => { + server = await createPathFileserver(); + fs = fsClient({ subject: `${server.service}.project-${project_id}` }); + }); + + it("create some files", async () => { + await fs.writeFile("a.txt", "abc"); + await fs.writeFile("b.txt", "b"); + await fs.writeFile("huge.txt", "b".repeat(1000)); + + // make b.txt old + await fs.utimes( + "b.txt", + (Date.now() - 60_000) / 1000, + (Date.now() - 60_000) / 1000, + ); + }); + + it("test useListing with many files and sorting", async () => { + let path = "", + sortField: SortField = "name", + sortDirection: SortDirection = "asc"; + const { result, rerender } = renderHook(() => + useListing({ fs, path, throttleUpdate: 0, sortField, sortDirection }), + ); + + await waitFor(() => { + expect(result.current.listing?.length).toEqual(3); + }); + expect(result.current.listing?.map(({ name }) => name)).toEqual([ + "a.txt", + "b.txt", + "huge.txt", + ]); + + sortDirection = "desc"; + sortField = "name"; + rerender(); + await waitFor(() => { + expect(result.current.listing?.map(({ name }) => name)).toEqual([ + "huge.txt", + "b.txt", + "a.txt", + ]); + }); + + sortDirection = "asc"; + sortField = "mtime"; + rerender(); + await waitFor(() => { + expect(result.current.listing?.map(({ name }) => name)).toEqual([ + "b.txt", + "a.txt", + "huge.txt", + ]); + }); + + sortDirection = "desc"; + sortField = "mtime"; + rerender(); + await waitFor(() => { + expect(result.current.listing?.map(({ name }) => name)).toEqual([ + "huge.txt", + "a.txt", + "b.txt", + ]); + }); + + sortDirection = "asc"; + sortField = "size"; + rerender(); + await waitFor(() => { + expect(result.current.listing?.map(({ name }) => name)).toEqual([ + "b.txt", + "a.txt", + "huge.txt", + ]); + }); + + sortDirection = "desc"; + sortField = "size"; + rerender(); + await waitFor(() => { + expect(result.current.listing?.map(({ name }) => name)).toEqual([ + "huge.txt", + "a.txt", + "b.txt", + ]); + }); + }); +}); + +afterAll(async () => { + await after(); + await cleanupFileservers(); +}); diff --git a/src/packages/test/test/setup.js b/src/packages/test/test/setup.js new file mode 100644 index 00000000000..fb85d307c60 --- /dev/null +++ b/src/packages/test/test/setup.js @@ -0,0 +1,5 @@ +require("@testing-library/jest-dom"); +process.env.COCALC_TEST_MODE = true; + +global.TextEncoder = require("util").TextEncoder; +global.TextDecoder = require("util").TextDecoder; diff --git a/src/packages/test/tsconfig.json b/src/packages/test/tsconfig.json new file mode 100644 index 00000000000..6e3f1bef814 --- /dev/null +++ b/src/packages/test/tsconfig.json @@ -0,0 +1,22 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "composite": true, + "incremental": true, + "rootDir": "./", + "outDir": "dist", + "sourceMap": true, + "lib": ["es5", "es6", "es2017", "dom"], + "declaration": true + }, + "exclude": ["dist", "node_modules"], + "types": ["jest", "@testing-library/jest-dom"], + "references": [ + { + "path": "../util", + "path": "../conat", + "path": "../backend", + "path": "../frontend" + } + ] +} diff --git a/src/packages/tsconfig.json b/src/packages/tsconfig.json index 70ea25757df..6edbb90e358 100644 --- a/src/packages/tsconfig.json +++ b/src/packages/tsconfig.json @@ -20,11 +20,11 @@ "strictNullChecks": true, "target": "es2020", "module": "commonjs" + }, + "watchOptions": { + "fallbackPolling": "dynamicPriority", + "synchronousWatchDirectory": true, + "watchDirectory": "useFsEvents", + "watchFile": "useFsEvents" } - // "watchOptions": { - // "fallbackPolling": "dynamicPriority", - // "synchronousWatchDirectory": true, - // "watchDirectory": "useFsEvents", - // "watchFile": "useFsEvents" - // } } diff --git a/src/packages/util/async-utils.ts b/src/packages/util/async-utils.ts index f0fed46ef41..d5142926e41 100644 --- a/src/packages/util/async-utils.ts +++ b/src/packages/util/async-utils.ts @@ -168,6 +168,12 @@ export class TimeoutError extends Error { } } +function captureStackWithoutPrinting() { + const obj = {} as any; + Error.captureStackTrace(obj, captureStackWithoutPrinting); + return obj.stack; +} + /* Wait for an event emitter to emit any event at all once. Returns array of args emitted by that event. If timeout_ms is 0 (the default) this can wait an unbounded @@ -178,12 +184,18 @@ export class TimeoutError extends Error { If the obj throws 'closed' before the event is emitted, then this throws an error, since clearly event can never be emitted. */ + +// Set DEBUG_ONCE to true and see a MUCH better stack trace about what +// caused once to throw in some cases! Do not leave this on though, +// since it uses extra time and memory grabbing a stack trace on every call. +const DEBUG_ONCE = false; export async function once( obj: EventEmitter, event: string, timeout_ms: number | undefined = 0, ): Promise { if (obj == null) throw Error("once -- obj is undefined"); + const stack = DEBUG_ONCE ? captureStackWithoutPrinting() : undefined; if (timeout_ms == null) { // clients might explicitly pass in undefined, but below we expect 0 to mean "no timeout" timeout_ms = 0; @@ -207,11 +219,17 @@ export async function once( function onClosed() { cleanup(); + if (DEBUG_ONCE) { + console.log(stack); + } reject(new TimeoutError(`once: "${event}" not emitted before "closed"`)); } function onTimeout() { cleanup(); + if (DEBUG_ONCE) { + console.log(stack); + } reject( new TimeoutError( `once: timeout of ${timeout_ms}ms waiting for "${event}"`, diff --git a/src/packages/util/db-schema/accounts.ts b/src/packages/util/db-schema/accounts.ts index fc7774d8d5a..27d8ad9a2d8 100644 --- a/src/packages/util/db-schema/accounts.ts +++ b/src/packages/util/db-schema/accounts.ts @@ -482,7 +482,12 @@ Table({ other_settings: { katex: true, confirm_close: false, - mask_files: true, + // mask_files -- note that there is a performance cost to this, e.g., 5ms if you have 10K files in + // a directory (basically it doubles the processing costs). + // It's also confusing and can be subtly wrong. Finally, it's almost never necessary due to us changing the defaults + // for running latex to put all the temp files in /tmp -- in general we should always put temp files in tmp anyways + // with all build processes. So mask_files is off by default if not explicitly selected. + mask_files: false, page_size: 500, standby_timeout_m: 5, default_file_sort: "name", diff --git a/src/packages/util/db-schema/llm-utils.ts b/src/packages/util/db-schema/llm-utils.ts index c5011867f68..0f3112a3222 100644 --- a/src/packages/util/db-schema/llm-utils.ts +++ b/src/packages/util/db-schema/llm-utils.ts @@ -434,15 +434,15 @@ export function getValidLanguageModelName({ } for (const free of [true, false]) { - const dflt = getDefaultLLM( + const defaultModel = getDefaultLLM( selectable_llms, filter, ollama, custom_openai, free, ); - if (dflt != null) { - return dflt; + if (defaultModel != null) { + return defaultModel; } } return DEFAULT_MODEL; diff --git a/src/packages/util/db-schema/projects.ts b/src/packages/util/db-schema/projects.ts index 088b9a655eb..2ab9a410da3 100644 --- a/src/packages/util/db-schema/projects.ts +++ b/src/packages/util/db-schema/projects.ts @@ -730,30 +730,10 @@ export interface CreateProjectOptions { // admins can specify the project_id - nobody else can -- useful for debugging. project_id?: string; -} - -interface BaseCopyOptions { - target_project_id?: string; - target_path?: string; // path into project; if not given, defaults to source path above. - overwrite_newer?: boolean; // if true, newer files in target are copied over (otherwise, uses rsync's --update) - delete_missing?: boolean; // if true, delete files in dest path not in source, **including** newer files - backup?: boolean; // make backup files - timeout?: number; // in **seconds**, not milliseconds - bwlimit?: number; - wait_until_done?: boolean; // by default, wait until done. false only gives the ID to query the status later - scheduled?: string | Date; // kucalc only: string (parseable by new Date()), or a Date - public?: boolean; // kucalc only: if true, may use the share server files rather than start the source project running - exclude?: string[]; // options passed to rsync via --exclude -} -export interface UserCopyOptions extends BaseCopyOptions { - account_id?: string; - src_project_id: string; - src_path: string; - // simulate copy taking at least this long -- useful for dev/debugging. - debug_delay_ms?: number; -} -// for copying files within and between projects -export interface CopyOptions extends BaseCopyOptions { - path: string; + // If given, files will be exact clone of those from src_project_id. + // account_id must be a collab on src_project_id. + // The implementation is highly efficient using "btrfs subvolume clone". + // Snapshots are not included in the clone. + src_project_id?: string; } diff --git a/src/packages/util/event-iterator.ts b/src/packages/util/event-iterator.ts index c3ca053301f..2d9ec62210e 100644 --- a/src/packages/util/event-iterator.ts +++ b/src/packages/util/event-iterator.ts @@ -1,7 +1,7 @@ /* LICENSE: MIT -This is a slight fork of +This is a slight fork of https://github.com/sapphiredev/utilities/tree/main/packages/event-iterator @@ -10,7 +10,7 @@ agree with the docs. I can see why. Upstream would capture ['arg1','arg2']] for an event emitter doing this emitter.emit('foo', 'arg1', 'arg2') - + But for our application we only want 'arg1'. I thus added a map option, which makes it easy to do what we want. */ @@ -46,6 +46,14 @@ export interface EventIteratorOptions { // called when iterator ends -- use to do cleanup. onEnd?: (iter?: EventIterator) => void; + + // Specifies the number of events to queue between iterations of the returned. + maxQueue?: number; + + // Either 'ignore' or 'throw' when there are more events to be queued than maxQueue allows. + // 'ignore' means overflow events are dropped and a warning is emitted, while + // 'throw' means to throw an exception. Default: 'ignore'. + overflow?: "ignore" | "throw"; } /** @@ -100,6 +108,11 @@ export class EventIterator */ readonly #limit: number; + readonly #maxQueue: number; + readonly #overflow?: "ignore" | "throw"; + + private resolveNext?: Function; + /** * The timer to track when this will idle out. */ @@ -124,6 +137,8 @@ export class EventIterator this.event = event; this.map = options.map ?? ((args) => args); this.#limit = options.limit ?? Infinity; + this.#maxQueue = options.maxQueue ?? Infinity; + this.#overflow = options.overflow ?? "ignore"; this.#idle = options.idle; this.filter = options.filter ?? ((): boolean => true); this.onEnd = options.onEnd; @@ -152,8 +167,8 @@ export class EventIterator */ public end(): void { if (this.#ended) return; + this.resolveNext?.(); this.#ended = true; - this.#queue = []; this.emitter.off(this.event, this.#push); const maxListeners = this.emitter.getMaxListeners(); @@ -165,19 +180,17 @@ export class EventIterator // aliases to match usage in NATS and CoCalc. close = this.end; stop = this.end; - - drain(): void { - // just immediately end - this.end(); - // [ ] TODO: for compat. I'm not sure what this should be - // or if it matters... - // console.log("WARNING: TODO -- event-iterator drain not implemented"); - } + // TODO/worry: drain doesn't do anything special to address outstanding + // requests like NATS did. Probably this isn't the place for it... + drain = this.end; /** * The next value that's received from the EventEmitter. */ public async next(): Promise> { + // if (this.verbose) { + // console.log("next", this.#queue); + // } if (this.err) { const err = this.err; delete this.err; @@ -226,10 +239,18 @@ export class EventIterator // Once it has received at least one value, we will clear the timer (if defined), // and resolve with the new value: - this.emitter.once(this.event, () => { - if (idleTimer) clearTimeout(idleTimer); + const handleEvent = () => { + delete this.resolveNext; + if (idleTimer) { + clearTimeout(idleTimer); + } resolve(this.next()); - }); + }; + this.emitter.once(this.event, handleEvent); + this.resolveNext = () => { + this.emitter.removeListener(this.event, handleEvent); + resolve({ done: true, value: undefined }); + }; }); } @@ -260,13 +281,38 @@ export class EventIterator * Pushes a value into the queue. */ protected push(...args): void { + // if (this.verbose) { + // console.log("push", args, this.#queue); + // } + if (this.err) { + return; + } try { const value = this.map(args); + if (this.#ended) { + // the this.map... call could have decided to end + // the iterator, by calling this.end() instead of returning a value. + if (value !== undefined) { + // not undefined so at least give the user the opportunity to get this final value. + this.#queue.push(value); + } + return; + } this.#queue.push(value); + while (this.#queue.length > this.#maxQueue && this.#queue.length > 0) { + if (this.#overflow == "throw") { + throw Error("maxQueue overflow"); + } + this.#queue.shift(); + } } catch (err) { this.err = err; // fake event to trigger handling of err this.emitter.emit(this.event); } } + + public queueSize(): number { + return this.#queue.length; + } } diff --git a/src/packages/util/jupyter/names.ts b/src/packages/util/jupyter/names.ts index e7d578494d2..f16106a862c 100644 --- a/src/packages/util/jupyter/names.ts +++ b/src/packages/util/jupyter/names.ts @@ -1,12 +1,23 @@ -import { meta_file } from "@cocalc/util/misc"; +import { meta_file, original_path } from "@cocalc/util/misc"; export const JUPYTER_POSTFIX = "jupyter2"; export const JUPYTER_SYNCDB_EXTENSIONS = `sage-${JUPYTER_POSTFIX}`; -// a.ipynb --> ".a.ipynb.sage-jupyter2" -export function syncdbPath(ipynbPath: string) { - if (!ipynbPath.endsWith(".ipynb")) { - throw Error(`ipynbPath must end with .ipynb but it is "${ipynbPath}"`); +// a.ipynb or .a.ipynb.sage-jupyter2 --> .a.ipynb.sage-jupyter2 +export function syncdbPath(path: string) { + if (path.endsWith(JUPYTER_POSTFIX)) { + return path; } - return meta_file(ipynbPath, JUPYTER_POSTFIX); + if (!path.endsWith(".ipynb")) { + throw Error(`must end with .ipynb but it is "${ipynbPath}"`); + } + return meta_file(path, JUPYTER_POSTFIX); +} + +// a.ipynb or .a.ipynb.sage-jupyter2 --> a.ipynb +export function ipynbPath(path: string) { + if (path.endsWith(".ipynb")) { + return path; + } + return original_path(path); } diff --git a/src/packages/util/misc.ts b/src/packages/util/misc.ts index 853b5be89ec..cf383069a95 100644 --- a/src/packages/util/misc.ts +++ b/src/packages/util/misc.ts @@ -335,6 +335,12 @@ export function uuidsha1(data: string): string { }); } +const SHA1_REGEXP = /^[a-f0-9]{40}$/; +export function isSha1(s: string): boolean { + return s.length === 40 && !!s.match(SHA1_REGEXP); +} + + // returns the number of keys of an object, e.g., {a:5, b:7, d:'hello'} --> 3 export function len(obj: object | undefined | null): number { if (obj == null) { @@ -501,7 +507,7 @@ export function trunc_left( sArg: T, max_length = 1024, ellipsis = ELLIPSIS, -): T | string { +): T | string { if (sArg == null) { return sArg; } @@ -2735,3 +2741,42 @@ export function uint8ArrayToBase64(uint8Array: Uint8Array) { } return btoa(binaryString); } + +// Inspired by https://github.com/etiennedi/kubernetes-resource-parser/tree/master +export function k8sCpuParser(input: string | number): number { + if (typeof input == "number") { + return input; + } + const milliMatch = input.match(/^([0-9]+)m$/); + if (milliMatch) { + return parseFloat(milliMatch[1]) / 1000; + } + return parseFloat(input); +} + +const memoryMultipliers = { + k: 1000, + M: 1000 ** 2, + G: 1000 ** 3, + T: 1000 ** 4, + P: 1000 ** 5, + E: 1000 ** 6, + Ki: 1024, + Mi: 1024 ** 2, + Gi: 1024 ** 3, + Ti: 1024 ** 4, + Pi: 1024 ** 5, + Ei: 1024 ** 6, +} as const; + +export function k8sMemoryParser(input: string | number): number { + if (typeof input == "number") { + return input; + } + const unitMatch = input.match(/^([0-9]+)([A-Za-z]{1,2})$/); + if (unitMatch) { + return parseInt(unitMatch[1], 10) * memoryMultipliers[unitMatch[2]]; + } + + return parseInt(input, 10); +} diff --git a/src/packages/util/package.json b/src/packages/util/package.json index 40f25349f82..be5aa5b10cc 100644 --- a/src/packages/util/package.json +++ b/src/packages/util/package.json @@ -71,7 +71,6 @@ }, "homepage": "https://github.com/sagemathinc/cocalc/tree/master/src/packages/util", "devDependencies": { - "@types/json-stable-stringify": "^1.0.32", "@types/lodash": "^4.14.202", "@types/node": "^18.16.14", "@types/seedrandom": "^3.0.8", diff --git a/src/packages/util/redux/Actions.ts b/src/packages/util/redux/Actions.ts index 27d9ffc0984..032303919e1 100644 --- a/src/packages/util/redux/Actions.ts +++ b/src/packages/util/redux/Actions.ts @@ -39,13 +39,9 @@ export class Actions { }; destroy = (): void => { - if (this.name == null) { - throw Error("unable to destroy actions because this.name is not defined"); - } - if (this.redux == null) { - throw Error( - `unable to destroy actions '${this.name}' since this.redux is not defined`, - ); + if (this.name == null || this.redux == null) { + // already closed + return; } // On the share server this.redux can be undefined at this point. this.redux.removeActions(this.name); diff --git a/src/packages/util/sanitize-software-envs.ts b/src/packages/util/sanitize-software-envs.ts index 074ba1ac4d7..8893d8b851f 100644 --- a/src/packages/util/sanitize-software-envs.ts +++ b/src/packages/util/sanitize-software-envs.ts @@ -118,19 +118,19 @@ export function sanitizeSoftwareEnv( return null; } - const swDflt = software["default"]; + const swDefault = software["default"]; // we check that the default is a string and that it exists in envs - const dflt = - typeof swDflt === "string" && envs[swDflt] != null - ? swDflt + const defaultSoftware = + typeof swDefault === "string" && envs[swDefault] != null + ? swDefault : Object.keys(envs)[0]; // this is a fallback entry, when projects were created before the software env was configured if (envs[DEFAULT_COMPUTE_IMAGE] == null) { - envs[DEFAULT_COMPUTE_IMAGE] = { ...envs[dflt], hidden: true }; + envs[DEFAULT_COMPUTE_IMAGE] = { ...envs[defaultSoftware], hidden: true }; } - return { groups, default: dflt, environments: envs }; + return { groups, default: defaultSoftware, environments: envs }; } function fallback(a: any, b: any, c?: string): any { diff --git a/src/packages/util/throttle.test.ts b/src/packages/util/throttle.test.ts new file mode 100644 index 00000000000..ac357c544f8 --- /dev/null +++ b/src/packages/util/throttle.test.ts @@ -0,0 +1,74 @@ +import { ThrottleString, Throttle } from "./throttle"; +import { delay } from "awaiting"; + +describe("a throttled string", () => { + let t; + let output = ""; + it("creates a throttled string", () => { + // emits 10 times a second or once very 100ms. + t = new ThrottleString(10); + t.on("data", (data) => { + output += data; + }); + }); + + it("write 3 times and wait 50ms and get nothing, then 70 more ms and get all", async () => { + t.write("a"); + t.write("b"); + t.write("c"); + await delay(50); + expect(output).toBe(""); + // this "d" also gets included -- it makes it in before the cutoff. + t.write("d"); + await delay(70); + expect(output).toBe("abcd"); + }); + + it("do the same again", async () => { + t.write("a"); + t.write("b"); + t.write("c"); + await delay(50); + expect(output).toBe("abcd"); + t.write("d"); + await delay(70); + expect(output).toBe("abcdabcd"); + }); +}); + +describe("a throttled list of objects", () => { + let t; + let output: any[] = []; + + it("creates a throttled any[]", () => { + // emits 10 times a second or once very 100ms. + t = new Throttle(10); + t.on("data", (data: any[]) => { + output = output.concat(data); + }); + }); + + it("write 3 times and wait 50ms and get nothing, then 70 more ms and get all", async () => { + t.write("a"); + t.write("b"); + t.write("c"); + await delay(50); + expect(output).toEqual([]); + // this "d" also gets included -- it makes it in before the cutoff. + t.write("d"); + await delay(70); + expect(output).toEqual(["a", "b", "c", "d"]); + }); + + it("do it again", async () => { + t.write("a"); + t.write("b"); + t.write("c"); + await delay(50); + expect(output).toEqual(["a", "b", "c", "d"]); + // this "d" also gets included -- it makes it in before the cutoff. + t.write("d"); + await delay(70); + expect(output).toEqual(["a", "b", "c", "d", "a", "b", "c", "d"]); + }); +}); diff --git a/src/packages/util/throttle.ts b/src/packages/util/throttle.ts index 2fff3d25969..92e64dfab9f 100644 --- a/src/packages/util/throttle.ts +++ b/src/packages/util/throttle.ts @@ -3,15 +3,22 @@ This is a really simple but incredibly useful little class. See packages/project/conat/terminal.ts for how to use it to make it so the terminal sends output at a rate of say "24 frames per second". + +This could also be called "buffering"... */ import { EventEmitter } from "events"; +const DEFAULT_MESSAGES_PER_SECOND = 24; + +// Throttling a string where use "+" to add more to our buffer export class ThrottleString extends EventEmitter { private buf: string = ""; private last = Date.now(); + private interval: number; - constructor(private interval: number) { + constructor(messagesPerSecond: number = DEFAULT_MESSAGES_PER_SECOND) { super(); + this.interval = 1000 / messagesPerSecond; } write = (data: string) => { @@ -34,20 +41,33 @@ export class ThrottleString extends EventEmitter { }; } -export class ThrottleAny extends EventEmitter { - private buf: any[] = []; +// Throttle a list of objects, where push them into an array to add more to our buffer. +export class Throttle extends EventEmitter { + private buf: T[] = []; private last = Date.now(); + private interval: number; - constructor(private interval: number) { + constructor(messagesPerSecond: number = DEFAULT_MESSAGES_PER_SECOND) { super(); + this.interval = 1000 / messagesPerSecond; } - write = (data: any) => { + // if you want data to be sent be sure to flush before closing + close = () => { + this.removeAllListeners(); + this.buf.length = 0; + }; + + write = (data: T) => { this.buf.push(data); + this.update(); + }; + + private update = () => { const now = Date.now(); const timeUntilEmit = this.interval - (now - this.last); if (timeUntilEmit > 0) { - setTimeout(() => this.write([]), timeUntilEmit); + setTimeout(() => this.update(), timeUntilEmit); } else { this.flush(); } diff --git a/src/packages/util/types/directory-listing.ts b/src/packages/util/types/directory-listing.ts index 4b9f58da669..11e82d54720 100644 --- a/src/packages/util/types/directory-listing.ts +++ b/src/packages/util/types/directory-listing.ts @@ -1,9 +1,15 @@ export interface DirectoryListingEntry { + // relative path (to containing directory) name: string; - isdir?: boolean; - issymlink?: boolean; - link_target?: string; // set if issymlink is true and we're able to determine the target of the link - size?: number; // bytes for file, number of entries for directory (*including* . and ..). - mtime?: number; + // number of *bytes* used to store this path. + size: number; + // last modification time in ms of this file + mtime: number; + // true if it is a directory + isDir?: boolean; + // true if it is a symlink + isSymLink?: boolean; + // set if issymlink is true and we're able to determine the target of the link + linkTarget?: string; error?: string; } diff --git a/src/packages/util/upgrades/consts.ts b/src/packages/util/upgrades/consts.ts index d0c6313eacc..106cc8d842c 100644 --- a/src/packages/util/upgrades/consts.ts +++ b/src/packages/util/upgrades/consts.ts @@ -27,7 +27,7 @@ export const MIN_DISK_GB = DISK_DEFAULT_GB; interface Values { min: number; - dflt: number; + default: number; max: number; } @@ -40,25 +40,25 @@ interface Limits { export const REGULAR: Limits = { cpu: { min: 1, - dflt: DEFAULT_CPU, + default: DEFAULT_CPU, max: MAX_CPU, }, ram: { min: 4, - dflt: RAM_DEFAULT_GB, + default: RAM_DEFAULT_GB, max: MAX_RAM_GB, }, disk: { min: MIN_DISK_GB, - dflt: DISK_DEFAULT_GB, + default: DISK_DEFAULT_GB, max: MAX_DISK_GB, }, } as const; export const BOOST: Limits = { - cpu: { min: 0, dflt: 0, max: MAX_CPU - 1 }, - ram: { min: 0, dflt: 0, max: MAX_RAM_GB - 1 }, - disk: { min: 0, dflt: 0, max: MAX_DISK_GB - 1 * DISK_DEFAULT_GB }, + cpu: { min: 0, default: 0, max: MAX_CPU - 1 }, + ram: { min: 0, default: 0, max: MAX_RAM_GB - 1 }, + disk: { min: 0, default: 0, max: MAX_DISK_GB - 1 * DISK_DEFAULT_GB }, } as const; // on-prem: this dedicated VM machine name is only used for cocalc-onprem diff --git a/src/packages/util/upgrades/quota.ts b/src/packages/util/upgrades/quota.ts index 8e532e38875..a7980a64284 100644 --- a/src/packages/util/upgrades/quota.ts +++ b/src/packages/util/upgrades/quota.ts @@ -637,21 +637,21 @@ function calcSiteLicenseQuotaIdleTimeout( // there is an old schema, inherited from SageMathCloud, etc. and newer iterations. // this helps by going from one schema to the newer one function upgrade2quota(up: Partial): RQuota { - const dflt_false = (x) => + const defaultFalse = (x) => x != null ? (typeof x === "boolean" ? x : to_int(x) >= 1) : false; - const dflt_num = (x) => + const defaultNumber = (x) => x != null ? (typeof x === "number" ? x : to_float(x)) : 0; return { - network: dflt_false(up.network), - member_host: dflt_false(up.member_host), - always_running: dflt_false(up.always_running), - disk_quota: dflt_num(up.disk_quota), - memory_limit: dflt_num(up.memory), - memory_request: dflt_num(up.memory_request), - cpu_limit: dflt_num(up.cores), - cpu_request: dflt_num(up.cpu_shares) / 1024, - privileged: dflt_false(up.privileged), - idle_timeout: dflt_num(up.mintime), + network: defaultFalse(up.network), + member_host: defaultFalse(up.member_host), + always_running: defaultFalse(up.always_running), + disk_quota: defaultNumber(up.disk_quota), + memory_limit: defaultNumber(up.memory), + memory_request: defaultNumber(up.memory_request), + cpu_limit: defaultNumber(up.cores), + cpu_request: defaultNumber(up.cpu_shares) / 1024, + privileged: defaultFalse(up.privileged), + idle_timeout: defaultNumber(up.mintime), dedicated_vm: false, // old schema has no dedicated_vm upgrades dedicated_disks: [] as DedicatedDisk[], // old schema has no dedicated_disk upgrades ext_rw: false, diff --git a/src/scripts/runoo b/src/scripts/runoo new file mode 100755 index 00000000000..4c0f8e93cca --- /dev/null +++ b/src/scripts/runoo @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +""" +This is just meant to be a quick and dirty python script so I can run other +scripts (e.g., a unit test runner) to get a sense if they are flaky or not.: + +runoo 20 python test.py # runs "python test.py" 20 times in parallel, ncpus at once + +""" + +import sys, os, time +from concurrent.futures import ProcessPoolExecutor, as_completed +import multiprocessing + +failed = False +def run_cmd(cmd, i): + global failed + if failed: + return + print('\n'*5) + print('*'*60) + print(f'* Loop: {i+1}/{n} , Time: {round(time.time() - start)}, Command: {cmd}') + print('*'*60) + if os.system(cmd): + failed = True + print('\n'*5) + print('*'*60) + print(f'* Command failed on run {i+1}**') + print('*'*60) + print('\n'*5) + sys.exit(1); + +if __name__ == '__main__': + n = int(sys.argv[1]) + cmd = ' '.join(sys.argv[2:]) + k = multiprocessing.cpu_count() + start = time.time() + + with ProcessPoolExecutor(max_workers=k) as executor: + futures = [executor.submit(run_cmd, cmd, i) for i in range(n)] + for f in as_completed(futures): + f.result() # Raises exception if failed + + print('\n'*5) + print('*'*60) + print(f"Successfully ran {n} times") + print('*'*60) + \ No newline at end of file diff --git a/src/smc_sagews/smc_sagews/tests/test_sagews_modes.py b/src/smc_sagews/smc_sagews/tests/test_sagews_modes.py index 0f87827740c..54ffd9a4f39 100644 --- a/src/smc_sagews/smc_sagews/tests/test_sagews_modes.py +++ b/src/smc_sagews/smc_sagews/tests/test_sagews_modes.py @@ -200,34 +200,34 @@ def test_bad_command(self, exec2): class TestShDefaultMode: - def test_start_sh_dflt(self, exec2): + def test_start_sh_default(self, exec2): exec2("%default_mode sh") - def test_multiline_dflt(self, exec2): + def test_multiline_default(self, exec2): exec2("FOO=hello\necho $FOO", pattern="^hello") def test_date(self, exec2): exec2("date +%Y-%m-%d", pattern=r'^\d{4}-\d{2}-\d{2}') - def test_capture_sh_01_dflt(self, exec2): + def test_capture_sh_01_default(self, exec2): exec2("%capture(stdout='output')\nuptime") - def test_capture_sh_02_dflt(self, exec2): + def test_capture_sh_02_default(self, exec2): exec2("%sage\noutput", pattern="up.*user.*load average") - def test_remember_settings_01_dflt(self, exec2): + def test_remember_settings_01_default(self, exec2): exec2("FOO='testing123'") - def test_remember_settings_02_dflt(self, exec2): + def test_remember_settings_02_default(self, exec2): exec2("echo $FOO", pattern=r"^testing123\s+") - def test_sh_display_dflt(self, execblob, image_file): + def test_sh_display_default(self, execblob, image_file): execblob("display < " + str(image_file), want_html=False) - def test_sh_autocomplete_01_dflt(self, exec2): + def test_sh_autocomplete_01_default(self, exec2): exec2("TESTVAR29=xyz") - def test_sh_autocomplete_02_dflt(self, execintrospect): + def test_sh_autocomplete_02_default(self, execintrospect): execintrospect('echo $TESTV', ["AR29"], '$TESTV') @@ -249,13 +249,13 @@ class TestRDefaultMode: def test_set_r_mode(self, exec2): exec2("%default_mode r") - def test_rdflt_assignment(self, exec2): + def test_rdefault_assignment(self, exec2): exec2("xx <- c(4,7,13)\nmean(xx)", html_pattern="^8$") - def test_dflt_capture_r_01(self, exec2): + def test_default_capture_r_01(self, exec2): exec2("%capture(stdout='output')\nsum(xx)") - def test_dflt_capture_r_02(self, exec2): + def test_default_capture_r_02(self, exec2): exec2("%sage\nprint(output)", "24\n") diff --git a/src/workspaces.py b/src/workspaces.py index 707edf20bc1..53011ac7f6a 100755 --- a/src/workspaces.py +++ b/src/workspaces.py @@ -126,6 +126,7 @@ def all_packages() -> List[str]: 'packages/file-server', 'packages/next', 'packages/hub', # hub won't build if next isn't already built + 'packages/test' ] for x in os.listdir('packages'): path = os.path.join("packages", x) @@ -303,7 +304,11 @@ def f(): print(f"TESTING {n}/{len(v)}: {path}") print("*") print("*" * 40) - test_cmd = "pnpm run --if-present test" + if args.test_github_ci and 'test-github-ci' in open( + os.path.join(package_path, 'package.json')).read(): + test_cmd = "pnpm run test-github-ci" + else: + test_cmd = "pnpm run --if-present test" if args.report: test_cmd += " --reporters=default --reporters=jest-junit" cmd(test_cmd, package_path) @@ -581,6 +586,11 @@ def packages_arg(parser): help= "how many times to retry a failed test suite before giving up; set to 0 to NOT retry" ) + subparser.add_argument( + '--test-github-ci', + const=True, + action="store_const", + help="run 'pnpm test-github-ci' if available instead of 'pnpm test'") subparser.add_argument('--report', action="store_const", const=True,