From 757edf8260e3f252972d3edf79075b2c8acbeaa8 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Wed, 27 Nov 2024 17:28:44 +0000 Subject: [PATCH 01/47] hack open function to write files in main thread --- src/PyodideWorker.js | 55 ++++- .../PyodideRunner/VisualOutputPane.jsx | 189 +++++++++++++----- 2 files changed, 184 insertions(+), 60 deletions(-) diff --git a/src/PyodideWorker.js b/src/PyodideWorker.js index b50981742..bb2a4cda8 100644 --- a/src/PyodideWorker.js +++ b/src/PyodideWorker.js @@ -4,7 +4,7 @@ const PyodideWorker = () => { // Import scripts dynamically based on the environment importScripts( - `${process.env.ASSETS_URL}/pyodide/shims/_internal_sense_hat.js`, + `${process.env.ASSETS_URL}/pyodide/shims/_internal_sense_hat.js` ); importScripts(`${process.env.ASSETS_URL}/pyodide/shims/pygal.js`); importScripts("https://cdn.jsdelivr.net/pyodide/v0.26.2/full/pyodide.js"); @@ -29,7 +29,7 @@ const PyodideWorker = () => { "Please refer to these code snippets for registering a service worker:", " - https://github.com/RaspberryPiFoundation/python-execution-prototypes/blob/fd2c50e032cba3bb0e92e19a88eb62e5b120fe7a/pyodide/index.html#L92-L98", " - https://github.com/RaspberryPiFoundation/python-execution-prototypes/blob/fd2c50e032cba3bb0e92e19a88eb62e5b120fe7a/pyodide/serviceworker.js", - ].join("\n"), + ].join("\n") ); } let pyodide, pyodidePromise, stdinBuffer, interruptBuffer, stopped; @@ -61,11 +61,12 @@ const PyodideWorker = () => { const runPython = async (python) => { stopped = false; await pyodide.loadPackage("pyodide_http"); + // pyodide.registerJsModule("basthon", fakeBasthonPackage); - await pyodide.runPythonAsync(` - import pyodide_http - pyodide_http.patch_all() - `); + // await pyodide.runPythonAsync(` + // import pyodide_http + // pyodide_http.patch_all() + // `); try { await withSupportForPackages(python, async () => { @@ -89,7 +90,7 @@ const PyodideWorker = () => { const withSupportForPackages = async ( python, - runPythonFn = async () => {}, + runPythonFn = async () => {} ) => { const imports = await pyodide._api.pyodide_code.find_imports(python).toJs(); await Promise.all(imports.map((name) => loadDependency(name))); @@ -162,7 +163,7 @@ const PyodideWorker = () => { enigma: { before: async () => { await pyodide.loadPackage( - `${process.env.ASSETS_URL}/pyodide/packages/py_enigma-0.1-py3-none-any.whl`, + `${process.env.ASSETS_URL}/pyodide/packages/py_enigma-0.1-py3-none-any.whl` ); }, after: () => {}, @@ -171,7 +172,7 @@ const PyodideWorker = () => { before: async () => { pyodide.registerJsModule("basthon", fakeBasthonPackage); await pyodide.loadPackage( - `${process.env.ASSETS_URL}/pyodide/packages/turtle-0.0.1-py3-none-any.whl`, + `${process.env.ASSETS_URL}/pyodide/packages/turtle-0.0.1-py3-none-any.whl` ); }, after: () => @@ -334,8 +335,10 @@ const PyodideWorker = () => { display_event: (event) => { const origin = event.toJs().get("display_type"); const content = event.toJs().get("content"); + const filename = event.toJs().get("filename"); - postMessage({ method: "handleVisual", origin, content }); + console.log({ origin, content, filename }); + postMessage({ method: "handleVisual", origin, content, filename }); }, locals: () => pyodide.runPython("globals()"), }, @@ -364,6 +367,8 @@ const PyodideWorker = () => { pyodide = await pyodidePromise; + pyodide.registerJsModule("basthon", fakeBasthonPackage); + await pyodide.runPythonAsync(` __old_input__ = input def __patched_input__(prompt=False): @@ -373,6 +378,36 @@ const PyodideWorker = () => { __builtins__.input = __patched_input__ `); + await pyodide.runPythonAsync(` + import basthon + import builtins + + # Save the original open function + _original_open = builtins.open + + def _custom_open(filename, mode="r", *args, **kwargs): + if "w" in mode or "a" in mode: + class CustomFile: + def __init__(self, filename): + self.filename = filename + self.content = "" + + def write(self, content): + self.content += content + # print(f"{self.filename} {self.content}") + basthon.kernel.display_event({ "display_type": "file", "filename": self.filename, "content": str({"filename": self.filename, "content": self.content}) }) + + def close(self): + pass + + return CustomFile(filename) + else: + return _original_open(filename, mode, *args, **kwargs) + + # Override the built-in open function + builtins.open = _custom_open + `); + if (supportsAllFeatures) { stdinBuffer = stdinBuffer || new Int32Array(new SharedArrayBuffer(1024 * 1024)); // 1 MiB diff --git a/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx b/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx index 339da793c..9db07dd7c 100644 --- a/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx +++ b/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx @@ -1,12 +1,15 @@ import React, { useEffect, useRef } from "react"; -import { useSelector } from "react-redux"; +import { useDispatch, useSelector } from "react-redux"; import AstroPiModel from "../../../../AstroPiModel/AstroPiModel"; import Highcharts from "highcharts"; +import { updateProjectComponent } from "../../../../../redux/EditorSlice"; const VisualOutputPane = ({ visuals, setVisuals }) => { const senseHatEnabled = useSelector((s) => s.editor.senseHatEnabled); const senseHatAlways = useSelector((s) => s.editor.senseHatAlwaysEnabled); + // const projectComponents = useSelector((s) => s.editor.project.components); const output = useRef(); + const dispatch = useDispatch(); useEffect(() => { if (visuals.length === 0) { @@ -16,6 +19,70 @@ const VisualOutputPane = ({ visuals, setVisuals }) => { } }, [visuals, setVisuals]); + const showVisuals = (visuals, output) => + visuals.map((v) => (v.showing ? v : showVisual(v, output))); + + const showVisual = (visual, output) => { + switch (visual.origin) { + case "sense_hat": + output.current.textContent = JSON.stringify(visual.content); + break; + case "pygal": + const chartContent = { + ...visual.content, + chart: { + ...visual.content.chart, + events: { + ...visual.content.chart.events, + load: function () { + this.renderTo.style.overflow = "visible"; + }, + }, + }, + tooltip: { + ...visual.content.tooltip, + formatter: + visual.content.chart.type === "pie" + ? function () { + return this.key + ": " + this.y; + } + : null, + }, + }; + Highcharts.chart(output.current, chartContent); + break; + case "turtle": + output.current.innerHTML = elementFromProps(visual.content).outerHTML; + break; + case "matplotlib": + // convert visual.content from Uint8Array to jpg + const img = document.createElement("img"); + img.style = "max-width: 100%; max-height: 100%;"; + img.src = `data:image/jpg;base64,${window.btoa( + String.fromCharCode(...new Uint8Array(visual.content)) + )}`; + output.current.innerHTML = img.outerHTML; + break; + case "file": + console.log("from the main thread:", visual); + const content = JSON.parse(visual.content.replace(/'/g, '"')); + const [name, extension] = content.filename.split("."); + dispatch( + updateProjectComponent({ + extension: extension, + name: name, + code: content.content, + }) + ); + break; + default: + throw new Error(`Unsupported origin: ${visual.origin}`); + } + + visual.showing = true; + return visual; + }; + return (
@@ -24,57 +91,79 @@ const VisualOutputPane = ({ visuals, setVisuals }) => { ); }; -const showVisuals = (visuals, output) => - visuals.map((v) => (v.showing ? v : showVisual(v, output))); +// const showVisuals = (visuals, output) => +// visuals.map((v) => (v.showing ? v : showVisual(v, output))); -const showVisual = (visual, output) => { - switch (visual.origin) { - case "sense_hat": - output.current.textContent = JSON.stringify(visual.content); - break; - case "pygal": - const chartContent = { - ...visual.content, - chart: { - ...visual.content.chart, - events: { - ...visual.content.chart.events, - load: function () { - this.renderTo.style.overflow = "visible"; - }, - }, - }, - tooltip: { - ...visual.content.tooltip, - formatter: - visual.content.chart.type === "pie" - ? function () { - return this.key + ": " + this.y; - } - : null, - }, - }; - Highcharts.chart(output.current, chartContent); - break; - case "turtle": - output.current.innerHTML = elementFromProps(visual.content).outerHTML; - break; - case "matplotlib": - // convert visual.content from Uint8Array to jpg - const img = document.createElement("img"); - img.style = "max-width: 100%; max-height: 100%;"; - img.src = `data:image/jpg;base64,${window.btoa( - String.fromCharCode(...new Uint8Array(visual.content)), - )}`; - output.current.innerHTML = img.outerHTML; - break; - default: - throw new Error(`Unsupported origin: ${visual.origin}`); - } +// const showVisual = (visual, output) => { +// switch (visual.origin) { +// case "sense_hat": +// output.current.textContent = JSON.stringify(visual.content); +// break; +// case "pygal": +// const chartContent = { +// ...visual.content, +// chart: { +// ...visual.content.chart, +// events: { +// ...visual.content.chart.events, +// load: function () { +// this.renderTo.style.overflow = "visible"; +// }, +// }, +// }, +// tooltip: { +// ...visual.content.tooltip, +// formatter: +// visual.content.chart.type === "pie" +// ? function () { +// return this.key + ": " + this.y; +// } +// : null, +// }, +// }; +// Highcharts.chart(output.current, chartContent); +// break; +// case "turtle": +// output.current.innerHTML = elementFromProps(visual.content).outerHTML; +// break; +// case "matplotlib": +// // convert visual.content from Uint8Array to jpg +// const img = document.createElement("img"); +// img.style = "max-width: 100%; max-height: 100%;"; +// img.src = `data:image/jpg;base64,${window.btoa( +// String.fromCharCode(...new Uint8Array(visual.content)) +// )}`; +// output.current.innerHTML = img.outerHTML; +// break; +// case "file": +// // const componentToUpdate = projectComponents.find( +// // (c) => c.name === visual.filename, +// // ); +// // if (componentToUpdate) { +// // dispatch( +// // updateProjectComponent({ +// // extension: extension, +// // name: fileName, +// // code: content, +// // }), +// // ); +// // } +// const [name, extension] = visual.filename.split("."); +// dispatch( +// updateProjectComponent({ +// extension: extension, +// name: name, +// code: visual.content, +// }) +// ); +// break; +// default: +// throw new Error(`Unsupported origin: ${visual.origin}`); +// } - visual.showing = true; - return visual; -}; +// visual.showing = true; +// return visual; +// }; const elementFromProps = (map) => { const tag = map.get("tag"); From e5d4b40490b8eef79f584702bf147a521c4266d5 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Wed, 27 Nov 2024 17:30:23 +0000 Subject: [PATCH 02/47] tidying --- .../PyodideRunner/VisualOutputPane.jsx | 74 ------------------- 1 file changed, 74 deletions(-) diff --git a/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx b/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx index 9db07dd7c..01bd4cd96 100644 --- a/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx +++ b/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx @@ -91,80 +91,6 @@ const VisualOutputPane = ({ visuals, setVisuals }) => { ); }; -// const showVisuals = (visuals, output) => -// visuals.map((v) => (v.showing ? v : showVisual(v, output))); - -// const showVisual = (visual, output) => { -// switch (visual.origin) { -// case "sense_hat": -// output.current.textContent = JSON.stringify(visual.content); -// break; -// case "pygal": -// const chartContent = { -// ...visual.content, -// chart: { -// ...visual.content.chart, -// events: { -// ...visual.content.chart.events, -// load: function () { -// this.renderTo.style.overflow = "visible"; -// }, -// }, -// }, -// tooltip: { -// ...visual.content.tooltip, -// formatter: -// visual.content.chart.type === "pie" -// ? function () { -// return this.key + ": " + this.y; -// } -// : null, -// }, -// }; -// Highcharts.chart(output.current, chartContent); -// break; -// case "turtle": -// output.current.innerHTML = elementFromProps(visual.content).outerHTML; -// break; -// case "matplotlib": -// // convert visual.content from Uint8Array to jpg -// const img = document.createElement("img"); -// img.style = "max-width: 100%; max-height: 100%;"; -// img.src = `data:image/jpg;base64,${window.btoa( -// String.fromCharCode(...new Uint8Array(visual.content)) -// )}`; -// output.current.innerHTML = img.outerHTML; -// break; -// case "file": -// // const componentToUpdate = projectComponents.find( -// // (c) => c.name === visual.filename, -// // ); -// // if (componentToUpdate) { -// // dispatch( -// // updateProjectComponent({ -// // extension: extension, -// // name: fileName, -// // code: content, -// // }), -// // ); -// // } -// const [name, extension] = visual.filename.split("."); -// dispatch( -// updateProjectComponent({ -// extension: extension, -// name: name, -// code: visual.content, -// }) -// ); -// break; -// default: -// throw new Error(`Unsupported origin: ${visual.origin}`); -// } - -// visual.showing = true; -// return visual; -// }; - const elementFromProps = (map) => { const tag = map.get("tag"); if (!tag) { From defa6193e41bf6eada81dc928975cafd1ed12291 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Wed, 11 Dec 2024 10:18:00 +0000 Subject: [PATCH 03/47] fixing rerun bug --- src/PyodideWorker.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/PyodideWorker.js b/src/PyodideWorker.js index bb2a4cda8..f0c024352 100644 --- a/src/PyodideWorker.js +++ b/src/PyodideWorker.js @@ -63,7 +63,7 @@ const PyodideWorker = () => { await pyodide.loadPackage("pyodide_http"); // pyodide.registerJsModule("basthon", fakeBasthonPackage); - // await pyodide.runPythonAsync(` + // await pyodide.runPythonAsync(` // import pyodide_http // pyodide_http.patch_all() // `); @@ -349,7 +349,7 @@ const PyodideWorker = () => { await pyodide.runPythonAsync(` # Clear all user-defined variables and modules for name in dir(): - if not name.startswith('_'): + if not name.startswith('_') and not name=='basthon': del globals()[name] `); postMessage({ method: "handleLoaded", stdinBuffer, interruptBuffer }); @@ -395,7 +395,7 @@ const PyodideWorker = () => { def write(self, content): self.content += content # print(f"{self.filename} {self.content}") - basthon.kernel.display_event({ "display_type": "file", "filename": self.filename, "content": str({"filename": self.filename, "content": self.content}) }) + basthon.kernel.display_event({ "display_type": "file", "filename": self.filename, "content": str({"filename": self.filename, "content": self.content, "mode": mode}) }) def close(self): pass From 5897d2768d60569b287cb57d4ada3e7640da2d74 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Wed, 11 Dec 2024 10:18:13 +0000 Subject: [PATCH 04/47] getting append mode working --- .../PythonRunner/PyodideRunner/VisualOutputPane.jsx | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx b/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx index 01bd4cd96..c72e7e8d6 100644 --- a/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx +++ b/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx @@ -9,6 +9,7 @@ const VisualOutputPane = ({ visuals, setVisuals }) => { const senseHatAlways = useSelector((s) => s.editor.senseHatAlwaysEnabled); // const projectComponents = useSelector((s) => s.editor.project.components); const output = useRef(); + const project = useSelector((state) => state.editor.project); const dispatch = useDispatch(); useEffect(() => { @@ -67,11 +68,21 @@ const VisualOutputPane = ({ visuals, setVisuals }) => { console.log("from the main thread:", visual); const content = JSON.parse(visual.content.replace(/'/g, '"')); const [name, extension] = content.filename.split("."); + let updatedContent; + if (content.mode === "w") { + updatedContent = content.content; + } else if (content.mode === "a") { + const componentToUpdate = project.components.find( + (item) => item.extension === extension && item.name === name + ); + updatedContent = componentToUpdate.content + "\n" + content.content; + } + dispatch( updateProjectComponent({ extension: extension, name: name, - code: content.content, + code: updatedContent, }) ); break; From b3689ebc09e75387b248cab4e2ec664aeaaf7485 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Wed, 11 Dec 2024 10:24:58 +0000 Subject: [PATCH 05/47] create the file if it does not already exist --- .../PyodideRunner/VisualOutputPane.jsx | 31 ++++++++++++------- src/redux/EditorSlice.js | 12 +++---- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx b/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx index c72e7e8d6..8b6d8ec8d 100644 --- a/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx +++ b/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx @@ -2,7 +2,10 @@ import React, { useEffect, useRef } from "react"; import { useDispatch, useSelector } from "react-redux"; import AstroPiModel from "../../../../AstroPiModel/AstroPiModel"; import Highcharts from "highcharts"; -import { updateProjectComponent } from "../../../../../redux/EditorSlice"; +import { + addProjectComponent, + updateProjectComponent, +} from "../../../../../redux/EditorSlice"; const VisualOutputPane = ({ visuals, setVisuals }) => { const senseHatEnabled = useSelector((s) => s.editor.senseHatEnabled); @@ -68,23 +71,29 @@ const VisualOutputPane = ({ visuals, setVisuals }) => { console.log("from the main thread:", visual); const content = JSON.parse(visual.content.replace(/'/g, '"')); const [name, extension] = content.filename.split("."); + const componentToUpdate = project.components.find( + (item) => item.extension === extension && item.name === name + ); let updatedContent; if (content.mode === "w") { updatedContent = content.content; } else if (content.mode === "a") { - const componentToUpdate = project.components.find( - (item) => item.extension === extension && item.name === name - ); updatedContent = componentToUpdate.content + "\n" + content.content; } - dispatch( - updateProjectComponent({ - extension: extension, - name: name, - code: updatedContent, - }) - ); + if (componentToUpdate) { + dispatch( + updateProjectComponent({ + extension, + name, + code: updatedContent, + }) + ); + } else { + dispatch( + addProjectComponent({ name, extension, content: updatedContent }) + ); + } break; default: throw new Error(`Unsupported origin: ${visual.origin}`); diff --git a/src/redux/EditorSlice.js b/src/redux/EditorSlice.js index ec5119c87..890a554a7 100644 --- a/src/redux/EditorSlice.js +++ b/src/redux/EditorSlice.js @@ -20,7 +20,7 @@ export const syncProject = (actionName) => autosave, assetsOnly, }, - { rejectWithValue }, + { rejectWithValue } ) => { const { createOrUpdateProject, @@ -75,7 +75,7 @@ export const syncProject = (actionName) => return false; } }, - }, + } ); export const loadProjectList = createAsyncThunk( @@ -90,7 +90,7 @@ export const loadProjectList = createAsyncThunk( page, links: parseLinkHeader(response.headers.link), }; - }, + } ); const initialState = { @@ -151,10 +151,10 @@ export const EditorSlice = createSlice({ .map((fileNames) => fileNames.includes(action.payload)) .indexOf(true); const closedFileIndex = state.openFiles[panelIndex].indexOf( - action.payload, + action.payload ); state.openFiles[panelIndex] = state.openFiles[panelIndex].filter( - (fileName) => fileName !== action.payload, + (fileName) => fileName !== action.payload ); if ( state.focussedFileIndices[panelIndex] >= @@ -197,7 +197,7 @@ export const EditorSlice = createSlice({ state.project.components.push({ name: action.payload.name, extension: action.payload.extension, - content: "", + content: action.payload.content || "", }); state.saving = "idle"; }, From eee4c1dfa53822b2bab824613bf566a0aa1974c3 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Wed, 11 Dec 2024 10:33:22 +0000 Subject: [PATCH 06/47] tidying --- .../Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx b/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx index 8b6d8ec8d..073d31f78 100644 --- a/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx +++ b/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx @@ -10,9 +10,8 @@ import { const VisualOutputPane = ({ visuals, setVisuals }) => { const senseHatEnabled = useSelector((s) => s.editor.senseHatEnabled); const senseHatAlways = useSelector((s) => s.editor.senseHatAlwaysEnabled); - // const projectComponents = useSelector((s) => s.editor.project.components); + const projectComponents = useSelector((s) => s.editor.project.components); const output = useRef(); - const project = useSelector((state) => state.editor.project); const dispatch = useDispatch(); useEffect(() => { @@ -71,7 +70,7 @@ const VisualOutputPane = ({ visuals, setVisuals }) => { console.log("from the main thread:", visual); const content = JSON.parse(visual.content.replace(/'/g, '"')); const [name, extension] = content.filename.split("."); - const componentToUpdate = project.components.find( + const componentToUpdate = projectComponents.find( (item) => item.extension === extension && item.name === name ); let updatedContent; From 2ea87a0e4fd28080d0ee2556943ff95a832af8ea Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Wed, 11 Dec 2024 13:13:40 +0000 Subject: [PATCH 07/47] getting for open as f and mode=x working --- src/PyodideWorker.js | 11 ++++++++++- .../PythonRunner/PyodideRunner/VisualOutputPane.jsx | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/PyodideWorker.js b/src/PyodideWorker.js index f0c024352..dbb882b51 100644 --- a/src/PyodideWorker.js +++ b/src/PyodideWorker.js @@ -381,12 +381,15 @@ const PyodideWorker = () => { await pyodide.runPythonAsync(` import basthon import builtins + import os # Save the original open function _original_open = builtins.open def _custom_open(filename, mode="r", *args, **kwargs): - if "w" in mode or "a" in mode: + if "x" in mode and os.path.exists(filename): + raise FileExistsError(f"File '{filename}' already exists") + if "w" in mode or "a" in mode or "x" in mode: class CustomFile: def __init__(self, filename): self.filename = filename @@ -400,6 +403,12 @@ const PyodideWorker = () => { def close(self): pass + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + return CustomFile(filename) else: return _original_open(filename, mode, *args, **kwargs) diff --git a/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx b/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx index 073d31f78..71c84c2d4 100644 --- a/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx +++ b/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx @@ -74,7 +74,7 @@ const VisualOutputPane = ({ visuals, setVisuals }) => { (item) => item.extension === extension && item.name === name ); let updatedContent; - if (content.mode === "w") { + if (content.mode === "w" || content.mode === "x") { updatedContent = content.content; } else if (content.mode === "a") { updatedContent = componentToUpdate.content + "\n" + content.content; From c6517911eedf2f0f34a1a7aa07126a1b2acfc668 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Tue, 24 Dec 2024 12:57:14 +0000 Subject: [PATCH 08/47] fixing "a" mode when file not already existing --- .../Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx b/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx index 71c84c2d4..c834c2c4f 100644 --- a/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx +++ b/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx @@ -77,7 +77,9 @@ const VisualOutputPane = ({ visuals, setVisuals }) => { if (content.mode === "w" || content.mode === "x") { updatedContent = content.content; } else if (content.mode === "a") { - updatedContent = componentToUpdate.content + "\n" + content.content; + updatedContent = + (componentToUpdate ? componentToUpdate.content + "\n" : "") + + content.content; } if (componentToUpdate) { From a41e9b3cf427638b18395a64013e93e22cb41571 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Tue, 24 Dec 2024 14:14:01 +0000 Subject: [PATCH 09/47] fixing pyodide http patch --- src/PyodideWorker.js | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/PyodideWorker.js b/src/PyodideWorker.js index dbb882b51..66f4065b2 100644 --- a/src/PyodideWorker.js +++ b/src/PyodideWorker.js @@ -60,13 +60,6 @@ const PyodideWorker = () => { const runPython = async (python) => { stopped = false; - await pyodide.loadPackage("pyodide_http"); - // pyodide.registerJsModule("basthon", fakeBasthonPackage); - - // await pyodide.runPythonAsync(` - // import pyodide_http - // pyodide_http.patch_all() - // `); try { await withSupportForPackages(python, async () => { @@ -378,6 +371,12 @@ const PyodideWorker = () => { __builtins__.input = __patched_input__ `); + await pyodide.loadPackage("pyodide-http"); + await pyodide.runPythonAsync(` + import pyodide_http + pyodide_http.patch_all() + `); + await pyodide.runPythonAsync(` import basthon import builtins From 40711798fb2c779facf2f1bdd94c2c8c67662b58 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Tue, 24 Dec 2024 14:19:11 +0000 Subject: [PATCH 10/47] updating changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ff853b266..ada99c374 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased + +### Added + +- Ability to write to files in `python` (#1146) + ## [0.28.13] - 2024-12-18 ### Changed From b44c1be96cc621d8e199d54707584ce1078f7a04 Mon Sep 17 00:00:00 2001 From: Lois Wells <88904316+loiswells97@users.noreply.github.com> Date: Tue, 24 Dec 2024 14:26:34 +0000 Subject: [PATCH 11/47] Update PyodideWorker.js commas --- src/PyodideWorker.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/PyodideWorker.js b/src/PyodideWorker.js index 66f4065b2..7cd235736 100644 --- a/src/PyodideWorker.js +++ b/src/PyodideWorker.js @@ -4,7 +4,7 @@ const PyodideWorker = () => { // Import scripts dynamically based on the environment importScripts( - `${process.env.ASSETS_URL}/pyodide/shims/_internal_sense_hat.js` + `${process.env.ASSETS_URL}/pyodide/shims/_internal_sense_hat.js`, ); importScripts(`${process.env.ASSETS_URL}/pyodide/shims/pygal.js`); importScripts("https://cdn.jsdelivr.net/pyodide/v0.26.2/full/pyodide.js"); @@ -29,7 +29,7 @@ const PyodideWorker = () => { "Please refer to these code snippets for registering a service worker:", " - https://github.com/RaspberryPiFoundation/python-execution-prototypes/blob/fd2c50e032cba3bb0e92e19a88eb62e5b120fe7a/pyodide/index.html#L92-L98", " - https://github.com/RaspberryPiFoundation/python-execution-prototypes/blob/fd2c50e032cba3bb0e92e19a88eb62e5b120fe7a/pyodide/serviceworker.js", - ].join("\n") + ].join("\n"), ); } let pyodide, pyodidePromise, stdinBuffer, interruptBuffer, stopped; @@ -83,7 +83,7 @@ const PyodideWorker = () => { const withSupportForPackages = async ( python, - runPythonFn = async () => {} + runPythonFn = async () => {}, ) => { const imports = await pyodide._api.pyodide_code.find_imports(python).toJs(); await Promise.all(imports.map((name) => loadDependency(name))); @@ -156,7 +156,7 @@ const PyodideWorker = () => { enigma: { before: async () => { await pyodide.loadPackage( - `${process.env.ASSETS_URL}/pyodide/packages/py_enigma-0.1-py3-none-any.whl` + `${process.env.ASSETS_URL}/pyodide/packages/py_enigma-0.1-py3-none-any.whl`, ); }, after: () => {}, @@ -165,7 +165,7 @@ const PyodideWorker = () => { before: async () => { pyodide.registerJsModule("basthon", fakeBasthonPackage); await pyodide.loadPackage( - `${process.env.ASSETS_URL}/pyodide/packages/turtle-0.0.1-py3-none-any.whl` + `${process.env.ASSETS_URL}/pyodide/packages/turtle-0.0.1-py3-none-any.whl`, ); }, after: () => From 2133244437e3024c5ccf979debe7952caa94d36f Mon Sep 17 00:00:00 2001 From: Lois Wells <88904316+loiswells97@users.noreply.github.com> Date: Tue, 24 Dec 2024 14:27:57 +0000 Subject: [PATCH 12/47] Update VisualOutputPane.jsx commas --- .../PythonRunner/PyodideRunner/VisualOutputPane.jsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx b/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx index c834c2c4f..b99ccc1d9 100644 --- a/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx +++ b/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx @@ -62,7 +62,7 @@ const VisualOutputPane = ({ visuals, setVisuals }) => { const img = document.createElement("img"); img.style = "max-width: 100%; max-height: 100%;"; img.src = `data:image/jpg;base64,${window.btoa( - String.fromCharCode(...new Uint8Array(visual.content)) + String.fromCharCode(...new Uint8Array(visual.content)), )}`; output.current.innerHTML = img.outerHTML; break; @@ -71,7 +71,7 @@ const VisualOutputPane = ({ visuals, setVisuals }) => { const content = JSON.parse(visual.content.replace(/'/g, '"')); const [name, extension] = content.filename.split("."); const componentToUpdate = projectComponents.find( - (item) => item.extension === extension && item.name === name + (item) => item.extension === extension && item.name === name, ); let updatedContent; if (content.mode === "w" || content.mode === "x") { @@ -88,11 +88,11 @@ const VisualOutputPane = ({ visuals, setVisuals }) => { extension, name, code: updatedContent, - }) + }), ); } else { dispatch( - addProjectComponent({ name, extension, content: updatedContent }) + addProjectComponent({ name, extension, content: updatedContent }), ); } break; From f6682824cbe2bd787cb6da1c0cc7dc92b3e36cef Mon Sep 17 00:00:00 2001 From: Lois Wells <88904316+loiswells97@users.noreply.github.com> Date: Tue, 24 Dec 2024 14:28:56 +0000 Subject: [PATCH 13/47] Update EditorSlice.js commas --- src/redux/EditorSlice.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/redux/EditorSlice.js b/src/redux/EditorSlice.js index 890a554a7..79ed03394 100644 --- a/src/redux/EditorSlice.js +++ b/src/redux/EditorSlice.js @@ -20,7 +20,7 @@ export const syncProject = (actionName) => autosave, assetsOnly, }, - { rejectWithValue } + { rejectWithValue }, ) => { const { createOrUpdateProject, @@ -75,7 +75,7 @@ export const syncProject = (actionName) => return false; } }, - } + }, ); export const loadProjectList = createAsyncThunk( @@ -90,7 +90,7 @@ export const loadProjectList = createAsyncThunk( page, links: parseLinkHeader(response.headers.link), }; - } + }, ); const initialState = { @@ -151,10 +151,10 @@ export const EditorSlice = createSlice({ .map((fileNames) => fileNames.includes(action.payload)) .indexOf(true); const closedFileIndex = state.openFiles[panelIndex].indexOf( - action.payload + action.payload, ); state.openFiles[panelIndex] = state.openFiles[panelIndex].filter( - (fileName) => fileName !== action.payload + (fileName) => fileName !== action.payload, ); if ( state.focussedFileIndices[panelIndex] >= From 83ca96034ed84b92a377bdf5ca69a7a8d81a7c70 Mon Sep 17 00:00:00 2001 From: Lois Wells <88904316+loiswells97@users.noreply.github.com> Date: Tue, 24 Dec 2024 14:30:26 +0000 Subject: [PATCH 14/47] Update VisualOutputPane.jsx hook deps --- .../Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx b/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx index b99ccc1d9..9096924f9 100644 --- a/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx +++ b/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx @@ -20,7 +20,7 @@ const VisualOutputPane = ({ visuals, setVisuals }) => { } else if (visuals.some((v) => !v.showing)) { setVisuals((visuals) => showVisuals(visuals, output)); } - }, [visuals, setVisuals]); + }, [visuals, setVisuals, showVisuals]); const showVisuals = (visuals, output) => visuals.map((v) => (v.showing ? v : showVisual(v, output))); From 9b4b1f2ced575848e9461439b972181311aac3c9 Mon Sep 17 00:00:00 2001 From: Lois Wells <88904316+loiswells97@users.noreply.github.com> Date: Tue, 24 Dec 2024 14:38:59 +0000 Subject: [PATCH 15/47] Update VisualOutputPane.jsx move useeffect hook --- .../PyodideRunner/VisualOutputPane.jsx | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx b/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx index 9096924f9..a7f82b9e9 100644 --- a/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx +++ b/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx @@ -14,14 +14,6 @@ const VisualOutputPane = ({ visuals, setVisuals }) => { const output = useRef(); const dispatch = useDispatch(); - useEffect(() => { - if (visuals.length === 0) { - output.current.innerHTML = ""; - } else if (visuals.some((v) => !v.showing)) { - setVisuals((visuals) => showVisuals(visuals, output)); - } - }, [visuals, setVisuals, showVisuals]); - const showVisuals = (visuals, output) => visuals.map((v) => (v.showing ? v : showVisual(v, output))); @@ -104,6 +96,14 @@ const VisualOutputPane = ({ visuals, setVisuals }) => { return visual; }; + useEffect(() => { + if (visuals.length === 0) { + output.current.innerHTML = ""; + } else if (visuals.some((v) => !v.showing)) { + setVisuals((visuals) => showVisuals(visuals, output)); + } + }, [visuals, setVisuals, showVisuals]); + return (
From 213a18d0c0ee9b67ad7f8ab054229b402c84fcab Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Tue, 24 Dec 2024 17:06:48 +0000 Subject: [PATCH 16/47] hopefully fix import problems caused by hacking the python import function --- src/PyodideWorker.js | 87 ++++++++++++++++++++++---------------------- 1 file changed, 43 insertions(+), 44 deletions(-) diff --git a/src/PyodideWorker.js b/src/PyodideWorker.js index 7cd235736..1ba4321f7 100644 --- a/src/PyodideWorker.js +++ b/src/PyodideWorker.js @@ -4,7 +4,7 @@ const PyodideWorker = () => { // Import scripts dynamically based on the environment importScripts( - `${process.env.ASSETS_URL}/pyodide/shims/_internal_sense_hat.js`, + `${process.env.ASSETS_URL}/pyodide/shims/_internal_sense_hat.js` ); importScripts(`${process.env.ASSETS_URL}/pyodide/shims/pygal.js`); importScripts("https://cdn.jsdelivr.net/pyodide/v0.26.2/full/pyodide.js"); @@ -29,7 +29,7 @@ const PyodideWorker = () => { "Please refer to these code snippets for registering a service worker:", " - https://github.com/RaspberryPiFoundation/python-execution-prototypes/blob/fd2c50e032cba3bb0e92e19a88eb62e5b120fe7a/pyodide/index.html#L92-L98", " - https://github.com/RaspberryPiFoundation/python-execution-prototypes/blob/fd2c50e032cba3bb0e92e19a88eb62e5b120fe7a/pyodide/serviceworker.js", - ].join("\n"), + ].join("\n") ); } let pyodide, pyodidePromise, stdinBuffer, interruptBuffer, stopped; @@ -83,7 +83,7 @@ const PyodideWorker = () => { const withSupportForPackages = async ( python, - runPythonFn = async () => {}, + runPythonFn = async () => {} ) => { const imports = await pyodide._api.pyodide_code.find_imports(python).toJs(); await Promise.all(imports.map((name) => loadDependency(name))); @@ -92,6 +92,44 @@ const PyodideWorker = () => { await pyodide.loadPackagesFromImports(python); checkIfStopped(); + await pyodide.runPythonAsync(` + import basthon + import builtins + import os + + # Save the original open function + _original_open = builtins.open + + def _custom_open(filename, mode="r", *args, **kwargs): + if "x" in mode and os.path.exists(filename): + raise FileExistsError(f"File '{filename}' already exists") + if "w" in mode or "a" in mode or "x" in mode: + class CustomFile: + def __init__(self, filename): + self.filename = filename + self.content = "" + + def write(self, content): + self.content += content + # print(f"{self.filename} {self.content}") + basthon.kernel.display_event({ "display_type": "file", "filename": self.filename, "content": str({"filename": self.filename, "content": self.content, "mode": mode}) }) + + def close(self): + pass + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + return CustomFile(filename) + else: + return _original_open(filename, mode, *args, **kwargs) + + # Override the built-in open function + builtins.open = _custom_open + `); await runPythonFn(); for (let name of imports) { @@ -156,7 +194,7 @@ const PyodideWorker = () => { enigma: { before: async () => { await pyodide.loadPackage( - `${process.env.ASSETS_URL}/pyodide/packages/py_enigma-0.1-py3-none-any.whl`, + `${process.env.ASSETS_URL}/pyodide/packages/py_enigma-0.1-py3-none-any.whl` ); }, after: () => {}, @@ -165,7 +203,7 @@ const PyodideWorker = () => { before: async () => { pyodide.registerJsModule("basthon", fakeBasthonPackage); await pyodide.loadPackage( - `${process.env.ASSETS_URL}/pyodide/packages/turtle-0.0.1-py3-none-any.whl`, + `${process.env.ASSETS_URL}/pyodide/packages/turtle-0.0.1-py3-none-any.whl` ); }, after: () => @@ -377,45 +415,6 @@ const PyodideWorker = () => { pyodide_http.patch_all() `); - await pyodide.runPythonAsync(` - import basthon - import builtins - import os - - # Save the original open function - _original_open = builtins.open - - def _custom_open(filename, mode="r", *args, **kwargs): - if "x" in mode and os.path.exists(filename): - raise FileExistsError(f"File '{filename}' already exists") - if "w" in mode or "a" in mode or "x" in mode: - class CustomFile: - def __init__(self, filename): - self.filename = filename - self.content = "" - - def write(self, content): - self.content += content - # print(f"{self.filename} {self.content}") - basthon.kernel.display_event({ "display_type": "file", "filename": self.filename, "content": str({"filename": self.filename, "content": self.content, "mode": mode}) }) - - def close(self): - pass - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.close() - - return CustomFile(filename) - else: - return _original_open(filename, mode, *args, **kwargs) - - # Override the built-in open function - builtins.open = _custom_open - `); - if (supportsAllFeatures) { stdinBuffer = stdinBuffer || new Int32Array(new SharedArrayBuffer(1024 * 1024)); // 1 MiB From d5be56f968b996cc2ec5c2cdc3d72f27faf92727 Mon Sep 17 00:00:00 2001 From: Lois Wells <88904316+loiswells97@users.noreply.github.com> Date: Tue, 24 Dec 2024 17:10:19 +0000 Subject: [PATCH 17/47] Update PyodideWorker.js linting --- src/PyodideWorker.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/PyodideWorker.js b/src/PyodideWorker.js index 1ba4321f7..ea0b3a5cc 100644 --- a/src/PyodideWorker.js +++ b/src/PyodideWorker.js @@ -4,7 +4,7 @@ const PyodideWorker = () => { // Import scripts dynamically based on the environment importScripts( - `${process.env.ASSETS_URL}/pyodide/shims/_internal_sense_hat.js` + `${process.env.ASSETS_URL}/pyodide/shims/_internal_sense_hat.js`, ); importScripts(`${process.env.ASSETS_URL}/pyodide/shims/pygal.js`); importScripts("https://cdn.jsdelivr.net/pyodide/v0.26.2/full/pyodide.js"); @@ -29,7 +29,7 @@ const PyodideWorker = () => { "Please refer to these code snippets for registering a service worker:", " - https://github.com/RaspberryPiFoundation/python-execution-prototypes/blob/fd2c50e032cba3bb0e92e19a88eb62e5b120fe7a/pyodide/index.html#L92-L98", " - https://github.com/RaspberryPiFoundation/python-execution-prototypes/blob/fd2c50e032cba3bb0e92e19a88eb62e5b120fe7a/pyodide/serviceworker.js", - ].join("\n") + ].join("\n"), ); } let pyodide, pyodidePromise, stdinBuffer, interruptBuffer, stopped; @@ -83,7 +83,7 @@ const PyodideWorker = () => { const withSupportForPackages = async ( python, - runPythonFn = async () => {} + runPythonFn = async () => {}, ) => { const imports = await pyodide._api.pyodide_code.find_imports(python).toJs(); await Promise.all(imports.map((name) => loadDependency(name))); @@ -194,7 +194,7 @@ const PyodideWorker = () => { enigma: { before: async () => { await pyodide.loadPackage( - `${process.env.ASSETS_URL}/pyodide/packages/py_enigma-0.1-py3-none-any.whl` + `${process.env.ASSETS_URL}/pyodide/packages/py_enigma-0.1-py3-none-any.whl`, ); }, after: () => {}, @@ -203,7 +203,7 @@ const PyodideWorker = () => { before: async () => { pyodide.registerJsModule("basthon", fakeBasthonPackage); await pyodide.loadPackage( - `${process.env.ASSETS_URL}/pyodide/packages/turtle-0.0.1-py3-none-any.whl` + `${process.env.ASSETS_URL}/pyodide/packages/turtle-0.0.1-py3-none-any.whl`, ); }, after: () => From 4513a856d1eba575df1bc2beeb4ed4e944027c77 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Thu, 2 Jan 2025 11:11:29 +0000 Subject: [PATCH 18/47] add file limits --- src/PyodideWorker.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/PyodideWorker.js b/src/PyodideWorker.js index ea0b3a5cc..3102415c7 100644 --- a/src/PyodideWorker.js +++ b/src/PyodideWorker.js @@ -100,10 +100,15 @@ const PyodideWorker = () => { # Save the original open function _original_open = builtins.open + MAX_FILES = 100 + MAX_FILE_SIZE = 8500000 + def _custom_open(filename, mode="r", *args, **kwargs): if "x" in mode and os.path.exists(filename): raise FileExistsError(f"File '{filename}' already exists") if "w" in mode or "a" in mode or "x" in mode: + if len(os.listdir()) > MAX_FILES and not os.path.exists(filename): + raise OSError(f"File system limit reached, no more than {MAX_FILES} files allowed") class CustomFile: def __init__(self, filename): self.filename = filename @@ -112,6 +117,8 @@ const PyodideWorker = () => { def write(self, content): self.content += content # print(f"{self.filename} {self.content}") + if len(self.content) > MAX_FILE_SIZE: + raise OSError(f"File '{self.filename}' exceeds maximum file size of {MAX_FILE_SIZE} bytes") basthon.kernel.display_event({ "display_type": "file", "filename": self.filename, "content": str({"filename": self.filename, "content": self.content, "mode": mode}) }) def close(self): From 82799cffba956236d65a5fcba67f7ff880d8570e Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Thu, 2 Jan 2025 11:53:43 +0000 Subject: [PATCH 19/47] tidying --- src/PyodideWorker.js | 1 - .../PyodideRunner/VisualOutputPane.jsx | 162 +++++++++--------- 2 files changed, 84 insertions(+), 79 deletions(-) diff --git a/src/PyodideWorker.js b/src/PyodideWorker.js index 3102415c7..a44e5f038 100644 --- a/src/PyodideWorker.js +++ b/src/PyodideWorker.js @@ -116,7 +116,6 @@ const PyodideWorker = () => { def write(self, content): self.content += content - # print(f"{self.filename} {self.content}") if len(self.content) > MAX_FILE_SIZE: raise OSError(f"File '{self.filename}' exceeds maximum file size of {MAX_FILE_SIZE} bytes") basthon.kernel.display_event({ "display_type": "file", "filename": self.filename, "content": str({"filename": self.filename, "content": self.content, "mode": mode}) }) diff --git a/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx b/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx index a7f82b9e9..dbe750cb4 100644 --- a/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx +++ b/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx @@ -1,4 +1,4 @@ -import React, { useEffect, useRef } from "react"; +import React, { useCallback, useEffect, useRef } from "react"; import { useDispatch, useSelector } from "react-redux"; import AstroPiModel from "../../../../AstroPiModel/AstroPiModel"; import Highcharts from "highcharts"; @@ -14,87 +14,93 @@ const VisualOutputPane = ({ visuals, setVisuals }) => { const output = useRef(); const dispatch = useDispatch(); - const showVisuals = (visuals, output) => - visuals.map((v) => (v.showing ? v : showVisual(v, output))); - - const showVisual = (visual, output) => { - switch (visual.origin) { - case "sense_hat": - output.current.textContent = JSON.stringify(visual.content); - break; - case "pygal": - const chartContent = { - ...visual.content, - chart: { - ...visual.content.chart, - events: { - ...visual.content.chart.events, - load: function () { - this.renderTo.style.overflow = "visible"; + const showVisual = useCallback( + (visual, output) => { + switch (visual.origin) { + case "sense_hat": + output.current.textContent = JSON.stringify(visual.content); + break; + case "pygal": + const chartContent = { + ...visual.content, + chart: { + ...visual.content.chart, + events: { + ...visual.content.chart.events, + load: function () { + this.renderTo.style.overflow = "visible"; + }, }, }, - }, - tooltip: { - ...visual.content.tooltip, - formatter: - visual.content.chart.type === "pie" - ? function () { - return this.key + ": " + this.y; - } - : null, - }, - }; - Highcharts.chart(output.current, chartContent); - break; - case "turtle": - output.current.innerHTML = elementFromProps(visual.content).outerHTML; - break; - case "matplotlib": - // convert visual.content from Uint8Array to jpg - const img = document.createElement("img"); - img.style = "max-width: 100%; max-height: 100%;"; - img.src = `data:image/jpg;base64,${window.btoa( - String.fromCharCode(...new Uint8Array(visual.content)), - )}`; - output.current.innerHTML = img.outerHTML; - break; - case "file": - console.log("from the main thread:", visual); - const content = JSON.parse(visual.content.replace(/'/g, '"')); - const [name, extension] = content.filename.split("."); - const componentToUpdate = projectComponents.find( - (item) => item.extension === extension && item.name === name, - ); - let updatedContent; - if (content.mode === "w" || content.mode === "x") { - updatedContent = content.content; - } else if (content.mode === "a") { - updatedContent = - (componentToUpdate ? componentToUpdate.content + "\n" : "") + - content.content; - } - - if (componentToUpdate) { - dispatch( - updateProjectComponent({ - extension, - name, - code: updatedContent, - }), - ); - } else { - dispatch( - addProjectComponent({ name, extension, content: updatedContent }), + tooltip: { + ...visual.content.tooltip, + formatter: + visual.content.chart.type === "pie" + ? function () { + return this.key + ": " + this.y; + } + : null, + }, + }; + Highcharts.chart(output.current, chartContent); + break; + case "turtle": + output.current.innerHTML = elementFromProps(visual.content).outerHTML; + break; + case "matplotlib": + // convert visual.content from Uint8Array to jpg + const img = document.createElement("img"); + img.style = "max-width: 100%; max-height: 100%;"; + img.src = `data:image/jpg;base64,${window.btoa( + String.fromCharCode(...new Uint8Array(visual.content)) + )}`; + output.current.innerHTML = img.outerHTML; + break; + case "file": + console.log("from the main thread:", visual); + const content = JSON.parse(visual.content.replace(/'/g, '"')); + const [name, extension] = content.filename.split("."); + const componentToUpdate = projectComponents.find( + (item) => item.extension === extension && item.name === name ); - } - break; - default: - throw new Error(`Unsupported origin: ${visual.origin}`); - } + let updatedContent; + if (content.mode === "w" || content.mode === "x") { + updatedContent = content.content; + } else if (content.mode === "a") { + updatedContent = + (componentToUpdate ? componentToUpdate.content + "\n" : "") + + content.content; + } - visual.showing = true; - return visual; - }; + if (componentToUpdate) { + dispatch( + updateProjectComponent({ + extension, + name, + code: updatedContent, + }) + ); + } else { + dispatch( + addProjectComponent({ name, extension, content: updatedContent }) + ); + } + break; + default: + throw new Error(`Unsupported origin: ${visual.origin}`); + } + + visual.showing = true; + return visual; + }, + [dispatch, projectComponents] + ); + + const showVisuals = useCallback( + (visuals, output) => + visuals.map((v) => (v.showing ? v : showVisual(v, output))), + [showVisual] + ); useEffect(() => { if (visuals.length === 0) { From a77390c1daa8c5f0361ea4e8bff95e18e8612928 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Thu, 2 Jan 2025 11:54:31 +0000 Subject: [PATCH 20/47] fixing linting in visual output pane --- .../PythonRunner/PyodideRunner/VisualOutputPane.jsx | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx b/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx index dbe750cb4..f7d2af531 100644 --- a/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx +++ b/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx @@ -52,7 +52,7 @@ const VisualOutputPane = ({ visuals, setVisuals }) => { const img = document.createElement("img"); img.style = "max-width: 100%; max-height: 100%;"; img.src = `data:image/jpg;base64,${window.btoa( - String.fromCharCode(...new Uint8Array(visual.content)) + String.fromCharCode(...new Uint8Array(visual.content)), )}`; output.current.innerHTML = img.outerHTML; break; @@ -61,7 +61,7 @@ const VisualOutputPane = ({ visuals, setVisuals }) => { const content = JSON.parse(visual.content.replace(/'/g, '"')); const [name, extension] = content.filename.split("."); const componentToUpdate = projectComponents.find( - (item) => item.extension === extension && item.name === name + (item) => item.extension === extension && item.name === name, ); let updatedContent; if (content.mode === "w" || content.mode === "x") { @@ -78,11 +78,11 @@ const VisualOutputPane = ({ visuals, setVisuals }) => { extension, name, code: updatedContent, - }) + }), ); } else { dispatch( - addProjectComponent({ name, extension, content: updatedContent }) + addProjectComponent({ name, extension, content: updatedContent }), ); } break; @@ -93,13 +93,13 @@ const VisualOutputPane = ({ visuals, setVisuals }) => { visual.showing = true; return visual; }, - [dispatch, projectComponents] + [dispatch, projectComponents], ); const showVisuals = useCallback( (visuals, output) => visuals.map((v) => (v.showing ? v : showVisual(v, output))), - [showVisual] + [showVisual], ); useEffect(() => { From 7bda5fb3b898e8ecd2d3f8d7121021f9a862270b Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Fri, 3 Jan 2025 17:22:02 +0000 Subject: [PATCH 21/47] fix bugs with reading infinite redefinition loop and writing not updating pyodide filesystem straightaway --- src/PyodideWorker.js | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/PyodideWorker.js b/src/PyodideWorker.js index a44e5f038..bb08ba598 100644 --- a/src/PyodideWorker.js +++ b/src/PyodideWorker.js @@ -40,6 +40,7 @@ const PyodideWorker = () => { switch (data.method) { case "writeFile": + console.log("writing to file in worker", data.filename, data.content); pyodide.FS.writeFile(data.filename, encoder.encode(data.content)); break; case "runPython": @@ -97,15 +98,12 @@ const PyodideWorker = () => { import builtins import os - # Save the original open function - _original_open = builtins.open - MAX_FILES = 100 MAX_FILE_SIZE = 8500000 def _custom_open(filename, mode="r", *args, **kwargs): if "x" in mode and os.path.exists(filename): - raise FileExistsError(f"File '{filename}' already exists") + raise FileExistsError(f"File '{filename}' already exists") if "w" in mode or "a" in mode or "x" in mode: if len(os.listdir()) > MAX_FILES and not os.path.exists(filename): raise OSError(f"File system limit reached, no more than {MAX_FILES} files allowed") @@ -118,6 +116,8 @@ const PyodideWorker = () => { self.content += content if len(self.content) > MAX_FILE_SIZE: raise OSError(f"File '{self.filename}' exceeds maximum file size of {MAX_FILE_SIZE} bytes") + with _original_open(self.filename, "w") as f: + f.write(self.content) basthon.kernel.display_event({ "display_type": "file", "filename": self.filename, "content": str({"filename": self.filename, "content": self.content, "mode": mode}) }) def close(self): @@ -415,6 +415,12 @@ const PyodideWorker = () => { __builtins__.input = __patched_input__ `); + await pyodide.runPythonAsync(` + import builtins + # Save the original open function + _original_open = builtins.open + `); + await pyodide.loadPackage("pyodide-http"); await pyodide.runPythonAsync(` import pyodide_http From f81cbee7c22b98e7b1a878c987abe224f2baa711 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Mon, 6 Jan 2025 16:46:58 +0000 Subject: [PATCH 22/47] refactoring file write handling out of visual output pane --- src/PyodideWorker.js | 13 +- .../PyodideRunner/PyodideRunner.jsx | 33 +++++ .../PyodideRunner/VisualOutputPane.jsx | 131 ++++++------------ 3 files changed, 87 insertions(+), 90 deletions(-) diff --git a/src/PyodideWorker.js b/src/PyodideWorker.js index bb08ba598..d2abbb091 100644 --- a/src/PyodideWorker.js +++ b/src/PyodideWorker.js @@ -40,7 +40,6 @@ const PyodideWorker = () => { switch (data.method) { case "writeFile": - console.log("writing to file in worker", data.filename, data.content); pyodide.FS.writeFile(data.filename, encoder.encode(data.content)); break; case "runPython": @@ -118,7 +117,7 @@ const PyodideWorker = () => { raise OSError(f"File '{self.filename}' exceeds maximum file size of {MAX_FILE_SIZE} bytes") with _original_open(self.filename, "w") as f: f.write(self.content) - basthon.kernel.display_event({ "display_type": "file", "filename": self.filename, "content": str({"filename": self.filename, "content": self.content, "mode": mode}) }) + basthon.kernel.write_file({ "filename": self.filename, "content": self.content, "mode": mode }) def close(self): pass @@ -372,10 +371,14 @@ const PyodideWorker = () => { display_event: (event) => { const origin = event.toJs().get("display_type"); const content = event.toJs().get("content"); - const filename = event.toJs().get("filename"); - console.log({ origin, content, filename }); - postMessage({ method: "handleVisual", origin, content, filename }); + postMessage({ method: "handleVisual", origin, content }); + }, + write_file: (event) => { + const filename = event.toJs().get("filename"); + const content = event.toJs().get("content"); + const mode = event.toJs().get("mode"); + postMessage({ method: "handleFileWrite", filename, content, mode }); }, locals: () => pyodide.runPython("globals()"), }, diff --git a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx index 157451968..e6adcdd89 100644 --- a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx +++ b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx @@ -8,6 +8,8 @@ import { setError, codeRunHandled, setLoadedRunner, + updateProjectComponent, + addProjectComponent, } from "../../../../../redux/EditorSlice"; import { Tab, Tabs, TabList, TabPanel } from "react-tabs"; import { useMediaQuery } from "react-responsive"; @@ -94,6 +96,9 @@ const PyodideRunner = ({ active }) => { data.info, ); break; + case "handleFileWrite": + handleFileWrite(data.filename, data.content, data.mode); + break; case "handleVisual": handleVisual(data.origin, data.content); break; @@ -194,6 +199,34 @@ const PyodideRunner = ({ active }) => { disableInput(); }; + const handleFileWrite = (filename, content, mode) => { + const [name, extension] = filename.split("."); + const componentToUpdate = projectCode.find( + (item) => item.extension === extension && item.name === name, + ); + let updatedContent; + if (mode === "w" || mode === "x") { + updatedContent = content; + } else if (mode === "a") { + updatedContent = + (componentToUpdate ? componentToUpdate.content + "\n" : "") + content; + } + + if (componentToUpdate) { + dispatch( + updateProjectComponent({ + extension, + name, + code: updatedContent, + }), + ); + } else { + dispatch( + addProjectComponent({ name, extension, content: updatedContent }), + ); + } + }; + const handleVisual = (origin, content) => { setHasVisual(true); setVisuals((array) => [...array, { origin, content }]); diff --git a/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx b/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx index f7d2af531..dadb12c6f 100644 --- a/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx +++ b/src/components/Editor/Runners/PythonRunner/PyodideRunner/VisualOutputPane.jsx @@ -1,100 +1,61 @@ import React, { useCallback, useEffect, useRef } from "react"; -import { useDispatch, useSelector } from "react-redux"; +import { useSelector } from "react-redux"; import AstroPiModel from "../../../../AstroPiModel/AstroPiModel"; import Highcharts from "highcharts"; -import { - addProjectComponent, - updateProjectComponent, -} from "../../../../../redux/EditorSlice"; const VisualOutputPane = ({ visuals, setVisuals }) => { const senseHatEnabled = useSelector((s) => s.editor.senseHatEnabled); const senseHatAlways = useSelector((s) => s.editor.senseHatAlwaysEnabled); - const projectComponents = useSelector((s) => s.editor.project.components); const output = useRef(); - const dispatch = useDispatch(); - const showVisual = useCallback( - (visual, output) => { - switch (visual.origin) { - case "sense_hat": - output.current.textContent = JSON.stringify(visual.content); - break; - case "pygal": - const chartContent = { - ...visual.content, - chart: { - ...visual.content.chart, - events: { - ...visual.content.chart.events, - load: function () { - this.renderTo.style.overflow = "visible"; - }, + const showVisual = useCallback((visual, output) => { + switch (visual.origin) { + case "sense_hat": + output.current.textContent = JSON.stringify(visual.content); + break; + case "pygal": + const chartContent = { + ...visual.content, + chart: { + ...visual.content.chart, + events: { + ...visual.content.chart.events, + load: function () { + this.renderTo.style.overflow = "visible"; }, }, - tooltip: { - ...visual.content.tooltip, - formatter: - visual.content.chart.type === "pie" - ? function () { - return this.key + ": " + this.y; - } - : null, - }, - }; - Highcharts.chart(output.current, chartContent); - break; - case "turtle": - output.current.innerHTML = elementFromProps(visual.content).outerHTML; - break; - case "matplotlib": - // convert visual.content from Uint8Array to jpg - const img = document.createElement("img"); - img.style = "max-width: 100%; max-height: 100%;"; - img.src = `data:image/jpg;base64,${window.btoa( - String.fromCharCode(...new Uint8Array(visual.content)), - )}`; - output.current.innerHTML = img.outerHTML; - break; - case "file": - console.log("from the main thread:", visual); - const content = JSON.parse(visual.content.replace(/'/g, '"')); - const [name, extension] = content.filename.split("."); - const componentToUpdate = projectComponents.find( - (item) => item.extension === extension && item.name === name, - ); - let updatedContent; - if (content.mode === "w" || content.mode === "x") { - updatedContent = content.content; - } else if (content.mode === "a") { - updatedContent = - (componentToUpdate ? componentToUpdate.content + "\n" : "") + - content.content; - } - - if (componentToUpdate) { - dispatch( - updateProjectComponent({ - extension, - name, - code: updatedContent, - }), - ); - } else { - dispatch( - addProjectComponent({ name, extension, content: updatedContent }), - ); - } - break; - default: - throw new Error(`Unsupported origin: ${visual.origin}`); - } + }, + tooltip: { + ...visual.content.tooltip, + formatter: + visual.content.chart.type === "pie" + ? function () { + return this.key + ": " + this.y; + } + : null, + }, + }; + Highcharts.chart(output.current, chartContent); + break; + case "turtle": + output.current.innerHTML = elementFromProps(visual.content).outerHTML; + break; + case "matplotlib": + // convert visual.content from Uint8Array to jpg + const img = document.createElement("img"); + img.style = "max-width: 100%; max-height: 100%;"; + img.src = `data:image/jpg;base64,${window.btoa( + String.fromCharCode(...new Uint8Array(visual.content)), + )}`; + output.current.innerHTML = img.outerHTML; + break; + default: + throw new Error(`Unsupported origin: ${visual.origin}`); + } - visual.showing = true; - return visual; - }, - [dispatch, projectComponents], - ); + visual.showing = true; + return visual; + }, []); const showVisuals = useCallback( (visuals, output) => From 81d7faa3d2208a61cbfa19da251febc7a5241fab Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Tue, 7 Jan 2025 09:25:45 +0000 Subject: [PATCH 23/47] fixing bug where components were not immediately updated --- .../Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx index e6adcdd89..03365b5e7 100644 --- a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx +++ b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx @@ -110,7 +110,7 @@ const PyodideRunner = ({ active }) => { } }; } - }, [pyodideWorker]); + }, [pyodideWorker, projectCode]); useEffect(() => { if (codeRunTriggered && active && output.current) { From 5b0dc66ede9faabe41490b383efb835d59dbe8aa Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Tue, 7 Jan 2025 12:18:34 +0000 Subject: [PATCH 24/47] change content in the focused file on file write --- .../Editor/EditorPanel/EditorPanel.jsx | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/src/components/Editor/EditorPanel/EditorPanel.jsx b/src/components/Editor/EditorPanel/EditorPanel.jsx index 951cfb70b..b98642767 100644 --- a/src/components/Editor/EditorPanel/EditorPanel.jsx +++ b/src/components/Editor/EditorPanel/EditorPanel.jsx @@ -27,6 +27,7 @@ const MAX_CHARACTERS = 8500000; const EditorPanel = ({ extension = "html", fileName = "index" }) => { const editor = useRef(); + const editorViewRef = useRef(); const project = useSelector((state) => state.editor.project); const readOnly = useSelector((state) => state.editor.readOnly); const [cookies] = useCookies(["theme", "fontSize"]); @@ -74,11 +75,11 @@ const EditorPanel = ({ extension = "html", fileName = "index" }) => { window.matchMedia("(prefers-color-scheme:dark)").matches); const editorTheme = isDarkMode ? editorDarkTheme : editorLightTheme; - useEffect(() => { - const file = project.components.find( - (item) => item.extension === extension && item.name === fileName, - ); + const file = project.components.find( + (item) => item.extension === extension && item.name === fileName, + ); + useEffect(() => { if (!file) { return; } @@ -123,6 +124,8 @@ const EditorPanel = ({ extension = "html", fileName = "index" }) => { parent: editor.current, }); + editorViewRef.current = view; + // 'aria-hidden' to fix keyboard access accessibility error view.scrollDOM.setAttribute("aria-hidden", "true"); @@ -138,6 +141,21 @@ const EditorPanel = ({ extension = "html", fileName = "index" }) => { }; }, [cookies]); + useEffect(() => { + if ( + editorViewRef.current && + file.content !== editorViewRef.current.state.doc.toString() + ) { + editorViewRef.current.dispatch({ + changes: { + from: 0, + to: editorViewRef.current.state.doc.length, + insert: file.content, + }, + }); + } + }, [file, editorViewRef]); + return ( <>
From 6214124eba053259edb93e4b86491e1bd796a5ff Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Tue, 7 Jan 2025 14:51:26 +0000 Subject: [PATCH 25/47] unit testing --- .../PyodideRunner/PyodideRunner.test.js | 102 ++++++++++++++++++ .../PyodideRunner/PyodideWorker.test.js | 32 +++--- src/redux/EditorSlice.test.js | 31 ++++++ 3 files changed, 153 insertions(+), 12 deletions(-) diff --git a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.test.js b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.test.js index c8bd31f40..fd828fd05 100644 --- a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.test.js +++ b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.test.js @@ -27,6 +27,7 @@ const project = { components: [ { name: "a", extension: "py", content: "print('a')" }, { name: "main", extension: "py", content: "print('hello')" }, + { name: "existing_file", extension: "txt", content: "hello" }, ], image_list: [ { filename: "image1.jpg", url: "http://example.com/image1.jpg" }, @@ -251,6 +252,107 @@ describe("When output is received", () => { }); }); +describe("When file write event is received", () => { + let worker; + beforeEach(() => { + render( + + , + , + ); + updateRunner({ project }); + worker = PyodideWorker.getLastInstance(); + }); + + test("it overwrites existing files in 'w' mode", () => { + worker.postMessageFromWorker({ + method: "handleFileWrite", + filename: "existing_file.txt", + content: "new content", + mode: "w", + }); + expect(dispatchSpy).toHaveBeenCalledWith({ + type: "editor/updateProjectComponent", + payload: { + name: "existing_file", + extension: "txt", + code: "new content", + }, + }); + }); + + test("it creates new file if not already existing in 'w' mode", () => { + worker.postMessageFromWorker({ + method: "handleFileWrite", + filename: "new_file.txt", + content: "new content", + mode: "w", + }); + + expect(dispatchSpy).toHaveBeenCalledWith({ + type: "editor/addProjectComponent", + payload: { + name: "new_file", + extension: "txt", + content: "new content", + }, + }); + }); + + test("it appends to existing files in 'a' mode", () => { + worker.postMessageFromWorker({ + method: "handleFileWrite", + filename: "existing_file.txt", + content: "new content", + mode: "a", + }); + expect(dispatchSpy).toHaveBeenCalledWith({ + type: "editor/updateProjectComponent", + payload: { + name: "existing_file", + extension: "txt", + code: "hello\nnew content", + }, + }); + }); + + test("it creates new file if not already existing in 'a' mode", () => { + worker.postMessageFromWorker({ + method: "handleFileWrite", + filename: "new_file.txt", + content: "new content", + mode: "a", + }); + + expect(dispatchSpy).toHaveBeenCalledWith({ + type: "editor/addProjectComponent", + payload: { + name: "new_file", + extension: "txt", + content: "new content", + }, + }); + }); + + test("it creates new file if not already existing in 'x' mode", () => { + worker.postMessageFromWorker({ + method: "handleFileWrite", + filename: "new_file.txt", + content: "new content", + mode: "x", + }); + + expect(dispatchSpy).toHaveBeenCalledWith({ + type: "editor/addProjectComponent", + payload: { + name: "new_file", + extension: "txt", + content: "new content", + }, + }); + }); +}); + describe("When visual output is received", () => { beforeEach(() => { render( diff --git a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideWorker.test.js b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideWorker.test.js index f54b07734..a9d688d0e 100644 --- a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideWorker.test.js +++ b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideWorker.test.js @@ -89,29 +89,23 @@ describe("PyodideWorker", () => { }); test("it patches the input function", async () => { - await worker.onmessage({ - data: { - method: "runPython", - python: "print('hello')", - }, - }); expect(pyodide.runPythonAsync).toHaveBeenCalledWith( expect.stringMatching(/__builtins__.input = __patched_input__/), ); }); test("it patches urllib and requests modules", async () => { - await worker.onmessage({ - data: { - method: "runPython", - python: "print('hello')", - }, - }); expect(pyodide.runPythonAsync).toHaveBeenCalledWith( expect.stringMatching(/pyodide_http.patch_all()/), ); }); + test("it saves original open function", async () => { + expect(pyodide.runPythonAsync).toHaveBeenCalledWith( + expect.stringMatching(/_original_open = builtins.open/), + ); + }); + test("it tries to load package from file system", async () => { pyodide._api.pyodide_code.find_imports = () => new MockPythonArray("numpy"); await worker.onmessage({ @@ -178,6 +172,20 @@ describe("PyodideWorker", () => { }); }); + test("it patches the open function", async () => { + await worker.onmessage({ + data: { + method: "runPython", + python: "print('hello')", + }, + }); + await waitFor(() => + expect(pyodide.runPythonAsync).toHaveBeenCalledWith( + expect.stringMatching(/builtins.open = _custom_open/), + ), + ); + }); + test("it runs the python code", async () => { await worker.onmessage({ data: { diff --git a/src/redux/EditorSlice.test.js b/src/redux/EditorSlice.test.js index 186dac7a8..cb24e8b3e 100644 --- a/src/redux/EditorSlice.test.js +++ b/src/redux/EditorSlice.test.js @@ -11,6 +11,7 @@ import reducer, { setIsOutputOnly, setErrorDetails, setReadOnly, + addProjectComponent, } from "./EditorSlice"; const mockCreateRemix = jest.fn(); @@ -104,6 +105,36 @@ test("Action setReadOnly correctly sets readOnly", () => { expect(reducer(previousState, setReadOnly(true))).toEqual(expectedState); }); +test("Action addProjectComponent adds component to project with correct content", () => { + const previousState = { + project: { + components: [], + }, + }; + const expectedState = { + project: { + components: [ + { + name: "main", + extension: "py", + content: "print('hello world')", + }, + ], + }, + saving: "idle", + }; + expect( + reducer( + previousState, + addProjectComponent({ + name: "main", + extension: "py", + content: "print('hello world')", + }), + ), + ).toEqual(expectedState); +}); + test("Showing rename modal sets file state and showing status", () => { const previousState = { renameFileModalShowing: false, From 307cc1355d5156d118a82e4fc162d90cc8441e11 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Tue, 7 Jan 2025 16:23:22 +0000 Subject: [PATCH 26/47] fixing line numbers --- src/PyodideWorker.js | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/src/PyodideWorker.js b/src/PyodideWorker.js index d2abbb091..77509e3a7 100644 --- a/src/PyodideWorker.js +++ b/src/PyodideWorker.js @@ -92,7 +92,8 @@ const PyodideWorker = () => { await pyodide.loadPackagesFromImports(python); checkIfStopped(); - await pyodide.runPythonAsync(` + await pyodide.runPythonAsync( + ` import basthon import builtins import os @@ -134,7 +135,9 @@ const PyodideWorker = () => { # Override the built-in open function builtins.open = _custom_open - `); + `, + { filename: "__custom_open__.py" }, + ); await runPythonFn(); for (let name of imports) { @@ -470,9 +473,19 @@ const PyodideWorker = () => { const parsePythonError = (error) => { const type = error.type; const [trace, info] = error.message.split(`${type}:`).map((s) => s?.trim()); + console.log(trace); + console.log(info); const lines = trace.split("\n"); + // if the third from last line matches /File "__custom_open__\.py", line (\d+)/g then strip off the last three lines + if ( + lines.length > 3 && + /File "__custom_open__\.py", line (\d+)/g.test(lines[lines.length - 3]) + ) { + lines.splice(-3, 3); + } + const snippetLine = lines[lines.length - 2]; // print("hi")invalid const caretLine = lines[lines.length - 1]; // ^^^^^^^ @@ -481,7 +494,9 @@ const PyodideWorker = () => { ? [snippetLine.slice(4), caretLine.slice(4)].join("\n") : ""; - const matches = [...trace.matchAll(/File "(.*)", line (\d+)/g)]; + const matches = [ + ...trace.matchAll(/File "(?!__custom_open__\.py)(.*)", line (\d+)/g), + ]; const match = matches[matches.length - 1]; const path = match ? match[1] : ""; From c929af0d9ab9f14d1100e3a77cebbe0d852d8dbe Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Tue, 7 Jan 2025 16:24:34 +0000 Subject: [PATCH 27/47] adding a couple of file write cypress tests --- cypress/e2e/spec-wc-pyodide.cy.js | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/cypress/e2e/spec-wc-pyodide.cy.js b/cypress/e2e/spec-wc-pyodide.cy.js index 858c308a9..ad5aa77cc 100644 --- a/cypress/e2e/spec-wc-pyodide.cy.js +++ b/cypress/e2e/spec-wc-pyodide.cy.js @@ -91,6 +91,34 @@ describe("Running the code with pyodide", () => { .should("contain", "Hello Lois"); }); + it("runs a simple program to write to a file", () => { + runCode('with open("output.txt", "w") as f:\n\tf.write("Hello world")'); + cy.get("editor-wc") + .shadow() + .contains(".files-list-item", "output.txt") + .click(); + cy.get("editor-wc") + .shadow() + .find(".cm-editor") + .should("contain", "Hello world"); + }); + + it("errors when trying to write to an existing file in 'x' mode", () => { + runCode('with open("output.txt", "w") as f:\n\tf.write("Hello world")'); + cy.get("editor-wc") + .shadow() + .find(".files-list-item") + .should("contain", "output.txt"); + runCode('with open("output.txt", "x") as f:\n\tf.write("Something else")'); + cy.get("editor-wc") + .shadow() + .find(".error-message__content") + .should( + "contain", + "FileExistsError: File 'output.txt' already exists on line 1 of main.py", + ); + }); + it("runs a simple program with a built-in python module", () => { runCode("from math import floor, pi\nprint(floor(pi))"); cy.get("editor-wc") From 91f95549b40d1e03c8dc5eb337e283d74d4e6e0d Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Tue, 7 Jan 2025 16:39:56 +0000 Subject: [PATCH 28/47] fixing test --- .../Runners/PythonRunner/PyodideRunner/PyodideWorker.test.js | 1 + 1 file changed, 1 insertion(+) diff --git a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideWorker.test.js b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideWorker.test.js index a9d688d0e..702cc4223 100644 --- a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideWorker.test.js +++ b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideWorker.test.js @@ -182,6 +182,7 @@ describe("PyodideWorker", () => { await waitFor(() => expect(pyodide.runPythonAsync).toHaveBeenCalledWith( expect.stringMatching(/builtins.open = _custom_open/), + { filename: "__custom_open__.py" }, ), ); }); From 7e0dbc2e1455b99e64a5d745c08fdd410ee4da95 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Wed, 8 Jan 2025 15:25:56 +0000 Subject: [PATCH 29/47] partial fix for cascading file update weirdness --- src/components/Editor/EditorPanel/EditorPanel.jsx | 11 +++++++++-- src/components/Editor/Output/Output.test.js | 2 ++ .../PythonRunner/PyodideRunner/PyodideRunner.jsx | 5 +++++ .../PythonRunner/PyodideRunner/PyodideRunner.test.js | 2 ++ src/redux/EditorSlice.js | 7 +++++++ 5 files changed, 25 insertions(+), 2 deletions(-) diff --git a/src/components/Editor/EditorPanel/EditorPanel.jsx b/src/components/Editor/EditorPanel/EditorPanel.jsx index b98642767..7933963a7 100644 --- a/src/components/Editor/EditorPanel/EditorPanel.jsx +++ b/src/components/Editor/EditorPanel/EditorPanel.jsx @@ -2,7 +2,10 @@ import "../../../assets/stylesheets/EditorPanel.scss"; import React, { useRef, useEffect, useContext, useState } from "react"; import { useSelector, useDispatch } from "react-redux"; -import { updateProjectComponent } from "../../../redux/EditorSlice"; +import { + setCascadeUpdate, + updateProjectComponent, +} from "../../../redux/EditorSlice"; import { useCookies } from "react-cookie"; import { useTranslation } from "react-i18next"; import { basicSetup } from "codemirror"; @@ -30,6 +33,7 @@ const EditorPanel = ({ extension = "html", fileName = "index" }) => { const editorViewRef = useRef(); const project = useSelector((state) => state.editor.project); const readOnly = useSelector((state) => state.editor.readOnly); + const cascadeUpdate = useSelector((state) => state.editor.cascadeUpdate); const [cookies] = useCookies(["theme", "fontSize"]); const dispatch = useDispatch(); const { t } = useTranslation(); @@ -42,6 +46,7 @@ const EditorPanel = ({ extension = "html", fileName = "index" }) => { extension: extension, name: fileName, code: content, + cascadeUpdate: false, }), ); }; @@ -143,6 +148,7 @@ const EditorPanel = ({ extension = "html", fileName = "index" }) => { useEffect(() => { if ( + cascadeUpdate && editorViewRef.current && file.content !== editorViewRef.current.state.doc.toString() ) { @@ -153,8 +159,9 @@ const EditorPanel = ({ extension = "html", fileName = "index" }) => { insert: file.content, }, }); + dispatch(setCascadeUpdate(false)); } - }, [file, editorViewRef]); + }, [file, cascadeUpdate, editorViewRef]); return ( <> diff --git a/src/components/Editor/Output/Output.test.js b/src/components/Editor/Output/Output.test.js index 0c23a4c37..2b736ac1f 100644 --- a/src/components/Editor/Output/Output.test.js +++ b/src/components/Editor/Output/Output.test.js @@ -26,6 +26,8 @@ describe("Output component", () => { project: { components: [], }, + focussedFileIndices: [0], + openFiles: [["main.py"]], }, auth: { user, diff --git a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx index 386657bda..cbecf07ea 100644 --- a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx +++ b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx @@ -53,6 +53,10 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => { const projectImages = useSelector((s) => s.editor.project.image_list); const projectCode = useSelector((s) => s.editor.project.components); const projectIdentifier = useSelector((s) => s.editor.project.identifier); + const focussedFileIndex = useSelector( + (state) => state.editor.focussedFileIndices, + )[0]; + const openFiles = useSelector((state) => state.editor.openFiles)[0]; const user = useSelector((s) => s.auth.user); const userId = user?.profile?.user; const isSplitView = useSelector((s) => s.editor.isSplitView); @@ -221,6 +225,7 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => { extension, name, code: updatedContent, + cascadeUpdate: openFiles[focussedFileIndex] === filename, }), ); } else { diff --git a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.test.js b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.test.js index d8bbba40f..5dddb9092 100644 --- a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.test.js +++ b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.test.js @@ -278,6 +278,7 @@ describe("When file write event is received", () => { name: "existing_file", extension: "txt", code: "new content", + cascadeUpdate: false, }, }); }); @@ -313,6 +314,7 @@ describe("When file write event is received", () => { name: "existing_file", extension: "txt", code: "hello\nnew content", + cascadeUpdate: false, }, }); }); diff --git a/src/redux/EditorSlice.js b/src/redux/EditorSlice.js index 79ed03394..f380ee37f 100644 --- a/src/redux/EditorSlice.js +++ b/src/redux/EditorSlice.js @@ -95,6 +95,7 @@ export const loadProjectList = createAsyncThunk( const initialState = { project: {}, + cascadeUpdate: false, readOnly: false, saveTriggered: false, saving: "idle", @@ -265,6 +266,7 @@ export const EditorSlice = createSlice({ const extension = action.payload.extension; const fileName = action.payload.name; const code = action.payload.code; + const cascadeUpdate = action.payload.cascadeUpdate; const mapped = state.project.components.map((item) => { if (item.extension !== extension || item.name !== fileName) { @@ -274,6 +276,7 @@ export const EditorSlice = createSlice({ return { ...item, ...{ content: code } }; }); state.project.components = mapped; + state.cascadeUpdate = cascadeUpdate; }, updateProjectName: (state, action) => { state.project.name = action.payload; @@ -295,6 +298,9 @@ export const EditorSlice = createSlice({ } state.saving = "idle"; }, + setCascadeUpdate: (state, action) => { + state.cascadeUpdate = action.payload; + }, setError: (state, action) => { state.error = action.payload; }, @@ -454,6 +460,7 @@ export const { setEmbedded, setIsOutputOnly, setBrowserPreview, + setCascadeUpdate, setError, setIsSplitView, setNameError, From 142f9178c1293864d608e985558be7451215e08a Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Wed, 8 Jan 2025 16:46:03 +0000 Subject: [PATCH 30/47] wip attempt to support binary write --- src/PyodideWorker.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/PyodideWorker.js b/src/PyodideWorker.js index 77509e3a7..f3c32d9f5 100644 --- a/src/PyodideWorker.js +++ b/src/PyodideWorker.js @@ -113,10 +113,16 @@ const PyodideWorker = () => { self.content = "" def write(self, content): - self.content += content + if "b" in mode: + self.content = content + write_mode = "wb" + else: + self.content += content + write_mode = "w" if len(self.content) > MAX_FILE_SIZE: raise OSError(f"File '{self.filename}' exceeds maximum file size of {MAX_FILE_SIZE} bytes") - with _original_open(self.filename, "w") as f: + + with _original_open(self.filename, write_mode) as f: f.write(self.content) basthon.kernel.write_file({ "filename": self.filename, "content": self.content, "mode": mode }) From 8a9a9aee386fa43add572b6154688608a22b2512 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Wed, 8 Jan 2025 17:17:18 +0000 Subject: [PATCH 31/47] initial ability to create images when logged in --- src/PyodideWorker.js | 2 ++ .../PyodideRunner/PyodideRunner.jsx | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/src/PyodideWorker.js b/src/PyodideWorker.js index f3c32d9f5..ff1bbd7b2 100644 --- a/src/PyodideWorker.js +++ b/src/PyodideWorker.js @@ -116,6 +116,8 @@ const PyodideWorker = () => { if "b" in mode: self.content = content write_mode = "wb" + print("writing binary file with content", content) + #return else: self.content += content write_mode = "w" diff --git a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx index cbecf07ea..aa8c9e544 100644 --- a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx +++ b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx @@ -10,6 +10,7 @@ import { setLoadedRunner, updateProjectComponent, addProjectComponent, + updateImages, } from "../../../../../redux/EditorSlice"; import { Tab, Tabs, TabList, TabPanel } from "react-tabs"; import { useMediaQuery } from "react-responsive"; @@ -206,11 +207,25 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => { disableInput(); }; - const handleFileWrite = (filename, content, mode) => { + const handleFileWrite = async (filename, content, mode) => { const [name, extension] = filename.split("."); const componentToUpdate = projectCode.find( (item) => item.extension === extension && item.name === name, ); + + if (mode === "wb") { + const { uploadImages } = ApiCallHandler({ + reactAppApiEndpoint, + }); + const response = await uploadImages( + projectIdentifier, + user.access_token, + // file object with the correct filename and binary content + [new File([content], filename, { type: "application/octet-stream" })], + ); + dispatch(updateImages(response.data.image_list)); + return; + } let updatedContent; if (mode === "w" || mode === "x") { updatedContent = content; From a69d4b45c0d9094b93d1c9611508a7fd640356f6 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Wed, 22 Jan 2025 17:36:14 +0000 Subject: [PATCH 32/47] add ability to overwrite existing images --- .../PyodideRunner/PyodideRunner.jsx | 109 +++++++++++------- src/utils/apiCallHandler.js | 12 ++ 2 files changed, 79 insertions(+), 42 deletions(-) diff --git a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx index aa8c9e544..ee47b6ba4 100644 --- a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx +++ b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx @@ -1,6 +1,12 @@ /* eslint-disable react-hooks/exhaustive-deps */ import "../../../../../assets/stylesheets/PythonRunner.scss"; -import React, { useContext, useEffect, useRef, useState } from "react"; +import React, { + useCallback, + useContext, + useEffect, + useRef, + useState, +} from "react"; import { useDispatch, useSelector } from "react-redux"; import { useTranslation } from "react-i18next"; import classNames from "classnames"; @@ -118,7 +124,7 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => { } }; } - }, [pyodideWorker, projectCode]); + }, [pyodideWorker, projectCode, projectImages]); useEffect(() => { if (codeRunTriggered && active && output.current) { @@ -207,48 +213,67 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => { disableInput(); }; - const handleFileWrite = async (filename, content, mode) => { - const [name, extension] = filename.split("."); - const componentToUpdate = projectCode.find( - (item) => item.extension === extension && item.name === name, - ); - - if (mode === "wb") { - const { uploadImages } = ApiCallHandler({ - reactAppApiEndpoint, - }); - const response = await uploadImages( - projectIdentifier, - user.access_token, - // file object with the correct filename and binary content - [new File([content], filename, { type: "application/octet-stream" })], + const handleFileWrite = useCallback( + async (filename, content, mode) => { + const [name, extension] = filename.split("."); + const componentToUpdate = projectCode.find( + (item) => item.extension === extension && item.name === name, ); - dispatch(updateImages(response.data.image_list)); - return; - } - let updatedContent; - if (mode === "w" || mode === "x") { - updatedContent = content; - } else if (mode === "a") { - updatedContent = - (componentToUpdate ? componentToUpdate.content + "\n" : "") + content; - } - if (componentToUpdate) { - dispatch( - updateProjectComponent({ - extension, - name, - code: updatedContent, - cascadeUpdate: openFiles[focussedFileIndex] === filename, - }), - ); - } else { - dispatch( - addProjectComponent({ name, extension, content: updatedContent }), - ); - } - }; + if (mode === "wb") { + const { uploadImages, updateImage } = ApiCallHandler({ + reactAppApiEndpoint, + }); + + const projectImageNames = projectImages.map((image) => image.filename); + console.log("Project Image Names: ", projectImageNames); + if (projectImageNames.includes(filename)) { + console.log("Image already exists"); + const response = await updateImage( + projectIdentifier, + user.access_token, + // file object with the correct filename and binary content + new File([content], filename, { type: "application/octet-stream" }), + ); + if (response.status === 200) { + dispatch(updateImages(response.data.image_list)); + } + return; + } + const response = await uploadImages( + projectIdentifier, + user.access_token, + // file object with the correct filename and binary content + [new File([content], filename, { type: "application/octet-stream" })], + ); + dispatch(updateImages(response.data.image_list)); + return; + } + let updatedContent; + if (mode === "w" || mode === "x") { + updatedContent = content; + } else if (mode === "a") { + updatedContent = + (componentToUpdate ? componentToUpdate.content + "\n" : "") + content; + } + + if (componentToUpdate) { + dispatch( + updateProjectComponent({ + extension, + name, + code: updatedContent, + cascadeUpdate: openFiles[focussedFileIndex] === filename, + }), + ); + } else { + dispatch( + addProjectComponent({ name, extension, content: updatedContent }), + ); + } + }, + [projectImages], + ); const handleVisual = (origin, content) => { if (showVisualOutputPanel) { diff --git a/src/utils/apiCallHandler.js b/src/utils/apiCallHandler.js index ead3394db..442c49778 100644 --- a/src/utils/apiCallHandler.js +++ b/src/utils/apiCallHandler.js @@ -106,6 +106,17 @@ const ApiCallHandler = ({ reactAppApiEndpoint }) => { ); }; + const updateImage = async (projectIdentifier, accessToken, image) => { + var formData = new FormData(); + formData.append("image", image, image.name); + + return await put( + `${host}/api/projects/${projectIdentifier}/images`, + formData, + { ...headers(accessToken), "Content-Type": "multipart/form-data" }, + ); + }; + const createError = async ( projectIdentifier, userId, @@ -137,6 +148,7 @@ const ApiCallHandler = ({ reactAppApiEndpoint }) => { loadAssets, readProjectList, uploadImages, + updateImage, createError, }; }; From fa34d9906581182b0610f50ec1c6068f92493b14 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Wed, 5 Mar 2025 10:33:07 +0000 Subject: [PATCH 33/47] fixinglinting --- .../Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx index 1e3ebb523..4d5d1809b 100644 --- a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx +++ b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx @@ -270,7 +270,7 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => { extension, name, content: updatedContent, - cascadeUpdate + cascadeUpdate, }), ); } else { From 8657571328f22b5c61c5a72f8b544b4d22dca324 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Wed, 5 Mar 2025 17:39:57 +0000 Subject: [PATCH 34/47] trying to get imageio imwrite working by queueing the write requests - still very much WIP --- src/PyodideWorker.js | 3 +- .../PyodideRunner/PyodideRunner.jsx | 150 ++++++++++++------ 2 files changed, 99 insertions(+), 54 deletions(-) diff --git a/src/PyodideWorker.js b/src/PyodideWorker.js index ff1bbd7b2..36241fc79 100644 --- a/src/PyodideWorker.js +++ b/src/PyodideWorker.js @@ -116,8 +116,6 @@ const PyodideWorker = () => { if "b" in mode: self.content = content write_mode = "wb" - print("writing binary file with content", content) - #return else: self.content += content write_mode = "w" @@ -126,6 +124,7 @@ const PyodideWorker = () => { with _original_open(self.filename, write_mode) as f: f.write(self.content) + print("the mode is", mode) basthon.kernel.write_file({ "filename": self.filename, "content": self.content, "mode": mode }) def close(self): diff --git a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx index 4d5d1809b..842a41974 100644 --- a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx +++ b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx @@ -27,6 +27,7 @@ import VisualOutputPane from "./VisualOutputPane"; import OutputViewToggle from "../OutputViewToggle"; import { SettingsContext } from "../../../../../utils/settings"; import RunnerControls from "../../../../RunButton/RunnerControls"; +import store from "../../../../../redux/stores/WebComponentStore"; const getWorkerURL = (url) => { const content = ` @@ -58,6 +59,7 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => { const stdinClosed = useRef(); const loadedRunner = useSelector((state) => state.editor.loadedRunner); const projectImages = useSelector((s) => s.editor.project.image_list); + const projectImageNames = projectImages.map((image) => image.filename); const projectCode = useSelector((s) => s.editor.project.components); const projectIdentifier = useSelector((s) => s.editor.project.identifier); const focussedFileIndex = useSelector( @@ -118,6 +120,7 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => { data.content, data.mode, cascadeUpdate, + // projectImages, ); break; case "handleVisual": @@ -131,7 +134,7 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => { } }; } - }, [pyodideWorker, projectCode, openFiles, focussedFileIndex]); + }, [pyodideWorker, projectCode, projectImages, openFiles, focussedFileIndex]); useEffect(() => { if (codeRunTriggered && active && output.current) { @@ -220,67 +223,110 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => { disableInput(); }; + const fileWriteQueue = useRef([]); // Queue to store file write requests + const isExecuting = useRef(false); + const handleFileWrite = useCallback( async (filename, content, mode, cascadeUpdate) => { - const [name, extension] = filename.split("."); - const componentToUpdate = projectCode.find( - (item) => item.extension === extension && item.name === name, - ); + // Add the file write request to the queue + fileWriteQueue.current.push({ + filename, + content, + mode, + cascadeUpdate, + projectImages, + }); - if (mode === "wb") { - const { uploadImages, updateImage } = ApiCallHandler({ - reactAppApiEndpoint, - }); - - const projectImageNames = projectImages.map((image) => image.filename); - console.log("Project Image Names: ", projectImageNames); - if (projectImageNames.includes(filename)) { - console.log("Image already exists"); - const response = await updateImage( - projectIdentifier, - user.access_token, - // file object with the correct filename and binary content - new File([content], filename, { type: "application/octet-stream" }), - ); - if (response.status === 200) { - dispatch(updateImages(response.data.image_list)); - } - return; - } - const response = await uploadImages( + // Process the queue if not already executing + if (!isExecuting.current) { + processFileWriteQueue(); + } + }, + [projectImages, projectImageNames], + ); + + const processFileWriteQueue = useCallback(async () => { + if (fileWriteQueue.current.length === 0) { + isExecuting.current = false; + return; + } + + isExecuting.current = true; + const { filename, content, mode, cascadeUpdate, projectImages } = + fileWriteQueue.current.shift(); + + const [name, extension] = filename.split("."); + const componentToUpdate = projectCode.find( + (item) => item.extension === extension && item.name === name, + ); + + if (mode === "wb" || mode === "w+b") { + const { uploadImages, updateImage } = ApiCallHandler({ + reactAppApiEndpoint, + }); + + console.log("the state of the store is: "); + console.log(store.getState()); + const projectImageNames = ( + store.getState().editor.project.image_list || [] + ).map((image) => image.filename); + console.log("Project Image Names: ", projectImageNames); + console.log(filename); + console.log(filename.split("/").pop()); + if (projectImageNames.includes(filename.split("/").pop())) { + console.log("Image already exists"); + const response = await updateImage( projectIdentifier, user.access_token, // file object with the correct filename and binary content - [new File([content], filename, { type: "application/octet-stream" })], + new File([content], filename, { type: "application/octet-stream" }), ); - dispatch(updateImages(response.data.image_list)); + if (response.status === 200) { + dispatch(updateImages(response.data.image_list)); + } + processFileWriteQueue(projectImageNames); // Process the next item in the queue return; } - let updatedContent; - if (mode === "w" || mode === "x") { - updatedContent = content; - } else if (mode === "a") { - updatedContent = - (componentToUpdate ? componentToUpdate.content + "\n" : "") + content; - } + const response = await uploadImages( + projectIdentifier, + user.access_token, + // file object with the correct filename and binary content + [new File([content], filename, { type: "application/octet-stream" })], + ); + dispatch(updateImages(response.data.image_list)); + processFileWriteQueue(projectImageNames); // Process the next item in the queue + return; + } + let updatedContent; + if (mode === "w" || mode === "x") { + updatedContent = content; + } else if (mode === "a") { + updatedContent = + (componentToUpdate ? componentToUpdate.content + "\n" : "") + content; + } - if (componentToUpdate) { - dispatch( - updateProjectComponent({ - extension, - name, - content: updatedContent, - cascadeUpdate, - }), - ); - } else { - dispatch( - addProjectComponent({ name, extension, content: updatedContent }), - ); - } - }, - [projectImages], - ); + if (componentToUpdate) { + dispatch( + updateProjectComponent({ + extension, + name, + content: updatedContent, + cascadeUpdate, + }), + ); + } else { + dispatch( + addProjectComponent({ name, extension, content: updatedContent }), + ); + } + + processFileWriteQueue(); // Process the next item in the queue + }, [projectImages, projectImageNames]); + + useEffect(() => { + const projectImageNames = projectImages.map((image) => image.filename); + console.log("Updated Project Image Names: ", projectImageNames); + }, [projectImages]); const handleVisual = (origin, content) => { if (showVisualOutputPanel) { From 22d412dc8686255e60d6b5f9e816fe78833ad083 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Wed, 19 Mar 2025 15:19:25 +0000 Subject: [PATCH 35/47] fixing append mode weirdness and finally getting images working!!!!! --- src/PyodideWorker.js | 22 +++++++++++++------ .../PyodideRunner/PyodideRunner.jsx | 2 +- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/PyodideWorker.js b/src/PyodideWorker.js index 36241fc79..8a127f276 100644 --- a/src/PyodideWorker.js +++ b/src/PyodideWorker.js @@ -97,6 +97,7 @@ const PyodideWorker = () => { import basthon import builtins import os + import mimetypes MAX_FILES = 100 MAX_FILE_SIZE = 8500000 @@ -110,21 +111,28 @@ const PyodideWorker = () => { class CustomFile: def __init__(self, filename): self.filename = filename - self.content = "" + type = mimetypes.guess_type(filename)[0] + if type and "text" in type: + self.content = "" + else: + self.content = b'' def write(self, content): - if "b" in mode: - self.content = content - write_mode = "wb" + if "a" in mode and os.path.exists(self.filename): + with _original_open(self.filename, "r") as f: + existing_content = f.read() + + if len(existing_content) > 0: + self.content += "\\n" + content + else: + self.content += content else: self.content += content - write_mode = "w" if len(self.content) > MAX_FILE_SIZE: raise OSError(f"File '{self.filename}' exceeds maximum file size of {MAX_FILE_SIZE} bytes") - with _original_open(self.filename, write_mode) as f: + with _original_open(self.filename, mode) as f: f.write(self.content) - print("the mode is", mode) basthon.kernel.write_file({ "filename": self.filename, "content": self.content, "mode": mode }) def close(self): diff --git a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx index 842a41974..5c4842ea4 100644 --- a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx +++ b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx @@ -302,7 +302,7 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => { updatedContent = content; } else if (mode === "a") { updatedContent = - (componentToUpdate ? componentToUpdate.content + "\n" : "") + content; + (componentToUpdate ? componentToUpdate.content : "") + content; } if (componentToUpdate) { From 638cb613681c9186711a86881a0400999889d3cf Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Wed, 19 Mar 2025 15:22:31 +0000 Subject: [PATCH 36/47] fixing a mode --- src/PyodideWorker.js | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/PyodideWorker.js b/src/PyodideWorker.js index 8a127f276..379bf8857 100644 --- a/src/PyodideWorker.js +++ b/src/PyodideWorker.js @@ -118,16 +118,7 @@ const PyodideWorker = () => { self.content = b'' def write(self, content): - if "a" in mode and os.path.exists(self.filename): - with _original_open(self.filename, "r") as f: - existing_content = f.read() - - if len(existing_content) > 0: - self.content += "\\n" + content - else: - self.content += content - else: - self.content += content + self.content += content if len(self.content) > MAX_FILE_SIZE: raise OSError(f"File '{self.filename}' exceeds maximum file size of {MAX_FILE_SIZE} bytes") From f74e8536d9623cefc5e64270d40dfa8db75a2388 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Wed, 19 Mar 2025 17:36:41 +0000 Subject: [PATCH 37/47] override imageio imread to allow builtin images to be loaded --- src/PyodideWorker.js | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/src/PyodideWorker.js b/src/PyodideWorker.js index 379bf8857..28ea4f203 100644 --- a/src/PyodideWorker.js +++ b/src/PyodideWorker.js @@ -373,6 +373,35 @@ const PyodideWorker = () => { `); }, }, + imageio: { + before: async () => { + await pyodide.loadPackage("imageio"); + await pyodide.loadPackage("requests"); + pyodide.runPython(` + import imageio.v3 as iio + import io + import requests + + # Store the original imread function to avoid recursion + #_original_imread = iio.imread + + def custom_imread(uri, *args, **kwargs): + split_uri = uri.split(":") + if split_uri[0] == "imageio": + + print("Redirecting request to an alternative URL") + new_url = f"https://raw.githubusercontent.com/imageio/imageio-binaries/master/images/{split_uri[1]}" + response = requests.get(new_url, stream=True) + response.raise_for_status() + return _original_imread(io.BytesIO(response.content), *args, **kwargs) # Use the original imread + return _original_imread(uri, *args, **kwargs) # Call the original function for all other cases + + # Override iio.imread + iio.imread = custom_imread + `); + }, + after: () => {}, + }, }; const fakeBasthonPackage = { @@ -433,6 +462,12 @@ const PyodideWorker = () => { _original_open = builtins.open `); + await pyodide.loadPackage("imageio"); + await pyodide.runPythonAsync(` + import imageio.v3 as iio + _original_imread = iio.imread + `); + await pyodide.loadPackage("pyodide-http"); await pyodide.runPythonAsync(` import pyodide_http From 92fd85861191fc8a8923c78ab758cb0203c2feab Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Thu, 20 Mar 2025 09:21:49 +0000 Subject: [PATCH 38/47] tidying --- .../Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx index 5c4842ea4..6f2df938a 100644 --- a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx +++ b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx @@ -59,7 +59,7 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => { const stdinClosed = useRef(); const loadedRunner = useSelector((state) => state.editor.loadedRunner); const projectImages = useSelector((s) => s.editor.project.image_list); - const projectImageNames = projectImages.map((image) => image.filename); + const projectImageNames = projectImages?.map((image) => image.filename); const projectCode = useSelector((s) => s.editor.project.components); const projectIdentifier = useSelector((s) => s.editor.project.identifier); const focussedFileIndex = useSelector( @@ -252,7 +252,7 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => { } isExecuting.current = true; - const { filename, content, mode, cascadeUpdate, projectImages } = + const { filename, content, mode, cascadeUpdate } = fileWriteQueue.current.shift(); const [name, extension] = filename.split("."); @@ -323,11 +323,6 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => { processFileWriteQueue(); // Process the next item in the queue }, [projectImages, projectImageNames]); - useEffect(() => { - const projectImageNames = projectImages.map((image) => image.filename); - console.log("Updated Project Image Names: ", projectImageNames); - }, [projectImages]); - const handleVisual = (origin, content) => { if (showVisualOutputPanel) { setHasVisual(true); From 176a98fe1139264143f9282415ba2424d85ddcc7 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Thu, 3 Apr 2025 10:37:54 +0100 Subject: [PATCH 39/47] wip: imageio imread failure investigations --- cypress/e2e/spec-wc-pyodide.cy.js | 2 +- src/PyodideWorker.js | 6 ++++++ .../Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx | 1 + 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/cypress/e2e/spec-wc-pyodide.cy.js b/cypress/e2e/spec-wc-pyodide.cy.js index 73bcbd314..6ca8b2499 100644 --- a/cypress/e2e/spec-wc-pyodide.cy.js +++ b/cypress/e2e/spec-wc-pyodide.cy.js @@ -140,7 +140,7 @@ describe("Running the code with pyodide", () => { cy.get("editor-wc") .shadow() .find(".cm-editor") - .should("contain", "Hello again world"); + .should("contain", "Hello worldHello again world"); }); it("runs a simple program with a built-in python module", () => { diff --git a/src/PyodideWorker.js b/src/PyodideWorker.js index 28ea4f203..ba0292803 100644 --- a/src/PyodideWorker.js +++ b/src/PyodideWorker.js @@ -376,6 +376,12 @@ const PyodideWorker = () => { imageio: { before: async () => { await pyodide.loadPackage("imageio"); + // if (!pyodide.micropip) { + // await pyodide.loadPackage("micropip"); + // pyodide.micropip = pyodide.pyimport("micropip"); + // } + // await pyodide.micropip.install("opencv-python"); + // // await pyodide.micropip.install("ffmpeg"); await pyodide.loadPackage("requests"); pyodide.runPython(` import imageio.v3 as iio diff --git a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx index 6f2df938a..66f7f19a0 100644 --- a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx +++ b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx @@ -229,6 +229,7 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => { const handleFileWrite = useCallback( async (filename, content, mode, cascadeUpdate) => { // Add the file write request to the queue + console.log(`Writing ${content} to ${filename}`); fileWriteQueue.current.push({ filename, content, From 2e6493640d200452ff28d62ddde97fdaa777cc11 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Wed, 16 Apr 2025 15:20:01 +0100 Subject: [PATCH 40/47] fixing writing images to the pyodide filesystem --- src/PyodideWorker.js | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/PyodideWorker.js b/src/PyodideWorker.js index ba0292803..913912c71 100644 --- a/src/PyodideWorker.js +++ b/src/PyodideWorker.js @@ -40,7 +40,12 @@ const PyodideWorker = () => { switch (data.method) { case "writeFile": - pyodide.FS.writeFile(data.filename, encoder.encode(data.content)); + if (data.content instanceof ArrayBuffer) { + const dataArray = new Uint8Array(data.content); + pyodide.FS.writeFile(data.filename, dataArray); + } else { + pyodide.FS.writeFile(data.filename, encoder.encode(data.content)); + } break; case "runPython": runPython(data.python); @@ -376,12 +381,6 @@ const PyodideWorker = () => { imageio: { before: async () => { await pyodide.loadPackage("imageio"); - // if (!pyodide.micropip) { - // await pyodide.loadPackage("micropip"); - // pyodide.micropip = pyodide.pyimport("micropip"); - // } - // await pyodide.micropip.install("opencv-python"); - // // await pyodide.micropip.install("ffmpeg"); await pyodide.loadPackage("requests"); pyodide.runPython(` import imageio.v3 as iio From eb82889c2f06f7b337c6cd678d86ce705d91d09f Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Wed, 16 Apr 2025 16:32:02 +0100 Subject: [PATCH 41/47] tidying and changelog --- .../Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx | 1 - 1 file changed, 1 deletion(-) diff --git a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx index 66f7f19a0..a96bf1844 100644 --- a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx +++ b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx @@ -120,7 +120,6 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => { data.content, data.mode, cascadeUpdate, - // projectImages, ); break; case "handleVisual": From 0c23ceac951545b16ff84efd342d6199d4b5dabe Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Wed, 16 Apr 2025 16:33:02 +0100 Subject: [PATCH 42/47] changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e42619de..b0bbe96af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,9 +6,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## Unreleased +### Added + +- Image read and write support (#1171) +- `imageio` support (#1171) + ### Fixed - Bugs in append mode for writing to files in python (#1200) +- Writing binary files to the `pyodide` filesystem (#1171) ## [0.29.1] - 2025-02-21 From 63b952a54ec675a12fa55139fc5b0ead76786b61 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Wed, 16 Apr 2025 16:51:08 +0100 Subject: [PATCH 43/47] more tidying --- src/PyodideWorker.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/PyodideWorker.js b/src/PyodideWorker.js index 913912c71..ec10692ec 100644 --- a/src/PyodideWorker.js +++ b/src/PyodideWorker.js @@ -393,8 +393,6 @@ const PyodideWorker = () => { def custom_imread(uri, *args, **kwargs): split_uri = uri.split(":") if split_uri[0] == "imageio": - - print("Redirecting request to an alternative URL") new_url = f"https://raw.githubusercontent.com/imageio/imageio-binaries/master/images/{split_uri[1]}" response = requests.get(new_url, stream=True) response.raise_for_status() From 24f5ce189e70a8c4863de51f5552acc27fa70599 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Thu, 24 Apr 2025 15:10:59 +0100 Subject: [PATCH 44/47] wip image write support when logged out --- .../PyodideRunner/PyodideRunner.jsx | 45 +++++++++++++------ .../ProjectImages/ProjectImages.jsx | 11 ++++- 2 files changed, 41 insertions(+), 15 deletions(-) diff --git a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx index a96bf1844..b9e48d953 100644 --- a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx +++ b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx @@ -275,25 +275,42 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => { console.log(filename.split("/").pop()); if (projectImageNames.includes(filename.split("/").pop())) { console.log("Image already exists"); - const response = await updateImage( + if (user) { + const response = await updateImage( + projectIdentifier, + user.access_token, + // file object with the correct filename and binary content + new File([content], filename, { type: "application/octet-stream" }), + ); + if (response.status === 200) { + dispatch(updateImages(response.data.image_list)); + } + } else { + const updatedImage = { + filename: filename.split("/").pop(), + content, + }; + const updatedImages = projectImages.map((image) => + image.filename === updatedImage.filename ? updatedImage : image, + ); + dispatch(updateImages(updatedImages)); + } + processFileWriteQueue(projectImageNames); // Process the next item in the queue + return; + } + if (user) { + const response = await uploadImages( projectIdentifier, user.access_token, // file object with the correct filename and binary content - new File([content], filename, { type: "application/octet-stream" }), + [new File([content], filename, { type: "application/octet-stream" })], ); - if (response.status === 200) { - dispatch(updateImages(response.data.image_list)); - } - processFileWriteQueue(projectImageNames); // Process the next item in the queue - return; + dispatch(updateImages(response.data.image_list)); + } else { + const newImage = { filename: filename.split("/").pop(), content }; + const updatedImages = [...projectImages, newImage]; + dispatch(updateImages(updatedImages)); } - const response = await uploadImages( - projectIdentifier, - user.access_token, - // file object with the correct filename and binary content - [new File([content], filename, { type: "application/octet-stream" })], - ); - dispatch(updateImages(response.data.image_list)); processFileWriteQueue(projectImageNames); // Process the next item in the queue return; } diff --git a/src/components/Menus/Sidebar/ImagePanel/ProjectImages/ProjectImages.jsx b/src/components/Menus/Sidebar/ImagePanel/ProjectImages/ProjectImages.jsx index 158762df5..f09803623 100644 --- a/src/components/Menus/Sidebar/ImagePanel/ProjectImages/ProjectImages.jsx +++ b/src/components/Menus/Sidebar/ImagePanel/ProjectImages/ProjectImages.jsx @@ -14,7 +14,16 @@ const ProjectImages = () => { data + String.fromCharCode(byte), + "", + ), + )}` + : image.url + } alt={image.filename} />
From cec4469a3c75f13ad2ca9eab037dd7308fc1fc59 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Wed, 11 Jun 2025 17:23:49 +0100 Subject: [PATCH 45/47] initial fix for serializability issues in redux and writing when user not logged in --- CHANGELOG.md | 13 ++++++-- .../PyodideRunner/PyodideRunner.jsx | 32 +++++++++++-------- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 56a917ca4..ba05e3cef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,11 +6,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ## Unreleased -## [0.30.1] - 2025-06-09 - ### Added + - Image read and write support (#1171) - `imageio` support (#1171) + +### Fixed + +- Writing binary files to the `pyodide` filesystem (#1171) + +## [0.30.1] - 2025-06-09 + +### Added + - Fake translation for stress testing (#1206) ### Changed @@ -20,7 +28,6 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Fixed - Styling issue on sidebar on mobile (#1194) -- Writing binary files to the `pyodide` filesystem (#1171) ## [0.30.0] - 2025-04-15 diff --git a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx index b9e48d953..c09324350 100644 --- a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx +++ b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx @@ -228,7 +228,6 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => { const handleFileWrite = useCallback( async (filename, content, mode, cascadeUpdate) => { // Add the file write request to the queue - console.log(`Writing ${content} to ${filename}`); fileWriteQueue.current.push({ filename, content, @@ -265,16 +264,11 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => { reactAppApiEndpoint, }); - console.log("the state of the store is: "); - console.log(store.getState()); + const projectImages = store.getState().editor.project.image_list || []; const projectImageNames = ( store.getState().editor.project.image_list || [] ).map((image) => image.filename); - console.log("Project Image Names: ", projectImageNames); - console.log(filename); - console.log(filename.split("/").pop()); if (projectImageNames.includes(filename.split("/").pop())) { - console.log("Image already exists"); if (user) { const response = await updateImage( projectIdentifier, @@ -288,7 +282,7 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => { } else { const updatedImage = { filename: filename.split("/").pop(), - content, + content: Array.from(content), // Convert ArrayBuffer to Array for storage in Redux to ensure serializability }; const updatedImages = projectImages.map((image) => image.filename === updatedImage.filename ? updatedImage : image, @@ -307,7 +301,10 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => { ); dispatch(updateImages(response.data.image_list)); } else { - const newImage = { filename: filename.split("/").pop(), content }; + const newImage = { + filename: filename.split("/").pop(), + content: Array.from(content), + }; // Convert ArrayBuffer to Array for storage in Redux to ensure serializability const updatedImages = [...projectImages, newImage]; dispatch(updateImages(updatedImages)); } @@ -358,11 +355,18 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => { stdinClosed.current = false; await Promise.allSettled( - projectImages.map(({ filename, url }) => - fetch(url) - .then((response) => response.arrayBuffer()) - .then((buffer) => writeFile(filename, buffer)), - ), + projectImages.map(({ filename, url, content }) => { + if (content && content.length) { + // If content is present, send it directly (convert to Uint8Array if needed) + const buffer = new Uint8Array(content).buffer; + return writeFile(filename, buffer); + } else { + // Otherwise, fetch from the URL + return fetch(url) + .then((response) => response.arrayBuffer()) + .then((buffer) => writeFile(filename, buffer)); + } + }), ); for (const { name, extension, content } of projectCode) { From e6469ec6a0fc28c9c4e33a5b082d25ee73de2156 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Fri, 4 Jul 2025 15:44:37 +0100 Subject: [PATCH 46/47] moving over to base64 instead of uint8array --- .../PyodideRunner/PyodideRunner.jsx | 28 +++++++++++-------- .../ProjectImages/ProjectImages.jsx | 7 +---- src/hooks/useProjectPersistence.js | 11 +++++++- src/utils/base64Helpers.js | 22 +++++++++++++++ 4 files changed, 50 insertions(+), 18 deletions(-) create mode 100644 src/utils/base64Helpers.js diff --git a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx index c09324350..c4ed289a2 100644 --- a/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx +++ b/src/components/Editor/Runners/PythonRunner/PyodideRunner/PyodideRunner.jsx @@ -28,6 +28,11 @@ import OutputViewToggle from "../OutputViewToggle"; import { SettingsContext } from "../../../../../utils/settings"; import RunnerControls from "../../../../RunButton/RunnerControls"; import store from "../../../../../redux/stores/WebComponentStore"; +import { + base64ToUint8Array, + uint8ArrayToBase64, +} from "../../../../../utils/base64Helpers"; +import { isOwner } from "../../../../../utils/projectHelpers"; const getWorkerURL = (url) => { const content = ` @@ -264,12 +269,13 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => { reactAppApiEndpoint, }); - const projectImages = store.getState().editor.project.image_list || []; - const projectImageNames = ( - store.getState().editor.project.image_list || [] - ).map((image) => image.filename); + const project = store.getState().editor.project; + const projectImages = project.image_list || []; + const projectImageNames = (project.image_list || []).map( + (image) => image.filename, + ); if (projectImageNames.includes(filename.split("/").pop())) { - if (user) { + if (user && isOwner(user, project)) { const response = await updateImage( projectIdentifier, user.access_token, @@ -282,7 +288,7 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => { } else { const updatedImage = { filename: filename.split("/").pop(), - content: Array.from(content), // Convert ArrayBuffer to Array for storage in Redux to ensure serializability + content: await uint8ArrayToBase64(content), // Convert Uint8Array to binary string for storage in Redux to ensure serializability }; const updatedImages = projectImages.map((image) => image.filename === updatedImage.filename ? updatedImage : image, @@ -292,7 +298,7 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => { processFileWriteQueue(projectImageNames); // Process the next item in the queue return; } - if (user) { + if (user && isOwner(user, project)) { const response = await uploadImages( projectIdentifier, user.access_token, @@ -303,8 +309,8 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => { } else { const newImage = { filename: filename.split("/").pop(), - content: Array.from(content), - }; // Convert ArrayBuffer to Array for storage in Redux to ensure serializability + content: uint8ArrayToBase64(content), // Convert Uint8Array to base64 string for storage in Redux to ensure serializability + }; const updatedImages = [...projectImages, newImage]; dispatch(updateImages(updatedImages)); } @@ -357,8 +363,8 @@ const PyodideRunner = ({ active, outputPanels = ["text", "visual"] }) => { await Promise.allSettled( projectImages.map(({ filename, url, content }) => { if (content && content.length) { - // If content is present, send it directly (convert to Uint8Array if needed) - const buffer = new Uint8Array(content).buffer; + // If content is present, send it directly (convert from base64 string to Uint8Array if needed) + const buffer = base64ToUint8Array(content).buffer; return writeFile(filename, buffer); } else { // Otherwise, fetch from the URL diff --git a/src/components/Menus/Sidebar/ImagePanel/ProjectImages/ProjectImages.jsx b/src/components/Menus/Sidebar/ImagePanel/ProjectImages/ProjectImages.jsx index f09803623..b51205818 100644 --- a/src/components/Menus/Sidebar/ImagePanel/ProjectImages/ProjectImages.jsx +++ b/src/components/Menus/Sidebar/ImagePanel/ProjectImages/ProjectImages.jsx @@ -16,12 +16,7 @@ const ProjectImages = () => { className="project-images__image" src={ image.content - ? `data:image/png;base64,${btoa( - new Uint8Array(Object.values(image.content)).reduce( - (data, byte) => data + String.fromCharCode(byte), - "", - ), - )}` + ? `data:image/png;base64,${image.content}` : image.url } alt={image.filename} diff --git a/src/hooks/useProjectPersistence.js b/src/hooks/useProjectPersistence.js index f5bc2ca65..e6ffbe3e6 100644 --- a/src/hooks/useProjectPersistence.js +++ b/src/hooks/useProjectPersistence.js @@ -7,6 +7,7 @@ import { syncProject, } from "../redux/EditorSlice"; import { showLoginPrompt, showSavePrompt } from "../utils/Notifications"; +import { base64ToUint8Array } from "../utils/base64Helpers"; const COMBINED_FILE_SIZE_SOFT_LIMIT = 1000000; @@ -52,7 +53,15 @@ export const useProjectPersistence = ({ await dispatch( syncProject("remix")({ ...params, - project, + project: { + ...project, + image_list: project.image_list.map((image) => ({ + ...image, + content: image.content + ? base64ToUint8Array(image.content) + : null, + })), + }, }), ); if (loadRemix) { diff --git a/src/utils/base64Helpers.js b/src/utils/base64Helpers.js new file mode 100644 index 000000000..7bca01171 --- /dev/null +++ b/src/utils/base64Helpers.js @@ -0,0 +1,22 @@ +export const uint8ArrayToBase64 = (uint8) => { + return new Promise((resolve, reject) => { + const blob = new Blob([uint8]); + const reader = new FileReader(); + reader.onload = function () { + // reader.result is a data URL: "data:application/octet-stream;base64,...." + // We only want the part after the comma + resolve(reader.result.split(",")[1]); + }; + reader.onerror = reject; + reader.readAsDataURL(blob); + }); +}; + +export const base64ToUint8Array = async (base64) => { + return new Uint8Array( + window + .atob(base64) + .split("") + .map((c) => c.charCodeAt(0)), + ); +}; From 56fdbac58b9cc3c13ebdf8defefea07e8dc30336 Mon Sep 17 00:00:00 2001 From: Lois Wells Date: Wed, 9 Jul 2025 11:55:42 +0100 Subject: [PATCH 47/47] bug fixing --- src/hooks/useProjectPersistence.js | 5 +---- src/utils/base64Helpers.js | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/src/hooks/useProjectPersistence.js b/src/hooks/useProjectPersistence.js index e6ffbe3e6..632cd3360 100644 --- a/src/hooks/useProjectPersistence.js +++ b/src/hooks/useProjectPersistence.js @@ -7,7 +7,6 @@ import { syncProject, } from "../redux/EditorSlice"; import { showLoginPrompt, showSavePrompt } from "../utils/Notifications"; -import { base64ToUint8Array } from "../utils/base64Helpers"; const COMBINED_FILE_SIZE_SOFT_LIMIT = 1000000; @@ -57,9 +56,7 @@ export const useProjectPersistence = ({ ...project, image_list: project.image_list.map((image) => ({ ...image, - content: image.content - ? base64ToUint8Array(image.content) - : null, + content: image.content, })), }, }), diff --git a/src/utils/base64Helpers.js b/src/utils/base64Helpers.js index 7bca01171..7690f10b5 100644 --- a/src/utils/base64Helpers.js +++ b/src/utils/base64Helpers.js @@ -12,7 +12,7 @@ export const uint8ArrayToBase64 = (uint8) => { }); }; -export const base64ToUint8Array = async (base64) => { +export const base64ToUint8Array = (base64) => { return new Uint8Array( window .atob(base64)