From 0ee40a88f7445c5538e48e9e13e6304b80083b7b Mon Sep 17 00:00:00 2001 From: Juniper Tyree <50025784+juntyr@users.noreply.github.com> Date: Thu, 24 Apr 2025 18:24:12 +0000 Subject: [PATCH 1/5] Start work on a libpressio codec wrapper --- Cargo.toml | 2 + codecs/pressio/Cargo.toml | 27 +++++++++++++ codecs/pressio/LICENSE | 1 + codecs/pressio/README.md | 38 ++++++++++++++++++ codecs/pressio/src/lib.rs | 82 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 150 insertions(+) create mode 100644 codecs/pressio/Cargo.toml create mode 120000 codecs/pressio/LICENSE create mode 100644 codecs/pressio/README.md create mode 100644 codecs/pressio/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 893be9c2..ff60f688 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ members = [ "codecs/linear-quantize", "codecs/log", "codecs/pco", + "codecs/pressio", "codecs/random-projection", "codecs/reinterpret", "codecs/round", @@ -62,6 +63,7 @@ numcodecs-jpeg2000 = { version = "0.2", path = "codecs/jpeg2000", default-featur numcodecs-linear-quantize = { version = "0.4", path = "codecs/linear-quantize", default-features = false } numcodecs-log = { version = "0.4", path = "codecs/log", default-features = false } numcodecs-pco = { version = "0.2", path = "codecs/pco", default-features = false } +numcodecs-pressio = { version = "0.1", path = "codecs/pressio", default-features = false } numcodecs-random-projection = { version = "0.3", path = "codecs/random-projection", default-features = false } numcodecs-reinterpret = { version = "0.3", path = "codecs/reinterpret", default-features = false } numcodecs-round = { version = "0.3", path = "codecs/round", default-features = false } diff --git a/codecs/pressio/Cargo.toml b/codecs/pressio/Cargo.toml new file mode 100644 index 00000000..2a8830c3 --- /dev/null +++ b/codecs/pressio/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "numcodecs-pressio" +version = "0.1.0" +edition = { workspace = true } +authors = { workspace = true } +repository = { workspace = true } +license = { workspace = true } +rust-version = { workspace = true } + +description = "libpressio codec wrapper for the numcodecs API" +readme = "README.md" +categories = ["compression", "encoding"] +keywords = ["libpressio", "numcodecs", "compression", "encoding"] + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +numcodecs = { workspace = true } +schemars = { workspace = true, features = ["derive", "preserve_order"] } +serde = { workspace = true, features = ["std", "derive"] } +thiserror = { workspace = true } + +# FIXME: move into workspace dependencies +libpressio = { git = "https://github.com/juntyr/libpressio-rs.git", rev = "ddceba6" } + +[lints] +workspace = true diff --git a/codecs/pressio/LICENSE b/codecs/pressio/LICENSE new file mode 120000 index 00000000..30cff740 --- /dev/null +++ b/codecs/pressio/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/codecs/pressio/README.md b/codecs/pressio/README.md new file mode 100644 index 00000000..2a211f67 --- /dev/null +++ b/codecs/pressio/README.md @@ -0,0 +1,38 @@ +[![CI Status]][workflow] [![MSRV]][repo] [![Latest Version]][crates.io] [![PyPi Release]][pypi] [![Rust Doc Crate]][docs.rs] [![Rust Doc Main]][docs] [![Read the Docs]][rtdocs] + +[CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main +[workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain + +[MSRV]: https://img.shields.io/badge/MSRV-1.85.0-blue +[repo]: https://github.com/juntyr/numcodecs-rs + +[Latest Version]: https://img.shields.io/crates/v/numcodecs-pressio +[crates.io]: https://crates.io/crates/numcodecs-pressio + +[PyPi Release]: https://img.shields.io/pypi/v/numcodecs-wasm-pressio.svg +[pypi]: https://pypi.python.org/pypi/numcodecs-wasm-pressio + +[Rust Doc Crate]: https://img.shields.io/docsrs/numcodecs-pressio +[docs.rs]: https://docs.rs/numcodecs-pressio/ + +[Rust Doc Main]: https://img.shields.io/badge/docs-main-blue +[docs]: https://juntyr.github.io/numcodecs-rs/numcodecs_pressio + +[Read the Docs]: https://img.shields.io/readthedocs/numcodecs-wasm?label=readthedocs +[rtdocs]: https://numcodecs-wasm.readthedocs.io/en/stable/api/numcodecs_wasm_pressio/ + +# numcodecs-pressio + +libpressio codec wrapper for the [`numcodecs`] API. + +[`numcodecs`]: https://docs.rs/numcodecs/0.2/numcodecs/ + +## License + +Licensed under the Mozilla Public License, Version 2.0 ([LICENSE](LICENSE) or https://www.mozilla.org/en-US/MPL/2.0/). + +## Funding + +The `numcodecs-pressio` crate has been developed as part of [ESiWACE3](https://www.esiwace.eu), the third phase of the Centre of Excellence in Simulation of Weather and Climate in Europe. + +Funded by the European Union. This work has received funding from the European High Performance Computing Joint Undertaking (JU) under grant agreement No 101093054. diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs new file mode 100644 index 00000000..cfe500cb --- /dev/null +++ b/codecs/pressio/src/lib.rs @@ -0,0 +1,82 @@ +//! [![CI Status]][workflow] [![MSRV]][repo] [![Latest Version]][crates.io] [![Rust Doc Crate]][docs.rs] [![Rust Doc Main]][docs] +//! +//! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/numcodecs-rs/ci.yml?branch=main +//! [workflow]: https://github.com/juntyr/numcodecs-rs/actions/workflows/ci.yml?query=branch%3Amain +//! +//! [MSRV]: https://img.shields.io/badge/MSRV-1.85.0-blue +//! [repo]: https://github.com/juntyr/numcodecs-rs +//! +//! [Latest Version]: https://img.shields.io/crates/v/numcodecs-pressio +//! [crates.io]: https://crates.io/crates/numcodecs-pressio +//! +//! [Rust Doc Crate]: https://img.shields.io/docsrs/numcodecs-pressio +//! [docs.rs]: https://docs.rs/numcodecs-pressio/ +//! +//! [Rust Doc Main]: https://img.shields.io/badge/docs-main-blue +//! [docs]: https://juntyr.github.io/numcodecs-rs/numcodecs_pressio +//! +//! libpressio codec wrapper for the [`numcodecs`] API. + +use numcodecs::{ + AnyArray, AnyArrayAssignError, AnyArrayView, AnyArrayViewMut, AnyCowArray, Codec, StaticCodec, + StaticCodecConfig, StaticCodecVersion, +}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +#[derive(Clone, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +/// Identity codec which applies the identity function, i.e. passes through the +/// input unchanged during encoding and decoding. +pub struct IdentityCodec { + /// The codec's encoding format version. Do not provide this parameter explicitly. + #[serde(default, rename = "_version")] + pub version: StaticCodecVersion<1, 0, 0>, +} + +impl Codec for IdentityCodec { + type Error = IdentityCodecError; + + fn encode(&self, data: AnyCowArray) -> Result { + Ok(data.into_owned()) + } + + fn decode(&self, encoded: AnyCowArray) -> Result { + Ok(encoded.into_owned()) + } + + fn decode_into( + &self, + encoded: AnyArrayView, + mut decoded: AnyArrayViewMut, + ) -> Result<(), Self::Error> { + Ok(decoded.assign(&encoded)?) + } +} + +impl StaticCodec for IdentityCodec { + const CODEC_ID: &'static str = "identity.rs"; + + type Config<'de> = Self; + + fn from_config(config: Self::Config<'_>) -> Self { + config + } + + fn get_config(&self) -> StaticCodecConfig { + StaticCodecConfig::from(self) + } +} + +#[derive(Debug, Error)] +/// Errors that may occur when applying the [`IdentityCodec`]. +pub enum IdentityCodecError { + /// [`IdentityCodec`] cannot decode into the provided array + #[error("Identity cannot decode into the provided array")] + MismatchedDecodeIntoArray { + /// The source of the error + #[from] + source: AnyArrayAssignError, + }, +} From f20df186f3eff3579c784f11a6b9de8b5a9a4466 Mon Sep 17 00:00:00 2001 From: Juniper Tyree <50025784+juntyr@users.noreply.github.com> Date: Fri, 25 Apr 2025 07:50:21 +0000 Subject: [PATCH 2/5] Try with some libpressio fixes --- codecs/pressio/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codecs/pressio/Cargo.toml b/codecs/pressio/Cargo.toml index 2a8830c3..bec78ed0 100644 --- a/codecs/pressio/Cargo.toml +++ b/codecs/pressio/Cargo.toml @@ -21,7 +21,7 @@ serde = { workspace = true, features = ["std", "derive"] } thiserror = { workspace = true } # FIXME: move into workspace dependencies -libpressio = { git = "https://github.com/juntyr/libpressio-rs.git", rev = "ddceba6" } +libpressio = { git = "https://github.com/juntyr/libpressio-rs.git", rev = "82afbea" } [lints] workspace = true From be882e65bd0251a348b370acea22b0eb22cbcf4b Mon Sep 17 00:00:00 2001 From: Juniper Tyree <50025784+juntyr@users.noreply.github.com> Date: Fri, 25 Apr 2025 08:10:37 +0000 Subject: [PATCH 3/5] Support building WASM codecs for local crates --- .github/workflows/ci.yml | 3 +++ crates/numcodecs-wasm-builder/src/main.rs | 21 ++++++++++++++++++--- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eab679a1..dae12153 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,6 +91,7 @@ jobs: cargo hack check --all \ --exclude numcodecs-python \ --exclude numcodecs-jpeg2000 \ + --exclude numcodecs-pressio \ --exclude numcodecs-sz3 \ --exclude numcodecs-tthresh \ --exclude numcodecs-zfp \ @@ -221,6 +222,7 @@ jobs: cargo hack clippy --all \ --exclude numcodecs-python \ --exclude numcodecs-jpeg2000 \ + --exclude numcodecs-pressio \ --exclude numcodecs-sz3 \ --exclude numcodecs-tthresh \ --exclude numcodecs-zfp \ @@ -236,6 +238,7 @@ jobs: cargo hack clippy --all \ --exclude numcodecs-python \ --exclude numcodecs-jpeg2000 \ + --exclude numcodecs-pressio \ --exclude numcodecs-sz3 \ --exclude numcodecs-tthresh \ --exclude numcodecs-zfp \ diff --git a/crates/numcodecs-wasm-builder/src/main.rs b/crates/numcodecs-wasm-builder/src/main.rs index 8fe9ef02..eeed60ae 100644 --- a/crates/numcodecs-wasm-builder/src/main.rs +++ b/crates/numcodecs-wasm-builder/src/main.rs @@ -22,6 +22,10 @@ struct Args { #[arg(long)] version: Version, + /// Path to the local version of the numcodecs codec crate to compile + #[arg(long)] + path: Option, + /// Path to the codec type to export, without the leading crate name #[arg(long)] codec: String, @@ -46,8 +50,13 @@ fn main() -> io::Result<()> { eprintln!("creating {target_dir:?}"); fs::create_dir_all(&target_dir)?; - let crate_dir = - create_codec_wasm_component_crate(&scratch_dir, &args.crate_, &args.version, &args.codec)?; + let crate_dir = create_codec_wasm_component_crate( + &scratch_dir, + &args.crate_, + &args.version, + args.path.as_deref(), + &args.codec, + )?; copy_buildenv_to_crate(&crate_dir)?; let nix_env = NixEnv::new(&crate_dir)?; @@ -70,6 +79,7 @@ fn create_codec_wasm_component_crate( scratch_dir: &Path, crate_: &str, version: &Version, + path: Option<&Path>, codec: &str, ) -> io::Result { let crate_dir = scratch_dir.join(format!("{crate_}-wasm-{version}")); @@ -80,6 +90,11 @@ fn create_codec_wasm_component_crate( } fs::create_dir_all(&crate_dir)?; + let path = path.map_or_else( + || String::new(), + |p| format!("path = \"{}\", ", p.display()), + ); + fs::write( crate_dir.join("Cargo.toml"), format!( @@ -96,7 +111,7 @@ edition = "2024" [dependencies] numcodecs-wasm-logging = {{ version = "0.1", default-features = false }} numcodecs-wasm-guest = {{ version = "0.2", default-features = false }} -numcodecs-my-codec = {{ package = "{crate_}", version = "{version}", default-features = false }} +numcodecs-my-codec = {{ package = "{crate_}", version = "{version}", {path}default-features = false }} "# ), )?; From 569782c6ed8abf6052b542cadc9f5b961b3b1af0 Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Fri, 25 Apr 2025 12:12:29 +0300 Subject: [PATCH 4/5] Update libpressio rev --- codecs/pressio/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/codecs/pressio/Cargo.toml b/codecs/pressio/Cargo.toml index bec78ed0..b3b7233e 100644 --- a/codecs/pressio/Cargo.toml +++ b/codecs/pressio/Cargo.toml @@ -21,7 +21,7 @@ serde = { workspace = true, features = ["std", "derive"] } thiserror = { workspace = true } # FIXME: move into workspace dependencies -libpressio = { git = "https://github.com/juntyr/libpressio-rs.git", rev = "82afbea" } +libpressio = { git = "https://github.com/juntyr/libpressio-rs.git", rev = "6d25c06", default-features = false } [lints] workspace = true From 90acf03bfc2a332ab28b5a50028c5beec42fa91e Mon Sep 17 00:00:00 2001 From: Juniper Tyree Date: Fri, 25 Apr 2025 12:29:15 +0300 Subject: [PATCH 5/5] Fix clippy lints --- codecs/pressio/src/lib.rs | 1 + crates/numcodecs-wasm-builder/src/main.rs | 5 +---- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/codecs/pressio/src/lib.rs b/codecs/pressio/src/lib.rs index cfe500cb..4026c773 100644 --- a/codecs/pressio/src/lib.rs +++ b/codecs/pressio/src/lib.rs @@ -17,6 +17,7 @@ //! //! libpressio codec wrapper for the [`numcodecs`] API. +use ::libpressio as _; use numcodecs::{ AnyArray, AnyArrayAssignError, AnyArrayView, AnyArrayViewMut, AnyCowArray, Codec, StaticCodec, StaticCodecConfig, StaticCodecVersion, diff --git a/crates/numcodecs-wasm-builder/src/main.rs b/crates/numcodecs-wasm-builder/src/main.rs index eeed60ae..124b3b9d 100644 --- a/crates/numcodecs-wasm-builder/src/main.rs +++ b/crates/numcodecs-wasm-builder/src/main.rs @@ -90,10 +90,7 @@ fn create_codec_wasm_component_crate( } fs::create_dir_all(&crate_dir)?; - let path = path.map_or_else( - || String::new(), - |p| format!("path = \"{}\", ", p.display()), - ); + let path = path.map_or_else(String::new, |p| format!("path = \"{}\", ", p.display())); fs::write( crate_dir.join("Cargo.toml"),