From cdee06ead7d92b633d0c4cf249394c8464dbfc14 Mon Sep 17 00:00:00 2001 From: Don MacAskill Date: Tue, 10 Jun 2025 15:59:01 -0700 Subject: [PATCH 01/11] Add support for custom CRC parameters Enables anyone to BYOP (bring your own parameters). See: https://github.com/awesomized/crc-fast-rust/issues/5 --- src/lib.rs | 170 +++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 160 insertions(+), 10 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index b4f7a8e..d90ee33 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -54,7 +54,7 @@ //! Implements the [std::io::Write](https://doc.rust-lang.org/std/io/trait.Write.html) trait for //! easier integration with existing code. //! -//! ```no_run +//! ```rust //! use std::env; //! use std::fs::File; //! use crc_fast::{Digest, CrcAlgorithm::Crc32IsoHdlc}; @@ -261,6 +261,19 @@ impl Digest { } } + /// Creates a new `Digest` instance with custom parameters. + #[inline(always)] + pub fn new_with_params(params: CrcParams) -> Self { + let calculator = Calculator::calculate as CalculatorFn; + + Self { + state: params.init, + amount: 0, + params, + calculator, + } + } + /// Updates the CRC state with the given data. #[inline(always)] pub fn update(&mut self, data: &[u8]) { @@ -361,6 +374,13 @@ pub fn checksum(algorithm: CrcAlgorithm, buf: &[u8]) -> u64 { calculator(params.init, buf, params) ^ params.xorout } +/// Computes the CRC checksum for the given data using the custom specified parameters. +pub fn checksum_with_params(params: CrcParams, buf: &[u8]) -> u64 { + let calculator = Calculator::calculate as CalculatorFn; + + calculator(params.init, buf, params) ^ params.xorout +} + /// Computes the CRC checksum for the given file using the specified algorithm. /// /// Appears to be much faster (~2X) than using Writer and io::*, at least on Apple M2 Ultra @@ -371,7 +391,7 @@ pub fn checksum(algorithm: CrcAlgorithm, buf: &[u8]) -> u64 { /// /// # Examples /// ### checksum_file -///```no_run +///```rust /// use std::env; /// use crc_fast::{checksum_file, CrcAlgorithm::Crc32IsoHdlc}; /// @@ -389,7 +409,32 @@ pub fn checksum_file( path: &str, chunk_size: Option, ) -> Result { - let mut digest = Digest::new(algorithm); + checksum_file_with_digest(Digest::new(algorithm), path, chunk_size) +} + +/// Computes the CRC checksum for the given file using the custom specified parameters. +/// +/// # Errors +/// +/// This function will return an error if the file cannot be read. +pub fn checksum_file_with_params( + params: CrcParams, + path: &str, + chunk_size: Option, +) -> Result { + checksum_file_with_digest(Digest::new_with_params(params), path, chunk_size) +} + +/// Computes the CRC checksum for the given file using the specified Digest. +/// +/// # Errors +/// +/// This function will return an error if the file cannot be read. +fn checksum_file_with_digest( + mut digest: Digest, + path: &str, + chunk_size: Option, +) -> Result { let mut file = File::open(path)?; // 512KiB KiB was fastest in my benchmarks on an Apple M2 Ultra @@ -540,19 +585,78 @@ mod lib { } } + #[test] + fn test_checksum_with_custom_params() { + // CRC-32 reflected + assert_eq!( + checksum_with_params(CRC32_ISCSI, TEST_CHECK_STRING), + CRC32_ISCSI.check, + ); + + // CRC-32 forward + assert_eq!( + checksum_with_params(CRC32_BZIP2, TEST_CHECK_STRING), + CRC32_BZIP2.check, + ); + + // CRC-64 reflected + assert_eq!( + checksum_with_params(CRC64_NVME, TEST_CHECK_STRING), + CRC64_NVME.check, + ); + + // CRC-64 forward + assert_eq!( + checksum_with_params(CRC64_ECMA_182, TEST_CHECK_STRING), + CRC64_ECMA_182.check, + ); + } + #[test] fn test_digest_updates_check() { for config in TEST_ALL_CONFIGS { let mut digest = Digest::new(config.get_algorithm()); - digest.update(b"123"); - digest.update(b"456"); - digest.update(b"789"); - let result = digest.finalize(); - - assert_eq!(result, config.get_check()); + check_digest(Digest::new(config.get_algorithm()), config.get_check()); } } + #[test] + fn test_digest_updates_check_with_custom_params() { + // CRC-32 reflected + check_digest( + Digest::new_with_params(CRC32_ISCSI), + CRC32_ISCSI.check, + ); + + // CRC-32 forward + check_digest( + Digest::new_with_params(CRC32_BZIP2), + CRC32_BZIP2.check, + ); + + // CRC-64 reflected + check_digest( + Digest::new_with_params(CRC64_NVME), + CRC64_NVME.check, + ); + + // CRC-64 forward + check_digest( + Digest::new_with_params(CRC64_ECMA_182), + CRC64_ECMA_182.check, + ); + } + + fn check_digest(mut digest: Digest, check: u64) { + digest.update(b"123"); + digest.update(b"456"); + digest.update(b"789"); + assert_eq!( + digest.finalize(), + check, + ); + } + #[test] fn test_small_all_lengths() { for config in TEST_ALL_CONFIGS { @@ -634,7 +738,7 @@ mod lib { // Create a test file with repeating zeros let test_file_path = "test/test_crc32_hash_file.bin"; let data = vec![0u8; 1024 * 1024]; // 1 MiB of zeros - if let Err(e) = std::fs::write(test_file_path, &data) { + if let Err(e) = write(test_file_path, &data) { eprintln!("Skipping test due to write error: {}", e); return; } @@ -647,6 +751,52 @@ mod lib { std::fs::remove_file(test_file_path).unwrap(); } + #[test] + fn test_checksum_file_with_custom_params() { + // Create a test file with repeating zeros + let test_file_path = "test/test_crc32_hash_file_custom.bin"; + let data = vec![0u8; 1024 * 1024]; // 1 MiB of zeros + if let Err(e) = write(test_file_path, &data) { + eprintln!("Skipping test due to write error: {}", e); + return; + } + + // CRC-32 reflected + check_file( + CRC32_ISCSI, + test_file_path, + CRC32_ISCSI.check, + ); + + // CRC-32 forward + check_file( + CRC32_BZIP2, + test_file_path, + CRC32_BZIP2.check, + ); + + // CRC-64 reflected + check_file( + CRC64_NVME, + test_file_path, + CRC64_NVME.check, + ); + + // CRC-64 forward + check_file( + CRC64_ECMA_182, + test_file_path, + CRC64_ECMA_182.check, + ); + + std::fs::remove_file(test_file_path).unwrap(); + } + + fn check_file(params: CrcParams, file_path: &str, check: u64) { + let result = checksum_file_with_params(params, file_path, None).unwrap(); + assert_eq!(result, check); + } + #[test] fn test_writer() { // Create a test file with repeating zeros From 99985e586bf0fbb5b8e8bde635acf32255b6c868 Mon Sep 17 00:00:00 2001 From: Don MacAskill Date: Tue, 10 Jun 2025 15:59:21 -0700 Subject: [PATCH 02/11] Add shared library support for custom CRC params --- libcrc_fast.h | 34 ++++++++++++++++++++++ src/ffi.rs | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+) diff --git a/libcrc_fast.h b/libcrc_fast.h index 0b5b395..6677535 100644 --- a/libcrc_fast.h +++ b/libcrc_fast.h @@ -50,6 +50,21 @@ typedef struct CrcFastDigestHandle { struct CrcFastDigest *_0; } CrcFastDigestHandle; +/** + * Custom CRC parameters + */ +typedef struct CrcFastParams { + enum CrcFastAlgorithm algorithm; + uint8_t width; + uint64_t poly; + uint64_t init; + bool refin; + bool refout; + uint64_t xorout; + uint64_t check; + uint64_t keys[23]; +} CrcFastParams; + #ifdef __cplusplus extern "C" { #endif // __cplusplus @@ -59,6 +74,11 @@ extern "C" { */ struct CrcFastDigestHandle *crc_fast_digest_new(enum CrcFastAlgorithm algorithm); +/** + * Creates a new Digest to compute CRC checksums using custom parameters + */ +struct CrcFastDigestHandle *crc_fast_digest_new_with_params(struct CrcFastParams params); + /** * Updates the Digest with data */ @@ -100,6 +120,13 @@ uint64_t crc_fast_digest_get_amount(struct CrcFastDigestHandle *handle); */ uint64_t crc_fast_checksum(enum CrcFastAlgorithm algorithm, const char *data, uintptr_t len); +/** + * Helper method to calculate a CRC checksum directly for data using custom parameters + */ +uint64_t crc_fast_checksum_with_params(struct CrcFastParams params, + const char *data, + uintptr_t len); + /** * Helper method to just calculate a CRC checksum directly for a file using algorithm */ @@ -107,6 +134,13 @@ uint64_t crc_fast_checksum_file(enum CrcFastAlgorithm algorithm, const uint8_t *path_ptr, uintptr_t path_len); +/** + * Helper method to calculate a CRC checksum directly for a file using custom parameters + */ +uint64_t crc_fast_checksum_file_with_params(struct CrcFastParams params, + const uint8_t *path_ptr, + uintptr_t path_len); + /** * Combine two CRC checksums using algorithm */ diff --git a/src/ffi.rs b/src/ffi.rs index 8f1327d..faf0a4f 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -12,6 +12,7 @@ use crate::{get_calculator_target, Digest}; use std::ffi::CStr; use std::os::raw::c_char; use std::slice; +use crate::structs::CrcParams; /// A handle to the Digest object #[repr(C)] @@ -68,6 +69,38 @@ impl From for CrcAlgorithm { } } +/// Custom CRC parameters +#[repr(C)] +pub struct CrcFastParams { + pub algorithm: CrcFastAlgorithm, + pub width: u8, + pub poly: u64, + pub init: u64, + pub refin: bool, + pub refout: bool, + pub xorout: u64, + pub check: u64, + pub keys: [u64; 23], +} + +// Convert from FFI struct to internal struct +impl From for CrcParams { + fn from(value: CrcFastParams) -> Self { + CrcParams { + algorithm: value.algorithm.into(), + name: "custom", // C interface doesn't need the name field + width: value.width, + poly: value.poly, + init: value.init, + refin: value.refin, + refout: value.refout, + xorout: value.xorout, + check: value.check, + keys: value.keys, + } + } +} + /// Creates a new Digest to compute CRC checksums using algorithm #[no_mangle] pub extern "C" fn crc_fast_digest_new(algorithm: CrcFastAlgorithm) -> *mut CrcFastDigestHandle { @@ -76,6 +109,14 @@ pub extern "C" fn crc_fast_digest_new(algorithm: CrcFastAlgorithm) -> *mut CrcFa Box::into_raw(handle) } +/// Creates a new Digest to compute CRC checksums using custom parameters +#[no_mangle] +pub extern "C" fn crc_fast_digest_new_with_params(params: CrcFastParams) -> *mut CrcFastDigestHandle { + let digest = Box::new(Digest::new_with_params(params.into())); + let handle = Box::new(CrcFastDigestHandle(Box::into_raw(digest))); + Box::into_raw(handle) +} + /// Updates the Digest with data #[no_mangle] pub extern "C" fn crc_fast_digest_update( @@ -197,6 +238,23 @@ pub extern "C" fn crc_fast_checksum( } } +/// Helper method to calculate a CRC checksum directly for data using custom parameters +#[no_mangle] +pub extern "C" fn crc_fast_checksum_with_params( + params: CrcFastParams, + data: *const c_char, + len: usize, +) -> u64 { + if data.is_null() { + return 0; + } + unsafe { + #[allow(clippy::unnecessary_cast)] + let bytes = slice::from_raw_parts(data as *const u8, len); + crate::checksum_with_params(params.into(), bytes) + } +} + /// Helper method to just calculate a CRC checksum directly for a file using algorithm #[no_mangle] pub extern "C" fn crc_fast_checksum_file( @@ -218,6 +276,27 @@ pub extern "C" fn crc_fast_checksum_file( } } +/// Helper method to calculate a CRC checksum directly for a file using custom parameters +#[no_mangle] +pub extern "C" fn crc_fast_checksum_file_with_params( + params: CrcFastParams, + path_ptr: *const u8, + path_len: usize, +) -> u64 { + if path_ptr.is_null() { + return 0; + } + + unsafe { + crate::checksum_file_with_params( + params.into(), + &convert_to_string(path_ptr, path_len), + None, + ) + .unwrap_or(0) // Return 0 on error instead of panicking + } +} + /// Combine two CRC checksums using algorithm #[no_mangle] pub extern "C" fn crc_fast_checksum_combine( From 2eb380fa91932a655b3a4f7a88e933d73bcf0408 Mon Sep 17 00:00:00 2001 From: Don MacAskill Date: Tue, 10 Jun 2025 16:05:22 -0700 Subject: [PATCH 03/11] Fix formatting --- src/ffi.rs | 8 +++++--- src/lib.rs | 44 ++++++++------------------------------------ 2 files changed, 13 insertions(+), 39 deletions(-) diff --git a/src/ffi.rs b/src/ffi.rs index faf0a4f..4311a60 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -7,12 +7,12 @@ #![cfg(any(target_arch = "aarch64", target_arch = "x86_64", target_arch = "x86"))] +use crate::structs::CrcParams; use crate::CrcAlgorithm; use crate::{get_calculator_target, Digest}; use std::ffi::CStr; use std::os::raw::c_char; use std::slice; -use crate::structs::CrcParams; /// A handle to the Digest object #[repr(C)] @@ -111,7 +111,9 @@ pub extern "C" fn crc_fast_digest_new(algorithm: CrcFastAlgorithm) -> *mut CrcFa /// Creates a new Digest to compute CRC checksums using custom parameters #[no_mangle] -pub extern "C" fn crc_fast_digest_new_with_params(params: CrcFastParams) -> *mut CrcFastDigestHandle { +pub extern "C" fn crc_fast_digest_new_with_params( + params: CrcFastParams, +) -> *mut CrcFastDigestHandle { let digest = Box::new(Digest::new_with_params(params.into())); let handle = Box::new(CrcFastDigestHandle(Box::into_raw(digest))); Box::into_raw(handle) @@ -293,7 +295,7 @@ pub extern "C" fn crc_fast_checksum_file_with_params( &convert_to_string(path_ptr, path_len), None, ) - .unwrap_or(0) // Return 0 on error instead of panicking + .unwrap_or(0) // Return 0 on error instead of panicking } } diff --git a/src/lib.rs b/src/lib.rs index d90ee33..62974de 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -623,22 +623,13 @@ mod lib { #[test] fn test_digest_updates_check_with_custom_params() { // CRC-32 reflected - check_digest( - Digest::new_with_params(CRC32_ISCSI), - CRC32_ISCSI.check, - ); + check_digest(Digest::new_with_params(CRC32_ISCSI), CRC32_ISCSI.check); // CRC-32 forward - check_digest( - Digest::new_with_params(CRC32_BZIP2), - CRC32_BZIP2.check, - ); + check_digest(Digest::new_with_params(CRC32_BZIP2), CRC32_BZIP2.check); // CRC-64 reflected - check_digest( - Digest::new_with_params(CRC64_NVME), - CRC64_NVME.check, - ); + check_digest(Digest::new_with_params(CRC64_NVME), CRC64_NVME.check); // CRC-64 forward check_digest( @@ -651,10 +642,7 @@ mod lib { digest.update(b"123"); digest.update(b"456"); digest.update(b"789"); - assert_eq!( - digest.finalize(), - check, - ); + assert_eq!(digest.finalize(), check,); } #[test] @@ -762,32 +750,16 @@ mod lib { } // CRC-32 reflected - check_file( - CRC32_ISCSI, - test_file_path, - CRC32_ISCSI.check, - ); + check_file(CRC32_ISCSI, test_file_path, CRC32_ISCSI.check); // CRC-32 forward - check_file( - CRC32_BZIP2, - test_file_path, - CRC32_BZIP2.check, - ); + check_file(CRC32_BZIP2, test_file_path, CRC32_BZIP2.check); // CRC-64 reflected - check_file( - CRC64_NVME, - test_file_path, - CRC64_NVME.check, - ); + check_file(CRC64_NVME, test_file_path, CRC64_NVME.check); // CRC-64 forward - check_file( - CRC64_ECMA_182, - test_file_path, - CRC64_ECMA_182.check, - ); + check_file(CRC64_ECMA_182, test_file_path, CRC64_ECMA_182.check); std::fs::remove_file(test_file_path).unwrap(); } From 54779da5565840538fef232aec697ff9b0c16a7d Mon Sep 17 00:00:00 2001 From: Don MacAskill Date: Tue, 10 Jun 2025 16:06:41 -0700 Subject: [PATCH 04/11] Remove dead code --- src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 62974de..14469e3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -615,7 +615,6 @@ mod lib { #[test] fn test_digest_updates_check() { for config in TEST_ALL_CONFIGS { - let mut digest = Digest::new(config.get_algorithm()); check_digest(Digest::new(config.get_algorithm()), config.get_check()); } } From 97d8d516342e196bcde8fbf60506cdef23186d14 Mon Sep 17 00:00:00 2001 From: Don MacAskill Date: Wed, 11 Jun 2025 11:09:14 -0700 Subject: [PATCH 05/11] Add more custom parameter support - Crc32Custom and Crc64Custom CrcAlgorithms - checksum_combine_with_custom_params() function - get_custom_params() function - test coverage for the above --- src/enums.rs | 2 + src/lib.rs | 123 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 125 insertions(+) diff --git a/src/enums.rs b/src/enums.rs index 02f487a..b1c50ac 100644 --- a/src/enums.rs +++ b/src/enums.rs @@ -43,12 +43,14 @@ impl Display for CrcAlgorithm { CrcAlgorithm::Crc32Bzip2 => write!(f, "{}", NAME_CRC32_BZIP2), CrcAlgorithm::Crc32CdRomEdc => write!(f, "{}", NAME_CRC32_CD_ROM_EDC), CrcAlgorithm::Crc32Cksum => write!(f, "{}", NAME_CRC32_CKSUM), + CrcAlgorithm::Crc32Custom => write!(f, "CRC-32/CUSTOM"), CrcAlgorithm::Crc32Iscsi => write!(f, "{}", NAME_CRC32_ISCSI), CrcAlgorithm::Crc32IsoHdlc => write!(f, "{}", NAME_CRC32_ISO_HDLC), CrcAlgorithm::Crc32Jamcrc => write!(f, "{}", NAME_CRC32_JAMCRC), CrcAlgorithm::Crc32Mef => write!(f, "{}", NAME_CRC32_MEF), CrcAlgorithm::Crc32Mpeg2 => write!(f, "{}", NAME_CRC32_MPEG_2), CrcAlgorithm::Crc32Xfer => write!(f, "{}", NAME_CRC32_XFER), + CrcAlgorithm::Crc64Custom => write!(f, "CRC-64/CUSTOM"), CrcAlgorithm::Crc64GoIso => write!(f, "{}", NAME_CRC64_GO_ISO), CrcAlgorithm::Crc64Ms => write!(f, "{}", NAME_CRC64_MS), CrcAlgorithm::Crc64Nvme => write!(f, "{}", NAME_CRC64_NVME), diff --git a/src/lib.rs b/src/lib.rs index 14469e3..7c23cff 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -144,12 +144,14 @@ pub enum CrcAlgorithm { Crc32Bzip2, Crc32CdRomEdc, Crc32Cksum, + Crc32Custom, // Custom CRC-32 implementation, not defined in consts Crc32Iscsi, Crc32IsoHdlc, Crc32Jamcrc, Crc32Mef, Crc32Mpeg2, Crc32Xfer, + Crc64Custom, // Custom CRC-64 implementation, not defined in consts Crc64Ecma182, Crc64GoIso, Crc64Ms, @@ -480,6 +482,52 @@ pub fn checksum_combine( combine::checksums(checksum1, checksum2, checksum2_len, params) } +/// Combines two CRC checksums using the custom specified parameters. +pub fn checksum_combine_with_custom_params( + params: CrcParams, + checksum1: u64, + checksum2: u64, + checksum2_len: u64, +) -> u64 { + combine::checksums(checksum1, checksum2, checksum2_len, params) +} + +/// Returns the custom CRC parameters for a given set of Rocksoft CRC parameters. +/// +/// Does not support mis-matched refin/refout parameters, so both must be true or both false. +/// +/// Rocksoft parameters for lots of variants: https://reveng.sourceforge.io/crc-catalogue/all.htm +pub fn get_custom_params( + name: &'static str, + width: u8, + poly: u64, + init: u64, + reflected: bool, + xorout: u64, + check: u64, +) -> CrcParams { + let keys = generate::keys(width, poly, reflected); + + let algorithm = match width { + 32 => CrcAlgorithm::Crc32Custom, + 64 => CrcAlgorithm::Crc64Custom, + _ => panic!("Unsupported width: {}", width), + }; + + CrcParams { + algorithm, + name, + width, + poly, + init, + refin: reflected, + refout: reflected, + xorout, + check, + keys, + } +} + /// Returns the target used to calculate the CRC checksum for the specified algorithm. /// /// These strings are informational only, not stable, and shouldn't be relied on to match across @@ -505,12 +553,18 @@ fn get_calculator_params(algorithm: CrcAlgorithm) -> (CalculatorFn, CrcParams) { CrcAlgorithm::Crc32Bzip2 => (Calculator::calculate as CalculatorFn, CRC32_BZIP2), CrcAlgorithm::Crc32CdRomEdc => (Calculator::calculate as CalculatorFn, CRC32_CD_ROM_EDC), CrcAlgorithm::Crc32Cksum => (Calculator::calculate as CalculatorFn, CRC32_CKSUM), + CrcAlgorithm::Crc32Custom => { + panic!("Custom CRC-32 requires parameters via get_custom_params()") + } CrcAlgorithm::Crc32Iscsi => (crc32_iscsi_calculator as CalculatorFn, CRC32_ISCSI), CrcAlgorithm::Crc32IsoHdlc => (crc32_iso_hdlc_calculator as CalculatorFn, CRC32_ISO_HDLC), CrcAlgorithm::Crc32Jamcrc => (Calculator::calculate as CalculatorFn, CRC32_JAMCRC), CrcAlgorithm::Crc32Mef => (Calculator::calculate as CalculatorFn, CRC32_MEF), CrcAlgorithm::Crc32Mpeg2 => (Calculator::calculate as CalculatorFn, CRC32_MPEG_2), CrcAlgorithm::Crc32Xfer => (Calculator::calculate as CalculatorFn, CRC32_XFER), + CrcAlgorithm::Crc64Custom => { + panic!("Custom CRC-64 requires parameters via get_custom_params()") + } CrcAlgorithm::Crc64Ecma182 => (Calculator::calculate as CalculatorFn, CRC64_ECMA_182), CrcAlgorithm::Crc64GoIso => (Calculator::calculate as CalculatorFn, CRC64_GO_ISO), CrcAlgorithm::Crc64Ms => (Calculator::calculate as CalculatorFn, CRC64_MS), @@ -560,6 +614,7 @@ mod lib { use super::*; use crate::test::consts::{TEST_ALL_CONFIGS, TEST_CHECK_STRING}; use crate::test::enums::AnyCrcTestConfig; + use crate::CrcAlgorithm::Crc32Iscsi; use cbindgen::Language::{Cxx, C}; use cbindgen::Style::Both; use rand::{rng, Rng}; @@ -612,6 +667,39 @@ mod lib { ); } + #[test] + fn test_get_custom_params() { + let custom_crc32 = get_custom_params( + "Custom CRC-32/ISCSI", + 32, + CRC32_ISCSI.poly, + CRC32_ISCSI.init, + CRC32_ISCSI.refin, + CRC32_ISCSI.xorout, + CRC32_ISCSI.check, + ); + + assert_eq!( + checksum_with_params(custom_crc32, TEST_CHECK_STRING), + CRC32_ISCSI.check, + ); + + let custom_crc64 = get_custom_params( + "Custom CRC-64/NVME", + 64, + CRC64_NVME.poly, + CRC64_NVME.init, + CRC64_NVME.refin, + CRC64_NVME.xorout, + CRC64_NVME.check, + ); + + assert_eq!( + checksum_with_params(custom_crc64, TEST_CHECK_STRING), + CRC64_NVME.check, + ); + } + #[test] fn test_digest_updates_check() { for config in TEST_ALL_CONFIGS { @@ -720,6 +808,41 @@ mod lib { } } + #[test] + fn test_combine_with_custom_params() { + // CRC-32 reflected + let checksum1 = checksum_with_params(CRC32_ISCSI, "1234".as_ref()); + let checksum2 = checksum_with_params(CRC32_ISCSI, "56789".as_ref()); + assert_eq!( + checksum_combine_with_custom_params(CRC32_ISCSI, checksum1, checksum2, 5), + CRC32_ISCSI.check, + ); + + // CRC-32 forward + let checksum1 = checksum_with_params(CRC32_BZIP2, "1234".as_ref()); + let checksum2 = checksum_with_params(CRC32_BZIP2, "56789".as_ref()); + assert_eq!( + checksum_combine_with_custom_params(CRC32_BZIP2, checksum1, checksum2, 5), + CRC32_BZIP2.check, + ); + + // CRC-64 reflected + let checksum1 = checksum_with_params(CRC64_NVME, "1234".as_ref()); + let checksum2 = checksum_with_params(CRC64_NVME, "56789".as_ref()); + assert_eq!( + checksum_combine_with_custom_params(CRC64_NVME, checksum1, checksum2, 5), + CRC64_NVME.check, + ); + + // CRC-64 forward + let checksum1 = checksum_with_params(CRC64_ECMA_182, "1234".as_ref()); + let checksum2 = checksum_with_params(CRC64_ECMA_182, "56789".as_ref()); + assert_eq!( + checksum_combine_with_custom_params(CRC64_ECMA_182, checksum1, checksum2, 5), + CRC64_ECMA_182.check, + ); + } + #[test] fn test_checksum_file() { // Create a test file with repeating zeros From 90538b788d43b27ce82c6f4d6264bbec9649a083 Mon Sep 17 00:00:00 2001 From: Don MacAskill Date: Wed, 11 Jun 2025 11:10:09 -0700 Subject: [PATCH 06/11] Add a command-line tool to calculate CRC params --- src/bin/get-custom-params.rs | 185 +++++++++++++++++++++++++++++++++++ 1 file changed, 185 insertions(+) create mode 100644 src/bin/get-custom-params.rs diff --git a/src/bin/get-custom-params.rs b/src/bin/get-custom-params.rs new file mode 100644 index 0000000..7b98aa0 --- /dev/null +++ b/src/bin/get-custom-params.rs @@ -0,0 +1,185 @@ +//! This is a simple program to get custom CRC parameters from the command line. + +use std::env; +use std::process::ExitCode; + +#[derive(Debug)] +struct Config { + width: Option, + polynomial: Option, + init: Option, + reflected: Option, + xorout: Option, + check: Option, + name: Option, +} + +impl Config { + fn new() -> Self { + Config { + width: None, + polynomial: None, + init: None, + reflected: None, + xorout: None, + check: None, + name: None, + } + } + + fn is_complete(&self) -> bool { + self.width.is_some() + && self.polynomial.is_some() + && self.init.is_some() + && self.reflected.is_some() + && self.xorout.is_some() + && self.check.is_some() + && self.name.is_some() + } +} + +fn parse_hex_or_decimal(s: &str) -> Result { + if s.starts_with("0x") || s.starts_with("0X") { + u64::from_str_radix(&s[2..], 16).map_err(|_| format!("Invalid hexadecimal value: {}", s)) + } else { + s.parse::() + .map_err(|_| format!("Invalid decimal value: {}", s)) + } +} + +fn parse_bool(s: &str) -> Result { + match s.to_lowercase().as_str() { + "true" | "1" | "yes" | "on" => Ok(true), + "false" | "0" | "no" | "off" => Ok(false), + _ => Err(format!("Invalid boolean value: {} (use true/false)", s)), + } +} + +fn parse_args(args: &[String]) -> Result { + let mut config = Config::new(); + let mut i = 1; // Skip program name + + while i < args.len() { + match args[i].as_str() { + "-n" => { + if i + 1 >= args.len() { + return Err("Missing value for -n (name)".to_string()); + } + config.name = Some(args[i + 1].clone()); + i += 2; + } + "-w" => { + if i + 1 >= args.len() { + return Err("Missing value for -w (width)".to_string()); + } + config.width = Some( + args[i + 1] + .parse::() + .map_err(|_| format!("Invalid width value: {}", args[i + 1]))?, + ); + i += 2; + } + "-p" => { + if i + 1 >= args.len() { + return Err("Missing value for -p (polynomial)".to_string()); + } + config.polynomial = Some(parse_hex_or_decimal(&args[i + 1])?); + i += 2; + } + "-i" => { + if i + 1 >= args.len() { + return Err("Missing value for -i (init)".to_string()); + } + config.init = Some(parse_hex_or_decimal(&args[i + 1])?); + i += 2; + } + "-r" => { + if i + 1 >= args.len() { + return Err("Missing value for -r (reflected)".to_string()); + } + config.reflected = Some(parse_bool(&args[i + 1])?); + i += 2; + } + "-x" => { + if i + 1 >= args.len() { + return Err("Missing value for -x (xorout)".to_string()); + } + config.xorout = Some(parse_hex_or_decimal(&args[i + 1])?); + i += 2; + } + "-c" => { + if i + 1 >= args.len() { + return Err("Missing value for -c (check)".to_string()); + } + config.check = Some(parse_hex_or_decimal(&args[i + 1])?); + i += 2; + } + arg => { + return Err(format!("Unknown argument: {}", arg)); + } + } + } + + Ok(config) +} + +fn print_usage() { + println!("Usage: get-custom-params -n -w -p -i -r -x -c "); + println!(); + println!("Example: get-custom-params -n CRC-32/ISCSI -w 32 -p 0x1edc6f41 -i 0xFFFFFFFF -r true -x 0xFFFFFFFF -c 0xe3069283"); + println!("Example: get-custom-params -n CRC-64/NVME -w 64 -p 0xad93d23594c93659 -i 0xffffffffffffffff -r true -x 0xffffffffffffffff -c 0xae8b14860a799888"); + println!(); + println!("Arguments:"); + println!(" -n Name of the CRC algorithm (e.g., CRC-32/ISCSI)"); + println!(" -w CRC width (number of bits)"); + println!(" -p CRC polynomial (hex or decimal)"); + println!(" -i Initial value (hex or decimal)"); + println!(" -r Reflected input/output (true/false)"); + println!(" -x XOR output value (hex or decimal)"); + println!(" -c Check value (hex or decimal)"); +} + +fn main() -> ExitCode { + let args: Vec = env::args().collect(); + + if args.len() == 1 { + print_usage(); + return ExitCode::from(1); + } + + let config = match parse_args(&args) { + Ok(config) => config, + Err(error) => { + eprintln!("Error: {}", error); + println!(); + print_usage(); + return ExitCode::from(1); + } + }; + + // Check if all required arguments are provided + if !config.is_complete() { + eprintln!("Error: All arguments are required"); + println!(); + print_usage(); + return ExitCode::from(1); + } + + let static_name: &'static str = Box::leak(config.name.unwrap().into_boxed_str()); + + let params = crc_fast::get_custom_params( + static_name, + config.width.unwrap() as u8, + config.polynomial.unwrap(), + config.init.unwrap(), + config.reflected.unwrap(), + config.xorout.unwrap(), + config.check.unwrap(), + ); + + println!(); + println!("{:#x?}", params); + println!(); + + ExitCode::from(0) +} From 17f40577842a9bb5fed717638752f7d858c489f3 Mon Sep 17 00:00:00 2001 From: Don MacAskill Date: Wed, 11 Jun 2025 11:16:08 -0700 Subject: [PATCH 07/11] Make CrcParams public Enables easier integration for consumers who would like to use custom CRC parameters, such as polynomials. --- src/algorithm.rs | 4 ++-- src/arch/mod.rs | 2 +- src/arch/software.rs | 2 +- src/combine.rs | 2 +- src/crc32/consts.rs | 2 +- src/crc64/consts.rs | 2 +- src/ffi.rs | 2 +- src/lib.rs | 24 +++++++++++++++++++----- src/structs.rs | 18 +----------------- src/test/enums.rs | 2 +- src/test/structs.rs | 2 +- src/traits.rs | 2 +- 12 files changed, 31 insertions(+), 33 deletions(-) diff --git a/src/algorithm.rs b/src/algorithm.rs index 20c394b..c999fe9 100644 --- a/src/algorithm.rs +++ b/src/algorithm.rs @@ -17,9 +17,9 @@ use crate::consts::CRC_CHUNK_SIZE; use crate::enums::{DataChunkProcessor, Reflector}; -use crate::structs::{CrcParams, CrcState}; +use crate::structs::CrcState; use crate::traits::{ArchOps, EnhancedCrcWidth}; -use crate::{crc32, crc64}; +use crate::{crc32, crc64, CrcParams}; /// Main entry point that works for both CRC-32 and CRC-64 #[inline] diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 22849e0..23aa21e 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -11,7 +11,7 @@ use std::arch::is_aarch64_feature_detected; #[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))] use crate::algorithm; -use crate::structs::CrcParams; +use crate::CrcParams; #[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))] use crate::structs::{Width32, Width64}; diff --git a/src/arch/software.rs b/src/arch/software.rs index a573d2d..5b8d32a 100644 --- a/src/arch/software.rs +++ b/src/arch/software.rs @@ -5,8 +5,8 @@ #![cfg(not(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64")))] use crate::consts::CRC_64_NVME; -use crate::structs::CrcParams; use crate::CrcAlgorithm; +use crate::CrcParams; use crc::Table; const RUST_CRC32_AIXM: crc::Crc> = diff --git a/src/combine.rs b/src/combine.rs index d4b2466..aa6fb4e 100644 --- a/src/combine.rs +++ b/src/combine.rs @@ -50,7 +50,7 @@ http://reveng.sourceforge.net/crc-catalogue/all.htm */ -use crate::structs::CrcParams; +use crate::CrcParams; /* Multiply the GF(2) vector vec by the GF(2) matrix mat, returning the resulting vector. The vector is stored as bits in a crc_t. The matrix is diff --git a/src/crc32/consts.rs b/src/crc32/consts.rs index 9c06015..27b5316 100644 --- a/src/crc32/consts.rs +++ b/src/crc32/consts.rs @@ -7,8 +7,8 @@ use crate::consts::{ NAME_CRC32_CD_ROM_EDC, NAME_CRC32_CKSUM, NAME_CRC32_ISCSI, NAME_CRC32_ISO_HDLC, NAME_CRC32_JAMCRC, NAME_CRC32_MEF, NAME_CRC32_MPEG_2, NAME_CRC32_XFER, }; -use crate::structs::CrcParams; use crate::CrcAlgorithm; +use crate::CrcParams; use crc::{ CRC_32_AIXM, CRC_32_AUTOSAR, CRC_32_BASE91_D, CRC_32_BZIP2, CRC_32_CD_ROM_EDC, CRC_32_CKSUM, CRC_32_ISCSI, CRC_32_ISO_HDLC, CRC_32_JAMCRC, CRC_32_MEF, CRC_32_MPEG_2, CRC_32_XFER, diff --git a/src/crc64/consts.rs b/src/crc64/consts.rs index c68e665..662a481 100644 --- a/src/crc64/consts.rs +++ b/src/crc64/consts.rs @@ -3,8 +3,8 @@ #![allow(dead_code)] use crate::consts::*; -use crate::structs::CrcParams; use crate::CrcAlgorithm; +use crate::CrcParams; use crc::{CRC_64_ECMA_182, CRC_64_GO_ISO, CRC_64_MS, CRC_64_REDIS, CRC_64_WE, CRC_64_XZ}; // width=64 poly=0x42f0e1eba9ea3693 init=0x0000000000000000 refin=false refout=false xorout=0x0000000000000000 check=0x6c40df5f0b497347 residue=0x0000000000000000 name="CRC-64/ECMA-182" diff --git a/src/ffi.rs b/src/ffi.rs index 4311a60..3399d25 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -7,8 +7,8 @@ #![cfg(any(target_arch = "aarch64", target_arch = "x86_64", target_arch = "x86"))] -use crate::structs::CrcParams; use crate::CrcAlgorithm; +use crate::CrcParams; use crate::{get_calculator_target, Digest}; use std::ffi::CStr; use std::os::raw::c_char; diff --git a/src/lib.rs b/src/lib.rs index 7c23cff..9ce8c90 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -116,7 +116,7 @@ use crate::crc32::fusion; use crate::crc64::consts::{ CRC64_ECMA_182, CRC64_GO_ISO, CRC64_MS, CRC64_NVME, CRC64_REDIS, CRC64_WE, CRC64_XZ, }; -use crate::structs::{Calculator, CrcParams}; +use crate::structs::Calculator; use crate::traits::CrcCalculator; use digest::{DynDigest, InvalidBufferSize}; use std::fs::File; @@ -161,6 +161,21 @@ pub enum CrcAlgorithm { Crc64Xz, } +/// Parameters for CRC computation, including polynomial, initial value, and other settings. +#[derive(Clone, Copy, Debug)] +pub struct CrcParams { + pub algorithm: CrcAlgorithm, + pub name: &'static str, + pub width: u8, + pub poly: u64, + pub init: u64, + pub refin: bool, + pub refout: bool, + pub xorout: u64, + pub check: u64, + pub keys: [u64; 23], +} + /// Type alias for a function pointer that represents a CRC calculation function. /// /// The function takes the following parameters: @@ -493,9 +508,9 @@ pub fn checksum_combine_with_custom_params( } /// Returns the custom CRC parameters for a given set of Rocksoft CRC parameters. -/// +/// /// Does not support mis-matched refin/refout parameters, so both must be true or both false. -/// +/// /// Rocksoft parameters for lots of variants: https://reveng.sourceforge.io/crc-catalogue/all.htm pub fn get_custom_params( name: &'static str, @@ -614,8 +629,7 @@ mod lib { use super::*; use crate::test::consts::{TEST_ALL_CONFIGS, TEST_CHECK_STRING}; use crate::test::enums::AnyCrcTestConfig; - use crate::CrcAlgorithm::Crc32Iscsi; - use cbindgen::Language::{Cxx, C}; + use cbindgen::Language::C; use cbindgen::Style::Both; use rand::{rng, Rng}; use std::fs::{read, write}; diff --git a/src/structs.rs b/src/structs.rs index fc4c152..fc44f2c 100644 --- a/src/structs.rs +++ b/src/structs.rs @@ -2,24 +2,8 @@ #![allow(dead_code)] -use crate::arch; use crate::traits::{CrcCalculator, CrcWidth}; -use crate::CrcAlgorithm; - -#[derive(Clone, Copy, Debug)] -pub struct CrcParams { - pub algorithm: CrcAlgorithm, - pub name: &'static str, - pub width: u8, - pub poly: u64, - pub init: u64, - pub refin: bool, - pub refout: bool, - pub xorout: u64, - pub check: u64, - pub keys: [u64; 23], -} - +use crate::{arch, CrcParams}; /// CRC-32 width implementation #[derive(Clone, Copy)] pub struct Width32; diff --git a/src/test/enums.rs b/src/test/enums.rs index d7f7e01..413ddea 100644 --- a/src/test/enums.rs +++ b/src/test/enums.rs @@ -3,9 +3,9 @@ #![cfg(test)] #![allow(dead_code)] -use crate::structs::CrcParams; use crate::test::structs::*; use crate::CrcAlgorithm; +use crate::CrcParams; use crc::Crc; pub enum AnyCrcTestConfig { diff --git a/src/test/structs.rs b/src/test/structs.rs index d11a04a..4e11b68 100644 --- a/src/test/structs.rs +++ b/src/test/structs.rs @@ -3,7 +3,7 @@ #![cfg(test)] #![allow(dead_code)] -use crate::structs::CrcParams; +use crate::CrcParams; use crc::{Crc, Table}; pub struct CrcTestConfig { diff --git a/src/traits.rs b/src/traits.rs index 4511d43..7eec0b5 100644 --- a/src/traits.rs +++ b/src/traits.rs @@ -5,7 +5,7 @@ #[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))] use crate::enums::Reflector; -use crate::structs::CrcParams; +use crate::CrcParams; #[cfg(any(target_arch = "x86", target_arch = "x86_64", target_arch = "aarch64"))] use crate::structs::CrcState; From 8c97d7e65cab87d1b2615a619c6dfe3f55cb49c9 Mon Sep 17 00:00:00 2001 From: Don MacAskill Date: Wed, 11 Jun 2025 11:29:28 -0700 Subject: [PATCH 08/11] Update custom CRC params support in FFI - Adds the Crc32Custom and Crc64Custom CrcFastAlgorithms - Adds the crc_fast_checksum_combine_with_custom_params() function - Adds the crc_fast_get_custom_params() function --- libcrc_fast.h | 21 +++++++++++++++++ src/ffi.rs | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/libcrc_fast.h b/libcrc_fast.h index 6677535..7d93499 100644 --- a/libcrc_fast.h +++ b/libcrc_fast.h @@ -19,12 +19,14 @@ typedef enum CrcFastAlgorithm { Crc32Bzip2, Crc32CdRomEdc, Crc32Cksum, + Crc32Custom, Crc32Iscsi, Crc32IsoHdlc, Crc32Jamcrc, Crc32Mef, Crc32Mpeg2, Crc32Xfer, + Crc64Custom, Crc64Ecma182, Crc64GoIso, Crc64Ms, @@ -149,6 +151,25 @@ uint64_t crc_fast_checksum_combine(enum CrcFastAlgorithm algorithm, uint64_t checksum2, uint64_t checksum2_len); +/** + * Combine two CRC checksums using custom parameters + */ +uint64_t crc_fast_checksum_combine_with_custom_params(struct CrcFastParams params, + uint64_t checksum1, + uint64_t checksum2, + uint64_t checksum2_len); + +/** + * Returns the custom CRC parameters for a given set of Rocksoft CRC parameters + */ +struct CrcFastParams crc_fast_get_custom_params(const char *name_ptr, + uint8_t width, + uint64_t poly, + uint64_t init, + bool reflected, + uint64_t xorout, + uint64_t check); + /** * Gets the target build properties (CPU architecture and fine-tuning parameters) for this algorithm */ diff --git a/src/ffi.rs b/src/ffi.rs index 3399d25..5bccafe 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -27,12 +27,14 @@ pub enum CrcFastAlgorithm { Crc32Bzip2, Crc32CdRomEdc, Crc32Cksum, + Crc32Custom, Crc32Iscsi, Crc32IsoHdlc, Crc32Jamcrc, Crc32Mef, Crc32Mpeg2, Crc32Xfer, + Crc64Custom, Crc64Ecma182, Crc64GoIso, Crc64Ms, @@ -52,12 +54,14 @@ impl From for CrcAlgorithm { CrcFastAlgorithm::Crc32Bzip2 => CrcAlgorithm::Crc32Bzip2, CrcFastAlgorithm::Crc32CdRomEdc => CrcAlgorithm::Crc32CdRomEdc, CrcFastAlgorithm::Crc32Cksum => CrcAlgorithm::Crc32Cksum, + CrcFastAlgorithm::Crc32Custom => CrcAlgorithm::Crc32Custom, CrcFastAlgorithm::Crc32Iscsi => CrcAlgorithm::Crc32Iscsi, CrcFastAlgorithm::Crc32IsoHdlc => CrcAlgorithm::Crc32IsoHdlc, CrcFastAlgorithm::Crc32Jamcrc => CrcAlgorithm::Crc32Jamcrc, CrcFastAlgorithm::Crc32Mef => CrcAlgorithm::Crc32Mef, CrcFastAlgorithm::Crc32Mpeg2 => CrcAlgorithm::Crc32Mpeg2, CrcFastAlgorithm::Crc32Xfer => CrcAlgorithm::Crc32Xfer, + CrcFastAlgorithm::Crc64Custom => CrcAlgorithm::Crc64Custom, CrcFastAlgorithm::Crc64Ecma182 => CrcAlgorithm::Crc64Ecma182, CrcFastAlgorithm::Crc64GoIso => CrcAlgorithm::Crc64GoIso, CrcFastAlgorithm::Crc64Ms => CrcAlgorithm::Crc64Ms, @@ -310,6 +314,66 @@ pub extern "C" fn crc_fast_checksum_combine( crate::checksum_combine(algorithm.into(), checksum1, checksum2, checksum2_len) } +/// Combine two CRC checksums using custom parameters +#[no_mangle] +pub extern "C" fn crc_fast_checksum_combine_with_custom_params( + params: CrcFastParams, + checksum1: u64, + checksum2: u64, + checksum2_len: u64, +) -> u64 { + crate::checksum_combine_with_custom_params(params.into(), checksum1, checksum2, checksum2_len) +} + +/// Returns the custom CRC parameters for a given set of Rocksoft CRC parameters +#[no_mangle] +pub extern "C" fn crc_fast_get_custom_params( + name_ptr: *const c_char, + width: u8, + poly: u64, + init: u64, + reflected: bool, + xorout: u64, + check: u64, +) -> CrcFastParams { + let name = if name_ptr.is_null() { + "custom" + } else { + unsafe { + CStr::from_ptr(name_ptr).to_str().unwrap_or("custom") + } + }; + + // Get the custom params from the library + let params = crate::get_custom_params( + // We need to use a static string for the name field + Box::leak(name.to_string().into_boxed_str()), + width, + poly, + init, + reflected, + xorout, + check, + ); + + // Convert to FFI struct + CrcFastParams { + algorithm: match width { + 32 => CrcFastAlgorithm::Crc32Custom, + 64 => CrcFastAlgorithm::Crc64Custom, + _ => panic!("Unsupported width: {}", width), + }, + width: params.width, + poly: params.poly, + init: params.init, + refin: params.refin, + refout: params.refout, + xorout: params.xorout, + check: params.check, + keys: params.keys, + } +} + /// Gets the target build properties (CPU architecture and fine-tuning parameters) for this algorithm #[no_mangle] pub extern "C" fn crc_fast_get_calculator_target(algorithm: CrcFastAlgorithm) -> *const c_char { From caa183709de805190fda028f28419ac97443d10f Mon Sep 17 00:00:00 2001 From: Don MacAskill Date: Wed, 11 Jun 2025 11:55:54 -0700 Subject: [PATCH 09/11] Add custom params support to the software fallback --- src/arch/software.rs | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/src/arch/software.rs b/src/arch/software.rs index 5b8d32a..4739a1b 100644 --- a/src/arch/software.rs +++ b/src/arch/software.rs @@ -7,7 +7,7 @@ use crate::consts::CRC_64_NVME; use crate::CrcAlgorithm; use crate::CrcParams; -use crc::Table; +use crc::{Algorithm, Table}; const RUST_CRC32_AIXM: crc::Crc> = crc::Crc::>::new(&crc::CRC_32_AIXM); @@ -78,6 +78,23 @@ pub(crate) fn update(state: u64, data: &[u8], params: CrcParams) -> u64 { CrcAlgorithm::Crc32Mef => RUST_CRC32_MEF, CrcAlgorithm::Crc32Mpeg2 => RUST_CRC32_MPEG_2, CrcAlgorithm::Crc32Xfer => RUST_CRC32_XFER, + CrcAlgorithm::Crc32Custom => { + let algorithm: Algorithm = Algorithm { + width: params.width as u8, + poly: params.poly as u32, + init: params.init as u32, + refin: params.refin, + refout: params.refout, + xorout: params.xorout as u32, + check: params.check as u32, + residue: 0x00000000, // unused in this context + }; + + // ugly, but the crc crate is difficult to work with... + let static_algorithm = Box::leak(Box::new(algorithm)); + + crc::Crc::>::new(static_algorithm) + } _ => panic!("Invalid algorithm for u32 CRC"), }; update_u32(state as u32, data, params) as u64 @@ -91,6 +108,23 @@ pub(crate) fn update(state: u64, data: &[u8], params: CrcParams) -> u64 { CrcAlgorithm::Crc64Redis => RUST_CRC64_REDIS, CrcAlgorithm::Crc64We => RUST_CRC64_WE, CrcAlgorithm::Crc64Xz => RUST_CRC64_XZ, + CrcAlgorithm::Crc64Custom => { + let algorithm: Algorithm = Algorithm { + width: params.width as u8, + poly: params.poly as u64, + init: params.init as u64, + refin: params.refin, + refout: params.refout, + xorout: params.xorout as u64, + check: params.check as u64, + residue: 0x0000000000000000, // unused in this context + }; + + // ugly, but the crc crate is difficult to work with... + let static_algorithm = Box::leak(Box::new(algorithm)); + + crc::Crc::>::new(static_algorithm) + } _ => panic!("Invalid algorithm for u64 CRC"), }; update_u64(state, data, params) From d771b8ea72b63a7babbb207169defa8d4ef7f67a Mon Sep 17 00:00:00 2001 From: Don MacAskill Date: Wed, 11 Jun 2025 11:56:00 -0700 Subject: [PATCH 10/11] Fix formatting --- src/ffi.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/ffi.rs b/src/ffi.rs index 5bccafe..f5cdf3e 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -339,9 +339,7 @@ pub extern "C" fn crc_fast_get_custom_params( let name = if name_ptr.is_null() { "custom" } else { - unsafe { - CStr::from_ptr(name_ptr).to_str().unwrap_or("custom") - } + unsafe { CStr::from_ptr(name_ptr).to_str().unwrap_or("custom") } }; // Get the custom params from the library From 786b4df15fb2f5a268fec6be2e49f158d80e295f Mon Sep 17 00:00:00 2001 From: Don MacAskill Date: Thu, 12 Jun 2025 14:15:18 -0700 Subject: [PATCH 11/11] Enable custom parameters via CrcParams constructor Replaces get_custom_params() with CrcParams::new(). https://github.com/awesomized/crc-fast-rust/pull/11#discussion_r2141600675 --- src/bin/get-custom-params.rs | 2 +- src/ffi.rs | 2 +- src/lib.rs | 191 ++++++++++++++++++++--------------- src/structs.rs | 42 +++++++- 4 files changed, 154 insertions(+), 83 deletions(-) diff --git a/src/bin/get-custom-params.rs b/src/bin/get-custom-params.rs index 7b98aa0..ab8c7f9 100644 --- a/src/bin/get-custom-params.rs +++ b/src/bin/get-custom-params.rs @@ -167,7 +167,7 @@ fn main() -> ExitCode { let static_name: &'static str = Box::leak(config.name.unwrap().into_boxed_str()); - let params = crc_fast::get_custom_params( + let params = crc_fast::CrcParams::new( static_name, config.width.unwrap() as u8, config.polynomial.unwrap(), diff --git a/src/ffi.rs b/src/ffi.rs index f5cdf3e..abc7a7b 100644 --- a/src/ffi.rs +++ b/src/ffi.rs @@ -343,7 +343,7 @@ pub extern "C" fn crc_fast_get_custom_params( }; // Get the custom params from the library - let params = crate::get_custom_params( + let params = CrcParams::new( // We need to use a static string for the name field Box::leak(name.to_string().into_boxed_str()), width, diff --git a/src/lib.rs b/src/lib.rs index 9ce8c90..790eca6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -507,42 +507,6 @@ pub fn checksum_combine_with_custom_params( combine::checksums(checksum1, checksum2, checksum2_len, params) } -/// Returns the custom CRC parameters for a given set of Rocksoft CRC parameters. -/// -/// Does not support mis-matched refin/refout parameters, so both must be true or both false. -/// -/// Rocksoft parameters for lots of variants: https://reveng.sourceforge.io/crc-catalogue/all.htm -pub fn get_custom_params( - name: &'static str, - width: u8, - poly: u64, - init: u64, - reflected: bool, - xorout: u64, - check: u64, -) -> CrcParams { - let keys = generate::keys(width, poly, reflected); - - let algorithm = match width { - 32 => CrcAlgorithm::Crc32Custom, - 64 => CrcAlgorithm::Crc64Custom, - _ => panic!("Unsupported width: {}", width), - }; - - CrcParams { - algorithm, - name, - width, - poly, - init, - refin: reflected, - refout: reflected, - xorout, - check, - keys, - } -} - /// Returns the target used to calculate the CRC checksum for the specified algorithm. /// /// These strings are informational only, not stable, and shouldn't be relied on to match across @@ -569,7 +533,7 @@ fn get_calculator_params(algorithm: CrcAlgorithm) -> (CalculatorFn, CrcParams) { CrcAlgorithm::Crc32CdRomEdc => (Calculator::calculate as CalculatorFn, CRC32_CD_ROM_EDC), CrcAlgorithm::Crc32Cksum => (Calculator::calculate as CalculatorFn, CRC32_CKSUM), CrcAlgorithm::Crc32Custom => { - panic!("Custom CRC-32 requires parameters via get_custom_params()") + panic!("Custom CRC-32 requires parameters via CrcParams::new()") } CrcAlgorithm::Crc32Iscsi => (crc32_iscsi_calculator as CalculatorFn, CRC32_ISCSI), CrcAlgorithm::Crc32IsoHdlc => (crc32_iso_hdlc_calculator as CalculatorFn, CRC32_ISO_HDLC), @@ -578,7 +542,7 @@ fn get_calculator_params(algorithm: CrcAlgorithm) -> (CalculatorFn, CrcParams) { CrcAlgorithm::Crc32Mpeg2 => (Calculator::calculate as CalculatorFn, CRC32_MPEG_2), CrcAlgorithm::Crc32Xfer => (Calculator::calculate as CalculatorFn, CRC32_XFER), CrcAlgorithm::Crc64Custom => { - panic!("Custom CRC-64 requires parameters via get_custom_params()") + panic!("Custom CRC-64 requires parameters via CrcParams::new()") } CrcAlgorithm::Crc64Ecma182 => (Calculator::calculate as CalculatorFn, CRC64_ECMA_182), CrcAlgorithm::Crc64GoIso => (Calculator::calculate as CalculatorFn, CRC64_GO_ISO), @@ -658,59 +622,49 @@ mod lib { fn test_checksum_with_custom_params() { // CRC-32 reflected assert_eq!( - checksum_with_params(CRC32_ISCSI, TEST_CHECK_STRING), + checksum_with_params(get_custom_crc32_reflected(), TEST_CHECK_STRING), CRC32_ISCSI.check, ); // CRC-32 forward assert_eq!( - checksum_with_params(CRC32_BZIP2, TEST_CHECK_STRING), + checksum_with_params(get_custom_crc32_forward(), TEST_CHECK_STRING), CRC32_BZIP2.check, ); // CRC-64 reflected assert_eq!( - checksum_with_params(CRC64_NVME, TEST_CHECK_STRING), + checksum_with_params(get_custom_crc64_reflected(), TEST_CHECK_STRING), CRC64_NVME.check, ); // CRC-64 forward assert_eq!( - checksum_with_params(CRC64_ECMA_182, TEST_CHECK_STRING), + checksum_with_params(get_custom_crc64_forward(), TEST_CHECK_STRING), CRC64_ECMA_182.check, ); } #[test] fn test_get_custom_params() { - let custom_crc32 = get_custom_params( - "Custom CRC-32/ISCSI", - 32, - CRC32_ISCSI.poly, - CRC32_ISCSI.init, - CRC32_ISCSI.refin, - CRC32_ISCSI.xorout, + assert_eq!( + checksum_with_params(get_custom_crc32_reflected(), TEST_CHECK_STRING), CRC32_ISCSI.check, ); assert_eq!( - checksum_with_params(custom_crc32, TEST_CHECK_STRING), - CRC32_ISCSI.check, + checksum_with_params(get_custom_crc32_forward(), TEST_CHECK_STRING), + CRC32_BZIP2.check, ); - let custom_crc64 = get_custom_params( - "Custom CRC-64/NVME", - 64, - CRC64_NVME.poly, - CRC64_NVME.init, - CRC64_NVME.refin, - CRC64_NVME.xorout, + assert_eq!( + checksum_with_params(get_custom_crc64_reflected(), TEST_CHECK_STRING), CRC64_NVME.check, ); assert_eq!( - checksum_with_params(custom_crc64, TEST_CHECK_STRING), - CRC64_NVME.check, + checksum_with_params(get_custom_crc64_forward(), TEST_CHECK_STRING), + CRC64_ECMA_182.check, ); } @@ -724,17 +678,26 @@ mod lib { #[test] fn test_digest_updates_check_with_custom_params() { // CRC-32 reflected - check_digest(Digest::new_with_params(CRC32_ISCSI), CRC32_ISCSI.check); + check_digest( + Digest::new_with_params(get_custom_crc32_reflected()), + CRC32_ISCSI.check, + ); // CRC-32 forward - check_digest(Digest::new_with_params(CRC32_BZIP2), CRC32_BZIP2.check); + check_digest( + Digest::new_with_params(get_custom_crc32_forward()), + CRC32_BZIP2.check, + ); // CRC-64 reflected - check_digest(Digest::new_with_params(CRC64_NVME), CRC64_NVME.check); + check_digest( + Digest::new_with_params(get_custom_crc64_reflected()), + CRC64_NVME.check, + ); // CRC-64 forward check_digest( - Digest::new_with_params(CRC64_ECMA_182), + Digest::new_with_params(get_custom_crc64_forward()), CRC64_ECMA_182.check, ); } @@ -825,34 +788,38 @@ mod lib { #[test] fn test_combine_with_custom_params() { // CRC-32 reflected - let checksum1 = checksum_with_params(CRC32_ISCSI, "1234".as_ref()); - let checksum2 = checksum_with_params(CRC32_ISCSI, "56789".as_ref()); + let crc32_params = get_custom_crc32_reflected(); + let checksum1 = checksum_with_params(crc32_params, "1234".as_ref()); + let checksum2 = checksum_with_params(crc32_params, "56789".as_ref()); assert_eq!( - checksum_combine_with_custom_params(CRC32_ISCSI, checksum1, checksum2, 5), + checksum_combine_with_custom_params(crc32_params, checksum1, checksum2, 5), CRC32_ISCSI.check, ); // CRC-32 forward - let checksum1 = checksum_with_params(CRC32_BZIP2, "1234".as_ref()); - let checksum2 = checksum_with_params(CRC32_BZIP2, "56789".as_ref()); + let crc32_params = get_custom_crc32_forward(); + let checksum1 = checksum_with_params(crc32_params, "1234".as_ref()); + let checksum2 = checksum_with_params(crc32_params, "56789".as_ref()); assert_eq!( - checksum_combine_with_custom_params(CRC32_BZIP2, checksum1, checksum2, 5), + checksum_combine_with_custom_params(crc32_params, checksum1, checksum2, 5), CRC32_BZIP2.check, ); // CRC-64 reflected - let checksum1 = checksum_with_params(CRC64_NVME, "1234".as_ref()); - let checksum2 = checksum_with_params(CRC64_NVME, "56789".as_ref()); + let crc64_params = get_custom_crc64_reflected(); + let checksum1 = checksum_with_params(crc64_params, "1234".as_ref()); + let checksum2 = checksum_with_params(crc64_params, "56789".as_ref()); assert_eq!( - checksum_combine_with_custom_params(CRC64_NVME, checksum1, checksum2, 5), + checksum_combine_with_custom_params(crc64_params, checksum1, checksum2, 5), CRC64_NVME.check, ); // CRC-64 forward - let checksum1 = checksum_with_params(CRC64_ECMA_182, "1234".as_ref()); - let checksum2 = checksum_with_params(CRC64_ECMA_182, "56789".as_ref()); + let crc64_params = get_custom_crc64_forward(); + let checksum1 = checksum_with_params(crc64_params, "1234".as_ref()); + let checksum2 = checksum_with_params(crc64_params, "56789".as_ref()); assert_eq!( - checksum_combine_with_custom_params(CRC64_ECMA_182, checksum1, checksum2, 5), + checksum_combine_with_custom_params(crc64_params, checksum1, checksum2, 5), CRC64_ECMA_182.check, ); } @@ -886,16 +853,32 @@ mod lib { } // CRC-32 reflected - check_file(CRC32_ISCSI, test_file_path, CRC32_ISCSI.check); + check_file( + get_custom_crc32_reflected(), + test_file_path, + CRC32_ISCSI.check, + ); // CRC-32 forward - check_file(CRC32_BZIP2, test_file_path, CRC32_BZIP2.check); + check_file( + get_custom_crc32_forward(), + test_file_path, + CRC32_BZIP2.check, + ); // CRC-64 reflected - check_file(CRC64_NVME, test_file_path, CRC64_NVME.check); + check_file( + get_custom_crc64_reflected(), + test_file_path, + CRC64_NVME.check, + ); // CRC-64 forward - check_file(CRC64_ECMA_182, test_file_path, CRC64_ECMA_182.check); + check_file( + get_custom_crc64_forward(), + test_file_path, + CRC64_ECMA_182.check, + ); std::fs::remove_file(test_file_path).unwrap(); } @@ -1058,4 +1041,52 @@ mod lib { Ok(()) } + + fn get_custom_crc32_reflected() -> CrcParams { + CrcParams::new( + "Custom CRC-32/ISCSI", + 32, + CRC32_ISCSI.poly, + CRC32_ISCSI.init, + CRC32_ISCSI.refin, + CRC32_ISCSI.xorout, + CRC32_ISCSI.check, + ) + } + + fn get_custom_crc32_forward() -> CrcParams { + CrcParams::new( + "Custom CRC-32/BZIP2", + 32, + CRC32_BZIP2.poly, + CRC32_BZIP2.init, + CRC32_BZIP2.refin, + CRC32_BZIP2.xorout, + CRC32_BZIP2.check, + ) + } + + fn get_custom_crc64_reflected() -> CrcParams { + CrcParams::new( + "Custom CRC-64/NVME", + 64, + CRC64_NVME.poly, + CRC64_NVME.init, + CRC64_NVME.refin, + CRC64_NVME.xorout, + CRC64_NVME.check, + ) + } + + fn get_custom_crc64_forward() -> CrcParams { + CrcParams::new( + "Custom CRC-64/ECMA-182", + 64, + CRC64_ECMA_182.poly, + CRC64_ECMA_182.init, + CRC64_ECMA_182.refin, + CRC64_ECMA_182.xorout, + CRC64_ECMA_182.check, + ) + } } diff --git a/src/structs.rs b/src/structs.rs index fc44f2c..a7ad4d5 100644 --- a/src/structs.rs +++ b/src/structs.rs @@ -3,10 +3,12 @@ #![allow(dead_code)] use crate::traits::{CrcCalculator, CrcWidth}; -use crate::{arch, CrcParams}; +use crate::{arch, generate, CrcAlgorithm, CrcParams}; + /// CRC-32 width implementation #[derive(Clone, Copy)] pub struct Width32; + impl CrcWidth for Width32 { const WIDTH: u32 = 32; type Value = u32; @@ -36,3 +38,41 @@ impl CrcCalculator for Calculator { unsafe { arch::update(state, data, params) } } } + +impl CrcParams { + /// Creates custom CRC parameters for a given set of Rocksoft CRC parameters. + /// + /// Does not support mis-matched refin/refout parameters, so both must be true or both false. + /// + /// Rocksoft parameters for lots of variants: https://reveng.sourceforge.io/crc-catalogue/all.htm + pub fn new( + name: &'static str, + width: u8, + poly: u64, + init: u64, + reflected: bool, + xorout: u64, + check: u64, + ) -> Self { + let keys = generate::keys(width, poly, reflected); + + let algorithm = match width { + 32 => CrcAlgorithm::Crc32Custom, + 64 => CrcAlgorithm::Crc64Custom, + _ => panic!("Unsupported width: {}", width), + }; + + Self { + algorithm, + name, + width, + poly, + init, + refin: reflected, + refout: reflected, + xorout, + check, + keys, + } + } +}