lc_framework/
lib.rs

1//! [![CI Status]][workflow] [![MSRV]][repo] [![Latest Version]][crates.io]
2//! [![Rust Doc Crate]][docs.rs] [![Rust Doc Main]][docs]
3//!
4//! [CI Status]: https://img.shields.io/github/actions/workflow/status/juntyr/lc-framework-rs/ci.yml?branch=main
5//! [workflow]: https://github.com/juntyr/lc-framework-rs/actions/workflows/ci.yml?query=branch%3Amain
6//!
7//! [MSRV]: https://img.shields.io/badge/MSRV-1.85.0-blue
8//! [repo]: https://github.com/juntyr/lc-framework-rs
9//!
10//! [Latest Version]: https://img.shields.io/crates/v/lc-framework
11//! [crates.io]: https://crates.io/crates/lc-framework
12//!
13//! [Rust Doc Crate]: https://img.shields.io/docsrs/lc-framework
14//! [docs.rs]: https://docs.rs/lc-framework/
15//!
16//! [Rust Doc Main]: https://img.shields.io/badge/docs-main-blue
17//! [docs]: https://juntyr.github.io/lc-framework-rs/lc_framework
18//!
19//! # lc-framework
20//!
21//! High-level Rust bindigs to the [LC] compression framework.
22//!
23//! [LC]: https://github.com/burtscher/LC-framework
24
25use std::ffi::c_longlong;
26
27/// Maximum number of components
28pub const MAX_COMPONENTS: usize = lc_framework_sys::MAX_STAGES;
29
30/// Maximum number of bytes
31pub const MAX_BYTES: usize = const {
32    #[allow(clippy::cast_possible_truncation)]
33    if std::mem::size_of::<c_longlong>() <= std::mem::size_of::<usize>() {
34        c_longlong::MAX as usize
35    } else {
36        usize::MAX
37    }
38};
39
40/// Compress the `input` data with LC using zero or more `preprocessors` and
41/// one or more `components`.
42///
43/// # Errors
44///
45/// Errors with
46/// - [`Error::TooFewComponents`] if no `components` are given
47/// - [`Error::TooManyComponents`] if too many `components` are given
48/// - [`Error::ExcessiveInputData`] if the `input` data is too large
49/// - [`Error::CompressionFailed`] if compression with LC failed
50/// - [`Error::ExcessiveCompressedData`] if the compressed data is too large
51pub fn compress(
52    preprocessors: &[Preprocessor],
53    components: &[Component],
54    input: &[u8],
55) -> Result<Vec<u8>, Error> {
56    let mut preprocessor_ids = Vec::with_capacity(preprocessors.len());
57    let mut preprocessor_params = Vec::new();
58    let mut preprocessor_params_num = Vec::with_capacity(preprocessors.len());
59    for preprocessor in preprocessors {
60        preprocessor_ids.push(preprocessor.as_id());
61        let preprocessor_nparams_sum = preprocessor_params.len();
62        preprocessor.push_params(&mut preprocessor_params);
63        preprocessor_params_num.push(preprocessor_params.len() - preprocessor_nparams_sum);
64    }
65
66    if components.is_empty() {
67        return Err(Error::TooFewComponents);
68    }
69
70    if components.len() > MAX_COMPONENTS {
71        return Err(Error::TooManyComponents);
72    }
73
74    let component_ids = components
75        .iter()
76        .copied()
77        .map(Component::as_id)
78        .collect::<Vec<_>>();
79
80    let input_size: c_longlong = input
81        .len()
82        .try_into()
83        .map_err(|_| Error::ExcessiveInputData)?;
84
85    let mut encoded_ptr = std::ptr::null_mut();
86    let mut encoded_size = 0;
87
88    #[expect(unsafe_code)]
89    // SAFETY: all pointers and lengths are valid
90    let status = unsafe {
91        lc_framework_sys::lc_compress(
92            preprocessor_ids.len(),
93            preprocessor_ids.as_ptr(),
94            preprocessor_params_num.as_ptr(),
95            preprocessor_params.as_ptr(),
96            component_ids.len(),
97            component_ids.as_ptr(),
98            input.as_ptr(),
99            input_size,
100            &raw mut encoded_ptr,
101            &raw mut encoded_size,
102        )
103    };
104
105    if status != 0 {
106        return Err(Error::CompressionFailed);
107    }
108
109    let encoded_len: usize = encoded_size
110        .try_into()
111        .map_err(|_| Error::ExcessiveCompressedData)?;
112
113    #[expect(unsafe_code)]
114    // SAFETY: all Vec elements are initialised via the copy
115    let encoded = unsafe {
116        let mut encoded = Vec::with_capacity(encoded_len);
117        std::ptr::copy_nonoverlapping(encoded_ptr.cast_const(), encoded.as_mut_ptr(), encoded_len);
118        encoded.set_len(encoded_len);
119        encoded
120    };
121
122    #[expect(unsafe_code)]
123    // SAFETY: encoded_ptr was allocated by LC
124    unsafe {
125        lc_framework_sys::lc_free_bytes(encoded_ptr);
126    }
127
128    Ok(encoded)
129}
130
131/// Dempress the `compressed` data with LC using zero or more `preprocessors`
132/// and one or more `components`.
133///
134/// The `compressed` data must have been [`compress`]ed using the same
135/// `preprocessors` and `components`.
136///
137/// # Errors
138///
139/// Errors with
140/// - [`Error::TooFewComponents`] if no `components` are given
141/// - [`Error::TooManyComponents`] if too many `components` are given
142/// - [`Error::ExcessiveCompressedData`] if the `compressed` data is too large
143/// - [`Error::DecompressionFailed`] if decompression with LC failed
144/// - [`Error::ExcessiveDecompressedData`] if the decompressed data is too
145///   large
146pub fn decompress(
147    preprocessors: &[Preprocessor],
148    components: &[Component],
149    compressed: &[u8],
150) -> Result<Vec<u8>, Error> {
151    let encoded = compressed;
152
153    let mut preprocessor_ids = Vec::with_capacity(preprocessors.len());
154    let mut preprocessor_params = Vec::new();
155    let mut preprocessor_params_num = Vec::with_capacity(preprocessors.len());
156    for preprocessor in preprocessors {
157        preprocessor_ids.push(preprocessor.as_id());
158        let preprocessor_nparams_sum = preprocessor_params.len();
159        preprocessor.push_params(&mut preprocessor_params);
160        preprocessor_params_num.push(preprocessor_params.len() - preprocessor_nparams_sum);
161    }
162
163    if components.is_empty() {
164        return Err(Error::TooFewComponents);
165    }
166
167    if components.len() > MAX_COMPONENTS {
168        return Err(Error::TooManyComponents);
169    }
170
171    let component_ids = components
172        .iter()
173        .copied()
174        .map(Component::as_id)
175        .collect::<Vec<_>>();
176
177    let encoded_size: c_longlong = encoded
178        .len()
179        .try_into()
180        .map_err(|_| Error::ExcessiveCompressedData)?;
181
182    let mut decoded_ptr = std::ptr::null_mut();
183    let mut decoded_size = 0;
184
185    #[expect(unsafe_code)]
186    // SAFETY: all pointers and lengths are valid
187    let status = unsafe {
188        lc_framework_sys::lc_decompress(
189            preprocessor_ids.len(),
190            preprocessor_ids.as_ptr(),
191            preprocessor_params_num.as_ptr(),
192            preprocessor_params.as_ptr(),
193            component_ids.len(),
194            component_ids.as_ptr(),
195            encoded.as_ptr(),
196            encoded_size,
197            &raw mut decoded_ptr,
198            &raw mut decoded_size,
199        )
200    };
201
202    if status != 0 {
203        return Err(Error::DecompressionFailed);
204    }
205
206    let decoded_len: usize = decoded_size
207        .try_into()
208        .map_err(|_| Error::ExcessiveDecompressedData)?;
209
210    #[expect(unsafe_code)]
211    // SAFETY: all Vec elements are initialised via the copy
212    let decoded = unsafe {
213        let mut decoded = Vec::with_capacity(decoded_len);
214        std::ptr::copy_nonoverlapping(decoded_ptr.cast_const(), decoded.as_mut_ptr(), decoded_len);
215        decoded.set_len(decoded_len);
216        decoded
217    };
218
219    #[expect(unsafe_code)]
220    // SAFETY: encoded_ptr was allocated by LC
221    unsafe {
222        lc_framework_sys::lc_free_bytes(decoded_ptr);
223    }
224
225    Ok(decoded)
226}
227
228#[derive(Debug, thiserror::Error)]
229/// Errors that can occur during compression and decompression with LC
230pub enum Error {
231    /// at least one component must be given
232    #[error("at least one component must be given")]
233    TooFewComponents,
234    /// too many components were given
235    #[error("at most {MAX_COMPONENTS} components must be given")]
236    TooManyComponents,
237    /// input data is too large
238    #[error("input data must not exceed {MAX_BYTES} bytes")]
239    ExcessiveInputData,
240    /// internal compression error
241    #[error("internal compression error")]
242    CompressionFailed,
243    /// compressed data is too large
244    #[error("compressed data must not exceed {MAX_BYTES} bytes")]
245    ExcessiveCompressedData,
246    /// internal decompression error
247    #[error("internal decompression error")]
248    DecompressionFailed,
249    /// decompressed data is too large
250    #[error("decompressed data must not exceed {MAX_BYTES} bytes")]
251    ExcessiveDecompressedData,
252}
253
254#[expect(missing_docs)]
255#[derive(Clone, Debug, PartialEq)]
256/// LC preprocessor
257pub enum Preprocessor {
258    Noop,
259    Lorenzo1D {
260        dtype: LorenzoDtype,
261    },
262    QuantizeErrorBound {
263        dtype: QuantizeDType,
264        kind: ErrorKind,
265        error_bound: f64,
266        threshold: Option<f64>,
267        decorrelation: Decorrelation,
268    },
269}
270
271impl Preprocessor {
272    const fn as_id(&self) -> lc_framework_sys::LC_CPUpreprocessor {
273        match self {
274            Self::Noop => lc_framework_sys::LC_CPUpreprocessor_NUL_CPUpreprocessor,
275            Self::Lorenzo1D {
276                dtype: LorenzoDtype::I32,
277            } => lc_framework_sys::LC_CPUpreprocessor_LOR1D_i32,
278            Self::QuantizeErrorBound {
279                dtype: QuantizeDType::F32,
280                kind: ErrorKind::Abs,
281                error_bound: _,
282                threshold: _,
283                decorrelation: Decorrelation::Zero,
284            } => lc_framework_sys::LC_CPUpreprocessor_QUANT_ABS_0_f32,
285            Self::QuantizeErrorBound {
286                dtype: QuantizeDType::F32,
287                kind: ErrorKind::Abs,
288                error_bound: _,
289                threshold: _,
290                decorrelation: Decorrelation::Random,
291            } => lc_framework_sys::LC_CPUpreprocessor_QUANT_ABS_R_f32,
292            Self::QuantizeErrorBound {
293                dtype: QuantizeDType::F32,
294                kind: ErrorKind::Noa,
295                error_bound: _,
296                threshold: _,
297                decorrelation: Decorrelation::Zero,
298            } => lc_framework_sys::LC_CPUpreprocessor_QUANT_NOA_0_f32,
299            Self::QuantizeErrorBound {
300                dtype: QuantizeDType::F32,
301                kind: ErrorKind::Noa,
302                error_bound: _,
303                threshold: _,
304                decorrelation: Decorrelation::Random,
305            } => lc_framework_sys::LC_CPUpreprocessor_QUANT_NOA_R_f32,
306            Self::QuantizeErrorBound {
307                dtype: QuantizeDType::F32,
308                kind: ErrorKind::Rel,
309                error_bound: _,
310                threshold: _,
311                decorrelation: Decorrelation::Zero,
312            } => lc_framework_sys::LC_CPUpreprocessor_QUANT_REL_0_f32,
313            Self::QuantizeErrorBound {
314                dtype: QuantizeDType::F32,
315                kind: ErrorKind::Rel,
316                error_bound: _,
317                threshold: _,
318                decorrelation: Decorrelation::Random,
319            } => lc_framework_sys::LC_CPUpreprocessor_QUANT_REL_R_f32,
320            Self::QuantizeErrorBound {
321                dtype: QuantizeDType::F64,
322                kind: ErrorKind::Abs,
323                error_bound: _,
324                threshold: _,
325                decorrelation: Decorrelation::Zero,
326            } => lc_framework_sys::LC_CPUpreprocessor_QUANT_ABS_0_f64,
327            Self::QuantizeErrorBound {
328                dtype: QuantizeDType::F64,
329                kind: ErrorKind::Abs,
330                error_bound: _,
331                threshold: _,
332                decorrelation: Decorrelation::Random,
333            } => lc_framework_sys::LC_CPUpreprocessor_QUANT_ABS_R_f64,
334            Self::QuantizeErrorBound {
335                dtype: QuantizeDType::F64,
336                kind: ErrorKind::Noa,
337                error_bound: _,
338                threshold: _,
339                decorrelation: Decorrelation::Zero,
340            } => lc_framework_sys::LC_CPUpreprocessor_QUANT_NOA_0_f64,
341            Self::QuantizeErrorBound {
342                dtype: QuantizeDType::F64,
343                kind: ErrorKind::Noa,
344                error_bound: _,
345                threshold: _,
346                decorrelation: Decorrelation::Random,
347            } => lc_framework_sys::LC_CPUpreprocessor_QUANT_NOA_R_f64,
348            Self::QuantizeErrorBound {
349                dtype: QuantizeDType::F64,
350                kind: ErrorKind::Rel,
351                error_bound: _,
352                threshold: _,
353                decorrelation: Decorrelation::Zero,
354            } => lc_framework_sys::LC_CPUpreprocessor_QUANT_REL_0_f64,
355            Self::QuantizeErrorBound {
356                dtype: QuantizeDType::F64,
357                kind: ErrorKind::Rel,
358                error_bound: _,
359                threshold: _,
360                decorrelation: Decorrelation::Random,
361            } => lc_framework_sys::LC_CPUpreprocessor_QUANT_REL_R_f64,
362        }
363    }
364
365    fn push_params(&self, params: &mut Vec<f64>) {
366        match self {
367            Self::Noop | Self::Lorenzo1D { dtype: _ } => (),
368            Self::QuantizeErrorBound {
369                dtype: _,
370                kind: _,
371                error_bound,
372                threshold,
373                decorrelation: _,
374            } => {
375                params.push(*error_bound);
376                if let Some(threshold) = threshold {
377                    params.push(*threshold);
378                }
379            }
380        }
381    }
382}
383
384#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
385/// LC error bound kind
386pub enum ErrorKind {
387    /// pointwise absolute error bound
388    Abs,
389    /// pointwise normalised absolute / data-range-relative error bound
390    Noa,
391    /// pointwise relative error bound
392    Rel,
393}
394
395#[expect(missing_docs)]
396#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
397/// LC quantisation decorrelation mode
398pub enum Decorrelation {
399    Zero,
400    Random,
401}
402
403#[expect(missing_docs)]
404#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
405/// LC Lorenzo preprocessor dtype
406pub enum LorenzoDtype {
407    I32,
408}
409
410#[expect(missing_docs)]
411#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
412/// LC quantization dtype
413pub enum QuantizeDType {
414    F32,
415    F64,
416}
417
418#[expect(missing_docs)]
419#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
420/// LC component
421pub enum Component {
422    Noop,
423    // mutators
424    TwosComplementToSignMagnitude { size: ElemSize },
425    TwosComplementToNegaBinary { size: ElemSize },
426    DebiasedExponentFractionSign { size: FloatSize },
427    DebiasedExponentSignFraction { size: FloatSize },
428    // shufflers
429    BitShuffle { size: ElemSize },
430    Tuple { size: TupleSize },
431    // predictors
432    Delta { size: ElemSize },
433    DeltaAsSignMagnitude { size: ElemSize },
434    DeltaAsNegaBinary { size: ElemSize },
435    // reducers
436    ChunkedLeadingZeroBitElimination { size: ElemSize },
437    HybridChunkedLeadingZeroBitElimination { size: ElemSize },
438    RepeatedAdaptiveRedundancyElimination { size: ElemSize },
439    RepeatedAdaptiveZeroElimination { size: ElemSize },
440    RunLengthEncoding { size: ElemSize },
441    RepeatedRedundancyElimination { size: ElemSize },
442    RepeatedZeroElimination { size: ElemSize },
443}
444
445impl Component {
446    #[expect(clippy::too_many_lines)]
447    const fn as_id(self) -> lc_framework_sys::LC_CPUcomponents {
448        match self {
449            Self::Noop => lc_framework_sys::LC_CPUcomponents_NUL_CPUcomponents,
450            // mutators
451            Self::TwosComplementToSignMagnitude { size: ElemSize::S1 } => {
452                lc_framework_sys::LC_CPUcomponents_TCMS_1
453            }
454            Self::TwosComplementToSignMagnitude { size: ElemSize::S2 } => {
455                lc_framework_sys::LC_CPUcomponents_TCMS_2
456            }
457            Self::TwosComplementToSignMagnitude { size: ElemSize::S4 } => {
458                lc_framework_sys::LC_CPUcomponents_TCMS_4
459            }
460            Self::TwosComplementToSignMagnitude { size: ElemSize::S8 } => {
461                lc_framework_sys::LC_CPUcomponents_TCMS_8
462            }
463            Self::TwosComplementToNegaBinary { size: ElemSize::S1 } => {
464                lc_framework_sys::LC_CPUcomponents_TCNB_1
465            }
466            Self::TwosComplementToNegaBinary { size: ElemSize::S2 } => {
467                lc_framework_sys::LC_CPUcomponents_TCNB_2
468            }
469            Self::TwosComplementToNegaBinary { size: ElemSize::S4 } => {
470                lc_framework_sys::LC_CPUcomponents_TCNB_4
471            }
472            Self::TwosComplementToNegaBinary { size: ElemSize::S8 } => {
473                lc_framework_sys::LC_CPUcomponents_TCNB_8
474            }
475            Self::DebiasedExponentFractionSign {
476                size: FloatSize::S4,
477            } => lc_framework_sys::LC_CPUcomponents_DBEFS_4,
478            Self::DebiasedExponentFractionSign {
479                size: FloatSize::S8,
480            } => lc_framework_sys::LC_CPUcomponents_DBEFS_8,
481            Self::DebiasedExponentSignFraction {
482                size: FloatSize::S4,
483            } => lc_framework_sys::LC_CPUcomponents_DBESF_4,
484            Self::DebiasedExponentSignFraction {
485                size: FloatSize::S8,
486            } => lc_framework_sys::LC_CPUcomponents_DBESF_8,
487            // shuffle
488            Self::BitShuffle { size: ElemSize::S1 } => lc_framework_sys::LC_CPUcomponents_BIT_1,
489            Self::BitShuffle { size: ElemSize::S2 } => lc_framework_sys::LC_CPUcomponents_BIT_2,
490            Self::BitShuffle { size: ElemSize::S4 } => lc_framework_sys::LC_CPUcomponents_BIT_4,
491            Self::BitShuffle { size: ElemSize::S8 } => lc_framework_sys::LC_CPUcomponents_BIT_8,
492            Self::Tuple {
493                size: TupleSize::S1x2,
494            } => lc_framework_sys::LC_CPUcomponents_TUPL2_1,
495            Self::Tuple {
496                size: TupleSize::S1x3,
497            } => lc_framework_sys::LC_CPUcomponents_TUPL3_1,
498            Self::Tuple {
499                size: TupleSize::S1x4,
500            } => lc_framework_sys::LC_CPUcomponents_TUPL4_1,
501            Self::Tuple {
502                size: TupleSize::S1x6,
503            } => lc_framework_sys::LC_CPUcomponents_TUPL6_1,
504            Self::Tuple {
505                size: TupleSize::S1x8,
506            } => lc_framework_sys::LC_CPUcomponents_TUPL8_1,
507            Self::Tuple {
508                size: TupleSize::S1x12,
509            } => lc_framework_sys::LC_CPUcomponents_TUPL12_1,
510            Self::Tuple {
511                size: TupleSize::S2x2,
512            } => lc_framework_sys::LC_CPUcomponents_TUPL2_2,
513            Self::Tuple {
514                size: TupleSize::S2x3,
515            } => lc_framework_sys::LC_CPUcomponents_TUPL3_2,
516            Self::Tuple {
517                size: TupleSize::S2x4,
518            } => lc_framework_sys::LC_CPUcomponents_TUPL4_2,
519            Self::Tuple {
520                size: TupleSize::S2x6,
521            } => lc_framework_sys::LC_CPUcomponents_TUPL6_2,
522            Self::Tuple {
523                size: TupleSize::S4x2,
524            } => lc_framework_sys::LC_CPUcomponents_TUPL2_4,
525            Self::Tuple {
526                size: TupleSize::S4x6,
527            } => lc_framework_sys::LC_CPUcomponents_TUPL6_4,
528            Self::Tuple {
529                size: TupleSize::S8x3,
530            } => lc_framework_sys::LC_CPUcomponents_TUPL3_8,
531            Self::Tuple {
532                size: TupleSize::S8x6,
533            } => lc_framework_sys::LC_CPUcomponents_TUPL6_8,
534            // predictors
535            Self::Delta { size: ElemSize::S1 } => lc_framework_sys::LC_CPUcomponents_DIFF_1,
536            Self::Delta { size: ElemSize::S2 } => lc_framework_sys::LC_CPUcomponents_DIFF_2,
537            Self::Delta { size: ElemSize::S4 } => lc_framework_sys::LC_CPUcomponents_DIFF_4,
538            Self::Delta { size: ElemSize::S8 } => lc_framework_sys::LC_CPUcomponents_DIFF_8,
539            Self::DeltaAsSignMagnitude { size: ElemSize::S1 } => {
540                lc_framework_sys::LC_CPUcomponents_DIFFMS_1
541            }
542            Self::DeltaAsSignMagnitude { size: ElemSize::S2 } => {
543                lc_framework_sys::LC_CPUcomponents_DIFFMS_2
544            }
545            Self::DeltaAsSignMagnitude { size: ElemSize::S4 } => {
546                lc_framework_sys::LC_CPUcomponents_DIFFMS_4
547            }
548            Self::DeltaAsSignMagnitude { size: ElemSize::S8 } => {
549                lc_framework_sys::LC_CPUcomponents_DIFFMS_8
550            }
551            Self::DeltaAsNegaBinary { size: ElemSize::S1 } => {
552                lc_framework_sys::LC_CPUcomponents_DIFFNB_1
553            }
554            Self::DeltaAsNegaBinary { size: ElemSize::S2 } => {
555                lc_framework_sys::LC_CPUcomponents_DIFFNB_2
556            }
557            Self::DeltaAsNegaBinary { size: ElemSize::S4 } => {
558                lc_framework_sys::LC_CPUcomponents_DIFFNB_4
559            }
560            Self::DeltaAsNegaBinary { size: ElemSize::S8 } => {
561                lc_framework_sys::LC_CPUcomponents_DIFFNB_8
562            }
563            // reducers
564            Self::ChunkedLeadingZeroBitElimination { size: ElemSize::S1 } => {
565                lc_framework_sys::LC_CPUcomponents_CLOG_1
566            }
567            Self::ChunkedLeadingZeroBitElimination { size: ElemSize::S2 } => {
568                lc_framework_sys::LC_CPUcomponents_CLOG_2
569            }
570            Self::ChunkedLeadingZeroBitElimination { size: ElemSize::S4 } => {
571                lc_framework_sys::LC_CPUcomponents_CLOG_4
572            }
573            Self::ChunkedLeadingZeroBitElimination { size: ElemSize::S8 } => {
574                lc_framework_sys::LC_CPUcomponents_CLOG_8
575            }
576            Self::HybridChunkedLeadingZeroBitElimination { size: ElemSize::S1 } => {
577                lc_framework_sys::LC_CPUcomponents_HCLOG_1
578            }
579            Self::HybridChunkedLeadingZeroBitElimination { size: ElemSize::S2 } => {
580                lc_framework_sys::LC_CPUcomponents_HCLOG_2
581            }
582            Self::HybridChunkedLeadingZeroBitElimination { size: ElemSize::S4 } => {
583                lc_framework_sys::LC_CPUcomponents_HCLOG_4
584            }
585            Self::HybridChunkedLeadingZeroBitElimination { size: ElemSize::S8 } => {
586                lc_framework_sys::LC_CPUcomponents_HCLOG_8
587            }
588            Self::RepeatedAdaptiveRedundancyElimination { size: ElemSize::S1 } => {
589                lc_framework_sys::LC_CPUcomponents_RARE_1
590            }
591            Self::RepeatedAdaptiveRedundancyElimination { size: ElemSize::S2 } => {
592                lc_framework_sys::LC_CPUcomponents_RARE_2
593            }
594            Self::RepeatedAdaptiveRedundancyElimination { size: ElemSize::S4 } => {
595                lc_framework_sys::LC_CPUcomponents_RARE_4
596            }
597            Self::RepeatedAdaptiveRedundancyElimination { size: ElemSize::S8 } => {
598                lc_framework_sys::LC_CPUcomponents_RARE_8
599            }
600            Self::RepeatedAdaptiveZeroElimination { size: ElemSize::S1 } => {
601                lc_framework_sys::LC_CPUcomponents_RAZE_1
602            }
603            Self::RepeatedAdaptiveZeroElimination { size: ElemSize::S2 } => {
604                lc_framework_sys::LC_CPUcomponents_RAZE_2
605            }
606            Self::RepeatedAdaptiveZeroElimination { size: ElemSize::S4 } => {
607                lc_framework_sys::LC_CPUcomponents_RAZE_4
608            }
609            Self::RepeatedAdaptiveZeroElimination { size: ElemSize::S8 } => {
610                lc_framework_sys::LC_CPUcomponents_RAZE_8
611            }
612            Self::RunLengthEncoding { size: ElemSize::S1 } => {
613                lc_framework_sys::LC_CPUcomponents_RLE_1
614            }
615            Self::RunLengthEncoding { size: ElemSize::S2 } => {
616                lc_framework_sys::LC_CPUcomponents_RLE_2
617            }
618            Self::RunLengthEncoding { size: ElemSize::S4 } => {
619                lc_framework_sys::LC_CPUcomponents_RLE_4
620            }
621            Self::RunLengthEncoding { size: ElemSize::S8 } => {
622                lc_framework_sys::LC_CPUcomponents_RLE_8
623            }
624            Self::RepeatedRedundancyElimination { size: ElemSize::S1 } => {
625                lc_framework_sys::LC_CPUcomponents_RRE_1
626            }
627            Self::RepeatedRedundancyElimination { size: ElemSize::S2 } => {
628                lc_framework_sys::LC_CPUcomponents_RRE_2
629            }
630            Self::RepeatedRedundancyElimination { size: ElemSize::S4 } => {
631                lc_framework_sys::LC_CPUcomponents_RRE_4
632            }
633            Self::RepeatedRedundancyElimination { size: ElemSize::S8 } => {
634                lc_framework_sys::LC_CPUcomponents_RRE_8
635            }
636            Self::RepeatedZeroElimination { size: ElemSize::S1 } => {
637                lc_framework_sys::LC_CPUcomponents_RZE_1
638            }
639            Self::RepeatedZeroElimination { size: ElemSize::S2 } => {
640                lc_framework_sys::LC_CPUcomponents_RZE_2
641            }
642            Self::RepeatedZeroElimination { size: ElemSize::S4 } => {
643                lc_framework_sys::LC_CPUcomponents_RZE_4
644            }
645            Self::RepeatedZeroElimination { size: ElemSize::S8 } => {
646                lc_framework_sys::LC_CPUcomponents_RZE_8
647            }
648        }
649    }
650}
651
652#[expect(missing_docs)]
653#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
654/// LC component element size, in bytes
655pub enum ElemSize {
656    S1,
657    S2,
658    S4,
659    S8,
660}
661
662#[expect(missing_docs)]
663#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
664/// LC component float element size, in bytes
665pub enum FloatSize {
666    S4,
667    S8,
668}
669
670#[expect(missing_docs)]
671#[derive(Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
672/// LC tuple component element size, in bytes x tuple length
673pub enum TupleSize {
674    S1x2,
675    S1x3,
676    S1x4,
677    S1x6,
678    S1x8,
679    S1x12,
680    S2x2,
681    S2x3,
682    S2x4,
683    S2x6,
684    S4x2,
685    S4x6,
686    S8x3,
687    S8x6,
688}
689
690#[cfg(test)]
691#[allow(clippy::unwrap_used)]
692mod tests {
693    use super::*;
694
695    #[test]
696    fn bit4_rle4() {
697        let preprocessors = &[];
698        let components = &[
699            Component::BitShuffle { size: ElemSize::S4 },
700            Component::RunLengthEncoding { size: ElemSize::S4 },
701        ];
702
703        let data = b"abcd";
704
705        let compressed = compress(preprocessors, components, data).unwrap();
706        let decompressed = decompress(preprocessors, components, &compressed).unwrap();
707
708        assert_eq!(decompressed, data);
709    }
710
711    #[test]
712    fn abs_error() {
713        let data = (0..100_u16)
714            .map(|x| f32::from(x) / 100.0)
715            .map(|x| std::f32::consts::PI * x)
716            .map(f32::cos)
717            .collect::<Vec<_>>();
718        #[expect(unsafe_code)]
719        // SAFETY:
720        // - read-only access to the underlying bytes
721        // - f32 is a plain-old data type
722        let data_bytes = unsafe {
723            std::slice::from_raw_parts(data.as_ptr().cast(), std::mem::size_of_val(data.as_slice()))
724        };
725
726        let error_bound = 0.1;
727
728        let preprocessors = &[Preprocessor::QuantizeErrorBound {
729            dtype: QuantizeDType::F32,
730            kind: ErrorKind::Abs,
731            error_bound,
732            threshold: None,
733            decorrelation: Decorrelation::Zero,
734        }];
735        let components = &[
736            Component::BitShuffle { size: ElemSize::S4 },
737            Component::RunLengthEncoding { size: ElemSize::S4 },
738        ];
739
740        let compressed = compress(preprocessors, components, data_bytes).unwrap();
741
742        for i in 0..std::mem::size_of::<c_longlong>() {
743            let mut compressed_unaligned = Vec::with_capacity(compressed.len() + i);
744            compressed_unaligned.extend(std::iter::repeat_n(b'\0', i));
745            compressed_unaligned.extend_from_slice(&compressed);
746
747            let decompressed_bytes = decompress(
748                preprocessors,
749                components,
750                compressed_unaligned.get(i..).unwrap(),
751            )
752            .unwrap();
753            assert_eq!(decompressed_bytes.len(), data_bytes.len());
754
755            #[expect(unsafe_code)]
756            // SAFETY: initialise aligned f32s with unaligned bytes
757            let decompressed = unsafe {
758                let mut decompressed = Vec::<f32>::with_capacity(data.len());
759                std::ptr::copy_nonoverlapping(
760                    decompressed_bytes.as_ptr(),
761                    decompressed.as_mut_ptr().cast(),
762                    data_bytes.len(),
763                );
764                decompressed.set_len(data.len());
765                decompressed
766            };
767
768            for (o, d) in data.iter().copied().zip(decompressed) {
769                assert!(f64::from((o - d).abs()) <= error_bound);
770            }
771        }
772    }
773}
lc_framework/lib.rs

lc_framework/
lib.rs
