1#![allow(clippy::multiple_crate_versions)] use ::zstd_sys as _;
25
26#[cfg(test)]
27use ::serde_json as _;
28
29use std::{
30 borrow::Cow,
31 fmt,
32 num::{NonZeroU16, NonZeroUsize},
33};
34
35use ndarray::{Array, Array1, ArrayBase, Axis, Data, Dimension, IxDyn, ShapeError};
36use num_traits::{Float, identities::Zero};
37use numcodecs::{
38 AnyArray, AnyArrayAssignError, AnyArrayDType, AnyArrayView, AnyArrayViewMut, AnyCowArray,
39 Codec, StaticCodec, StaticCodecConfig, StaticCodecVersion,
40};
41use schemars::{JsonSchema, Schema, SchemaGenerator, json_schema};
42use serde::{Deserialize, Deserializer, Serialize, Serializer};
43use thiserror::Error;
44
45type QpetSperrCodecVersion = StaticCodecVersion<0, 1, 0>;
46
47#[derive(Clone, Serialize, Deserialize, JsonSchema)]
48#[schemars(deny_unknown_fields)]
50pub struct QpetSperrCodec {
58 #[serde(flatten)]
60 pub mode: QpetSperrCompressionMode,
61 #[serde(default, rename = "_version")]
63 pub version: QpetSperrCodecVersion,
64}
65
66#[derive(Clone, Serialize, Deserialize, JsonSchema)]
67#[serde(tag = "mode")]
69pub enum QpetSperrCompressionMode {
70 #[serde(rename = "qoi-symbolic")]
72 SymbolicQuantityOfInterest {
73 qoi: String,
75 #[serde(default = "default_qoi_block_size")]
78 qoi_block_size: NonZeroU16,
79 qoi_pwe: Positive<f64>,
82 #[serde(default = "default_sperr_chunks")]
84 sperr_chunks: (NonZeroUsize, NonZeroUsize, NonZeroUsize),
85 #[serde(default)]
87 data_pwe: Option<Positive<f64>>,
88 #[serde(default = "default_qoi_k")]
90 qoi_k: Positive<f64>,
91 #[serde(default)]
93 high_prec: bool,
94 },
95}
96
97const fn default_qoi_block_size() -> NonZeroU16 {
98 const NON_ZERO_ONE: NonZeroU16 = NonZeroU16::MIN;
99 NON_ZERO_ONE
101}
102
103const fn default_sperr_chunks() -> (NonZeroUsize, NonZeroUsize, NonZeroUsize) {
104 const NON_ZERO_256: NonZeroUsize = NonZeroUsize::MIN.saturating_add(255);
105 (NON_ZERO_256, NON_ZERO_256, NON_ZERO_256)
106}
107
108const fn default_qoi_k() -> Positive<f64> {
109 Positive(3.0)
111}
112
113impl Codec for QpetSperrCodec {
114 type Error = QpetSperrCodecError;
115
116 fn encode(&self, data: AnyCowArray) -> Result<AnyArray, Self::Error> {
117 match data {
118 AnyCowArray::F32(data) => Ok(AnyArray::U8(
119 Array1::from(compress(data, &self.mode)?).into_dyn(),
120 )),
121 AnyCowArray::F64(data) => Ok(AnyArray::U8(
122 Array1::from(compress(data, &self.mode)?).into_dyn(),
123 )),
124 encoded => Err(QpetSperrCodecError::UnsupportedDtype(encoded.dtype())),
125 }
126 }
127
128 fn decode(&self, encoded: AnyCowArray) -> Result<AnyArray, Self::Error> {
129 let AnyCowArray::U8(encoded) = encoded else {
130 return Err(QpetSperrCodecError::EncodedDataNotBytes {
131 dtype: encoded.dtype(),
132 });
133 };
134
135 if !matches!(encoded.shape(), [_]) {
136 return Err(QpetSperrCodecError::EncodedDataNotOneDimensional {
137 shape: encoded.shape().to_vec(),
138 });
139 }
140
141 decompress(&AnyCowArray::U8(encoded).as_bytes())
142 }
143
144 fn decode_into(
145 &self,
146 encoded: AnyArrayView,
147 mut decoded: AnyArrayViewMut,
148 ) -> Result<(), Self::Error> {
149 let decoded_in = self.decode(encoded.cow())?;
150
151 Ok(decoded.assign(&decoded_in)?)
152 }
153}
154
155impl StaticCodec for QpetSperrCodec {
156 const CODEC_ID: &'static str = "qpet-sperr.rs";
157
158 type Config<'de> = Self;
159
160 fn from_config(config: Self::Config<'_>) -> Self {
161 config
162 }
163
164 fn get_config(&self) -> StaticCodecConfig<'_, Self> {
165 StaticCodecConfig::from(self)
166 }
167}
168
169#[derive(Debug, Error)]
170pub enum QpetSperrCodecError {
172 #[error("QpetSperr does not support the dtype {0}")]
174 UnsupportedDtype(AnyArrayDType),
175 #[error("QpetSperr failed to encode the header")]
177 HeaderEncodeFailed {
178 source: QpetSperrHeaderError,
180 },
181 #[error("QpetSperr failed to encode the data")]
183 QpetSperrEncodeFailed {
184 source: QpetSperrCodingError,
186 },
187 #[error("QpetSperr failed to encode a slice")]
189 SliceEncodeFailed {
190 source: QpetSperrSliceError,
192 },
193 #[error(
196 "QpetSperr can only decode one-dimensional byte arrays but received an array of dtype {dtype}"
197 )]
198 EncodedDataNotBytes {
199 dtype: AnyArrayDType,
201 },
202 #[error(
205 "QpetSperr can only decode one-dimensional byte arrays but received a byte array of shape {shape:?}"
206 )]
207 EncodedDataNotOneDimensional {
208 shape: Vec<usize>,
210 },
211 #[error("QpetSperr failed to decode the header")]
213 HeaderDecodeFailed {
214 source: QpetSperrHeaderError,
216 },
217 #[error("QpetSperr failed to decode a slice")]
219 SliceDecodeFailed {
220 source: QpetSperrSliceError,
222 },
223 #[error("QpetSperr failed to decode from an excessive number of slices")]
225 DecodeTooManySlices,
226 #[error("QpetSperr failed to decode the data")]
228 SperrDecodeFailed {
229 source: QpetSperrCodingError,
231 },
232 #[error("QpetSperr decoded into an invalid shape not matching the data size")]
234 DecodeInvalidShape {
235 source: ShapeError,
237 },
238 #[error("QpetSperr cannot decode into the provided array")]
240 MismatchedDecodeIntoArray {
241 #[from]
243 source: AnyArrayAssignError,
244 },
245}
246
247#[derive(Debug, Error)]
248#[error(transparent)]
249pub struct QpetSperrHeaderError(postcard::Error);
251
252#[derive(Debug, Error)]
253#[error(transparent)]
254pub struct QpetSperrSliceError(postcard::Error);
256
257#[derive(Debug, Error)]
258#[error(transparent)]
259pub struct QpetSperrCodingError(qpet_sperr::Error);
261
262#[allow(clippy::missing_panics_doc)]
274pub fn compress<T: QpetSperrElement, S: Data<Elem = T>, D: Dimension>(
275 data: ArrayBase<S, D>,
276 mode: &QpetSperrCompressionMode,
277) -> Result<Vec<u8>, QpetSperrCodecError> {
278 let mut encoded = postcard::to_extend(
279 &CompressionHeader {
280 dtype: T::DTYPE,
281 shape: Cow::Borrowed(data.shape()),
282 version: StaticCodecVersion,
283 },
284 Vec::new(),
285 )
286 .map_err(|err| QpetSperrCodecError::HeaderEncodeFailed {
287 source: QpetSperrHeaderError(err),
288 })?;
289
290 if data.is_empty() {
292 return Ok(encoded);
293 }
294
295 let mut chunk_size = Vec::from(data.shape());
296 let (width, height, depth) = match *chunk_size.as_mut_slice() {
297 [ref mut rest @ .., depth, height, width] => {
298 for r in rest {
299 *r = 1;
300 }
301 (width, height, depth)
302 }
303 [height, width] => (width, height, 1),
304 [width] => (width, 1, 1),
305 [] => (1, 1, 1),
306 };
307
308 for mut slice in data.into_dyn().exact_chunks(chunk_size.as_slice()) {
309 while slice.ndim() < 3 {
310 slice = slice.insert_axis(Axis(0));
311 }
312 #[allow(clippy::unwrap_used)]
313 let slice = slice.into_shape_with_order((depth, height, width)).unwrap();
316
317 let QpetSperrCompressionMode::SymbolicQuantityOfInterest {
318 qoi,
319 qoi_block_size,
320 qoi_pwe,
321 sperr_chunks,
322 data_pwe,
323 qoi_k,
324 high_prec,
325 } = mode;
326
327 let encoded_slice = qpet_sperr::compress_3d(
328 slice,
329 qpet_sperr::CompressionMode::SymbolicQuantityOfInterest {
330 qoi: qoi.as_str(),
331 qoi_block_size: *qoi_block_size,
332 qoi_pwe: qoi_pwe.0,
333 data_pwe: data_pwe.map(|data_pwe| data_pwe.0),
334 qoi_k: qoi_k.0,
335 high_prec: *high_prec,
336 },
337 (
338 sperr_chunks.0.get(),
339 sperr_chunks.1.get(),
340 sperr_chunks.2.get(),
341 ),
342 )
343 .map_err(|err| QpetSperrCodecError::QpetSperrEncodeFailed {
344 source: QpetSperrCodingError(err),
345 })?;
346
347 encoded = postcard::to_extend(encoded_slice.as_slice(), encoded).map_err(|err| {
348 QpetSperrCodecError::SliceEncodeFailed {
349 source: QpetSperrSliceError(err),
350 }
351 })?;
352 }
353
354 Ok(encoded)
355}
356
357pub fn decompress(encoded: &[u8]) -> Result<AnyArray, QpetSperrCodecError> {
370 fn decompress_typed<T: QpetSperrElement>(
371 mut encoded: &[u8],
372 shape: &[usize],
373 ) -> Result<Array<T, IxDyn>, QpetSperrCodecError> {
374 let mut decoded = Array::<T, _>::zeros(shape);
375
376 let mut chunk_size = Vec::from(shape);
377 let (width, height, depth) = match *chunk_size.as_mut_slice() {
378 [ref mut rest @ .., depth, height, width] => {
379 for r in rest {
380 *r = 1;
381 }
382 (width, height, depth)
383 }
384 [height, width] => (width, height, 1),
385 [width] => (width, 1, 1),
386 [] => (1, 1, 1),
387 };
388
389 for mut slice in decoded.exact_chunks_mut(chunk_size.as_slice()) {
390 let (encoded_slice, rest) =
391 postcard::take_from_bytes::<Cow<[u8]>>(encoded).map_err(|err| {
392 QpetSperrCodecError::SliceDecodeFailed {
393 source: QpetSperrSliceError(err),
394 }
395 })?;
396 encoded = rest;
397
398 while slice.ndim() < 3 {
399 slice = slice.insert_axis(Axis(0));
400 }
401 #[allow(clippy::unwrap_used)]
402 let slice = slice.into_shape_with_order((depth, height, width)).unwrap();
405
406 qpet_sperr::decompress_into_3d(&encoded_slice, slice).map_err(|err| {
407 QpetSperrCodecError::SperrDecodeFailed {
408 source: QpetSperrCodingError(err),
409 }
410 })?;
411 }
412
413 if !encoded.is_empty() {
414 return Err(QpetSperrCodecError::DecodeTooManySlices);
415 }
416
417 Ok(decoded)
418 }
419
420 let (header, encoded) =
421 postcard::take_from_bytes::<CompressionHeader>(encoded).map_err(|err| {
422 QpetSperrCodecError::HeaderDecodeFailed {
423 source: QpetSperrHeaderError(err),
424 }
425 })?;
426
427 if header.shape.iter().copied().product::<usize>() == 0 {
429 return match header.dtype {
430 QpetSperrDType::F32 => Ok(AnyArray::F32(Array::zeros(&*header.shape))),
431 QpetSperrDType::F64 => Ok(AnyArray::F64(Array::zeros(&*header.shape))),
432 };
433 }
434
435 match header.dtype {
436 QpetSperrDType::F32 => Ok(AnyArray::F32(decompress_typed(encoded, &header.shape)?)),
437 QpetSperrDType::F64 => Ok(AnyArray::F64(decompress_typed(encoded, &header.shape)?)),
438 }
439}
440
441pub trait QpetSperrElement: qpet_sperr::Element + Zero {
443 const DTYPE: QpetSperrDType;
445}
446
447impl QpetSperrElement for f32 {
448 const DTYPE: QpetSperrDType = QpetSperrDType::F32;
449}
450impl QpetSperrElement for f64 {
451 const DTYPE: QpetSperrDType = QpetSperrDType::F64;
452}
453
454#[expect(clippy::derive_partial_eq_without_eq)] #[derive(Copy, Clone, PartialEq, PartialOrd, Hash)]
456pub struct Positive<T: Float>(T);
458
459impl<T: Float> Positive<T> {
460 #[must_use]
461 pub const fn get(self) -> T {
463 self.0
464 }
465}
466
467impl Serialize for Positive<f64> {
468 fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
469 serializer.serialize_f64(self.0)
470 }
471}
472
473impl<'de> Deserialize<'de> for Positive<f64> {
474 fn deserialize<D: Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
475 let x = f64::deserialize(deserializer)?;
476
477 if x > 0.0 {
478 Ok(Self(x))
479 } else {
480 Err(serde::de::Error::invalid_value(
481 serde::de::Unexpected::Float(x),
482 &"a positive value",
483 ))
484 }
485 }
486}
487
488impl JsonSchema for Positive<f64> {
489 fn schema_name() -> Cow<'static, str> {
490 Cow::Borrowed("PositiveF64")
491 }
492
493 fn schema_id() -> Cow<'static, str> {
494 Cow::Borrowed(concat!(module_path!(), "::", "Positive<f64>"))
495 }
496
497 fn json_schema(_gen: &mut SchemaGenerator) -> Schema {
498 json_schema!({
499 "type": "number",
500 "exclusiveMinimum": 0.0
501 })
502 }
503}
504
505#[derive(Serialize, Deserialize)]
506struct CompressionHeader<'a> {
507 dtype: QpetSperrDType,
508 #[serde(borrow)]
509 shape: Cow<'a, [usize]>,
510 version: QpetSperrCodecVersion,
511}
512
513#[derive(Copy, Clone, Debug, Serialize, Deserialize)]
515#[expect(missing_docs)]
516pub enum QpetSperrDType {
517 #[serde(rename = "f32", alias = "float32")]
518 F32,
519 #[serde(rename = "f64", alias = "float64")]
520 F64,
521}
522
523impl fmt::Display for QpetSperrDType {
524 fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
525 fmt.write_str(match self {
526 Self::F32 => "f32",
527 Self::F64 => "f64",
528 })
529 }
530}
531
532#[cfg(test)]
533#[allow(clippy::unwrap_used)]
534mod tests {
535 use std::f64;
536
537 use ndarray::{Ix0, Ix1, Ix2, Ix3, Ix4};
538
539 use super::*;
540
541 #[test]
542 fn zero_length() {
543 let encoded = compress(
544 Array::<f32, _>::from_shape_vec([3, 0], vec![]).unwrap(),
545 &QpetSperrCompressionMode::SymbolicQuantityOfInterest {
546 qoi: String::from("x"),
547 qoi_block_size: default_qoi_block_size(),
548 qoi_pwe: Positive(42.0),
549 sperr_chunks: default_sperr_chunks(),
550 data_pwe: None,
551 qoi_k: default_qoi_k(),
552 high_prec: false,
553 },
554 )
555 .unwrap();
556 let decoded = decompress(&encoded).unwrap();
557
558 assert_eq!(decoded.dtype(), AnyArrayDType::F32);
559 assert!(decoded.is_empty());
560 assert_eq!(decoded.shape(), &[3, 0]);
561 }
562
563 #[test]
564 fn small_2d() {
565 let encoded = compress(
566 Array::<f32, _>::from_shape_vec([1, 1], vec![42.0]).unwrap(),
567 &QpetSperrCompressionMode::SymbolicQuantityOfInterest {
568 qoi: String::from("x"),
569 qoi_block_size: default_qoi_block_size(),
570 qoi_pwe: Positive(42.0),
571 sperr_chunks: default_sperr_chunks(),
572 data_pwe: None,
573 qoi_k: default_qoi_k(),
574 high_prec: false,
575 },
576 )
577 .unwrap();
578 let decoded = decompress(&encoded).unwrap();
579
580 assert_eq!(decoded.dtype(), AnyArrayDType::F32);
581 assert_eq!(decoded.len(), 1);
582 assert_eq!(decoded.shape(), &[1, 1]);
583 }
584
585 #[test]
586 fn large_3d() {
587 let encoded = compress(
588 Array::<f64, _>::zeros((64, 64, 64)),
589 &QpetSperrCompressionMode::SymbolicQuantityOfInterest {
590 qoi: String::from("x"),
591 qoi_block_size: default_qoi_block_size(),
592 qoi_pwe: Positive(42.0),
593 sperr_chunks: default_sperr_chunks(),
594 data_pwe: None,
595 qoi_k: default_qoi_k(),
596 high_prec: false,
597 },
598 )
599 .unwrap();
600 let decoded = decompress(&encoded).unwrap();
601
602 assert_eq!(decoded.dtype(), AnyArrayDType::F64);
603 assert_eq!(decoded.len(), 64 * 64 * 64);
604 assert_eq!(decoded.shape(), &[64, 64, 64]);
605 }
606
607 #[test]
608 fn all_modes() {
609 for mode in [QpetSperrCompressionMode::SymbolicQuantityOfInterest {
610 qoi: String::from("x^2"),
611 qoi_block_size: default_qoi_block_size(),
612 qoi_pwe: Positive(0.1),
613 sperr_chunks: default_sperr_chunks(),
614 data_pwe: None,
615 qoi_k: default_qoi_k(),
616 high_prec: false,
617 }] {
618 let encoded = compress(Array::<f64, _>::zeros((64, 64, 64)), &mode).unwrap();
619 let decoded = decompress(&encoded).unwrap();
620
621 assert_eq!(decoded.dtype(), AnyArrayDType::F64);
622 assert_eq!(decoded.len(), 64 * 64 * 64);
623 assert_eq!(decoded.shape(), &[64, 64, 64]);
624 }
625 }
626
627 #[test]
628 fn many_dimensions() {
629 for data in [
630 Array::<f32, Ix0>::from_shape_vec([], vec![42.0])
631 .unwrap()
632 .into_dyn(),
633 Array::<f32, Ix1>::from_shape_vec([2], vec![1.0, 2.0])
634 .unwrap()
635 .into_dyn(),
636 Array::<f32, Ix2>::from_shape_vec([2, 2], vec![1.0, 2.0, 3.0, 4.0])
637 .unwrap()
638 .into_dyn(),
639 Array::<f32, Ix3>::from_shape_vec(
640 [2, 2, 2],
641 vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0],
642 )
643 .unwrap()
644 .into_dyn(),
645 Array::<f32, Ix4>::from_shape_vec(
646 [2, 2, 2, 2],
647 vec![
648 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0,
649 15.0, 16.0,
650 ],
651 )
652 .unwrap()
653 .into_dyn(),
654 ] {
655 let encoded = compress(
656 data.view(),
657 &QpetSperrCompressionMode::SymbolicQuantityOfInterest {
658 qoi: String::from("x"),
659 qoi_block_size: default_qoi_block_size(),
660 qoi_pwe: Positive(f64::EPSILON),
661 sperr_chunks: default_sperr_chunks(),
662 data_pwe: None,
663 qoi_k: default_qoi_k(),
664 high_prec: false,
665 },
666 )
667 .unwrap();
668 let decoded = decompress(&encoded).unwrap();
669
670 assert_eq!(decoded, AnyArray::F32(data));
671 }
672 }
673
674 #[test]
675 fn zero_square_qoi() {
676 let encoded = compress(
677 Array::<f64, _>::zeros((64, 64, 1)),
678 &QpetSperrCompressionMode::SymbolicQuantityOfInterest {
679 qoi: String::from("x^2"),
680 qoi_block_size: default_qoi_block_size(),
681 qoi_pwe: Positive(0.1),
682 sperr_chunks: default_sperr_chunks(),
683 data_pwe: None,
684 qoi_k: default_qoi_k(),
685 high_prec: false,
686 },
687 )
688 .unwrap();
689 let decoded = decompress(&encoded).unwrap();
690
691 assert_eq!(decoded.dtype(), AnyArrayDType::F64);
692 assert_eq!(decoded.len(), 64 * 64 * 1);
693 assert_eq!(decoded.shape(), &[64, 64, 1]);
694 }
695}