From 89e20c9705dd7e76e9e855ebef26130b18d5900c Mon Sep 17 00:00:00 2001 From: Lachlan Deakin Date: Mon, 18 Mar 2024 07:58:53 +1100 Subject: [PATCH] Use pcodec bounded size representation --- src/array/codec/array_to_bytes/pcodec.rs | 2 ++ .../array_to_bytes/pcodec/pcodec_codec.rs | 27 ++++++++++++++++--- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/src/array/codec/array_to_bytes/pcodec.rs b/src/array/codec/array_to_bytes/pcodec.rs index 0ef37b0f..72b77e2e 100644 --- a/src/array/codec/array_to_bytes/pcodec.rs +++ b/src/array/codec/array_to_bytes/pcodec.rs @@ -227,11 +227,13 @@ mod tests { ChunkRepresentation::new(chunk_shape, data_type, fill_value).unwrap(); let bytes: Vec = (0..chunk_representation.size()).map(|s| s as u8).collect(); + let max_encoded_size = codec.compute_encoded_size(&chunk_representation)?; let encoded = codec.encode( bytes.clone(), &chunk_representation, &CodecOptions::default(), )?; + assert!((encoded.len() as u64) <= max_encoded_size.size().unwrap()); let decoded = codec .decode(encoded, &chunk_representation, &CodecOptions::default()) .unwrap(); diff --git a/src/array/codec/array_to_bytes/pcodec/pcodec_codec.rs b/src/array/codec/array_to_bytes/pcodec/pcodec_codec.rs index 5513488b..c05dce4d 100644 --- a/src/array/codec/array_to_bytes/pcodec/pcodec_codec.rs +++ b/src/array/codec/array_to_bytes/pcodec/pcodec_codec.rs @@ -1,4 +1,4 @@ -use pco::{ChunkConfig, FloatMultSpec, IntMultSpec, PagingSpec}; +use pco::{standalone::guarantee::file_size, ChunkConfig, FloatMultSpec, IntMultSpec, PagingSpec}; use crate::{ array::{ @@ -211,9 +211,28 @@ impl ArrayToBytesCodecTraits for PcodecCodec { fn compute_encoded_size( &self, - _decoded_representation: &ChunkRepresentation, + decoded_representation: &ChunkRepresentation, ) -> Result { - // FIXME: pcodec is likely bounded, but it doesn't have a nice API to figure out what the bounded size is - Ok(BytesRepresentation::UnboundedSize) + let data_type = decoded_representation.data_type(); + let mut num_elements = decoded_representation.num_elements_usize(); + if data_type == &DataType::Complex64 || data_type == &DataType::Complex128 { + num_elements *= 2; + } + + let size = match data_type { + DataType::UInt32 | DataType::Int32 | DataType::Float32 | DataType::Complex64 => Ok( + file_size::(num_elements, &self.chunk_config.paging_spec) + .map_err(|err| CodecError::from(err.to_string()))?, + ), + DataType::UInt64 | DataType::Int64 | DataType::Float64 | DataType::Complex128 => Ok( + file_size::(num_elements, &self.chunk_config.paging_spec) + .map_err(|err| CodecError::from(err.to_string()))?, + ), + _ => Err(CodecError::UnsupportedDataType( + data_type.clone(), + IDENTIFIER.to_string(), + )), + }?; + Ok(BytesRepresentation::BoundedSize(size.try_into().unwrap())) } }