Skip to content

Commit 66ad5df

Browse files
committed
feat: compress zarr traces with zstd
1 parent cb1be48 commit 66ad5df

File tree

1 file changed

+24
-0
lines changed

1 file changed

+24
-0
lines changed

src/storage/zarr/common.rs

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ use std::{collections::HashMap, num::NonZero};
44

55
use anyhow::Result;
66
use nuts_storable::{ItemType, Value};
7+
use zarrs::array::codec::{BloscCodec, BloscCodecConfiguration, BloscCodecConfigurationV1};
78
use zarrs::array::{Array, ArrayBuilder, DataType, FillValue};
89
use zarrs::metadata_ext::data_type::NumpyTimeUnit;
910

@@ -232,7 +233,30 @@ pub fn create_arrays<TStorage: ?Sized>(
232233
.chain(std::iter::once(draw_chunk_size))
233234
.chain(extra_shape)
234235
.collect();
236+
237+
let codec = {
238+
if let Some(typesize) = zarr_type.fixed_size() {
239+
let config = BloscCodecConfiguration::V1(BloscCodecConfigurationV1 {
240+
cname: zarrs::array::codec::BloscCompressor::Zstd,
241+
clevel: 3u8.try_into().unwrap(),
242+
shuffle: zarrs::array::codec::BloscShuffleMode::Shuffle,
243+
blocksize: 0,
244+
typesize: Some(typesize),
245+
});
246+
BloscCodec::new_with_configuration(&config)?
247+
} else {
248+
let config = BloscCodecConfiguration::V1(BloscCodecConfigurationV1 {
249+
cname: zarrs::array::codec::BloscCompressor::Zstd,
250+
clevel: 3u8.try_into().unwrap(),
251+
shuffle: zarrs::array::codec::BloscShuffleMode::NoShuffle,
252+
blocksize: 0,
253+
typesize: None,
254+
});
255+
BloscCodec::new_with_configuration(&config)?
256+
}
257+
};
235258
let array = ArrayBuilder::new(shape, grid, zarr_type, fill_value)
259+
.bytes_to_bytes_codecs(vec![Arc::new(codec)])
236260
.dimension_names(Some(dims))
237261
.build(store.clone(), &format!("{}/{}", group_path, name))?;
238262
arrays.insert(name.to_string(), array);

0 commit comments

Comments
 (0)