diff --git a/zarrs_opendal/CHANGELOG.md b/zarrs_opendal/CHANGELOG.md index 70680f68..e2f6fac2 100644 --- a/zarrs_opendal/CHANGELOG.md +++ b/zarrs_opendal/CHANGELOG.md @@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Changed - - **Breaking**: Bump maximum supported `opendal` to 0.51 + - **Breaking**: Bump `opendal` to 0.51 ## [0.4.0] - 2024-11-15 diff --git a/zarrs_opendal/Cargo.toml b/zarrs_opendal/Cargo.toml index cf656322..a7a39c85 100644 --- a/zarrs_opendal/Cargo.toml +++ b/zarrs_opendal/Cargo.toml @@ -18,7 +18,7 @@ workspace = true [dependencies] async-trait = "0.1.74" futures = "0.3.29" -opendal = { version = ">=0.46,<0.52", default-features = false } +opendal = { version = ">=0.51,<0.52", default-features = false } zarrs_storage = { workspace = true, features = ["async"] } [dev-dependencies] diff --git a/zarrs_opendal/doc/version_compatibility_matrix.md b/zarrs_opendal/doc/version_compatibility_matrix.md index 7d3306f7..4f0bfd4a 100644 --- a/zarrs_opendal/doc/version_compatibility_matrix.md +++ b/zarrs_opendal/doc/version_compatibility_matrix.md @@ -1,6 +1,6 @@ | [zarrs_opendal] | [opendal] | [zarrs] ([zarrs_storage]) | | --------------- | --------- | ------------------------- | -| 0.5 | 0.46-0.51 | 0.18.x (0.3.x) | +| 0.5 | 0.51-0.51 | 0.18.x (0.3.x) | | 0.4 | 0.46-0.50 | 0.18.x (0.3.x) | | 0.3 | 0.46-0.50 | 0.17.x (0.2.x) | diff --git a/zarrs_opendal/src/async.rs b/zarrs_opendal/src/async.rs index 956ee8cc..b9332828 100644 --- a/zarrs_opendal/src/async.rs +++ b/zarrs_opendal/src/async.rs @@ -83,7 +83,7 @@ impl AsyncWritableStorageTraits for AsyncOpendalStore { } async fn erase(&self, key: &StoreKey) -> Result<(), StorageError> { - handle_result(self.operator.remove(vec![key.to_string()]).await) + handle_result(self.operator.delete(key.as_str()).await) } async fn erase_prefix(&self, prefix: &StorePrefix) -> Result<(), StorageError> { @@ -157,23 +157,44 @@ impl AsyncListableStorageTraits for AsyncOpendalStore { } async fn size_prefix(&self, prefix: &StorePrefix) -> Result { - handle_result_notfound( + let Some(files) = handle_result_notfound( self.operator .list_with(prefix.as_str()) .recursive(true) - .metakey(opendal::Metakey::ContentLength) .await, )? - .map_or_else( - || Ok(0), - |list| { - let size = list - .into_iter() - .map(|entry| entry.metadata().content_length()) - .sum::(); - Ok(size) - }, - ) + else { + return Ok(0); + }; + + if self + .operator + .info() + .full_capability() + .list_has_content_length + { + let size = files + .into_iter() + .filter_map(|entry| { + if entry.metadata().is_file() { + Some(entry.metadata().content_length()) + } else { + None + } + }) + .sum::(); + Ok(size) + } else { + // TODO: concurrent + let mut size = 0; + for entry in files { + let meta = handle_result(self.operator.stat(entry.path()).await)?; + if meta.is_file() { + size += meta.content_length(); + } + } + Ok(size) + } } } diff --git a/zarrs_opendal/src/sync.rs b/zarrs_opendal/src/sync.rs index 84417168..e626f3d1 100644 --- a/zarrs_opendal/src/sync.rs +++ b/zarrs_opendal/src/sync.rs @@ -71,7 +71,7 @@ impl WritableStorageTraits for OpendalStore { } fn erase(&self, key: &StoreKey) -> Result<(), StorageError> { - handle_result(self.operator.remove(vec![key.to_string()])) + handle_result(self.operator.delete(key.as_str())) } fn erase_prefix(&self, prefix: &StorePrefix) -> Result<(), StorageError> { @@ -145,23 +145,44 @@ impl ListableStorageTraits for OpendalStore { } fn size_prefix(&self, prefix: &StorePrefix) -> Result { - handle_result_notfound( + let Some(files) = handle_result_notfound( self.operator .list_with(prefix.as_str()) .recursive(true) - .metakey(opendal::Metakey::ContentLength) .call(), )? - .map_or_else( - || Ok(0), - |list| { - let size = list - .into_iter() - .map(|entry| entry.metadata().content_length()) - .sum::(); - Ok(size) - }, - ) + else { + return Ok(0); + }; + + if self + .operator + .info() + .full_capability() + .list_has_content_length + { + let size = files + .into_iter() + .filter_map(|entry| { + if entry.metadata().is_file() { + Some(entry.metadata().content_length()) + } else { + None + } + }) + .sum::(); + Ok(size) + } else { + // TODO: concurrent + let mut size = 0; + for entry in files { + let meta = handle_result(self.operator.stat(entry.path()))?; + if meta.is_file() { + size += meta.content_length(); + } + } + Ok(size) + } } }