chore(zarrs_opendal): bump minimum supported opendal to 0.51

LDeakin · Dec 23, 2024 · ec62841 · ec62841
1 parent 724b334
commit ec62841
Show file tree

Hide file tree

Showing 5 changed files with 71 additions and 29 deletions.
diff --git a/zarrs_opendal/CHANGELOG.md b/zarrs_opendal/CHANGELOG.md
@@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Changed
- - **Breaking**: Bump maximum supported `opendal` to 0.51
+ - **Breaking**: Bump `opendal` to 0.51
 
 ## [0.4.0] - 2024-11-15
 

diff --git a/zarrs_opendal/Cargo.toml b/zarrs_opendal/Cargo.toml
@@ -18,7 +18,7 @@ workspace = true
 [dependencies]
 async-trait = "0.1.74"
 futures = "0.3.29"
-opendal = { version = ">=0.46,<0.52", default-features = false }
+opendal = { version = ">=0.51,<0.52", default-features = false }
 zarrs_storage = { workspace = true, features = ["async"] }
 
 [dev-dependencies]

diff --git a/zarrs_opendal/doc/version_compatibility_matrix.md b/zarrs_opendal/doc/version_compatibility_matrix.md
@@ -1,6 +1,6 @@
 | [zarrs_opendal] | [opendal] | [zarrs] ([zarrs_storage]) |
 | --------------- | --------- | ------------------------- |
-| 0.5             | 0.46-0.51 | 0.18.x (0.3.x)            |
+| 0.5             | 0.51-0.51 | 0.18.x (0.3.x)            |
 | 0.4             | 0.46-0.50 | 0.18.x (0.3.x)            |
 | 0.3             | 0.46-0.50 | 0.17.x (0.2.x)            |
 

diff --git a/zarrs_opendal/src/async.rs b/zarrs_opendal/src/async.rs
@@ -83,7 +83,7 @@ impl AsyncWritableStorageTraits for AsyncOpendalStore {
     }
 
     async fn erase(&self, key: &StoreKey) -> Result<(), StorageError> {
-        handle_result(self.operator.remove(vec![key.to_string()]).await)
+        handle_result(self.operator.delete(key.as_str()).await)
     }
 
     async fn erase_prefix(&self, prefix: &StorePrefix) -> Result<(), StorageError> {
@@ -157,23 +157,44 @@ impl AsyncListableStorageTraits for AsyncOpendalStore {
     }
 
     async fn size_prefix(&self, prefix: &StorePrefix) -> Result<u64, StorageError> {
-        handle_result_notfound(
+        let Some(files) = handle_result_notfound(
             self.operator
                 .list_with(prefix.as_str())
                 .recursive(true)
-                .metakey(opendal::Metakey::ContentLength)
                 .await,
         )?
-        .map_or_else(
-            || Ok(0),
-            |list| {
-                let size = list
-                    .into_iter()
-                    .map(|entry| entry.metadata().content_length())
-                    .sum::<u64>();
-                Ok(size)
-            },
-        )
+        else {
+            return Ok(0);
+        };
+
+        if self
+            .operator
+            .info()
+            .full_capability()
+            .list_has_content_length
+        {
+            let size = files
+                .into_iter()
+                .filter_map(|entry| {
+                    if entry.metadata().is_file() {
+                        Some(entry.metadata().content_length())
+                    } else {
+                        None
+                    }
+                })
+                .sum::<u64>();
+            Ok(size)
+        } else {
+            // TODO: concurrent
+            let mut size = 0;
+            for entry in files {
+                let meta = handle_result(self.operator.stat(entry.path()).await)?;
+                if meta.is_file() {
+                    size += meta.content_length();
+                }
+            }
+            Ok(size)
+        }
     }
 }
 

diff --git a/zarrs_opendal/src/sync.rs b/zarrs_opendal/src/sync.rs
@@ -71,7 +71,7 @@ impl WritableStorageTraits for OpendalStore {
     }
 
     fn erase(&self, key: &StoreKey) -> Result<(), StorageError> {
-        handle_result(self.operator.remove(vec![key.to_string()]))
+        handle_result(self.operator.delete(key.as_str()))
     }
 
     fn erase_prefix(&self, prefix: &StorePrefix) -> Result<(), StorageError> {
@@ -145,23 +145,44 @@ impl ListableStorageTraits for OpendalStore {
     }
 
     fn size_prefix(&self, prefix: &StorePrefix) -> Result<u64, StorageError> {
-        handle_result_notfound(
+        let Some(files) = handle_result_notfound(
             self.operator
                 .list_with(prefix.as_str())
                 .recursive(true)
-                .metakey(opendal::Metakey::ContentLength)
                 .call(),
         )?
-        .map_or_else(
-            || Ok(0),
-            |list| {
-                let size = list
-                    .into_iter()
-                    .map(|entry| entry.metadata().content_length())
-                    .sum::<u64>();
-                Ok(size)
-            },
-        )
+        else {
+            return Ok(0);
+        };
+
+        if self
+            .operator
+            .info()
+            .full_capability()
+            .list_has_content_length
+        {
+            let size = files
+                .into_iter()
+                .filter_map(|entry| {
+                    if entry.metadata().is_file() {
+                        Some(entry.metadata().content_length())
+                    } else {
+                        None
+                    }
+                })
+                .sum::<u64>();
+            Ok(size)
+        } else {
+            // TODO: concurrent
+            let mut size = 0;
+            for entry in files {
+                let meta = handle_result(self.operator.stat(entry.path()))?;
+                if meta.is_file() {
+                    size += meta.content_length();
+                }
+            }
+            Ok(size)
+        }
     }
 }