-
Notifications
You must be signed in to change notification settings - Fork 590
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
support retry_unknown_service_error for opendal s3
- Loading branch information
Showing
4 changed files
with
88 additions
and
27 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -567,6 +567,7 @@ impl<OS: ObjectStore> MonitoredObjectStore<OS> { | |
pub async fn upload(&self, path: &str, obj: Bytes) -> ObjectResult<()> { | ||
let operation_type = OperationType::Upload; | ||
let operation_type_str = operation_type.as_str(); | ||
let media_type = self.media_type(); | ||
This comment has been minimized.
Sorry, something went wrong.
This comment has been minimized.
Sorry, something went wrong.
hzxa21
Author
Collaborator
|
||
|
||
self.object_store_metrics | ||
.write_bytes | ||
|
@@ -578,7 +579,7 @@ impl<OS: ObjectStore> MonitoredObjectStore<OS> { | |
let _timer = self | ||
.object_store_metrics | ||
.operation_latency | ||
.with_label_values(&[self.media_type(), operation_type_str]) | ||
.with_label_values(&[media_type, operation_type_str]) | ||
.start_timer(); | ||
|
||
let builder = || async { | ||
|
@@ -593,6 +594,7 @@ impl<OS: ObjectStore> MonitoredObjectStore<OS> { | |
&self.config, | ||
operation_type, | ||
self.object_store_metrics.clone(), | ||
media_type, | ||
) | ||
.await; | ||
|
||
|
@@ -630,10 +632,12 @@ impl<OS: ObjectStore> MonitoredObjectStore<OS> { | |
pub async fn read(&self, path: &str, range: impl ObjectRangeBounds) -> ObjectResult<Bytes> { | ||
let operation_type = OperationType::Read; | ||
let operation_type_str = operation_type.as_str(); | ||
let media_type = self.media_type(); | ||
|
||
let _timer = self | ||
.object_store_metrics | ||
.operation_latency | ||
.with_label_values(&[self.media_type(), operation_type_str]) | ||
.with_label_values(&[media_type, operation_type_str]) | ||
.start_timer(); | ||
|
||
let builder = || async { | ||
|
@@ -648,6 +652,7 @@ impl<OS: ObjectStore> MonitoredObjectStore<OS> { | |
&self.config, | ||
operation_type, | ||
self.object_store_metrics.clone(), | ||
media_type, | ||
) | ||
.await; | ||
|
||
|
@@ -701,6 +706,7 @@ impl<OS: ObjectStore> MonitoredObjectStore<OS> { | |
&self.config, | ||
operation_type, | ||
self.object_store_metrics.clone(), | ||
media_type, | ||
) | ||
.await; | ||
|
||
|
@@ -719,10 +725,11 @@ impl<OS: ObjectStore> MonitoredObjectStore<OS> { | |
pub async fn metadata(&self, path: &str) -> ObjectResult<ObjectMetadata> { | ||
let operation_type = OperationType::Metadata; | ||
let operation_type_str = operation_type.as_str(); | ||
let media_type = self.media_type(); | ||
let _timer = self | ||
.object_store_metrics | ||
.operation_latency | ||
.with_label_values(&[self.media_type(), operation_type_str]) | ||
.with_label_values(&[media_type, operation_type_str]) | ||
.start_timer(); | ||
|
||
let builder = || async { | ||
|
@@ -737,6 +744,7 @@ impl<OS: ObjectStore> MonitoredObjectStore<OS> { | |
&self.config, | ||
operation_type, | ||
self.object_store_metrics.clone(), | ||
media_type, | ||
) | ||
.await; | ||
|
||
|
@@ -747,10 +755,12 @@ impl<OS: ObjectStore> MonitoredObjectStore<OS> { | |
pub async fn delete(&self, path: &str) -> ObjectResult<()> { | ||
let operation_type = OperationType::Delete; | ||
let operation_type_str = operation_type.as_str(); | ||
let media_type = self.media_type(); | ||
|
||
let _timer = self | ||
.object_store_metrics | ||
.operation_latency | ||
.with_label_values(&[self.media_type(), operation_type_str]) | ||
.with_label_values(&[media_type, operation_type_str]) | ||
.start_timer(); | ||
|
||
let builder = || async { | ||
|
@@ -765,6 +775,7 @@ impl<OS: ObjectStore> MonitoredObjectStore<OS> { | |
&self.config, | ||
operation_type, | ||
self.object_store_metrics.clone(), | ||
media_type, | ||
) | ||
.await; | ||
|
||
|
@@ -775,6 +786,8 @@ impl<OS: ObjectStore> MonitoredObjectStore<OS> { | |
async fn delete_objects(&self, paths: &[String]) -> ObjectResult<()> { | ||
let operation_type = OperationType::DeleteObjects; | ||
let operation_type_str = operation_type.as_str(); | ||
let media_type = self.media_type(); | ||
|
||
let _timer = self | ||
.object_store_metrics | ||
.operation_latency | ||
|
@@ -793,6 +806,7 @@ impl<OS: ObjectStore> MonitoredObjectStore<OS> { | |
&self.config, | ||
operation_type, | ||
self.object_store_metrics.clone(), | ||
media_type, | ||
) | ||
.await; | ||
|
||
|
@@ -803,11 +817,12 @@ impl<OS: ObjectStore> MonitoredObjectStore<OS> { | |
pub async fn list(&self, prefix: &str) -> ObjectResult<ObjectMetadataIter> { | ||
let operation_type = OperationType::List; | ||
let operation_type_str = operation_type.as_str(); | ||
let media_type = self.media_type(); | ||
|
||
let _timer = self | ||
.object_store_metrics | ||
.operation_latency | ||
.with_label_values(&[self.media_type(), operation_type_str]) | ||
.with_label_values(&[media_type, operation_type_str]) | ||
.start_timer(); | ||
|
||
let builder = || async { | ||
|
@@ -822,6 +837,7 @@ impl<OS: ObjectStore> MonitoredObjectStore<OS> { | |
&self.config, | ||
operation_type, | ||
self.object_store_metrics.clone(), | ||
media_type, | ||
) | ||
.await; | ||
|
||
|
@@ -1101,20 +1117,26 @@ struct RetryCondition { | |
operation_type: OperationType, | ||
retry_count: usize, | ||
metrics: Arc<ObjectStoreMetrics>, | ||
retry_opendal_s3_unknown_error: bool, | ||
} | ||
|
||
impl RetryCondition { | ||
fn new(operation_type: OperationType, metrics: Arc<ObjectStoreMetrics>) -> Self { | ||
fn new( | ||
operation_type: OperationType, | ||
metrics: Arc<ObjectStoreMetrics>, | ||
retry_opendal_s3_unknown_error: bool, | ||
) -> Self { | ||
Self { | ||
operation_type, | ||
retry_count: 0, | ||
metrics, | ||
retry_opendal_s3_unknown_error, | ||
} | ||
} | ||
|
||
#[inline(always)] | ||
fn should_retry_inner(&mut self, err: &ObjectError) -> bool { | ||
let should_retry = err.should_retry(); | ||
let should_retry = err.should_retry(self.retry_opendal_s3_unknown_error); | ||
if should_retry { | ||
self.retry_count += 1; | ||
} | ||
|
@@ -1145,6 +1167,7 @@ async fn retry_request<F, T, B>( | |
config: &ObjectStoreConfig, | ||
operation_type: OperationType, | ||
object_store_metrics: Arc<ObjectStoreMetrics>, | ||
media_type: &'static str, | ||
) -> ObjectResult<T> | ||
where | ||
B: Fn() -> F, | ||
|
@@ -1155,7 +1178,13 @@ where | |
Duration::from_millis(get_attempt_timeout_by_type(config, operation_type)); | ||
let operation_type_str = operation_type.as_str(); | ||
|
||
let retry_condition = RetryCondition::new(operation_type, object_store_metrics); | ||
let retry_condition = RetryCondition::new( | ||
operation_type, | ||
object_store_metrics, | ||
config.s3.developer.retry_unknown_service_error | ||
&& (media_type == opendal_engine::EngineType::S3.as_str() | ||
|| media_type == opendal_engine::EngineType::Minio.as_str()), | ||
); | ||
|
||
let f = || async { | ||
let future = builder(); | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
nits: How about using the same variable name?
media_type
andengine_type
are too similar.