From b873884c2bf472af02d74324fc560d7e3cc9e03d Mon Sep 17 00:00:00 2001 From: chenmoneygithub Date: Fri, 6 Sep 2024 11:46:00 -0700 Subject: [PATCH] Reduce system metrics logging frequency --- composer/loggers/mlflow_logger.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/composer/loggers/mlflow_logger.py b/composer/loggers/mlflow_logger.py index 7a8acdd511..b87161d0f8 100644 --- a/composer/loggers/mlflow_logger.py +++ b/composer/loggers/mlflow_logger.py @@ -174,9 +174,9 @@ def __init__( if log_system_metrics: # Set system metrics sampling interval and samples before logging so that system metrics - # are collected every 5s, and aggregated over 3 samples before being logged - # (logging per 15s). - mlflow.set_system_metrics_samples_before_logging(3) + # are collected every 5s, and aggregated over 6 samples before being logged + # (logging per 30s). + mlflow.set_system_metrics_samples_before_logging(6) mlflow.set_system_metrics_sampling_interval(5) self._rank_zero_only = rank_zero_only @@ -545,7 +545,11 @@ def register_model_with_run_id( """ if self._enabled: from mlflow.exceptions import MlflowException - from mlflow.protos.databricks_pb2 import ALREADY_EXISTS, RESOURCE_ALREADY_EXISTS, ErrorCode + from mlflow.protos.databricks_pb2 import ( + ALREADY_EXISTS, + RESOURCE_ALREADY_EXISTS, + ErrorCode, + ) full_name = f'{self.model_registry_prefix}.{name}' if len(self.model_registry_prefix) > 0 else name