Skip to content

Commit

Permalink
Reduce system metrics logging frequency (#3604)
Browse files Browse the repository at this point in the history
  • Loading branch information
chenmoneygithub authored Sep 6, 2024
1 parent 7d34d66 commit 7dfd4d3
Showing 1 changed file with 8 additions and 4 deletions.
12 changes: 8 additions & 4 deletions composer/loggers/mlflow_logger.py
Original file line number Diff line number Diff line change
Expand Up @@ -174,9 +174,9 @@ def __init__(

if log_system_metrics:
# Set system metrics sampling interval and samples before logging so that system metrics
# are collected every 5s, and aggregated over 3 samples before being logged
# (logging per 15s).
mlflow.set_system_metrics_samples_before_logging(3)
# are collected every 5s, and aggregated over 6 samples before being logged
# (logging per 30s).
mlflow.set_system_metrics_samples_before_logging(6)
mlflow.set_system_metrics_sampling_interval(5)

self._rank_zero_only = rank_zero_only
Expand Down Expand Up @@ -545,7 +545,11 @@ def register_model_with_run_id(
"""
if self._enabled:
from mlflow.exceptions import MlflowException
from mlflow.protos.databricks_pb2 import ALREADY_EXISTS, RESOURCE_ALREADY_EXISTS, ErrorCode
from mlflow.protos.databricks_pb2 import (
ALREADY_EXISTS,
RESOURCE_ALREADY_EXISTS,
ErrorCode,
)

full_name = f'{self.model_registry_prefix}.{name}' if len(self.model_registry_prefix) > 0 else name

Expand Down

0 comments on commit 7dfd4d3

Please sign in to comment.