From be9c36fce7799ea542d25a6f333e8729f1c67ee1 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sat, 20 Mar 2021 23:24:38 +0100 Subject: [PATCH 1/3] Speed up runner registration by using ExtensionManager directly instead of creating new DriverManager instance per extension. Creating new instance is not needed and results in disk / pkg resources scan per runner (extension) which is slow. This also substantially reduces debug log noise on service start up. --- st2common/st2common/bootstrap/runnersregistrar.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/st2common/st2common/bootstrap/runnersregistrar.py b/st2common/st2common/bootstrap/runnersregistrar.py index bb99389433..6c6e20e0c8 100644 --- a/st2common/st2common/bootstrap/runnersregistrar.py +++ b/st2common/st2common/bootstrap/runnersregistrar.py @@ -15,7 +15,6 @@ from __future__ import absolute_import -from stevedore.driver import DriverManager from stevedore.extension import ExtensionManager from st2common import log as logging @@ -41,15 +40,13 @@ def register_runners(experimental=False, fail_on_failure=True): runner_count = 0 manager = ExtensionManager(namespace=RUNNERS_NAMESPACE, invoke_on_load=False) - extension_names = manager.names() - for name in extension_names: + # NOTE: We use ExtensionManager directly instead of DriverManager per extension since that is + # much faster and allows us to reduce stevedore loading overead for each runner + for extension in manager.extensions: + name = extension.name LOG.debug('Found runner "%s"' % (name)) - - manager = DriverManager( - namespace=RUNNERS_NAMESPACE, invoke_on_load=False, name=name - ) - runner_metadata = manager.driver.get_metadata() + runner_metadata = extension.plugin.get_metadata() runner_count += register_runner(runner_metadata, experimental) LOG.debug("End : register runners") From 90921097810659cd68e223340a3988059d9fe30d Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sat, 20 Mar 2021 23:39:43 +0100 Subject: [PATCH 2/3] Add changelog entry. --- CHANGELOG.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index c86827565e..72367a3fe2 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -156,6 +156,13 @@ Improvements Contributed by @Kami. +* Speed up service start up time by speeding up runners registration on service start up by + re-using existing stevedore ``ExtensionManager`` instance instead of instantiating new + ``DriverManager`` instance per extension which is not necessary and it's slow since it requires + disk / pkg resources scan for each extension. (improvement) #5198 + + Contributed by @Kami. + Fixed ~~~~~ From 6698226e7acc6e324a981ab2f664cd56fbcf8146 Mon Sep 17 00:00:00 2001 From: Tomaz Muraus Date: Sun, 21 Mar 2021 16:12:53 +0100 Subject: [PATCH 3/3] Fix typo. --- st2common/st2common/bootstrap/runnersregistrar.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/st2common/st2common/bootstrap/runnersregistrar.py b/st2common/st2common/bootstrap/runnersregistrar.py index 6c6e20e0c8..3d30346b98 100644 --- a/st2common/st2common/bootstrap/runnersregistrar.py +++ b/st2common/st2common/bootstrap/runnersregistrar.py @@ -42,7 +42,7 @@ def register_runners(experimental=False, fail_on_failure=True): manager = ExtensionManager(namespace=RUNNERS_NAMESPACE, invoke_on_load=False) # NOTE: We use ExtensionManager directly instead of DriverManager per extension since that is - # much faster and allows us to reduce stevedore loading overead for each runner + # much faster and allows us to reduce stevedore loading overhead for each runner for extension in manager.extensions: name = extension.name LOG.debug('Found runner "%s"' % (name)) @@ -96,7 +96,6 @@ def register_runner(runner_type, experimental): runner_type_model.id = runner_type_db.id try: - runner_type_db = RunnerType.add_or_update(runner_type_model) extra = {"runner_type_db": runner_type_db}