ST2 Metrics Collection

CHANGELOG.rst

@@ -26,20 +26,6 @@ Added
   deployments to override ``COMPONENTS`` and ``ST2_CONF`` in a global location
   so ``st2ctl`` can start/stop/restart selected components and utilize a non-default
   location for ``st2.conf``.
-  (new feature) #4027
-
-  Contributed by Nick Maludy (Encore Technologies).
-* Add support for new optional ``content_version`` runner parameter to the Python and Local Shell
-  Script runner. This parameter can contain a git commit hash / tag / branch from a pack git
-  repository and runner will ensure this revision of the pack content (Python action / local shell
-  script action) is used for a particular action execution.
-
-  Keep in mind that providing this parameter only ensures a particular revision of the pack content
-  is used. Python runner virtual environment and dependencies are outside of this scope.
-
-  Note: To be able to utilize this functionality, git version >= 2.5.0 must be installed on the
-  system.
-  (new feature) #3997
 
 Changed
 ~~~~~~~
@@ -64,6 +50,8 @@ Changed
   (improvement)
 * Throw a more user-friendly error when writing pack data files to disk and when an invalid file
   path is provided (e.g. path is outside the pack directory, etc.). (improvement) #4039 #4046
+* Added metrics for collecting performance and health information about
+  the various ST2 services and functions.
 
 Fixed
 ~~~~~

conf/st2.conf.sample

@@ -180,6 +180,16 @@ connection_retries = 10
 # URL of all the nodes in a messaging service cluster.
 cluster_urls =  # comma separated list allowed here.
 
+[metrics]
+# Destination server to connect to if driver requires connection.
+host = 127.0.0.1
+# Enable metrics collection.
+enable = False
+# Driver type for metrics collection.
+driver = statsd
+# Destination port to connect to if driver requires connection.
+port = 8125
+
 [mistral]
 # URL Mistral uses to talk back to the API.If not provided it defaults to public API URL. Note: This needs to be a base URL without API version (e.g. http://127.0.0.1:9101)
 api_url = None

conf/st2.dev.conf

@@ -111,3 +111,9 @@ jitter_interval = 0
 
 [packs]
 enable_common_libs = True
+
+[metrics]
+enable = True
+driver = statsd
+host = 127.0.0.1
+port = 8125

contrib/runners/python_runner/python_runner/python_runner.py

@@ -53,6 +53,10 @@
 from st2common.runners.utils import make_read_and_store_stream_func
 
 from python_runner import python_action_wrapper
+from st2common.metrics.metrics import CounterWithTimer
+
+from st2common.constants.metrics import PYTHON_RUNNER_EXECUTION, PYTHON_WRAPPER_EXECUTION
+
 
 __all__ = [
     'PythonRunner',
@@ -119,7 +123,8 @@ def pre_run(self):
         if self._log_level == PYTHON_RUNNER_DEFAULT_LOG_LEVEL:
             self._log_level = cfg.CONF.actionrunner.python_runner_log_level
 
-    def run(self, action_parameters):
+    @CounterWithTimer(PYTHON_RUNNER_EXECUTION)
+    def run(self, action_parameters, **kwargs):
         LOG.debug('Running pythonrunner.')
         LOG.debug('Getting pack name.')
         pack = self.get_pack_ref()
@@ -249,18 +254,21 @@ def run(self, action_parameters):
 
         LOG.debug('Running command: PATH=%s PYTHONPATH=%s %s' % (env['PATH'], env['PYTHONPATH'],
                                                                  command_string))
-        exit_code, stdout, stderr, timed_out = run_command(cmd=args,
-                                                           stdin=stdin,
-                                                           stdout=subprocess.PIPE,
-                                                           stderr=subprocess.PIPE,
-                                                           shell=False,
-                                                           env=env,
-                                                           timeout=self._timeout,
-                                                           read_stdout_func=read_and_store_stdout,
-                                                           read_stderr_func=read_and_store_stderr,
-                                                           read_stdout_buffer=stdout,
-                                                           read_stderr_buffer=stderr,
-                                                           stdin_value=stdin_params)
+        with CounterWithTimer(PYTHON_WRAPPER_EXECUTION):
+            exit_code, stdout, stderr, timed_out = run_command(
+                cmd=args,
+                stdin=stdin,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                shell=False,
+                env=env,
+                timeout=self._timeout,
+                read_stdout_func=read_and_store_stdout,
+                read_stderr_func=read_and_store_stderr,
+                read_stdout_buffer=stdout,
+                read_stderr_buffer=stderr,
+                stdin_value=stdin_params
+            )
         LOG.debug('Returning values: %s, %s, %s, %s', exit_code, stdout, stderr, timed_out)
         LOG.debug('Returning.')
         return self._get_output_values(exit_code, stdout, stderr, timed_out)

contrib/runners/python_runner/tests/unit/test_pythonrunner.py

@@ -24,7 +24,6 @@
 import unittest2
 from oslo_config import cfg
 
-from python_runner import python_runner
 from st2actions.container.base import RunnerContainer
 from st2common.runners.base_action import Action
 from st2common.runners.utils import get_action_class_instance
@@ -35,13 +34,17 @@
 from st2common.constants.action import MAX_PARAM_LENGTH
 from st2common.constants.pack import SYSTEM_PACK_NAME
 from st2common.persistence.execution import ActionExecutionOutput
+from st2common import config as st2common_config
 from python_runner.python_action_wrapper import PythonActionWrapper
 from st2tests.base import RunnerTestCase
 from st2tests.base import CleanDbTestCase
 from st2tests.base import blocking_eventlet_spawn
 from st2tests.base import make_mock_stream_readline
 import st2tests.base as tests_base
 
+st2common_config.parse_args(args=[])
+
+from python_runner import python_runner
 
 PASCAL_ROW_ACTION_PATH = os.path.join(tests_base.get_resources_path(), 'packs',
                                       'pythonactions/actions/pascal_row.py')

fixed-requirements.txt

@@ -39,6 +39,7 @@ prompt-toolkit==1.0.15
 tooz==1.60.0
 zake==0.2.2
 routes==2.4.1
+statsd==3.2.2
 flex==6.12.0
 webob==1.7.4
 prance==0.9.0

requirements.txt

@@ -26,6 +26,7 @@ oslo.utils<=3.33.0,>=3.15.0
 paramiko==2.4.0
 passlib==1.7.1
 prettytable
+prometheus_client==0.1.1
 prompt-toolkit==1.0.15
 psutil
 pyinotify==0.9.6
@@ -44,6 +45,7 @@ routes==2.4.1
 semver==2.7.7
 six==1.10.0
 sseclient==0.0.18
+statsd==3.2.2
 stevedore==1.28.0
 tooz==1.60.0
 unittest2

st2common/in-requirements.txt

@@ -27,3 +27,5 @@ routes
 flex
 webob
 jsonpath-rw
+prometheus_client
+statsd

st2common/st2common/config.py

@@ -390,6 +390,19 @@ def register_opts(ignore_errors=False):
     cli_opts = [debug, profile, use_debugger]
     do_register_cli_opts(cli_opts, ignore_errors=ignore_errors)
 
+    # Metrics Options stream options
+    metrics_opts = [
+        cfg.BoolOpt('enable', default=False,
+                    help='Enable metrics collection.'),
+        cfg.StrOpt('driver', default='statsd',
+                   help='Driver type for metrics collection.'),
+        cfg.StrOpt('host', default='127.0.0.1',
+                   help='Destination server to connect to if driver requires connection.'),
+        cfg.IntOpt('port', default=8125,
+                   help='Destination port to connect to if driver requires connection.'),
+    ]
+    do_register_opts(metrics_opts, group='metrics', ignore_errors=ignore_errors)
+
 
 def parse_args(args=None):
     register_opts()

st2common/st2common/constants/metrics.py

@@ -0,0 +1,22 @@
+# Licensed to the StackStorm, Inc ('StackStorm') under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+METRICS_COUNTER_SUFFIX = "_counter"
+METRICS_TIMER_SUFFIX = "_timer"
+
+PYTHON_RUNNER_EXECUTION = "python_runner_execution"
+PYTHON_WRAPPER_EXECUTION = "python_wrapper_execution"
+
+METRICS_REGISTER_RUNNER = "register_runner"

st2common/st2common/metrics/__init__.py

@@ -0,0 +1,14 @@
+# Licensed to the StackStorm, Inc ('StackStorm') under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

st2common/st2common/metrics/drivers/__init__.py

@@ -0,0 +1,14 @@
+# Licensed to the StackStorm, Inc ('StackStorm') under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

st2common/st2common/metrics/drivers/prometheus.py

@@ -0,0 +1,48 @@
+# Licensed to the StackStorm, Inc ('StackStorm') under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from prometheus_client import Histogram, Gauge
+
+from st2common.metrics.metrics import BaseMetricsDriver
+
+
+class PrometheusDriver(BaseMetricsDriver):
+    """ Base class for driver implementations for metric collection
+    """
+    def __init__(self):
+        pass
+
+    def time(self, key, time):
+        """ Timer metric
+        """
+        prometheus_histogram = Histogram(  # pylint: disable=no-value-for-parameter
+            key
+        )
+        prometheus_histogram.observe(time)
+
+    def inc_counter(self, key, amount=1):
+        """ Increment counter
+        """
+        prometheus_counter = Gauge(  # pylint: disable=no-value-for-parameter
+            key
+        )
+        prometheus_counter.inc(amount)
+
+    def dec_counter(self, key, amount=1):
+        """ Decrement metric
+        """
+        prometheus_counter = Gauge(  # pylint: disable=no-value-for-parameter
+            key
+        )
+        prometheus_counter.dec(amount)

st2common/st2common/metrics/drivers/statsd_driver.py

@@ -0,0 +1,47 @@
+# Licensed to the StackStorm, Inc ('StackStorm') under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from numbers import Number
+from oslo_config import cfg
+import statsd
+
+from st2common.metrics.metrics import BaseMetricsDriver
+
+
+class StatsdDriver(BaseMetricsDriver):
+    """ StatsD Implementation of the metrics driver
+    """
+    def __init__(self):
+        self._connection = statsd.StatsClient(cfg.CONF.metrics.host, cfg.CONF.metrics.port)
+
+    def time(self, key, time):
+        """ Timer metric
+        """
+        assert isinstance(key, str)
+        assert isinstance(time, Number)
+        self._connection.timing(key, time)
+
+    def inc_counter(self, key, amount=1):
+        """ Increment counter
+        """
+        assert isinstance(key, str)
+        assert isinstance(amount, Number)
+        self._connection.incr(key, amount)
+
+    def dec_counter(self, key, amount=1):
+        """ Decrement metric
+        """
+        assert isinstance(key, str)
+        assert isinstance(amount, Number)
+        self._connection.decr(key, amount)