Merge pull request #4016 from StackStorm/pack_install_use_python3_bin…

…ary_parameter

Add new --python3 flag to st2 pack install CLI command

CHANGELOG.rst

@@ -13,6 +13,14 @@ Added
 * Add support for utf-8 / unicode characters in the pack config files. (improvement) #3980 #3989
 
   Contributed by @sumkire.
+* Add new ``--python3`` flag to ``st2 pack install`` CLI command and ``python3`` parameter to
+  ``packs.{install,setup_virtualenv}`` actions. When the value of this parameter is True, it
+  uses ``python3`` binary when creating virtual environment for that pack (based on the value of
+  ``actionrunner.python3_binary`` config option).
+
+  Note: For this feature to work, Python 3 needs to be installed on the system and ``virtualenv``
+  package installed on the system needs to support Python 3 (it needs to be a recent version).
+  (new feature) #4016 #3922
 
 Changed
 ~~~~~~~

conf/st2.conf.sample

@@ -13,15 +13,17 @@ python_runner_log_level = DEBUG
 # Internal pool size for dispatcher used by workflow actions.
 workflows_pool_size = 40
 # Virtualenv binary which should be used to create pack virtualenvs.
-virtualenv_binary = /data/stanley/virtualenv/bin/virtualenv
+virtualenv_binary = /usr/bin/virtualenv
+# Python 3 binary which will be used by Python actions for packs which use Python 3 virtual environment
+python3_binary = /usr/bin/python3
 # location of the logging.conf file
 logging = conf/logging.conf
 # True to store and stream action output (stdout and stderr) in real-time.
 stream_output = True
 # List of virtualenv options to be passsed to "virtualenv" command that creates pack virtualenv.
 virtualenv_opts = --system-site-packages # comma separated list allowed here.
 # Python binary which will be used by Python actions.
-python_binary = /data/stanley/virtualenv/bin/python
+python_binary = /usr/bin/python
 
 [api]
 # List of origins allowed for api, auth and stream

contrib/packs/actions/install.meta.yaml

@@ -27,3 +27,8 @@
       description: "Set to True to force install the pack and skip StackStorm version compatibility check and also delete and ignore lock file if one exists."
       required: false
       default: false
+    python3:
+      type: "boolean"
+      description: "Use Python 3 binary when creating a virtualenv for this pack."
+      required: false
+      default: false

contrib/packs/actions/pack_mgmt/setup_virtualenv.py

@@ -74,7 +74,7 @@ def __init__(self, config=None, action_service=None):
         if self.proxy_ca_bundle_path and not os.environ.get('proxy_ca_bundle_path', None):
             os.environ['no_proxy'] = self.no_proxy
 
-    def run(self, packs, update=False):
+    def run(self, packs, update=False, python3=False):
         """
         :param packs: A list of packs to create the environment for.
         :type: packs: ``list``
@@ -85,7 +85,7 @@ def run(self, packs, update=False):
 
         for pack_name in packs:
             setup_pack_virtualenv(pack_name=pack_name, update=update, logger=self.logger,
-                                  proxy_config=self.proxy_config)
+                                  proxy_config=self.proxy_config, use_python3=python3)
 
         message = ('Successfuly set up virtualenv for the following packs: %s' %
                    (', '.join(packs)))

contrib/packs/actions/setup_virtualenv.yaml

@@ -7,12 +7,18 @@
   parameters:
     packs:
       type: "array"
+      description: "List of packs to create virtualenv for."
       items:
         type: "string"
     update:
       type: "boolean"
       default: false
       description: "Check this option if the virtual environment already exists and if you only want to perform an update and installation of new dependencies. If you don't check this option, the virtual environment will be destroyed then re-created. If you check this and the virtual environment doesn't exist, it will create it."
+    python3:
+      type: "boolean"
+      description: "Use Python 3 binary when creating a virtualenv for this pack."
+      required: false
+      default: false
     env:
       type: "object"
       description: "Optional environment variables"

contrib/packs/actions/workflows/install.yaml

@@ -19,6 +19,7 @@
       parameters:
         packs: "{{ __results['make a prerun'].result }}"
         env: "{{env}}"
+        python3: "{{python3}}"
       on-success: "register pack"
     -
       name: "register pack"

st2api/st2api/controllers/v1/packs.py

@@ -98,6 +98,7 @@ class PackInstallController(ActionExecutionsControllerMixin):
     def post(self, pack_install_request, requester_user=None):
         parameters = {
             'packs': pack_install_request.packs,
+            'python3': pack_install_request.python3
         }
 
         if pack_install_request.force:

st2client/st2client/commands/pack.py

@@ -185,14 +185,18 @@ def __init__(self, resource, *args, **kwargs):
                                  metavar='pack',
                                  help='Name of the %s in Exchange, or a git repo URL.' %
                                  resource.get_plural_display_name().lower())
+        self.parser.add_argument('--python3',
+                                 action='store_true',
+                                 default=False,
+                                 help='Use Python 3 binary for pack virtual environment.')
         self.parser.add_argument('--force',
                                  action='store_true',
                                  default=False,
                                  help='Force pack installation.')
 
     def run(self, args, **kwargs):
         self._get_content_counts_for_pack(args, **kwargs)
-        return self.manager.install(args.packs, force=args.force, **kwargs)
+        return self.manager.install(args.packs, python3=args.python3, force=args.force, **kwargs)
 
     def _get_content_counts_for_pack(self, args, **kwargs):
         # Global content list, excluding "tests"

st2client/st2client/models/core.py

@@ -476,11 +476,12 @@ class AsyncRequest(Resource):
 
 class PackResourceManager(ResourceManager):
     @add_auth_token_to_kwargs_from_env
-    def install(self, packs, force=False, **kwargs):
+    def install(self, packs, force=False, python3=False, **kwargs):
         url = '/%s/install' % (self.resource.get_url_path_name())
         payload = {
             'packs': packs,
-            'force': force
+            'force': force,
+            'python3': python3
         }
         response = self.client.post(url, payload, **kwargs)
         if response.status_code != http_client.OK:

st2common/st2common/config.py

@@ -19,6 +19,7 @@
 import sys
 
 from oslo_config import cfg
+from distutils.spawn import find_executable
 
 from st2common.constants.system import VERSION_STRING
 from st2common.constants.runners import PYTHON_RUNNER_DEFAULT_LOG_LEVEL
@@ -221,8 +222,10 @@ def register_opts(ignore_errors=False):
 
     # Runner options
     default_python_bin_path = sys.executable
+    default_python3_bin_path = find_executable('python3')
     base_dir = os.path.dirname(os.path.realpath(default_python_bin_path))
     default_virtualenv_bin_path = os.path.join(base_dir, 'virtualenv')
+
     action_runner_opts = [
         # Common runner options
         cfg.StrOpt('logging', default='conf/logging.conf',
@@ -231,6 +234,9 @@ def register_opts(ignore_errors=False):
         # Python runner options
         cfg.StrOpt('python_binary', default=default_python_bin_path,
                    help='Python binary which will be used by Python actions.'),
+        cfg.StrOpt('python3_binary', default=default_python3_bin_path,
+                   help=('Python 3 binary which will be used by Python actions for packs which '
+                         'use Python 3 virtual environment')),
         cfg.StrOpt('virtualenv_binary', default=default_virtualenv_bin_path,
                    help='Virtualenv binary which should be used to create pack virtualenvs.'),
         cfg.StrOpt('python_runner_log_level',

st2common/st2common/openapi.yaml

@@ -4607,6 +4607,10 @@ definitions:
         type: array
         items:
           type: string
+      python3:
+        description: Use Python 3 binary for pack virtual environment.
+        type: boolean
+        default: false
       force:
         description: Force pack installation.
         type: boolean

st2common/st2common/openapi.yaml.j2

@@ -4603,6 +4603,10 @@ definitions:
         type: array
         items:
           type: string
+      python3:
+        description: Use Python 3 binary for pack virtual environment.
+        type: boolean
+        default: false
       force:
         description: Force pack installation.
         type: boolean

st2common/st2common/util/virtualenvs.py

@@ -41,7 +41,8 @@
 
 
 def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True,
-                          include_setuptools=True, include_wheel=True, proxy_config=None):
+                          include_setuptools=True, include_wheel=True, proxy_config=None,
+                          use_python3=False):
 
     """
     Setup virtual environment for the provided pack.
@@ -54,6 +55,9 @@ def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True
 
     :param logger: Optional logger instance to use. If not provided it defaults to the module
                    level logger.
+
+    :param use_python3: Use Python3 binary when creating virtualenv for this pack.
+    :type use_python3: ``bool``
     """
     logger = logger or LOG
 
@@ -82,7 +86,8 @@ def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True
         # 1. Create virtual environment
         logger.debug('Creating virtualenv for pack "%s" in "%s"' % (pack_name, virtualenv_path))
         create_virtualenv(virtualenv_path=virtualenv_path, logger=logger, include_pip=include_pip,
-                          include_setuptools=include_setuptools, include_wheel=include_wheel)
+                          include_setuptools=include_setuptools, include_wheel=include_wheel,
+                          use_python3=use_python3)
 
     # 2. Install base requirements which are common to all the packs
     logger.debug('Installing base requirements')
@@ -110,7 +115,7 @@ def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True
 
 
 def create_virtualenv(virtualenv_path, logger=None, include_pip=True, include_setuptools=True,
-                      include_wheel=True):
+                      include_wheel=True, use_python3=False):
     """
     :param include_pip: Include pip binary and package in the newely created virtual environment.
     :type include_pip: ``bool``
@@ -121,11 +126,15 @@ def create_virtualenv(virtualenv_path, logger=None, include_pip=True, include_se
 
     :param include_wheel: Include wheel in the newely created virtual environment.
     :type include_wheel : ``bool``
+
+    :param use_python3: Use Python3 binary when creating virtualenv for this pack.
+    :type use_python3: ``bool``
     """
 
     logger = logger or LOG
 
     python_binary = cfg.CONF.actionrunner.python_binary
+    python3_binary = cfg.CONF.actionrunner.python3_binary
     virtualenv_binary = cfg.CONF.actionrunner.virtualenv_binary
     virtualenv_opts = cfg.CONF.actionrunner.virtualenv_opts
 
@@ -138,7 +147,18 @@ def create_virtualenv(virtualenv_path, logger=None, include_pip=True, include_se
     logger.debug('Creating virtualenv in "%s" using Python binary "%s"' %
                  (virtualenv_path, python_binary))
 
-    cmd = [virtualenv_binary, '-p', python_binary]
+    cmd = [virtualenv_binary]
+
+    if use_python3 and not python3_binary:
+        msg = ('Requested to use Python 3, but python3 binary not found on the system or '
+               'actionrunner.python3 config option is not configured correctly.')
+        raise ValueError(msg)
+
+    if use_python3:
+        cmd.extend(['-p', python3_binary])
+    else:
+        cmd.extend(['-p', python_binary])
+
     cmd.extend(virtualenv_opts)
 
     if not include_pip:

st2common/tests/unit/test_virtualenvs.py

@@ -290,6 +290,41 @@ def test_install_requirements_with_https_proxy_no_cert(self):
         }
         virtualenvs.run_command.assert_called_once_with(**expected_args)
 
+    @mock.patch.object(virtualenvs, 'run_command')
+    def test_setup_pack_virtualenv_use_python3_binary(self, mock_run_command):
+        mock_run_command.return_value = 0, '', ''
+
+        cfg.CONF.set_override(name='python_binary', group='actionrunner',
+                              override='/usr/bin/python2.7')
+        cfg.CONF.set_override(name='python3_binary', group='actionrunner',
+                              override='/usr/bin/python3')
+
+        pack_name = 'dummy_pack_2'
+
+        # Python 2
+        setup_pack_virtualenv(pack_name=pack_name, update=False,
+                              include_setuptools=False, include_wheel=False,
+                              use_python3=False)
+
+        actual_cmd = mock_run_command.call_args_list[0][1]['cmd']
+        actual_cmd = ' '.join(actual_cmd)
+
+        self.assertEqual(mock_run_command.call_count, 2)
+        self.assertTrue('-p /usr/bin/python2.7' in actual_cmd)
+
+        mock_run_command.reset_mock()
+
+        # Python 3
+        setup_pack_virtualenv(pack_name=pack_name, update=False,
+                              include_setuptools=False, include_wheel=False,
+                              use_python3=True)
+
+        actual_cmd = mock_run_command.call_args_list[0][1]['cmd']
+        actual_cmd = ' '.join(actual_cmd)
+
+        self.assertEqual(mock_run_command.call_count, 2)
+        self.assertTrue('-p /usr/bin/python3' in actual_cmd)
+
     def assertVirtulenvExists(self, virtualenv_dir):
         self.assertTrue(os.path.exists(virtualenv_dir))
         self.assertTrue(os.path.isdir(virtualenv_dir))

tools/config_gen.py

@@ -64,8 +64,9 @@
 # set them to static values to ensure consistent and stable output
 STATIC_OPTION_VALUES = {
     'actionrunner': {
-        'virtualenv_binary': '/data/stanley/virtualenv/bin/virtualenv',
-        'python_binary': '/data/stanley/virtualenv/bin/python'
+        'virtualenv_binary': '/usr/bin/virtualenv',
+        'python_binary': '/usr/bin/python',
+        'python3_binary': '/usr/bin/python3'
     },
     'webui': {
         'webui_base_url': 'https://localhost'