diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ced938ed9e..8210ac8baf 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -13,6 +13,14 @@ Added * Add support for utf-8 / unicode characters in the pack config files. (improvement) #3980 #3989 Contributed by @sumkire. +* Add new ``--python3`` flag to ``st2 pack install`` CLI command and ``python3`` parameter to + ``packs.{install,setup_virtualenv}`` actions. When the value of this parameter is True, it + uses ``python3`` binary when creating virtual environment for that pack (based on the value of + ``actionrunner.python3_binary`` config option). + + Note: For this feature to work, Python 3 needs to be installed on the system and ``virtualenv`` + package installed on the system needs to support Python 3 (it needs to be a recent version). + (new feature) #4016 #3922 Changed ~~~~~~~ diff --git a/conf/st2.conf.sample b/conf/st2.conf.sample index eb607bc38f..5bebfc0630 100644 --- a/conf/st2.conf.sample +++ b/conf/st2.conf.sample @@ -13,7 +13,9 @@ python_runner_log_level = DEBUG # Internal pool size for dispatcher used by workflow actions. workflows_pool_size = 40 # Virtualenv binary which should be used to create pack virtualenvs. -virtualenv_binary = /data/stanley/virtualenv/bin/virtualenv +virtualenv_binary = /usr/bin/virtualenv +# Python 3 binary which will be used by Python actions for packs which use Python 3 virtual environment +python3_binary = /usr/bin/python3 # location of the logging.conf file logging = conf/logging.conf # True to store and stream action output (stdout and stderr) in real-time. @@ -21,7 +23,7 @@ stream_output = True # List of virtualenv options to be passsed to "virtualenv" command that creates pack virtualenv. virtualenv_opts = --system-site-packages # comma separated list allowed here. # Python binary which will be used by Python actions. -python_binary = /data/stanley/virtualenv/bin/python +python_binary = /usr/bin/python [api] # List of origins allowed for api, auth and stream diff --git a/contrib/packs/actions/install.meta.yaml b/contrib/packs/actions/install.meta.yaml index 3958ec00bb..46632de72a 100644 --- a/contrib/packs/actions/install.meta.yaml +++ b/contrib/packs/actions/install.meta.yaml @@ -27,3 +27,8 @@ description: "Set to True to force install the pack and skip StackStorm version compatibility check and also delete and ignore lock file if one exists." required: false default: false + python3: + type: "boolean" + description: "Use Python 3 binary when creating a virtualenv for this pack." + required: false + default: false diff --git a/contrib/packs/actions/pack_mgmt/setup_virtualenv.py b/contrib/packs/actions/pack_mgmt/setup_virtualenv.py index 57e5bb2dd4..1231aa1282 100644 --- a/contrib/packs/actions/pack_mgmt/setup_virtualenv.py +++ b/contrib/packs/actions/pack_mgmt/setup_virtualenv.py @@ -74,7 +74,7 @@ def __init__(self, config=None, action_service=None): if self.proxy_ca_bundle_path and not os.environ.get('proxy_ca_bundle_path', None): os.environ['no_proxy'] = self.no_proxy - def run(self, packs, update=False): + def run(self, packs, update=False, python3=False): """ :param packs: A list of packs to create the environment for. :type: packs: ``list`` @@ -85,7 +85,7 @@ def run(self, packs, update=False): for pack_name in packs: setup_pack_virtualenv(pack_name=pack_name, update=update, logger=self.logger, - proxy_config=self.proxy_config) + proxy_config=self.proxy_config, use_python3=python3) message = ('Successfuly set up virtualenv for the following packs: %s' % (', '.join(packs))) diff --git a/contrib/packs/actions/setup_virtualenv.yaml b/contrib/packs/actions/setup_virtualenv.yaml index 2c05b02b84..52771c7fca 100644 --- a/contrib/packs/actions/setup_virtualenv.yaml +++ b/contrib/packs/actions/setup_virtualenv.yaml @@ -7,12 +7,18 @@ parameters: packs: type: "array" + description: "List of packs to create virtualenv for." items: type: "string" update: type: "boolean" default: false description: "Check this option if the virtual environment already exists and if you only want to perform an update and installation of new dependencies. If you don't check this option, the virtual environment will be destroyed then re-created. If you check this and the virtual environment doesn't exist, it will create it." + python3: + type: "boolean" + description: "Use Python 3 binary when creating a virtualenv for this pack." + required: false + default: false env: type: "object" description: "Optional environment variables" diff --git a/contrib/packs/actions/workflows/install.yaml b/contrib/packs/actions/workflows/install.yaml index e8373d6da3..e2dbd7ea03 100644 --- a/contrib/packs/actions/workflows/install.yaml +++ b/contrib/packs/actions/workflows/install.yaml @@ -19,6 +19,7 @@ parameters: packs: "{{ __results['make a prerun'].result }}" env: "{{env}}" + python3: "{{python3}}" on-success: "register pack" - name: "register pack" diff --git a/st2api/st2api/controllers/v1/packs.py b/st2api/st2api/controllers/v1/packs.py index 85c2123306..0765193f2e 100644 --- a/st2api/st2api/controllers/v1/packs.py +++ b/st2api/st2api/controllers/v1/packs.py @@ -98,6 +98,7 @@ class PackInstallController(ActionExecutionsControllerMixin): def post(self, pack_install_request, requester_user=None): parameters = { 'packs': pack_install_request.packs, + 'python3': pack_install_request.python3 } if pack_install_request.force: diff --git a/st2client/st2client/commands/pack.py b/st2client/st2client/commands/pack.py index dcda4821ab..502f769b36 100644 --- a/st2client/st2client/commands/pack.py +++ b/st2client/st2client/commands/pack.py @@ -185,6 +185,10 @@ def __init__(self, resource, *args, **kwargs): metavar='pack', help='Name of the %s in Exchange, or a git repo URL.' % resource.get_plural_display_name().lower()) + self.parser.add_argument('--python3', + action='store_true', + default=False, + help='Use Python 3 binary for pack virtual environment.') self.parser.add_argument('--force', action='store_true', default=False, @@ -192,7 +196,7 @@ def __init__(self, resource, *args, **kwargs): def run(self, args, **kwargs): self._get_content_counts_for_pack(args, **kwargs) - return self.manager.install(args.packs, force=args.force, **kwargs) + return self.manager.install(args.packs, python3=args.python3, force=args.force, **kwargs) def _get_content_counts_for_pack(self, args, **kwargs): # Global content list, excluding "tests" diff --git a/st2client/st2client/models/core.py b/st2client/st2client/models/core.py index e9829b7140..39f67dce23 100644 --- a/st2client/st2client/models/core.py +++ b/st2client/st2client/models/core.py @@ -476,11 +476,12 @@ class AsyncRequest(Resource): class PackResourceManager(ResourceManager): @add_auth_token_to_kwargs_from_env - def install(self, packs, force=False, **kwargs): + def install(self, packs, force=False, python3=False, **kwargs): url = '/%s/install' % (self.resource.get_url_path_name()) payload = { 'packs': packs, - 'force': force + 'force': force, + 'python3': python3 } response = self.client.post(url, payload, **kwargs) if response.status_code != http_client.OK: diff --git a/st2common/st2common/config.py b/st2common/st2common/config.py index 2f3f9024f6..254d6d40c0 100644 --- a/st2common/st2common/config.py +++ b/st2common/st2common/config.py @@ -19,6 +19,7 @@ import sys from oslo_config import cfg +from distutils.spawn import find_executable from st2common.constants.system import VERSION_STRING from st2common.constants.runners import PYTHON_RUNNER_DEFAULT_LOG_LEVEL @@ -221,8 +222,10 @@ def register_opts(ignore_errors=False): # Runner options default_python_bin_path = sys.executable + default_python3_bin_path = find_executable('python3') base_dir = os.path.dirname(os.path.realpath(default_python_bin_path)) default_virtualenv_bin_path = os.path.join(base_dir, 'virtualenv') + action_runner_opts = [ # Common runner options cfg.StrOpt('logging', default='conf/logging.conf', @@ -231,6 +234,9 @@ def register_opts(ignore_errors=False): # Python runner options cfg.StrOpt('python_binary', default=default_python_bin_path, help='Python binary which will be used by Python actions.'), + cfg.StrOpt('python3_binary', default=default_python3_bin_path, + help=('Python 3 binary which will be used by Python actions for packs which ' + 'use Python 3 virtual environment')), cfg.StrOpt('virtualenv_binary', default=default_virtualenv_bin_path, help='Virtualenv binary which should be used to create pack virtualenvs.'), cfg.StrOpt('python_runner_log_level', diff --git a/st2common/st2common/openapi.yaml b/st2common/st2common/openapi.yaml index 263914e1b3..a143a16965 100644 --- a/st2common/st2common/openapi.yaml +++ b/st2common/st2common/openapi.yaml @@ -4607,6 +4607,10 @@ definitions: type: array items: type: string + python3: + description: Use Python 3 binary for pack virtual environment. + type: boolean + default: false force: description: Force pack installation. type: boolean diff --git a/st2common/st2common/openapi.yaml.j2 b/st2common/st2common/openapi.yaml.j2 index 67e0c64519..2df04e80bf 100644 --- a/st2common/st2common/openapi.yaml.j2 +++ b/st2common/st2common/openapi.yaml.j2 @@ -4603,6 +4603,10 @@ definitions: type: array items: type: string + python3: + description: Use Python 3 binary for pack virtual environment. + type: boolean + default: false force: description: Force pack installation. type: boolean diff --git a/st2common/st2common/util/virtualenvs.py b/st2common/st2common/util/virtualenvs.py index f01a7a0074..edf59613d4 100644 --- a/st2common/st2common/util/virtualenvs.py +++ b/st2common/st2common/util/virtualenvs.py @@ -41,7 +41,8 @@ def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True, - include_setuptools=True, include_wheel=True, proxy_config=None): + include_setuptools=True, include_wheel=True, proxy_config=None, + use_python3=False): """ Setup virtual environment for the provided pack. @@ -54,6 +55,9 @@ def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True :param logger: Optional logger instance to use. If not provided it defaults to the module level logger. + + :param use_python3: Use Python3 binary when creating virtualenv for this pack. + :type use_python3: ``bool`` """ logger = logger or LOG @@ -82,7 +86,8 @@ def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True # 1. Create virtual environment logger.debug('Creating virtualenv for pack "%s" in "%s"' % (pack_name, virtualenv_path)) create_virtualenv(virtualenv_path=virtualenv_path, logger=logger, include_pip=include_pip, - include_setuptools=include_setuptools, include_wheel=include_wheel) + include_setuptools=include_setuptools, include_wheel=include_wheel, + use_python3=use_python3) # 2. Install base requirements which are common to all the packs logger.debug('Installing base requirements') @@ -110,7 +115,7 @@ def setup_pack_virtualenv(pack_name, update=False, logger=None, include_pip=True def create_virtualenv(virtualenv_path, logger=None, include_pip=True, include_setuptools=True, - include_wheel=True): + include_wheel=True, use_python3=False): """ :param include_pip: Include pip binary and package in the newely created virtual environment. :type include_pip: ``bool`` @@ -121,11 +126,15 @@ def create_virtualenv(virtualenv_path, logger=None, include_pip=True, include_se :param include_wheel: Include wheel in the newely created virtual environment. :type include_wheel : ``bool`` + + :param use_python3: Use Python3 binary when creating virtualenv for this pack. + :type use_python3: ``bool`` """ logger = logger or LOG python_binary = cfg.CONF.actionrunner.python_binary + python3_binary = cfg.CONF.actionrunner.python3_binary virtualenv_binary = cfg.CONF.actionrunner.virtualenv_binary virtualenv_opts = cfg.CONF.actionrunner.virtualenv_opts @@ -138,7 +147,18 @@ def create_virtualenv(virtualenv_path, logger=None, include_pip=True, include_se logger.debug('Creating virtualenv in "%s" using Python binary "%s"' % (virtualenv_path, python_binary)) - cmd = [virtualenv_binary, '-p', python_binary] + cmd = [virtualenv_binary] + + if use_python3 and not python3_binary: + msg = ('Requested to use Python 3, but python3 binary not found on the system or ' + 'actionrunner.python3 config option is not configured correctly.') + raise ValueError(msg) + + if use_python3: + cmd.extend(['-p', python3_binary]) + else: + cmd.extend(['-p', python_binary]) + cmd.extend(virtualenv_opts) if not include_pip: diff --git a/st2common/tests/unit/test_virtualenvs.py b/st2common/tests/unit/test_virtualenvs.py index f165c76fac..a082392f9f 100644 --- a/st2common/tests/unit/test_virtualenvs.py +++ b/st2common/tests/unit/test_virtualenvs.py @@ -290,6 +290,41 @@ def test_install_requirements_with_https_proxy_no_cert(self): } virtualenvs.run_command.assert_called_once_with(**expected_args) + @mock.patch.object(virtualenvs, 'run_command') + def test_setup_pack_virtualenv_use_python3_binary(self, mock_run_command): + mock_run_command.return_value = 0, '', '' + + cfg.CONF.set_override(name='python_binary', group='actionrunner', + override='/usr/bin/python2.7') + cfg.CONF.set_override(name='python3_binary', group='actionrunner', + override='/usr/bin/python3') + + pack_name = 'dummy_pack_2' + + # Python 2 + setup_pack_virtualenv(pack_name=pack_name, update=False, + include_setuptools=False, include_wheel=False, + use_python3=False) + + actual_cmd = mock_run_command.call_args_list[0][1]['cmd'] + actual_cmd = ' '.join(actual_cmd) + + self.assertEqual(mock_run_command.call_count, 2) + self.assertTrue('-p /usr/bin/python2.7' in actual_cmd) + + mock_run_command.reset_mock() + + # Python 3 + setup_pack_virtualenv(pack_name=pack_name, update=False, + include_setuptools=False, include_wheel=False, + use_python3=True) + + actual_cmd = mock_run_command.call_args_list[0][1]['cmd'] + actual_cmd = ' '.join(actual_cmd) + + self.assertEqual(mock_run_command.call_count, 2) + self.assertTrue('-p /usr/bin/python3' in actual_cmd) + def assertVirtulenvExists(self, virtualenv_dir): self.assertTrue(os.path.exists(virtualenv_dir)) self.assertTrue(os.path.isdir(virtualenv_dir)) diff --git a/tools/config_gen.py b/tools/config_gen.py index 52f24fb83e..7c54fec30b 100755 --- a/tools/config_gen.py +++ b/tools/config_gen.py @@ -64,8 +64,9 @@ # set them to static values to ensure consistent and stable output STATIC_OPTION_VALUES = { 'actionrunner': { - 'virtualenv_binary': '/data/stanley/virtualenv/bin/virtualenv', - 'python_binary': '/data/stanley/virtualenv/bin/python' + 'virtualenv_binary': '/usr/bin/virtualenv', + 'python_binary': '/usr/bin/python', + 'python3_binary': '/usr/bin/python3' }, 'webui': { 'webui_base_url': 'https://localhost'