Enable ci-integration tests on GHA (incl fixing a few tests)

.github/workflows/ci.yaml

@@ -14,6 +14,7 @@ on:
     type: [opened, reopened, edited]
     branches:
       # Only for PRs targeting those branches
+      - main
       - master
       - v[0-9]+.[0-9]+
   schedule:
@@ -44,28 +45,45 @@ jobs:
           # - name: 'Micro Benchmarks'
           #   task: 'micro-benchmarks'
           #   python-version: '3.6'
-          # Integration tests are not working yet, still done in Travis
-          # - name: 'Integration Tests'
-          #   task: 'ci-integration'
+          - name: 'Integration Tests'
+            task: 'ci-integration'
+            python-version: '3.6'
     services:
       mongo:
         image: mongo:4.0
         ports:
           - 27017:27017
-      # Can't use RabbitMQ here for Integrations because we rely on custom config
-      # and SSL certs that are in the repo. In GHA, these services are started first
-      # before the code is checked out, so this is a non-starter, we need to do it
-      # manually below (TODO)
+
+      # In GHA, these services are started first before the code is checked out.
+      # We use bitnami images to facilitate reconfiguring RabbitMQ during ci-integration tests.
+      # We rely on custom config and SSL certs that are in the repo.
+      # Many images require config in env vars (which we can't change during the test job)
+      # or they require config in entrypoint args (which we can't override for GHA services)
+      # bitnami builds ways to get config files from mounted volumes.
       rabbitmq:
-        # use the -management version so it has the management tools installed
-        image: rabbitmq:3.8-management
+        image: bitnami/rabbitmq:3.8
+        volumes:
+          - /home/runner/rabbitmq_conf:/bitnami/conf  # RABBITMQ_MOUNTED_CONF_DIR
+        env:
+          # tell bitnami/rabbitmq to enable this by default
+          RABBITMQ_PLUGINS: rabbitmq_management
+          RABBITMQ_USERNAME: guest
+          RABBITMQ_PASSWORD: guest
+
+        # These are strictly docker options, not entrypoint args (GHA restriction)
+        options: >-
+          --name rabbitmq
+
         ports:
-          # SSL port
-          - 5671:5671
-          # standard port
-          - 5672:5672
-          # management port
-          - 15672:15672
+          # These 6 ports are exposed by bitnami/rabbitmq (see https://www.rabbitmq.com/networking.html#ports)
+          # host_port:container_port/protocol
+          - 5671:5671/tcp   # AMQP SSL port
+          - 5672:5672/tcp   # AMQP standard port
+          - 15672:15672/tcp # Management: HTTP, CLI
+          #- 15671:15671/tcp # Management: SSL port
+          #- 25672:25672/tcp # inter-node or CLI
+          #- 4369:4369/tcp   # epmd
+
     env:
       TASK: '${{ matrix.task }}'
 
@@ -114,14 +132,34 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: '${{ matrix.python-version }}'
+      - name: Get date components for use in cache-keys
+        id: date
+        run: |
+          echo "::set-output name=year::$(/bin/date -u "+%Y")"
+          echo "::set-output name=month::$(/bin/date -u "+%m")"
+          echo "::set-output name=week::$(/bin/date -u "+%U")"
       - uses: actions/cache@v2
         with:
           path: |
             ~/.cache/pip
             virtualenv
-          key: ${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt') }}
+            ~/virtualenv
+          # TODO: maybe make the virtualenv a partial cache to exclude st2*?
+          # !virtualenv/lib/python*/site-packages/st2*
+          # !virtualenv/bin/st2*
+          key: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-python-${{ matrix.python }}-
+      - uses: actions/cache@v2
+        with:
+          path: |
+            /var/cache/apt/archives/*.deb
+            /var/cache/apt/archives/partial/*.deb
+            /var/cache/apt/*.bin
+          key: ${{ runner.os }}-apt-${{ steps.date.outputs.year }}-${{ steps.date.outputs.week }}
           restore-keys: |
-            ${{ runner.os }}-${{ matrix.python }}-
+            ${{ runner.os }}-apt-${{ steps.date.outputs.year }}-
+            ${{ runner.os }}-apt-
       - name: Install apt depedencies
         run: |
           # install dev dependencies for Python YAML and LDAP packages
@@ -132,7 +170,19 @@ jobs:
         run: |
           # Note: Use the verison of virtualenv pinned in fixed-requirements.txt so we
           #       only have to update it one place when we change the version
-          pip install --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
+          # Note: Use --user to avoid polluting system site-packages (which breaks one of our tests)
+          # TODO: simplify this once fixed in contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
+          if [[ ! -f ~/virtualenv/bin/virtualenv ]]; then  # use the cached version whenever possible
+            pip install --user --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
+            virtualenv --no-download ~/virtualenv
+            ~/virtualenv/bin/pip install --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
+            # drop the --user install virtualenv to prevent polluting tests
+            pip freeze --user | xargs pip uninstall -y
+            mkdir -p ~/.local/bin
+            ln -s ~/virtualenv/bin/virtualenv ~/.local/bin/virtualenv
+          fi
+          which virtualenv
+          virtualenv --version
       - name: Install requirements
         run: |
           ./scripts/travis/install-requirements.sh
@@ -149,34 +199,43 @@ jobs:
         run: |
           echo "$ST2_CI_REPO_PATH"
           sudo ST2_CI_REPO_PATH="${ST2_CI_REPO_PATH}" scripts/travis/permissions-workaround.sh
-      - name: Setup RabbitMQ (NOT WORKING YET)
-        if: "${{ env.TASK == 'ci-integration' }}"
+      - name: Reconfigure RabbitMQ
+        if: "${{ env.TASK == 'ci-unit' || env.TASK == 'ci-integration' }}"
+        # bitnami image allows (see bitnami/rabbitmq readme):
+        # Here we're copying a rabbitmq.config file which won't do anything.
+        # We need to switch to custom.conf or advanced.config.
+        timeout-minutes: 2  # may die if rabbitmq fails to start
         run: |
+          set -x
           # Use custom RabbitMQ config which enables SSL / TLS listener on port 5671 with test certs
-          # Travis runs as the 'travis' user, GitHub actions run as the 'runner' user,
-          # And the cert filepaths are slightly different between the two.
-          # Example:
-          #   Travis-CI:      /home/travis/build/StackStorm/st2/st2tests/st2tests/fixtures/ssl_certs/ca/ca_certificate_bundle.pem
-          #   GitHub Actions: /home/runner/work/st2/st2/st2tests/st2tests/fixtures/ssl_certs/ca/ca_certificate_bundle.pem
-          sed -i 's|/home/travis/build/StackStorm|/home/runner/work/st2|g' scripts/travis/rabbitmq.config
-          # Now that we've manged the config file, install it
-          sudo cp scripts/travis/rabbitmq.config /etc/rabbitmq/rabbitmq.config
-          # Install rabbitmq_management RabbitMQ plugin
-          sudo service rabbitmq-server restart
-          sleep 5
-          sudo rabbitmq-plugins enable rabbitmq_management
+          sudo cp scripts/github/rabbitmq.conf /home/runner/rabbitmq_conf/custom.conf
+          # The code is checked out after the container is already up, so we don't mount them.
+          # We copy those certs into the dir that is mounted to /bitnami/conf
+          sudo cp -r st2tests/st2tests/fixtures/ssl_certs /home/runner/rabbitmq_conf/
+          # refresh rabbitmq config - based on ENTRYPOINT logic
+          docker exec rabbitmq bash -c 'cat /bitnami/conf/custom.conf >> /opt/bitnami/rabbitmq/etc/rabbitmq/rabbitmq.conf'
+          # sleep to prevent interleaved output in GHA logs
+          docker exec rabbitmq cat /opt/bitnami/rabbitmq/etc/rabbitmq/rabbitmq.conf && sleep 0.1
+          echo
+          echo restarting rabbitmq container
+          docker restart rabbitmq
+          # wait for rabbitmq container to restart
+          until [ "$(docker inspect -f {{.State.Running}} rabbitmq)" == "true" ]; do sleep 0.1; done
+          echo enabled RabbitMQ plugins:
+          # print plugins list to: (1) ease debugging, (2) pause till rabbitmq is really running
+          docker exec rabbitmq rabbitmq-plugins list -e
+          echo
           sudo wget http://guest:guest@localhost:15672/cli/rabbitmqadmin -O /usr/local/bin/rabbitmqadmin
           sudo chmod +x /usr/local/bin/rabbitmqadmin
-          sudo service rabbitmq-server restart
-          # chmod to make glob work (*.log to avoid log dir)
-          sudo chmod a+rx /var/log/rabbitmq
-          sudo tail -n 30 /var/log/rabbitmq/*.log
+          # print logs from stdout (RABBITMQ_LOGS=-)
+          docker logs --tail=20 rabbitmq
       - name: Print versions
         run: |
           # Print various binary versions
           git --version
           pip --version
           pip list
+          virtualenv --version
           # Print out various environment variables info
           make play
       - name: make

.travis.yml

@@ -4,6 +4,7 @@ language: python
 
 branches:
   only:
+    - main
     - master
     - /^v[0-9]+\.[0-9]+$/
 
@@ -85,6 +86,12 @@ cache:
     #- .tox/
 
 install:
+  # This triggers the same behavior on travis as we found on github because it installs six in the system-site-packages
+  #- /opt/python/3.6.7/bin/pip install --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
+  #- mkdir -p ~/.local/bin
+  #- ln -s /opt/python/3.6.7/bin/virtualenv ~/.local/bin/virtualenv
+  - which virtualenv
+  - virtualenv --version
   - ./scripts/travis/install-requirements.sh
   # prep a travis-specific dev conf file that uses travis instead of stanley
   - cp conf/st2.dev.conf "${ST2_CONF}" ; sed -i -e "s/stanley/${ST2_CI_USER}/" "${ST2_CONF}"
@@ -108,16 +115,6 @@ before_script:
   - ls -l /etc/rabbitmq
   - ls -l /home/travis
   - ls -l /home/travis/build
-  - ls -l /home/travis/build/StackStorm
-  - ls -l /home/travis/build/StackStorm/st2
-  - ls -l /home/travis/build/StackStorm/st2/st2tests
-  - ls -l /home/travis/build/StackStorm/st2/st2tests/st2tests
-  - ls -l /home/travis/build/StackStorm/st2/st2tests/st2tests/fixtures
-  - ls -l /home/travis/build/StackStorm/st2/st2tests/st2tests/fixtures/ssl_certs
-  - ls -l /home/travis/build/StackStorm/st2/st2tests/st2tests/fixtures/ssl_certs/ca
-  - ls -l /home/travis/build/StackStorm/st2/st2tests/st2tests/fixtures/ssl_certs/server
-  - cat /home/travis/build/StackStorm/st2/st2tests/st2tests/fixtures/ssl_certs/ca/ca_certificate_bundle.pem
-  - cat /home/travis/build/StackStorm/st2/st2tests/st2tests/fixtures/ssl_certs/server/server_certificate.pem
   # Install rabbitmq_management RabbitMQ plugin
   - sudo service rabbitmq-server restart ; sleep 5
   - sudo rabbitmq-plugins enable rabbitmq_management

contrib/runners/python_runner/python_runner/python_action_wrapper.py

@@ -359,6 +359,12 @@ def _get_action_instance(self):
 
         stdin_data = sys.stdin.readline().strip()
 
+        if not stdin_data:
+            # This could indicate that parent process (e.g. process which runs the tests has
+            # incorrectly opened the stdin and that one is then inherited by the process which is
+            # spawning it which will cause issues)
+            raise ValueError("Received no valid parameters data from sys.stdin")
+
         try:
             stdin_parameters = orjson.loads(stdin_data)
             stdin_parameters = stdin_parameters.get("parameters", {})

contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py

@@ -142,14 +142,21 @@ def test_stdin_params_timeout_no_stdin_data_provided(self):
             "python %s --pack=dummy --file-path=%s --config='%s' "
             "--stdin-parameters" % (WRAPPER_SCRIPT_PATH, file_path, config)
         )
-        exit_code, stdout, stderr = run_command(command_string, shell=True)
+        exit_code, stdout, stderr = run_command(
+            command_string, shell=True, close_fds=True
+        )
 
-        expected_msg = (
+        # Depending on how tests are spawned, sys.stdin may be opened and this will cause issues
+        # with this tests so we simply check for two different errors which are considered
+        # acceptable.
+        expected_msg_1 = (
             "ValueError: No input received and timed out while waiting for parameters "
             "from stdin"
         )
+        expected_msg_2 = "ValueError: Received no valid parameters data from sys.stdin"
+
         self.assertEqual(exit_code, 1)
-        self.assertIn(expected_msg, stderr)
+        self.assertTrue(expected_msg_1 in stderr or expected_msg_2 in stderr)
 
     def test_stdin_params_invalid_format_friendly_error(self):
         config = {}

contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py

@@ -40,6 +40,12 @@
 
 
 class PythonRunnerBehaviorTestCase(CleanFilesTestCase, CleanDbTestCase):
+
+    # If you need these logs, then you probably also want to uncomment
+    # extra debug log messages in st2common/st2common/util/virtualenvs.py
+    # and pass --logging-level=DEBUG to nosetests
+    # DISPLAY_LOG_MESSAGES = True
+
     def setUp(self):
         super(PythonRunnerBehaviorTestCase, self).setUp()
         config.parse_args()
@@ -74,6 +80,15 @@ def test_priority_of_loading_library_after_setup_pack_virtualenv(self):
         (_, output, _) = self._run_action(
             pack_name, "get_library_path.py", {"module": "six"}
         )
+        # FIXME: This test fails if system site-packages has six because
+        # it won't get installed in the virtualenv (w/ --system-site-packages)
+        # system site-packages is never from a virtualenv.
+        # Travis has python installed in /opt/python/3.6.7
+        # with a no-system-site-packages virtualenv at /home/travis/virtualenv/python3.6.7
+        # GitHub Actions python is in /opt/hostedtoolcache/Python/3.6.13/x64/
+        # But ther isn't a virtualenv, so when we pip installed `virtualenv`,
+        # (which depends on, and therefore installs `six`)
+        # we installed it in system-site-packages not an intermediate virtualenv
         self.assertEqual(output["result"].find(self.virtualenvs_path), 0)
 
         # Conversely, this expects that 'mock' module file-path is not under sandbox library,

scripts/github/rabbitmq.conf

@@ -0,0 +1,11 @@
+# bitnami/rabbitmq configuration file (gets merged with rabbitmq.conf)
+listeners.ssl.default            = 5671
+# /bitnami/conf is a directory mounted into the bitnami/rabbitmq container
+ssl_options.cacertfile           = /bitnami/conf/ssl_certs/ca/ca_certificate_bundle.pem
+ssl_options.certfile             = /bitnami/conf/ssl_certs/server/server_certificate.pem
+ssl_options.keyfile              = /bitnami/conf/ssl_certs/server/private_key.pem
+ssl_options.verify               = verify_peer
+ssl_options.fail_if_no_peer_cert = false
+
+# this is "insecure" but it doesn't matter for CI, and it simplifies integration test machinery
+loopback_users = none

scripts/travis/prepare-integration.sh

@@ -16,6 +16,9 @@ st2 --version
 # Clean up old screen log files
 rm -f logs/screen-*.log
 
+# ::group::/::endgroup:: is helpful github actions syntax to fold this section.
+echo ::group::launchdev.sh start -x
+
 # start dev environment in screens
 ./tools/launchdev.sh start -x
 
@@ -28,6 +31,9 @@ echo " === START: Catting screen process log files. ==="
 cat logs/screen-*.log
 echo " === END: Catting screen process log files. ==="
 
+# github actions: fold for launchdev.sh start -x
+echo ::endgroup::
+
 # Setup the virtualenv for the examples pack which is required for orquesta integration tests.
 st2 run packs.setup_virtualenv packs=examples
 
@@ -39,3 +45,6 @@ chmod 777 logs/*
 # root needs to access write some lock files when creating virtualenvs
 # o=other; X=only set execute bit if user execute bit is set (eg on dirs)
 chmod -R o+rwX ./virtualenv/
+# newer virtualenv versions are putting lock files under ~/.local
+# as this script runs with sudo, HOME is actually the CI user's home
+chmod -R o+rwX ${HOME}/.local/share/virtualenv

scripts/travis/rabbitmq.config

@@ -1,9 +1,9 @@
 [
   {rabbit, [
      {ssl_listeners, [5671]},
-     {ssl_options, [{cacertfile, "/home/travis/build/StackStorm/st2/st2tests/st2tests/fixtures/ssl_certs/ca/ca_certificate_bundle.pem"},
-                    {certfile,   "/home/travis/build/StackStorm/st2/st2tests/st2tests/fixtures/ssl_certs/server/server_certificate.pem"},
-                    {keyfile,    "/home/travis/build/StackStorm/st2/st2tests/st2tests/fixtures/ssl_certs/server/private_key.pem"},
+     {ssl_options, [{cacertfile, "/home/travis/build/st2sandbox/st2/st2tests/st2tests/fixtures/ssl_certs/ca/ca_certificate_bundle.pem"},
+                    {certfile,   "/home/travis/build/st2sandbox/st2/st2tests/st2tests/fixtures/ssl_certs/server/server_certificate.pem"},
+                    {keyfile,    "/home/travis/build/st2sandbox/st2/st2tests/st2tests/fixtures/ssl_certs/server/private_key.pem"},
                     {verify,     verify_peer},
                     {fail_if_no_peer_cert, false}]}
    ]}

st2common/st2common/config.py

@@ -393,8 +393,13 @@ def register_opts(ignore_errors=False):
 
     # Runner options
     default_python_bin_path = sys.executable
-    base_dir = os.path.dirname(os.path.realpath(default_python_bin_path))
+    # If the virtualenv uses a symlinked python, then try using virtualenv from that venv
+    # first before looking for virtualenv installed in python's system-site-packages.
+    base_dir = os.path.dirname(default_python_bin_path)
     default_virtualenv_bin_path = os.path.join(base_dir, "virtualenv")
+    if not os.path.exists(default_virtualenv_bin_path):
+        base_dir = os.path.dirname(os.path.realpath(default_python_bin_path))
+        default_virtualenv_bin_path = os.path.join(base_dir, "virtualenv")
 
     action_runner_opts = [
         # Common runner options

st2common/st2common/util/shell.py

@@ -47,6 +47,7 @@ def run_command(
     shell=False,
     cwd=None,
     env=None,
+    close_fds=None,
 ):
     """
     Run the provided command in a subprocess and wait until it completes.
@@ -73,6 +74,9 @@ def run_command(
                 environment from the current process is inherited.
     :type env: ``dict``
 
+    :param close_fds: True to close all the fds. By default when None is provided we rely on
+                      default upstream behavior which may be Python version specific.
+
     :rtype: ``tuple`` (exit_code, stdout, stderr)
     """
     if not isinstance(cmd, (list, tuple) + six.string_types):
@@ -83,6 +87,10 @@ def run_command(
     if not env:
         env = os.environ.copy()
 
+    kwargs = {}
+    if close_fds is not None:
+        kwargs["close_fds"] = close_fds
+
     process = concurrency.subprocess_popen(
         args=cmd,
         stdin=stdin,
@@ -91,6 +99,7 @@ def run_command(
         env=env,
         cwd=cwd,
         shell=shell,
+        **kwargs,
     )
     stdout, stderr = process.communicate()
     exit_code = process.returncode