From cef01c3944a4395ed1908b674e1a8d5282233851 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Fri, 19 Mar 2021 23:54:34 -0500
Subject: [PATCH 001/103] use bitnami/rabbitmq service container in GHA

---
 .github/workflows/ci.yaml | 75 +++++++++++++++++++++++++--------------
 1 file changed, 49 insertions(+), 26 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 9fb0b97385..b56fbfc3dc 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -44,27 +44,39 @@ jobs:
           #   task: 'micro-benchmarks'
           #   python-version: '3.6'
           # Integration tests are not working yet, still done in Travis
-          # - name: 'Integration Tests'
-          #   task: 'ci-integration'
+          - name: 'Integration Tests'
+            task: 'ci-integration'
     services:
       mongo:
         image: mongo:4.0
         ports:
           - 27017:27017
-      # Can't use RabbitMQ here for Integrations because we rely on custom config
-      # and SSL certs that are in the repo. In GHA, these services are started first
-      # before the code is checked out, so this is a non-starter, we need to do it
-      # manually below (TODO)
+
+			# In GHA, these services are started first before the code is checked out.
+			# We use bitnami images to facilitate reconfiguring RabbitMQ during ci-integration tests.
+      # We rely on custom config and SSL certs that are in the repo.
+			# Many images require config in env vars (which we can't change during the test job)
+			# or they require config in entrypoint args (which we can't override for GHA services)
+			# bitnami builds ways to get config files from mounted volumes.
       rabbitmq:
-        # use the -management version so it has the management tools installed
-        image: rabbitmq:3.8-management
+				image: bitnami/rabbitmq:3.8  # includes mangement tools by default
+        volumes:
+          - /home/runner/rabbitmq_conf:/bitnami/conf  # RABBITMQ_MOUNTED_CONF_DIR
+
+        # These are strictly docker options, not entrypoint args (GHA restriction)
+        options: >-
+          --name rabbitmq
+
         ports:
-          # SSL port
-          - 5671:5671
-          # standard port
-          - 5672:5672
-          # management port
-          - 15672:15672
+          # these will assign a random free host port
+          # These 6 ports are exposed by bitnami/rabbitmq (see https://www.rabbitmq.com/networking.html#ports)
+          #- 4369/tcp  # epmd
+          - 5671/tcp   # AMQP SSL port
+          - 5672/tcp   # AMQP standard port
+          - 15671/tcp  # Management: SSL port
+          - 15672/tcp  # Management: HTTP, CLI
+          #- 25672/tcp  # inter-node or CLI
+
     env:
       TASK: '${{ matrix.task }}'
 
@@ -148,8 +160,15 @@ jobs:
         run: |
           echo "$ST2_CI_REPO_PATH"
           sudo ST2_CI_REPO_PATH="${ST2_CI_REPO_PATH}" scripts/travis/permissions-workaround.sh
-      - name: Setup RabbitMQ (NOT WORKING YET)
+      - name: Setup RabbitMQ
         if: "${{ env.TASK == 'ci-integration' }}"
+        # bitnami image allows (see bitnami/rabbitmq readme):
+        #   - replace generated rabbitmq.conf entirely
+        #   - generate rabbitmq.conf then merge the custom.conf setings on top
+        #   - add advanced.config (erlang) and/or rabbitmq-env.conf
+        # Here we're copying a rabbitmq.config file which won't do anything.
+        # We need to switch to custom.conf or advanced.config.
+        timeout-minutes: 2  # may die if rabbitmq fails to start
         run: |
           # Use custom RabbitMQ config which enables SSL / TLS listener on port 5671 with test certs
           # Travis runs as the 'travis' user, GitHub actions run as the 'runner' user,
@@ -157,19 +176,23 @@ jobs:
           # Example:
           #   Travis-CI:      /home/travis/build/StackStorm/st2/st2tests/st2tests/fixtures/ssl_certs/ca/ca_certificate_bundle.pem
           #   GitHub Actions: /home/runner/work/st2/st2/st2tests/st2tests/fixtures/ssl_certs/ca/ca_certificate_bundle.pem
-          sed -i 's|/home/travis/build/StackStorm|/home/runner/work/st2|g' scripts/travis/rabbitmq.config
-          # Now that we've manged the config file, install it
-          sudo cp scripts/travis/rabbitmq.config /etc/rabbitmq/rabbitmq.config
+          #   service container: /bitnami/conf/ssl_certs/ca/certificate_bundle.pem
+          sed -i 's|/home/travis/.*/ssl_certs/|/bitnami/conf/ssl_certs/|g' scripts/travis/rabbitmq.config
+          # Now that we've manged the config file, install it and the ssl certs
+          sudo cp scripts/travis/rabbitmq.config /home/runner/rabbitmq_conf/rabbitmq.config
+          sudo cp -r st2tests/st2tests/fixtures/ssl_certs /home/runner/rabbitmq_conf/
+          docker restart rabbitmq
+          # wait for rabbitmq to restart
+          until [ "$(docker inspect -f {{.State.Running}} rabbitmq)" == "true" ]; do sleep 0.1; done
           # Install rabbitmq_management RabbitMQ plugin
-          sudo service rabbitmq-server restart
-          sleep 5
-          sudo rabbitmq-plugins enable rabbitmq_management
-          sudo wget http://guest:guest@localhost:15672/cli/rabbitmqadmin -O /usr/local/bin/rabbitmqadmin
+          #sudo service rabbitmq-server restart
+          #sleep 5
+          #sudo rabbitmq-plugins enable rabbitmq_management
+          sudo wget http://guest:guest@localhost:${{ job.services.rabbitmq.ports[15672] }}/cli/rabbitmqadmin -O /usr/local/bin/rabbitmqadmin
           sudo chmod +x /usr/local/bin/rabbitmqadmin
-          sudo service rabbitmq-server restart
-          # chmod to make glob work (*.log to avoid log dir)
-          sudo chmod a+rx /var/log/rabbitmq
-          sudo tail -n 30 /var/log/rabbitmq/*.log
+          #sudo service rabbitmq-server restart
+          # may need to escape the asterisk
+          docker exec tail -n 30 /var/log/rabbitmq/*.log
       - name: Print versions
         run: |
           # Print various binary versions

From 005585b271045e880b83908289701ae30b8488cd Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 02:03:48 -0500
Subject: [PATCH 002/103] no random ports

---
 .github/workflows/ci.yaml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index b56fbfc3dc..39a8ef69be 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -68,14 +68,14 @@ jobs:
           --name rabbitmq
 
         ports:
-          # these will assign a random free host port
           # These 6 ports are exposed by bitnami/rabbitmq (see https://www.rabbitmq.com/networking.html#ports)
-          #- 4369/tcp  # epmd
-          - 5671/tcp   # AMQP SSL port
-          - 5672/tcp   # AMQP standard port
-          - 15671/tcp  # Management: SSL port
-          - 15672/tcp  # Management: HTTP, CLI
-          #- 25672/tcp  # inter-node or CLI
+          # host_port:container_port/protocol
+          #- 4369:4369/tcp  # epmd
+          - 5671:5671/tcp   # AMQP SSL port
+          - 5672:5672/tcp   # AMQP standard port
+          - 15671:15671/tcp  # Management: SSL port
+          - 15672:15672/tcp  # Management: HTTP, CLI
+          #- 25672:25672/tcp  # inter-node or CLI
 
     env:
       TASK: '${{ matrix.task }}'
@@ -188,7 +188,7 @@ jobs:
           #sudo service rabbitmq-server restart
           #sleep 5
           #sudo rabbitmq-plugins enable rabbitmq_management
-          sudo wget http://guest:guest@localhost:${{ job.services.rabbitmq.ports[15672] }}/cli/rabbitmqadmin -O /usr/local/bin/rabbitmqadmin
+          sudo wget http://guest:guest@localhost:15672/cli/rabbitmqadmin -O /usr/local/bin/rabbitmqadmin
           sudo chmod +x /usr/local/bin/rabbitmqadmin
           #sudo service rabbitmq-server restart
           # may need to escape the asterisk

From afb3550fac4b249058895808629d4b51f74fde0a Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 02:28:53 -0500
Subject: [PATCH 003/103] use separate rabbitmq.conf file on github

---
 .github/workflows/ci.yaml    | 21 ++++++++-------------
 scripts/github/rabbitmq.conf |  7 +++++++
 2 files changed, 15 insertions(+), 13 deletions(-)
 create mode 100644 scripts/github/rabbitmq.conf

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 39a8ef69be..5949972490 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -62,6 +62,8 @@ jobs:
 				image: bitnami/rabbitmq:3.8  # includes mangement tools by default
         volumes:
           - /home/runner/rabbitmq_conf:/bitnami/conf  # RABBITMQ_MOUNTED_CONF_DIR
+          # this won't exist until checked out, but the checkout should then populate it.
+          - /home/runner/work/st2/st2/st2tests/st2tests/fixtures/ssl_certs:/st2/ssl_certs
 
         # These are strictly docker options, not entrypoint args (GHA restriction)
         options: >-
@@ -163,24 +165,17 @@ jobs:
       - name: Setup RabbitMQ
         if: "${{ env.TASK == 'ci-integration' }}"
         # bitnami image allows (see bitnami/rabbitmq readme):
-        #   - replace generated rabbitmq.conf entirely
-        #   - generate rabbitmq.conf then merge the custom.conf setings on top
-        #   - add advanced.config (erlang) and/or rabbitmq-env.conf
         # Here we're copying a rabbitmq.config file which won't do anything.
         # We need to switch to custom.conf or advanced.config.
         timeout-minutes: 2  # may die if rabbitmq fails to start
         run: |
           # Use custom RabbitMQ config which enables SSL / TLS listener on port 5671 with test certs
-          # Travis runs as the 'travis' user, GitHub actions run as the 'runner' user,
-          # And the cert filepaths are slightly different between the two.
-          # Example:
-          #   Travis-CI:      /home/travis/build/StackStorm/st2/st2tests/st2tests/fixtures/ssl_certs/ca/ca_certificate_bundle.pem
-          #   GitHub Actions: /home/runner/work/st2/st2/st2tests/st2tests/fixtures/ssl_certs/ca/ca_certificate_bundle.pem
-          #   service container: /bitnami/conf/ssl_certs/ca/certificate_bundle.pem
-          sed -i 's|/home/travis/.*/ssl_certs/|/bitnami/conf/ssl_certs/|g' scripts/travis/rabbitmq.config
-          # Now that we've manged the config file, install it and the ssl certs
-          sudo cp scripts/travis/rabbitmq.config /home/runner/rabbitmq_conf/rabbitmq.config
-          sudo cp -r st2tests/st2tests/fixtures/ssl_certs /home/runner/rabbitmq_conf/
+          # These certs are mounted in a volume on the service container to /st2/ssl_certs
+          # bitnami/rabbitmq image supports 3 ways to adjust rabbitmq config:
+          #   - merge custom.conf into a generated rabbitmq.conf (we use this)
+          #   - replace generated rabbitmq.conf entirely
+          #   - add advanced.config (erlang) and/or rabbitmq-env.conf
+          sudo cp scripts/github/rabbitmq.conf /home/runner/rabbitmq_conf/custom.conf
           docker restart rabbitmq
           # wait for rabbitmq to restart
           until [ "$(docker inspect -f {{.State.Running}} rabbitmq)" == "true" ]; do sleep 0.1; done
diff --git a/scripts/github/rabbitmq.conf b/scripts/github/rabbitmq.conf
new file mode 100644
index 0000000000..5c3c96dc73
--- /dev/null
+++ b/scripts/github/rabbitmq.conf
@@ -0,0 +1,7 @@
+# bitnami/rabbitmq configuration file (gets merged with rabbitmq.conf)
+listeners.ssl.default            = 5671
+ssl_options.cacertfile           = /st2/ca/ca_certificate_bundle.pem
+ssl_options.certfile             = /st2/server/server_certificate.pem
+ssl_options.keyfile              = /st2/server/private_key.pem
+ssl_options.verify               = verify_peer
+ssl_options.fail_if_no_peer_cert = false

From ae1e3eae004cc6bb360de7a70789ca4cdc2c273f Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 02:34:05 -0500
Subject: [PATCH 004/103] ci: cleanup tabbitmq_management plugin steps

---
 .github/workflows/ci.yaml | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 5949972490..12a988d2aa 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -179,13 +179,9 @@ jobs:
           docker restart rabbitmq
           # wait for rabbitmq to restart
           until [ "$(docker inspect -f {{.State.Running}} rabbitmq)" == "true" ]; do sleep 0.1; done
-          # Install rabbitmq_management RabbitMQ plugin
-          #sudo service rabbitmq-server restart
-          #sleep 5
-          #sudo rabbitmq-plugins enable rabbitmq_management
+          # bitnami/rabbitmq enables rabbitmq_management RabbitMQ plugin by default
           sudo wget http://guest:guest@localhost:15672/cli/rabbitmqadmin -O /usr/local/bin/rabbitmqadmin
           sudo chmod +x /usr/local/bin/rabbitmqadmin
-          #sudo service rabbitmq-server restart
           # may need to escape the asterisk
           docker exec tail -n 30 /var/log/rabbitmq/*.log
       - name: Print versions

From 570a860c0c3b23e67dfddd68c3e5b6fa98e866c2 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 02:36:01 -0500
Subject: [PATCH 005/103] ci: temporarily disable all but ci-integration

---
 .github/workflows/ci.yaml | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 12a988d2aa..77c24f6d51 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -14,6 +14,7 @@ on:
     type: [opened, reopened, edited]
     branches:
       # Only for PRs targeting those branches
+      - main
       - master
       - v[0-9]+.[0-9]+
   schedule:
@@ -28,18 +29,18 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - name: 'Lint Checks'
-            task: 'ci-checks'
-            python-version: '3.6'
-          - name: 'Compile'
-            task: 'ci-compile'
-            python-version: '3.6'
-          - name: 'Pack Tests'
-            task: 'ci-packs-tests'
-            python-version: '3.6'
-          - name: 'Unit Tests'
-            task: 'ci-unit'
-            python-version: '3.6'
+          #- name: 'Lint Checks'
+          #  task: 'ci-checks'
+          #  python-version: '3.6'
+          #- name: 'Compile'
+          #  task: 'ci-compile'
+          #  python-version: '3.6'
+          #- name: 'Pack Tests'
+          #  task: 'ci-packs-tests'
+          #  python-version: '3.6'
+          #- name: 'Unit Tests'
+          #  task: 'ci-unit'
+          #  python-version: '3.6'
           # - name: 'Micro Benchmarks'
           #   task: 'micro-benchmarks'
           #   python-version: '3.6'

From 78400bc8a770f4b7a09f94c872b1517b6746afcd Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 02:39:27 -0500
Subject: [PATCH 006/103] fix whitespace

---
 .github/workflows/ci.yaml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 77c24f6d51..5c5875200a 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -53,14 +53,14 @@ jobs:
         ports:
           - 27017:27017
 
-			# In GHA, these services are started first before the code is checked out.
-			# We use bitnami images to facilitate reconfiguring RabbitMQ during ci-integration tests.
+      # In GHA, these services are started first before the code is checked out.
+      # We use bitnami images to facilitate reconfiguring RabbitMQ during ci-integration tests.
       # We rely on custom config and SSL certs that are in the repo.
-			# Many images require config in env vars (which we can't change during the test job)
-			# or they require config in entrypoint args (which we can't override for GHA services)
-			# bitnami builds ways to get config files from mounted volumes.
+      # Many images require config in env vars (which we can't change during the test job)
+      # or they require config in entrypoint args (which we can't override for GHA services)
+      # bitnami builds ways to get config files from mounted volumes.
       rabbitmq:
-				image: bitnami/rabbitmq:3.8  # includes mangement tools by default
+        image: bitnami/rabbitmq:3.8  # includes mangement tools by default
         volumes:
           - /home/runner/rabbitmq_conf:/bitnami/conf  # RABBITMQ_MOUNTED_CONF_DIR
           # this won't exist until checked out, but the checkout should then populate it.

From f26391926e7412b78ee9cf4a85c24d5d4bb42170 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 02:47:03 -0500
Subject: [PATCH 007/103] ci: cp ssl_certs into rabbitmq container

---
 .github/workflows/ci.yaml    | 7 ++++---
 scripts/github/rabbitmq.conf | 7 ++++---
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 5c5875200a..0ecb8867f7 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -63,8 +63,6 @@ jobs:
         image: bitnami/rabbitmq:3.8  # includes mangement tools by default
         volumes:
           - /home/runner/rabbitmq_conf:/bitnami/conf  # RABBITMQ_MOUNTED_CONF_DIR
-          # this won't exist until checked out, but the checkout should then populate it.
-          - /home/runner/work/st2/st2/st2tests/st2tests/fixtures/ssl_certs:/st2/ssl_certs
 
         # These are strictly docker options, not entrypoint args (GHA restriction)
         options: >-
@@ -171,12 +169,15 @@ jobs:
         timeout-minutes: 2  # may die if rabbitmq fails to start
         run: |
           # Use custom RabbitMQ config which enables SSL / TLS listener on port 5671 with test certs
-          # These certs are mounted in a volume on the service container to /st2/ssl_certs
           # bitnami/rabbitmq image supports 3 ways to adjust rabbitmq config:
           #   - merge custom.conf into a generated rabbitmq.conf (we use this)
           #   - replace generated rabbitmq.conf entirely
           #   - add advanced.config (erlang) and/or rabbitmq-env.conf
           sudo cp scripts/github/rabbitmq.conf /home/runner/rabbitmq_conf/custom.conf
+          # The code is checked out after the container is already up, so we don't mount them.
+          # We copy those certs into the dir that is mounted to /bitnami/conf
+          sudo cp -r st2tests/st2tests/fixtures/ssl_certs /home/runner/rabbitmq_conf/
+          
           docker restart rabbitmq
           # wait for rabbitmq to restart
           until [ "$(docker inspect -f {{.State.Running}} rabbitmq)" == "true" ]; do sleep 0.1; done
diff --git a/scripts/github/rabbitmq.conf b/scripts/github/rabbitmq.conf
index 5c3c96dc73..d7197e8d88 100644
--- a/scripts/github/rabbitmq.conf
+++ b/scripts/github/rabbitmq.conf
@@ -1,7 +1,8 @@
 # bitnami/rabbitmq configuration file (gets merged with rabbitmq.conf)
 listeners.ssl.default            = 5671
-ssl_options.cacertfile           = /st2/ca/ca_certificate_bundle.pem
-ssl_options.certfile             = /st2/server/server_certificate.pem
-ssl_options.keyfile              = /st2/server/private_key.pem
+# /bitnami/conf is a directory mounted into the bitnami/rabbitmq container
+ssl_options.cacertfile           = /bitnami/conf/ssl_certs/ca/ca_certificate_bundle.pem
+ssl_options.certfile             = /bitnami/conf/ssl_certs/server/server_certificate.pem
+ssl_options.keyfile              = /bitnami/conf/ssl_certs/server/private_key.pem
 ssl_options.verify               = verify_peer
 ssl_options.fail_if_no_peer_cert = false

From d215c1ece53bc376db50f16c828dcc9a66e475ff Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 02:57:21 -0500
Subject: [PATCH 008/103] debug

---
 .github/workflows/ci.yaml | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 0ecb8867f7..4a327f226d 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -144,10 +144,10 @@ jobs:
         run: |
           # Note: Use the verison of virtualenv pinned in fixed-requirements.txt so we
           #       only have to update it one place when we change the version
-          pip install --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
+          echo pip install --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
       - name: Install requirements
         run: |
-          ./scripts/travis/install-requirements.sh
+          echo ./scripts/travis/install-requirements.sh
       - name: Setup integration tests
         run: |
           # prep a ci-specific dev conf file that uses runner instead of stanley
@@ -177,11 +177,14 @@ jobs:
           # The code is checked out after the container is already up, so we don't mount them.
           # We copy those certs into the dir that is mounted to /bitnami/conf
           sudo cp -r st2tests/st2tests/fixtures/ssl_certs /home/runner/rabbitmq_conf/
-          
+          ls -l /home/runner/rabbitmq_conf
+          echo START restart rabbitmq container...
           docker restart rabbitmq
           # wait for rabbitmq to restart
           until [ "$(docker inspect -f {{.State.Running}} rabbitmq)" == "true" ]; do sleep 0.1; done
           # bitnami/rabbitmq enables rabbitmq_management RabbitMQ plugin by default
+          echo DONE restarted rabbitmq container
+          docker port rabbitmq
           sudo wget http://guest:guest@localhost:15672/cli/rabbitmqadmin -O /usr/local/bin/rabbitmqadmin
           sudo chmod +x /usr/local/bin/rabbitmqadmin
           # may need to escape the asterisk

From 21a499cf8850fe2c91076b2ea19fb669cde8c31a Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 03:02:52 -0500
Subject: [PATCH 009/103] reenable management plugin

---
 .github/workflows/ci.yaml | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 4a327f226d..f69025e585 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -60,9 +60,12 @@ jobs:
       # or they require config in entrypoint args (which we can't override for GHA services)
       # bitnami builds ways to get config files from mounted volumes.
       rabbitmq:
-        image: bitnami/rabbitmq:3.8  # includes mangement tools by default
+        image: bitnami/rabbitmq:3.8
         volumes:
           - /home/runner/rabbitmq_conf:/bitnami/conf  # RABBITMQ_MOUNTED_CONF_DIR
+        env:
+          # tell bitnami/rabbitmq to enable this by default
+          RABBITMQ_PLUGINS: rabbitmq_management
 
         # These are strictly docker options, not entrypoint args (GHA restriction)
         options: >-
@@ -74,7 +77,7 @@ jobs:
           #- 4369:4369/tcp  # epmd
           - 5671:5671/tcp   # AMQP SSL port
           - 5672:5672/tcp   # AMQP standard port
-          - 15671:15671/tcp  # Management: SSL port
+          #- 15671:15671/tcp  # Management: SSL port
           - 15672:15672/tcp  # Management: HTTP, CLI
           #- 25672:25672/tcp  # inter-node or CLI
 
@@ -182,7 +185,6 @@ jobs:
           docker restart rabbitmq
           # wait for rabbitmq to restart
           until [ "$(docker inspect -f {{.State.Running}} rabbitmq)" == "true" ]; do sleep 0.1; done
-          # bitnami/rabbitmq enables rabbitmq_management RabbitMQ plugin by default
           echo DONE restarted rabbitmq container
           docker port rabbitmq
           sudo wget http://guest:guest@localhost:15672/cli/rabbitmqadmin -O /usr/local/bin/rabbitmqadmin

From c36963f828193d9c30f7fb475698c0fdc43899ab Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 03:08:17 -0500
Subject: [PATCH 010/103] debug

---
 .github/workflows/ci.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index f69025e585..74b6cbc86a 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -181,12 +181,12 @@ jobs:
           # We copy those certs into the dir that is mounted to /bitnami/conf
           sudo cp -r st2tests/st2tests/fixtures/ssl_certs /home/runner/rabbitmq_conf/
           ls -l /home/runner/rabbitmq_conf
-          echo START restart rabbitmq container...
+          echo restarting rabbitmq container
           docker restart rabbitmq
           # wait for rabbitmq to restart
           until [ "$(docker inspect -f {{.State.Running}} rabbitmq)" == "true" ]; do sleep 0.1; done
-          echo DONE restarted rabbitmq container
-          docker port rabbitmq
+          echo This is the final rabbitmq.conf file:
+          docker exec rabbitmq cat /opt/bitnami/rabbitmq/etc/rabbitmq/rabbitmq.conf
           sudo wget http://guest:guest@localhost:15672/cli/rabbitmqadmin -O /usr/local/bin/rabbitmqadmin
           sudo chmod +x /usr/local/bin/rabbitmqadmin
           # may need to escape the asterisk

From 821cbbc7d60156438a73a3b3869e45b811c6b049 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 10:29:44 -0500
Subject: [PATCH 011/103] debug

---
 .github/workflows/ci.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 74b6cbc86a..2622a2daaf 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -141,8 +141,8 @@ jobs:
         run: |
           # install dev dependencies for Python YAML and LDAP packages
           # https://github.com/StackStorm/st2-auth-ldap
-          sudo apt-get -y update
-          sudo apt-get -f -y install libldap2-dev libsasl2-dev libssl-dev libyaml-dev ldap-utils
+          echo sudo apt-get -y update
+          echo sudo apt-get -f -y install libldap2-dev libsasl2-dev libssl-dev libyaml-dev ldap-utils
       - name: Install virtualenv
         run: |
           # Note: Use the verison of virtualenv pinned in fixed-requirements.txt so we
@@ -182,9 +182,11 @@ jobs:
           sudo cp -r st2tests/st2tests/fixtures/ssl_certs /home/runner/rabbitmq_conf/
           ls -l /home/runner/rabbitmq_conf
           echo restarting rabbitmq container
+          docker exec rabbitmq-plugins list
           docker restart rabbitmq
           # wait for rabbitmq to restart
           until [ "$(docker inspect -f {{.State.Running}} rabbitmq)" == "true" ]; do sleep 0.1; done
+          docker exec rabbitmq-plugins list
           echo This is the final rabbitmq.conf file:
           docker exec rabbitmq cat /opt/bitnami/rabbitmq/etc/rabbitmq/rabbitmq.conf
           sudo wget http://guest:guest@localhost:15672/cli/rabbitmqadmin -O /usr/local/bin/rabbitmqadmin

From d8b5ae320bd4fc79eeecd84ffacb316582029fd4 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 10:31:45 -0500
Subject: [PATCH 012/103] debug

---
 .github/workflows/ci.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 2622a2daaf..1c89a2ea49 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -182,11 +182,11 @@ jobs:
           sudo cp -r st2tests/st2tests/fixtures/ssl_certs /home/runner/rabbitmq_conf/
           ls -l /home/runner/rabbitmq_conf
           echo restarting rabbitmq container
-          docker exec rabbitmq-plugins list
+          docker exec rabbitmq rabbitmq-plugins list
           docker restart rabbitmq
           # wait for rabbitmq to restart
           until [ "$(docker inspect -f {{.State.Running}} rabbitmq)" == "true" ]; do sleep 0.1; done
-          docker exec rabbitmq-plugins list
+          docker exec rabbitmq rabbitmq-plugins list
           echo This is the final rabbitmq.conf file:
           docker exec rabbitmq cat /opt/bitnami/rabbitmq/etc/rabbitmq/rabbitmq.conf
           sudo wget http://guest:guest@localhost:15672/cli/rabbitmqadmin -O /usr/local/bin/rabbitmqadmin

From cfb8c6c48ca459621bc1ad694d35dd6fbf93efa9 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 10:37:13 -0500
Subject: [PATCH 013/103] debug

---
 .github/workflows/ci.yaml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 1c89a2ea49..7ba1eece90 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -182,17 +182,17 @@ jobs:
           sudo cp -r st2tests/st2tests/fixtures/ssl_certs /home/runner/rabbitmq_conf/
           ls -l /home/runner/rabbitmq_conf
           echo restarting rabbitmq container
-          docker exec rabbitmq rabbitmq-plugins list
           docker restart rabbitmq
           # wait for rabbitmq to restart
           until [ "$(docker inspect -f {{.State.Running}} rabbitmq)" == "true" ]; do sleep 0.1; done
-          docker exec rabbitmq rabbitmq-plugins list
+          echo enabled RabbitMQ plugins:
+          docker exec rabbitmq rabbitmq-plugins list -e
           echo This is the final rabbitmq.conf file:
           docker exec rabbitmq cat /opt/bitnami/rabbitmq/etc/rabbitmq/rabbitmq.conf
           sudo wget http://guest:guest@localhost:15672/cli/rabbitmqadmin -O /usr/local/bin/rabbitmqadmin
           sudo chmod +x /usr/local/bin/rabbitmqadmin
           # may need to escape the asterisk
-          docker exec tail -n 30 /var/log/rabbitmq/*.log
+          docker exec rabbitmq tail -n 30 /var/log/rabbitmq/*.log
       - name: Print versions
         run: |
           # Print various binary versions
@@ -204,7 +204,7 @@ jobs:
       - name: make
         # use: script -e -c to print colors
         run: |
-          script -e -c "make ${TASK}"
+          echo script -e -c "make ${TASK}"
       - name: Nightly
         # Run any additional nightly checks only as part of a nightly (cron) build
         if: "${{ env.IS_NIGHTLY_BUILD == 'yes' }}"
@@ -214,7 +214,7 @@ jobs:
         # NOTE: We only generate and submit coverage report for master and version branches and only when the build succeeds (default on GitHub Actions, this was not the case on Travis so we had to explicitly check success)
         if: "${{ success() && ((env.TASK == 'ci-unit') || (env.TASK == 'ci-integration')) && (env.ENABLE_COVERAGE == 'yes') }}"
         run: |
-          ./scripts/travis/submit-codecov-coverage.sh
+          echo ./scripts/travis/submit-codecov-coverage.sh
   slack-notification:
     name: Slack notification for failed master builds
     if: always()

From 6b688addda8e45001188bf16e02c9f8590b271c4 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 10:42:30 -0500
Subject: [PATCH 014/103] logs

---
 .github/workflows/ci.yaml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 7ba1eece90..258248e1fb 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -192,7 +192,10 @@ jobs:
           sudo wget http://guest:guest@localhost:15672/cli/rabbitmqadmin -O /usr/local/bin/rabbitmqadmin
           sudo chmod +x /usr/local/bin/rabbitmqadmin
           # may need to escape the asterisk
-          docker exec rabbitmq tail -n 30 /var/log/rabbitmq/*.log
+          echo docker logs
+          docker logs --tail 30 rabbitmq
+          echo rmq logs
+          docker exec rabbitmq bash -c 'tail -n 30 /var/log/rabbitmq/*.log'
       - name: Print versions
         run: |
           # Print various binary versions

From eba896f61cecb8a7b86c7c3db0f8182e964e06db Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 10:50:56 -0500
Subject: [PATCH 015/103] debug

---
 .github/workflows/ci.yaml | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 258248e1fb..9d44687acd 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -171,6 +171,7 @@ jobs:
         # We need to switch to custom.conf or advanced.config.
         timeout-minutes: 2  # may die if rabbitmq fails to start
         run: |
+          set -x
           # Use custom RabbitMQ config which enables SSL / TLS listener on port 5671 with test certs
           # bitnami/rabbitmq image supports 3 ways to adjust rabbitmq config:
           #   - merge custom.conf into a generated rabbitmq.conf (we use this)
@@ -181,20 +182,24 @@ jobs:
           # We copy those certs into the dir that is mounted to /bitnami/conf
           sudo cp -r st2tests/st2tests/fixtures/ssl_certs /home/runner/rabbitmq_conf/
           ls -l /home/runner/rabbitmq_conf
+          docker exec rabbitmq ls -la /opt/bitnami/rabbitmq/etc/rabbitmq /bitnami/conf
+          echo
+          echo This is the pre-restart rabbitmq.conf file:
+          docker exec rabbitmq cat /opt/bitnami/rabbitmq/etc/rabbitmq/rabbitmq.conf
+          echo
           echo restarting rabbitmq container
           docker restart rabbitmq
-          # wait for rabbitmq to restart
+          # wait for rabbitmq container to restart
           until [ "$(docker inspect -f {{.State.Running}} rabbitmq)" == "true" ]; do sleep 0.1; done
           echo enabled RabbitMQ plugins:
+          # print plugins list to: (1) ease debugging, (2) pause till rabbitmq is really running
           docker exec rabbitmq rabbitmq-plugins list -e
           echo This is the final rabbitmq.conf file:
           docker exec rabbitmq cat /opt/bitnami/rabbitmq/etc/rabbitmq/rabbitmq.conf
+          echo
           sudo wget http://guest:guest@localhost:15672/cli/rabbitmqadmin -O /usr/local/bin/rabbitmqadmin
           sudo chmod +x /usr/local/bin/rabbitmqadmin
-          # may need to escape the asterisk
-          echo docker logs
-          docker logs --tail 30 rabbitmq
-          echo rmq logs
+          # print logs
           docker exec rabbitmq bash -c 'tail -n 30 /var/log/rabbitmq/*.log'
       - name: Print versions
         run: |

From 6ed3e48ad65ce46d4d21e652b7caaf7f1367de0b Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 10:55:40 -0500
Subject: [PATCH 016/103] debug

---
 .github/workflows/ci.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 9d44687acd..adacd649f1 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -182,7 +182,7 @@ jobs:
           # We copy those certs into the dir that is mounted to /bitnami/conf
           sudo cp -r st2tests/st2tests/fixtures/ssl_certs /home/runner/rabbitmq_conf/
           ls -l /home/runner/rabbitmq_conf
-          docker exec rabbitmq ls -la /opt/bitnami/rabbitmq/etc/rabbitmq /bitnami/conf
+          docker exec rabbitmq ls -la /opt/bitnami/rabbitmq/etc/rabbitmq /bitnami/conf /bitnami/rabbitmq/conf
           echo
           echo This is the pre-restart rabbitmq.conf file:
           docker exec rabbitmq cat /opt/bitnami/rabbitmq/etc/rabbitmq/rabbitmq.conf
@@ -194,6 +194,7 @@ jobs:
           echo enabled RabbitMQ plugins:
           # print plugins list to: (1) ease debugging, (2) pause till rabbitmq is really running
           docker exec rabbitmq rabbitmq-plugins list -e
+          docker exec rabbitmq ls -la /opt/bitnami/rabbitmq/etc/rabbitmq /bitnami/conf /bitnami/rabbitmq/conf
           echo This is the final rabbitmq.conf file:
           docker exec rabbitmq cat /opt/bitnami/rabbitmq/etc/rabbitmq/rabbitmq.conf
           echo

From 0433adf0f8e6cd5937d612f13287e448d7a158fd Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 11:02:16 -0500
Subject: [PATCH 017/103] debug

---
 .github/workflows/ci.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index adacd649f1..c64c972635 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -182,7 +182,7 @@ jobs:
           # We copy those certs into the dir that is mounted to /bitnami/conf
           sudo cp -r st2tests/st2tests/fixtures/ssl_certs /home/runner/rabbitmq_conf/
           ls -l /home/runner/rabbitmq_conf
-          docker exec rabbitmq ls -la /opt/bitnami/rabbitmq/etc/rabbitmq /bitnami/conf /bitnami/rabbitmq/conf
+          docker exec rabbitmq ls -la /opt/bitnami/rabbitmq/etc/rabbitmq /bitnami/conf
           echo
           echo This is the pre-restart rabbitmq.conf file:
           docker exec rabbitmq cat /opt/bitnami/rabbitmq/etc/rabbitmq/rabbitmq.conf
@@ -194,7 +194,7 @@ jobs:
           echo enabled RabbitMQ plugins:
           # print plugins list to: (1) ease debugging, (2) pause till rabbitmq is really running
           docker exec rabbitmq rabbitmq-plugins list -e
-          docker exec rabbitmq ls -la /opt/bitnami/rabbitmq/etc/rabbitmq /bitnami/conf /bitnami/rabbitmq/conf
+          docker exec rabbitmq ls -la /opt/bitnami/rabbitmq/etc/rabbitmq /bitnami/conf
           echo This is the final rabbitmq.conf file:
           docker exec rabbitmq cat /opt/bitnami/rabbitmq/etc/rabbitmq/rabbitmq.conf
           echo

From 550a509d79765731673d9097beb915eeb59872be Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 11:23:56 -0500
Subject: [PATCH 018/103] debug

---
 .github/workflows/ci.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index c64c972635..ca72ed8715 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -185,9 +185,10 @@ jobs:
           docker exec rabbitmq ls -la /opt/bitnami/rabbitmq/etc/rabbitmq /bitnami/conf
           echo
           echo This is the pre-restart rabbitmq.conf file:
-          docker exec rabbitmq cat /opt/bitnami/rabbitmq/etc/rabbitmq/rabbitmq.conf
+          docker exec rabbitmq cat /opt/bitnami/rabbitmq/etc/rabbitmq/rabbitmq.conf && sleep 1
           echo
           echo restarting rabbitmq container
+          docker events 2>&1 > docker-events.log &
           docker restart rabbitmq
           # wait for rabbitmq container to restart
           until [ "$(docker inspect -f {{.State.Running}} rabbitmq)" == "true" ]; do sleep 0.1; done
@@ -195,8 +196,9 @@ jobs:
           # print plugins list to: (1) ease debugging, (2) pause till rabbitmq is really running
           docker exec rabbitmq rabbitmq-plugins list -e
           docker exec rabbitmq ls -la /opt/bitnami/rabbitmq/etc/rabbitmq /bitnami/conf
+          cat docker-events.log && sleep 1
           echo This is the final rabbitmq.conf file:
-          docker exec rabbitmq cat /opt/bitnami/rabbitmq/etc/rabbitmq/rabbitmq.conf
+          docker exec rabbitmq cat /opt/bitnami/rabbitmq/etc/rabbitmq/rabbitmq.conf && sleep 1
           echo
           sudo wget http://guest:guest@localhost:15672/cli/rabbitmqadmin -O /usr/local/bin/rabbitmqadmin
           sudo chmod +x /usr/local/bin/rabbitmqadmin

From c5c2214ceaffecffd463b0668a077a8d31456496 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 11:32:56 -0500
Subject: [PATCH 019/103] debug

---
 .github/workflows/ci.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index ca72ed8715..cb149160a1 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -189,11 +189,13 @@ jobs:
           echo
           echo restarting rabbitmq container
           docker events 2>&1 > docker-events.log &
+          docker ps -ef
           docker restart rabbitmq
           # wait for rabbitmq container to restart
           until [ "$(docker inspect -f {{.State.Running}} rabbitmq)" == "true" ]; do sleep 0.1; done
           echo enabled RabbitMQ plugins:
           # print plugins list to: (1) ease debugging, (2) pause till rabbitmq is really running
+          docker ps -ef
           docker exec rabbitmq rabbitmq-plugins list -e
           docker exec rabbitmq ls -la /opt/bitnami/rabbitmq/etc/rabbitmq /bitnami/conf
           cat docker-events.log && sleep 1

From e0c8ea764ee3de4238ae1338f83db02a556a84ec Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 11:35:05 -0500
Subject: [PATCH 020/103] debug

---
 .github/workflows/ci.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index cb149160a1..6564e25e72 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -189,13 +189,13 @@ jobs:
           echo
           echo restarting rabbitmq container
           docker events 2>&1 > docker-events.log &
-          docker ps -ef
+          docker exec rabbitmq ps -ef
           docker restart rabbitmq
           # wait for rabbitmq container to restart
           until [ "$(docker inspect -f {{.State.Running}} rabbitmq)" == "true" ]; do sleep 0.1; done
           echo enabled RabbitMQ plugins:
           # print plugins list to: (1) ease debugging, (2) pause till rabbitmq is really running
-          docker ps -ef
+          docker exec rabbitmq ps -ef
           docker exec rabbitmq rabbitmq-plugins list -e
           docker exec rabbitmq ls -la /opt/bitnami/rabbitmq/etc/rabbitmq /bitnami/conf
           cat docker-events.log && sleep 1

From fc8d235a7b4ee00aba6bc39dde0394070fbc77f7 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 12:12:29 -0500
Subject: [PATCH 021/103] debug

---
 .github/workflows/ci.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 6564e25e72..31e009d07a 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -190,7 +190,9 @@ jobs:
           echo restarting rabbitmq container
           docker events 2>&1 > docker-events.log &
           docker exec rabbitmq ps -ef
-          docker restart rabbitmq
+          # docker restart rabbitmq
+          docker stop rabbitmq
+          docker start rabbitmq
           # wait for rabbitmq container to restart
           until [ "$(docker inspect -f {{.State.Running}} rabbitmq)" == "true" ]; do sleep 0.1; done
           echo enabled RabbitMQ plugins:

From 771dbf88701710bebae074aef4af54685d346234 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 12:17:19 -0500
Subject: [PATCH 022/103] debug

---
 .github/workflows/ci.yaml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 31e009d07a..221f12f64e 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -48,10 +48,10 @@ jobs:
           - name: 'Integration Tests'
             task: 'ci-integration'
     services:
-      mongo:
-        image: mongo:4.0
-        ports:
-          - 27017:27017
+#      mongo:
+#        image: mongo:4.0
+#        ports:
+#          - 27017:27017
 
       # In GHA, these services are started first before the code is checked out.
       # We use bitnami images to facilitate reconfiguring RabbitMQ during ci-integration tests.
@@ -189,10 +189,10 @@ jobs:
           echo
           echo restarting rabbitmq container
           docker events 2>&1 > docker-events.log &
-          docker exec rabbitmq ps -ef
-          # docker restart rabbitmq
-          docker stop rabbitmq
-          docker start rabbitmq
+          docker exec rabbitmq ps -ef && sleep 1
+          docker ps --all && sleep 1
+          docker restart rabbitmq
+          docker ps --all && sleep 1
           # wait for rabbitmq container to restart
           until [ "$(docker inspect -f {{.State.Running}} rabbitmq)" == "true" ]; do sleep 0.1; done
           echo enabled RabbitMQ plugins:

From 7d28568949c17205114e83e211acce8545b78417 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 12:19:18 -0500
Subject: [PATCH 023/103] debug

---
 .github/workflows/ci.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 221f12f64e..6358fedddd 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -189,10 +189,10 @@ jobs:
           echo
           echo restarting rabbitmq container
           docker events 2>&1 > docker-events.log &
-          docker exec rabbitmq ps -ef && sleep 1
-          docker ps --all && sleep 1
+          docker exec rabbitmq ps -ef && sleep 0.25
+          docker ps --no-trunc && sleep 0.25
           docker restart rabbitmq
-          docker ps --all && sleep 1
+          docker ps --no-trunc && sleep 0.25
           # wait for rabbitmq container to restart
           until [ "$(docker inspect -f {{.State.Running}} rabbitmq)" == "true" ]; do sleep 0.1; done
           echo enabled RabbitMQ plugins:

From cc886d25dd92195d572fe72309df21b7de5854fa Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 12:39:31 -0500
Subject: [PATCH 024/103] manual custom.conf

---
 .github/workflows/ci.yaml | 20 ++++----------------
 1 file changed, 4 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 6358fedddd..0c8953797d 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -173,36 +173,24 @@ jobs:
         run: |
           set -x
           # Use custom RabbitMQ config which enables SSL / TLS listener on port 5671 with test certs
-          # bitnami/rabbitmq image supports 3 ways to adjust rabbitmq config:
-          #   - merge custom.conf into a generated rabbitmq.conf (we use this)
-          #   - replace generated rabbitmq.conf entirely
-          #   - add advanced.config (erlang) and/or rabbitmq-env.conf
           sudo cp scripts/github/rabbitmq.conf /home/runner/rabbitmq_conf/custom.conf
           # The code is checked out after the container is already up, so we don't mount them.
           # We copy those certs into the dir that is mounted to /bitnami/conf
           sudo cp -r st2tests/st2tests/fixtures/ssl_certs /home/runner/rabbitmq_conf/
-          ls -l /home/runner/rabbitmq_conf
-          docker exec rabbitmq ls -la /opt/bitnami/rabbitmq/etc/rabbitmq /bitnami/conf
           echo
-          echo This is the pre-restart rabbitmq.conf file:
-          docker exec rabbitmq cat /opt/bitnami/rabbitmq/etc/rabbitmq/rabbitmq.conf && sleep 1
+          echo refresh rabbitmq config - based on ENTRYPOINT logic
+          docker exec rabbitmq bash -c 'cat /bitnami/conf/custom.conf >> /opt/bitnami/rabbitmq/etc/rabbitmq/rabbitmq.conf'
           echo
           echo restarting rabbitmq container
-          docker events 2>&1 > docker-events.log &
-          docker exec rabbitmq ps -ef && sleep 0.25
-          docker ps --no-trunc && sleep 0.25
           docker restart rabbitmq
-          docker ps --no-trunc && sleep 0.25
           # wait for rabbitmq container to restart
           until [ "$(docker inspect -f {{.State.Running}} rabbitmq)" == "true" ]; do sleep 0.1; done
           echo enabled RabbitMQ plugins:
           # print plugins list to: (1) ease debugging, (2) pause till rabbitmq is really running
-          docker exec rabbitmq ps -ef
           docker exec rabbitmq rabbitmq-plugins list -e
-          docker exec rabbitmq ls -la /opt/bitnami/rabbitmq/etc/rabbitmq /bitnami/conf
-          cat docker-events.log && sleep 1
+          echo
           echo This is the final rabbitmq.conf file:
-          docker exec rabbitmq cat /opt/bitnami/rabbitmq/etc/rabbitmq/rabbitmq.conf && sleep 1
+          docker exec rabbitmq cat /opt/bitnami/rabbitmq/etc/rabbitmq/rabbitmq.conf && sleep 0.25
           echo
           sudo wget http://guest:guest@localhost:15672/cli/rabbitmqadmin -O /usr/local/bin/rabbitmqadmin
           sudo chmod +x /usr/local/bin/rabbitmqadmin

From f5ae1ee3e3c7e77d70d504e5cdaa2e314ccdfb34 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 13:44:35 -0500
Subject: [PATCH 025/103] manual custom.conf

---
 .github/workflows/ci.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 0c8953797d..96ae9e6891 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -180,6 +180,8 @@ jobs:
           echo
           echo refresh rabbitmq config - based on ENTRYPOINT logic
           docker exec rabbitmq bash -c 'cat /bitnami/conf/custom.conf >> /opt/bitnami/rabbitmq/etc/rabbitmq/rabbitmq.conf'
+          # sleep to prevent interleaved output in GHA logs
+          docker exec rabbitmq cat /opt/bitnami/rabbitmq/etc/rabbitmq/rabbitmq.conf && sleep 0.1
           echo
           echo restarting rabbitmq container
           docker restart rabbitmq
@@ -189,12 +191,10 @@ jobs:
           # print plugins list to: (1) ease debugging, (2) pause till rabbitmq is really running
           docker exec rabbitmq rabbitmq-plugins list -e
           echo
-          echo This is the final rabbitmq.conf file:
-          docker exec rabbitmq cat /opt/bitnami/rabbitmq/etc/rabbitmq/rabbitmq.conf && sleep 0.25
-          echo
           sudo wget http://guest:guest@localhost:15672/cli/rabbitmqadmin -O /usr/local/bin/rabbitmqadmin
           sudo chmod +x /usr/local/bin/rabbitmqadmin
           # print logs
+          docker exec rabbitmq ls /var/log /var/log/rabbitmq
           docker exec rabbitmq bash -c 'tail -n 30 /var/log/rabbitmq/*.log'
       - name: Print versions
         run: |

From 86d05e42ae8f0faf408f60228f7900fba2133090 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 14:15:46 -0500
Subject: [PATCH 026/103] manual custom.conf

---
 .github/workflows/ci.yaml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 96ae9e6891..44b5642737 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -193,9 +193,10 @@ jobs:
           echo
           sudo wget http://guest:guest@localhost:15672/cli/rabbitmqadmin -O /usr/local/bin/rabbitmqadmin
           sudo chmod +x /usr/local/bin/rabbitmqadmin
-          # print logs
-          docker exec rabbitmq ls /var/log /var/log/rabbitmq
-          docker exec rabbitmq bash -c 'tail -n 30 /var/log/rabbitmq/*.log'
+          # print logs RABBITMQ_LOGS=-
+          docker logs --tail=30 rabbitmq
+          docker exec rabbitmq ls /var/log /opt/bitnami/rabbitmq/var/log/rabbitmq
+          docker exec rabbitmq bash -c 'tail -n 30 /opt/bitnami/rabbitmq/var/log/rabbitmq/*.log'
       - name: Print versions
         run: |
           # Print various binary versions

From 2eb713214d4f6ce829101a358e089e8fa7f8a7d7 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 14:30:28 -0500
Subject: [PATCH 027/103] log

---
 .github/workflows/ci.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 44b5642737..c7118e795c 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -196,7 +196,7 @@ jobs:
           # print logs RABBITMQ_LOGS=-
           docker logs --tail=30 rabbitmq
           docker exec rabbitmq ls /var/log /opt/bitnami/rabbitmq/var/log/rabbitmq
-          docker exec rabbitmq bash -c 'tail -n 30 /opt/bitnami/rabbitmq/var/log/rabbitmq/*.log'
+          docker exec rabbitmq bash -c 'tail -n 30 /opt/bitnami/rabbitmq/var/log/rabbitmq/*'
       - name: Print versions
         run: |
           # Print various binary versions

From e0795bc85cfa9effd6461e8183a012a205599944 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 14:34:17 -0500
Subject: [PATCH 028/103] log

---
 .github/workflows/ci.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index c7118e795c..89f0dd96c5 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -195,8 +195,8 @@ jobs:
           sudo chmod +x /usr/local/bin/rabbitmqadmin
           # print logs RABBITMQ_LOGS=-
           docker logs --tail=30 rabbitmq
-          docker exec rabbitmq ls /var/log /opt/bitnami/rabbitmq/var/log/rabbitmq
-          docker exec rabbitmq bash -c 'tail -n 30 /opt/bitnami/rabbitmq/var/log/rabbitmq/*'
+          docker exec rabbitmq ls /var/log /opt/bitnami/rabbitmq/var/log/rabbitmq/log
+          docker exec rabbitmq bash -c 'tail -n 30 /opt/bitnami/rabbitmq/var/log/rabbitmq/log/*'
       - name: Print versions
         run: |
           # Print various binary versions

From 6b8455b59227ac3f8c89ff6843b4724b85fa8684 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 14:41:44 -0500
Subject: [PATCH 029/103] log

---
 .github/workflows/ci.yaml | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 89f0dd96c5..9d788100ee 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -189,14 +189,12 @@ jobs:
           until [ "$(docker inspect -f {{.State.Running}} rabbitmq)" == "true" ]; do sleep 0.1; done
           echo enabled RabbitMQ plugins:
           # print plugins list to: (1) ease debugging, (2) pause till rabbitmq is really running
-          docker exec rabbitmq rabbitmq-plugins list -e
+          docker exec rabbitmq rabbitmq-plugins list
           echo
           sudo wget http://guest:guest@localhost:15672/cli/rabbitmqadmin -O /usr/local/bin/rabbitmqadmin
           sudo chmod +x /usr/local/bin/rabbitmqadmin
-          # print logs RABBITMQ_LOGS=-
-          docker logs --tail=30 rabbitmq
-          docker exec rabbitmq ls /var/log /opt/bitnami/rabbitmq/var/log/rabbitmq/log
-          docker exec rabbitmq bash -c 'tail -n 30 /opt/bitnami/rabbitmq/var/log/rabbitmq/log/*'
+          # print logs from stdout (RABBITMQ_LOGS=-)
+          docker logs --tail=20 rabbitmq
       - name: Print versions
         run: |
           # Print various binary versions

From 90096792c7fb83959603dbd674ccf5d610428bc7 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 14:48:20 -0500
Subject: [PATCH 030/103] reenable

---
 .github/workflows/ci.yaml | 33 ++++++++++++++++-----------------
 1 file changed, 16 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 9d788100ee..3bf8e1cd48 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -48,10 +48,10 @@ jobs:
           - name: 'Integration Tests'
             task: 'ci-integration'
     services:
-#      mongo:
-#        image: mongo:4.0
-#        ports:
-#          - 27017:27017
+      mongo:
+        image: mongo:4.0
+        ports:
+          - 27017:27017
 
       # In GHA, these services are started first before the code is checked out.
       # We use bitnami images to facilitate reconfiguring RabbitMQ during ci-integration tests.
@@ -74,12 +74,12 @@ jobs:
         ports:
           # These 6 ports are exposed by bitnami/rabbitmq (see https://www.rabbitmq.com/networking.html#ports)
           # host_port:container_port/protocol
-          #- 4369:4369/tcp  # epmd
           - 5671:5671/tcp   # AMQP SSL port
           - 5672:5672/tcp   # AMQP standard port
-          #- 15671:15671/tcp  # Management: SSL port
-          - 15672:15672/tcp  # Management: HTTP, CLI
-          #- 25672:25672/tcp  # inter-node or CLI
+          - 15672:15672/tcp # Management: HTTP, CLI
+          #- 15671:15671/tcp # Management: SSL port
+          #- 25672:25672/tcp # inter-node or CLI
+          #- 4369:4369/tcp   # epmd
 
     env:
       TASK: '${{ matrix.task }}'
@@ -141,16 +141,16 @@ jobs:
         run: |
           # install dev dependencies for Python YAML and LDAP packages
           # https://github.com/StackStorm/st2-auth-ldap
-          echo sudo apt-get -y update
-          echo sudo apt-get -f -y install libldap2-dev libsasl2-dev libssl-dev libyaml-dev ldap-utils
+          sudo apt-get -y update
+          sudo apt-get -f -y install libldap2-dev libsasl2-dev libssl-dev libyaml-dev ldap-utils
       - name: Install virtualenv
         run: |
           # Note: Use the verison of virtualenv pinned in fixed-requirements.txt so we
           #       only have to update it one place when we change the version
-          echo pip install --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
+          pip install --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
       - name: Install requirements
         run: |
-          echo ./scripts/travis/install-requirements.sh
+          ./scripts/travis/install-requirements.sh
       - name: Setup integration tests
         run: |
           # prep a ci-specific dev conf file that uses runner instead of stanley
@@ -177,8 +177,7 @@ jobs:
           # The code is checked out after the container is already up, so we don't mount them.
           # We copy those certs into the dir that is mounted to /bitnami/conf
           sudo cp -r st2tests/st2tests/fixtures/ssl_certs /home/runner/rabbitmq_conf/
-          echo
-          echo refresh rabbitmq config - based on ENTRYPOINT logic
+          # refresh rabbitmq config - based on ENTRYPOINT logic
           docker exec rabbitmq bash -c 'cat /bitnami/conf/custom.conf >> /opt/bitnami/rabbitmq/etc/rabbitmq/rabbitmq.conf'
           # sleep to prevent interleaved output in GHA logs
           docker exec rabbitmq cat /opt/bitnami/rabbitmq/etc/rabbitmq/rabbitmq.conf && sleep 0.1
@@ -189,7 +188,7 @@ jobs:
           until [ "$(docker inspect -f {{.State.Running}} rabbitmq)" == "true" ]; do sleep 0.1; done
           echo enabled RabbitMQ plugins:
           # print plugins list to: (1) ease debugging, (2) pause till rabbitmq is really running
-          docker exec rabbitmq rabbitmq-plugins list
+          docker exec rabbitmq rabbitmq-plugins list -e
           echo
           sudo wget http://guest:guest@localhost:15672/cli/rabbitmqadmin -O /usr/local/bin/rabbitmqadmin
           sudo chmod +x /usr/local/bin/rabbitmqadmin
@@ -206,7 +205,7 @@ jobs:
       - name: make
         # use: script -e -c to print colors
         run: |
-          echo script -e -c "make ${TASK}"
+          script -e -c "make ${TASK}"
       - name: Nightly
         # Run any additional nightly checks only as part of a nightly (cron) build
         if: "${{ env.IS_NIGHTLY_BUILD == 'yes' }}"
@@ -216,7 +215,7 @@ jobs:
         # NOTE: We only generate and submit coverage report for master and version branches and only when the build succeeds (default on GitHub Actions, this was not the case on Travis so we had to explicitly check success)
         if: "${{ success() && ((env.TASK == 'ci-unit') || (env.TASK == 'ci-integration')) && (env.ENABLE_COVERAGE == 'yes') }}"
         run: |
-          echo ./scripts/travis/submit-codecov-coverage.sh
+          ./scripts/travis/submit-codecov-coverage.sh
   slack-notification:
     name: Slack notification for failed master builds
     if: always()

From b81b15385bb9cc5e431a69c1b32e82441a7fb78e Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 14:59:26 -0500
Subject: [PATCH 031/103] remove git clone progress reporting (reduce CI noise)

---
 tools/launchdev.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tools/launchdev.sh b/tools/launchdev.sh
index 9e6fc9bc46..060a2d5c0e 100755
--- a/tools/launchdev.sh
+++ b/tools/launchdev.sh
@@ -153,7 +153,8 @@ function st2start(){
         cp -Rp ./contrib/examples $PACKS_BASE_DIR
         # Clone st2tests in /tmp directory.
         pushd /tmp
-        git clone https://github.com/StackStorm/st2tests.git
+		# -q = no progress bar (better for CI)
+        git clone -q https://github.com/StackStorm/st2tests.git
         ret=$?
         if [ ${ret} -eq 0 ]; then
             cp -Rp ./st2tests/packs/fixtures $PACKS_BASE_DIR

From b80831e986ccf75d6587fc9ae8a0794e4a9df3a4 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 15:00:20 -0500
Subject: [PATCH 032/103] add python-version for ci-integration tests

---
 .github/workflows/ci.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 3bf8e1cd48..34c586c93a 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -44,9 +44,9 @@ jobs:
           # - name: 'Micro Benchmarks'
           #   task: 'micro-benchmarks'
           #   python-version: '3.6'
-          # Integration tests are not working yet, still done in Travis
           - name: 'Integration Tests'
             task: 'ci-integration'
+            python-version: '3.6'
     services:
       mongo:
         image: mongo:4.0

From d395a358c4f5c4dadac431453007c1d9b0b86045 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 15:33:55 -0500
Subject: [PATCH 033/103] add echo above git clone -q

---
 tools/launchdev.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tools/launchdev.sh b/tools/launchdev.sh
index 060a2d5c0e..6ed8840aaa 100755
--- a/tools/launchdev.sh
+++ b/tools/launchdev.sh
@@ -153,7 +153,8 @@ function st2start(){
         cp -Rp ./contrib/examples $PACKS_BASE_DIR
         # Clone st2tests in /tmp directory.
         pushd /tmp
-		# -q = no progress bar (better for CI)
+		echo Cloning https://github.com/StackStorm/st2tests.git
+		# -q = no progress reporting (better for CI). Errors will still print.
         git clone -q https://github.com/StackStorm/st2tests.git
         ret=$?
         if [ ${ret} -eq 0 ]; then

From 894380270466bd7195c45542b977ad96f03a08c1 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 15:34:25 -0500
Subject: [PATCH 034/103] add default rabbitmq user/pass

---
 .github/workflows/ci.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 34c586c93a..34d5499fc9 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -66,6 +66,9 @@ jobs:
         env:
           # tell bitnami/rabbitmq to enable this by default
           RABBITMQ_PLUGINS: rabbitmq_management
+          # TODO: don't hardcode the user as guest:guest in tests - maybe specify via an env var
+          RABBITMQ_DEFAULT_USER: guest
+          RABBITMQ_DEFAULT_PASS: guest
 
         # These are strictly docker options, not entrypoint args (GHA restriction)
         options: >-

From dbae76e0ba2b02b1eab0fbdcd0a260ca55c53798 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 15:57:25 -0500
Subject: [PATCH 035/103] debug

---
 .github/workflows/ci.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 34d5499fc9..8ca48da27e 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -67,6 +67,7 @@ jobs:
           # tell bitnami/rabbitmq to enable this by default
           RABBITMQ_PLUGINS: rabbitmq_management
           # TODO: don't hardcode the user as guest:guest in tests - maybe specify via an env var
+          RABBITMQ_SECURE_PASSWORD: "no"
           RABBITMQ_DEFAULT_USER: guest
           RABBITMQ_DEFAULT_PASS: guest
 
@@ -209,6 +210,9 @@ jobs:
         # use: script -e -c to print colors
         run: |
           script -e -c "make ${TASK}"
+          success=$?
+          docker logs --tail=100 rabbitmq
+          exit $success
       - name: Nightly
         # Run any additional nightly checks only as part of a nightly (cron) build
         if: "${{ env.IS_NIGHTLY_BUILD == 'yes' }}"

From 1a7eb21eda3bbffae6d035a4ea3d13ef72fb8430 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 16:02:48 -0500
Subject: [PATCH 036/103] debug

---
 .github/workflows/ci.yaml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 8ca48da27e..5d9aedbd5c 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -209,10 +209,8 @@ jobs:
       - name: make
         # use: script -e -c to print colors
         run: |
-          script -e -c "make ${TASK}"
-          success=$?
-          docker logs --tail=100 rabbitmq
-          exit $success
+          script -e -c "make ${TASK}" || docker logs --tail=100 rabbitmq
+          exit 5
       - name: Nightly
         # Run any additional nightly checks only as part of a nightly (cron) build
         if: "${{ env.IS_NIGHTLY_BUILD == 'yes' }}"

From 10587786bd8bca41c072fcbebf57d87dfaa5d814 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 16:36:34 -0500
Subject: [PATCH 037/103] do not hardcode RMQ user/pass in tests

---
 .github/workflows/ci.yaml                     | 10 +++--
 .../integration/test_rabbitmq_ssl_listener.py | 39 +++++++------------
 2 files changed, 21 insertions(+), 28 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 5d9aedbd5c..c9e23ca359 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -66,10 +66,10 @@ jobs:
         env:
           # tell bitnami/rabbitmq to enable this by default
           RABBITMQ_PLUGINS: rabbitmq_management
-          # TODO: don't hardcode the user as guest:guest in tests - maybe specify via an env var
+          # Since we're using a container, we can't use guest:guest
           RABBITMQ_SECURE_PASSWORD: "no"
-          RABBITMQ_DEFAULT_USER: guest
-          RABBITMQ_DEFAULT_PASS: guest
+          RABBITMQ_USERNAME: st2
+          RABBITMQ_PASSWORD: st2
 
         # These are strictly docker options, not entrypoint args (GHA restriction)
         options: >-
@@ -104,6 +104,10 @@ jobs:
 
       # Name of the user who is running the CI (on GitHub Actions this is 'runner')
       ST2_CI_USER: 'runner'
+
+      # Since we're using a container, we can't use guest:guest
+      RABBITMQ_USERNAME: st2
+      RABBITMQ_PASSWORD: st2
     steps:
       - name: Custom Environment Setup
         # built-in GitHub Actions environment variables
diff --git a/st2common/tests/integration/test_rabbitmq_ssl_listener.py b/st2common/tests/integration/test_rabbitmq_ssl_listener.py
index e64a22995d..4e3eba6801 100644
--- a/st2common/tests/integration/test_rabbitmq_ssl_listener.py
+++ b/st2common/tests/integration/test_rabbitmq_ssl_listener.py
@@ -31,6 +31,11 @@
 
 CERTS_FIXTURES_PATH = os.path.join(get_fixtures_base_path(), "ssl_certs/")
 ST2_CI = os.environ.get("ST2_CI", "false").lower() == "true"
+RMQ_USER = os.environ.get("RABBITMQ_USERNAME", "guest")
+RMQ_PASS = os.environ.get("RABBITMQ_PASSWORD", "guest")
+RMQ_HOST = os.environ.get("RABBITMQ_HOST", "127.0.0.1")
+RMQ_PORT = os.environ.get("RABBITMQ_PORT", "5671")
+RMQ_URL = f"amqp://{RMQ_USER}:{RMQ_PASS}@{RMQ_HOST}:{RMQ_PORT}/"
 
 NON_SSL_LISTENER_PORT = 5672
 SSL_LISTENER_PORT = 5671
@@ -52,9 +57,7 @@ def setUp(self):
         cfg.CONF.set_override(name="ssl_cert_reqs", override=None, group="messaging")
 
     def test_non_ssl_connection_on_ssl_listener_port_failure(self):
-        connection = transport_utils.get_connection(
-            urls="amqp://guest:guest@127.0.0.1:5671/"
-        )
+        connection = transport_utils.get_connection(urls=RMQ_URL)
 
         expected_msg_1 = (
             "[Errno 104]"  # followed by: ' Connection reset by peer' or ' ECONNRESET'
@@ -80,7 +83,7 @@ def test_non_ssl_connection_on_ssl_listener_port_failure(self):
 
     def test_ssl_connection_on_ssl_listener_success(self):
         # Using query param notation
-        urls = "amqp://guest:guest@127.0.0.1:5671/?ssl=true"
+        urls = f"{RMQ_URL}?ssl=true"
         connection = transport_utils.get_connection(urls=urls)
 
         try:
@@ -93,9 +96,7 @@ def test_ssl_connection_on_ssl_listener_success(self):
         # Using messaging.ssl config option
         cfg.CONF.set_override(name="ssl", override=True, group="messaging")
 
-        connection = transport_utils.get_connection(
-            urls="amqp://guest:guest@127.0.0.1:5671/"
-        )
+        connection = transport_utils.get_connection(urls=RMQ_URL)
 
         try:
             self.assertTrue(connection.connect())
@@ -117,9 +118,7 @@ def test_ssl_connection_ca_certs_provided(self):
             name="ssl_cert_reqs", override="required", group="messaging"
         )
 
-        connection = transport_utils.get_connection(
-            urls="amqp://guest:guest@127.0.0.1:5671/"
-        )
+        connection = transport_utils.get_connection(urls=RMQ_URL)
 
         try:
             self.assertTrue(connection.connect())
@@ -139,9 +138,7 @@ def test_ssl_connection_ca_certs_provided(self):
             name="ssl_ca_certs", override=ca_cert_path, group="messaging"
         )
 
-        connection = transport_utils.get_connection(
-            urls="amqp://guest:guest@127.0.0.1:5671/"
-        )
+        connection = transport_utils.get_connection(urls=RMQ_URL)
 
         expected_msg = r"\[SSL: CERTIFICATE_VERIFY_FAILED\] certificate verify failed"
         self.assertRaisesRegexp(ssl.SSLError, expected_msg, connection.connect)
@@ -156,9 +153,7 @@ def test_ssl_connection_ca_certs_provided(self):
             name="ssl_ca_certs", override=ca_cert_path, group="messaging"
         )
 
-        connection = transport_utils.get_connection(
-            urls="amqp://guest:guest@127.0.0.1:5671/"
-        )
+        connection = transport_utils.get_connection(urls=RMQ_URL)
 
         expected_msg = r"\[SSL: CERTIFICATE_VERIFY_FAILED\] certificate verify failed"
         self.assertRaisesRegexp(ssl.SSLError, expected_msg, connection.connect)
@@ -172,9 +167,7 @@ def test_ssl_connection_ca_certs_provided(self):
             name="ssl_ca_certs", override=ca_cert_path, group="messaging"
         )
 
-        connection = transport_utils.get_connection(
-            urls="amqp://guest:guest@127.0.0.1:5671/"
-        )
+        connection = transport_utils.get_connection(urls=RMQ_URL)
 
         try:
             self.assertTrue(connection.connect())
@@ -204,9 +197,7 @@ def test_ssl_connect_client_side_cert_authentication(self):
             name="ssl_ca_certs", override=ca_cert_path, group="messaging"
         )
 
-        connection = transport_utils.get_connection(
-            urls="amqp://guest:guest@127.0.0.1:5671/"
-        )
+        connection = transport_utils.get_connection(urls=RMQ_URL)
 
         try:
             self.assertTrue(connection.connect())
@@ -235,9 +226,7 @@ def test_ssl_connect_client_side_cert_authentication(self):
             name="ssl_ca_certs", override=ca_cert_path, group="messaging"
         )
 
-        connection = transport_utils.get_connection(
-            urls="amqp://guest:guest@127.0.0.1:5671/"
-        )
+        connection = transport_utils.get_connection(urls=RMQ_URL)
 
         expected_msg = r"\[X509: KEY_VALUES_MISMATCH\] key values mismatch"
         self.assertRaisesRegexp(ssl.SSLError, expected_msg, connection.connect)

From 4948c40564c1bf96edd19485a0149e013e0fccdb Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 16:48:56 -0500
Subject: [PATCH 038/103] rmq user

---
 .github/workflows/ci.yaml | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index c9e23ca359..51993bf893 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -93,7 +93,9 @@ jobs:
       COLUMNS: '120'
       PYLINT_CONCURRENCY: '2'
 
-      # CI st2.conf (with ST2_CI_USER user instead of stanley)
+      # CI st2.conf
+      #   - with ST2_CI_USER user instead of stanley
+      #   - with RABBITMQ_USERNAME:RABBITMQ_PASSWORD instead of guest:guest
       ST2_CONF: 'conf/st2.ci.conf'
 
       # Tell StackStorm that we are indeed in CI mode, previously we hard coded a Travis specific
@@ -161,10 +163,12 @@ jobs:
           ./scripts/travis/install-requirements.sh
       - name: Setup integration tests
         run: |
-          # prep a ci-specific dev conf file that uses runner instead of stanley
-          # this user is the username of the user in GitHub actions, used for SSH, etc during
-          # integration tests (important)
-          cp conf/st2.dev.conf "${ST2_CONF}" ; sed -i -e "s/stanley/${ST2_CI_USER}/" "${ST2_CONF}"
+          # prep a ci-specific dev conf file that
+          #   - uses runner instead of stanley
+          #     this user is the username of the user in GitHub actions,
+          #     used for SSH, etc during integration tests (important)
+          #   - uses a non-guest user/pass for RabbitMQ since we can't use guest outside of the container
+          cp conf/st2.dev.conf "${ST2_CONF}" ; sed -i -e "s/stanley/${ST2_CI_USER}/" -e "s/guest:guest/${RABBITMQ_USERNAME}:${RABBITMQ_PASSWORD}/" "${ST2_CONF}"
           scripts/travis/add-itest-user-key.sh
           sudo .circle/add-itest-user.sh
       - name: Permissions Workaround

From 141e7e0d82f60564f397817f956b1abd35bfb449 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 17:21:04 -0500
Subject: [PATCH 039/103] override more rabbitmq test urls

---
 .../integration/test_rabbitmq_ssl_listener.py   |  2 ++
 st2tests/st2tests/config.py                     | 17 +++++++++++++++++
 2 files changed, 19 insertions(+)

diff --git a/st2common/tests/integration/test_rabbitmq_ssl_listener.py b/st2common/tests/integration/test_rabbitmq_ssl_listener.py
index 4e3eba6801..a523098c05 100644
--- a/st2common/tests/integration/test_rabbitmq_ssl_listener.py
+++ b/st2common/tests/integration/test_rabbitmq_ssl_listener.py
@@ -31,6 +31,8 @@
 
 CERTS_FIXTURES_PATH = os.path.join(get_fixtures_base_path(), "ssl_certs/")
 ST2_CI = os.environ.get("ST2_CI", "false").lower() == "true"
+
+# If CI has RabbitMQ running in a container, then we can't use guest.
 RMQ_USER = os.environ.get("RABBITMQ_USERNAME", "guest")
 RMQ_PASS = os.environ.get("RABBITMQ_PASSWORD", "guest")
 RMQ_HOST = os.environ.get("RABBITMQ_HOST", "127.0.0.1")
diff --git a/st2tests/st2tests/config.py b/st2tests/st2tests/config.py
index b140357839..9aa6c41a99 100644
--- a/st2tests/st2tests/config.py
+++ b/st2tests/st2tests/config.py
@@ -57,6 +57,7 @@ def _override_config_opts(coordinator_noop=False):
     _override_db_opts()
     _override_common_opts()
     _override_api_opts()
+    _override_messaging_opts()
     _override_keyvalue_opts()
     _override_scheduler_opts()
     _override_workflow_engine_opts()
@@ -104,6 +105,22 @@ def _override_api_opts():
     )
 
 
+def _override_messaging_opts():
+    # If CI has RabbitMQ running in a container, then we can't use guest.
+    rmq_user = os.environ.get("RABBITMQ_USERNAME", "guest")
+    rmq_pass = os.environ.get("RABBITMQ_PASSWORD", "guest")
+    rmq_host = os.environ.get("RABBITMQ_HOST", "127.0.0.1")
+    rmq_port = os.environ.get("RABBITMQ_PORT", "5672")
+    override_rmq_url = f"amqp://{RMQ_USER}:{RMQ_PASS}@{RMQ_HOST}:{RMQ_PORT}//"
+
+    if override_rmq_url != "amqp://guest:guest@127.0.0.1:5672//":
+        CONF.set_override(
+            name="url",
+            override=rmq_url,
+            group="messaging",
+        )
+
+
 def _override_keyvalue_opts():
     current_file_path = os.path.dirname(__file__)
     rel_st2_base_path = os.path.join(current_file_path, "../..")

From c4b78a678ddc12027885eb948f93abfe3e932936 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 17:21:21 -0500
Subject: [PATCH 040/103] debug

---
 .github/workflows/ci.yaml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 51993bf893..8ef976274c 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -217,8 +217,7 @@ jobs:
       - name: make
         # use: script -e -c to print colors
         run: |
-          script -e -c "make ${TASK}" || docker logs --tail=100 rabbitmq
-          exit 5
+          script -e -c "make ${TASK}"
       - name: Nightly
         # Run any additional nightly checks only as part of a nightly (cron) build
         if: "${{ env.IS_NIGHTLY_BUILD == 'yes' }}"

From fa6d3047998dea8431f0a5fb68f8de338a96854c Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 17:24:08 -0500
Subject: [PATCH 041/103] whitespace

---
 tools/launchdev.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/launchdev.sh b/tools/launchdev.sh
index 6ed8840aaa..d50ab9a4f4 100755
--- a/tools/launchdev.sh
+++ b/tools/launchdev.sh
@@ -153,8 +153,8 @@ function st2start(){
         cp -Rp ./contrib/examples $PACKS_BASE_DIR
         # Clone st2tests in /tmp directory.
         pushd /tmp
-		echo Cloning https://github.com/StackStorm/st2tests.git
-		# -q = no progress reporting (better for CI). Errors will still print.
+        echo Cloning https://github.com/StackStorm/st2tests.git
+        # -q = no progress reporting (better for CI). Errors will still print.
         git clone -q https://github.com/StackStorm/st2tests.git
         ret=$?
         if [ ${ret} -eq 0 ]; then

From c97e9c190c08fa27ac7113bf8899917e88e95791 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 17:35:49 -0500
Subject: [PATCH 042/103] fix vars

---
 st2tests/st2tests/config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/st2tests/st2tests/config.py b/st2tests/st2tests/config.py
index 9aa6c41a99..db2df67132 100644
--- a/st2tests/st2tests/config.py
+++ b/st2tests/st2tests/config.py
@@ -111,7 +111,7 @@ def _override_messaging_opts():
     rmq_pass = os.environ.get("RABBITMQ_PASSWORD", "guest")
     rmq_host = os.environ.get("RABBITMQ_HOST", "127.0.0.1")
     rmq_port = os.environ.get("RABBITMQ_PORT", "5672")
-    override_rmq_url = f"amqp://{RMQ_USER}:{RMQ_PASS}@{RMQ_HOST}:{RMQ_PORT}//"
+    override_rmq_url = f"amqp://{rmq_user}:{rmq_pass}@{rmq_host}:{rmq_port}//"
 
     if override_rmq_url != "amqp://guest:guest@127.0.0.1:5672//":
         CONF.set_override(

From beacbd88551fa9888b0b89edf44df6670e906e83 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 17:41:31 -0500
Subject: [PATCH 043/103] fix var

---
 st2tests/st2tests/config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/st2tests/st2tests/config.py b/st2tests/st2tests/config.py
index db2df67132..e7e84f47d5 100644
--- a/st2tests/st2tests/config.py
+++ b/st2tests/st2tests/config.py
@@ -116,7 +116,7 @@ def _override_messaging_opts():
     if override_rmq_url != "amqp://guest:guest@127.0.0.1:5672//":
         CONF.set_override(
             name="url",
-            override=rmq_url,
+            override=override_rmq_url,
             group="messaging",
         )
 

From ebbc44faf5207c5836a429efbe14fa69585128ea Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 18:06:13 -0500
Subject: [PATCH 044/103] fix integration test conf files on the fly

---
 .github/workflows/ci.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 8ef976274c..27def5fd4f 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -184,6 +184,9 @@ jobs:
         timeout-minutes: 2  # may die if rabbitmq fails to start
         run: |
           set -x
+          # Since we're using a container, we can't use guest:guest
+          # Integration tests hardcode paths to conf/st2.tests*.conf so we modify in place.
+          sed -i -e "s/guest:guest/${RABBITMQ_USERNAME}:${RABBITMQ_PASSWORD}/" conf/st2.tests*.conf
           # Use custom RabbitMQ config which enables SSL / TLS listener on port 5671 with test certs
           sudo cp scripts/github/rabbitmq.conf /home/runner/rabbitmq_conf/custom.conf
           # The code is checked out after the container is already up, so we don't mount them.

From a99c1bd94cabd628c393f498220884de31a2b657 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 18:43:20 -0500
Subject: [PATCH 045/103] fix integration tests

---
 .github/workflows/ci.yaml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 27def5fd4f..3cbb5287e2 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -176,7 +176,7 @@ jobs:
         run: |
           echo "$ST2_CI_REPO_PATH"
           sudo ST2_CI_REPO_PATH="${ST2_CI_REPO_PATH}" scripts/travis/permissions-workaround.sh
-      - name: Setup RabbitMQ
+      - name: Finish Integration test setup (RabbitMQ, st2.tests.conf)
         if: "${{ env.TASK == 'ci-integration' }}"
         # bitnami image allows (see bitnami/rabbitmq readme):
         # Here we're copying a rabbitmq.config file which won't do anything.
@@ -184,9 +184,12 @@ jobs:
         timeout-minutes: 2  # may die if rabbitmq fails to start
         run: |
           set -x
+          ls -l /tmp /etc/ssl /etc/ssl/certs || true
           # Since we're using a container, we can't use guest:guest
           # Integration tests hardcode paths to conf/st2.tests*.conf so we modify in place.
           sed -i -e "s/guest:guest/${RABBITMQ_USERNAME}:${RABBITMQ_PASSWORD}/" conf/st2.tests*.conf
+          # use a temp directory that is actually writeable (for pack installation tests)
+          sed -i -e "s/base_path = .*/base_path = ${{ runner.temp }}/" conf/st2.tests*.conf
           # Use custom RabbitMQ config which enables SSL / TLS listener on port 5671 with test certs
           sudo cp scripts/github/rabbitmq.conf /home/runner/rabbitmq_conf/custom.conf
           # The code is checked out after the container is already up, so we don't mount them.

From ced48424763ff559ae13c9b7d2da361d20a73439 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 18:50:14 -0500
Subject: [PATCH 046/103] debug

---
 .github/workflows/ci.yaml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 3cbb5287e2..11bb5f8ea5 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -184,12 +184,13 @@ jobs:
         timeout-minutes: 2  # may die if rabbitmq fails to start
         run: |
           set -x
-          ls -l /tmp /etc/ssl /etc/ssl/certs || true
+          ls -l /etc/ssl/certs || true
+          ls -ld /tmp || true
           # Since we're using a container, we can't use guest:guest
           # Integration tests hardcode paths to conf/st2.tests*.conf so we modify in place.
           sed -i -e "s/guest:guest/${RABBITMQ_USERNAME}:${RABBITMQ_PASSWORD}/" conf/st2.tests*.conf
           # use a temp directory that is actually writeable (for pack installation tests)
-          sed -i -e "s/base_path = .*/base_path = ${{ runner.temp }}/" conf/st2.tests*.conf
+          sed -i -e "s|base_path = .*|base_path = ${{ runner.temp }}|" conf/st2.tests*.conf
           # Use custom RabbitMQ config which enables SSL / TLS listener on port 5671 with test certs
           sudo cp scripts/github/rabbitmq.conf /home/runner/rabbitmq_conf/custom.conf
           # The code is checked out after the container is already up, so we don't mount them.

From a49c7f8df8afede1bc8ac531b99ff7e34afb13ad Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 19:16:14 -0500
Subject: [PATCH 047/103] Switch tests to check w/ CA that is present in GHA

---
 st2common/tests/integration/test_rabbitmq_ssl_listener.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/st2common/tests/integration/test_rabbitmq_ssl_listener.py b/st2common/tests/integration/test_rabbitmq_ssl_listener.py
index a523098c05..2dd9209efc 100644
--- a/st2common/tests/integration/test_rabbitmq_ssl_listener.py
+++ b/st2common/tests/integration/test_rabbitmq_ssl_listener.py
@@ -131,7 +131,7 @@ def test_ssl_connection_ca_certs_provided(self):
 
         # 2. Validate server cert against other CA bundle (failure)
         # CA bundle which was not used to sign the server cert
-        ca_cert_path = os.path.join("/etc/ssl/certs/thawte_Primary_Root_CA.pem")
+        ca_cert_path = os.path.join("/etc/ssl/certs/SecureTrust_CA.pem")
 
         cfg.CONF.set_override(
             name="ssl_cert_reqs", override="required", group="messaging"
@@ -146,7 +146,7 @@ def test_ssl_connection_ca_certs_provided(self):
         self.assertRaisesRegexp(ssl.SSLError, expected_msg, connection.connect)
 
         # 3. Validate server cert against other CA bundle (failure)
-        ca_cert_path = os.path.join("/etc/ssl/certs/thawte_Primary_Root_CA.pem")
+        ca_cert_path = os.path.join("/etc/ssl/certs/SecureTrust_CA.pem")
 
         cfg.CONF.set_override(
             name="ssl_cert_reqs", override="optional", group="messaging"
@@ -162,7 +162,7 @@ def test_ssl_connection_ca_certs_provided(self):
 
         # 4. Validate server cert against other CA bundle (failure)
         # We use invalid bundle but cert_reqs is none
-        ca_cert_path = os.path.join("/etc/ssl/certs/thawte_Primary_Root_CA.pem")
+        ca_cert_path = os.path.join("/etc/ssl/certs/SecureTrust_CA.pem")
 
         cfg.CONF.set_override(name="ssl_cert_reqs", override="none", group="messaging")
         cfg.CONF.set_override(

From de4207253758c22c9e51ed22b56e5d0cd95a81fe Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 19:16:26 -0500
Subject: [PATCH 048/103] debug

---
 .github/workflows/ci.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 11bb5f8ea5..b35b0e29af 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -185,12 +185,12 @@ jobs:
         run: |
           set -x
           ls -l /etc/ssl/certs || true
-          ls -ld /tmp || true
           # Since we're using a container, we can't use guest:guest
           # Integration tests hardcode paths to conf/st2.tests*.conf so we modify in place.
           sed -i -e "s/guest:guest/${RABBITMQ_USERNAME}:${RABBITMQ_PASSWORD}/" conf/st2.tests*.conf
+          # ls -ld /tmp || true
           # use a temp directory that is actually writeable (for pack installation tests)
-          sed -i -e "s|base_path = .*|base_path = ${{ runner.temp }}|" conf/st2.tests*.conf
+          #sed -i -e "s|base_path = .*|base_path = ${{ runner.temp }}|" conf/st2.tests*.conf
           # Use custom RabbitMQ config which enables SSL / TLS listener on port 5671 with test certs
           sudo cp scripts/github/rabbitmq.conf /home/runner/rabbitmq_conf/custom.conf
           # The code is checked out after the container is already up, so we don't mount them.

From cf611a56f9bd1d24f19c08f6b847c7b64f9b3227 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 19:55:37 -0500
Subject: [PATCH 049/103] debug

---
 .github/workflows/ci.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index b35b0e29af..e0a580c8f0 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -184,7 +184,7 @@ jobs:
         timeout-minutes: 2  # may die if rabbitmq fails to start
         run: |
           set -x
-          ls -l /etc/ssl/certs || true
+          ls -ld /home/runner/.local/share/virtualenv/py_info /home/runner/.local/share/virtualenv /home/runner/.local/share /home/runner/.local /home/runner /home / || true
           # Since we're using a container, we can't use guest:guest
           # Integration tests hardcode paths to conf/st2.tests*.conf so we modify in place.
           sed -i -e "s/guest:guest/${RABBITMQ_USERNAME}:${RABBITMQ_PASSWORD}/" conf/st2.tests*.conf

From b41591196361633b970313c93e48e2c57df21433 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 20:02:27 -0500
Subject: [PATCH 050/103] try working around permissions issue

---
 .github/workflows/ci.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index e0a580c8f0..36d639d1f5 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -176,6 +176,7 @@ jobs:
         run: |
           echo "$ST2_CI_REPO_PATH"
           sudo ST2_CI_REPO_PATH="${ST2_CI_REPO_PATH}" scripts/travis/permissions-workaround.sh
+          sudo chmod -R o+w /home/runner/.local/share/virtualenv
       - name: Finish Integration test setup (RabbitMQ, st2.tests.conf)
         if: "${{ env.TASK == 'ci-integration' }}"
         # bitnami image allows (see bitnami/rabbitmq readme):

From 406276e63a57f74d600d3615ee444964efee8abd Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 22:04:19 -0500
Subject: [PATCH 051/103] debug

---
 tools/launchdev.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tools/launchdev.sh b/tools/launchdev.sh
index d50ab9a4f4..6009605ab8 100755
--- a/tools/launchdev.sh
+++ b/tools/launchdev.sh
@@ -55,6 +55,7 @@ function init(){
     CURRENT_DIR=`pwd`
     CURRENT_USER=`whoami`
     CURRENT_USER_GROUP=`id -gn`
+	echo "Current user:group = ${CURRENT_USER}:${CURRENT_USER_GROUP}"
 
     if [[ (${COMMAND_PATH} == /*) ]] ;
     then

From 1aa5dd70b4bb53a2e29172edc788c142cbd702c3 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 22:18:39 -0500
Subject: [PATCH 052/103] improve GHA folding

---
 scripts/travis/prepare-integration.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/scripts/travis/prepare-integration.sh b/scripts/travis/prepare-integration.sh
index dc6efa83a4..c46c71f813 100755
--- a/scripts/travis/prepare-integration.sh
+++ b/scripts/travis/prepare-integration.sh
@@ -16,6 +16,9 @@ st2 --version
 # Clean up old screen log files
 rm -f logs/screen-*.log
 
+# ::group::/::endgroup:: is helpful github actions syntax to fold this section.
+echo ::group::launchdev.sh start -x
+
 # start dev environment in screens
 ./tools/launchdev.sh start -x
 
@@ -28,6 +31,9 @@ echo " === START: Catting screen process log files. ==="
 cat logs/screen-*.log
 echo " === END: Catting screen process log files. ==="
 
+# github actions: fold for launchdev.sh start -x
+echo ::endgroup::
+
 # Setup the virtualenv for the examples pack which is required for orquesta integration tests.
 st2 run packs.setup_virtualenv packs=examples
 

From e5ccd7643325e06331b68864cf82dda580b30277 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 22:52:18 -0500
Subject: [PATCH 053/103] debug

---
 .github/workflows/ci.yaml | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 36d639d1f5..b270bb1020 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -176,7 +176,7 @@ jobs:
         run: |
           echo "$ST2_CI_REPO_PATH"
           sudo ST2_CI_REPO_PATH="${ST2_CI_REPO_PATH}" scripts/travis/permissions-workaround.sh
-          sudo chmod -R o+w /home/runner/.local/share/virtualenv
+          # sudo chmod -R o+w /home/runner/.local/share/virtualenv
       - name: Finish Integration test setup (RabbitMQ, st2.tests.conf)
         if: "${{ env.TASK == 'ci-integration' }}"
         # bitnami image allows (see bitnami/rabbitmq readme):
@@ -189,7 +189,6 @@ jobs:
           # Since we're using a container, we can't use guest:guest
           # Integration tests hardcode paths to conf/st2.tests*.conf so we modify in place.
           sed -i -e "s/guest:guest/${RABBITMQ_USERNAME}:${RABBITMQ_PASSWORD}/" conf/st2.tests*.conf
-          # ls -ld /tmp || true
           # use a temp directory that is actually writeable (for pack installation tests)
           #sed -i -e "s|base_path = .*|base_path = ${{ runner.temp }}|" conf/st2.tests*.conf
           # Use custom RabbitMQ config which enables SSL / TLS listener on port 5671 with test certs
@@ -225,7 +224,11 @@ jobs:
       - name: make
         # use: script -e -c to print colors
         run: |
-          script -e -c "make ${TASK}"
+          sudo ls -l /tmp /home/runner/work/st2/st2/virtualenv/bin/python /home/runner/.local/share/virtualenv/py_info/* || true
+          script -e -c "make ${TASK}" || true
+          sudo ls -l /tmp /home/runner/work/st2/st2/virtualenv/bin/python /home/runner/.local/share/virtualenv/py_info/* || true
+          sudo ls -ld /home/runner/.local/share/virtualenv/py_info /home/runner/.local/share/virtualenv /home/runner/.local/share /home/runner/.local /home/runner /home /
+          exit 5
       - name: Nightly
         # Run any additional nightly checks only as part of a nightly (cron) build
         if: "${{ env.IS_NIGHTLY_BUILD == 'yes' }}"

From d14f0cf4e569da328fa25a3e7cfd70bb88de4171 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 22:52:42 -0500
Subject: [PATCH 054/103] apt download cache

---
 .github/workflows/ci.yaml | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index b270bb1020..f9d1bf96bb 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -139,6 +139,12 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: '${{ matrix.python-version }}'
+      - name: Get date components for use in cache-keys
+        id: date
+        run: |
+          echo "::set-output name=year::$(/bin/date -u "+%Y")"
+          echo "::set-output name=month::$(/bin/date -u "+%m")"
+          echo "::set-output name=week::$(/bin/date -u "+%U")"
       - uses: actions/cache@v2
         with:
           path: |
@@ -147,6 +153,16 @@ jobs:
           key: ${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt') }}
           restore-keys: |
             ${{ runner.os }}-${{ matrix.python }}-
+      - uses: actions/cache@v2
+        with:
+          path: |
+            /var/cache/apt/archives/*.deb
+            /var/cache/apt/archives/partial/*.deb
+            /var/cache/apt/*.bin
+          key: apt-${{ runner.os }}-${{ steps.date.outputs.year }}-${{ steps.date.outputs.week }}
+          restore-keys: |
+            apt-${{ runner.os }}-${{ steps.date.outputs.year }}-
+            apt-${{ runner.os }}-
       - name: Install apt depedencies
         run: |
           # install dev dependencies for Python YAML and LDAP packages

From 976933c9be0936410793668c6b71b740eb966961 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 23:18:25 -0500
Subject: [PATCH 055/103] fix virtualenv lock files issue again

---
 scripts/travis/prepare-integration.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/travis/prepare-integration.sh b/scripts/travis/prepare-integration.sh
index c46c71f813..7c05c89820 100755
--- a/scripts/travis/prepare-integration.sh
+++ b/scripts/travis/prepare-integration.sh
@@ -45,3 +45,5 @@ chmod 777 logs/*
 # root needs to access write some lock files when creating virtualenvs
 # o=other; X=only set execute bit if user execute bit is set (eg on dirs)
 chmod -R o+rwX ./virtualenv/
+# newer virtualenv versions are putting lock files under ~/.local
+chmod -R o+rwX ~${ST2_CI_USER}/.local/share/virtualenv

From 1c6442d22123b7d3ab062168a05bb7bc3432d886 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 23:31:28 -0500
Subject: [PATCH 056/103] bust python cache

---
 .github/workflows/ci.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index f9d1bf96bb..7ed392ca7b 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -150,9 +150,9 @@ jobs:
           path: |
             ~/.cache/pip
             virtualenv
-          key: ${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt') }}
+          key: py-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt') }}
           restore-keys: |
-            ${{ runner.os }}-${{ matrix.python }}-
+            py-${{ runner.os }}-${{ matrix.python }}-
       - uses: actions/cache@v2
         with:
           path: |

From bda229140a8ab7f9231177d19f01fd5e0d8865ba Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 23:31:43 -0500
Subject: [PATCH 057/103] fix $HOME usage

---
 scripts/travis/prepare-integration.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/travis/prepare-integration.sh b/scripts/travis/prepare-integration.sh
index 7c05c89820..ba61659190 100755
--- a/scripts/travis/prepare-integration.sh
+++ b/scripts/travis/prepare-integration.sh
@@ -46,4 +46,5 @@ chmod 777 logs/*
 # o=other; X=only set execute bit if user execute bit is set (eg on dirs)
 chmod -R o+rwX ./virtualenv/
 # newer virtualenv versions are putting lock files under ~/.local
-chmod -R o+rwX ~${ST2_CI_USER}/.local/share/virtualenv
+# as this script runs with sudo, HOME is actually the CI user's home
+chmod -R o+rwX ${HOME}/.local/share/virtualenv

From fb13e08ee86257c3ecc24b872024fff7198da555 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 23:39:19 -0500
Subject: [PATCH 058/103] drop debug lines

---
 .github/workflows/ci.yaml | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 7ed392ca7b..35ec1dbc38 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -201,7 +201,6 @@ jobs:
         timeout-minutes: 2  # may die if rabbitmq fails to start
         run: |
           set -x
-          ls -ld /home/runner/.local/share/virtualenv/py_info /home/runner/.local/share/virtualenv /home/runner/.local/share /home/runner/.local /home/runner /home / || true
           # Since we're using a container, we can't use guest:guest
           # Integration tests hardcode paths to conf/st2.tests*.conf so we modify in place.
           sed -i -e "s/guest:guest/${RABBITMQ_USERNAME}:${RABBITMQ_PASSWORD}/" conf/st2.tests*.conf
@@ -240,11 +239,7 @@ jobs:
       - name: make
         # use: script -e -c to print colors
         run: |
-          sudo ls -l /tmp /home/runner/work/st2/st2/virtualenv/bin/python /home/runner/.local/share/virtualenv/py_info/* || true
-          script -e -c "make ${TASK}" || true
-          sudo ls -l /tmp /home/runner/work/st2/st2/virtualenv/bin/python /home/runner/.local/share/virtualenv/py_info/* || true
-          sudo ls -ld /home/runner/.local/share/virtualenv/py_info /home/runner/.local/share/virtualenv /home/runner/.local/share /home/runner/.local /home/runner /home /
-          exit 5
+          script -e -c "make ${TASK}"
       - name: Nightly
         # Run any additional nightly checks only as part of a nightly (cron) build
         if: "${{ env.IS_NIGHTLY_BUILD == 'yes' }}"

From e527bfb4519cfface6465af95efa18b0e494e505 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sat, 20 Mar 2021 23:47:46 -0500
Subject: [PATCH 059/103] go back to guest:guest with rabbitmq

---
 .github/workflows/ci.yaml                     | 19 ++++++++++---------
 scripts/github/rabbitmq.conf                  |  3 +++
 .../integration/test_rabbitmq_ssl_listener.py |  7 +------
 st2tests/st2tests/config.py                   | 17 -----------------
 4 files changed, 14 insertions(+), 32 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 35ec1dbc38..af29a9ba7f 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -67,9 +67,11 @@ jobs:
           # tell bitnami/rabbitmq to enable this by default
           RABBITMQ_PLUGINS: rabbitmq_management
           # Since we're using a container, we can't use guest:guest
-          RABBITMQ_SECURE_PASSWORD: "no"
-          RABBITMQ_USERNAME: st2
-          RABBITMQ_PASSWORD: st2
+          # RABBITMQ_SECURE_PASSWORD: "no"
+          # RABBITMQ_USERNAME: st2
+          # RABBITMQ_PASSWORD: st2
+          RABBITMQ_USERNAME: guest
+          RABBITMQ_PASSWORD: guest
 
         # These are strictly docker options, not entrypoint args (GHA restriction)
         options: >-
@@ -108,8 +110,8 @@ jobs:
       ST2_CI_USER: 'runner'
 
       # Since we're using a container, we can't use guest:guest
-      RABBITMQ_USERNAME: st2
-      RABBITMQ_PASSWORD: st2
+      # RABBITMQ_USERNAME: st2
+      # RABBITMQ_PASSWORD: st2
     steps:
       - name: Custom Environment Setup
         # built-in GitHub Actions environment variables
@@ -184,7 +186,8 @@ jobs:
           #     this user is the username of the user in GitHub actions,
           #     used for SSH, etc during integration tests (important)
           #   - uses a non-guest user/pass for RabbitMQ since we can't use guest outside of the container
-          cp conf/st2.dev.conf "${ST2_CONF}" ; sed -i -e "s/stanley/${ST2_CI_USER}/" -e "s/guest:guest/${RABBITMQ_USERNAME}:${RABBITMQ_PASSWORD}/" "${ST2_CONF}"
+          cp conf/st2.dev.conf "${ST2_CONF}" ; sed -i -e "s/stanley/${ST2_CI_USER}/" "${ST2_CONF}"
+          # cp conf/st2.dev.conf "${ST2_CONF}" ; sed -i -e "s/stanley/${ST2_CI_USER}/" -e "s/guest:guest/${RABBITMQ_USERNAME}:${RABBITMQ_PASSWORD}/" "${ST2_CONF}"
           scripts/travis/add-itest-user-key.sh
           sudo .circle/add-itest-user.sh
       - name: Permissions Workaround
@@ -203,9 +206,7 @@ jobs:
           set -x
           # Since we're using a container, we can't use guest:guest
           # Integration tests hardcode paths to conf/st2.tests*.conf so we modify in place.
-          sed -i -e "s/guest:guest/${RABBITMQ_USERNAME}:${RABBITMQ_PASSWORD}/" conf/st2.tests*.conf
-          # use a temp directory that is actually writeable (for pack installation tests)
-          #sed -i -e "s|base_path = .*|base_path = ${{ runner.temp }}|" conf/st2.tests*.conf
+          ###sed -i -e "s/guest:guest/${RABBITMQ_USERNAME}:${RABBITMQ_PASSWORD}/" conf/st2.tests*.conf
           # Use custom RabbitMQ config which enables SSL / TLS listener on port 5671 with test certs
           sudo cp scripts/github/rabbitmq.conf /home/runner/rabbitmq_conf/custom.conf
           # The code is checked out after the container is already up, so we don't mount them.
diff --git a/scripts/github/rabbitmq.conf b/scripts/github/rabbitmq.conf
index d7197e8d88..1c8032292b 100644
--- a/scripts/github/rabbitmq.conf
+++ b/scripts/github/rabbitmq.conf
@@ -6,3 +6,6 @@ ssl_options.certfile             = /bitnami/conf/ssl_certs/server/server_certifi
 ssl_options.keyfile              = /bitnami/conf/ssl_certs/server/private_key.pem
 ssl_options.verify               = verify_peer
 ssl_options.fail_if_no_peer_cert = false
+
+# this is "insecure" but it doesn't matter for CI, and it simplifies integration test machinery
+loopback_users = none
diff --git a/st2common/tests/integration/test_rabbitmq_ssl_listener.py b/st2common/tests/integration/test_rabbitmq_ssl_listener.py
index 2dd9209efc..f591a463fe 100644
--- a/st2common/tests/integration/test_rabbitmq_ssl_listener.py
+++ b/st2common/tests/integration/test_rabbitmq_ssl_listener.py
@@ -32,12 +32,7 @@
 CERTS_FIXTURES_PATH = os.path.join(get_fixtures_base_path(), "ssl_certs/")
 ST2_CI = os.environ.get("ST2_CI", "false").lower() == "true"
 
-# If CI has RabbitMQ running in a container, then we can't use guest.
-RMQ_USER = os.environ.get("RABBITMQ_USERNAME", "guest")
-RMQ_PASS = os.environ.get("RABBITMQ_PASSWORD", "guest")
-RMQ_HOST = os.environ.get("RABBITMQ_HOST", "127.0.0.1")
-RMQ_PORT = os.environ.get("RABBITMQ_PORT", "5671")
-RMQ_URL = f"amqp://{RMQ_USER}:{RMQ_PASS}@{RMQ_HOST}:{RMQ_PORT}/"
+RMQ_URL = "amqp://guest:guest@127.0.0.1:5671/"
 
 NON_SSL_LISTENER_PORT = 5672
 SSL_LISTENER_PORT = 5671
diff --git a/st2tests/st2tests/config.py b/st2tests/st2tests/config.py
index e7e84f47d5..b140357839 100644
--- a/st2tests/st2tests/config.py
+++ b/st2tests/st2tests/config.py
@@ -57,7 +57,6 @@ def _override_config_opts(coordinator_noop=False):
     _override_db_opts()
     _override_common_opts()
     _override_api_opts()
-    _override_messaging_opts()
     _override_keyvalue_opts()
     _override_scheduler_opts()
     _override_workflow_engine_opts()
@@ -105,22 +104,6 @@ def _override_api_opts():
     )
 
 
-def _override_messaging_opts():
-    # If CI has RabbitMQ running in a container, then we can't use guest.
-    rmq_user = os.environ.get("RABBITMQ_USERNAME", "guest")
-    rmq_pass = os.environ.get("RABBITMQ_PASSWORD", "guest")
-    rmq_host = os.environ.get("RABBITMQ_HOST", "127.0.0.1")
-    rmq_port = os.environ.get("RABBITMQ_PORT", "5672")
-    override_rmq_url = f"amqp://{rmq_user}:{rmq_pass}@{rmq_host}:{rmq_port}//"
-
-    if override_rmq_url != "amqp://guest:guest@127.0.0.1:5672//":
-        CONF.set_override(
-            name="url",
-            override=override_rmq_url,
-            group="messaging",
-        )
-
-
 def _override_keyvalue_opts():
     current_file_path = os.path.dirname(__file__)
     rel_st2_base_path = os.path.join(current_file_path, "../..")

From 9100e77b6eb4ea97f51b2fb74463de9915aef92b Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 21 Mar 2021 09:19:11 -0500
Subject: [PATCH 060/103] debug

---
 .../test_python_action_process_wrapper.py     |  6 ++++++
 .../integration/test_pythonrunner_behavior.py | 19 +++++++++++++++++++
 2 files changed, 25 insertions(+)

diff --git a/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py b/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py
index e1d39361a2..52816aa1a8 100644
--- a/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py
+++ b/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py
@@ -39,11 +39,14 @@
 import unittest2
 from distutils.spawn import find_executable
 
+from st2common import log as logging
 from st2common.util.shell import run_command
 from six.moves import range
 
 __all__ = ["PythonRunnerActionWrapperProcessTestCase"]
 
+LOG = logging.getLogger(__name__)
+
 # Maximum limit for the process wrapper script execution time (in seconds)
 WRAPPER_PROCESS_RUN_TIME_UPPER_LIMIT = 0.31
 
@@ -142,7 +145,10 @@ def test_stdin_params_timeout_no_stdin_data_provided(self):
             "python %s --pack=dummy --file-path=%s --config='%s' "
             "--stdin-parameters" % (WRAPPER_SCRIPT_PATH, file_path, config)
         )
+        LOG.debug(self.command_string)
         exit_code, stdout, stderr = run_command(command_string, shell=True)
+        LOG.debug(f"stdout=\n{stdout}")
+        LOG.debug(f"stderr=\n{stderr}")
 
         expected_msg = (
             "ValueError: No input received and timed out while waiting for parameters "
diff --git a/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py b/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
index a6d300be23..eeb5782167 100644
--- a/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
+++ b/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
@@ -18,12 +18,14 @@
 """
 
 import os
+import pwd
 import mock
 import tempfile
 
 from oslo_config import cfg
 
 from python_runner import python_runner
+from st2common import log as logging
 from st2common.util.virtualenvs import setup_pack_virtualenv
 from st2tests import config
 from st2tests.base import CleanFilesTestCase
@@ -32,6 +34,8 @@
 
 __all__ = ["PythonRunnerBehaviorTestCase"]
 
+LOG = logging.getLogger(__name__)
+
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
 WRAPPER_SCRIPT_PATH = os.path.join(
     BASE_DIR, "../../../python_runner/python_runner/python_action_wrapper.py"
@@ -49,6 +53,13 @@ def setUp(self):
 
         self.base_path = dir_path
         self.virtualenvs_path = os.path.join(self.base_path, "virtualenvs/")
+        ls = os.stat(self.virtualenvs_path)
+        LOG.debug(
+            f"{self.virtualenvs_path} exists={os.path.exists(self.virtualenvs_path)} "
+            f"perms={oct(ls.st_mode)[-3:]} owner_uid={ls.st_uid} owner_gid={ls.st_gid} "
+            f"owner={pwd.getpwuid(ls.st_uid)[0]} "
+            f"uid={os.getuid()} user={pwd.getpwuid(os.getuid())[0]}"
+        )
 
         # Make sure dir is deleted on tearDown
         self.to_delete_directories.append(self.base_path)
@@ -68,6 +79,14 @@ def test_priority_of_loading_library_after_setup_pack_virtualenv(self):
         # requirements.txt wihch only writes 'six' module.
         setup_pack_virtualenv(pack_name=pack_name)
         self.assertTrue(os.path.exists(os.path.join(self.virtualenvs_path, pack_name)))
+        ls = os.stat(os.path.join(self.virtualenvs_path, pack_name))
+        LOG.debug(
+            f"{os.path.join(self.virtualenvs_path, pack_name)} "
+            f"exists={os.path.exists(os.path.join(self.virtualenvs_path, pack_name))} "
+            f"perms={oct(ls.st_mode)[-3:]} owner_uid={ls.st_uid} owner_gid={ls.st_gid} "
+            f"owner={pwd.getpwuid(ls.st_uid)[0]} "
+            f"uid={os.getuid()} user={pwd.getpwuid(os.getuid())[0]}"
+        )
 
         # This test suite expects that loaded six module is located under the virtualenv library,
         # because 'six' is written in the requirements.txt of 'test_library_dependencies' pack.

From 219090b06f0fdea18d1dfcfb8ffc9a5c6de927f8 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 21 Mar 2021 09:21:16 -0500
Subject: [PATCH 061/103] finish reverting rmq guest:guest stuff

---
 .../integration/test_rabbitmq_ssl_listener.py | 36 +++++++++++++------
 1 file changed, 25 insertions(+), 11 deletions(-)

diff --git a/st2common/tests/integration/test_rabbitmq_ssl_listener.py b/st2common/tests/integration/test_rabbitmq_ssl_listener.py
index f591a463fe..224a50d4a4 100644
--- a/st2common/tests/integration/test_rabbitmq_ssl_listener.py
+++ b/st2common/tests/integration/test_rabbitmq_ssl_listener.py
@@ -32,8 +32,6 @@
 CERTS_FIXTURES_PATH = os.path.join(get_fixtures_base_path(), "ssl_certs/")
 ST2_CI = os.environ.get("ST2_CI", "false").lower() == "true"
 
-RMQ_URL = "amqp://guest:guest@127.0.0.1:5671/"
-
 NON_SSL_LISTENER_PORT = 5672
 SSL_LISTENER_PORT = 5671
 
@@ -54,7 +52,9 @@ def setUp(self):
         cfg.CONF.set_override(name="ssl_cert_reqs", override=None, group="messaging")
 
     def test_non_ssl_connection_on_ssl_listener_port_failure(self):
-        connection = transport_utils.get_connection(urls=RMQ_URL)
+        connection = transport_utils.get_connection(
+            urls="amqp://guest:guest@127.0.0.1:5671/"
+        )
 
         expected_msg_1 = (
             "[Errno 104]"  # followed by: ' Connection reset by peer' or ' ECONNRESET'
@@ -80,7 +80,7 @@ def test_non_ssl_connection_on_ssl_listener_port_failure(self):
 
     def test_ssl_connection_on_ssl_listener_success(self):
         # Using query param notation
-        urls = f"{RMQ_URL}?ssl=true"
+        urls = "amqp://guest:guest@127.0.0.1:5671/?ssl=true"
         connection = transport_utils.get_connection(urls=urls)
 
         try:
@@ -93,7 +93,9 @@ def test_ssl_connection_on_ssl_listener_success(self):
         # Using messaging.ssl config option
         cfg.CONF.set_override(name="ssl", override=True, group="messaging")
 
-        connection = transport_utils.get_connection(urls=RMQ_URL)
+        connection = transport_utils.get_connection(
+            urls="amqp://guest:guest@127.0.0.1:5671/"
+        )
 
         try:
             self.assertTrue(connection.connect())
@@ -115,7 +117,9 @@ def test_ssl_connection_ca_certs_provided(self):
             name="ssl_cert_reqs", override="required", group="messaging"
         )
 
-        connection = transport_utils.get_connection(urls=RMQ_URL)
+        connection = transport_utils.get_connection(
+            urls="amqp://guest:guest@127.0.0.1:5671/"
+        )
 
         try:
             self.assertTrue(connection.connect())
@@ -135,7 +139,9 @@ def test_ssl_connection_ca_certs_provided(self):
             name="ssl_ca_certs", override=ca_cert_path, group="messaging"
         )
 
-        connection = transport_utils.get_connection(urls=RMQ_URL)
+        connection = transport_utils.get_connection(
+            urls="amqp://guest:guest@127.0.0.1:5671/"
+        )
 
         expected_msg = r"\[SSL: CERTIFICATE_VERIFY_FAILED\] certificate verify failed"
         self.assertRaisesRegexp(ssl.SSLError, expected_msg, connection.connect)
@@ -150,7 +156,9 @@ def test_ssl_connection_ca_certs_provided(self):
             name="ssl_ca_certs", override=ca_cert_path, group="messaging"
         )
 
-        connection = transport_utils.get_connection(urls=RMQ_URL)
+        connection = transport_utils.get_connection(
+            urls="amqp://guest:guest@127.0.0.1:5671/"
+        )
 
         expected_msg = r"\[SSL: CERTIFICATE_VERIFY_FAILED\] certificate verify failed"
         self.assertRaisesRegexp(ssl.SSLError, expected_msg, connection.connect)
@@ -164,7 +172,9 @@ def test_ssl_connection_ca_certs_provided(self):
             name="ssl_ca_certs", override=ca_cert_path, group="messaging"
         )
 
-        connection = transport_utils.get_connection(urls=RMQ_URL)
+        connection = transport_utils.get_connection(
+            urls="amqp://guest:guest@127.0.0.1:5671/"
+        )
 
         try:
             self.assertTrue(connection.connect())
@@ -194,7 +204,9 @@ def test_ssl_connect_client_side_cert_authentication(self):
             name="ssl_ca_certs", override=ca_cert_path, group="messaging"
         )
 
-        connection = transport_utils.get_connection(urls=RMQ_URL)
+        connection = transport_utils.get_connection(
+            urls="amqp://guest:guest@127.0.0.1:5671/"
+        )
 
         try:
             self.assertTrue(connection.connect())
@@ -223,7 +235,9 @@ def test_ssl_connect_client_side_cert_authentication(self):
             name="ssl_ca_certs", override=ca_cert_path, group="messaging"
         )
 
-        connection = transport_utils.get_connection(urls=RMQ_URL)
+        connection = transport_utils.get_connection(
+            urls="amqp://guest:guest@127.0.0.1:5671/"
+        )
 
         expected_msg = r"\[X509: KEY_VALUES_MISMATCH\] key values mismatch"
         self.assertRaisesRegexp(ssl.SSLError, expected_msg, connection.connect)

From 2bbb65c72edf3bcf824677e0c1ef93f59b9d4e5a Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 21 Mar 2021 10:06:20 -0500
Subject: [PATCH 062/103] debug

---
 .../integration/test_python_action_process_wrapper.py |  2 +-
 .../tests/integration/test_pythonrunner_behavior.py   | 11 +++++++----
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py b/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py
index 52816aa1a8..101c365fec 100644
--- a/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py
+++ b/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py
@@ -145,7 +145,7 @@ def test_stdin_params_timeout_no_stdin_data_provided(self):
             "python %s --pack=dummy --file-path=%s --config='%s' "
             "--stdin-parameters" % (WRAPPER_SCRIPT_PATH, file_path, config)
         )
-        LOG.debug(self.command_string)
+        LOG.debug(command_string)
         exit_code, stdout, stderr = run_command(command_string, shell=True)
         LOG.debug(f"stdout=\n{stdout}")
         LOG.debug(f"stderr=\n{stderr}")
diff --git a/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py b/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
index eeb5782167..39a77ab959 100644
--- a/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
+++ b/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
@@ -53,12 +53,8 @@ def setUp(self):
 
         self.base_path = dir_path
         self.virtualenvs_path = os.path.join(self.base_path, "virtualenvs/")
-        ls = os.stat(self.virtualenvs_path)
         LOG.debug(
             f"{self.virtualenvs_path} exists={os.path.exists(self.virtualenvs_path)} "
-            f"perms={oct(ls.st_mode)[-3:]} owner_uid={ls.st_uid} owner_gid={ls.st_gid} "
-            f"owner={pwd.getpwuid(ls.st_uid)[0]} "
-            f"uid={os.getuid()} user={pwd.getpwuid(os.getuid())[0]}"
         )
 
         # Make sure dir is deleted on tearDown
@@ -79,6 +75,13 @@ def test_priority_of_loading_library_after_setup_pack_virtualenv(self):
         # requirements.txt wihch only writes 'six' module.
         setup_pack_virtualenv(pack_name=pack_name)
         self.assertTrue(os.path.exists(os.path.join(self.virtualenvs_path, pack_name)))
+        ls_v = os.stat(self.virtualenvs_path)
+        LOG.debug(
+            f"{self.virtualenvs_path} exists={os.path.exists(self.virtualenvs_path)} "
+            f"perms={oct(ls_v.st_mode)[-3:]} owner_uid={ls_v.st_uid} owner_gid={ls_v.st_gid} "
+            f"owner={pwd.getpwuid(ls_v.st_uid)[0]} "
+            f"uid={os.getuid()} user={pwd.getpwuid(os.getuid())[0]}"
+        )
         ls = os.stat(os.path.join(self.virtualenvs_path, pack_name))
         LOG.debug(
             f"{os.path.join(self.virtualenvs_path, pack_name)} "

From aecf27f715980b0228508605ffe27f833d89836e Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 21 Mar 2021 12:21:32 -0500
Subject: [PATCH 063/103] test logging

---
 .../tests/integration/test_python_action_process_wrapper.py     | 2 +-
 .../tests/integration/test_pythonrunner_behavior.py             | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py b/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py
index 101c365fec..16c98fcd37 100644
--- a/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py
+++ b/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py
@@ -35,11 +35,11 @@
 from __future__ import absolute_import
 import os
 import json
+import logging
 
 import unittest2
 from distutils.spawn import find_executable
 
-from st2common import log as logging
 from st2common.util.shell import run_command
 from six.moves import range
 
diff --git a/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py b/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
index 39a77ab959..cb932c9147 100644
--- a/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
+++ b/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
@@ -17,6 +17,7 @@
 This tests whether an action which is python-script behaves as we expect.
 """
 
+import logging
 import os
 import pwd
 import mock
@@ -25,7 +26,6 @@
 from oslo_config import cfg
 
 from python_runner import python_runner
-from st2common import log as logging
 from st2common.util.virtualenvs import setup_pack_virtualenv
 from st2tests import config
 from st2tests.base import CleanFilesTestCase

From e10d98dea3126e3c55ddabd12706f53076741361 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 21 Mar 2021 12:50:03 -0500
Subject: [PATCH 064/103] use print instead of logging

---
 .../integration/test_python_action_process_wrapper.py    | 9 +++------
 .../tests/integration/test_pythonrunner_behavior.py      | 9 +++------
 2 files changed, 6 insertions(+), 12 deletions(-)

diff --git a/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py b/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py
index 16c98fcd37..7be43c8e88 100644
--- a/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py
+++ b/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py
@@ -35,7 +35,6 @@
 from __future__ import absolute_import
 import os
 import json
-import logging
 
 import unittest2
 from distutils.spawn import find_executable
@@ -45,8 +44,6 @@
 
 __all__ = ["PythonRunnerActionWrapperProcessTestCase"]
 
-LOG = logging.getLogger(__name__)
-
 # Maximum limit for the process wrapper script execution time (in seconds)
 WRAPPER_PROCESS_RUN_TIME_UPPER_LIMIT = 0.31
 
@@ -145,10 +142,10 @@ def test_stdin_params_timeout_no_stdin_data_provided(self):
             "python %s --pack=dummy --file-path=%s --config='%s' "
             "--stdin-parameters" % (WRAPPER_SCRIPT_PATH, file_path, config)
         )
-        LOG.debug(command_string)
+        print(command_string)
         exit_code, stdout, stderr = run_command(command_string, shell=True)
-        LOG.debug(f"stdout=\n{stdout}")
-        LOG.debug(f"stderr=\n{stderr}")
+        print(f"stdout=\n{stdout}")
+        print(f"stderr=\n{stderr}")
 
         expected_msg = (
             "ValueError: No input received and timed out while waiting for parameters "
diff --git a/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py b/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
index cb932c9147..a4846c724a 100644
--- a/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
+++ b/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
@@ -17,7 +17,6 @@
 This tests whether an action which is python-script behaves as we expect.
 """
 
-import logging
 import os
 import pwd
 import mock
@@ -34,8 +33,6 @@
 
 __all__ = ["PythonRunnerBehaviorTestCase"]
 
-LOG = logging.getLogger(__name__)
-
 BASE_DIR = os.path.dirname(os.path.abspath(__file__))
 WRAPPER_SCRIPT_PATH = os.path.join(
     BASE_DIR, "../../../python_runner/python_runner/python_action_wrapper.py"
@@ -53,7 +50,7 @@ def setUp(self):
 
         self.base_path = dir_path
         self.virtualenvs_path = os.path.join(self.base_path, "virtualenvs/")
-        LOG.debug(
+        print(
             f"{self.virtualenvs_path} exists={os.path.exists(self.virtualenvs_path)} "
         )
 
@@ -76,14 +73,14 @@ def test_priority_of_loading_library_after_setup_pack_virtualenv(self):
         setup_pack_virtualenv(pack_name=pack_name)
         self.assertTrue(os.path.exists(os.path.join(self.virtualenvs_path, pack_name)))
         ls_v = os.stat(self.virtualenvs_path)
-        LOG.debug(
+        print(
             f"{self.virtualenvs_path} exists={os.path.exists(self.virtualenvs_path)} "
             f"perms={oct(ls_v.st_mode)[-3:]} owner_uid={ls_v.st_uid} owner_gid={ls_v.st_gid} "
             f"owner={pwd.getpwuid(ls_v.st_uid)[0]} "
             f"uid={os.getuid()} user={pwd.getpwuid(os.getuid())[0]}"
         )
         ls = os.stat(os.path.join(self.virtualenvs_path, pack_name))
-        LOG.debug(
+        print(
             f"{os.path.join(self.virtualenvs_path, pack_name)} "
             f"exists={os.path.exists(os.path.join(self.virtualenvs_path, pack_name))} "
             f"perms={oct(ls.st_mode)[-3:]} owner_uid={ls.st_uid} owner_gid={ls.st_gid} "

From f473b7de120c33201d3122a8c52af6f4f33f9f21 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 21 Mar 2021 13:16:29 -0500
Subject: [PATCH 065/103] debug

---
 .github/workflows/ci.yaml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index af29a9ba7f..406ea0818a 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -240,6 +240,12 @@ jobs:
       - name: make
         # use: script -e -c to print colors
         run: |
+          ls -ld /tmp
+          sudo ls -ld /tmp
+          ls -l /tmp
+          sudo ls -l /tmp
+          mount
+          sudo mount
           script -e -c "make ${TASK}"
       - name: Nightly
         # Run any additional nightly checks only as part of a nightly (cron) build

From 1d97d293e0ab473270919fae1c757fd3d65b4bc5 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 21 Mar 2021 15:12:52 -0500
Subject: [PATCH 066/103] debug

---
 Makefile                                                     | 4 ++--
 .../tests/integration/test_pythonrunner_behavior.py          | 5 +++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 99707ee24f..1549d502ef 100644
--- a/Makefile
+++ b/Makefile
@@ -63,14 +63,14 @@ ifndef PYLINT_CONCURRENCY
 	PYLINT_CONCURRENCY := 1
 endif
 
-NOSE_OPTS := --rednose --immediate --with-parallel --nocapture
+NOSE_OPTS := --rednose --immediate --with-parallel --nocapture --nologcapture
 
 ifndef NOSE_TIME
 	NOSE_TIME := yes
 endif
 
 ifeq ($(NOSE_TIME),yes)
-	NOSE_OPTS := --rednose --immediate --with-parallel --with-timer --nocapture
+	NOSE_OPTS := --rednose --immediate --with-parallel --with-timer --nocapture --nologcapture
 	NOSE_WITH_TIMER := 1
 endif
 
diff --git a/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py b/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
index a4846c724a..738f4fa88b 100644
--- a/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
+++ b/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
@@ -74,14 +74,14 @@ def test_priority_of_loading_library_after_setup_pack_virtualenv(self):
         self.assertTrue(os.path.exists(os.path.join(self.virtualenvs_path, pack_name)))
         ls_v = os.stat(self.virtualenvs_path)
         print(
-            f"{self.virtualenvs_path} exists={os.path.exists(self.virtualenvs_path)} "
+            f"\n{self.virtualenvs_path} exists={os.path.exists(self.virtualenvs_path)} "
             f"perms={oct(ls_v.st_mode)[-3:]} owner_uid={ls_v.st_uid} owner_gid={ls_v.st_gid} "
             f"owner={pwd.getpwuid(ls_v.st_uid)[0]} "
             f"uid={os.getuid()} user={pwd.getpwuid(os.getuid())[0]}"
         )
         ls = os.stat(os.path.join(self.virtualenvs_path, pack_name))
         print(
-            f"{os.path.join(self.virtualenvs_path, pack_name)} "
+            f"\n{os.path.join(self.virtualenvs_path, pack_name)} "
             f"exists={os.path.exists(os.path.join(self.virtualenvs_path, pack_name))} "
             f"perms={oct(ls.st_mode)[-3:]} owner_uid={ls.st_uid} owner_gid={ls.st_gid} "
             f"owner={pwd.getpwuid(ls.st_uid)[0]} "
@@ -93,6 +93,7 @@ def test_priority_of_loading_library_after_setup_pack_virtualenv(self):
         (_, output, _) = self._run_action(
             pack_name, "get_library_path.py", {"module": "six"}
         )
+        print(f"\noutput={output}")
         self.assertEqual(output["result"].find(self.virtualenvs_path), 0)
 
         # Conversely, this expects that 'mock' module file-path is not under sandbox library,

From 3a2fcca6b2e9b61575841c933261555eeff3ff95 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 21 Mar 2021 15:39:40 -0500
Subject: [PATCH 067/103] debug

---
 Makefile                                                  | 6 +++---
 .../tests/integration/test_pythonrunner_behavior.py       | 8 +++++---
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/Makefile b/Makefile
index 1549d502ef..c1fed617eb 100644
--- a/Makefile
+++ b/Makefile
@@ -63,14 +63,14 @@ ifndef PYLINT_CONCURRENCY
 	PYLINT_CONCURRENCY := 1
 endif
 
-NOSE_OPTS := --rednose --immediate --with-parallel --nocapture --nologcapture
+NOSE_OPTS := --rednose --immediate --with-parallel --nocapture
 
 ifndef NOSE_TIME
 	NOSE_TIME := yes
 endif
 
 ifeq ($(NOSE_TIME),yes)
-	NOSE_OPTS := --rednose --immediate --with-parallel --with-timer --nocapture --nologcapture
+	NOSE_OPTS := --rednose --immediate --with-parallel --with-timer --nocapture
 	NOSE_WITH_TIMER := 1
 endif
 
@@ -841,7 +841,7 @@ endif
 		echo "==========================================================="; \
 		echo "Running integration tests in" $$component; \
 		echo "-----------------------------------------------------------"; \
-		. $(VIRTUALENV_DIR)/bin/activate; \
+		echo skipping . $(VIRTUALENV_DIR)/bin/activate; \
 		    COVERAGE_FILE=.coverage.integration.$$(echo $$component | tr '/' '.') \
 		    nosetests $(NOSE_OPTS) -s -v $(NOSE_COVERAGE_FLAGS) \
 		    $(NOSE_COVERAGE_PACKAGES) \
diff --git a/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py b/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
index 738f4fa88b..1893378ce8 100644
--- a/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
+++ b/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
@@ -51,7 +51,7 @@ def setUp(self):
         self.base_path = dir_path
         self.virtualenvs_path = os.path.join(self.base_path, "virtualenvs/")
         print(
-            f"{self.virtualenvs_path} exists={os.path.exists(self.virtualenvs_path)} "
+            f"\n{self.virtualenvs_path} exists={os.path.exists(self.virtualenvs_path)} "
         )
 
         # Make sure dir is deleted on tearDown
@@ -74,19 +74,21 @@ def test_priority_of_loading_library_after_setup_pack_virtualenv(self):
         self.assertTrue(os.path.exists(os.path.join(self.virtualenvs_path, pack_name)))
         ls_v = os.stat(self.virtualenvs_path)
         print(
-            f"\n{self.virtualenvs_path} exists={os.path.exists(self.virtualenvs_path)} "
+            f"{self.virtualenvs_path} exists={os.path.exists(self.virtualenvs_path)} "
             f"perms={oct(ls_v.st_mode)[-3:]} owner_uid={ls_v.st_uid} owner_gid={ls_v.st_gid} "
             f"owner={pwd.getpwuid(ls_v.st_uid)[0]} "
             f"uid={os.getuid()} user={pwd.getpwuid(os.getuid())[0]}"
         )
         ls = os.stat(os.path.join(self.virtualenvs_path, pack_name))
         print(
-            f"\n{os.path.join(self.virtualenvs_path, pack_name)} "
+            f"{os.path.join(self.virtualenvs_path, pack_name)} "
             f"exists={os.path.exists(os.path.join(self.virtualenvs_path, pack_name))} "
             f"perms={oct(ls.st_mode)[-3:]} owner_uid={ls.st_uid} owner_gid={ls.st_gid} "
             f"owner={pwd.getpwuid(ls.st_uid)[0]} "
             f"uid={os.getuid()} user={pwd.getpwuid(os.getuid())[0]}"
         )
+        p = os.path.join(self.virtualenvs_path, pack_name, 'lib', 'python3.6', 'site-packages', 'six.py')
+        print(f"{p} exists={os.path.exists(p)}")
 
         # This test suite expects that loaded six module is located under the virtualenv library,
         # because 'six' is written in the requirements.txt of 'test_library_dependencies' pack.

From 8595f87e8d11b4e445453f0ddc536ac0a52d7e28 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 21 Mar 2021 15:52:55 -0500
Subject: [PATCH 068/103] debug

---
 Makefile                                                  | 8 ++++----
 .../tests/integration/test_pythonrunner_behavior.py       | 2 +-
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/Makefile b/Makefile
index c1fed617eb..f1c7ecf97e 100644
--- a/Makefile
+++ b/Makefile
@@ -842,10 +842,10 @@ endif
 		echo "Running integration tests in" $$component; \
 		echo "-----------------------------------------------------------"; \
 		echo skipping . $(VIRTUALENV_DIR)/bin/activate; \
-		    COVERAGE_FILE=.coverage.integration.$$(echo $$component | tr '/' '.') \
-		    nosetests $(NOSE_OPTS) -s -v $(NOSE_COVERAGE_FLAGS) \
-		    $(NOSE_COVERAGE_PACKAGES) \
-		    $$component/tests/integration || exit 1; \
+		echo     COVERAGE_FILE=.coverage.integration.$$(echo $$component | tr '/' '.') \
+		echo     nosetests $(NOSE_OPTS) -s -v $(NOSE_COVERAGE_FLAGS) \
+		echo     $(NOSE_COVERAGE_PACKAGES) \
+		echo     $$component/tests/integration || exit 1; \
 		echo "-----------------------------------------------------------"; \
 		echo "Done integration running tests in" $$component; \
 		echo "==========================================================="; \
diff --git a/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py b/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
index 1893378ce8..b5fc7adc7d 100644
--- a/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
+++ b/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
@@ -88,7 +88,7 @@ def test_priority_of_loading_library_after_setup_pack_virtualenv(self):
             f"uid={os.getuid()} user={pwd.getpwuid(os.getuid())[0]}"
         )
         p = os.path.join(self.virtualenvs_path, pack_name, 'lib', 'python3.6', 'site-packages', 'six.py')
-        print(f"{p} exists={os.path.exists(p)}")
+        print(f"\n{p} exists={os.path.exists(p)}")
 
         # This test suite expects that loaded six module is located under the virtualenv library,
         # because 'six' is written in the requirements.txt of 'test_library_dependencies' pack.

From 5ccff08882f16601b53cdbd397445f37b2c7bb6f Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 21 Mar 2021 16:25:23 -0500
Subject: [PATCH 069/103] debug

---
 .../tests/integration/test_pythonrunner_behavior.py   | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py b/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
index b5fc7adc7d..67c108cda1 100644
--- a/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
+++ b/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
@@ -41,6 +41,9 @@
 
 
 class PythonRunnerBehaviorTestCase(CleanFilesTestCase, CleanDbTestCase):
+
+    DISPLAY_LOG_MESSAGES = True
+
     def setUp(self):
         super(PythonRunnerBehaviorTestCase, self).setUp()
         config.parse_args()
@@ -87,8 +90,12 @@ def test_priority_of_loading_library_after_setup_pack_virtualenv(self):
             f"owner={pwd.getpwuid(ls.st_uid)[0]} "
             f"uid={os.getuid()} user={pwd.getpwuid(os.getuid())[0]}"
         )
-        p = os.path.join(self.virtualenvs_path, pack_name, 'lib', 'python3.6', 'site-packages', 'six.py')
-        print(f"\n{p} exists={os.path.exists(p)}")
+        lpy = os.path.join(self.virtualenvs_path, pack_name, 'lib', 'python3.6')
+        print(f"\n{lpy} exists={os.path.exists(lpy)}")
+        sp = os.path.join(self.virtualenvs_path, pack_name, 'lib', 'python3.6', 'site-packages')
+        print(f"{sp} exists={os.path.exists(sp)}")
+        p_six = os.path.join(self.virtualenvs_path, pack_name, 'lib', 'python3.6', 'site-packages', 'six.py')
+        print(f"{p_six} exists={os.path.exists(p_six)}")
 
         # This test suite expects that loaded six module is located under the virtualenv library,
         # because 'six' is written in the requirements.txt of 'test_library_dependencies' pack.

From a35833c8c057fafcff199fa7acbf6ae1521b0cdc Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 21 Mar 2021 16:49:06 -0500
Subject: [PATCH 070/103] debug

---
 Makefile                                             |  2 +-
 .../tests/integration/test_pythonrunner_behavior.py  | 12 ------------
 2 files changed, 1 insertion(+), 13 deletions(-)

diff --git a/Makefile b/Makefile
index f1c7ecf97e..c8c51e1694 100644
--- a/Makefile
+++ b/Makefile
@@ -860,7 +860,7 @@ endif
 		echo "==========================================================="; \
 		. $(VIRTUALENV_DIR)/bin/activate; \
 		    COVERAGE_FILE=.coverage.integration.$$(echo $$component | tr '/' '.') \
-			nosetests $(NOSE_OPTS) -s -v \
+			nosetests $(NOSE_OPTS) -s -v --logging-level=DEBUG \
 			$(NOSE_COVERAGE_FLAGS) $(NOSE_COVERAGE_PACKAGES) $$component/tests/integration || exit 1; \
 	done
 	@echo
diff --git a/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py b/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
index 67c108cda1..4d8f00c664 100644
--- a/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
+++ b/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
@@ -53,9 +53,6 @@ def setUp(self):
 
         self.base_path = dir_path
         self.virtualenvs_path = os.path.join(self.base_path, "virtualenvs/")
-        print(
-            f"\n{self.virtualenvs_path} exists={os.path.exists(self.virtualenvs_path)} "
-        )
 
         # Make sure dir is deleted on tearDown
         self.to_delete_directories.append(self.base_path)
@@ -75,13 +72,6 @@ def test_priority_of_loading_library_after_setup_pack_virtualenv(self):
         # requirements.txt wihch only writes 'six' module.
         setup_pack_virtualenv(pack_name=pack_name)
         self.assertTrue(os.path.exists(os.path.join(self.virtualenvs_path, pack_name)))
-        ls_v = os.stat(self.virtualenvs_path)
-        print(
-            f"{self.virtualenvs_path} exists={os.path.exists(self.virtualenvs_path)} "
-            f"perms={oct(ls_v.st_mode)[-3:]} owner_uid={ls_v.st_uid} owner_gid={ls_v.st_gid} "
-            f"owner={pwd.getpwuid(ls_v.st_uid)[0]} "
-            f"uid={os.getuid()} user={pwd.getpwuid(os.getuid())[0]}"
-        )
         ls = os.stat(os.path.join(self.virtualenvs_path, pack_name))
         print(
             f"{os.path.join(self.virtualenvs_path, pack_name)} "
@@ -90,8 +80,6 @@ def test_priority_of_loading_library_after_setup_pack_virtualenv(self):
             f"owner={pwd.getpwuid(ls.st_uid)[0]} "
             f"uid={os.getuid()} user={pwd.getpwuid(os.getuid())[0]}"
         )
-        lpy = os.path.join(self.virtualenvs_path, pack_name, 'lib', 'python3.6')
-        print(f"\n{lpy} exists={os.path.exists(lpy)}")
         sp = os.path.join(self.virtualenvs_path, pack_name, 'lib', 'python3.6', 'site-packages')
         print(f"{sp} exists={os.path.exists(sp)}")
         p_six = os.path.join(self.virtualenvs_path, pack_name, 'lib', 'python3.6', 'site-packages', 'six.py')

From 7265f7acc0b3c9f0fbb9c8d13798af5fcc352128 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 21 Mar 2021 17:05:03 -0500
Subject: [PATCH 071/103] check virtualenv binary in use

---
 .github/workflows/ci.yaml     | 4 ++++
 st2common/st2common/config.py | 2 ++
 2 files changed, 6 insertions(+)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 406ea0818a..fce21f65a7 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -235,6 +235,10 @@ jobs:
           git --version
           pip --version
           pip list
+          which virtualenv
+          virtualenv --version
+          virtualenv/bin/virtualenv --version
+          ls -l virtualenv/bin
           # Print out various environment variables info
           make play
       - name: make
diff --git a/st2common/st2common/config.py b/st2common/st2common/config.py
index e7b30a9a7c..fe0fa5057f 100644
--- a/st2common/st2common/config.py
+++ b/st2common/st2common/config.py
@@ -393,6 +393,8 @@ def register_opts(ignore_errors=False):
 
     # Runner options
     default_python_bin_path = sys.executable
+    # FIXME: If the virtualenv uses a symlinked python, then this won't use
+    # what is installed in the virtualenv, it will use the system virtualenv.
     base_dir = os.path.dirname(os.path.realpath(default_python_bin_path))
     default_virtualenv_bin_path = os.path.join(base_dir, "virtualenv")
 

From f8fc994047fa3d168c269aac653b4f38e480c5fa Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 21 Mar 2021 17:16:33 -0500
Subject: [PATCH 072/103] debug

---
 st2common/st2common/util/virtualenvs.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/st2common/st2common/util/virtualenvs.py b/st2common/st2common/util/virtualenvs.py
index 62cfc99b52..e4ed26cbce 100644
--- a/st2common/st2common/util/virtualenvs.py
+++ b/st2common/st2common/util/virtualenvs.py
@@ -283,6 +283,7 @@ def install_requirements(
         " ".join(cmd),
     )
     exit_code, stdout, stderr = run_command(cmd=cmd, env=env)
+    logger.debug(f"\n==========\nstdout=\n{stdout}\n==========\nstderr=\n{stderr}")
 
     if exit_code != 0:
         stdout = to_ascii(stdout)

From 38aa0269de044fe8113aee794f5e632a9bfe7293 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 21 Mar 2021 17:59:40 -0500
Subject: [PATCH 073/103] Use virtualenv from st2 virtualenv by default

If a virtualenv was built with symlinks instead of copies of the python
binary, as happens by default on GHA integration tests, then we end up
using the system virtualenv binary instead of what we installed.

Currently GHA has the same version of virtualenv, but we build
virtualenvs with --system-site-packages, and we need it to use the st2
virtualenv as the "system" dir instead of the GHA provided one. So,
don't get the realpath of the python binary.
---
 st2common/st2common/config.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/st2common/st2common/config.py b/st2common/st2common/config.py
index fe0fa5057f..19d5ebb249 100644
--- a/st2common/st2common/config.py
+++ b/st2common/st2common/config.py
@@ -395,7 +395,8 @@ def register_opts(ignore_errors=False):
     default_python_bin_path = sys.executable
     # FIXME: If the virtualenv uses a symlinked python, then this won't use
     # what is installed in the virtualenv, it will use the system virtualenv.
-    base_dir = os.path.dirname(os.path.realpath(default_python_bin_path))
+    #base_dir = os.path.dirname(os.path.realpath(default_python_bin_path))
+    base_dir = os.path.dirname(default_python_bin_path)
     default_virtualenv_bin_path = os.path.join(base_dir, "virtualenv")
 
     action_runner_opts = [

From b9a5674afd3edec6d02ba13ca20ad69da4754190 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 21 Mar 2021 18:48:36 -0500
Subject: [PATCH 074/103] try in travis

---
 .travis.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.travis.yml b/.travis.yml
index ebb2e9bc94..82edb47a73 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -4,6 +4,7 @@ language: python
 
 branches:
   only:
+    - main
     - master
     - /^v[0-9]+\.[0-9]+$/
 

From 536e81e1ed343ea85e428a84f7296a0a49f251cf Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 21 Mar 2021 18:54:12 -0500
Subject: [PATCH 075/103] ugh

---
 .travis.yml                    | 10 ----------
 scripts/travis/rabbitmq.config |  6 +++---
 2 files changed, 3 insertions(+), 13 deletions(-)

diff --git a/.travis.yml b/.travis.yml
index 82edb47a73..31e183c7ea 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -109,16 +109,6 @@ before_script:
   - ls -l /etc/rabbitmq
   - ls -l /home/travis
   - ls -l /home/travis/build
-  - ls -l /home/travis/build/StackStorm
-  - ls -l /home/travis/build/StackStorm/st2
-  - ls -l /home/travis/build/StackStorm/st2/st2tests
-  - ls -l /home/travis/build/StackStorm/st2/st2tests/st2tests
-  - ls -l /home/travis/build/StackStorm/st2/st2tests/st2tests/fixtures
-  - ls -l /home/travis/build/StackStorm/st2/st2tests/st2tests/fixtures/ssl_certs
-  - ls -l /home/travis/build/StackStorm/st2/st2tests/st2tests/fixtures/ssl_certs/ca
-  - ls -l /home/travis/build/StackStorm/st2/st2tests/st2tests/fixtures/ssl_certs/server
-  - cat /home/travis/build/StackStorm/st2/st2tests/st2tests/fixtures/ssl_certs/ca/ca_certificate_bundle.pem
-  - cat /home/travis/build/StackStorm/st2/st2tests/st2tests/fixtures/ssl_certs/server/server_certificate.pem
   # Install rabbitmq_management RabbitMQ plugin
   - sudo service rabbitmq-server restart ; sleep 5
   - sudo rabbitmq-plugins enable rabbitmq_management
diff --git a/scripts/travis/rabbitmq.config b/scripts/travis/rabbitmq.config
index d23be26ec5..0b9cd71f8f 100644
--- a/scripts/travis/rabbitmq.config
+++ b/scripts/travis/rabbitmq.config
@@ -1,9 +1,9 @@
 [
   {rabbit, [
      {ssl_listeners, [5671]},
-     {ssl_options, [{cacertfile, "/home/travis/build/StackStorm/st2/st2tests/st2tests/fixtures/ssl_certs/ca/ca_certificate_bundle.pem"},
-                    {certfile,   "/home/travis/build/StackStorm/st2/st2tests/st2tests/fixtures/ssl_certs/server/server_certificate.pem"},
-                    {keyfile,    "/home/travis/build/StackStorm/st2/st2tests/st2tests/fixtures/ssl_certs/server/private_key.pem"},
+     {ssl_options, [{cacertfile, "/home/travis/build/st2sandbox/st2/st2tests/st2tests/fixtures/ssl_certs/ca/ca_certificate_bundle.pem"},
+                    {certfile,   "/home/travis/build/st2sandbox/st2/st2tests/st2tests/fixtures/ssl_certs/server/server_certificate.pem"},
+                    {keyfile,    "/home/travis/build/st2sandbox/st2/st2tests/st2tests/fixtures/ssl_certs/server/private_key.pem"},
                     {verify,     verify_peer},
                     {fail_if_no_peer_cert, false}]}
    ]}

From 8f9eb68205f6e287e83b6e156a91b27f9c736476 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 21 Mar 2021 19:12:56 -0500
Subject: [PATCH 076/103] DEEEEBBBBUUUUUUUGGGGG

---
 .../tests/integration/test_pythonrunner_behavior.py         | 6 +++---
 st2common/st2common/util/virtualenvs.py                     | 1 +
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py b/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
index 4d8f00c664..a5bf0f92ad 100644
--- a/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
+++ b/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
@@ -74,16 +74,16 @@ def test_priority_of_loading_library_after_setup_pack_virtualenv(self):
         self.assertTrue(os.path.exists(os.path.join(self.virtualenvs_path, pack_name)))
         ls = os.stat(os.path.join(self.virtualenvs_path, pack_name))
         print(
-            f"{os.path.join(self.virtualenvs_path, pack_name)} "
+            f"\n{os.path.join(self.virtualenvs_path, pack_name)} "
             f"exists={os.path.exists(os.path.join(self.virtualenvs_path, pack_name))} "
             f"perms={oct(ls.st_mode)[-3:]} owner_uid={ls.st_uid} owner_gid={ls.st_gid} "
             f"owner={pwd.getpwuid(ls.st_uid)[0]} "
             f"uid={os.getuid()} user={pwd.getpwuid(os.getuid())[0]}"
         )
         sp = os.path.join(self.virtualenvs_path, pack_name, 'lib', 'python3.6', 'site-packages')
-        print(f"{sp} exists={os.path.exists(sp)}")
+        print(f"\n{sp} exists={os.path.exists(sp)}")
         p_six = os.path.join(self.virtualenvs_path, pack_name, 'lib', 'python3.6', 'site-packages', 'six.py')
-        print(f"{p_six} exists={os.path.exists(p_six)}")
+        print(f"\n{p_six} exists={os.path.exists(p_six)}")
 
         # This test suite expects that loaded six module is located under the virtualenv library,
         # because 'six' is written in the requirements.txt of 'test_library_dependencies' pack.
diff --git a/st2common/st2common/util/virtualenvs.py b/st2common/st2common/util/virtualenvs.py
index e4ed26cbce..3b89385251 100644
--- a/st2common/st2common/util/virtualenvs.py
+++ b/st2common/st2common/util/virtualenvs.py
@@ -330,6 +330,7 @@ def install_requirement(virtualenv_path, requirement, proxy_config=None, logger=
         "Installing requirement %s with command %s.", requirement, " ".join(cmd)
     )
     exit_code, stdout, stderr = run_command(cmd=cmd, env=env)
+    logger.debug(f"\n==========\nstdout=\n{stdout}\n==========\nstderr=\n{stderr}")
 
     if exit_code != 0:
         raise Exception(

From 90909d823fb6d719f6cb8e2fbacbf7b5abf8d70e Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 21 Mar 2021 19:56:47 -0500
Subject: [PATCH 077/103] virtualenv version

---
 .github/workflows/ci.yaml | 5 +++++
 .travis.yml               | 1 +
 Makefile                  | 2 ++
 3 files changed, 8 insertions(+)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index fce21f65a7..c649dacbea 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -173,9 +173,14 @@ jobs:
           sudo apt-get -f -y install libldap2-dev libsasl2-dev libssl-dev libyaml-dev ldap-utils
       - name: Install virtualenv
         run: |
+          ls /opt/hostedtoolcache/Python/3.6.13/x64/lib/python3.6/site-packages
+          which virtualenv || true
+          virtualenv --version || true
           # Note: Use the verison of virtualenv pinned in fixed-requirements.txt so we
           #       only have to update it one place when we change the version
           pip install --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
+          which virtualenv
+          virtualenv --version
       - name: Install requirements
         run: |
           ./scripts/travis/install-requirements.sh
diff --git a/.travis.yml b/.travis.yml
index 31e183c7ea..4d172d55ed 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -87,6 +87,7 @@ cache:
 
 install:
   - ./scripts/travis/install-requirements.sh
+  - ls -l /home/travis/build/st2sandbox/st2/virtualenv/bin
   # prep a travis-specific dev conf file that uses travis instead of stanley
   - cp conf/st2.dev.conf "${ST2_CONF}" ; sed -i -e "s/stanley/${ST2_CI_USER}/" "${ST2_CONF}"
   - sudo scripts/travis/add-itest-user-key.sh
diff --git a/Makefile b/Makefile
index c8c51e1694..48599eed7b 100644
--- a/Makefile
+++ b/Makefile
@@ -674,6 +674,8 @@ virtualenv:
 	@echo
 	@echo "==================== virtualenv ===================="
 	@echo
+	which virtualenv
+	virtualenv --version
 	test -f $(VIRTUALENV_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) $(VIRTUALENV_DIR) --no-download
 
 	# Setup PYTHONPATH in bash activate script...

From 52a8e8aec9f979e9c01bed6d2543436287f90d51 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 21 Mar 2021 20:00:04 -0500
Subject: [PATCH 078/103] virtualenv version

---
 .travis.yml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.travis.yml b/.travis.yml
index 4d172d55ed..ff3ba9fd01 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -86,6 +86,11 @@ cache:
     #- .tox/
 
 install:
+  - which virtualenv || true
+  - virtualenv --version || true
+  - pip install --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
+  - which virtualenv
+  - virtualenv --version
   - ./scripts/travis/install-requirements.sh
   - ls -l /home/travis/build/st2sandbox/st2/virtualenv/bin
   # prep a travis-specific dev conf file that uses travis instead of stanley

From d17b0d03533428113aa131a5862351e9e26ba52a Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 21 Mar 2021 20:06:37 -0500
Subject: [PATCH 079/103] more ls

---
 .github/workflows/ci.yaml | 1 +
 .travis.yml               | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index c649dacbea..ea79b9b4f2 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -181,6 +181,7 @@ jobs:
           pip install --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
           which virtualenv
           virtualenv --version
+          ls /opt/hostedtoolcache/Python/3.6.13/x64/lib/python3.6/site-packages
       - name: Install requirements
         run: |
           ./scripts/travis/install-requirements.sh
diff --git a/.travis.yml b/.travis.yml
index ff3ba9fd01..73c1b89ebb 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -86,11 +86,13 @@ cache:
     #- .tox/
 
 install:
+  - ls /home/travis/virtualenv/python3.6.7/lib/python3.6/site-packages
   - which virtualenv || true
   - virtualenv --version || true
   - pip install --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
   - which virtualenv
   - virtualenv --version
+  - ls /home/travis/virtualenv/python3.6.7/lib/python3.6/site-packages
   - ./scripts/travis/install-requirements.sh
   - ls -l /home/travis/build/st2sandbox/st2/virtualenv/bin
   # prep a travis-specific dev conf file that uses travis instead of stanley

From c129abe3bbc333a3b17eed9a4cb31e63784c6273 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 21 Mar 2021 20:36:12 -0500
Subject: [PATCH 080/103] virtualenv config?

---
 .github/workflows/ci.yaml | 6 ++++--
 .travis.yml               | 6 ++++--
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index ea79b9b4f2..2d8ad43b2c 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -173,7 +173,7 @@ jobs:
           sudo apt-get -f -y install libldap2-dev libsasl2-dev libssl-dev libyaml-dev ldap-utils
       - name: Install virtualenv
         run: |
-          ls /opt/hostedtoolcache/Python/3.6.13/x64/lib/python3.6/site-packages
+          #ls /opt/hostedtoolcache/Python/3.6.13/x64/lib/python3.6/site-packages
           which virtualenv || true
           virtualenv --version || true
           # Note: Use the verison of virtualenv pinned in fixed-requirements.txt so we
@@ -181,7 +181,9 @@ jobs:
           pip install --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
           which virtualenv
           virtualenv --version
-          ls /opt/hostedtoolcache/Python/3.6.13/x64/lib/python3.6/site-packages
+          #ls /opt/hostedtoolcache/Python/3.6.13/x64/lib/python3.6/site-packages
+          virtualenv --help
+          exit 1
       - name: Install requirements
         run: |
           ./scripts/travis/install-requirements.sh
diff --git a/.travis.yml b/.travis.yml
index 73c1b89ebb..866381565b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -86,13 +86,15 @@ cache:
     #- .tox/
 
 install:
-  - ls /home/travis/virtualenv/python3.6.7/lib/python3.6/site-packages
+  #- ls /home/travis/virtualenv/python3.6.7/lib/python3.6/site-packages
   - which virtualenv || true
   - virtualenv --version || true
   - pip install --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
   - which virtualenv
   - virtualenv --version
-  - ls /home/travis/virtualenv/python3.6.7/lib/python3.6/site-packages
+  - virtualenv --help
+  - exit 1
+  #- ls /home/travis/virtualenv/python3.6.7/lib/python3.6/site-packages
   - ./scripts/travis/install-requirements.sh
   - ls -l /home/travis/build/st2sandbox/st2/virtualenv/bin
   # prep a travis-specific dev conf file that uses travis instead of stanley

From 5defee7eef7c4d26815d7608326ffa4ce8f55bc6 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 21 Mar 2021 20:46:47 -0500
Subject: [PATCH 081/103] investigate base python

---
 .github/workflows/ci.yaml | 3 ++-
 .travis.yml               | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 2d8ad43b2c..d352221d1d 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -182,7 +182,8 @@ jobs:
           which virtualenv
           virtualenv --version
           #ls /opt/hostedtoolcache/Python/3.6.13/x64/lib/python3.6/site-packages
-          virtualenv --help
+          #virtualenv --help # no config file
+          ls -l /opt/hostedtoolcache/Python/3.6*/x64/bin/python*
           exit 1
       - name: Install requirements
         run: |
diff --git a/.travis.yml b/.travis.yml
index 866381565b..34945889da 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -92,7 +92,8 @@ install:
   - pip install --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
   - which virtualenv
   - virtualenv --version
-  - virtualenv --help
+  #- virtualenv --help # no config file
+  - ls -l /home/travis/virtualenv/python3.6*/bin/python*
   - exit 1
   #- ls /home/travis/virtualenv/python3.6.7/lib/python3.6/site-packages
   - ./scripts/travis/install-requirements.sh

From db07be6d6227f481c68bbb3f1f24daa6602254c0 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 21 Mar 2021 20:56:10 -0500
Subject: [PATCH 082/103] investigate base python

---
 .github/workflows/ci.yaml | 1 +
 .travis.yml               | 1 +
 2 files changed, 2 insertions(+)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index d352221d1d..13dcdc60da 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -184,6 +184,7 @@ jobs:
           #ls /opt/hostedtoolcache/Python/3.6.13/x64/lib/python3.6/site-packages
           #virtualenv --help # no config file
           ls -l /opt/hostedtoolcache/Python/3.6*/x64/bin/python*
+          /opt/hostedtoolcache/Python/3.6.13/x64/bin/python -c 'import sysconfig; print(sysconfig.get_config_vars())'
           exit 1
       - name: Install requirements
         run: |
diff --git a/.travis.yml b/.travis.yml
index 34945889da..7b4f5183f0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -94,6 +94,7 @@ install:
   - virtualenv --version
   #- virtualenv --help # no config file
   - ls -l /home/travis/virtualenv/python3.6*/bin/python*
+  - /home/travis/virtualenv/python3.6.7/bin/python -c 'import sysconfig; print(sysconfig.get_config_vars())'
   - exit 1
   #- ls /home/travis/virtualenv/python3.6.7/lib/python3.6/site-packages
   - ./scripts/travis/install-requirements.sh

From 9cb1ef8c47aa1656bae6c8df07ebb3944bdd838b Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 21 Mar 2021 21:25:10 -0500
Subject: [PATCH 083/103] investigate travis pyvenv.cfg

---
 .github/workflows/ci.yaml | 2 ++
 .travis.yml               | 9 +++++++--
 2 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 13dcdc60da..501946fdc4 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -173,6 +173,8 @@ jobs:
           sudo apt-get -f -y install libldap2-dev libsasl2-dev libssl-dev libyaml-dev ldap-utils
       - name: Install virtualenv
         run: |
+          ls /opt/hostedtoolcache/Pythonn/3.6.13/x64
+          exit 1
           #ls /opt/hostedtoolcache/Python/3.6.13/x64/lib/python3.6/site-packages
           which virtualenv || true
           virtualenv --version || true
diff --git a/.travis.yml b/.travis.yml
index 7b4f5183f0..0054ca85b2 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -86,6 +86,10 @@ cache:
     #- .tox/
 
 install:
+  - ls -l /opt/python/3.6*/bin/python*
+  - ls /opt/python/3.6.7/ /home/travis/virtualenv/python3.6.7/
+  - cat /home/travis/virtualenv/python3.6.7/pyvenv.cfg
+  - exit 1
   #- ls /home/travis/virtualenv/python3.6.7/lib/python3.6/site-packages
   - which virtualenv || true
   - virtualenv --version || true
@@ -93,8 +97,9 @@ install:
   - which virtualenv
   - virtualenv --version
   #- virtualenv --help # no config file
-  - ls -l /home/travis/virtualenv/python3.6*/bin/python*
-  - /home/travis/virtualenv/python3.6.7/bin/python -c 'import sysconfig; print(sysconfig.get_config_vars())'
+  #- ls -l /home/travis/virtualenv/python3.6*/bin/python*
+  #- /home/travis/virtualenv/python3.6.7/bin/python -c 'import sysconfig; print(sysconfig.get_config_vars())'
+  #- /opt/python/3.6.7/bin/python -c 'import sysconfig; print(sysconfig.get_config_vars())'
   - exit 1
   #- ls /home/travis/virtualenv/python3.6.7/lib/python3.6/site-packages
   - ./scripts/travis/install-requirements.sh

From 92eb77e1a993b4918ac623fd913196cee23812ac Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 21 Mar 2021 21:29:51 -0500
Subject: [PATCH 084/103] investigate travis virtualenv

---
 .github/workflows/ci.yaml | 3 ++-
 .travis.yml               | 6 ++++--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 501946fdc4..29f43f4dd3 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -173,7 +173,8 @@ jobs:
           sudo apt-get -f -y install libldap2-dev libsasl2-dev libssl-dev libyaml-dev ldap-utils
       - name: Install virtualenv
         run: |
-          ls /opt/hostedtoolcache/Pythonn/3.6.13/x64
+          ls /opt/hostedtoolcache/Python/3.6.13/x64
+          ls /opt/hostedtoolcache/Python/3.6.13/x64/lib/python3.6
           exit 1
           #ls /opt/hostedtoolcache/Python/3.6.13/x64/lib/python3.6/site-packages
           which virtualenv || true
diff --git a/.travis.yml b/.travis.yml
index 0054ca85b2..e1662b7d5d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -87,8 +87,10 @@ cache:
 
 install:
   - ls -l /opt/python/3.6*/bin/python*
-  - ls /opt/python/3.6.7/ /home/travis/virtualenv/python3.6.7/
-  - cat /home/travis/virtualenv/python3.6.7/pyvenv.cfg
+  #- ls /opt/python/3.6.7/ /home/travis/virtualenv/python3.6.7/
+  - ls /opt/python/3.6.7/lib/python3.6/
+  - ls -l /home/travis/virtualenv/python3.6.7/lib/python3.6/
+  #- cat /home/travis/virtualenv/python3.6.7/pyvenv.cfg
   - exit 1
   #- ls /home/travis/virtualenv/python3.6.7/lib/python3.6/site-packages
   - which virtualenv || true

From 34b97ff0fe4d85053936f749e4b0f20954dc57ae Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 21 Mar 2021 21:36:38 -0500
Subject: [PATCH 085/103] try installing virtualenv in --user on github

---
 .github/workflows/ci.yaml | 12 ++----------
 .travis.yml               | 13 -------------
 2 files changed, 2 insertions(+), 23 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 29f43f4dd3..3f2c51fe9e 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -173,22 +173,14 @@ jobs:
           sudo apt-get -f -y install libldap2-dev libsasl2-dev libssl-dev libyaml-dev ldap-utils
       - name: Install virtualenv
         run: |
-          ls /opt/hostedtoolcache/Python/3.6.13/x64
-          ls /opt/hostedtoolcache/Python/3.6.13/x64/lib/python3.6
-          exit 1
-          #ls /opt/hostedtoolcache/Python/3.6.13/x64/lib/python3.6/site-packages
           which virtualenv || true
           virtualenv --version || true
           # Note: Use the verison of virtualenv pinned in fixed-requirements.txt so we
           #       only have to update it one place when we change the version
-          pip install --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
+          # Note: Use --user to avoid polluting system site-packages
+          pip install --user --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
           which virtualenv
           virtualenv --version
-          #ls /opt/hostedtoolcache/Python/3.6.13/x64/lib/python3.6/site-packages
-          #virtualenv --help # no config file
-          ls -l /opt/hostedtoolcache/Python/3.6*/x64/bin/python*
-          /opt/hostedtoolcache/Python/3.6.13/x64/bin/python -c 'import sysconfig; print(sysconfig.get_config_vars())'
-          exit 1
       - name: Install requirements
         run: |
           ./scripts/travis/install-requirements.sh
diff --git a/.travis.yml b/.travis.yml
index e1662b7d5d..ff3ba9fd01 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -86,24 +86,11 @@ cache:
     #- .tox/
 
 install:
-  - ls -l /opt/python/3.6*/bin/python*
-  #- ls /opt/python/3.6.7/ /home/travis/virtualenv/python3.6.7/
-  - ls /opt/python/3.6.7/lib/python3.6/
-  - ls -l /home/travis/virtualenv/python3.6.7/lib/python3.6/
-  #- cat /home/travis/virtualenv/python3.6.7/pyvenv.cfg
-  - exit 1
-  #- ls /home/travis/virtualenv/python3.6.7/lib/python3.6/site-packages
   - which virtualenv || true
   - virtualenv --version || true
   - pip install --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
   - which virtualenv
   - virtualenv --version
-  #- virtualenv --help # no config file
-  #- ls -l /home/travis/virtualenv/python3.6*/bin/python*
-  #- /home/travis/virtualenv/python3.6.7/bin/python -c 'import sysconfig; print(sysconfig.get_config_vars())'
-  #- /opt/python/3.6.7/bin/python -c 'import sysconfig; print(sysconfig.get_config_vars())'
-  - exit 1
-  #- ls /home/travis/virtualenv/python3.6.7/lib/python3.6/site-packages
   - ./scripts/travis/install-requirements.sh
   - ls -l /home/travis/build/st2sandbox/st2/virtualenv/bin
   # prep a travis-specific dev conf file that uses travis instead of stanley

From cc47fe545965f4c95677951531115372d73dfcdc Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 21 Mar 2021 21:44:21 -0500
Subject: [PATCH 086/103] rm --user packages

---
 .github/workflows/ci.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 3f2c51fe9e..a0772d42d2 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -184,6 +184,9 @@ jobs:
       - name: Install requirements
         run: |
           ./scripts/travis/install-requirements.sh
+          # drop the --user install virtualenv to prevent polluting tests
+          ls /home/runner/.local/lib/python3.6/site-packages
+          rm -rf /home/runner/.local/lib/python3.6/site-packages
       - name: Setup integration tests
         run: |
           # prep a ci-specific dev conf file that

From 641853ff333dd8b2f19710e0b160cf1fbcb56fb5 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 21 Mar 2021 23:21:27 -0500
Subject: [PATCH 087/103] pip uninstall --user

---
 .github/workflows/ci.yaml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index a0772d42d2..362e6e1e76 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -152,6 +152,8 @@ jobs:
           path: |
             ~/.cache/pip
             virtualenv
+          #  !virtualenv/lib/python*/site-packages/st2*
+          #  !virtualenv/bin/st2*
           key: py-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt') }}
           restore-keys: |
             py-${{ runner.os }}-${{ matrix.python }}-
@@ -185,8 +187,7 @@ jobs:
         run: |
           ./scripts/travis/install-requirements.sh
           # drop the --user install virtualenv to prevent polluting tests
-          ls /home/runner/.local/lib/python3.6/site-packages
-          rm -rf /home/runner/.local/lib/python3.6/site-packages
+          pip freeze --user | xargs pip uninstall -y
       - name: Setup integration tests
         run: |
           # prep a ci-specific dev conf file that

From 80f33fe21b83727a5e6800dff8428739b090a486 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 21 Mar 2021 23:37:56 -0500
Subject: [PATCH 088/103] virtualenv

---
 .github/workflows/ci.yaml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 362e6e1e76..129c74c2e4 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -181,13 +181,16 @@ jobs:
           #       only have to update it one place when we change the version
           # Note: Use --user to avoid polluting system site-packages
           pip install --user --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
+          virtualenv --no-download ~/virtualenv
+          ~/virtualenv/bin/pip install --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
+          # drop the --user install virtualenv to prevent polluting tests
+          pip freeze --user | xargs pip uninstall -y
+          ln -s ~/virtualenv/bin/virtualenv ~/.local/bin/virtualenv
           which virtualenv
           virtualenv --version
       - name: Install requirements
         run: |
           ./scripts/travis/install-requirements.sh
-          # drop the --user install virtualenv to prevent polluting tests
-          pip freeze --user | xargs pip uninstall -y
       - name: Setup integration tests
         run: |
           # prep a ci-specific dev conf file that

From 4b7d9e167ab95b1bd940ce394b2b3cb6de101e8f Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Sun, 21 Mar 2021 23:41:49 -0500
Subject: [PATCH 089/103] virtualenv

---
 .github/workflows/ci.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 129c74c2e4..baef114a6a 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -185,6 +185,7 @@ jobs:
           ~/virtualenv/bin/pip install --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
           # drop the --user install virtualenv to prevent polluting tests
           pip freeze --user | xargs pip uninstall -y
+          mkdir -p ~/.local/bin
           ln -s ~/virtualenv/bin/virtualenv ~/.local/bin/virtualenv
           which virtualenv
           virtualenv --version

From e2c1f95e8010f3c360b41a7579cd371822cbe338 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Mon, 22 Mar 2021 00:05:01 -0500
Subject: [PATCH 090/103] Try to recreate failure on Travis

---
 .travis.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index ff3ba9fd01..e266f7650f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -88,7 +88,9 @@ cache:
 install:
   - which virtualenv || true
   - virtualenv --version || true
-  - pip install --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
+  - /opt/python/3.6.7/bin/pip install --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
+  - mkdir -p ~/.local/bin
+  - ln -s /opt/python/3.6.7/bin/virtualenv ~/.local/bin/virtualenv
   - which virtualenv
   - virtualenv --version
   - ./scripts/travis/install-requirements.sh

From c3395ea52cd3c54c118c726e456c527ea443992e Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Mon, 22 Mar 2021 10:01:00 -0500
Subject: [PATCH 091/103] leave TODO/FIXME around broken test

---
 .github/workflows/ci.yaml                     |  5 ++---
 .travis.yml                                   |  9 ++++----
 .../integration/test_pythonrunner_behavior.py | 22 ++++++++-----------
 3 files changed, 15 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index baef114a6a..e953e70270 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -175,11 +175,10 @@ jobs:
           sudo apt-get -f -y install libldap2-dev libsasl2-dev libssl-dev libyaml-dev ldap-utils
       - name: Install virtualenv
         run: |
-          which virtualenv || true
-          virtualenv --version || true
           # Note: Use the verison of virtualenv pinned in fixed-requirements.txt so we
           #       only have to update it one place when we change the version
-          # Note: Use --user to avoid polluting system site-packages
+          # Note: Use --user to avoid polluting system site-packages (which breaks one of our tests)
+          # TODO: simplify this once fixed in contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
           pip install --user --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
           virtualenv --no-download ~/virtualenv
           ~/virtualenv/bin/pip install --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
diff --git a/.travis.yml b/.travis.yml
index e266f7650f..5693d16cf5 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -86,11 +86,10 @@ cache:
     #- .tox/
 
 install:
-  - which virtualenv || true
-  - virtualenv --version || true
-  - /opt/python/3.6.7/bin/pip install --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
-  - mkdir -p ~/.local/bin
-  - ln -s /opt/python/3.6.7/bin/virtualenv ~/.local/bin/virtualenv
+  # This triggers the same behavior on travis as we found on github because it installs six in the system-site-packages
+  #- /opt/python/3.6.7/bin/pip install --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
+  #- mkdir -p ~/.local/bin
+  #- ln -s /opt/python/3.6.7/bin/virtualenv ~/.local/bin/virtualenv
   - which virtualenv
   - virtualenv --version
   - ./scripts/travis/install-requirements.sh
diff --git a/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py b/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
index a5bf0f92ad..2952f0b51c 100644
--- a/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
+++ b/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
@@ -72,25 +72,21 @@ def test_priority_of_loading_library_after_setup_pack_virtualenv(self):
         # requirements.txt wihch only writes 'six' module.
         setup_pack_virtualenv(pack_name=pack_name)
         self.assertTrue(os.path.exists(os.path.join(self.virtualenvs_path, pack_name)))
-        ls = os.stat(os.path.join(self.virtualenvs_path, pack_name))
-        print(
-            f"\n{os.path.join(self.virtualenvs_path, pack_name)} "
-            f"exists={os.path.exists(os.path.join(self.virtualenvs_path, pack_name))} "
-            f"perms={oct(ls.st_mode)[-3:]} owner_uid={ls.st_uid} owner_gid={ls.st_gid} "
-            f"owner={pwd.getpwuid(ls.st_uid)[0]} "
-            f"uid={os.getuid()} user={pwd.getpwuid(os.getuid())[0]}"
-        )
-        sp = os.path.join(self.virtualenvs_path, pack_name, 'lib', 'python3.6', 'site-packages')
-        print(f"\n{sp} exists={os.path.exists(sp)}")
-        p_six = os.path.join(self.virtualenvs_path, pack_name, 'lib', 'python3.6', 'site-packages', 'six.py')
-        print(f"\n{p_six} exists={os.path.exists(p_six)}")
 
         # This test suite expects that loaded six module is located under the virtualenv library,
         # because 'six' is written in the requirements.txt of 'test_library_dependencies' pack.
         (_, output, _) = self._run_action(
             pack_name, "get_library_path.py", {"module": "six"}
         )
-        print(f"\noutput={output}")
+        # FIXME: This test fails if system site-packages has six because
+        # it won't get installed in the virtualenv (w/ --system-site-packages)
+        # system site-packages is never from a virtualenv.
+        # Travis has python installed in /opt/python/3.6.7
+        # with a no-system-site-packages virtualenv at /home/travis/virtualenv/python3.6.7
+        # GitHub Actions python is in /opt/hostedtoolcache/Python/3.6.13/x64/
+        # But ther isn't a virtualenv, so when we pip installed `virtualenv`,
+        # (which depends on, and therefore installs `six`)
+        # we installed it in system-site-packages not an intermediate virtualenv
         self.assertEqual(output["result"].find(self.virtualenvs_path), 0)
 
         # Conversely, this expects that 'mock' module file-path is not under sandbox library,

From 72afad4fc1b3dd81f884b00dafcae16ba3ea834e Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Mon, 22 Mar 2021 11:03:59 -0500
Subject: [PATCH 092/103] backwards compat default virtualenv bin

---
 st2common/st2common/config.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/st2common/st2common/config.py b/st2common/st2common/config.py
index 19d5ebb249..de0c0dcfde 100644
--- a/st2common/st2common/config.py
+++ b/st2common/st2common/config.py
@@ -393,11 +393,13 @@ def register_opts(ignore_errors=False):
 
     # Runner options
     default_python_bin_path = sys.executable
-    # FIXME: If the virtualenv uses a symlinked python, then this won't use
-    # what is installed in the virtualenv, it will use the system virtualenv.
-    #base_dir = os.path.dirname(os.path.realpath(default_python_bin_path))
+    # If the virtualenv uses a symlinked python, then try using virtualenv from that venv
+    # first before looking for virtualenv installed in python's system-site-packages.
     base_dir = os.path.dirname(default_python_bin_path)
     default_virtualenv_bin_path = os.path.join(base_dir, "virtualenv")
+    if not os.path.exists(default_virtualenv_bin_python):
+        base_dir = os.path.dirname(os.path.realpath(default_python_bin_path))
+        default_virtualenv_bin_path = os.path.join(base_dir, "virtualenv")
 
     action_runner_opts = [
         # Common runner options

From 9bb6643483f9f04e5cab441f0efabbe7a6a82568 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Mon, 22 Mar 2021 11:09:16 -0500
Subject: [PATCH 093/103] cleanup

---
 .github/workflows/ci.yaml                     | 55 ++++++-------------
 .travis.yml                                   |  1 -
 Makefile                                      |  2 -
 .../integration/test_pythonrunner_behavior.py |  6 +-
 st2common/st2common/util/virtualenvs.py       | 10 +++-
 5 files changed, 30 insertions(+), 44 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index e953e70270..5c2b455e87 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -95,9 +95,7 @@ jobs:
       COLUMNS: '120'
       PYLINT_CONCURRENCY: '2'
 
-      # CI st2.conf
-      #   - with ST2_CI_USER user instead of stanley
-      #   - with RABBITMQ_USERNAME:RABBITMQ_PASSWORD instead of guest:guest
+      # CI st2.conf (with ST2_CI_USER user instead of stanley)
       ST2_CONF: 'conf/st2.ci.conf'
 
       # Tell StackStorm that we are indeed in CI mode, previously we hard coded a Travis specific
@@ -108,10 +106,6 @@ jobs:
 
       # Name of the user who is running the CI (on GitHub Actions this is 'runner')
       ST2_CI_USER: 'runner'
-
-      # Since we're using a container, we can't use guest:guest
-      # RABBITMQ_USERNAME: st2
-      # RABBITMQ_PASSWORD: st2
     steps:
       - name: Custom Environment Setup
         # built-in GitHub Actions environment variables
@@ -152,11 +146,12 @@ jobs:
           path: |
             ~/.cache/pip
             virtualenv
+            ~/virtualenv
           #  !virtualenv/lib/python*/site-packages/st2*
           #  !virtualenv/bin/st2*
-          key: py-${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt') }}
+          key: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt') }}
           restore-keys: |
-            py-${{ runner.os }}-${{ matrix.python }}-
+            ${{ runner.os }}-python-${{ matrix.python }}-
       - uses: actions/cache@v2
         with:
           path: |
@@ -165,8 +160,8 @@ jobs:
             /var/cache/apt/*.bin
           key: apt-${{ runner.os }}-${{ steps.date.outputs.year }}-${{ steps.date.outputs.week }}
           restore-keys: |
-            apt-${{ runner.os }}-${{ steps.date.outputs.year }}-
-            apt-${{ runner.os }}-
+            ${{ runner.os }}-apt-${{ steps.date.outputs.year }}-
+            ${{ runner.os }}-apt-
       - name: Install apt depedencies
         run: |
           # install dev dependencies for Python YAML and LDAP packages
@@ -179,13 +174,15 @@ jobs:
           #       only have to update it one place when we change the version
           # Note: Use --user to avoid polluting system site-packages (which breaks one of our tests)
           # TODO: simplify this once fixed in contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
-          pip install --user --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
-          virtualenv --no-download ~/virtualenv
-          ~/virtualenv/bin/pip install --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
-          # drop the --user install virtualenv to prevent polluting tests
-          pip freeze --user | xargs pip uninstall -y
-          mkdir -p ~/.local/bin
-          ln -s ~/virtualenv/bin/virtualenv ~/.local/bin/virtualenv
+          if [[ ! -f ~/virtualenv/bin/virtualenv ]]; then  # use the cached version whenever possible
+            pip install --user --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
+            virtualenv --no-download ~/virtualenv
+            ~/virtualenv/bin/pip install --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
+            # drop the --user install virtualenv to prevent polluting tests
+            pip freeze --user | xargs pip uninstall -y
+            mkdir -p ~/.local/bin
+            ln -s ~/virtualenv/bin/virtualenv ~/.local/bin/virtualenv
+          fi
           which virtualenv
           virtualenv --version
       - name: Install requirements
@@ -193,13 +190,10 @@ jobs:
           ./scripts/travis/install-requirements.sh
       - name: Setup integration tests
         run: |
-          # prep a ci-specific dev conf file that
-          #   - uses runner instead of stanley
-          #     this user is the username of the user in GitHub actions,
-          #     used for SSH, etc during integration tests (important)
-          #   - uses a non-guest user/pass for RabbitMQ since we can't use guest outside of the container
+          # prep a ci-specific dev conf file that uses runner instead of stanley
+          # this user is the username of the user in GitHub actions, used for SSH, etc during
+          # integration tests (important)
           cp conf/st2.dev.conf "${ST2_CONF}" ; sed -i -e "s/stanley/${ST2_CI_USER}/" "${ST2_CONF}"
-          # cp conf/st2.dev.conf "${ST2_CONF}" ; sed -i -e "s/stanley/${ST2_CI_USER}/" -e "s/guest:guest/${RABBITMQ_USERNAME}:${RABBITMQ_PASSWORD}/" "${ST2_CONF}"
           scripts/travis/add-itest-user-key.sh
           sudo .circle/add-itest-user.sh
       - name: Permissions Workaround
@@ -207,7 +201,6 @@ jobs:
         run: |
           echo "$ST2_CI_REPO_PATH"
           sudo ST2_CI_REPO_PATH="${ST2_CI_REPO_PATH}" scripts/travis/permissions-workaround.sh
-          # sudo chmod -R o+w /home/runner/.local/share/virtualenv
       - name: Finish Integration test setup (RabbitMQ, st2.tests.conf)
         if: "${{ env.TASK == 'ci-integration' }}"
         # bitnami image allows (see bitnami/rabbitmq readme):
@@ -216,9 +209,6 @@ jobs:
         timeout-minutes: 2  # may die if rabbitmq fails to start
         run: |
           set -x
-          # Since we're using a container, we can't use guest:guest
-          # Integration tests hardcode paths to conf/st2.tests*.conf so we modify in place.
-          ###sed -i -e "s/guest:guest/${RABBITMQ_USERNAME}:${RABBITMQ_PASSWORD}/" conf/st2.tests*.conf
           # Use custom RabbitMQ config which enables SSL / TLS listener on port 5671 with test certs
           sudo cp scripts/github/rabbitmq.conf /home/runner/rabbitmq_conf/custom.conf
           # The code is checked out after the container is already up, so we don't mount them.
@@ -247,21 +237,12 @@ jobs:
           git --version
           pip --version
           pip list
-          which virtualenv
           virtualenv --version
-          virtualenv/bin/virtualenv --version
-          ls -l virtualenv/bin
           # Print out various environment variables info
           make play
       - name: make
         # use: script -e -c to print colors
         run: |
-          ls -ld /tmp
-          sudo ls -ld /tmp
-          ls -l /tmp
-          sudo ls -l /tmp
-          mount
-          sudo mount
           script -e -c "make ${TASK}"
       - name: Nightly
         # Run any additional nightly checks only as part of a nightly (cron) build
diff --git a/.travis.yml b/.travis.yml
index 5693d16cf5..ccc38914ea 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -93,7 +93,6 @@ install:
   - which virtualenv
   - virtualenv --version
   - ./scripts/travis/install-requirements.sh
-  - ls -l /home/travis/build/st2sandbox/st2/virtualenv/bin
   # prep a travis-specific dev conf file that uses travis instead of stanley
   - cp conf/st2.dev.conf "${ST2_CONF}" ; sed -i -e "s/stanley/${ST2_CI_USER}/" "${ST2_CONF}"
   - sudo scripts/travis/add-itest-user-key.sh
diff --git a/Makefile b/Makefile
index 48599eed7b..c8c51e1694 100644
--- a/Makefile
+++ b/Makefile
@@ -674,8 +674,6 @@ virtualenv:
 	@echo
 	@echo "==================== virtualenv ===================="
 	@echo
-	which virtualenv
-	virtualenv --version
 	test -f $(VIRTUALENV_DIR)/bin/activate || virtualenv --python=$(PYTHON_VERSION) $(VIRTUALENV_DIR) --no-download
 
 	# Setup PYTHONPATH in bash activate script...
diff --git a/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py b/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
index 2952f0b51c..a694ffceac 100644
--- a/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
+++ b/contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
@@ -18,7 +18,6 @@
 """
 
 import os
-import pwd
 import mock
 import tempfile
 
@@ -42,7 +41,10 @@
 
 class PythonRunnerBehaviorTestCase(CleanFilesTestCase, CleanDbTestCase):
 
-    DISPLAY_LOG_MESSAGES = True
+    # If you need these logs, then you probably also want to uncomment
+    # extra debug log messages in st2common/st2common/util/virtualenvs.py
+    # and pass --logging-level=DEBUG to nosetests
+    # DISPLAY_LOG_MESSAGES = True
 
     def setUp(self):
         super(PythonRunnerBehaviorTestCase, self).setUp()
diff --git a/st2common/st2common/util/virtualenvs.py b/st2common/st2common/util/virtualenvs.py
index 3b89385251..ad64f1e3af 100644
--- a/st2common/st2common/util/virtualenvs.py
+++ b/st2common/st2common/util/virtualenvs.py
@@ -283,7 +283,10 @@ def install_requirements(
         " ".join(cmd),
     )
     exit_code, stdout, stderr = run_command(cmd=cmd, env=env)
-    logger.debug(f"\n==========\nstdout=\n{stdout}\n==========\nstderr=\n{stderr}")
+
+    # Normally we don't want this, even in debug logs. But it is useful to
+    # investigate pip behavior changes & broken virtualenv integration tests.
+    # logger.debug(f"\npip stdout=\n{stdout}")
 
     if exit_code != 0:
         stdout = to_ascii(stdout)
@@ -330,7 +333,10 @@ def install_requirement(virtualenv_path, requirement, proxy_config=None, logger=
         "Installing requirement %s with command %s.", requirement, " ".join(cmd)
     )
     exit_code, stdout, stderr = run_command(cmd=cmd, env=env)
-    logger.debug(f"\n==========\nstdout=\n{stdout}\n==========\nstderr=\n{stderr}")
+
+    # Normally we don't want this, even in debug logs. But it is useful to
+    # investigate pip behavior changes & broken virtualenv integration tests.
+    # logger.debug(f"\npip stdout=\n{stdout}")
 
     if exit_code != 0:
         raise Exception(

From 8f444764b424418add5dbb91dd2518d0c93341df Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Mon, 22 Mar 2021 14:07:53 -0500
Subject: [PATCH 094/103] cleanup

---
 tools/launchdev.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/launchdev.sh b/tools/launchdev.sh
index 6009605ab8..3d4a97b26e 100755
--- a/tools/launchdev.sh
+++ b/tools/launchdev.sh
@@ -55,7 +55,7 @@ function init(){
     CURRENT_DIR=`pwd`
     CURRENT_USER=`whoami`
     CURRENT_USER_GROUP=`id -gn`
-	echo "Current user:group = ${CURRENT_USER}:${CURRENT_USER_GROUP}"
+    echo "Current user:group = ${CURRENT_USER}:${CURRENT_USER_GROUP}"
 
     if [[ (${COMMAND_PATH} == /*) ]] ;
     then

From a815e25c58a9ecfc7f1abcbd84955c67cd98e67f Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Mon, 22 Mar 2021 14:09:46 -0500
Subject: [PATCH 095/103] fix typo

---
 st2common/st2common/config.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/st2common/st2common/config.py b/st2common/st2common/config.py
index de0c0dcfde..d97cf08546 100644
--- a/st2common/st2common/config.py
+++ b/st2common/st2common/config.py
@@ -397,7 +397,7 @@ def register_opts(ignore_errors=False):
     # first before looking for virtualenv installed in python's system-site-packages.
     base_dir = os.path.dirname(default_python_bin_path)
     default_virtualenv_bin_path = os.path.join(base_dir, "virtualenv")
-    if not os.path.exists(default_virtualenv_bin_python):
+    if not os.path.exists(default_virtualenv_bin_path):
         base_dir = os.path.dirname(os.path.realpath(default_python_bin_path))
         default_virtualenv_bin_path = os.path.join(base_dir, "virtualenv")
 

From 499851eba1772f6c58e062f2e60e85a5a1db957c Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Mon, 22 Mar 2021 15:02:25 -0500
Subject: [PATCH 096/103] try py 3.6.7 on gha

---
 .github/workflows/ci.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 5c2b455e87..8f4d3c6af0 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -44,6 +44,9 @@ jobs:
           # - name: 'Micro Benchmarks'
           #   task: 'micro-benchmarks'
           #   python-version: '3.6'
+          - name: 'Integration Tests 3.6.7 (travis version)'
+            task: 'ci-integration'
+            python-version: '3.6.7'
           - name: 'Integration Tests'
             task: 'ci-integration'
             python-version: '3.6'

From 3e1bc4b50da8b136ed1c74023adb1456f0121ed6 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Mon, 22 Mar 2021 15:53:41 -0500
Subject: [PATCH 097/103] debug

---
 .../python_runner/python_runner/python_action_wrapper.py       | 3 +++
 .../tests/integration/test_python_action_process_wrapper.py    | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/contrib/runners/python_runner/python_runner/python_action_wrapper.py b/contrib/runners/python_runner/python_runner/python_action_wrapper.py
index 06158496be..007ceb0312 100644
--- a/contrib/runners/python_runner/python_runner/python_action_wrapper.py
+++ b/contrib/runners/python_runner/python_runner/python_action_wrapper.py
@@ -347,7 +347,9 @@ def _get_action_instance(self):
     if args.stdin_parameters:
         LOG.debug("Getting parameters from stdin")
 
+        print(f"\nBEFORE: type(sys.stdin)={type(sys.stdin)}\nsys.stdin={sys.stdin}\nsys.stdin.closed={sys.stdin.closed}")
         i, _, _ = select.select([sys.stdin], [], [], READ_STDIN_INPUT_TIMEOUT)
+        print(f"\nAFTER: type(sys.stdin)={type(sys.stdin)}\nsys.stdin={sys.stdin}\nsys.stdin.closed={sys.stdin.closed}")
 
         if not i:
             raise ValueError(
@@ -356,6 +358,7 @@ def _get_action_instance(self):
                     "parameters from stdin"
                 )
             )
+        print(i)
 
         stdin_data = sys.stdin.readline().strip()
 
diff --git a/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py b/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py
index 7be43c8e88..495fb3331b 100644
--- a/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py
+++ b/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py
@@ -142,7 +142,7 @@ def test_stdin_params_timeout_no_stdin_data_provided(self):
             "python %s --pack=dummy --file-path=%s --config='%s' "
             "--stdin-parameters" % (WRAPPER_SCRIPT_PATH, file_path, config)
         )
-        print(command_string)
+        print(f"\ncommand_string=\n{command_string}")
         exit_code, stdout, stderr = run_command(command_string, shell=True)
         print(f"stdout=\n{stdout}")
         print(f"stderr=\n{stderr}")

From 746faccdf5e27f45144fc3b9719eb931f1deefa8 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Mon, 22 Mar 2021 16:09:52 -0500
Subject: [PATCH 098/103] debug

---
 .../python_runner/python_runner/python_action_wrapper.py      | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/contrib/runners/python_runner/python_runner/python_action_wrapper.py b/contrib/runners/python_runner/python_runner/python_action_wrapper.py
index 007ceb0312..5795d02e77 100644
--- a/contrib/runners/python_runner/python_runner/python_action_wrapper.py
+++ b/contrib/runners/python_runner/python_runner/python_action_wrapper.py
@@ -349,7 +349,7 @@ def _get_action_instance(self):
 
         print(f"\nBEFORE: type(sys.stdin)={type(sys.stdin)}\nsys.stdin={sys.stdin}\nsys.stdin.closed={sys.stdin.closed}")
         i, _, _ = select.select([sys.stdin], [], [], READ_STDIN_INPUT_TIMEOUT)
-        print(f"\nAFTER: type(sys.stdin)={type(sys.stdin)}\nsys.stdin={sys.stdin}\nsys.stdin.closed={sys.stdin.closed}")
+        print(f"\nAFTER: type(sys.stdin)={type(sys.stdin)}\nsys.stdin={sys.stdin}\nsys.stdin.closed={sys.stdin.closed}\ni={i}")
 
         if not i:
             raise ValueError(
@@ -358,7 +358,7 @@ def _get_action_instance(self):
                     "parameters from stdin"
                 )
             )
-        print(i)
+        print("after not i")
 
         stdin_data = sys.stdin.readline().strip()
 

From 727d9293e4a10578767e0b42a1f42c2fb271bdd2 Mon Sep 17 00:00:00 2001
From: Jacob Floyd <cognifloyd@gmail.com>
Date: Mon, 22 Mar 2021 16:45:45 -0500
Subject: [PATCH 099/103] is it the kernel?

---
 .github/workflows/ci.yaml                                | 9 +++++----
 .travis.yml                                              | 4 ++++
 .../python_runner/python_runner/python_action_wrapper.py | 2 ++
 3 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 8f4d3c6af0..1cc41d977f 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -44,12 +44,9 @@ jobs:
           # - name: 'Micro Benchmarks'
           #   task: 'micro-benchmarks'
           #   python-version: '3.6'
-          - name: 'Integration Tests 3.6.7 (travis version)'
-            task: 'ci-integration'
-            python-version: '3.6.7'
           - name: 'Integration Tests'
             task: 'ci-integration'
-            python-version: '3.6'
+            python-version: '3.6.7'  # temporarily match travis CI version; TODO use 3.6
     services:
       mongo:
         image: mongo:4.0
@@ -173,6 +170,10 @@ jobs:
           sudo apt-get -f -y install libldap2-dev libsasl2-dev libssl-dev libyaml-dev ldap-utils
       - name: Install virtualenv
         run: |
+          uname -a
+          which python
+          python --version
+          python -c 'import sys, select; i,_,_=select.select([sys.stdin],[],[],2);print(i)'
           # Note: Use the verison of virtualenv pinned in fixed-requirements.txt so we
           #       only have to update it one place when we change the version
           # Note: Use --user to avoid polluting system site-packages (which breaks one of our tests)
diff --git a/.travis.yml b/.travis.yml
index ccc38914ea..142aee60a7 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -90,6 +90,10 @@ install:
   #- /opt/python/3.6.7/bin/pip install --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
   #- mkdir -p ~/.local/bin
   #- ln -s /opt/python/3.6.7/bin/virtualenv ~/.local/bin/virtualenv
+  - uname -a
+  - which python
+  - python --version
+  - python -c 'import sys, select; i,_,_=select.select([sys.stdin],[],[],2);print(i)'
   - which virtualenv
   - virtualenv --version
   - ./scripts/travis/install-requirements.sh
diff --git a/contrib/runners/python_runner/python_runner/python_action_wrapper.py b/contrib/runners/python_runner/python_runner/python_action_wrapper.py
index 5795d02e77..6903b13b3e 100644
--- a/contrib/runners/python_runner/python_runner/python_action_wrapper.py
+++ b/contrib/runners/python_runner/python_runner/python_action_wrapper.py
@@ -347,6 +347,8 @@ def _get_action_instance(self):
     if args.stdin_parameters:
         LOG.debug("Getting parameters from stdin")
 
+        # travis has ubuntu xenial (16.04.6 LTS) with kernel 4.15.0-1077-gcp
+        # gha has ubuntu focal (20.04.2 LTS) with kernel 5.4
         print(f"\nBEFORE: type(sys.stdin)={type(sys.stdin)}\nsys.stdin={sys.stdin}\nsys.stdin.closed={sys.stdin.closed}")
         i, _, _ = select.select([sys.stdin], [], [], READ_STDIN_INPUT_TIMEOUT)
         print(f"\nAFTER: type(sys.stdin)={type(sys.stdin)}\nsys.stdin={sys.stdin}\nsys.stdin.closed={sys.stdin.closed}\ni={i}")

From 47405ee0794f14a66c70652704bbd38e0d9baa98 Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Mon, 22 Mar 2021 23:11:53 +0100
Subject: [PATCH 100/103] Add debugging change.

---
 .../python_runner/python_runner/python_action_wrapper.py        | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/contrib/runners/python_runner/python_runner/python_action_wrapper.py b/contrib/runners/python_runner/python_runner/python_action_wrapper.py
index 6903b13b3e..48bae417a7 100644
--- a/contrib/runners/python_runner/python_runner/python_action_wrapper.py
+++ b/contrib/runners/python_runner/python_runner/python_action_wrapper.py
@@ -363,6 +363,8 @@ def _get_action_instance(self):
         print("after not i")
 
         stdin_data = sys.stdin.readline().strip()
+        print("stdin data: %s" % (stdin_data))
+        print(len(stdin_data)
 
         try:
             stdin_parameters = orjson.loads(stdin_data)

From b38ad36fce5786e1e8e71cf73af78de4c7f30055 Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Mon, 22 Mar 2021 23:14:28 +0100
Subject: [PATCH 101/103] test change.

---
 .../python_runner/python_runner/python_action_wrapper.py      | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/contrib/runners/python_runner/python_runner/python_action_wrapper.py b/contrib/runners/python_runner/python_runner/python_action_wrapper.py
index 48bae417a7..873eac0b2d 100644
--- a/contrib/runners/python_runner/python_runner/python_action_wrapper.py
+++ b/contrib/runners/python_runner/python_runner/python_action_wrapper.py
@@ -364,7 +364,9 @@ def _get_action_instance(self):
 
         stdin_data = sys.stdin.readline().strip()
         print("stdin data: %s" % (stdin_data))
-        print(len(stdin_data)
+        print(len(stdin_data))
+        print(stdin_data)
+        print(stdin_data)
 
         try:
             stdin_parameters = orjson.loads(stdin_data)

From 98ad5e52290fe51d06965a6d6cb72be4a14aa69d Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Mon, 22 Mar 2021 23:29:59 +0100
Subject: [PATCH 102/103] Test close_fds=True.

---
 .../tests/integration/test_python_action_process_wrapper.py | 5 ++++-
 st2common/st2common/util/shell.py                           | 6 ++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py b/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py
index 495fb3331b..0f04fcf7cb 100644
--- a/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py
+++ b/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py
@@ -143,7 +143,10 @@ def test_stdin_params_timeout_no_stdin_data_provided(self):
             "--stdin-parameters" % (WRAPPER_SCRIPT_PATH, file_path, config)
         )
         print(f"\ncommand_string=\n{command_string}")
-        exit_code, stdout, stderr = run_command(command_string, shell=True)
+        import subprocess
+        import sys
+        print(sys.stdin)
+        exit_code, stdout, stderr = run_command(command_string, shell=True, close_fds=False)
         print(f"stdout=\n{stdout}")
         print(f"stderr=\n{stderr}")
 
diff --git a/st2common/st2common/util/shell.py b/st2common/st2common/util/shell.py
index 113f7ae162..8f28e1c227 100644
--- a/st2common/st2common/util/shell.py
+++ b/st2common/st2common/util/shell.py
@@ -47,6 +47,7 @@ def run_command(
     shell=False,
     cwd=None,
     env=None,
+    close_fds=None,
 ):
     """
     Run the provided command in a subprocess and wait until it completes.
@@ -83,6 +84,10 @@ def run_command(
     if not env:
         env = os.environ.copy()
 
+    kwargs = {}
+    if close_fds is not None:
+        kwargs["close_fds"] = close_fds
+
     process = concurrency.subprocess_popen(
         args=cmd,
         stdin=stdin,
@@ -91,6 +96,7 @@ def run_command(
         env=env,
         cwd=cwd,
         shell=shell,
+        **kwargs,
     )
     stdout, stderr = process.communicate()
     exit_code = process.returncode

From 5d0c3793096ad3171a016e89cf2e98e731034229 Mon Sep 17 00:00:00 2001
From: Tomaz Muraus <tomaz@tomaz.me>
Date: Mon, 22 Mar 2021 23:51:35 +0100
Subject: [PATCH 103/103] Remove testing changes, add workaround for failing
 test.

---
 .../python_runner/python_action_wrapper.py    | 13 ++++++-------
 .../test_python_action_process_wrapper.py     | 19 ++++++++++---------
 st2common/st2common/util/shell.py             |  3 +++
 3 files changed, 19 insertions(+), 16 deletions(-)

diff --git a/contrib/runners/python_runner/python_runner/python_action_wrapper.py b/contrib/runners/python_runner/python_runner/python_action_wrapper.py
index 873eac0b2d..a0ef94d048 100644
--- a/contrib/runners/python_runner/python_runner/python_action_wrapper.py
+++ b/contrib/runners/python_runner/python_runner/python_action_wrapper.py
@@ -349,9 +349,7 @@ def _get_action_instance(self):
 
         # travis has ubuntu xenial (16.04.6 LTS) with kernel 4.15.0-1077-gcp
         # gha has ubuntu focal (20.04.2 LTS) with kernel 5.4
-        print(f"\nBEFORE: type(sys.stdin)={type(sys.stdin)}\nsys.stdin={sys.stdin}\nsys.stdin.closed={sys.stdin.closed}")
         i, _, _ = select.select([sys.stdin], [], [], READ_STDIN_INPUT_TIMEOUT)
-        print(f"\nAFTER: type(sys.stdin)={type(sys.stdin)}\nsys.stdin={sys.stdin}\nsys.stdin.closed={sys.stdin.closed}\ni={i}")
 
         if not i:
             raise ValueError(
@@ -360,13 +358,14 @@ def _get_action_instance(self):
                     "parameters from stdin"
                 )
             )
-        print("after not i")
 
         stdin_data = sys.stdin.readline().strip()
-        print("stdin data: %s" % (stdin_data))
-        print(len(stdin_data))
-        print(stdin_data)
-        print(stdin_data)
+
+        if not stdin_data:
+            # This could indicate that parent process (e.g. process which runs the tests has
+            # incorrectly opened the stdin and that one is then inherited by the process which is
+            # spawning it which will cause issues)
+            raise ValueError("Received no valid parameters data from sys.stdin")
 
         try:
             stdin_parameters = orjson.loads(stdin_data)
diff --git a/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py b/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py
index 0f04fcf7cb..c97a380fb2 100644
--- a/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py
+++ b/contrib/runners/python_runner/tests/integration/test_python_action_process_wrapper.py
@@ -142,20 +142,21 @@ def test_stdin_params_timeout_no_stdin_data_provided(self):
             "python %s --pack=dummy --file-path=%s --config='%s' "
             "--stdin-parameters" % (WRAPPER_SCRIPT_PATH, file_path, config)
         )
-        print(f"\ncommand_string=\n{command_string}")
-        import subprocess
-        import sys
-        print(sys.stdin)
-        exit_code, stdout, stderr = run_command(command_string, shell=True, close_fds=False)
-        print(f"stdout=\n{stdout}")
-        print(f"stderr=\n{stderr}")
+        exit_code, stdout, stderr = run_command(
+            command_string, shell=True, close_fds=True
+        )
 
-        expected_msg = (
+        # Depending on how tests are spawned, sys.stdin may be opened and this will cause issues
+        # with this tests so we simply check for two different errors which are considered
+        # acceptable.
+        expected_msg_1 = (
             "ValueError: No input received and timed out while waiting for parameters "
             "from stdin"
         )
+        expected_msg_2 = "ValueError: Received no valid parameters data from sys.stdin"
+
         self.assertEqual(exit_code, 1)
-        self.assertIn(expected_msg, stderr)
+        self.assertTrue(expected_msg_1 in stderr or expected_msg_2 in stderr)
 
     def test_stdin_params_invalid_format_friendly_error(self):
         config = {}
diff --git a/st2common/st2common/util/shell.py b/st2common/st2common/util/shell.py
index 8f28e1c227..171085be74 100644
--- a/st2common/st2common/util/shell.py
+++ b/st2common/st2common/util/shell.py
@@ -74,6 +74,9 @@ def run_command(
                 environment from the current process is inherited.
     :type env: ``dict``
 
+    :param close_fds: True to close all the fds. By default when None is provided we rely on
+                      default upstream behavior which may be Python version specific.
+
     :rtype: ``tuple`` (exit_code, stdout, stderr)
     """
     if not isinstance(cmd, (list, tuple) + six.string_types):