Skip to content

Commit

Permalink
Merge pull request #5202 from st2sandbox/ci-integration-to-gha
Browse files Browse the repository at this point in the history
Enable ci-integration tests on GHA (incl fixing a few tests)
  • Loading branch information
blag authored Mar 23, 2021
2 parents 5b41a40 + 28833ec commit 007beed
Show file tree
Hide file tree
Showing 14 changed files with 187 additions and 47 deletions.
131 changes: 95 additions & 36 deletions .github/workflows/ci.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -44,28 +44,45 @@ jobs:
# - name: 'Micro Benchmarks'
# task: 'micro-benchmarks'
# python-version: '3.6'
# Integration tests are not working yet, still done in Travis
# - name: 'Integration Tests'
# task: 'ci-integration'
- name: 'Integration Tests'
task: 'ci-integration'
python-version: '3.6'
services:
mongo:
image: mongo:4.0
ports:
- 27017:27017
# Can't use RabbitMQ here for Integrations because we rely on custom config
# and SSL certs that are in the repo. In GHA, these services are started first
# before the code is checked out, so this is a non-starter, we need to do it
# manually below (TODO)

# In GHA, these services are started first before the code is checked out.
# We use bitnami images to facilitate reconfiguring RabbitMQ during ci-integration tests.
# We rely on custom config and SSL certs that are in the repo.
# Many images require config in env vars (which we can't change during the test job)
# or they require config in entrypoint args (which we can't override for GHA services)
# bitnami builds ways to get config files from mounted volumes.
rabbitmq:
# use the -management version so it has the management tools installed
image: rabbitmq:3.8-management
image: bitnami/rabbitmq:3.8
volumes:
- /home/runner/rabbitmq_conf:/bitnami/conf # RABBITMQ_MOUNTED_CONF_DIR
env:
# tell bitnami/rabbitmq to enable this by default
RABBITMQ_PLUGINS: rabbitmq_management
RABBITMQ_USERNAME: guest
RABBITMQ_PASSWORD: guest

# These are strictly docker options, not entrypoint args (GHA restriction)
options: >-
--name rabbitmq
ports:
# SSL port
- 5671:5671
# standard port
- 5672:5672
# management port
- 15672:15672
# These 6 ports are exposed by bitnami/rabbitmq (see https://www.rabbitmq.com/networking.html#ports)
# host_port:container_port/protocol
- 5671:5671/tcp # AMQP SSL port
- 5672:5672/tcp # AMQP standard port
- 15672:15672/tcp # Management: HTTP, CLI
#- 15671:15671/tcp # Management: SSL port
#- 25672:25672/tcp # inter-node or CLI
#- 4369:4369/tcp # epmd

env:
TASK: '${{ matrix.task }}'

Expand Down Expand Up @@ -114,14 +131,34 @@ jobs:
uses: actions/setup-python@v2
with:
python-version: '${{ matrix.python-version }}'
- name: Get date components for use in cache-keys
id: date
run: |
echo "::set-output name=year::$(/bin/date -u "+%Y")"
echo "::set-output name=month::$(/bin/date -u "+%m")"
echo "::set-output name=week::$(/bin/date -u "+%U")"
- uses: actions/cache@v2
with:
path: |
~/.cache/pip
virtualenv
key: ${{ runner.os }}-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt') }}
~/virtualenv
# TODO: maybe make the virtualenv a partial cache to exclude st2*?
# !virtualenv/lib/python*/site-packages/st2*
# !virtualenv/bin/st2*
key: ${{ runner.os }}-python-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'test-requirements.txt') }}
restore-keys: |
${{ runner.os }}-python-${{ matrix.python }}-
- uses: actions/cache@v2
with:
path: |
/var/cache/apt/archives/*.deb
/var/cache/apt/archives/partial/*.deb
/var/cache/apt/*.bin
key: ${{ runner.os }}-apt-${{ steps.date.outputs.year }}-${{ steps.date.outputs.week }}
restore-keys: |
${{ runner.os }}-${{ matrix.python }}-
${{ runner.os }}-apt-${{ steps.date.outputs.year }}-
${{ runner.os }}-apt-
- name: Install apt depedencies
run: |
# install dev dependencies for Python YAML and LDAP packages
Expand All @@ -130,9 +167,22 @@ jobs:
sudo apt-get -f -y install libldap2-dev libsasl2-dev libssl-dev libyaml-dev ldap-utils
- name: Install virtualenv
run: |
set -x
# Note: Use the verison of virtualenv pinned in fixed-requirements.txt so we
# only have to update it one place when we change the version
pip install --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
# Note: Use --user to avoid polluting system site-packages (which breaks one of our tests)
# TODO: simplify this once fixed in contrib/runners/python_runner/tests/integration/test_pythonrunner_behavior.py
if [[ ! -f ~/virtualenv/bin/virtualenv ]]; then # use the cached version whenever possible
pip install --user --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
virtualenv --no-download ~/virtualenv
~/virtualenv/bin/pip install --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
# drop the --user install virtualenv to prevent polluting tests
pip freeze --user | xargs pip uninstall -y
fi
mkdir -p ~/.local/bin
ln -s ~/virtualenv/bin/virtualenv ~/.local/bin/virtualenv
which virtualenv
virtualenv --version
- name: Install requirements
run: |
./scripts/travis/install-requirements.sh
Expand All @@ -149,34 +199,43 @@ jobs:
run: |
echo "$ST2_CI_REPO_PATH"
sudo ST2_CI_REPO_PATH="${ST2_CI_REPO_PATH}" scripts/travis/permissions-workaround.sh
- name: Setup RabbitMQ (NOT WORKING YET)
if: "${{ env.TASK == 'ci-integration' }}"
- name: Reconfigure RabbitMQ
if: "${{ env.TASK == 'ci-unit' || env.TASK == 'ci-integration' }}"
# bitnami image allows (see bitnami/rabbitmq readme):
# Here we're copying a rabbitmq.config file which won't do anything.
# We need to switch to custom.conf or advanced.config.
timeout-minutes: 2 # may die if rabbitmq fails to start
run: |
set -x
# Use custom RabbitMQ config which enables SSL / TLS listener on port 5671 with test certs
# Travis runs as the 'travis' user, GitHub actions run as the 'runner' user,
# And the cert filepaths are slightly different between the two.
# Example:
# Travis-CI: /home/travis/build/StackStorm/st2/st2tests/st2tests/fixtures/ssl_certs/ca/ca_certificate_bundle.pem
# GitHub Actions: /home/runner/work/st2/st2/st2tests/st2tests/fixtures/ssl_certs/ca/ca_certificate_bundle.pem
sed -i 's|/home/travis/build/StackStorm|/home/runner/work/st2|g' scripts/travis/rabbitmq.config
# Now that we've manged the config file, install it
sudo cp scripts/travis/rabbitmq.config /etc/rabbitmq/rabbitmq.config
# Install rabbitmq_management RabbitMQ plugin
sudo service rabbitmq-server restart
sleep 5
sudo rabbitmq-plugins enable rabbitmq_management
sudo cp scripts/github/rabbitmq.conf /home/runner/rabbitmq_conf/custom.conf
# The code is checked out after the container is already up, so we don't mount them.
# We copy those certs into the dir that is mounted to /bitnami/conf
sudo cp -r st2tests/st2tests/fixtures/ssl_certs /home/runner/rabbitmq_conf/
# refresh rabbitmq config - based on ENTRYPOINT logic
docker exec rabbitmq bash -c 'cat /bitnami/conf/custom.conf >> /opt/bitnami/rabbitmq/etc/rabbitmq/rabbitmq.conf'
# sleep to prevent interleaved output in GHA logs
docker exec rabbitmq cat /opt/bitnami/rabbitmq/etc/rabbitmq/rabbitmq.conf && sleep 0.1
echo
echo restarting rabbitmq container
docker restart rabbitmq
# wait for rabbitmq container to restart
until [ "$(docker inspect -f {{.State.Running}} rabbitmq)" == "true" ]; do sleep 0.1; done
echo enabled RabbitMQ plugins:
# print plugins list to: (1) ease debugging, (2) pause till rabbitmq is really running
docker exec rabbitmq rabbitmq-plugins list -e
echo
sudo wget http://guest:guest@localhost:15672/cli/rabbitmqadmin -O /usr/local/bin/rabbitmqadmin
sudo chmod +x /usr/local/bin/rabbitmqadmin
sudo service rabbitmq-server restart
# chmod to make glob work (*.log to avoid log dir)
sudo chmod a+rx /var/log/rabbitmq
sudo tail -n 30 /var/log/rabbitmq/*.log
# print logs from stdout (RABBITMQ_LOGS=-)
docker logs --tail=20 rabbitmq
- name: Print versions
run: |
# Print various binary versions
git --version
pip --version
pip list
virtualenv --version
# Print out various environment variables info
make play
- name: make
Expand Down
6 changes: 6 additions & 0 deletions .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,12 @@ cache:
#- .tox/

install:
# This triggers the same behavior on travis as we found on github because it installs six in the system-site-packages
#- /opt/python/3.6.7/bin/pip install --upgrade --force-reinstall $(grep "^virtualenv" fixed-requirements.txt)
#- mkdir -p ~/.local/bin
#- ln -s /opt/python/3.6.7/bin/virtualenv ~/.local/bin/virtualenv
- which virtualenv
- virtualenv --version
- ./scripts/travis/install-requirements.sh
# prep a travis-specific dev conf file that uses travis instead of stanley
- cp conf/st2.dev.conf "${ST2_CONF}" ; sed -i -e "s/stanley/${ST2_CI_USER}/" "${ST2_CONF}"
Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -674,7 +674,7 @@ check-dependency-conflicts:
@echo
# Verify there are no conflicting dependencies
cat st2*/requirements.txt contrib/runners/*/requirements.txt | sort -u > req.txt && \
$(VIRTUALENV_DIR)/bin/pip-compile req.txt; \
$(VIRTUALENV_DIR)/bin/pip-compile req.txt || exit 1; \
if [[ -e req.txt ]]; then rm req.txt; fi

.PHONY: virtualenv
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -359,6 +359,12 @@ def _get_action_instance(self):

stdin_data = sys.stdin.readline().strip()

if not stdin_data:
# This could indicate that parent process (e.g. process which runs the tests has
# incorrectly opened the stdin and that one is then inherited by the process which is
# spawning it which will cause issues)
raise ValueError("Received no valid parameters data from sys.stdin")

try:
stdin_parameters = orjson.loads(stdin_data)
stdin_parameters = stdin_parameters.get("parameters", {})
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -142,14 +142,21 @@ def test_stdin_params_timeout_no_stdin_data_provided(self):
"python %s --pack=dummy --file-path=%s --config='%s' "
"--stdin-parameters" % (WRAPPER_SCRIPT_PATH, file_path, config)
)
exit_code, stdout, stderr = run_command(command_string, shell=True)
exit_code, stdout, stderr = run_command(
command_string, shell=True, close_fds=True
)

expected_msg = (
# Depending on how tests are spawned, sys.stdin may be opened and this will cause issues
# with this tests so we simply check for two different errors which are considered
# acceptable.
expected_msg_1 = (
"ValueError: No input received and timed out while waiting for parameters "
"from stdin"
)
expected_msg_2 = "ValueError: Received no valid parameters data from sys.stdin"

self.assertEqual(exit_code, 1)
self.assertIn(expected_msg, stderr)
self.assertTrue(expected_msg_1 in stderr or expected_msg_2 in stderr)

def test_stdin_params_invalid_format_friendly_error(self):
config = {}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,12 @@


class PythonRunnerBehaviorTestCase(CleanFilesTestCase, CleanDbTestCase):

# If you need these logs, then you probably also want to uncomment
# extra debug log messages in st2common/st2common/util/virtualenvs.py
# and pass --logging-level=DEBUG to nosetests
# DISPLAY_LOG_MESSAGES = True

def setUp(self):
super(PythonRunnerBehaviorTestCase, self).setUp()
config.parse_args()
Expand Down Expand Up @@ -74,6 +80,15 @@ def test_priority_of_loading_library_after_setup_pack_virtualenv(self):
(_, output, _) = self._run_action(
pack_name, "get_library_path.py", {"module": "six"}
)
# FIXME: This test fails if system site-packages has six because
# it won't get installed in the virtualenv (w/ --system-site-packages)
# system site-packages is never from a virtualenv.
# Travis has python installed in /opt/python/3.6.7
# with a no-system-site-packages virtualenv at /home/travis/virtualenv/python3.6.7
# GitHub Actions python is in /opt/hostedtoolcache/Python/3.6.13/x64/
# But ther isn't a virtualenv, so when we pip installed `virtualenv`,
# (which depends on, and therefore installs `six`)
# we installed it in system-site-packages not an intermediate virtualenv
self.assertEqual(output["result"].find(self.virtualenvs_path), 0)

# Conversely, this expects that 'mock' module file-path is not under sandbox library,
Expand Down
11 changes: 11 additions & 0 deletions scripts/github/rabbitmq.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# bitnami/rabbitmq configuration file (gets merged with rabbitmq.conf)
listeners.ssl.default = 5671
# /bitnami/conf is a directory mounted into the bitnami/rabbitmq container
ssl_options.cacertfile = /bitnami/conf/ssl_certs/ca/ca_certificate_bundle.pem
ssl_options.certfile = /bitnami/conf/ssl_certs/server/server_certificate.pem
ssl_options.keyfile = /bitnami/conf/ssl_certs/server/private_key.pem
ssl_options.verify = verify_peer
ssl_options.fail_if_no_peer_cert = false

# this is "insecure" but it doesn't matter for CI, and it simplifies integration test machinery
loopback_users = none
9 changes: 9 additions & 0 deletions scripts/travis/prepare-integration.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,9 @@ st2 --version
# Clean up old screen log files
rm -f logs/screen-*.log

# ::group::/::endgroup:: is helpful github actions syntax to fold this section.
echo ::group::launchdev.sh start -x

# start dev environment in screens
./tools/launchdev.sh start -x

Expand All @@ -28,6 +31,9 @@ echo " === START: Catting screen process log files. ==="
cat logs/screen-*.log
echo " === END: Catting screen process log files. ==="

# github actions: fold for launchdev.sh start -x
echo ::endgroup::

# Setup the virtualenv for the examples pack which is required for orquesta integration tests.
st2 run packs.setup_virtualenv packs=examples

Expand All @@ -39,3 +45,6 @@ chmod 777 logs/*
# root needs to access write some lock files when creating virtualenvs
# o=other; X=only set execute bit if user execute bit is set (eg on dirs)
chmod -R o+rwX ./virtualenv/
# newer virtualenv versions are putting lock files under ~/.local
# as this script runs with sudo, HOME is actually the CI user's home
chmod -R o+rwX ${HOME}/.local/share/virtualenv
7 changes: 6 additions & 1 deletion st2common/st2common/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -393,8 +393,13 @@ def register_opts(ignore_errors=False):

# Runner options
default_python_bin_path = sys.executable
base_dir = os.path.dirname(os.path.realpath(default_python_bin_path))
# If the virtualenv uses a symlinked python, then try using virtualenv from that venv
# first before looking for virtualenv installed in python's system-site-packages.
base_dir = os.path.dirname(default_python_bin_path)
default_virtualenv_bin_path = os.path.join(base_dir, "virtualenv")
if not os.path.exists(default_virtualenv_bin_path):
base_dir = os.path.dirname(os.path.realpath(default_python_bin_path))
default_virtualenv_bin_path = os.path.join(base_dir, "virtualenv")

action_runner_opts = [
# Common runner options
Expand Down
9 changes: 9 additions & 0 deletions st2common/st2common/util/shell.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ def run_command(
shell=False,
cwd=None,
env=None,
close_fds=None,
):
"""
Run the provided command in a subprocess and wait until it completes.
Expand All @@ -73,6 +74,9 @@ def run_command(
environment from the current process is inherited.
:type env: ``dict``
:param close_fds: True to close all the fds. By default when None is provided we rely on
default upstream behavior which may be Python version specific.
:rtype: ``tuple`` (exit_code, stdout, stderr)
"""
if not isinstance(cmd, (list, tuple) + six.string_types):
Expand All @@ -83,6 +87,10 @@ def run_command(
if not env:
env = os.environ.copy()

kwargs = {}
if close_fds is not None:
kwargs["close_fds"] = close_fds

process = concurrency.subprocess_popen(
args=cmd,
stdin=stdin,
Expand All @@ -91,6 +99,7 @@ def run_command(
env=env,
cwd=cwd,
shell=shell,
**kwargs,
)
stdout, stderr = process.communicate()
exit_code = process.returncode
Expand Down
8 changes: 8 additions & 0 deletions st2common/st2common/util/virtualenvs.py
Original file line number Diff line number Diff line change
Expand Up @@ -284,6 +284,10 @@ def install_requirements(
)
exit_code, stdout, stderr = run_command(cmd=cmd, env=env)

# Normally we don't want this, even in debug logs. But it is useful to
# investigate pip behavior changes & broken virtualenv integration tests.
# logger.debug(f"\npip stdout=\n{stdout}")

if exit_code != 0:
stdout = to_ascii(stdout)
stderr = to_ascii(stderr)
Expand Down Expand Up @@ -330,6 +334,10 @@ def install_requirement(virtualenv_path, requirement, proxy_config=None, logger=
)
exit_code, stdout, stderr = run_command(cmd=cmd, env=env)

# Normally we don't want this, even in debug logs. But it is useful to
# investigate pip behavior changes & broken virtualenv integration tests.
# logger.debug(f"\npip stdout=\n{stdout}")

if exit_code != 0:
raise Exception(
'Failed to install requirement "%s": %s' % (requirement, stdout)
Expand Down
Loading

0 comments on commit 007beed

Please sign in to comment.