From ab57c502bbf4863f0c9fd0bdc068a99083960297 Mon Sep 17 00:00:00 2001 From: Aurelien Gateau Date: Mon, 30 Jan 2023 18:20:26 +0100 Subject: [PATCH 1/2] chore(ci): Make sure our /data directory can be used as a git repository Without marking the repository safe, git can refuses to work on it because it is not owned by the user running the ggshield command. --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 7e1f16ec31..da10ae3905 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,6 +31,7 @@ RUN set -ex; \ mkdir /data; chmod 777 /data USER app +RUN git config --global --add safe.directory /data WORKDIR /data VOLUME [ "/data" ] From 0d5f6f2e18c3e59e28e5225bdbdeda000b34b33d Mon Sep 17 00:00:00 2001 From: Aurelien Gateau Date: Fri, 27 Jan 2023 11:46:21 +0100 Subject: [PATCH 2/2] test: test our GitHub actions Define separate action directories for testing, because we want to use different Docker images (:unstable instead of :latest) and we want to update ggshield in these images. Add a script to generate the endpoint.sh scripts of the four actions. Fixes #304 --- .github/workflows/main.yml | 32 +++++-- actions-unstable/README.md | 8 ++ actions-unstable/iac/Dockerfile | 5 ++ actions-unstable/iac/action.yml | 26 ++++++ actions-unstable/iac/entrypoint.sh | 51 +++++++++++ actions-unstable/secret/Dockerfile | 5 ++ actions-unstable/secret/action.yml | 26 ++++++ actions-unstable/secret/entrypoint.sh | 51 +++++++++++ actions/iac/entrypoint.sh | 16 +++- actions/secret/entrypoint.sh | 16 +++- scripts/README.md | 12 +++ .../action-entrypoint-generator | 86 +++++++++++++++++++ .../update-ggshield.sh | 33 +++++++ 13 files changed, 359 insertions(+), 8 deletions(-) create mode 100644 actions-unstable/README.md create mode 100644 actions-unstable/iac/Dockerfile create mode 100644 actions-unstable/iac/action.yml create mode 100755 actions-unstable/iac/entrypoint.sh create mode 100644 actions-unstable/secret/Dockerfile create mode 100644 actions-unstable/secret/action.yml create mode 100755 actions-unstable/secret/entrypoint.sh create mode 100755 scripts/action-entrypoint-generator/action-entrypoint-generator create mode 100644 scripts/action-entrypoint-generator/update-ggshield.sh diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4e199c42e3..45033bd61b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -152,16 +152,16 @@ jobs: dist packages - scanning: - name: Test GitHub secret action + test_github_secret_scan_action: + name: Test GitHub action for `secret scan` runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v2 with: fetch-depth: 0 - - name: Scan commits - uses: ./actions/secret + - name: Scan commits for hardcoded secrets + uses: ./actions-unstable/secret env: GITHUB_PUSH_BEFORE_SHA: ${{ github.event.before }} GITHUB_PUSH_BASE_SHA: ${{ github.event.base }} @@ -169,12 +169,31 @@ jobs: GITHUB_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }} GITGUARDIAN_API_URL: ${{ secrets.GITGUARDIAN_API_URL }} + GITGUARDIAN_GGSHIELD_REF: ${{ github.ref }} + + test_github_iac_scan_action: + name: Test GitHub action for `iac scan` + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + fetch-depth: 0 + - name: Scan commits for IaC vulnerabilities + uses: ./actions-unstable/iac + with: + args: . + env: + GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }} + GITGUARDIAN_API_URL: ${{ secrets.GITGUARDIAN_API_URL }} + GITGUARDIAN_GGSHIELD_REF: ${{ github.ref }} dockerhub-unstable: name: Push Docker image to Docker Hub runs-on: ubuntu-latest if: github.ref == 'refs/heads/main' && github.event_name == 'push' - needs: [lint, build, scanning] + needs: + [lint, build, test_github_iac_scan_action, test_github_secret_scan_action] steps: - name: Checkout uses: actions/checkout@v2 @@ -190,7 +209,8 @@ jobs: name: Push Docker image to GitHub Packages runs-on: ubuntu-latest if: github.ref == 'refs/heads/main' && github.event_name == 'push' - needs: [lint, build, scanning] + needs: + [lint, build, test_github_iac_scan_action, test_github_secret_scan_action] steps: - name: Check out the repo uses: actions/checkout@v2 diff --git a/actions-unstable/README.md b/actions-unstable/README.md new file mode 100644 index 0000000000..cee27ee102 --- /dev/null +++ b/actions-unstable/README.md @@ -0,0 +1,8 @@ +This directory contains GitHub actions used internally by our CI to simulate the "real" GitHub actions (the ones defined in the `actions` directory) using the latest versions of GGShield. + +They differ from the real GitHub actions by: + +- Using the `gitguardian/ggshield:unstable` Docker image instead of `gitguardian/ggshield:latest`. +- Having the possibility to override the installed GGShield version. + +These actions are not meant to be used outside of GGShield CI. diff --git a/actions-unstable/iac/Dockerfile b/actions-unstable/iac/Dockerfile new file mode 100644 index 0000000000..e773db16a9 --- /dev/null +++ b/actions-unstable/iac/Dockerfile @@ -0,0 +1,5 @@ +FROM gitguardian/ggshield:unstable + +COPY entrypoint.sh /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/actions-unstable/iac/action.yml b/actions-unstable/iac/action.yml new file mode 100644 index 0000000000..8cc24b3fef --- /dev/null +++ b/actions-unstable/iac/action.yml @@ -0,0 +1,26 @@ +name: 'ggshield iac scan' +description: 'Scan commits for iac vulnerabilities' +author: GitGuardian + +inputs: + args: + description: | + Arguments to be passed to ggshield iac scan + Options: + --exit-zero Always return a 0 (non-error) status code, even if issues + are found. The env var GITGUARDIAN_EXIT_ZERO can also be used + to set this option. + --minimum-severity [LOW|MEDIUM|HIGH|CRITICAL] + Minimum severity of the policies + --ignore-policy, --ipo TEXT Policies to exclude from the results. + --ignore-path, --ipa PATH Do not scan the specified paths. + --json JSON output. + required: false +branding: + icon: 'shield' + color: 'blue' +runs: + using: 'docker' + image: 'Dockerfile' + args: + - ${{ inputs.args }} diff --git a/actions-unstable/iac/entrypoint.sh b/actions-unstable/iac/entrypoint.sh new file mode 100755 index 0000000000..ec306de3b5 --- /dev/null +++ b/actions-unstable/iac/entrypoint.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -euo pipefail +# AUTOGENERATED FILE, DO NOT EDIT! +# This file has been generated by the `action-entrypoint-generator` script +# defined in `scripts/action-entrypoint-generator`. To make changes to this +# file, modify the script and rerun it. + +# GitHub overrides $HOME. Set it back to the home directory of our `app` user, +# otherwise the call to `git config` fails. +export HOME=/home/app + +# Mark the current directory as safe. If we don't do this, git commands fail +# because the source in $PWD is owned by a different user than our `app` user. +git config --global --add safe.directory "$PWD" + +progress() { + echo -e "\033[34m$*\033[0m" +} + +update_ggshield() { + local old_pwd="$PWD" + + cd "$HOME" + local venv_dir=$HOME/venv + + progress "Cloning ggshield $GITGUARDIAN_GGSHIELD_REF" + git clone --depth 1 https://github.com/gitguardian/ggshield + cd ggshield + git fetch origin "$GITGUARDIAN_GGSHIELD_REF" + git checkout FETCH_HEAD + + progress "Creating venv in $venv_dir" + python -m venv "$venv_dir" + . "$venv_dir/bin/activate" + + progress "Installing in venv" + pip install . + + cd "$old_pwd" + + progress "ggshield=$(which ggshield)" +} + +if [ -n "${GITGUARDIAN_GGSHIELD_REF:-}" ] ; then + update_ggshield +else + progress "Using ggshield from image" +fi + +args=("$@") +ggshield iac scan -v ${args[@]} diff --git a/actions-unstable/secret/Dockerfile b/actions-unstable/secret/Dockerfile new file mode 100644 index 0000000000..e773db16a9 --- /dev/null +++ b/actions-unstable/secret/Dockerfile @@ -0,0 +1,5 @@ +FROM gitguardian/ggshield:unstable + +COPY entrypoint.sh /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/actions-unstable/secret/action.yml b/actions-unstable/secret/action.yml new file mode 100644 index 0000000000..12161cea03 --- /dev/null +++ b/actions-unstable/secret/action.yml @@ -0,0 +1,26 @@ +name: 'ggshield scan' +description: 'Scan commits for hardcoded secrets and security policy breaks.' +author: GitGuardian + +branding: + icon: 'shield' + color: 'blue' + +inputs: + args: + description: | + Arguments to be passed to ggshield secret scan + Options: + --json Output results in JSON format [default: False] + --show-secrets Show secrets in plaintext instead of hiding them. + --all-policies Present fails of all policies (Filenames, FileExtensions, + Secret Detection). By default, only Secret Detection is shown. + --exit-zero Always return a 0 (non-error) status code, even if incidents are found. + -b, --banlist-detector TEXT Exclude results from a detector. + required: false + +runs: + using: 'docker' + image: 'Dockerfile' + args: + - ${{ inputs.args }} diff --git a/actions-unstable/secret/entrypoint.sh b/actions-unstable/secret/entrypoint.sh new file mode 100755 index 0000000000..36fd2c346e --- /dev/null +++ b/actions-unstable/secret/entrypoint.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +set -euo pipefail +# AUTOGENERATED FILE, DO NOT EDIT! +# This file has been generated by the `action-entrypoint-generator` script +# defined in `scripts/action-entrypoint-generator`. To make changes to this +# file, modify the script and rerun it. + +# GitHub overrides $HOME. Set it back to the home directory of our `app` user, +# otherwise the call to `git config` fails. +export HOME=/home/app + +# Mark the current directory as safe. If we don't do this, git commands fail +# because the source in $PWD is owned by a different user than our `app` user. +git config --global --add safe.directory "$PWD" + +progress() { + echo -e "\033[34m$*\033[0m" +} + +update_ggshield() { + local old_pwd="$PWD" + + cd "$HOME" + local venv_dir=$HOME/venv + + progress "Cloning ggshield $GITGUARDIAN_GGSHIELD_REF" + git clone --depth 1 https://github.com/gitguardian/ggshield + cd ggshield + git fetch origin "$GITGUARDIAN_GGSHIELD_REF" + git checkout FETCH_HEAD + + progress "Creating venv in $venv_dir" + python -m venv "$venv_dir" + . "$venv_dir/bin/activate" + + progress "Installing in venv" + pip install . + + cd "$old_pwd" + + progress "ggshield=$(which ggshield)" +} + +if [ -n "${GITGUARDIAN_GGSHIELD_REF:-}" ] ; then + update_ggshield +else + progress "Using ggshield from image" +fi + +args=("$@") +ggshield secret scan -v ${args[@]} ci diff --git a/actions/iac/entrypoint.sh b/actions/iac/entrypoint.sh index 48497546d1..31e2106bb2 100755 --- a/actions/iac/entrypoint.sh +++ b/actions/iac/entrypoint.sh @@ -1,4 +1,18 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash +set -euo pipefail +# AUTOGENERATED FILE, DO NOT EDIT! +# This file has been generated by the `action-entrypoint-generator` script +# defined in `scripts/action-entrypoint-generator`. To make changes to this +# file, modify the script and rerun it. + +# GitHub overrides $HOME. Set it back to the home directory of our `app` user, +# otherwise the call to `git config` fails. +export HOME=/home/app + +# Mark the current directory as safe. If we don't do this, git commands fail +# because the source in $PWD is owned by a different user than our `app` user. +git config --global --add safe.directory "$PWD" + args=("$@") ggshield iac scan -v ${args[@]} diff --git a/actions/secret/entrypoint.sh b/actions/secret/entrypoint.sh index 28c396d282..bc1a8f7448 100755 --- a/actions/secret/entrypoint.sh +++ b/actions/secret/entrypoint.sh @@ -1,4 +1,18 @@ -#! /usr/bin/env bash +#!/usr/bin/env bash +set -euo pipefail +# AUTOGENERATED FILE, DO NOT EDIT! +# This file has been generated by the `action-entrypoint-generator` script +# defined in `scripts/action-entrypoint-generator`. To make changes to this +# file, modify the script and rerun it. + +# GitHub overrides $HOME. Set it back to the home directory of our `app` user, +# otherwise the call to `git config` fails. +export HOME=/home/app + +# Mark the current directory as safe. If we don't do this, git commands fail +# because the source in $PWD is owned by a different user than our `app` user. +git config --global --add safe.directory "$PWD" + args=("$@") ggshield secret scan -v ${args[@]} ci diff --git a/scripts/README.md b/scripts/README.md index 7a7e719a03..ee9e88d5af 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -6,6 +6,18 @@ This directory contains scripts to help with ggshield development. Build .pyz, .deb and .rpm packages. +## action-entrypoint-generator/action-entrypoint-generator + +We have two GitHub actions: one to run `secret scan` and another to run `iac scan`. They are defined in `actions/secret` and `actions/iac`. These two actions use the `gitguardian/ggshield:latest` Docker image defined at the root of the repository, which provides a way to run GGShield inside a Docker container. + +To test that changes made inside pull requests do not cause problems with our GitHub actions, we have a separate set of actions: `actions-unstable/secret` and `actions-unstable/iac`. The difference between these actions and those defined in `actions` are the following: + +- They use the `gitguardian/ggshield:unstable` image instead of `gitguardian/ggshield:latest` as the base image, so that we test more recent code. + +- If the `$GITGUARDIAN_GGSHIELD_REF` environment variable is set, then they install GGShield from this ref, so that we test the code from the pull request and not the code currently in the `main` branch. + +Since the Dockerfile syntax does not make it easy to reuse code outside of the directory they are defined in, the `action-entrypoint-generator/action-entrypoint-generator` script generates the four possible `entrypoint.sh` files. + ## push-to-cloudsmith Publish the .deb and .rpm built by build-packages to Cloudsmith. diff --git a/scripts/action-entrypoint-generator/action-entrypoint-generator b/scripts/action-entrypoint-generator/action-entrypoint-generator new file mode 100755 index 0000000000..175f4ba16e --- /dev/null +++ b/scripts/action-entrypoint-generator/action-entrypoint-generator @@ -0,0 +1,86 @@ +#!/usr/bin/env python +""" +Generate the entrypoint.sh files for our GitHub actions. +""" +import argparse +import sys +from pathlib import Path + + +TEMPLATE = """#!/usr/bin/env bash +set -euo pipefail +# AUTOGENERATED FILE, DO NOT EDIT! +# This file has been generated by the `action-entrypoint-generator` script +# defined in `scripts/action-entrypoint-generator`. To make changes to this +# file, modify the script and rerun it. + +# GitHub overrides $HOME. Set it back to the home directory of our `app` user, +# otherwise the call to `git config` fails. +export HOME=/home/app + +# Mark the current directory as safe. If we don't do this, git commands fail +# because the source in $PWD is owned by a different user than our `app` user. +git config --global --add safe.directory "$PWD" + +@UPDATE_GGSHIELD@ +args=("$@") +@COMMAND@ +""" + +# This script contains the code used by the unstable actions to update ggshield +UPDATE_GGSHIELD_CODE = (Path(__file__).parent / "update-ggshield.sh").read_text() + +COMMAND_FOR_VERTICAL = { + "secret": "ggshield secret scan -v ${args[@]} ci", + "iac": "ggshield iac scan -v ${args[@]}", +} + +DEFAULT_OUTPUT_DIR = Path(__file__).parent.parent.parent + + +def generate(base_output_dir: Path, stable: bool, vertical: str): + dct = { + "UPDATE_GGSHIELD": "" if stable else UPDATE_GGSHIELD_CODE, + "COMMAND": COMMAND_FOR_VERTICAL[vertical], + } + content = TEMPLATE + for key, value in dct.items(): + content = content.replace(f"@{key}@", value) + + output_dir = ( + base_output_dir / ("actions" if stable else "actions-unstable") / vertical + ) + output_dir.mkdir(exist_ok=True, parents=True) + output_file = output_dir / "entrypoint.sh" + print(f"Creating {output_file}") + output_file.write_text(content) + + +def main(): + assert (DEFAULT_OUTPUT_DIR / "actions").exists() + + parser = argparse.ArgumentParser( + formatter_class=argparse.RawDescriptionHelpFormatter, description=__doc__ + ) + + parser.add_argument( + "-o", + "--output", + default=DEFAULT_OUTPUT_DIR, + help="write actions to OUTPUT_DIR", + metavar="OUTPUT_DIR", + ) + + args = parser.parse_args() + + output_dir = Path(args.output) + assert output_dir.is_dir() + for stable in False, True: + for vertical in COMMAND_FOR_VERTICAL.keys(): + generate(output_dir, stable=stable, vertical=vertical) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/action-entrypoint-generator/update-ggshield.sh b/scripts/action-entrypoint-generator/update-ggshield.sh new file mode 100644 index 0000000000..21d626ccbd --- /dev/null +++ b/scripts/action-entrypoint-generator/update-ggshield.sh @@ -0,0 +1,33 @@ +progress() { + echo -e "\033[34m$*\033[0m" +} + +update_ggshield() { + local old_pwd="$PWD" + + cd "$HOME" + local venv_dir=$HOME/venv + + progress "Cloning ggshield $GITGUARDIAN_GGSHIELD_REF" + git clone --depth 1 https://github.com/gitguardian/ggshield + cd ggshield + git fetch origin "$GITGUARDIAN_GGSHIELD_REF" + git checkout FETCH_HEAD + + progress "Creating venv in $venv_dir" + python -m venv "$venv_dir" + . "$venv_dir/bin/activate" + + progress "Installing in venv" + pip install . + + cd "$old_pwd" + + progress "ggshield=$(which ggshield)" +} + +if [ -n "${GITGUARDIAN_GGSHIELD_REF:-}" ] ; then + update_ggshield +else + progress "Using ggshield from image" +fi