Merge pull request #462 from GitGuardian/agateau/test-github-actions

Test our GitHub actions

.github/workflows/main.yml

@@ -152,29 +152,48 @@ jobs:
             dist
             packages
 
-  scanning:
-    name: Test GitHub secret action
+  test_github_secret_scan_action:
+    name: Test GitHub action for `secret scan`
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
-      - name: Scan commits
-        uses: ./actions/secret
+      - name: Scan commits for hardcoded secrets
+        uses: ./actions-unstable/secret
         env:
           GITHUB_PUSH_BEFORE_SHA: ${{ github.event.before }}
           GITHUB_PUSH_BASE_SHA: ${{ github.event.base }}
           GITHUB_PULL_BASE_SHA: ${{ github.event.pull_request.base.sha }}
           GITHUB_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
           GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }}
           GITGUARDIAN_API_URL: ${{ secrets.GITGUARDIAN_API_URL }}
+          GITGUARDIAN_GGSHIELD_REF: ${{ github.ref }}
+
+  test_github_iac_scan_action:
+    name: Test GitHub action for `iac scan`
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Scan commits for IaC vulnerabilities
+        uses: ./actions-unstable/iac
+        with:
+          args: .
+        env:
+          GITGUARDIAN_API_KEY: ${{ secrets.GITGUARDIAN_API_KEY }}
+          GITGUARDIAN_API_URL: ${{ secrets.GITGUARDIAN_API_URL }}
+          GITGUARDIAN_GGSHIELD_REF: ${{ github.ref }}
 
   dockerhub-unstable:
     name: Push Docker image to Docker Hub
     runs-on: ubuntu-latest
     if: github.ref == 'refs/heads/main' && github.event_name == 'push'
-    needs: [lint, build, scanning]
+    needs:
+      [lint, build, test_github_iac_scan_action, test_github_secret_scan_action]
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -190,7 +209,8 @@ jobs:
     name: Push Docker image to GitHub Packages
     runs-on: ubuntu-latest
     if: github.ref == 'refs/heads/main' && github.event_name == 'push'
-    needs: [lint, build, scanning]
+    needs:
+      [lint, build, test_github_iac_scan_action, test_github_secret_scan_action]
     steps:
       - name: Check out the repo
         uses: actions/checkout@v2

Dockerfile

@@ -31,6 +31,7 @@ RUN set -ex; \
     mkdir /data; chmod 777 /data
 
 USER app
+RUN git config --global --add safe.directory /data
 
 WORKDIR /data
 VOLUME [ "/data" ]

actions-unstable/README.md

@@ -0,0 +1,8 @@
+This directory contains GitHub actions used internally by our CI to simulate the "real" GitHub actions (the ones defined in the `actions` directory) using the latest versions of GGShield.
+
+They differ from the real GitHub actions by:
+
+- Using the `gitguardian/ggshield:unstable` Docker image instead of `gitguardian/ggshield:latest`.
+- Having the possibility to override the installed GGShield version.
+
+These actions are not meant to be used outside of GGShield CI.

actions-unstable/iac/Dockerfile

@@ -0,0 +1,5 @@
+FROM gitguardian/ggshield:unstable
+
+COPY entrypoint.sh /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

actions-unstable/iac/action.yml

@@ -0,0 +1,26 @@
+name: 'ggshield iac scan'
+description: 'Scan commits for iac vulnerabilities'
+author: GitGuardian <[email protected]>
+
+inputs:
+  args:
+    description: |
+      Arguments to be passed to ggshield iac scan
+      Options:
+        --exit-zero                     Always return a 0 (non-error) status code, even if issues
+                                        are found. The env var GITGUARDIAN_EXIT_ZERO can also be used
+                                        to set this option.
+        --minimum-severity              [LOW|MEDIUM|HIGH|CRITICAL]
+                                        Minimum severity of the policies
+        --ignore-policy, --ipo TEXT     Policies to exclude from the results.
+        --ignore-path, --ipa PATH       Do not scan the specified paths.
+        --json                          JSON output.
+    required: false
+branding:
+  icon: 'shield'
+  color: 'blue'
+runs:
+  using: 'docker'
+  image: 'Dockerfile'
+  args:
+    - ${{ inputs.args }}

actions-unstable/iac/entrypoint.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+set -euo pipefail
+# AUTOGENERATED FILE, DO NOT EDIT!
+# This file has been generated by the `action-entrypoint-generator` script
+# defined in `scripts/action-entrypoint-generator`. To make changes to this
+# file, modify the script and rerun it.
+
+# GitHub overrides $HOME. Set it back to the home directory of our `app` user,
+# otherwise the call to `git config` fails.
+export HOME=/home/app
+
+# Mark the current directory as safe. If we don't do this, git commands fail
+# because the source in $PWD is owned by a different user than our `app` user.
+git config --global --add safe.directory "$PWD"
+
+progress() {
+    echo -e "\033[34m$*\033[0m"
+}
+
+update_ggshield() {
+    local old_pwd="$PWD"
+
+    cd "$HOME"
+    local venv_dir=$HOME/venv
+
+    progress "Cloning ggshield $GITGUARDIAN_GGSHIELD_REF"
+    git clone --depth 1 https://github.com/gitguardian/ggshield
+    cd ggshield
+    git fetch origin "$GITGUARDIAN_GGSHIELD_REF"
+    git checkout FETCH_HEAD
+
+    progress "Creating venv in $venv_dir"
+    python -m venv "$venv_dir"
+    . "$venv_dir/bin/activate"
+
+    progress "Installing in venv"
+    pip install .
+
+    cd "$old_pwd"
+
+    progress "ggshield=$(which ggshield)"
+}
+
+if [ -n "${GITGUARDIAN_GGSHIELD_REF:-}" ] ; then
+    update_ggshield
+else
+    progress "Using ggshield from image"
+fi
+
+args=("$@")
+ggshield iac scan -v ${args[@]}

actions-unstable/secret/Dockerfile

@@ -0,0 +1,5 @@
+FROM gitguardian/ggshield:unstable
+
+COPY entrypoint.sh /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

actions-unstable/secret/action.yml

@@ -0,0 +1,26 @@
+name: 'ggshield scan'
+description: 'Scan commits for hardcoded secrets and security policy breaks.'
+author: GitGuardian <[email protected]>
+
+branding:
+  icon: 'shield'
+  color: 'blue'
+
+inputs:
+  args:
+    description: |
+      Arguments to be passed to ggshield secret scan
+      Options:        
+        --json                        Output results in JSON format  [default: False]
+        --show-secrets                Show secrets in plaintext instead of hiding them.
+        --all-policies                Present fails of all policies (Filenames, FileExtensions,
+                                      Secret Detection). By default, only Secret Detection is shown.
+        --exit-zero                   Always return a 0 (non-error) status code, even if incidents are found.
+        -b, --banlist-detector TEXT   Exclude results from a detector.
+    required: false
+
+runs:
+  using: 'docker'
+  image: 'Dockerfile'
+  args:
+    - ${{ inputs.args }}

actions-unstable/secret/entrypoint.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+set -euo pipefail
+# AUTOGENERATED FILE, DO NOT EDIT!
+# This file has been generated by the `action-entrypoint-generator` script
+# defined in `scripts/action-entrypoint-generator`. To make changes to this
+# file, modify the script and rerun it.
+
+# GitHub overrides $HOME. Set it back to the home directory of our `app` user,
+# otherwise the call to `git config` fails.
+export HOME=/home/app
+
+# Mark the current directory as safe. If we don't do this, git commands fail
+# because the source in $PWD is owned by a different user than our `app` user.
+git config --global --add safe.directory "$PWD"
+
+progress() {
+    echo -e "\033[34m$*\033[0m"
+}
+
+update_ggshield() {
+    local old_pwd="$PWD"
+
+    cd "$HOME"
+    local venv_dir=$HOME/venv
+
+    progress "Cloning ggshield $GITGUARDIAN_GGSHIELD_REF"
+    git clone --depth 1 https://github.com/gitguardian/ggshield
+    cd ggshield
+    git fetch origin "$GITGUARDIAN_GGSHIELD_REF"
+    git checkout FETCH_HEAD
+
+    progress "Creating venv in $venv_dir"
+    python -m venv "$venv_dir"
+    . "$venv_dir/bin/activate"
+
+    progress "Installing in venv"
+    pip install .
+
+    cd "$old_pwd"
+
+    progress "ggshield=$(which ggshield)"
+}
+
+if [ -n "${GITGUARDIAN_GGSHIELD_REF:-}" ] ; then
+    update_ggshield
+else
+    progress "Using ggshield from image"
+fi
+
+args=("$@")
+ggshield secret scan -v ${args[@]} ci

actions/iac/entrypoint.sh

@@ -1,4 +1,18 @@
-#! /usr/bin/env bash
+#!/usr/bin/env bash
+set -euo pipefail
+# AUTOGENERATED FILE, DO NOT EDIT!
+# This file has been generated by the `action-entrypoint-generator` script
+# defined in `scripts/action-entrypoint-generator`. To make changes to this
+# file, modify the script and rerun it.
+
+# GitHub overrides $HOME. Set it back to the home directory of our `app` user,
+# otherwise the call to `git config` fails.
+export HOME=/home/app
+
+# Mark the current directory as safe. If we don't do this, git commands fail
+# because the source in $PWD is owned by a different user than our `app` user.
+git config --global --add safe.directory "$PWD"
+
 
 args=("$@")
 ggshield iac scan -v ${args[@]}

actions/secret/entrypoint.sh

@@ -1,4 +1,18 @@
-#! /usr/bin/env bash
+#!/usr/bin/env bash
+set -euo pipefail
+# AUTOGENERATED FILE, DO NOT EDIT!
+# This file has been generated by the `action-entrypoint-generator` script
+# defined in `scripts/action-entrypoint-generator`. To make changes to this
+# file, modify the script and rerun it.
+
+# GitHub overrides $HOME. Set it back to the home directory of our `app` user,
+# otherwise the call to `git config` fails.
+export HOME=/home/app
+
+# Mark the current directory as safe. If we don't do this, git commands fail
+# because the source in $PWD is owned by a different user than our `app` user.
+git config --global --add safe.directory "$PWD"
+
 
 args=("$@")
 ggshield secret scan -v ${args[@]} ci

scripts/README.md

@@ -6,6 +6,18 @@ This directory contains scripts to help with ggshield development.
 
 Build .pyz, .deb and .rpm packages.
 
+## action-entrypoint-generator/action-entrypoint-generator
+
+We have two GitHub actions: one to run `secret scan` and another to run `iac scan`. They are defined in `actions/secret` and `actions/iac`. These two actions use the `gitguardian/ggshield:latest` Docker image defined at the root of the repository, which provides a way to run GGShield inside a Docker container.
+
+To test that changes made inside pull requests do not cause problems with our GitHub actions, we have a separate set of actions: `actions-unstable/secret` and `actions-unstable/iac`. The difference between these actions and those defined in `actions` are the following:
+
+- They use the `gitguardian/ggshield:unstable` image instead of `gitguardian/ggshield:latest` as the base image, so that we test more recent code.
+
+- If the `$GITGUARDIAN_GGSHIELD_REF` environment variable is set, then they install GGShield from this ref, so that we test the code from the pull request and not the code currently in the `main` branch.
+
+Since the Dockerfile syntax does not make it easy to reuse code outside of the directory they are defined in, the `action-entrypoint-generator/action-entrypoint-generator` script generates the four possible `entrypoint.sh` files.
+
 ## push-to-cloudsmith
 
 Publish the .deb and .rpm built by build-packages to Cloudsmith.

scripts/action-entrypoint-generator/action-entrypoint-generator

@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+"""
+Generate the entrypoint.sh files for our GitHub actions.
+"""
+import argparse
+import sys
+from pathlib import Path
+
+
+TEMPLATE = """#!/usr/bin/env bash
+set -euo pipefail
+# AUTOGENERATED FILE, DO NOT EDIT!
+# This file has been generated by the `action-entrypoint-generator` script
+# defined in `scripts/action-entrypoint-generator`. To make changes to this
+# file, modify the script and rerun it.
+
+# GitHub overrides $HOME. Set it back to the home directory of our `app` user,
+# otherwise the call to `git config` fails.
+export HOME=/home/app
+
+# Mark the current directory as safe. If we don't do this, git commands fail
+# because the source in $PWD is owned by a different user than our `app` user.
+git config --global --add safe.directory "$PWD"
+
+@UPDATE_GGSHIELD@
+args=("$@")
+@COMMAND@
+"""
+
+# This script contains the code used by the unstable actions to update ggshield
+UPDATE_GGSHIELD_CODE = (Path(__file__).parent / "update-ggshield.sh").read_text()
+
+COMMAND_FOR_VERTICAL = {
+    "secret": "ggshield secret scan -v ${args[@]} ci",
+    "iac": "ggshield iac scan -v ${args[@]}",
+}
+
+DEFAULT_OUTPUT_DIR = Path(__file__).parent.parent.parent
+
+
+def generate(base_output_dir: Path, stable: bool, vertical: str):
+    dct = {
+        "UPDATE_GGSHIELD": "" if stable else UPDATE_GGSHIELD_CODE,
+        "COMMAND": COMMAND_FOR_VERTICAL[vertical],
+    }
+    content = TEMPLATE
+    for key, value in dct.items():
+        content = content.replace(f"@{key}@", value)
+
+    output_dir = (
+        base_output_dir / ("actions" if stable else "actions-unstable") / vertical
+    )
+    output_dir.mkdir(exist_ok=True, parents=True)
+    output_file = output_dir / "entrypoint.sh"
+    print(f"Creating {output_file}")
+    output_file.write_text(content)
+
+
+def main():
+    assert (DEFAULT_OUTPUT_DIR / "actions").exists()
+
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.RawDescriptionHelpFormatter, description=__doc__
+    )
+
+    parser.add_argument(
+        "-o",
+        "--output",
+        default=DEFAULT_OUTPUT_DIR,
+        help="write actions to OUTPUT_DIR",
+        metavar="OUTPUT_DIR",
+    )
+
+    args = parser.parse_args()
+
+    output_dir = Path(args.output)
+    assert output_dir.is_dir()
+    for stable in False, True:
+        for vertical in COMMAND_FOR_VERTICAL.keys():
+            generate(output_dir, stable=stable, vertical=vertical)
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())