ci.yml

name: CI

on:
  push:
    branches:
      - main
  workflow_dispatch:
  pull_request:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

env:
  CARGO_INCREMENTAL: 0
  CARGO_NET_RETRY: 10
  RUSTUP_MAX_RETRIES: 10
  RUST_LOG: info
  RUST_BACKTRACE: 1
  RUSTFLAGS: "-D warnings"
  CARGO_TERM_COLOR: always
  CICD_INTERMEDIATES_DIR: "_cicd-intermediates"
  XDG_CACHE_HOME: ${{ github.workspace }}/.cache
  PYTEST_ADDOPTS: "--color=yes"

  #
  # Select a profile that is used for building the binary. The profile optimizes for certain use-cases.
  # For distribution builds we want to reduce the size of the binary as much as possible. Whereas in
  # regular CI builds we just want the fastest build possible.
  #
  # We switch based on the branch that is being built. If it's the main branch or a tag, we use the `dist`.
  #
  # Inspiration was taken from this blog: https://arusahni.net/blog/2020/03/optimizing-rust-binary-size.html
  #
  CARGO_BUILD_PROFILE: ci

jobs:
  # Check if the code has changed in such a way that a rebuild is needed.
  determine_changes:
    name: "determine changes"
    runs-on: ubuntu-latest
    outputs:
      # Flag that is raised when any code is changed
      code: ${{ steps.changed.outputs.code_any_changed }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - uses: tj-actions/changed-files@v45
        id: changed
        with:
          files_yaml: |
            code:
              - "**/*"
              - "!assets/**"
              - "!docs/**"
              - "!install/**"
              - "!assets/**"
              - "!**/*.md"

  pre-commit:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up pixi
        uses: prefix-dev/setup-pixi@v0.8.1
        with:
          environments: lint
      - name: pre-commit
        run: pixi run pre-commit-run --color=always --show-diff-on-failure
        env:
          # As the rust GitHub action is better at the rust jobs it can be skipped in this job.
          SKIP: clippy,fmt

  # Format the all rust code and make sure it's formatted correctly.
  format:
    name: "cargo fmt | ubuntu"
    needs: determine_changes
    runs-on: ubuntu-latest
    if: ${{ needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/main' }}
    steps:
      - uses: actions/checkout@v4
      - name: "Install Rustfmt"
        run: rustup component add rustfmt
      - name: "rustfmt"
        run: cargo fmt --all --check

  # Check that all the code references are correct.
  check-rustdoc-links:
    name: "cargo rustdoc | ubuntu"
    needs: determine_changes
    runs-on: ubuntu-latest
    if: ${{ needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/main' }}
    steps:
      - uses: actions/checkout@v4
      - uses: Swatinem/rust-cache@v2
        with:
          save-if: ${{ github.ref == 'refs/heads/main' }}
      - run: |
          for package in $(cargo metadata --no-deps --format-version=1 | jq -r '.packages[] | .name'); do
            cargo rustdoc -p "$package" --all-features -- -D warnings -W unreachable-pub
          done

  # Run `cargo clippy` on the entire codebase.
  lint:
    name: "cargo clippy | ubuntu"
    needs: determine_changes
    runs-on: ubuntu-latest
    if: ${{ needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/main' }}
    steps:
      - uses: actions/checkout@v4
      - uses: Swatinem/rust-cache@v2
        with:
          save-if: ${{ github.ref == 'refs/heads/main' }}
      - name: "Install Rust toolchain"
        run: rustup component add clippy
      - name: Run clippy
        run: cargo clippy --all-targets --workspace --locked

  # Checks for dependencies that are not used in the codebase
  cargo-machete:
    name: Cargo Machete
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Machete
        uses: bnjbvr/cargo-machete@main


  #
  # Run tests on important platforms.
  #

  cargo-test-linux:
    name: "cargo test | ubuntu"
    timeout-minutes: 15
    needs: determine_changes
    if: ${{ needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/main' }}
    runs-on: 8core_ubuntu_latest_runner
    steps:
      - uses: actions/checkout@v4
      - uses: prefix-dev/setup-pixi@v0.8.1
        with:
          cache: ${{ github.ref == 'refs/heads/main' }}
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: ". -> target/pixi"
          key: ${{ hashFiles('pixi.lock') }}
          save-if: ${{ github.ref == 'refs/heads/main' }}
      - name: Test pixi
        run: pixi run test-slow

  cargo-test-macos-aarch64:
    name: "cargo test | macos aarch64"
    timeout-minutes: 15
    needs: determine_changes
    if: ${{ needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/main' }}
    runs-on: macos-14
    steps:
      - uses: actions/checkout@v4
      - uses: prefix-dev/setup-pixi@v0.8.1
        with:
          cache: ${{ github.ref == 'refs/heads/main' }}
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: ". -> target/pixi"
          key: ${{ hashFiles('pixi.lock') }}
          save-if: ${{ github.ref == 'refs/heads/main' }}
      - name: Test pixi
        run: pixi run test-slow

  cargo-test-macos-x86_64:
    name: "cargo test | macos x86_64"
    timeout-minutes: 15
    needs: determine_changes
    if: ${{ needs.determine_changes.outputs.code == 'true' && github.ref == 'refs/heads/main' }} # Only run on the main branch
    runs-on: macos-13
    steps:
      - uses: actions/checkout@v4
      - uses: prefix-dev/setup-pixi@v0.8.1
        with:
          cache: ${{ github.ref == 'refs/heads/main' }}
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: ". -> target/pixi"
          key: ${{ hashFiles('pixi.lock') }}
          save-if: ${{ github.ref == 'refs/heads/main' }}
      - name: Test pixi
        run: pixi run test-slow

  cargo-test-windows:
    name: "cargo test | windows"
    timeout-minutes: 15
    needs: determine_changes
    if: ${{ needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/main' }}
    runs-on: 16core_windows_latest_runner
    steps:
      - uses: actions/checkout@v4
      - name: Create Dev Drive using ReFS
        run: ${{ github.workspace }}/.github/workflows/setup-dev-drive.ps1
      - uses: prefix-dev/setup-pixi@v0.8.1
        with:
          cache: ${{ github.ref == 'refs/heads/main' }}
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: ". -> target/pixi"
          key: ${{ hashFiles('pixi.lock') }}
          save-if: ${{ github.ref == 'refs/heads/main' }}
      - name: Test pixi
        run: pixi run test-slow

  #
  # Builds the binary artifacts on different platforms
  #

  build-binary-linux-x86_64:
    needs: determine_changes
    if: ${{ needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/main' }}
    runs-on: 8core_ubuntu_latest_runner
    name: "build binary | linux x86_64"
    steps:
        - uses: actions/checkout@v4
        - uses: rui314/setup-mold@v1
        - name: "Setup musl"
          run: |
            sudo apt-get install musl-tools
            rustup target add x86_64-unknown-linux-musl
        - uses: Swatinem/rust-cache@v2
        - name: "Build"
          run: >
            cargo build
            --locked
            --target x86_64-unknown-linux-musl
            --profile $CARGO_BUILD_PROFILE
            --features self_update
        - name: "Upload binary"
          uses: actions/upload-artifact@v4
          with:
            name: pixi-linux-x86_64-${{ github.sha }}
            path: ./target/x86_64-unknown-linux-musl/${{ env.CARGO_BUILD_PROFILE }}/pixi
            retention-days: 1

  build-binary-macos-aarch64:
    needs: determine_changes
    if: ${{ needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/main' }}
    runs-on: macos-14
    name: "build binary | macos aarch64"
    steps:
      - uses: actions/checkout@v4
      - uses: rui314/setup-mold@v1
      - uses: Swatinem/rust-cache@v2
      - name: "Build"
        run: >
          cargo build
          --locked
          --profile $CARGO_BUILD_PROFILE
          --features self_update
      - name: "Upload binary"
        uses: actions/upload-artifact@v4
        with:
          name: pixi-macos-aarch64-${{ github.sha }}
          path: ./target/${{ env.CARGO_BUILD_PROFILE }}/pixi
          retention-days: 1

  build-binary-macos-x86_64:
    needs: determine_changes
    if: ${{ needs.determine_changes.outputs.code == 'true' && github.ref == 'refs/heads/main' }} # Only run on the main branch
    runs-on: macos-13
    name: "build binary | macos x86_64"
    steps:
      - uses: actions/checkout@v4
      - uses: rui314/setup-mold@v1
      - uses: Swatinem/rust-cache@v2
      - name: "Build"
        run: >
          cargo build
          --locked
          --profile $CARGO_BUILD_PROFILE
          --features self_update
      - name: "Upload binary"
        uses: actions/upload-artifact@v4
        with:
          name: pixi-macos-x86_64-${{ github.sha }}
          path: ./target/${{ env.CARGO_BUILD_PROFILE }}/pixi
          retention-days: 1

  build-binary-windows-x86_64:
    needs: determine_changes
    if: ${{ needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/main' }}
    runs-on: 16core_windows_latest_runner
    name: "build binary | windows x86_64"
    steps:
      - uses: actions/checkout@v4
      - name: Create Dev Drive using ReFS
        run: ${{ github.workspace }}/.github/workflows/setup-dev-drive.ps1
      - name: Copy Git Repo to Dev Drive
        run: |
          Copy-Item -Path "${{ github.workspace }}" -Destination "${{ env.PIXI_WORKSPACE }}" -Recurse
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: ${{ env.PIXI_WORKSPACE }}
      - name: "Build"
        working-directory: ${{ env.PIXI_WORKSPACE }}
        run: >
          cargo build
          --locked
          --profile $env:CARGO_BUILD_PROFILE
          --features self_update
      - name: "Upload binary"
        uses: actions/upload-artifact@v4
        with:
          name: pixi-windows-x86_64-${{ github.sha }}
          path: ${{ env.PIXI_WORKSPACE }}/target/${{ env.CARGO_BUILD_PROFILE }}/pixi.exe
          retention-days: 1

  build-binary-windows-aarch64:
    needs: determine_changes
    if: ${{ needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/main' }}
    runs-on: 16core_windows_latest_runner
    name: "build binary | windows aarch64"
    steps:
      - uses: actions/checkout@v4
      - name: Create Dev Drive using ReFS
        run: ${{ github.workspace }}/.github/workflows/setup-dev-drive.ps1
      - name: Copy Git Repo to Dev Drive
        run: |
          Copy-Item -Path "${{ github.workspace }}" -Destination "${{ env.PIXI_WORKSPACE }}" -Recurse
      - name: "Install Rust toolchain"
        run: rustup target add aarch64-pc-windows-msvc
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: ${{ env.PIXI_WORKSPACE }}
      - name: "Build"
        working-directory: ${{ env.PIXI_WORKSPACE }}
        run: >
          cargo build
          --locked
          --target aarch64-pc-windows-msvc
          --profile $env:CARGO_BUILD_PROFILE
          --features self_update
      - name: "Upload binary"
        uses: actions/upload-artifact@v4
        with:
          name: pixi-windows-aarch64-${{ github.sha }}
          path: ${{ env.PIXI_WORKSPACE }}/target/aarch64-pc-windows-msvc/${{ env.CARGO_BUILD_PROFILE }}/pixi.exe
          retention-days: 1

  #
  # Run integration tests on important platforms
  #

  test-integration-windows-x86_64:
    timeout-minutes: 30
    name: pytest integration | windows x86_64
    runs-on: windows-latest
    needs: build-binary-windows-x86_64
    env:
      TARGET_RELEASE: "target/pixi/release"
    steps:
      - uses: actions/checkout@v4
      - name: Create Dev Drive using ReFS
        run: ${{ github.workspace }}/.github/workflows/setup-dev-drive.ps1
      - name: Copy Git Repo to Dev Drive
        run: |
          Copy-Item -Path "${{ github.workspace }}" -Destination "${{ env.PIXI_WORKSPACE }}" -Recurse
          echo "${{ env.PIXI_WORKSPACE }}/${{ env.TARGET_RELEASE }}" | Out-File -Append -Encoding utf8 -FilePath $env:GITHUB_PATH
      - name: Download binary from build
        uses: actions/download-artifact@v4
        with:
          name: pixi-windows-x86_64-${{ github.sha }}
          path: ${{ env.PIXI_WORKSPACE }}/${{ env.TARGET_RELEASE }}
      - name: Verify pixi installation
        run: pixi --version

      - name: Install pixi
        working-directory: ${{ env.PIXI_WORKSPACE }}
        run: pixi install -v

      - name: Run integration tests
        working-directory: ${{ env.PIXI_WORKSPACE }}
        run: pixi run --locked test-integration-ci

      - name: Run long running integration tests
        if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'test:extra_slow') }}
        run: pixi run --locked test-integration-ci -m "extra_slow"

      - name: Test examples
        shell: bash
        working-directory: ${{ env.PIXI_WORKSPACE }}
        run: bash tests/scripts/test-examples.sh

      - name: "Checkout Deltares/Ribasim"
        uses: actions/checkout@v4
        with:
          repository: Deltares/Ribasim
          path: ribasim
      - name: "Copy Deltares/Ribasim to Dev Drive"
        run: Copy-Item -Path "${{ github.workspace }}/ribasim" -Destination "${{ env.PIXI_WORKSPACE }}/ribasim" -Recurse
      - name: "Install Deltares/Ribasim"
        run: pixi install
        working-directory: ${{ env.PIXI_WORKSPACE }}/ribasim

      - name: "Checkout quantco/polarify"
        uses: actions/checkout@v4
        with:
          repository: quantco/polarify
          path: polarify
      - name: "Copy quantco/polarify to Dev Drive"
        run: Copy-Item -Path "${{ github.workspace }}/polarify" -Destination "${{ env.PIXI_WORKSPACE }}/polarify" -Recurse
      - name: "Install quantco/polarify"
        run: pixi install
        working-directory: ${{ env.PIXI_WORKSPACE }}/polarify

  test-integration-macos-aarch64:
    timeout-minutes: 30
    name: pytest integration | macos aarch64
    runs-on: macos-14
    needs: build-binary-macos-aarch64
    env:
      TARGET_RELEASE: "${{ github.workspace }}/target/pixi/release"
    steps:
      - uses: actions/checkout@v4
      - name: Download binary from build
        uses: actions/download-artifact@v4
        with:
          name: pixi-macos-aarch64-${{ github.sha }}
          path: ${{ env.TARGET_RELEASE }}
      - name: Setup unix binary, add to github path
        run: |
          chmod a+x ${{ env.TARGET_RELEASE }}/pixi
          echo "${{ env.TARGET_RELEASE }}" >> $GITHUB_PATH
      - name: Verify pixi installation
        run: pixi --version

      - name: Install pixi
        run: pixi install -v

      - name: Run integration tests
        run: pixi run --locked test-integration-ci

      - name: Run long running integration tests
        if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'test:extra_slow') }}
        run: pixi run --locked test-integration-ci -m "extra_slow"

      - name: Test examples
        run: bash tests/scripts/test-examples.sh

      - name: Test export
        run: pixi run --locked test-export

      - name: "Checkout Deltares/Ribasim"
        uses: actions/checkout@v4
        with:
          repository: Deltares/Ribasim
          path: ribasim
      - name: "Install Deltares/Ribasim"
        run: pixi install
        working-directory: ribasim

      - name: "Checkout quantco/polarify"
        uses: actions/checkout@v4
        with:
          repository: quantco/polarify
          path: polarify
      - name: "Install quantco/polarify"
        run: pixi install
        working-directory: polarify

  test-integration-linux-x86_64:
    timeout-minutes: 30
    name: pytest integration | linux x86_64
    runs-on: 8core_ubuntu_latest_runner
    needs: build-binary-linux-x86_64
    env:
      TARGET_RELEASE: "${{ github.workspace }}/target/pixi/release"
    steps:
      - uses: actions/checkout@v4
      - name: Download binary from build
        uses: actions/download-artifact@v4
        with:
          name: pixi-linux-x86_64-${{ github.sha }}
          path: ${{ env.TARGET_RELEASE }}
      - name: Setup unix binary, add to github path
        run: |
          chmod a+x ${{ env.TARGET_RELEASE }}/pixi
          echo "${{ env.TARGET_RELEASE }}" >> $GITHUB_PATH
      - name: Verify pixi installation
        run: pixi --version

      - name: Install pixi
        run: pixi install -v

      - name: Run integration tests
        run: pixi run --locked test-integration-ci

      - name: Run long running integration tests
        if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'test:extra_slow') }}
        run: pixi run --locked test-integration-ci -m "extra_slow"

      - name: "Test examples"
        run: bash tests/scripts/test-examples.sh

      - name: "Test export"
        run: pixi run --locked test-export

      - name: "Checkout nerfstudio-project/nerfstudio"
        uses: actions/checkout@v4
        with:
          repository: nerfstudio-project/nerfstudio
          path: nerfstudio
      - name: "Install nerfstudio-project/nerfstudio"
        run: pixi install
        working-directory: nerfstudio

      - name: "Checkout Deltares/Ribasim"
        uses: actions/checkout@v4
        with:
          repository: Deltares/Ribasim
          path: ribasim
      - name: "Install Deltares/Ribasim"
        run: pixi install
        working-directory: ribasim

      - name: "Checkout quantco/polarify"
        uses: actions/checkout@v4
        with:
          repository: quantco/polarify
          path: polarify
      - name: "Install quantco/polarify"
        run: pixi install
        working-directory: polarify

  #
  # Install a number of common wheels on some platforms
  #

  test-common-wheels-linux-x86_64:
    name: "test wheel installation | linux x86_64"
    needs:
      - build-binary-linux-x86_64
    uses: ./.github/workflows/test_common_wheels.yml
    with:
        sha: ${{ github.sha }}
        arch: linux-x86_64
        runs-on: 8core_ubuntu_latest_runner

  test-common-wheels-windows-x86_64:
    name: "test wheel installation | windows x86_64"
    needs:
      - build-binary-windows-x86_64
    uses: ./.github/workflows/test_common_wheels.yml
    with:
      sha: ${{ github.sha }}
      arch: windows-x86_64
      runs-on: windows-latest

  test-common-wheels-macos-aarch64:
    name: "test wheel installation | macos aarch64"
    needs:
      - build-binary-macos-aarch64
    uses: ./.github/workflows/test_common_wheels.yml
    with:
      sha: ${{ github.sha }}
      arch: macos-aarch64
      runs-on: macos-14