docs: add pytorch integration guide. #340
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| pull_request: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref_name }}-${{ github.event.pull_request.number || github.sha }} | |
| cancel-in-progress: true | |
| env: | |
| CARGO_INCREMENTAL: 0 | |
| CARGO_NET_RETRY: 10 | |
| RUSTUP_MAX_RETRIES: 10 | |
| RUST_LOG: info | |
| RUST_BACKTRACE: 1 | |
| RUSTFLAGS: "-D warnings" | |
| CARGO_TERM_COLOR: always | |
| CICD_INTERMEDIATES_DIR: "_cicd-intermediates" | |
| XDG_CACHE_HOME: ${{ github.workspace }}/.cache | |
| PYTEST_ADDOPTS: "--color=yes" | |
| # | |
| # Select a profile that is used for building the binary. The profile optimizes for certain use-cases. | |
| # For distribution builds we want to reduce the size of the binary as much as possible. Whereas in | |
| # regular CI builds we just want the fastest build possible. | |
| # | |
| # We switch based on the branch that is being built. If it's the main branch or a tag, we use the `dist`. | |
| # | |
| # Inspiration was taken from this blog: https://arusahni.net/blog/2020/03/optimizing-rust-binary-size.html | |
| # | |
| CARGO_BUILD_PROFILE: ci | |
| jobs: | |
| # Check if the code has changed in such a way that a rebuild is needed. | |
| determine_changes: | |
| name: "determine changes" | |
| runs-on: ubuntu-latest | |
| outputs: | |
| # Flag that is raised when any code is changed | |
| code: ${{ steps.changed.outputs.code_any_changed }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - uses: tj-actions/changed-files@v45 | |
| id: changed | |
| with: | |
| files_yaml: | | |
| code: | |
| - "**/*" | |
| - "!assets/**" | |
| - "!docs/**" | |
| - "!install/**" | |
| - "!assets/**" | |
| - "!**/*.md" | |
| pre-commit: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up pixi | |
| uses: prefix-dev/[email protected] | |
| with: | |
| environments: lint | |
| - name: pre-commit | |
| run: pixi run pre-commit-run --color=always --show-diff-on-failure | |
| env: | |
| # As the rust GitHub action is better at the rust jobs it can be skipped in this job. | |
| SKIP: clippy,fmt | |
| # Format the all rust code and make sure it's formatted correctly. | |
| format: | |
| name: "cargo fmt | ubuntu" | |
| needs: determine_changes | |
| runs-on: ubuntu-latest | |
| if: ${{ needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/main' }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: "Install Rustfmt" | |
| run: rustup component add rustfmt | |
| - name: "rustfmt" | |
| run: cargo fmt --all --check | |
| # Check that all the code references are correct. | |
| check-rustdoc-links: | |
| name: "cargo rustdoc | ubuntu" | |
| needs: determine_changes | |
| runs-on: ubuntu-latest | |
| if: ${{ needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/main' }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| save-if: ${{ github.ref == 'refs/heads/main' }} | |
| - run: | | |
| for package in $(cargo metadata --no-deps --format-version=1 | jq -r '.packages[] | .name'); do | |
| cargo rustdoc -p "$package" --all-features -- -D warnings -W unreachable-pub | |
| done | |
| # Run `cargo clippy` on the entire codebase. | |
| lint: | |
| name: "cargo clippy | ubuntu" | |
| needs: determine_changes | |
| runs-on: ubuntu-latest | |
| if: ${{ needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/main' }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| save-if: ${{ github.ref == 'refs/heads/main' }} | |
| - name: "Install Rust toolchain" | |
| run: rustup component add clippy | |
| - name: Run clippy | |
| run: cargo clippy --all-targets --workspace --locked | |
| # Checks for dependencies that are not used in the codebase | |
| cargo-machete: | |
| name: Cargo Machete | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Machete | |
| uses: bnjbvr/cargo-machete@main | |
| # | |
| # Run tests on important platforms. | |
| # | |
| cargo-test-linux: | |
| name: "cargo test | ubuntu" | |
| timeout-minutes: 15 | |
| needs: determine_changes | |
| if: ${{ needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/main' }} | |
| runs-on: 8core_ubuntu_latest_runner | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: prefix-dev/[email protected] | |
| with: | |
| cache: ${{ github.ref == 'refs/heads/main' }} | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: ". -> target/pixi" | |
| key: ${{ hashFiles('pixi.lock') }} | |
| save-if: ${{ github.ref == 'refs/heads/main' }} | |
| - name: Test pixi | |
| run: pixi run test-slow | |
| cargo-test-macos-aarch64: | |
| name: "cargo test | macos aarch64" | |
| timeout-minutes: 15 | |
| needs: determine_changes | |
| if: ${{ needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/main' }} | |
| runs-on: macos-14 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: prefix-dev/[email protected] | |
| with: | |
| cache: ${{ github.ref == 'refs/heads/main' }} | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: ". -> target/pixi" | |
| key: ${{ hashFiles('pixi.lock') }} | |
| save-if: ${{ github.ref == 'refs/heads/main' }} | |
| - name: Test pixi | |
| run: pixi run test-slow | |
| cargo-test-macos-x86_64: | |
| name: "cargo test | macos x86_64" | |
| timeout-minutes: 15 | |
| needs: determine_changes | |
| if: ${{ needs.determine_changes.outputs.code == 'true' && github.ref == 'refs/heads/main' }} # Only run on the main branch | |
| runs-on: macos-13 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: prefix-dev/[email protected] | |
| with: | |
| cache: ${{ github.ref == 'refs/heads/main' }} | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: ". -> target/pixi" | |
| key: ${{ hashFiles('pixi.lock') }} | |
| save-if: ${{ github.ref == 'refs/heads/main' }} | |
| - name: Test pixi | |
| run: pixi run test-slow | |
| cargo-test-windows: | |
| name: "cargo test | windows" | |
| timeout-minutes: 15 | |
| needs: determine_changes | |
| if: ${{ needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/main' }} | |
| runs-on: 16core_windows_latest_runner | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Create Dev Drive using ReFS | |
| run: ${{ github.workspace }}/.github/workflows/setup-dev-drive.ps1 | |
| - uses: prefix-dev/[email protected] | |
| with: | |
| cache: ${{ github.ref == 'refs/heads/main' }} | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: ". -> target/pixi" | |
| key: ${{ hashFiles('pixi.lock') }} | |
| save-if: ${{ github.ref == 'refs/heads/main' }} | |
| - name: Test pixi | |
| run: pixi run test-slow | |
| # | |
| # Builds the binary artifacts on different platforms | |
| # | |
| build-binary-linux-x86_64: | |
| needs: determine_changes | |
| if: ${{ needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/main' }} | |
| runs-on: 8core_ubuntu_latest_runner | |
| name: "build binary | linux x86_64" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: rui314/setup-mold@v1 | |
| - name: "Setup musl" | |
| run: | | |
| sudo apt-get install musl-tools | |
| rustup target add x86_64-unknown-linux-musl | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: "Build" | |
| run: > | |
| cargo build | |
| --locked | |
| --target x86_64-unknown-linux-musl | |
| --profile $CARGO_BUILD_PROFILE | |
| --features self_update | |
| - name: "Upload binary" | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: pixi-linux-x86_64-${{ github.sha }} | |
| path: ./target/x86_64-unknown-linux-musl/${{ env.CARGO_BUILD_PROFILE }}/pixi | |
| retention-days: 1 | |
| build-binary-macos-aarch64: | |
| needs: determine_changes | |
| if: ${{ needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/main' }} | |
| runs-on: macos-14 | |
| name: "build binary | macos aarch64" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: rui314/setup-mold@v1 | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: "Build" | |
| run: > | |
| cargo build | |
| --locked | |
| --profile $CARGO_BUILD_PROFILE | |
| --features self_update | |
| - name: "Upload binary" | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: pixi-macos-aarch64-${{ github.sha }} | |
| path: ./target/${{ env.CARGO_BUILD_PROFILE }}/pixi | |
| retention-days: 1 | |
| build-binary-macos-x86_64: | |
| needs: determine_changes | |
| if: ${{ needs.determine_changes.outputs.code == 'true' && github.ref == 'refs/heads/main' }} # Only run on the main branch | |
| runs-on: macos-13 | |
| name: "build binary | macos x86_64" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: rui314/setup-mold@v1 | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: "Build" | |
| run: > | |
| cargo build | |
| --locked | |
| --profile $CARGO_BUILD_PROFILE | |
| --features self_update | |
| - name: "Upload binary" | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: pixi-macos-x86_64-${{ github.sha }} | |
| path: ./target/${{ env.CARGO_BUILD_PROFILE }}/pixi | |
| retention-days: 1 | |
| build-binary-windows-x86_64: | |
| needs: determine_changes | |
| if: ${{ needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/main' }} | |
| runs-on: 16core_windows_latest_runner | |
| name: "build binary | windows x86_64" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Create Dev Drive using ReFS | |
| run: ${{ github.workspace }}/.github/workflows/setup-dev-drive.ps1 | |
| - name: Copy Git Repo to Dev Drive | |
| run: | | |
| Copy-Item -Path "${{ github.workspace }}" -Destination "${{ env.PIXI_WORKSPACE }}" -Recurse | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: ${{ env.PIXI_WORKSPACE }} | |
| - name: "Build" | |
| working-directory: ${{ env.PIXI_WORKSPACE }} | |
| run: > | |
| cargo build | |
| --locked | |
| --profile $env:CARGO_BUILD_PROFILE | |
| --features self_update | |
| - name: "Upload binary" | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: pixi-windows-x86_64-${{ github.sha }} | |
| path: ${{ env.PIXI_WORKSPACE }}/target/${{ env.CARGO_BUILD_PROFILE }}/pixi.exe | |
| retention-days: 1 | |
| build-binary-windows-aarch64: | |
| needs: determine_changes | |
| if: ${{ needs.determine_changes.outputs.code == 'true' || github.ref == 'refs/heads/main' }} | |
| runs-on: 16core_windows_latest_runner | |
| name: "build binary | windows aarch64" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Create Dev Drive using ReFS | |
| run: ${{ github.workspace }}/.github/workflows/setup-dev-drive.ps1 | |
| - name: Copy Git Repo to Dev Drive | |
| run: | | |
| Copy-Item -Path "${{ github.workspace }}" -Destination "${{ env.PIXI_WORKSPACE }}" -Recurse | |
| - name: "Install Rust toolchain" | |
| run: rustup target add aarch64-pc-windows-msvc | |
| - uses: Swatinem/rust-cache@v2 | |
| with: | |
| workspaces: ${{ env.PIXI_WORKSPACE }} | |
| - name: "Build" | |
| working-directory: ${{ env.PIXI_WORKSPACE }} | |
| run: > | |
| cargo build | |
| --locked | |
| --target aarch64-pc-windows-msvc | |
| --profile $env:CARGO_BUILD_PROFILE | |
| --features self_update | |
| - name: "Upload binary" | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: pixi-windows-aarch64-${{ github.sha }} | |
| path: ${{ env.PIXI_WORKSPACE }}/target/aarch64-pc-windows-msvc/${{ env.CARGO_BUILD_PROFILE }}/pixi.exe | |
| retention-days: 1 | |
| # | |
| # Run integration tests on important platforms | |
| # | |
| test-integration-windows-x86_64: | |
| timeout-minutes: 30 | |
| name: pytest integration | windows x86_64 | |
| runs-on: windows-latest | |
| needs: build-binary-windows-x86_64 | |
| env: | |
| TARGET_RELEASE: "target/pixi/release" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Create Dev Drive using ReFS | |
| run: ${{ github.workspace }}/.github/workflows/setup-dev-drive.ps1 | |
| - name: Copy Git Repo to Dev Drive | |
| run: | | |
| Copy-Item -Path "${{ github.workspace }}" -Destination "${{ env.PIXI_WORKSPACE }}" -Recurse | |
| echo "${{ env.PIXI_WORKSPACE }}/${{ env.TARGET_RELEASE }}" | Out-File -Append -Encoding utf8 -FilePath $env:GITHUB_PATH | |
| - name: Download binary from build | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: pixi-windows-x86_64-${{ github.sha }} | |
| path: ${{ env.PIXI_WORKSPACE }}/${{ env.TARGET_RELEASE }} | |
| - name: Verify pixi installation | |
| run: pixi --version | |
| - name: Install pixi | |
| working-directory: ${{ env.PIXI_WORKSPACE }} | |
| run: pixi install -v | |
| - name: Run integration tests | |
| working-directory: ${{ env.PIXI_WORKSPACE }} | |
| run: pixi run --locked test-integration-ci | |
| - name: Run long running integration tests | |
| if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'test:extra_slow') }} | |
| run: pixi run --locked test-integration-ci -m "extra_slow" | |
| - name: Test examples | |
| shell: bash | |
| working-directory: ${{ env.PIXI_WORKSPACE }} | |
| run: bash tests/scripts/test-examples.sh | |
| - name: "Checkout Deltares/Ribasim" | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: Deltares/Ribasim | |
| path: ribasim | |
| - name: "Copy Deltares/Ribasim to Dev Drive" | |
| run: Copy-Item -Path "${{ github.workspace }}/ribasim" -Destination "${{ env.PIXI_WORKSPACE }}/ribasim" -Recurse | |
| - name: "Install Deltares/Ribasim" | |
| run: pixi install | |
| working-directory: ${{ env.PIXI_WORKSPACE }}/ribasim | |
| - name: "Checkout quantco/polarify" | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: quantco/polarify | |
| path: polarify | |
| - name: "Copy quantco/polarify to Dev Drive" | |
| run: Copy-Item -Path "${{ github.workspace }}/polarify" -Destination "${{ env.PIXI_WORKSPACE }}/polarify" -Recurse | |
| - name: "Install quantco/polarify" | |
| run: pixi install | |
| working-directory: ${{ env.PIXI_WORKSPACE }}/polarify | |
| test-integration-macos-aarch64: | |
| timeout-minutes: 30 | |
| name: pytest integration | macos aarch64 | |
| runs-on: macos-14 | |
| needs: build-binary-macos-aarch64 | |
| env: | |
| TARGET_RELEASE: "${{ github.workspace }}/target/pixi/release" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download binary from build | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: pixi-macos-aarch64-${{ github.sha }} | |
| path: ${{ env.TARGET_RELEASE }} | |
| - name: Setup unix binary, add to github path | |
| run: | | |
| chmod a+x ${{ env.TARGET_RELEASE }}/pixi | |
| echo "${{ env.TARGET_RELEASE }}" >> $GITHUB_PATH | |
| - name: Verify pixi installation | |
| run: pixi --version | |
| - name: Install pixi | |
| run: pixi install -v | |
| - name: Run integration tests | |
| run: pixi run --locked test-integration-ci | |
| - name: Run long running integration tests | |
| if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'test:extra_slow') }} | |
| run: pixi run --locked test-integration-ci -m "extra_slow" | |
| - name: Test examples | |
| run: bash tests/scripts/test-examples.sh | |
| - name: Test export | |
| run: pixi run --locked test-export | |
| - name: "Checkout Deltares/Ribasim" | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: Deltares/Ribasim | |
| path: ribasim | |
| - name: "Install Deltares/Ribasim" | |
| run: pixi install | |
| working-directory: ribasim | |
| - name: "Checkout quantco/polarify" | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: quantco/polarify | |
| path: polarify | |
| - name: "Install quantco/polarify" | |
| run: pixi install | |
| working-directory: polarify | |
| test-integration-linux-x86_64: | |
| timeout-minutes: 30 | |
| name: pytest integration | linux x86_64 | |
| runs-on: 8core_ubuntu_latest_runner | |
| needs: build-binary-linux-x86_64 | |
| env: | |
| TARGET_RELEASE: "${{ github.workspace }}/target/pixi/release" | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Download binary from build | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: pixi-linux-x86_64-${{ github.sha }} | |
| path: ${{ env.TARGET_RELEASE }} | |
| - name: Setup unix binary, add to github path | |
| run: | | |
| chmod a+x ${{ env.TARGET_RELEASE }}/pixi | |
| echo "${{ env.TARGET_RELEASE }}" >> $GITHUB_PATH | |
| - name: Verify pixi installation | |
| run: pixi --version | |
| - name: Install pixi | |
| run: pixi install -v | |
| - name: Run integration tests | |
| run: pixi run --locked test-integration-ci | |
| - name: Run long running integration tests | |
| if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'test:extra_slow') }} | |
| run: pixi run --locked test-integration-ci -m "extra_slow" | |
| - name: "Test examples" | |
| run: bash tests/scripts/test-examples.sh | |
| - name: "Test export" | |
| run: pixi run --locked test-export | |
| - name: "Checkout nerfstudio-project/nerfstudio" | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: nerfstudio-project/nerfstudio | |
| path: nerfstudio | |
| - name: "Install nerfstudio-project/nerfstudio" | |
| run: pixi install | |
| working-directory: nerfstudio | |
| - name: "Checkout Deltares/Ribasim" | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: Deltares/Ribasim | |
| path: ribasim | |
| - name: "Install Deltares/Ribasim" | |
| run: pixi install | |
| working-directory: ribasim | |
| - name: "Checkout quantco/polarify" | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: quantco/polarify | |
| path: polarify | |
| - name: "Install quantco/polarify" | |
| run: pixi install | |
| working-directory: polarify | |
| # | |
| # Install a number of common wheels on some platforms | |
| # | |
| test-common-wheels-linux-x86_64: | |
| name: "test wheel installation | linux x86_64" | |
| needs: | |
| - build-binary-linux-x86_64 | |
| uses: ./.github/workflows/test_common_wheels.yml | |
| with: | |
| sha: ${{ github.sha }} | |
| arch: linux-x86_64 | |
| runs-on: 8core_ubuntu_latest_runner | |
| test-common-wheels-windows-x86_64: | |
| name: "test wheel installation | windows x86_64" | |
| needs: | |
| - build-binary-windows-x86_64 | |
| uses: ./.github/workflows/test_common_wheels.yml | |
| with: | |
| sha: ${{ github.sha }} | |
| arch: windows-x86_64 | |
| runs-on: windows-latest | |
| test-common-wheels-macos-aarch64: | |
| name: "test wheel installation | macos aarch64" | |
| needs: | |
| - build-binary-macos-aarch64 | |
| uses: ./.github/workflows/test_common_wheels.yml | |
| with: | |
| sha: ${{ github.sha }} | |
| arch: macos-aarch64 | |
| runs-on: macos-14 |