ENH: Autoupdate hooks [pre-commit.ci] #1742
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: HI-ML HI-ML-Azure PR | |
| on: | |
| push: | |
| branches: [ main ] | |
| tags: | |
| - '*' | |
| pull_request: | |
| branches: [ main ] | |
| paths: | |
| - "hi-ml-azure/**" | |
| - "hi-ml/**" | |
| - ".github/workflows/hi-ml-pr.yml" | |
| # Cancel previous runs of this workflow that are still in progress. | |
| concurrency: | |
| group: ${{ github.ref }}/hi-ml-pr | |
| cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} | |
| permissions: | |
| # This is required for requesting the Azure login token | |
| id-token: write | |
| # This is required for actions/checkout | |
| contents: read | |
| env: | |
| pythonVersion: 3.9 | |
| HIML_TENANT_ID: ${{ secrets.HIML_TENANT_ID }} | |
| HIML_RESOURCE_GROUP: ${{ secrets.HIML_RESOURCE_GROUP }} | |
| HIML_SUBSCRIPTION_ID: ${{ secrets.HIML_SUBSCRIPTION_ID }} | |
| HIML_WORKSPACE_NAME: ${{ secrets.HIML_WORKSPACE_NAME }} | |
| HIML_SERVICE_PRINCIPAL_ID: ${{ secrets.HIML_SERVICE_PRINCIPAL_ID }} | |
| HIML_DIST_ARTIFACT_SUFFIX: '-dist' | |
| HIML_PACKAGE_NAME_ARTIFACT_SUFFIX: '-package_name' | |
| HIML_VERSION_ARTIFACT_SUFFIX: '-latest_version' | |
| # Set the AML experiment name for all AML jobs submitted during tests. Github.ref looks like | |
| # "refs/pull/123/merge" for PR builds. | |
| HIML_EXPERIMENT_NAME: ${{ github.ref }} | |
| PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} | |
| PYPI_TEST_API_TOKEN: ${{ secrets.PYPI_TEST_API_TOKEN }} | |
| jobs: | |
| cancel-azureml: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Azure login using OpenID Connect | |
| uses: ./.github/actions/azure-login | |
| - name: Cancel previous AzureML runs | |
| uses: ./.github/actions/cancel_azureml_jobs | |
| himl-flake8: | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| lfs: true | |
| - name: Set up Python ${{ env.pythonVersion }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.pythonVersion }} | |
| - name: flake8 | |
| run: | | |
| make pip_test | |
| make flake8 | |
| himl-mypy: | |
| runs-on: ubuntu-20.04 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| lfs: true | |
| - name: Set up Python ${{ env.pythonVersion }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.pythonVersion }} | |
| - name: mypy | |
| run: | | |
| make pip_test | |
| make mypy | |
| # himl-pyright: | |
| # runs-on: ubuntu-20.04 | |
| # steps: | |
| # - uses: actions/checkout@v3 | |
| # with: | |
| # lfs: true | |
| # - uses: actions/setup-node@v3 | |
| # with: | |
| # node-version: '14' | |
| # - uses: conda-incubator/setup-miniconda@v2 | |
| # - name: pyright | |
| # shell: bash -l {0} | |
| # run: | | |
| # conda info | |
| # make pyright | |
| himl-pytest-fast: | |
| runs-on: ubuntu-20.04 | |
| strategy: | |
| matrix: | |
| folder: [ hi-ml, hi-ml-azure ] | |
| # This will let all parts of the matrix run, to collect as many errors as possible, rather than aborting after first fail | |
| fail-fast: false | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| lfs: true | |
| - name: Set up Python ${{ env.pythonVersion }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.pythonVersion }} | |
| - name: Install hi-ml-azure in editable mode | |
| if: ${{ matrix.folder != 'hi-ml-azure' }} | |
| run: | | |
| cd hi-ml-azure | |
| # Install local package in editable mode | |
| make pip_local | |
| - name: Install hi-ml in editable mode | |
| if: ${{ matrix.folder != 'hi-ml-azure' && matrix.folder != 'hi-ml' }} | |
| run: | | |
| cd hi-ml | |
| # Install local package in editable mode | |
| make pip_local | |
| - name: Azure login using OpenID Connect | |
| uses: ./.github/actions/azure-login | |
| - name: Test with pytest, fast only | |
| run: | | |
| cd ${{ matrix.folder }} | |
| make pip_test | |
| # Install local package in editable mode | |
| make pip_local | |
| # Run fast tests | |
| make pytest_fast | |
| himl-build-python: | |
| runs-on: ubuntu-20.04 | |
| needs: [ himl-pytest-fast ] | |
| strategy: | |
| matrix: | |
| folder: [ hi-ml, hi-ml-azure ] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| lfs: true | |
| - name: Set up Python ${{ env.pythonVersion }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.pythonVersion }} | |
| - name: Build Distribution | |
| run: | | |
| cd ${{ matrix.folder }} | |
| make build | |
| - name: Upload package artifacts | |
| uses: ./.github/actions/upload_package_artifacts | |
| with: | |
| folder: ${{ matrix.folder }} | |
| himl-test-artifact-pkg: | |
| runs-on: ubuntu-20.04 | |
| needs: [ himl-build-python, cancel-azureml ] | |
| strategy: | |
| matrix: | |
| folder: [ hi-ml, hi-ml-azure ] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| lfs: true | |
| - name: Set up Python ${{ env.pythonVersion }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.pythonVersion }} | |
| - name: Download dependent package artifacts | |
| id: download_dep | |
| if: ${{ matrix.folder != 'hi-ml-azure' }} | |
| uses: ./.github/actions/download_package_artifacts | |
| with: | |
| folder: hi-ml-azure | |
| - name: Install dependencies | |
| if: ${{ matrix.folder != 'hi-ml-azure' }} | |
| run: | | |
| cd hi-ml-azure | |
| make pip_upgrade | |
| # Install package | |
| pip install ${{ steps.download_dep.outputs.package_filename }} | |
| - name: Download package artifacts | |
| id: download | |
| uses: ./.github/actions/download_package_artifacts | |
| with: | |
| folder: ${{ matrix.folder }} | |
| - name: Azure login using OpenID Connect | |
| uses: ./.github/actions/azure-login | |
| - name: Install artifact and test | |
| run: | | |
| cd ${{ matrix.folder }} | |
| make pip_upgrade | |
| # Install package | |
| pip install ${{ steps.download.outputs.package_filename }} | |
| # Set env vars so wheel is sent to azureml as a private package | |
| # This had to be deprecated because the upload always uses SAS tokens encrypted with the now disabled storage account key | |
| # echo "HIML_AZURE_WHEEL_FILENAME=${{ steps.download.outputs.package_filename }}" >> $GITHUB_ENV | |
| # Test with pytest | |
| make pytest_and_coverage | |
| - name: Upload coverage report to Codecov | |
| # Coverage should also be uploaded if tests still fail | |
| if: always() | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| flags: ${{ matrix.folder }} | |
| himl_smoke_helloworld_v1: | |
| runs-on: ubuntu-20.04 | |
| needs: [ cancel-azureml ] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| lfs: true | |
| - name: Set up Python ${{ env.pythonVersion }} | |
| uses: ./.github/actions/prepare_himl_python_env | |
| with: | |
| python-version: ${{ env.pythonVersion }} | |
| - name: Azure login using OpenID Connect | |
| uses: ./.github/actions/azure-login | |
| - name: Run smoke_helloworld_v1 | |
| run: | | |
| cd hi-ml | |
| make smoke_helloworld_v1 | |
| himl_smoke_helloworld_v2: | |
| runs-on: ubuntu-20.04 | |
| needs: [ cancel-azureml ] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| lfs: true | |
| - name: Set up Python ${{ env.pythonVersion }} | |
| uses: ./.github/actions/prepare_himl_python_env | |
| with: | |
| python-version: ${{ env.pythonVersion }} | |
| - name: Azure login using OpenID Connect | |
| uses: ./.github/actions/azure-login | |
| - name: Run smoke_helloworld_v2 | |
| run: | | |
| cd hi-ml | |
| make smoke_helloworld_v2 | |
| himl_smoke_helloworld_v1_2nodes: | |
| runs-on: ubuntu-20.04 | |
| needs: [ cancel-azureml ] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| lfs: true | |
| - name: Set up Python ${{ env.pythonVersion }} | |
| uses: ./.github/actions/prepare_himl_python_env | |
| with: | |
| python-version: ${{ env.pythonVersion }} | |
| - name: Azure login using OpenID Connect | |
| uses: ./.github/actions/azure-login | |
| - name: Run smoke_helloworld_v1_2nodes | |
| run: | | |
| cd hi-ml | |
| make smoke_helloworld_v1_2nodes | |
| himl_smoke_helloworld_v2_2nodes: | |
| runs-on: ubuntu-20.04 | |
| needs: [ cancel-azureml ] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| lfs: true | |
| - name: Set up Python ${{ env.pythonVersion }} | |
| uses: ./.github/actions/prepare_himl_python_env | |
| with: | |
| python-version: ${{ env.pythonVersion }} | |
| - name: Azure login using OpenID Connect | |
| uses: ./.github/actions/azure-login | |
| - name: Run smoke_helloworld_v2_2nodes | |
| run: | | |
| cd hi-ml | |
| make smoke_helloworld_v2_2nodes | |
| himl-smoke-tests-completed: | |
| # This job is just a placeholder to ensure that all smoke tests have completed before | |
| # publishing the package. Reference this job rather than the individual smoke tests. | |
| runs-on: ubuntu-20.04 | |
| needs: [ | |
| himl_smoke_helloworld_v1, | |
| himl_smoke_helloworld_v2, | |
| himl_smoke_helloworld_v1_2nodes, | |
| himl_smoke_helloworld_v2_2nodes, | |
| ] | |
| steps: | |
| - name: Smoke tests completed | |
| run: | | |
| echo "All hi-ml smoke tests have passed" | |
| himl-publish-testpypi-pkg: | |
| runs-on: ubuntu-20.04 | |
| needs: [ himl-test-artifact-pkg, himl-mypy, himl-flake8, himl-smoke-tests-completed ] | |
| if: "!startsWith(github.ref, 'refs/tags/v')" | |
| strategy: | |
| matrix: | |
| folder: [ hi-ml, hi-ml-azure ] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| lfs: true | |
| - name: Download package artifacts | |
| id: download | |
| uses: ./.github/actions/download_package_artifacts | |
| with: | |
| folder: ${{ matrix.folder }} | |
| - name: Publish package to TestPyPI | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| user: __token__ | |
| password: ${{ env.PYPI_TEST_API_TOKEN }} | |
| packages_dir: ${{ matrix.folder }}/dist/ | |
| repository_url: https://test.pypi.org/legacy/ | |
| verbose: true | |
| - name: Show published TestPyPI package version | |
| run: | | |
| echo "Install this package version via:" | |
| echo "pip install --extra-""index-url https://test.pypi.org/simple ${{ steps.download.outputs.package_version }}" | |
| himl-publish-pypi-pkg: | |
| runs-on: ubuntu-20.04 | |
| needs: [ himl-test-artifact-pkg, himl-mypy, himl-flake8, himl-smoke-tests-completed ] | |
| strategy: | |
| matrix: | |
| folder: [ hi-ml, hi-ml-azure ] | |
| if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| lfs: true | |
| - name: Download package artifacts | |
| id: download | |
| uses: ./.github/actions/download_package_artifacts | |
| with: | |
| folder: ${{ matrix.folder }} | |
| - name: Publish package to PyPI | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| user: __token__ | |
| password: ${{ env.PYPI_API_TOKEN }} | |
| packages_dir: ${{ matrix.folder }}/dist/ | |
| himl-test-pypi-pkg: | |
| runs-on: ubuntu-20.04 | |
| needs: [ himl-publish-pypi-pkg ] | |
| strategy: | |
| matrix: | |
| folder: [ hi-ml, hi-ml-azure ] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| lfs: true | |
| - name: Set up Python ${{ env.pythonVersion }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.pythonVersion }} | |
| - name: Download package artifacts | |
| id: download | |
| uses: ./.github/actions/download_package_artifacts | |
| with: | |
| folder: ${{ matrix.folder }} | |
| - name: Install dependencies | |
| if: ${{ matrix.folder != 'hi-ml-azure' }} | |
| run: | | |
| cd hi-ml-azure | |
| # Install local package in editable mode | |
| make pip_local | |
| - name: Azure login using OpenID Connect | |
| uses: ./.github/actions/azure-login | |
| - name: Install PyPI package and test | |
| run: | | |
| cd ${{ matrix.folder }} | |
| make pip_upgrade | |
| make pip_test | |
| while ! pip install ${{ steps.download.outputs.package_version }} | |
| do | |
| echo waiting for pypi | |
| sleep 10 | |
| done | |
| # Test with pytest, fast only | |
| make pytest_fast | |
| hi-ml-pr-completed: | |
| runs-on: ubuntu-20.04 | |
| if: "!startsWith(github.ref, 'refs/tags/v')" | |
| needs: [ himl-publish-testpypi-pkg ] | |
| steps: | |
| - name: Final PR Job | |
| run: | | |
| echo "All hi-ml PR jobs have passed" |