pytest #7526
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: pytest | |
| on: | |
| pull_request: | |
| merge_group: | |
| types: | |
| - checks_requested | |
| workflow_dispatch: | |
| env: | |
| PUDL_OUTPUT: /home/runner/pudl-work/output/ | |
| PUDL_INPUT: /home/runner/pudl-work/input/ | |
| DAGSTER_HOME: /home/runner/pudl-work/dagster_home/ | |
| ZENODO_SANDBOX_TOKEN_PUBLISH: ${{ secrets.ZENODO_SANDBOX_TOKEN_PUBLISH }} | |
| jobs: | |
| change_filter: | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pull-requests: read | |
| outputs: | |
| # 2025-07-17: because merge_group is an Object and run_code_checks is not a conditional, need to explicitly check for null-ness instead of relying on truthiness. | |
| run_code_checks: ${{ github.event_name =='workflow_dispatch' || (steps.filter.outputs.code == 'true' && (github.event.merge_group != null)) }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 0 | |
| - uses: dorny/paths-filter@v3 | |
| id: filter | |
| with: | |
| # this will raise a warning, but it works: https://github.com/dorny/paths-filter/issues/225 | |
| predicate-quantifier: "every" | |
| filters: | | |
| code: | |
| - '!dbt/seeds/etl_full_row_counts.csv' | |
| - '!devtools/**' | |
| - '!docker/**' | |
| - '!docs/**' | |
| - '!.github/**/*.md' | |
| - '!.github/FUNDING.yml' | |
| - '!.github/actionlint.yml' | |
| - '!.github/dependabot.yml' | |
| - '!.github/release.yml' | |
| - '!.github/ISSUE_TEMPLATE/*.md' | |
| - '!.github/ISSUE_TEMPLATE/*.yml' | |
| - '!.github/workflows/bot-auto-merge.yml' | |
| - '!.github/workflows/build-deploy-docs.yml' | |
| - '!.github/workflows/build-deploy-pudl.yml' | |
| - '!.github/workflows/com-dev-notify.yml' | |
| - '!.github/workflows/docker-build-test.yml' | |
| - '!.github/workflows/q-update-issue-scheduler.yml' | |
| - '!.github/workflows/release.yml' | |
| - '!.github/workflows/update-lockfiles.yml' | |
| - '!.github/workflows/zenodo-cache-sync.yml' | |
| - '!.github/workflows/zenodo-data-release.yml' | |
| - '!AGENTS.md' | |
| - '!README.rst' | |
| - '!CONTRIBUTING.rst' | |
| - '!LICENSE.txt' | |
| - '!CITATION.cff' | |
| - '!.zenodo.json' | |
| - '!**.gitignore' | |
| - '!.git-blame-ignore-revs' | |
| - '!.pre-commit-config.yaml' | |
| - '!.readthedocs.yaml' | |
| - '!.mypy.ini' | |
| - '!terraform/**' | |
| - '!test/unit/**' | |
| - name: print out components | |
| run: | | |
| echo "event name (${{ github.event_name }}) is workflow dispatch: ${{ github.event_name == 'workflow_dispatch' }}" | |
| echo "found code changes: ${{ steps.filter.outputs.code }}" | |
| echo "merge_group ${{ github.event.merge_group }} is not null: ${{ github.event.merge_group != null }}" | |
| ci-docs: | |
| permissions: | |
| contents: read | |
| id-token: write | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 2 | |
| - name: Install pudl environment using pixi | |
| uses: prefix-dev/[email protected] | |
| with: | |
| locked: true | |
| cache: true | |
| cache-write: ${{ github.event_name == 'push' && github.ref_name == 'main' }} | |
| - name: Make input, output and dagster dirs | |
| run: mkdir -p ${{ env.PUDL_OUTPUT }} ${{ env.PUDL_INPUT}} ${{ env.DAGSTER_HOME }} | |
| - name: Lint and build PUDL documentation with Sphinx | |
| run: | | |
| pixi run docs-build | |
| - name: Upload docs coverage artifact | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: coverage-docs | |
| include-hidden-files: true | |
| path: .coverage | |
| ci-unit: | |
| permissions: | |
| contents: read | |
| id-token: write | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 2 | |
| - name: Install pudl environment using pixi | |
| uses: prefix-dev/[email protected] | |
| with: | |
| locked: true | |
| cache: true | |
| cache-write: ${{ github.event_name == 'push' && github.ref_name == 'main' }} | |
| - name: Make input, output and dagster dirs | |
| run: mkdir -p ${{ env.PUDL_OUTPUT }} ${{ env.PUDL_INPUT}} ${{ env.DAGSTER_HOME }} | |
| - name: Run PUDL unit tests and collect test coverage | |
| run: | | |
| pixi run pytest-unit | |
| - name: Upload unit tests coverage artifact | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: coverage-unit | |
| include-hidden-files: true | |
| path: .coverage | |
| ci-integration: | |
| # The blacksmith-4vcpu-ubuntu-2404 runner has 4vCPUs, 16GB RAM, 80 GB disk. | |
| # It's much cheaper and a little faster than the 8vCPU GitHub runner, which | |
| # is the smallest GitHub runner we seemt to be able to run our integration | |
| # tests on. | |
| # Dashboard at https://app.blacksmith.sh/catalyst-cooperative/ | |
| # runs-on: ubuntu-latest-8core | |
| runs-on: blacksmith-4vcpu-ubuntu-2404 | |
| needs: change_filter | |
| if: ${{ needs.change_filter.outputs.run_code_checks == 'true' }} | |
| permissions: | |
| contents: read | |
| id-token: write | |
| strategy: | |
| fail-fast: false | |
| defaults: | |
| run: | |
| shell: bash -l {0} | |
| steps: | |
| - name: Collect Workflow Telemetry | |
| uses: catchpoint/workflow-telemetry-action@v2 | |
| - uses: actions/checkout@v6 | |
| with: | |
| fetch-depth: 2 | |
| - name: Install pudl environment using pixi | |
| uses: prefix-dev/[email protected] | |
| with: | |
| locked: true | |
| cache: true | |
| cache-write: ${{ github.event_name == 'push' && github.ref_name == 'main' }} | |
| - name: Restore Zenodo datastore from cache if possible | |
| uses: actions/cache@v5 | |
| id: cache-zenodo-datastore | |
| with: | |
| path: ${{ env.PUDL_INPUT }} | |
| key: zenodo-datastore-${{ hashFiles('src/pudl/package_data/settings/zenodo_dois.yml') }} | |
| restore-keys: | | |
| zenodo-datastore- | |
| - name: Make input, output and dagster dirs | |
| run: mkdir -p ${{ env.PUDL_OUTPUT }} ${{ env.PUDL_INPUT}} ${{ env.DAGSTER_HOME }} | |
| - name: Run the integration tests | |
| run: | | |
| pixi run pytest-integration | |
| - name: Upload integration test coverage artifact | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: coverage-integration | |
| include-hidden-files: true | |
| path: .coverage | |
| - name: Upload dbt test failure database artifact | |
| if: always() | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: dbt-failures | |
| path: pudl_dbt_tests.duckdb | |
| - name: Log post-test Zenodo datastore contents | |
| run: find ${{ env.PUDL_INPUT }} | |
| ci-coverage: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - ci-docs | |
| - ci-unit | |
| - ci-integration | |
| - change_filter | |
| if: ${{ needs.change_filter.outputs.run_code_checks == 'true' }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - name: Download coverage | |
| id: download-coverage | |
| uses: actions/download-artifact@v7 | |
| with: | |
| path: coverage | |
| - name: List downloaded files | |
| run: | | |
| find coverage -type f | |
| - name: Install pudl environment using pixi | |
| uses: prefix-dev/[email protected] | |
| with: | |
| locked: true | |
| cache: true | |
| - name: Combine coverage data, output XML, and report | |
| run: | | |
| pixi run coverage combine coverage/*/.coverage | |
| pixi run coverage xml --fail-under=0 | |
| pixi run coverage report --fail-under=0 | |
| - name: Upload XML coverage report to CodeCov | |
| uses: codecov/codecov-action@v5 | |
| with: | |
| disable_search: true | |
| files: ./coverage.xml | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| fail_ci_if_error: true # optional (default = false) | |
| plugins: noop | |
| verbose: true | |
| - name: Run coverage report again but fail if we're under the minimum | |
| run: | | |
| pixi run coverage report |