Move change detection to separate workflow in CI (#122336)
https://github.com/python/cpython/commit/e60ee11cb51b87deeb22ad125717bd0d0dc... commit: e60ee11cb51b87deeb22ad125717bd0d0dc10fa8 branch: main author: Sviatoslav Sydorenko (Святослав Сидоренко) <wk@sydorenko.org.ua> committer: hugovk <1324225+hugovk@users.noreply.github.com> date: 2024-07-31T11:32:16+03:00 summary: Move change detection to separate workflow in CI (#122336) files: A .github/workflows/reusable-change-detection.yml M .github/workflows/build.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6568b50c3a6a13..4f3995a020e31b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,8 +1,5 @@ name: Tests -# gh-84728: "paths-ignore" is not used to skip documentation-only PRs, because -# it prevents to mark a job as mandatory. A PR cannot be merged if a job is -# mandatory but not scheduled because of "paths-ignore". on: workflow_dispatch: push: @@ -23,121 +20,19 @@ concurrency: jobs: check_source: - name: 'Check for source changes' - runs-on: ubuntu-latest - timeout-minutes: 10 - outputs: - # Some of the referenced steps set outputs conditionally and there may be - # cases when referencing them evaluates to empty strings. It is nice to - # work with proper booleans so they have to be evaluated through JSON - # conversion in the expressions. However, empty strings used like that - # may trigger all sorts of undefined and hard-to-debug behaviors in - # GitHub Actions CI/CD. To help with this, all of the outputs set here - # that are meant to be used as boolean flags (and not arbitrary strings), - # MUST have fallbacks with default values set. A common pattern would be - # to add ` || false` to all such expressions here, in the output - # definitions. They can then later be safely used through the following - # idiom in job conditionals and other expressions. Here's some examples: - # - # if: fromJSON(needs.check_source.outputs.run-docs) - # - # ${{ - # fromJSON(needs.check_source.outputs.run_tests) - # && 'truthy-branch' - # || 'falsy-branch' - # }} - # - run-docs: ${{ steps.docs-changes.outputs.run-docs || false }} - run-win-msi: ${{ steps.win-msi-changes.outputs.run-win-msi || false }} - run_tests: ${{ steps.check.outputs.run_tests || false }} - run_hypothesis: ${{ steps.check.outputs.run_hypothesis || false }} - run_cifuzz: ${{ steps.check.outputs.run_cifuzz || false }} - config_hash: ${{ steps.config_hash.outputs.hash }} # str - steps: - - uses: actions/checkout@v4 - - name: Check for source changes - id: check - run: | - if [ -z "$GITHUB_BASE_REF" ]; then - echo "run_tests=true" >> $GITHUB_OUTPUT - else - git fetch origin $GITHUB_BASE_REF --depth=1 - # git diff "origin/$GITHUB_BASE_REF..." (3 dots) may be more - # reliable than git diff "origin/$GITHUB_BASE_REF.." (2 dots), - # but it requires to download more commits (this job uses - # "git fetch --depth=1"). - # - # git diff "origin/$GITHUB_BASE_REF..." (3 dots) works with Git - # 2.26, but Git 2.28 is stricter and fails with "no merge base". - # - # git diff "origin/$GITHUB_BASE_REF.." (2 dots) should be enough on - # GitHub, since GitHub starts by merging origin/$GITHUB_BASE_REF - # into the PR branch anyway. - # - # https://github.com/python/core-workflow/issues/373 - git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc|^\.pre-commit-config\.yaml$|\.ruff\.toml$|\.md$|mypy\.ini$)' && echo "run_tests=true" >> $GITHUB_OUTPUT || true - fi - - # Check if we should run hypothesis tests - GIT_BRANCH=${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}} - echo $GIT_BRANCH - if $(echo "$GIT_BRANCH" | grep -q -w '3\.\(8\|9\|10\|11\)'); then - echo "Branch too old for hypothesis tests" - echo "run_hypothesis=false" >> $GITHUB_OUTPUT - else - echo "Run hypothesis tests" - echo "run_hypothesis=true" >> $GITHUB_OUTPUT - fi - - # oss-fuzz maintains a configuration for fuzzing the main branch of - # CPython, so CIFuzz should be run only for code that is likely to be - # merged into the main branch; compatibility with older branches may - # be broken. - FUZZ_RELEVANT_FILES='(\.c$|\.h$|\.cpp$|^configure$|^\.github/workflows/build\.yml$|^Modules/_xxtestfuzz)' - if [ "$GITHUB_BASE_REF" = "main" ] && [ "$(git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qE $FUZZ_RELEVANT_FILES; echo $?)" -eq 0 ]; then - # The tests are pretty slow so they are executed only for PRs - # changing relevant files. - echo "Run CIFuzz tests" - echo "run_cifuzz=true" >> $GITHUB_OUTPUT - else - echo "Branch too old for CIFuzz tests; or no C files were changed" - echo "run_cifuzz=false" >> $GITHUB_OUTPUT - fi - - name: Compute hash for config cache key - id: config_hash - run: | - echo "hash=${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}" >> $GITHUB_OUTPUT - - name: Get a list of the changed documentation-related files - if: github.event_name == 'pull_request' - id: changed-docs-files - uses: Ana06/get-changed-files@v2.3.0 - with: - filter: | - Doc/** - Misc/** - .github/workflows/reusable-docs.yml - format: csv # works for paths with spaces - - name: Check for docs changes - if: >- - github.event_name == 'pull_request' - && steps.changed-docs-files.outputs.added_modified_renamed != '' - id: docs-changes - run: | - echo "run-docs=true" >> "${GITHUB_OUTPUT}" - - name: Get a list of the MSI installer-related files - id: changed-win-msi-files - uses: Ana06/get-changed-files@v2.3.0 - with: - filter: | - Tools/msi/** - .github/workflows/reusable-windows-msi.yml - format: csv # works for paths with spaces - - name: Check for changes in MSI installer-related files - if: >- - steps.changed-win-msi-files.outputs.added_modified_renamed != '' - id: win-msi-changes - run: | - echo "run-win-msi=true" >> "${GITHUB_OUTPUT}" + name: Change detection + # To use boolean outputs from this job, parse them as JSON. + # Here's some examples: + # + # if: fromJSON(needs.check_source.outputs.run-docs) + # + # ${{ + # fromJSON(needs.check_source.outputs.run_tests) + # && 'truthy-branch' + # || 'falsy-branch' + # }} + # + uses: ./.github/workflows/reusable-change-detection.yml check-docs: name: Docs diff --git a/.github/workflows/reusable-change-detection.yml b/.github/workflows/reusable-change-detection.yml new file mode 100644 index 00000000000000..25c789d335efc8 --- /dev/null +++ b/.github/workflows/reusable-change-detection.yml @@ -0,0 +1,150 @@ +--- + +name: Change detection + +on: # yamllint disable-line rule:truthy + workflow_call: + outputs: + # Some of the referenced steps set outputs conditionally and there may be + # cases when referencing them evaluates to empty strings. It is nice to + # work with proper booleans so they have to be evaluated through JSON + # conversion in the expressions. However, empty strings used like that + # may trigger all sorts of undefined and hard-to-debug behaviors in + # GitHub Actions CI/CD. To help with this, all of the outputs set here + # that are meant to be used as boolean flags (and not arbitrary strings), + # MUST have fallbacks with default values set. A common pattern would be + # to add ` || false` to all such expressions here, in the output + # definitions. They can then later be safely used through the following + # idiom in job conditionals and other expressions. Here's some examples: + # + # if: fromJSON(needs.change-detection.outputs.run-docs) + # + # ${{ + # fromJSON(needs.change-detection.outputs.run-tests) + # && 'truthy-branch' + # || 'falsy-branch' + # }} + # + config_hash: + description: Config hash value for use in cache keys + value: ${{ jobs.compute-changes.outputs.config-hash }} # str + run-docs: + description: Whether to build the docs + value: ${{ jobs.compute-changes.outputs.run-docs || false }} # bool + run_tests: + description: Whether to run the regular tests + value: ${{ jobs.compute-changes.outputs.run-tests || false }} # bool + run-win-msi: + description: Whether to run the MSI installer smoke tests + value: >- # bool + ${{ jobs.compute-changes.outputs.run-win-msi || false }} + run_hypothesis: + description: Whether to run the Hypothesis tests + value: >- # bool + ${{ jobs.compute-changes.outputs.run-hypothesis || false }} + run_cifuzz: + description: Whether to run the CIFuzz job + value: >- # bool + ${{ jobs.compute-changes.outputs.run-cifuzz || false }} + +jobs: + compute-changes: + name: Compute changed files + runs-on: ubuntu-latest + timeout-minutes: 10 + outputs: + config-hash: ${{ steps.config-hash.outputs.hash }} + run-cifuzz: ${{ steps.check.outputs.run-cifuzz }} + run-docs: ${{ steps.docs-changes.outputs.run-docs }} + run-hypothesis: ${{ steps.check.outputs.run-hypothesis }} + run-tests: ${{ steps.check.outputs.run-tests }} + run-win-msi: ${{ steps.win-msi-changes.outputs.run-win-msi }} + steps: + - run: >- + echo '${{ github.event_name }}' + - uses: actions/checkout@v4 + - name: Check for source changes + id: check + run: | + if [ -z "$GITHUB_BASE_REF" ]; then + echo "run-tests=true" >> $GITHUB_OUTPUT + else + git fetch origin $GITHUB_BASE_REF --depth=1 + # git diff "origin/$GITHUB_BASE_REF..." (3 dots) may be more + # reliable than git diff "origin/$GITHUB_BASE_REF.." (2 dots), + # but it requires to download more commits (this job uses + # "git fetch --depth=1"). + # + # git diff "origin/$GITHUB_BASE_REF..." (3 dots) works with Git + # 2.26, but Git 2.28 is stricter and fails with "no merge base". + # + # git diff "origin/$GITHUB_BASE_REF.." (2 dots) should be enough on + # GitHub, since GitHub starts by merging origin/$GITHUB_BASE_REF + # into the PR branch anyway. + # + # https://github.com/python/core-workflow/issues/373 + git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc|^\.pre-commit-config\.yaml$|\.ruff\.toml$|\.md$|mypy\.ini$)' && echo "run-tests=true" >> $GITHUB_OUTPUT || true + fi + + # Check if we should run hypothesis tests + GIT_BRANCH=${GITHUB_BASE_REF:-${GITHUB_REF#refs/heads/}} + echo $GIT_BRANCH + if $(echo "$GIT_BRANCH" | grep -q -w '3\.\(8\|9\|10\|11\)'); then + echo "Branch too old for hypothesis tests" + echo "run-hypothesis=false" >> $GITHUB_OUTPUT + else + echo "Run hypothesis tests" + echo "run-hypothesis=true" >> $GITHUB_OUTPUT + fi + + # oss-fuzz maintains a configuration for fuzzing the main branch of + # CPython, so CIFuzz should be run only for code that is likely to be + # merged into the main branch; compatibility with older branches may + # be broken. + FUZZ_RELEVANT_FILES='(\.c$|\.h$|\.cpp$|^configure$|^\.github/workflows/build\.yml$|^Modules/_xxtestfuzz)' + if [ "$GITHUB_BASE_REF" = "main" ] && [ "$(git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qE $FUZZ_RELEVANT_FILES; echo $?)" -eq 0 ]; then + # The tests are pretty slow so they are executed only for PRs + # changing relevant files. + echo "Run CIFuzz tests" + echo "run-cifuzz=true" >> $GITHUB_OUTPUT + else + echo "Branch too old for CIFuzz tests; or no C files were changed" + echo "run-cifuzz=false" >> $GITHUB_OUTPUT + fi + - name: Compute hash for config cache key + id: config-hash + run: | + echo "hash=${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}" >> $GITHUB_OUTPUT + - name: Get a list of the changed documentation-related files + if: github.event_name == 'pull_request' + id: changed-docs-files + uses: Ana06/get-changed-files@v2.3.0 + with: + filter: | + Doc/** + Misc/** + .github/workflows/reusable-docs.yml + format: csv # works for paths with spaces + - name: Check for docs changes + if: >- + github.event_name == 'pull_request' + && steps.changed-docs-files.outputs.added_modified_renamed != '' + id: docs-changes + run: | + echo "run-docs=true" >> "${GITHUB_OUTPUT}" + - name: Get a list of the MSI installer-related files + id: changed-win-msi-files + uses: Ana06/get-changed-files@v2.3.0 + with: + filter: | + Tools/msi/** + .github/workflows/reusable-windows-msi.yml + format: csv # works for paths with spaces + - name: Check for changes in MSI installer-related files + if: >- + steps.changed-win-msi-files.outputs.added_modified_renamed != '' + id: win-msi-changes + run: | + echo "run-win-msi=true" >> "${GITHUB_OUTPUT}" + +...
participants (1)
-
hugovk