horovod · tgaddair · Oct 9, 2021 · Oct 8, 2021 · Oct 8, 2021
diff --git a/.github/gen-workflow-ci.py b/.github/gen-workflow-ci.py
@@ -113,21 +113,36 @@ def workflow_header() -> str:
                 f'\n'
                 f'on:\n'
                 f'  schedule:\n'
+                f'    # run a build on master (this does not publish test results or cancel concurrent builds)\n'
                 f'    - cron: \'0 10 * * *\' # everyday at 10am\n'
                 f'  push:\n'
                 f'    # only consider push to master and tags\n'
                 f'    # otherwise modify job.config.outputs.push\n'
                 f'    branches: [ master ]\n'
                 f'    tags: [ \'v*.*.*\' ]\n'
                 f'  pull_request:\n'
+                f'    # only consider pull requests into master\n'
                 f'    branches: [ master ]\n'
                 f'\n'
                 f'concurrency:\n'
-                f'  # github.ref means something like refs/heads/master or refs/tags/v0.22.1 or the branch.\n'
-                f'  # This helps to not cancel concurrent runs on master and a tag that share the same commit\n'
-                f'  # On master, head_ref is empty, so we use the SHA of the commit, this means\n'
-                f'  # individual commits to master will not be cancelled, but tagged\n'
-                f'  group: ci-${{{{ github.ref }}}}-${{{{ github.head_ref || github.sha }}}}\n'
+                f'  # This controls which concurrent builds to cancel:\n'
+                f'  # - we do not want any concurrent builds on a branch (pull_request)\n'
+                f'  # - we do not want concurrent builds on the same commit on master (push)\n'
+                f'  # - we do not want concurrent builds on the same commit on a tag (push)\n'
+                f'  # - we allow concurrent runs on the same commit on master and its tag (push)\n'
+                f'  # - we allow concurrent runs on the same commit on master (push) and a scheduled build (schedule)\n'
+                f'  #\n'
+                f'  # A pull_request event only runs on branch commit, a push event only on master and tag commit.\n'
+                f'  # A schedule event only runs on master HEAD commit.\n'
+                f'  #\n'
+                f'  # Expression github.ref means something like refs/heads/master or refs/tags/v0.22.1 or the branch.\n'
+                f'  # This helps to not cancel concurrent runs on master or a tag that share the same commit.\n'
+                f'  # Expression github.head_ref refers to the branch of the pull request.\n'
+                f'  # On master, github.head_ref is empty, so we use the SHA of the commit, this means individual\n'
+                f'  # commits to master will not be cancelled, while there can only be one concurrent build on a branch.\n'
+                f'  #\n'
+                f'  # We include the event name to we allow for concurrent scheduled and master builds.\n'
+                f'  group: ci-${{{{ github.event_name }}}}-${{{{ github.ref }}}}-${{{{ github.head_ref || github.sha }}}}\n'
                 f'  cancel-in-progress: true\n'
                 f'\n')
 
@@ -138,6 +153,8 @@ def jobs(*jobs: str) -> str:
                '    steps:\n' \
                '    - name: Debug Action\n' \
                '      uses: hmarr/debug-action@v1.0.0\n' \
+               '    - name: Debug Concurrency\n' \
+               '      run: echo "ci-${{ github.event_name }}-${{ github.ref }}-${{ github.head_ref || github.sha }}"\n' \
                '\n' \
                '  event_file:\n' \
                '    name: "Event File"\n' \
@@ -487,14 +504,22 @@ def trigger_buildkite_job(id: str, name: str, needs: List[str], mode: str) -> st
                 f'      ( github.event_name != \'pull_request\' || github.event.pull_request.head.repo.full_name == github.repository )\n'
                 f'\n'
                 f'    steps:\n'
+                f'      - name: Configure Buildkite Build\n'
+                f'        id: config\n'
+                f'        run: |\n'
+                f'          if [[ "${{{{ github.event_name }}}}" == "schedule" ]]\n'
+                f'          then\n'
+                f'            # we add this label to the branch used by Buildkite to avoid it cancelling one of concurrent schedule and push builds on master\n'
+                f'            echo "::set-output name=branch-label:: (schedule)"\n'
+                f'          fi\n'
+                f'\n'
                 f'      - name: Trigger Buildkite Pipeline\n'
                 f'        id: build\n'
                 f'        uses: EnricoMi/trigger-pipeline-action@master\n'
                 f'        env:\n'
                 f'          PIPELINE: "horovod/horovod"\n'
                 f'          # COMMIT is taken from GITHUB_SHA\n'
-                f'          # BRANCH falls back to GITHUB_REF if empty\n'
-                f'          BRANCH: "${{{{ github.event.pull_request.head.ref }}}}"\n'
+                f'          BRANCH: "${{{{ github.event.pull_request.head.ref || github.ref }}}}${{{{ steps.config.outputs.branch-label }}}}"\n'
                 f'          # empty MESSAGE will be filled by Buildkite\n'
                 f'          BUILDKITE_API_ACCESS_TOKEN: ${{{{ secrets.BUILDKITE_TOKEN }}}}\n'
                 f'          BUILD_ENV_VARS: "{{\\"PIPELINE_MODE\\": \\"{mode}\\"}}"\n'

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -4,21 +4,36 @@ name: CI
 
 on:
   schedule:
+    # run a build on master (this does not publish test results or cancel concurrent builds)
     - cron: '0 10 * * *' # everyday at 10am
   push:
     # only consider push to master and tags
     # otherwise modify job.config.outputs.push
     branches: [ master ]
     tags: [ 'v*.*.*' ]
   pull_request:
+    # only consider pull requests into master
     branches: [ master ]
 
 concurrency:
-  # github.ref means something like refs/heads/master or refs/tags/v0.22.1 or the branch.
-  # This helps to not cancel concurrent runs on master and a tag that share the same commit
-  # On master, head_ref is empty, so we use the SHA of the commit, this means
-  # individual commits to master will not be cancelled, but tagged
-  group: ci-${{ github.ref }}-${{ github.head_ref || github.sha }}
+  # This controls which concurrent builds to cancel:
+  # - we do not want any concurrent builds on a branch (pull_request)
+  # - we do not want concurrent builds on the same commit on master (push)
+  # - we do not want concurrent builds on the same commit on a tag (push)
+  # - we allow concurrent runs on the same commit on master and its tag (push)
+  # - we allow concurrent runs on the same commit on master (push) and a scheduled build (schedule)
+  #
+  # A pull_request event only runs on branch commit, a push event only on master and tag commit.
+  # A schedule event only runs on master HEAD commit.
+  #
+  # Expression github.ref means something like refs/heads/master or refs/tags/v0.22.1 or the branch.
+  # This helps to not cancel concurrent runs on master or a tag that share the same commit.
+  # Expression github.head_ref refers to the branch of the pull request.
+  # On master, github.head_ref is empty, so we use the SHA of the commit, this means individual
+  # commits to master will not be cancelled, while there can only be one concurrent build on a branch.
+  #
+  # We include the event name to we allow for concurrent scheduled and master builds.
+  group: ci-${{ github.event_name }}-${{ github.ref }}-${{ github.head_ref || github.sha }}
   cancel-in-progress: true
 
 jobs:
@@ -27,6 +42,8 @@ jobs:
     steps:
     - name: Debug Action
       uses: hmarr/debug-action@v1.0.0
+    - name: Debug Concurrency
+      run: echo "ci-${{ github.event_name }}-${{ github.ref }}-${{ github.head_ref || github.sha }}"
 
   event_file:
     name: "Event File"
@@ -3432,14 +3449,22 @@ jobs:
       ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository )
 
     steps:
+      - name: Configure Buildkite Build
+        id: config
+        run: |
+          if [[ "${{ github.event_name }}" == "schedule" ]]
+          then
+            # we add this label to the branch used by Buildkite to avoid it cancelling one of concurrent schedule and push builds on master
+            echo "::set-output name=branch-label:: (schedule)"
+          fi
+
       - name: Trigger Buildkite Pipeline
         id: build
         uses: EnricoMi/trigger-pipeline-action@master
         env:
           PIPELINE: "horovod/horovod"
           # COMMIT is taken from GITHUB_SHA
-          # BRANCH falls back to GITHUB_REF if empty
-          BRANCH: "${{ github.event.pull_request.head.ref }}"
+          BRANCH: "${{ github.event.pull_request.head.ref || github.ref }}${{ steps.config.outputs.branch-label }}"
           # empty MESSAGE will be filled by Buildkite
           BUILDKITE_API_ACCESS_TOKEN: ${{ secrets.BUILDKITE_TOKEN }}
           BUILD_ENV_VARS: "{\"PIPELINE_MODE\": \"GPU NON HEADS\"}"
@@ -3482,14 +3507,22 @@ jobs:
       ( github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository )
 
     steps:
+      - name: Configure Buildkite Build
+        id: config
+        run: |
+          if [[ "${{ github.event_name }}" == "schedule" ]]
+          then
+            # we add this label to the branch used by Buildkite to avoid it cancelling one of concurrent schedule and push builds on master
+            echo "::set-output name=branch-label:: (schedule)"
+          fi
+
       - name: Trigger Buildkite Pipeline
         id: build
         uses: EnricoMi/trigger-pipeline-action@master
         env:
           PIPELINE: "horovod/horovod"
           # COMMIT is taken from GITHUB_SHA
-          # BRANCH falls back to GITHUB_REF if empty
-          BRANCH: "${{ github.event.pull_request.head.ref }}"
+          BRANCH: "${{ github.event.pull_request.head.ref || github.ref }}${{ steps.config.outputs.branch-label }}"
           # empty MESSAGE will be filled by Buildkite
           BUILDKITE_API_ACCESS_TOKEN: ${{ secrets.BUILDKITE_TOKEN }}
           BUILD_ENV_VARS: "{\"PIPELINE_MODE\": \"GPU HEADS\"}"