Run SOMISANA forecast models on MIMS #149
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run SOMISANA forecast models on MIMS | |
| on: | |
| schedule: | |
| - cron: '0 6,18 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| run_date: | |
| description: 'Date and time for T0 for the run in format YYYYMMDD_HH' | |
| required: false | |
| default: '' | |
| type: string | |
| build_images: | |
| description: 'Run the image builds?' | |
| required: true | |
| default: 'true' | |
| type: boolean | |
| jobs: | |
| # note that the jobs aren't executed in the order they are written below | |
| # they are executed in the order depending on the 'needs' attribute of each job | |
| # start by building our docker images needed in the workflow | |
| build_images: | |
| # conditional execution when run manually as a workflow_dispatch | |
| # saves time when testing something and we don't want to build the images every time | |
| # but it'll get executed every time when run as a scheduled workflow | |
| if: ${{ github.event.schedule || github.event.inputs.build_images == 'true' }} | |
| strategy: | |
| matrix: # the 'matrix' strategy allows us to build all three docker images in parallel using the same reusable workflow | |
| image_id: ['cli', 'run', 'matlab'] | |
| uses: ./.github/workflows/build_images.yml # Path to your reusable workflow | |
| with: | |
| IMAGE_ID: ${{ matrix.image_id }} | |
| # ensure that we don't keep building up a stash of images, which can use a lot of resources | |
| # we'll need to add more of these as we expand onto new servers | |
| prune_images_mims1: | |
| needs: [build_images] | |
| runs-on: mims1 | |
| steps: | |
| - name: remove all unused docker images and containers | |
| run: | | |
| docker system prune -f | |
| # set some environment variables | |
| envs: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| BRANCH_REF: ${{ steps.BRANCH_REF.outputs.value }} | |
| RUN_DATE: ${{ steps.calculate_date.outputs.value }} | |
| steps: | |
| - name: Calculate run_date | |
| id: calculate_date | |
| run: | | |
| input_run_date=${{ github.event.inputs.run_date || 'unspecified' }} | |
| if [[ ${{ github.event_name }} == 'workflow_dispatch' && ${input_run_date} != 'unspecified' ]] | |
| then | |
| run_date="${{ github.event.inputs.run_date }}" # Use provided run_date | |
| else | |
| # automatically set the run_date by finding the nearest 6 hourly time stamp in the past | |
| # Get the current time in UTC | |
| current_time=$(date -u +'%Y%m%d_%H') | |
| # Extract the hour and calculate the nearest multiple of 6 in the past | |
| hour=$(echo ${current_time:9:2} | awk '{print int($1 - ($1%6))}') | |
| # Correct hour formatting (ensure leading zero) | |
| hour=$(printf "%02d" $hour) | |
| # Assemble the run_date | |
| run_date=$(echo ${current_time:0:8}_${hour}) | |
| fi | |
| echo "value=$run_date" >> $GITHUB_OUTPUT | |
| # Dynamically set the branch ref to the currently executing branch | |
| - name: Set the BRANCH_REF | |
| id: BRANCH_REF | |
| run: | | |
| echo "value=${GITHUB_REF##*/}" >> $GITHUB_OUTPUT | |
| # everything below here runs using the `mims1` self-hosted runner | |
| # This is the `computecroco01` server with 128 cpu's and 256 G ram, dedictated to running SOMISANA's operational models | |
| # It is possible that we may want to use an additional modelling server (which will be a separate node on MIMS) | |
| # In that event, we could put all the code below here in a new reusable workflow called run_ops_mims1.yml | |
| # And then set up another one called run_ops_mims2.yml set up in the same way but obviously running different models/domains | |
| # (note you'll also have to include another `git pull` command at the end of build_images.yml to make sure the latest images are available on the new server) | |
| # | |
| # download all the data we'll need to force the models | |
| # no need to provide any model specific inputs as we hard code the extent to cover the whole EEZ | |
| get_forcing: | |
| needs: [envs, build_images] | |
| if: ${{ always() }} # Always run even if build_images is not executed (but it'll wait for build_images if it is) | |
| uses: ./.github/workflows/get_forcing.yml # Path to your reusable workflow | |
| with: | |
| RUNNER_NAME: mims1 | |
| RUN_DATE: ${{ needs.envs.outputs.RUN_DATE }} | |
| BRANCH_REF: ${{ needs.envs.outputs.BRANCH_REF }} | |
| HDAYS: 5 | |
| FDAYS: 5 | |
| secrets: inherit | |
| # prepare croco config dirs | |
| prep_configdirs: | |
| needs: [envs,get_forcing] | |
| if: ${{ always() }} # Always run even if build_images is not executed (but it'll wait for build_images if it is) (this job needs get_forcing, which needs build_images) | |
| strategy: | |
| matrix: | |
| # running as a matrix strategy allows us to prepare different domains in parallel inside a single job | |
| domain: ['algoa_01','swcape_02'] | |
| uses: ./.github/workflows/prep_configdir.yml # Path to your reusable workflow | |
| with: | |
| RUN_DATE: ${{ needs.envs.outputs.RUN_DATE }} | |
| BRANCH_REF: ${{ needs.envs.outputs.BRANCH_REF }} | |
| RUNNER_NAME: mims1 | |
| MODEL: croco_v1.3.1 | |
| DOMAIN: ${{ matrix.domain }} | |
| # prepare croco forcing files, run the model, do the postprocessing | |
| # | |
| # we're using separate jobs for the different domains, rather than using the matrix strategy, as if one of the domains fails, then it cancels all others running in parallel... to separating the jobs makes it more robust... and also easier to follow in the actions log on github | |
| # we're also seprating jobs for gfs and saws forcing, rather than using the matrix strategy, as they have different HDAYS and FDAYS inputs (otherwise we could have included both in a single job using the matrix strategy) | |
| # we will probably be able to use the matrix strategy for the different ogcm forcings though (let's see how this goes) | |
| run_swcape_gfs: | |
| needs: [envs,get_forcing,prep_configdirs] | |
| if: ${{ always() }} # Always run even if build_images is not executed (but it'll wait for build_images if it is) | |
| strategy: | |
| matrix: | |
| # matrix strategy allows us to run different model forcings in parallel | |
| frc: [''] | |
| ogcm: ['MERCATOR'] | |
| uses: ./.github/workflows/run_croco.yml # Path to your reusable workflow | |
| with: | |
| RUN_DATE: ${{ needs.envs.outputs.RUN_DATE }} | |
| BRANCH_REF: ${{ needs.envs.outputs.BRANCH_REF }} | |
| RUNNER_NAME: mims1 | |
| MODEL: croco_v1.3.1 | |
| DOMAIN: swcape_02 | |
| COMP: C01 | |
| INP: I99 | |
| BLK: GFS | |
| FRC: ${{ matrix.frc }} | |
| OGCM: ${{ matrix.ogcm }} | |
| HDAYS: 5 | |
| FDAYS: 5 | |
| run_algoa_gfs: | |
| needs: [envs,get_forcing,prep_configdirs] | |
| if: ${{ always() }} # Always run even if build_images is not executed (but it'll wait for build_images if it is) | |
| strategy: | |
| matrix: | |
| # matrix strategy allows us to run different model forcings in parallel | |
| frc: [''] | |
| ogcm: ['MERCATOR'] | |
| uses: ./.github/workflows/run_croco.yml # Path to your reusable workflow | |
| with: | |
| RUN_DATE: ${{ needs.envs.outputs.RUN_DATE }} | |
| BRANCH_REF: ${{ needs.envs.outputs.BRANCH_REF }} | |
| RUNNER_NAME: mims1 | |
| MODEL: croco_v1.3.1 | |
| DOMAIN: algoa_01 | |
| COMP: C01 | |
| INP: I99 | |
| BLK: GFS | |
| FRC: ${{ matrix.frc }} | |
| OGCM: ${{ matrix.ogcm }} | |
| HDAYS: 5 | |
| FDAYS: 5 | |