Skip to content

#514 correct tag given to invoke arguments to only use name and not i… #6153

#514 correct tag given to invoke arguments to only use name and not i…

#514 correct tag given to invoke arguments to only use name and not i… #6153

Workflow file for this run

# -----------------------------------------------------------------------------
# BSD 3-Clause License
#
# Copyright (c) 2023-2025, Science and Technology Facilities Council.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
# * Redistributions of source code must retain the above copyright notice, this
# list of conditions and the following disclaimer.
#
# * Redistributions in binary form must reproduce the above copyright notice,
# this list of conditions and the following disclaimer in the documentation
# and/or other materials provided with the distribution.
#
# * Neither the name of the copyright holder nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
# -----------------------------------------------------------------------------
# Author S. Siso, STFC Daresbury Lab
# This workflow will use a self-hosted runner to perform the more expensive
# integrations tests that are not run on GHA systems.
name: LFRic Integration Tests
on:
push
env:
LFRIC_APPS_REV: 14921
LFRIC_CORE_REV: 53845
PYTHON_VERSION: 3.14
GCC_VERSION: 14
NVHPC_VERSION: 25.1
NUM_PARALLEL: 16
jobs:
run_if_on_mirror:
if: ${{ github.repository == 'stfc/PSyclone-mirror' }}
runs-on: self-hosted
outputs:
lfric_dist_mem: ${{ steps.lfric_dist_mem.outputs.time }}
lfric_omp_offload: ${{ steps.lfric_omp_offload.outputs.time }}
lfric_omp_offload_build: ${{ steps.lfric_omp_offload.outputs.build_time }}
lfric_acc_offload: ${{ steps.lfric_acc_offload.outputs.time }}
lfric_all_trans: ${{ steps.lfric_all_trans.outputs.time }}
lfric_colour_tiling: ${{ steps.lfric_colour_tiling.outputs.time }}
lfric_atm: ${{ steps.lfric_atm.outputs.time }}
lfric_atm_build: ${{ steps.lfric_atm.outputs.build_time }}
steps:
- uses: actions/checkout@v3
with:
submodules: recursive
# This is required to get the commit history for merge commits for
# the ci-skip check below.
fetch-depth: '0'
- name: Check for [skip ci] in commit message
uses: mstachniuk/ci-skip@v1
with:
# This setting causes the tests to 'fail' if [skip ci] is specified
fail-fast: true
commit-filter: '[skip ci]'
- name: Install dependencies
run: |
# Use a specific version of Python (rather than the system-wide one).
module load python/${PYTHON_VERSION}
python -m venv .runner_venv
. .runner_venv/bin/activate
python -m pip install --upgrade pip
# Uncomment the below to use the submodule version of fparser rather
# than the latest release from pypi.
# pip install external/fparser
pip install .[test]
# Fetch the specified version of LFRic apps
source /archive/psyclone-spack/psyclone-spack-Jun25/spack-repo/share/spack/setup-env.sh
spack load fcm/xyl5qxp
cd /archive/psyclone-tests/latest-run
rm -rf lfric_apps lfric_core
fcm co --username ${{ secrets.UKMO_USER }} --password '${{ secrets.UKMO_PASSW }}' \
-r ${LFRIC_APPS_REV} https://code.metoffice.gov.uk/svn/lfric_apps/main/trunk/ lfric_apps
# Disable XIOS in the gungho_model example configuration
cd lfric_apps/applications/gungho_model/example
sed -i "s|use_xios_io=.*|use_xios_io=.false.|g" configuration.nml
# We need to patch lfric_core to resolve issues with the nvfortran compiler
# see https://code.metoffice.gov.uk/trac/lfric/ticket/4621
cd /archive/psyclone-tests/latest-run
fcm co --username ${{ secrets.UKMO_USER }} --password '${{ secrets.UKMO_PASSW }}' \
-r ${LFRIC_CORE_REV} https://code.metoffice.gov.uk/svn/lfric/LFRic/trunk/ lfric_core
cd lfric_core
svn patch ${GITHUB_WORKSPACE}/examples/lfric/scripts/KGOs/nvidia_lfric_core.patch
# PSyclone, compile and run MetOffice LFRic with 6 MPI ranks.
- name: LFRic passthrough (with DistributedMemory)
id: lfric_dist_mem
run: |
# Set up environment
source /archive/psyclone-spack/psyclone-spack-Jun25/spack-repo/share/spack/setup-env.sh
spack load lfric-build-environment%gcc@${GCC_VERSION}
source .runner_venv/bin/activate
export PSYCLONE_LFRIC_DIR=${GITHUB_WORKSPACE}/examples/lfric/scripts
# TODO #3206: The no-annex-dof configuration fails in LFRic because some kernels lie in
# their metadata
# export PSYCLONE_CONFIG_FILE=${PSYCLONE_LFRIC_DIR}/KGOs/lfric_psyclone_no_annexed.cfg
export PSYCLONE_CONFIG_FILE=${PSYCLONE_LFRIC_DIR}/KGOs/lfric_psyclone.cfg
export LFRIC_DIR=/archive/psyclone-tests/latest-run/lfric_apps
cd ${LFRIC_DIR}
# Use un-patched LFRic core
sed -i "s|lfric_core_rev=.*|lfric_core_rev=${LFRIC_CORE_REV}|g" dependencies.sh
sed -i "s|lfric_core_sources=.*|lfric_core_sources=|g" dependencies.sh
# Ensure no transformations are applied by supplying an empty
# transformation script.
export OPT_DIR=${LFRIC_DIR}/applications/gungho_model/optimisation/psyclone-test/psykal
mkdir -p ${OPT_DIR}
cp ${PSYCLONE_LFRIC_DIR}/does_nothing.py ${OPT_DIR}/global.py
# Clean previous version and compile again
rm -rf applications/gungho_model/working
./build/local_build.py -j ${NUM_PARALLEL} -a gungho_model -v -p psyclone-test -w working-passthrough
# Run
cd applications/gungho_model/example
rm -f timer.txt gungho_model-checksums.txt # In case there were from a previous run
# Number of OMP threads have to be set even when psyclone does not inject OpenMP
# because some psykal-lite routines have it.
export OMP_NUM_THREADS=1
mpirun -n 12 ../bin/gungho_model configuration.nml
python ${PSYCLONE_LFRIC_DIR}/compare_ouput.py ${PSYCLONE_LFRIC_DIR}/KGOs/gungho_model-checksums.txt gungho_model-checksums.txt
cat timer.txt
export VAR_TIME=$(grep "gungho_model" timer.txt | cut -d'|' -f5)
echo "time=${VAR_TIME}" >> "${GITHUB_OUTPUT}"
# PSyclone, compile and run MetOffice gungho_model on GPU
- name: LFRic GungHo with OpenMP offload
id: lfric_omp_offload
run: |
# Set up environment
source /archive/psyclone-spack/psyclone-spack-Jun25/spack-repo/share/spack/setup-env.sh
spack load lfric-build-environment%nvhpc@${NVHPC_VERSION}
spack load nvhpc
source .runner_venv/bin/activate
export PSYCLONE_LFRIC_DIR=${GITHUB_WORKSPACE}/examples/lfric/scripts
export PSYCLONE_CONFIG_FILE=${PSYCLONE_LFRIC_DIR}/KGOs/lfric_psyclone.cfg
export LFRIC_DIR=/archive/psyclone-tests/latest-run/lfric_apps
cd ${LFRIC_DIR}
# Use the patched LFRic core to support nvfortran
sed -i "s|lfric_core_rev=.*|lfric_core_rev=|g" dependencies.sh
sed -i "s|lfric_core_sources=.*|lfric_core_sources=/archive/psyclone-tests/latest-run/lfric_core|g" dependencies.sh
# PSyclone scripts must now be under 'optimisation/psykal' and be called 'global.py'
export OPT_DIR=${LFRIC_DIR}/applications/gungho_model/optimisation/psyclone-test/psykal
mkdir -p ${OPT_DIR}
cp ${PSYCLONE_LFRIC_DIR}/gpu_offloading.py ${OPT_DIR}/global.py
# Clean previous version and compile again
rm -rf applications/gungho_model/working
export BUILD_START="${SECONDS}"
LFRIC_OFFLOAD_DIRECTIVES=omp ./build/local_build.py -j ${NUM_PARALLEL} -a gungho_model -p psyclone-test -w working-omp-offload
export BUILD_ELAPSED=$((${SECONDS}-${BUILD_START}))
cd applications/gungho_model/example
rm -f timer.txt gungho_model-checksums.txt # In case there were from a previous run
export OMP_NUM_THREADS=12
export CUDA_VISIBLE_DEVICES=1
mpirun -n 1 ../bin/gungho_model configuration.nml
python ${PSYCLONE_LFRIC_DIR}/compare_ouput.py ${PSYCLONE_LFRIC_DIR}/KGOs/gungho_model-checksums.txt gungho_model-checksums.txt
cat timer.txt
export VAR_TIME=$(grep "gungho_model" timer.txt | cut -d'|' -f5)
echo "time=${VAR_TIME}" >> "${GITHUB_OUTPUT}"
echo "build_time=${BUILD_ELAPSED}" >> "${GITHUB_OUTPUT}"
# PSyclone, compile and run MetOffice gungho_model on GPU
- name: LFRic GungHo with OpenACC offload
id: lfric_acc_offload
run: |
# Set up environment
source /archive/psyclone-spack/psyclone-spack-Jun25/spack-repo/share/spack/setup-env.sh
spack load lfric-build-environment%nvhpc@${NVHPC_VERSION}
spack load nvhpc
source .runner_venv/bin/activate
export PSYCLONE_LFRIC_DIR=${GITHUB_WORKSPACE}/examples/lfric/scripts
export PSYCLONE_CONFIG_FILE=${PSYCLONE_LFRIC_DIR}/KGOs/lfric_psyclone.cfg
export LFRIC_DIR=/archive/psyclone-tests/latest-run/lfric_apps
cd ${LFRIC_DIR}
# Use the patched LFRic core to support nvfortran
sed -i "s|lfric_core_rev=.*|lfric_core_rev=|g" dependencies.sh
sed -i "s|lfric_core_sources=.*|lfric_core_sources=/archive/psyclone-tests/latest-run/lfric_core|g" dependencies.sh
# PSyclone scripts must now be under 'optimisation/psykal' and be called 'global.py'
export OPT_DIR=${LFRIC_DIR}/applications/gungho_model/optimisation/psyclone-test/psykal
mkdir -p ${OPT_DIR}
cp ${PSYCLONE_LFRIC_DIR}/gpu_offloading.py ${OPT_DIR}/global.py
# Clean previous version and compile again
rm -rf applications/gungho_model/working
LFRIC_OFFLOAD_DIRECTIVES=acc ./build/local_build.py -j ${NUM_PARALLEL} -a gungho_model -p psyclone-test -w working-acc-offload
cd applications/gungho_model/example
rm -f timer.txt gungho_model-checksums.txt # In case there were from a previous run
export OMP_NUM_THREADS=12
export CUDA_VISIBLE_DEVICES=1
mpirun -n 1 ../bin/gungho_model configuration.nml
python ${PSYCLONE_LFRIC_DIR}/compare_ouput.py ${PSYCLONE_LFRIC_DIR}/KGOs/gungho_model-checksums.txt gungho_model-checksums.txt
cat timer.txt
export VAR_TIME=$(grep "gungho_model" timer.txt | cut -d'|' -f5)
echo "time=${VAR_TIME}" >> "${GITHUB_OUTPUT}"
# PSyclone, compile and run MetOffice LFRic with all optimisations and 6 mpi ranks each with 2 OpenMP threads
- name: LFRic with all transformations
id: lfric_all_trans
run: |
# Set up environment
source /archive/psyclone-spack/psyclone-spack-Jun25/spack-repo/share/spack/setup-env.sh
spack load lfric-build-environment%gcc@${GCC_VERSION}
source .runner_venv/bin/activate
export PSYCLONE_LFRIC_DIR=${GITHUB_WORKSPACE}/examples/lfric/scripts
export PSYCLONE_CONFIG_FILE=${PSYCLONE_LFRIC_DIR}/KGOs/lfric_psyclone.cfg
export LFRIC_DIR=/archive/psyclone-tests/latest-run/lfric_apps
cd ${LFRIC_DIR}
# Use un-patched LFRic core
sed -i "s|lfric_core_rev=.*|lfric_core_rev=${LFRIC_CORE_REV}|g" dependencies.sh
sed -i "s|lfric_core_sources=.*|lfric_core_sources=|g" dependencies.sh
# PSyclone scripts must now be under 'optimisation/psykal' and be called 'global.py'
export OPT_DIR=${LFRIC_DIR}/applications/gungho_model/optimisation/psyclone-test/psykal
mkdir -p ${OPT_DIR}
cp ${PSYCLONE_LFRIC_DIR}/everything_everywhere_all_at_once.py ${OPT_DIR}/global.py
# Clean previous version and compile again
rm -rf applications/gungho_model/working
./build/local_build.py -j ${NUM_PARALLEL} -a gungho_model -p psyclone-test -v -w working-all-trans
# Run
cd applications/gungho_model/example
rm -f timer.txt gungho_model-checksums.txt # In case there were from a previous run
export OMP_NUM_THREADS=2
mpirun -n 6 ../bin/gungho_model configuration.nml
python ${PSYCLONE_LFRIC_DIR}/compare_ouput.py ${PSYCLONE_LFRIC_DIR}/KGOs/gungho_model-checksums.txt gungho_model-checksums.txt
cat timer.txt
export VAR_TIME=$(grep "gungho_model" timer.txt | cut -d'|' -f5)
echo "time=${VAR_TIME}" >> "${GITHUB_OUTPUT}"
# PSyclone, compile and run MetOffice LFRic with all optimisations and 6 mpi ranks each with 2 OpenMP threads
- name: LFRic with colour-tiling and OpenMP
id: lfric_colour_tiling
run: |
# Set up environment
source /archive/psyclone-spack/psyclone-spack-Jun25/spack-repo/share/spack/setup-env.sh
spack load lfric-build-environment%gcc@${GCC_VERSION}
source .runner_venv/bin/activate
export PSYCLONE_LFRIC_DIR=${GITHUB_WORKSPACE}/examples/lfric/scripts
export PSYCLONE_CONFIG_FILE=${PSYCLONE_LFRIC_DIR}/KGOs/lfric_psyclone.cfg
export LFRIC_DIR=/archive/psyclone-tests/latest-run/lfric_apps
cd ${LFRIC_DIR}
# Use un-patched LFRic core
sed -i "s|lfric_core_rev=.*|lfric_core_rev=${LFRIC_CORE_REV}|g" dependencies.sh
sed -i "s|lfric_core_sources=.*|lfric_core_sources=|g" dependencies.sh
# PSyclone scripts must now be under 'optimisation/psykal' and be called 'global.py'
export OPT_DIR=${LFRIC_DIR}/applications/gungho_model/optimisation/psyclone-test/psykal
mkdir -p ${OPT_DIR}
cp ${PSYCLONE_LFRIC_DIR}/tiledcolouring_and_omp.py ${OPT_DIR}/global.py
# Clean previous version and compile again
rm -rf applications/gungho_model/working
./build/local_build.py -j ${NUM_PARALLEL} -a gungho_model -p psyclone-test -v -w working-omp-tiling
# Run
cd applications/gungho_model/example
rm -f timer.txt gungho_model-checksums.txt # In case there were from a previous run
export OMP_NUM_THREADS=2
mpirun -n 6 ../bin/gungho_model configuration.nml
python ${PSYCLONE_LFRIC_DIR}/compare_ouput.py ${PSYCLONE_LFRIC_DIR}/KGOs/gungho_model-checksums.txt gungho_model-checksums.txt
cat timer.txt
export VAR_TIME=$(grep "gungho_model" timer.txt | cut -d'|' -f5)
echo "time=${VAR_TIME}" >> "${GITHUB_OUTPUT}"
- name: LFRic Atm PSyclone transmutation
id: lfric_atm
run: |
# Set up environment
source /archive/psyclone-spack/psyclone-spack-Jun25/spack-repo/share/spack/setup-env.sh
spack load lfric-build-environment%gcc@${GCC_VERSION}
source .runner_venv/bin/activate
export PSYCLONE_LFRIC_DIR=${GITHUB_WORKSPACE}/examples/lfric/scripts
export LFRIC_DIR=/archive/psyclone-tests/latest-run/lfric_apps
cd ${LFRIC_DIR}
# Use un-patched LFRic core
sed -i "s|lfric_core_rev=.*|lfric_core_rev=${LFRIC_CORE_REV}|g" dependencies.sh
sed -i "s|lfric_core_sources=.*|lfric_core_sources=|g" dependencies.sh
# Build the 'meto-ex1a' platform, that includes psyclone-transmutation
export BUILD_START="${SECONDS}"
./build/local_build.py -a lfric_atm -p meto-ex1a -j 16 -w workdir-lfric-atm-transmuted
export BUILD_ELAPSED=$((${SECONDS}-${BUILD_START}))
# Run the app with 8 threads and no MPI
# (with MPI it currently fails to initialise the mesh - even with no psyclone)
cd applications/lfric_atm/example/
OMP_NUM_THREADS=8 mpirun -n 1 ../bin/lfric_atm configuration.nml
# And compare the results
python ${PSYCLONE_LFRIC_DIR}/compare_ouput.py ${PSYCLONE_LFRIC_DIR}/KGOs/lfric_atm-checksums.txt lfric_atm-checksums.txt
cat timer.txt
export VAR_TIME=$(grep "lfric_atm" timer.txt | cut -d'|' -f5)
echo "time=${VAR_TIME}" >> "${GITHUB_OUTPUT}"
echo "build_time=${BUILD_ELAPSED}" >> "${GITHUB_OUTPUT}"
upload_if_on_mirror:
if: ${{ github.repository == 'stfc/PSyclone-mirror' }}
runs-on: ubuntu-latest
needs: run_if_on_mirror
steps:
- name: Install mongosh
run: |
sudo apt-get install gnupg
wget -qO- https://www.mongodb.org/static/pgp/server-8.0.asc | sudo tee /etc/apt/trusted.gpg.d/server-8.0.asc
echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu noble/mongodb-org/8.0 multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-8.0.list
sudo apt-get update
sudo apt-get install -y mongodb-mongosh
- name: Upload results
run: |
export COMMON_FIELDS='branch_name: "'"$GITHUB_REF_NAME"'", commit: "'"$GITHUB_SHA"'", date: new Date()'
export COMMON_FIELDS=${COMMON_FIELDS}', github_job: "'"$GITHUB_RUN_ID"'"-"'"$GITHUB_RUN_ATTEMPT"'"'
export COMMON_FIELDS=${COMMON_FIELDS}', lfric_apps_version: '"$LFRIC_APPS_REV"', system: "Rupert"'
mongosh \
"mongodb+srv://cluster0.x8ncpxi.mongodb.net/PerformanceMonitoring" \
--quiet --apiVersion 1 --username ${{ secrets.MONGODB_USERNAME }} \
--password ${{ secrets.MONGODB_PASSWORD }} \
--eval 'db.GitHub_CI.insertMany([
{
ci_test: "LFRic Passthrough with DM",
compiler:"gfortran@'"$GCC_VERSION"'",
elapsed_time: '"${{needs.run_if_on_mirror.outputs.lfric_dist_mem}}"',
'"${COMMON_FIELDS}"'
},
{
ci_test: "LFRic OpenMP offloading",
compiler:"nvhpc@'"$NVHPC_VERSION"'",
elapsed_time: '"${{needs.run_if_on_mirror.outputs.lfric_omp_offload}}"',
'"${COMMON_FIELDS}"'
},
{
ci_test: "LFRic OpenMP offloading build time",
compiler:"nvhpc@'"$NVHPC_VERSION"'",
elapsed_time: '"${{needs.run_if_on_mirror.outputs.lfric_omp_offload_build}}"',
'"${COMMON_FIELDS}"'
},
{
ci_test: "LFRic OpenACC",
compiler:"nvhpc@'"$NVHPC_VERSION"'",
elapsed_time: '"${{needs.run_if_on_mirror.outputs.lfric_acc_offload}}"',
'"${COMMON_FIELDS}"'
},
{
ci_test: "LFRic all transformations",
compiler:"gfortran@'"$GCC_VERSION"'",
elapsed_time: '"${{needs.run_if_on_mirror.outputs.lfric_all_trans}}"',
'"${COMMON_FIELDS}"'
},
{
ci_test: "LFRic colour-tiling",
compiler:"gfortran@'"$GCC_VERSION"'",
elapsed_time: '"${{needs.run_if_on_mirror.outputs.lfric_colour_tiling}}"',
'"${COMMON_FIELDS}"'
},
{
ci_test: "LFRic_atm",
compiler:"gfortran@'"$GCC_VERSION"'",
elapsed_time: '"${{needs.run_if_on_mirror.outputs.lfric_atm}}"',
'"${COMMON_FIELDS}"'
},
{
ci_test: "LFRic_atm build time",
compiler:"gfortran@'"$GCC_VERSION"'",
elapsed_time: '"${{needs.run_if_on_mirror.outputs.lfric_atm_build}}"',
'"${COMMON_FIELDS}"'
}
])'