diff --git a/.github/workflows/test_eessi.yml b/.github/workflows/test_eessi.yml index 04195dd619..3d18e56816 100644 --- a/.github/workflows/test_eessi.yml +++ b/.github/workflows/test_eessi.yml @@ -10,16 +10,18 @@ jobs: fail-fast: false matrix: EESSI_VERSION: - - 2021.12 + - 2023.06 EESSI_SOFTWARE_SUBDIR: - aarch64/generic - - aarch64/graviton2 - - aarch64/graviton3 + - aarch64/neoverse_n1 + - aarch64/neoverse_v1 - x86_64/amd/zen2 - x86_64/amd/zen3 - x86_64/intel/haswell - x86_64/intel/skylake_avx512 - x86_64/generic + EASYSTACK_FILE: + - eessi-2023.06-eb-4.7.2-2021a.yml steps: - name: Check out software-layer repository uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 @@ -41,7 +43,7 @@ jobs: export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort echo "just run check_missing_installations.sh (should use eessi-${{matrix.EESSI_VERSION}}.yml)" - ./check_missing_installations.sh + ./check_missing_installations.sh ${{matrix.EASYSTACK_FILE}} - name: Test check_missing_installations.sh with missing package (GCC/8.3.0) run: | @@ -52,18 +54,15 @@ jobs: export EESSI_OS_TYPE=linux export EESSI_SOFTWARE_SUBDIR=${{matrix.EESSI_SOFTWARE_SUBDIR}} env | grep ^EESSI | sort - echo "modify eessi-${{matrix.EESSI_VERSION}}.yml by adding a missing package (GCC/8.3.0)" - echo " GCC:" >> eessi-${{matrix.EESSI_VERSION}}.yml - echo " toolchains:" >> eessi-${{matrix.EESSI_VERSION}}.yml - echo " SYSTEM:" >> eessi-${{matrix.EESSI_VERSION}}.yml - echo " versions: '8.3.0'" >> eessi-${{matrix.EESSI_VERSION}}.yml - tail -n 4 eessi-${{matrix.EESSI_VERSION}}.yml + echo "modify easystack file by adding a missing package (GCC/8.3.0)" + echo " - GCC-8.3.0:" >> ${{matrix.EASYSTACK_FILE}} + tail -n 5 ${{matrix.EASYSTACK_FILE}} # note, check_missing_installations.sh exits 1 if a package was # missing, which is intepreted as false (exit code based, not # boolean logic), hence when the script exits 0 if no package was # missing it is interpreted as true, thus the test did not capture # the missing package - if ./check_missing_installations.sh; then + if ./check_missing_installations.sh ${{matrix.EASYSTACK_FILE}}; then echo "did NOT capture missing package; test FAILED" exit 1 else diff --git a/.github/workflows/tests_scripts.yml b/.github/workflows/tests_scripts.yml index 5c0b3893ae..8ad4928993 100644 --- a/.github/workflows/tests_scripts.yml +++ b/.github/workflows/tests_scripts.yml @@ -92,6 +92,6 @@ jobs: # since create_directory_tarballs.sh must be accessible from within build container cp -a * /tmp/ cd /tmp - ./build_container.sh run /tmp/$USER/EESSI /tmp/create_directory_tarballs.sh 2021.12 + ./build_container.sh run /tmp/$USER/EESSI /tmp/create_directory_tarballs.sh 2023.06 # check if tarballs have been produced ls -l *.tar.gz diff --git a/EESSI-pilot-install-software.sh b/EESSI-pilot-install-software.sh index d9bcf20231..287688a61f 100755 --- a/EESSI-pilot-install-software.sh +++ b/EESSI-pilot-install-software.sh @@ -137,235 +137,29 @@ else echo_green ">> MODULEPATH set up: ${MODULEPATH}" fi -REQ_EB_VERSION='4.5.0' - -# load EasyBuild module (will be installed if it's not available yet) -source ${TOPDIR}/load_easybuild_module.sh ${REQ_EB_VERSION} - -echo_green "All set, let's start installing some software in ${EASYBUILD_INSTALLPATH}..." - -# install Java with fixed custom easyblock that uses patchelf to ensure right glibc is picked up, -# see https://github.com/EESSI/software-layer/issues/123 -# and https://github.com/easybuilders/easybuild-easyblocks/pull/2557 -ok_msg="Java installed, off to a good (?) start!" -fail_msg="Failed to install Java, woopsie..." -$EB Java-11.eb --robot --include-easyblocks-from-pr 2557 -check_exit_code $? "${ok_msg}" "${fail_msg}" - -# install GCC for foss/2020a -export GCC_EC="GCC-9.3.0.eb" -echo ">> Starting slow with ${GCC_EC}..." -ok_msg="${GCC_EC} installed, yippy! Off to a good start..." -fail_msg="Installation of ${GCC_EC} failed!" -# pull in easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14453, -# which includes patch to fix build of GCC 9.3 when recent kernel headers are in place -$EB ${GCC_EC} --robot --from-pr 14453 GCCcore-9.3.0.eb -check_exit_code $? "${ok_msg}" "${fail_msg}" - -# install CMake with custom easyblock that patches CMake when --sysroot is used -echo ">> Install CMake with fixed easyblock to take into account --sysroot" -ok_msg="CMake installed!" -fail_msg="Installation of CMake failed, what the ..." -$EB CMake-3.16.4-GCCcore-9.3.0.eb --robot --include-easyblocks-from-pr 2248 -check_exit_code $? "${ok_msg}" "${fail_msg}" - -# If we're building OpenBLAS for GENERIC, we need https://github.com/easybuilders/easybuild-easyblocks/pull/1946 -echo ">> Installing OpenBLAS..." -ok_msg="Done with OpenBLAS!" -fail_msg="Installation of OpenBLAS failed!" -if [[ $GENERIC -eq 1 ]]; then - echo_yellow ">> Using https://github.com/easybuilders/easybuild-easyblocks/pull/1946 to build generic OpenBLAS." - openblas_include_easyblocks_from_pr="--include-easyblocks-from-pr 1946" -else - openblas_include_easyblocks_from_pr='' -fi -$EB $openblas_include_easyblocks_from_pr OpenBLAS-0.3.9-GCC-9.3.0.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing OpenMPI..." -ok_msg="OpenMPI installed, w00!" -fail_msg="Installation of OpenMPI failed, that's not good..." -$EB OpenMPI-4.0.3-GCC-9.3.0.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -# install Python -echo ">> Install Python 2.7.18 and Python 3.8.2..." -ok_msg="Python 2.7.18 and 3.8.2 installed, yaay!" -fail_msg="Installation of Python failed, oh no..." -$EB Python-2.7.18-GCCcore-9.3.0.eb Python-3.8.2-GCCcore-9.3.0.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing Perl..." -ok_msg="Perl installed, making progress..." -fail_msg="Installation of Perl failed, this never happens..." -# use enhanced Perl easyblock from https://github.com/easybuilders/easybuild-easyblocks/pull/2640 -# to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...) -$EB Perl-5.30.2-GCCcore-9.3.0.eb --robot --include-easyblocks-from-pr 2640 -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing Qt5..." -ok_msg="Qt5 installed, phieuw, that was a big one!" -fail_msg="Installation of Qt5 failed, that's frustrating..." -$EB Qt5-5.14.1-GCCcore-9.3.0.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -# skip test step when installing SciPy-bundle on aarch64, -# to dance around problem with broken numpy tests; -# cfr. https://github.com/easybuilders/easybuild-easyconfigs/issues/11959 -echo ">> Installing SciPy-bundle" -ok_msg="SciPy-bundle installed, yihaa!" -fail_msg="SciPy-bundle installation failed, bummer..." -SCIPY_EC=SciPy-bundle-2020.03-foss-2020a-Python-3.8.2.eb -if [[ "$(uname -m)" == "aarch64" ]]; then - $EB $SCIPY_EC --robot --skip-test-step -else - $EB $SCIPY_EC --robot -fi -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing GROMACS..." -ok_msg="GROMACS installed, wow!" -fail_msg="Installation of GROMACS failed, damned..." -$EB GROMACS-2020.1-foss-2020a-Python-3.8.2.eb GROMACS-2020.4-foss-2020a-Python-3.8.2.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -# note: compiling OpenFOAM is memory hungry (16GB is not enough with 8 cores)! -# 32GB is sufficient to build with 16 cores -echo ">> Installing OpenFOAM (twice!)..." -ok_msg="OpenFOAM installed, now we're talking!" -fail_msg="Installation of OpenFOAM failed, we were so close..." -$EB OpenFOAM-8-foss-2020a.eb OpenFOAM-v2006-foss-2020a.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then - echo ">> Installing QuantumESPRESSO..." - ok_msg="QuantumESPRESSO installed, let's go quantum!" - fail_msg="Installation of QuantumESPRESSO failed, did somebody observe it?!" - $EB QuantumESPRESSO-6.6-foss-2020a.eb --robot - check_exit_code $? "${ok_msg}" "${fail_msg}" -fi +for eb_version in '4.7.2'; do -echo ">> Installing R 4.0.0 (better be patient)..." -ok_msg="R installed, wow!" -fail_msg="Installation of R failed, so sad..." -$EB R-4.0.0-foss-2020a.eb --robot --parallel-extensions-install --experimental -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing Bioconductor 3.11 bundle..." -ok_msg="Bioconductor installed, enjoy!" -fail_msg="Installation of Bioconductor failed, that's annoying..." -$EB R-bundle-Bioconductor-3.11-foss-2020a-R-4.0.0.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing TensorFlow 2.3.1..." -ok_msg="TensorFlow 2.3.1 installed, w00!" -fail_msg="Installation of TensorFlow failed, why am I not surprised..." -$EB TensorFlow-2.3.1-foss-2020a-Python-3.8.2.eb --robot --include-easyblocks-from-pr 2218 -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing Horovod 0.21.3..." -ok_msg="Horovod installed! Go do some parallel training!" -fail_msg="Horovod installation failed. There comes the headache..." -$EB Horovod-0.21.3-foss-2020a-TensorFlow-2.3.1-Python-3.8.2.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -if [ ! "${EESSI_CPU_FAMILY}" = "ppc64le" ]; then - - echo ">> Installing code-server 3.7.3..." - ok_msg="code-server 3.7.3 installed, now you can use VS Code!" - fail_msg="Installation of code-server failed, that's going to be hard to fix..." - $EB code-server-3.7.3.eb --robot - check_exit_code $? "${ok_msg}" "${fail_msg}" -fi + # load EasyBuild module (will be installed if it's not available yet) + source ${TOPDIR}/load_easybuild_module.sh ${eb_version} -echo ">> Installing RStudio-Server 1.3.1093..." -ok_msg="RStudio-Server installed, enjoy!" -fail_msg="Installation of RStudio-Server failed, might be OS deps..." -$EB RStudio-Server-1.3.1093-foss-2020a-Java-11-R-4.0.0.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing OSU-Micro-Benchmarks 5.6.3..." -ok_msg="OSU-Micro-Benchmarks installed, yihaa!" -fail_msg="Installation of OSU-Micro-Benchmarks failed, that's unexpected..." -$EB OSU-Micro-Benchmarks-5.6.3-gompi-2020a.eb -r -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing Spark 3.1.1..." -ok_msg="Spark installed, set off the fireworks!" -fail_msg="Installation of Spark failed, no fireworks this time..." -$EB Spark-3.1.1-foss-2020a-Python-3.8.2.eb -r -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing IPython 7.15.0..." -ok_msg="IPython installed, launch your Jupyter Notebooks!" -fail_msg="Installation of IPython failed, that's unexpected..." -$EB IPython-7.15.0-foss-2020a-Python-3.8.2.eb -r -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing WRF 3.9.1.1..." -ok_msg="WRF installed, it's getting hot in here!" -fail_msg="Installation of WRF failed, that's unexpected..." -OMPI_MCA_pml=ucx UCX_TLS=tcp $EB WRF-3.9.1.1-foss-2020a-dmpar.eb -r --include-easyblocks-from-pr 2648 -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing R 4.1.0 (better be patient)..." -ok_msg="R installed, wow!" -fail_msg="Installation of R failed, so sad..." -$EB --from-pr 14821 X11-20210518-GCCcore-10.3.0.eb -r && $EB --from-pr 16011 R-4.1.0-foss-2021a.eb --robot --parallel-extensions-install --experimental -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing Nextflow 22.10.1..." -ok_msg="Nextflow installed, the work must flow..." -fail_msg="Installation of Nextflow failed, that's unexpected..." -$EB -r --from-pr 16531 Nextflow-22.10.1.eb -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing OSU-Micro-Benchmarks/5.7.1-gompi-2021a..." -ok_msg="OSU-Micro-Benchmarks installed, yihaa!" -fail_msg="Installation of OSU-Micro-Benchmarks failed, that's unexpected..." -$EB OSU-Micro-Benchmarks-5.7.1-gompi-2021a.eb -r -check_exit_code $? "${ok_msg}" "${fail_msg}" - -echo ">> Installing EasyBuild 4.5.1..." -ok_msg="EasyBuild v4.5.1 installed" -fail_msg="EasyBuild v4.5.1 failed to install" -$EB --from-pr 14545 --include-easyblocks-from-pr 2805 -check_exit_code $? "${ok_msg}" "${fail_msg}" - -LMOD_IGNORE_CACHE=1 module swap EasyBuild/4.5.1 -check_exit_code $? "Swapped to EasyBuild/4.5.1" "Couldn't swap to EasyBuild/4.5.1" - -echo ">> Installing SciPy-bundle with foss/2021a..." -ok_msg="SciPy-bundle with foss/2021a installed, welcome to the modern age" -fail_msg="Installation of SciPy-bundle with foss/2021a failed, back to the stone age..." -# use GCCcore easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14454 -# which includes patch to fix installation with recent Linux kernel headers -$EB --from-pr 14454 GCCcore-10.3.0.eb --robot -# use enhanced Perl easyblock from https://github.com/easybuilders/easybuild-easyblocks/pull/2640 -# to avoid trouble when using long installation prefix (for example with EESSI pilot 2021.12 on skylake_avx512...) -$EB Perl-5.32.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2640 -# use enhanced CMake easyblock to patch CMake's UnixPaths.cmake script if --sysroot is set -# from https://github.com/easybuilders/easybuild-easyblocks/pull/2248 -$EB CMake-3.20.1-GCCcore-10.3.0.eb --robot --include-easyblocks-from-pr 2248 -# use Rust easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/14584 -# that includes patch to fix bootstrap problem when using alternate sysroot -$EB --from-pr 14584 Rust-1.52.1-GCCcore-10.3.0.eb --robot -# use OpenBLAS easyconfig from https://github.com/easybuilders/easybuild-easyconfigs/pull/15885 -# which includes a patch to fix installation on POWER -$EB $openblas_include_easyblocks_from_pr --from-pr 15885 OpenBLAS-0.3.15-GCC-10.3.0.eb --robot -# ignore failing FlexiBLAS tests when building on POWER; -# some tests are failing due to a segmentation fault due to "invalid memory reference", -# see also https://github.com/easybuilders/easybuild-easyconfigs/pull/12476; -# using -fstack-protector-strong -fstack-clash-protection should fix that, -# but it doesn't for some reason when building for ppc64le/generic... -if [ "${EESSI_SOFTWARE_SUBDIR}" = "ppc64le/generic" ]; then - $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb --ignore-test-failure -else - $EB FlexiBLAS-3.0.4-GCC-10.3.0.eb -fi + echo_green "All set, let's start installing some software with EasyBuild v${eb_version} in ${EASYBUILD_INSTALLPATH}..." -$EB SciPy-bundle-2021.05-foss-2021a.eb --robot -check_exit_code $? "${ok_msg}" "${fail_msg}" + for gen in '2021a'; do + + es="eessi-${EESSI_PILOT_VERSION}-eb-${eb_version}-${gen}.yml" + + if [ -f ${es} ]; then + echo_green "Feeding easystack file ${es} to EasyBuild..." + + ${EB} --easystack ${TOPDIR}/${es} --robot + + $TOPDIR/check_missing_installations.sh ${TOPDIR}/${es} + else + fatal_error "Easystack file ${es} not found!" + fi + done + +done ### add packages here @@ -378,7 +172,5 @@ fi $TOPDIR/update_lmod_cache.sh ${EPREFIX} ${EASYBUILD_INSTALLPATH} -$TOPDIR/check_missing_installations.sh - echo ">> Cleaning up ${TMPDIR}..." rm -r ${TMPDIR} diff --git a/bot/check-result.sh b/bot/check-result.sh new file mode 100755 index 0000000000..182b8555a8 --- /dev/null +++ b/bot/check-result.sh @@ -0,0 +1,492 @@ +#!/bin/bash +# +# Script to check the result of building the EESSI software layer. +# Intended use is that it is called by a (batch) job running on a compute +# node. +# +# This script is part of the EESSI software layer, see +# https://github.com/EESSI/software-layer.git +# +# author: Thomas Roeblitz (@trz42) +# +# license: GPLv2 +# + +# result cases + +# - SUCCESS (all of) +# - working directory contains slurm-JOBID.out file +# - working directory contains eessi*tar.gz +# - no message ERROR +# - no message FAILED +# - no message ' required modules missing:' +# - one or more of 'No missing installations' +# - message regarding created tarball +# - FAILED (one of ... implemented as NOT SUCCESS) +# - no slurm-JOBID.out file +# - no tarball +# - message with ERROR +# - message with FAILED +# - message with ' required modules missing:' +# - no message regarding created tarball + +# stop as soon as something fails +# set -e + +TOPDIR=$(dirname $(realpath $0)) + +source ${TOPDIR}/../scripts/utils.sh +source ${TOPDIR}/../scripts/cfg_files.sh + +# defaults +export JOB_CFG_FILE="${JOB_CFG_FILE_OVERRIDE:=./cfg/job.cfg}" + +# check if ${JOB_CFG_FILE} exists +if [[ ! -r "${JOB_CFG_FILE}" ]]; then + echo_red "job config file (JOB_CFG_FILE=${JOB_CFG_FILE}) does not exist or not readable" +else + echo "bot/check-result.sh: showing ${JOB_CFG_FILE} from software-layer side" + cat ${JOB_CFG_FILE} + + echo "bot/check-result.sh: obtaining configuration settings from '${JOB_CFG_FILE}'" + cfg_load ${JOB_CFG_FILE} +fi + +display_help() { + echo "usage: $0 [OPTIONS]" + echo " OPTIONS:" + echo " -h | --help - display this usage information [default: false]" + echo " -v | --verbose - display more information [default: false]" +} + +# set defaults for command line arguments +VERBOSE=0 + +POSITIONAL_ARGS=() + +while [[ $# -gt 0 ]]; do + case $1 in + -h|--help) + display_help + exit 0 + ;; + -v|--verbose) + VERBOSE=1 + shift 1 + ;; + --) + shift + POSITIONAL_ARGS+=("$@") # save positional args + break + ;; + -*|--*) + fatal_error "Unknown option: $1" "${CMDLINE_ARG_UNKNOWN_EXITCODE}" + ;; + *) # No more options + POSITIONAL_ARGS+=("$1") # save positional arg + shift + ;; + esac +done + +set -- "${POSITIONAL_ARGS[@]}" + +job_dir=${PWD} + +[[ ${VERBOSE} -ne 0 ]] && echo ">> analysing job in directory ${job_dir}" + +job_out="slurm-${SLURM_JOB_ID}.out" +[[ ${VERBOSE} -ne 0 ]] && echo ">> searching for job output file(s) matching '"${job_out}"'" +if [[ -f ${job_out} ]]; then + SLURM=1 + [[ ${VERBOSE} -ne 0 ]] && echo " found slurm output file '"${job_out}"'" +else + SLURM=0 + [[ ${VERBOSE} -ne 0 ]] && echo " Slurm output file '"${job_out}"' NOT found" +fi + +ERROR=-1 +if [[ ${SLURM} -eq 1 ]]; then + GP_error='ERROR: ' + grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_error}") + [[ $? -eq 0 ]] && ERROR=1 || ERROR=0 + # have to be careful to not add searched for pattern into slurm out file + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_error}"'" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +fi + +FAILED=-1 +if [[ ${SLURM} -eq 1 ]]; then + GP_failed='FAILED: ' + grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_failed}") + [[ $? -eq 0 ]] && FAILED=1 || FAILED=0 + # have to be careful to not add searched for pattern into slurm out file + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_failed}"'" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +fi + +MISSING=-1 +if [[ ${SLURM} -eq 1 ]]; then + GP_req_missing=' required modules missing:' + grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_req_missing}") + [[ $? -eq 0 ]] && MISSING=1 || MISSING=0 + # have to be careful to not add searched for pattern into slurm out file + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_req_missing}"'" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +fi + +NO_MISSING=-1 +if [[ ${SLURM} -eq 1 ]]; then + GP_no_missing='No missing installations' + grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_no_missing}") + [[ $? -eq 0 ]] && NO_MISSING=1 || NO_MISSING=0 + # have to be careful to not add searched for pattern into slurm out file + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_no_missing}"'" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +fi + +TGZ=-1 +TARBALL= +if [[ ${SLURM} -eq 1 ]]; then + GP_tgz_created="\.tar\.gz created!" + grep_out=$(grep -v "^>> searching for " ${job_dir}/${job_out} | grep "${GP_tgz_created}" | sort -u) + if [[ $? -eq 0 ]]; then + TGZ=1 + TARBALL=$(echo ${grep_out} | sed -e 's@^.*/\(eessi[^/ ]*\) .*$@\1@') + else + TGZ=0 + fi + # have to be careful to not add searched for pattern into slurm out file + [[ ${VERBOSE} -ne 0 ]] && echo ">> searching for '"${GP_tgz_created}"'" + [[ ${VERBOSE} -ne 0 ]] && echo "${grep_out}" +fi + +[[ ${VERBOSE} -ne 0 ]] && echo "SUMMARY: ${job_dir}/${job_out}" +[[ ${VERBOSE} -ne 0 ]] && echo " : ()" +[[ ${VERBOSE} -ne 0 ]] && echo " ERROR......: $([[ $ERROR -eq 1 ]] && echo 'yes' || echo 'no') (no)" +[[ ${VERBOSE} -ne 0 ]] && echo " FAILED.....: $([[ $FAILED -eq 1 ]] && echo 'yes' || echo 'no') (no)" +[[ ${VERBOSE} -ne 0 ]] && echo " REQ_MISSING: $([[ $MISSING -eq 1 ]] && echo 'yes' || echo 'no') (no)" +[[ ${VERBOSE} -ne 0 ]] && echo " NO_MISSING.: $([[ $NO_MISSING -eq 1 ]] && echo 'yes' || echo 'no') (yes)" +[[ ${VERBOSE} -ne 0 ]] && echo " TGZ_CREATED: $([[ $TGZ -eq 1 ]] && echo 'yes' || echo 'no') (yes)" + +job_result_file=_bot_job${SLURM_JOB_ID}.result + +if [[ ${SLURM} -eq 1 ]] && \ + [[ ${ERROR} -eq 0 ]] && \ + [[ ${FAILED} -eq 0 ]] && \ + [[ ${MISSING} -eq 0 ]] && \ + [[ ${NO_MISSING} -eq 1 ]] && \ + [[ ${TGZ} -eq 1 ]] && \ + [[ ! -z ${TARBALL} ]]; then + # SUCCESS + status="SUCCESS" + summary=":grin: SUCCESS" +else + # FAILURE + status="FAILURE" + summary=":cry: FAILURE" +fi + +### Example details/descriptions +# Note, final string must not contain any line breaks. Below example include +# line breaks for the sake of readability. In case of FAILURE, the structure is +# very similar (incl. information about Artefacts if any was produced), however, +# under Details some lines will be marked with :heavy_multiplication_x: +#
+# :grin: SUCCESS _(click triangle for details)_ +#
+#
_Details_
+#
+# :heavy_check_mark: job output file slurm-4682.out
+# :heavy_check_mark: no message matching ERROR:
+# :heavy_check_mark: no message matching FAILED:
+# :heavy_check_mark: no message matching required modules missing:
+# :heavy_check_mark: found message(s) matching No missing installations
+# :heavy_check_mark: found message matching tar.gz created!
+#
+#
_Artefacts_
+#
+#
+# eessi-2023.06-software-linux-x86_64-generic-1682696567.tar.gz +# size: 234 MiB (245366784 bytes)
+# entries: 1234
+# modules under _2023.06/software/linux/x86_64/generic/modules/all/_
+#
+#           GCC/9.3.0.lua
+# GCC/10.3.0.lua
+# OpenSSL/1.1.lua +#
+# software under _2023.06/software/linux/x86_64/generic/software/_ +#
+#           GCC/9.3.0/
+# CMake/3.20.1-GCCcore-10.3.0/
+# OpenMPI/4.1.1-GCC-10.3.0/ +#
+# other under _2023.06/software/linux/x86_64/generic/_ +#
+#           .lmod/cache/spiderT.lua
+# .lmod/cache/spiderT.luac_5.1
+# .lmod/cache/timestamp +#
+#
+#
+#
+#
+# +#
+# :cry: FAILURE _(click triangle for details)_ +#
+#
_Details_
+#
+# :heavy_check_mark: job output file slurm-4682.out
+# :heavy_multiplication_x: no message matching ERROR:
+# :heavy_check_mark: no message matching FAILED:
+# :heavy_multiplication_x: no message matching required modules missing:
+# :heavy_check_mark: found message(s) matching No missing installations
+# :heavy_check_mark: found message matching tar.gz created!
+#
+#
_Artefacts_
+#
+# No artefacts were created or found. +#
+#
+#
+### + +# construct and write complete PR comment details: implements third alternative +comment_template="
__SUMMARY_FMT__
__DETAILS_FMT____ARTEFACTS_FMT__
" +comment_summary_fmt="__SUMMARY__ _(click triangle for details)_" +comment_details_fmt="
_Details_
__DETAILS_LIST__
" +comment_success_item_fmt=":heavy_check_mark: __ITEM__" +comment_failure_item_fmt=":heavy_multiplication_x: __ITEM__" +comment_artefacts_fmt="
_Artefacts_
__ARTEFACTS_LIST__
" +comment_artefact_details_fmt="
__ARTEFACT_SUMMARY____ARTEFACT_DETAILS__
" + +function print_br_item() { + format="${1}" + item="${2}" + echo -n "${format//__ITEM__/${item}}
" +} + +function print_br_item2() { + format="${1}" + item="${2}" + item2="${3}" + format1="${format//__ITEM__/${item}}" + echo -n "${format1//__ITEM2__/${item2}}
" +} + +function print_code_item() { + format="${1}" + item="${2}" + echo -n "${format//__ITEM__/${item}}" +} + +function print_dd_item() { + format="${1}" + item="${2}" + echo -n "
${format//__ITEM__/${item}}
" +} + +function print_list_item() { + format="${1}" + item="${2}" + echo -n "
  • ${format//__ITEM__/${item}}
  • " +} + +function print_pre_item() { + format="${1}" + item="${2}" + echo -n "
    ${format//__ITEM__/${item}}
    " +} + +function success() { + format="${comment_success_item_fmt}" + item="$1" + print_br_item "${format}" "${item}" +} + +function failure() { + format="${comment_failure_item_fmt}" + item="$1" + print_br_item "${format}" "${item}" +} + +function add_detail() { + actual=${1} + expected=${2} + success_msg="${3}" + failure_msg="${4}" + if [[ ${actual} -eq ${expected} ]]; then + success "${success_msg}" + else + failure "${failure_msg}" + fi +} + +echo "[RESULT]" > ${job_result_file} +echo -n "comment_description = " >> ${job_result_file} + +# construct values for placeholders in comment_template: +# - __SUMMARY_FMT__ -> variable $comment_summary +# - __DETAILS_FMT__ -> variable $comment_details +# - __ARTEFACTS_FMT__ -> variable $comment_artefacts + +comment_summary="${comment_summary_fmt/__SUMMARY__/${summary}}" + +# first construct comment_details_list, abbreviated CoDeList +# then use it to set comment_details +CoDeList="" + +success_msg="job output file ${job_out}" +failure_msg="no job output file ${job_out}" +CoDeList=${CoDeList}$(add_detail ${SLURM} 1 "${success_msg}" "${failure_msg}") + +success_msg="no message matching ${GP_error}" +failure_msg="found message matching ${GP_error}" +CoDeList=${CoDeList}$(add_detail ${ERROR} 0 "${success_msg}" "${failure_msg}") + +success_msg="no message matching ${GP_failed}" +failure_msg="found message matching ${GP_failed}" +CoDeList=${CoDeList}$(add_detail ${FAILED} 0 "${success_msg}" "${failure_msg}") + +success_msg="no message matching ${GP_req_missing}" +failure_msg="found message matching ${GP_req_missing}" +CoDeList=${CoDeList}$(add_detail ${MISSING} 0 "${success_msg}" "${failure_msg}") + +success_msg="found message(s) matching ${GP_no_missing}" +failure_msg="no message matching ${GP_no_missing}" +CoDeList=${CoDeList}$(add_detail ${NO_MISSING} 1 "${success_msg}" "${failure_msg}") + +success_msg="found message matching ${GP_tgz_created}" +failure_msg="no message matching ${GP_tgz_created}" +CoDeList=${CoDeList}$(add_detail ${TGZ} 1 "${success_msg}" "${failure_msg}") + +comment_details="${comment_details_fmt/__DETAILS_LIST__/${CoDeList}}" + + +# first construct comment_artefacts_list, abbreviated CoArList +# then use it to set comment_artefacts +CoArList="" + +# TARBALL should only contain a single tarball +if [[ ! -z ${TARBALL} ]]; then + # Example of the detailed information for a tarball. The actual result MUST be a + # single line (no '\n') or it would break the structure of the markdown table + # that holds status updates of a bot job. + # + #
    + #
    + # eessi-2023.06-software-linux-x86_64-generic-1682696567.tar.gz + # size: 234 MiB (245366784 bytes)
    + # entries: 1234
    + # modules under _2023.06/software/linux/x86_64/intel/cascadelake/modules/all/_
    + #
    +    #       GCC/9.3.0.lua
    + # GCC/10.3.0.lua
    + # OpenSSL/1.1.lua + #
    + # software under _2023.06/software/linux/x86_64/intel/cascadelake/software/_ + #
    +    #       GCC/9.3.0/
    + # CMake/3.20.1-GCCcore-10.3.0/
    + # OpenMPI/4.1.1-GCC-10.3.0/ + #
    + # other under _2023.06/software/linux/x86_64/intel/cascadelake/_ + #
    +    #       .lmod/cache/spiderT.lua
    + # .lmod/cache/spiderT.luac_5.1
    + # .lmod/cache/timestamp + #
    + #
    + #
    + size="$(stat --dereference --printf=%s ${TARBALL})" + size_mib=$((${size} >> 20)) + tmpfile=$(mktemp --tmpdir=. tarfiles.XXXX) + tar tf ${TARBALL} > ${tmpfile} + entries=$(cat ${tmpfile} | wc -l) + # determine prefix from job config: VERSION/software/OS_TYPE/CPU_FAMILY/ARCHITECTURE + # e.g., 2023.06/software/linux/x86_64/intel/skylake_avx512 + # cfg/job.cfg contains (only the attributes to be used are shown below): + # [repository] + # repo_version = 2023.06 + # [architecture] + # os_type = linux + # software_subdir = x86_64/intel/skylake_avx512 + repo_version=$(cfg_get_value "repository" "repo_version") + os_type=$(cfg_get_value "architecture" "os_type") + software_subdir=$(cfg_get_value "architecture" "software_subdir") + prefix="${repo_version}/software/${os_type}/${software_subdir}" + + # extract directories/entries from tarball content + modules_entries=$(grep "${prefix}/modules" ${tmpfile}) + software_entries=$(grep "${prefix}/software" ${tmpfile}) + other_entries=$(cat ${tmpfile} | grep -v "${prefix}/modules" | grep -v "${prefix}/software") + other_shortened=$(echo "${other_entries}" | sed -e "s@^.*${prefix}/@@" | sort -u) + modules=$(echo "${modules_entries}" | grep "/all/.*/.*lua$" | sed -e 's@^.*/\([^/]*/[^/]*.lua\)$@\1@' | sort -u) + software_pkgs=$(echo "${software_entries}" | sed -e "s@${prefix}/software/@@" | awk -F/ '{if (NR >= 2) {print $1 "/" $2}}' | sort -u) + + artefact_summary="$(print_code_item '__ITEM__' ${TARBALL})" + CoArList="" + CoArList="${CoArList}$(print_br_item2 'size: __ITEM__ MiB (__ITEM2__ bytes)' ${size_mib} ${size})" + CoArList="${CoArList}$(print_br_item 'entries: __ITEM__' ${entries})" + CoArList="${CoArList}$(print_br_item 'modules under ___ITEM___' ${prefix}/modules/all)" + CoArList="${CoArList}
    "
    +    if [[ ! -z ${modules} ]]; then
    +        while IFS= read -r mod ; do
    +            CoArList="${CoArList}$(print_br_item '__ITEM__' ${mod})"
    +        done <<< "${modules}"
    +    else
    +        CoArList="${CoArList}$(print_br_item '__ITEM__' 'no module files in tarball')"
    +    fi
    +    CoArList="${CoArList}
    " + CoArList="${CoArList}$(print_br_item 'software under ___ITEM___' ${prefix}/software)" + CoArList="${CoArList}
    "
    +    if [[ ! -z ${software_pkgs} ]]; then
    +        while IFS= read -r sw_pkg ; do
    +            CoArList="${CoArList}$(print_br_item '__ITEM__' ${sw_pkg})"
    +        done <<< "${software_pkgs}"
    +    else
    +        CoArList="${CoArList}$(print_br_item '__ITEM__' 'no software packages in tarball')"
    +    fi
    +    CoArList="${CoArList}
    " + CoArList="${CoArList}$(print_br_item 'other under ___ITEM___' ${prefix})" + CoArList="${CoArList}
    "
    +    if [[ ! -z ${other_shortened} ]]; then
    +        while IFS= read -r other ; do
    +            CoArList="${CoArList}$(print_br_item '__ITEM__' ${other})"
    +        done <<< "${other_shortened}"
    +    else
    +        CoArList="${CoArList}$(print_br_item '__ITEM__' 'no other files in tarball')"
    +    fi
    +    CoArList="${CoArList}
    " +else + CoArList="${CoArList}$(print_dd_item 'No artefacts were created or found.' '')" +fi + +comment_artefacts_details="${comment_artefact_details_fmt/__ARTEFACT_SUMMARY__/${artefact_summary}}" +comment_artefacts_details="${comment_artefacts_details/__ARTEFACT_DETAILS__/${CoArList}}" +comment_artefacts="${comment_artefacts_fmt/__ARTEFACTS_LIST__/${comment_artefacts_details}}" + +# now put all pieces together creating comment_details from comment_template +comment_description=${comment_template/__SUMMARY_FMT__/${comment_summary}} +comment_description=${comment_description/__DETAILS_FMT__/${comment_details}} +comment_description=${comment_description/__ARTEFACTS_FMT__/${comment_artefacts}} + +echo "${comment_description}" >> ${job_result_file} + +# add overall result: SUCCESS, FAILURE, UNKNOWN + artefacts +# - this should make use of subsequent steps such as deploying a tarball more +# efficient +echo "status = ${status}" >> ${job_result_file} +echo "artefacts = " >> ${job_result_file} +echo "${TARBALL}" | sed -e 's/^/ /g' >> ${job_result_file} + +# remove tmpfile +if [[ -f ${tmpfile} ]]; then + rm ${tmpfile} +fi + +# exit script with value that reflects overall job result: SUCCESS (0), FAILURE (1) +test "${status}" == "SUCCESS" +exit $? diff --git a/check_missing_installations.sh b/check_missing_installations.sh index 4a5316c09f..3627d1d0b5 100755 --- a/check_missing_installations.sh +++ b/check_missing_installations.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Script to check for missing installations in EESSI pilot software stack (version 2021.12) +# Script to check for missing installations in EESSI pilot software stack (version 2023.06) # # author: Kenneth Hoste (@boegel) # author: Thomas Roeblitz (@trz42) @@ -10,10 +10,11 @@ TOPDIR=$(dirname $(realpath $0)) -if [ -z ${EESSI_PILOT_VERSION} ]; then - echo "ERROR: \${EESSI_PILOT_VERSION} must be set to run $0!" >&2 +if [ $# -ne 1 ]; then + echo "ERROR: Usage: $0 " >&2 exit 1 fi +easystack=$1 LOCAL_TMPDIR=$(mktemp -d) @@ -23,9 +24,7 @@ source $TOPDIR/configure_easybuild echo ">> Checking for missing installations in ${EASYBUILD_INSTALLPATH}..." eb_missing_out=$LOCAL_TMPDIR/eb_missing.out -# we need to use --from-pr to pull in some easyconfigs that are not available in EasyBuild version being used -# PR #16531: Nextflow-22.10.1.eb -${EB:-eb} --from-pr 16531 --easystack eessi-${EESSI_PILOT_VERSION}.yml --experimental --missing 2>&1 | tee ${eb_missing_out} +${EB:-eb} --easystack ${easystack} --missing 2>&1 | tee ${eb_missing_out} exit_code=${PIPESTATUS[0]} ok_msg="Command 'eb --missing ...' succeeded, analysing output..." diff --git a/configure_easybuild b/configure_easybuild index 245553f342..23f3920154 100644 --- a/configure_easybuild +++ b/configure_easybuild @@ -36,3 +36,6 @@ fi export EASYBUILD_FILTER_DEPS=$DEPS_TO_FILTER export EASYBUILD_MODULE_EXTENSIONS=1 + +# need to enable use of experimental features, since we're using easystack files +export EASYBUILD_EXPERIMENTAL=1 diff --git a/eb_hooks.py b/eb_hooks.py index 2fba925b01..787e990e75 100644 --- a/eb_hooks.py +++ b/eb_hooks.py @@ -11,6 +11,13 @@ from easybuild.tools.systemtools import AARCH64, POWER, X86_64, get_cpu_architecture, get_cpu_features from easybuild.tools.toolchain.compiler import OPTARCH_GENERIC +# prefer importing LooseVersion from easybuild.tools, but fall back to distuils in case EasyBuild <= 4.7.0 is used +try: + from easybuild.tools import LooseVersion +except ImportError: + from distutils.version import LooseVersion + + EESSI_RPATH_OVERRIDE_ATTR = 'orig_rpath_override_dirs' @@ -56,13 +63,6 @@ def parse_hook(ec, *args, **kwargs): PARSE_HOOKS[ec.name](ec, eprefix) -def pre_configure_hook(self, *args, **kwargs): - """Main pre-configure hook: trigger custom functions based on software name.""" - - if self.name in PRE_CONFIGURE_HOOKS: - PRE_CONFIGURE_HOOKS[self.name](self, *args, **kwargs) - - def pre_prepare_hook(self, *args, **kwargs): """Main pre-prepare hook: trigger custom functions.""" @@ -91,7 +91,7 @@ def pre_prepare_hook(self, *args, **kwargs): mpi_family, rpath_override_dirs) -def gcc_postprepare(self, *args, **kwargs): +def post_prepare_hook_gcc_prefixed_ld_rpath_wrapper(self, *args, **kwargs): """ Post-configure hook for GCCcore: - copy RPATH wrapper script for linker commands to also have a wrapper in place with system type prefix like 'x86_64-pc-linux-gnu' @@ -121,6 +121,7 @@ def gcc_postprepare(self, *args, **kwargs): else: raise EasyBuildError("GCCcore-specific hook triggered for non-GCCcore easyconfig?!") + def post_prepare_hook(self, *args, **kwargs): """Main post-prepare hook: trigger custom functions.""" @@ -134,7 +135,7 @@ def post_prepare_hook(self, *args, **kwargs): POST_PREPARE_HOOKS[self.name](self, *args, **kwargs) -def cgal_toolchainopts_precise(ec, eprefix): +def parse_hook_cgal_toolchainopts_precise(ec, eprefix): """Enable 'precise' rather than 'strict' toolchain option for CGAL on POWER.""" if ec.name == 'CGAL': if get_cpu_architecture() == POWER: @@ -147,7 +148,7 @@ def cgal_toolchainopts_precise(ec, eprefix): raise EasyBuildError("CGAL-specific hook triggered for non-CGAL easyconfig?!") -def fontconfig_add_fonts(ec, eprefix): +def parse_hook_fontconfig_add_fonts(ec, eprefix): """Inject --with-add-fonts configure option for fontconfig.""" if ec.name == 'fontconfig': # make fontconfig aware of fonts included with compat layer @@ -158,7 +159,7 @@ def fontconfig_add_fonts(ec, eprefix): raise EasyBuildError("fontconfig-specific hook triggered for non-fontconfig easyconfig?!") -def ucx_eprefix(ec, eprefix): +def parse_hook_ucx_eprefix(ec, eprefix): """Make UCX aware of compatibility layer via additional configuration options.""" if ec.name == 'UCX': ec.update('configopts', '--with-sysroot=%s' % eprefix) @@ -174,7 +175,19 @@ def pre_configure_hook(self, *args, **kwargs): PRE_CONFIGURE_HOOKS[self.name](self, *args, **kwargs) -def libfabric_disable_psm3_x86_64_generic(self, *args, **kwargs): +def pre_configure_hook_openblas_optarch_generic(self, *args, **kwargs): + """ + Pre-configure hook for OpenBLAS: add DYNAMIC_ARCH=1 to build/test/install options when using --optarch=GENERIC + """ + if self.name == 'OpenBLAS': + if build_option('optarch') == OPTARCH_GENERIC: + for step in ('build', 'test', 'install'): + self.cfg.update(f'{step}opts', "DYNAMIC_ARCH=1") + else: + raise EasyBuildError("OpenBLAS-specific hook triggered for non-OpenBLAS easyconfig?!") + + +def pre_configure_hook_libfabric_disable_psm3_x86_64_generic(self, *args, **kwargs): """Add --disable-psm3 to libfabric configure options when building with --optarch=GENERIC on x86_64.""" if self.name == 'libfabric': if get_cpu_architecture() == X86_64: @@ -187,7 +200,7 @@ def libfabric_disable_psm3_x86_64_generic(self, *args, **kwargs): raise EasyBuildError("libfabric-specific hook triggered for non-libfabric easyconfig?!") -def metabat_preconfigure(self, *args, **kwargs): +def pre_configure_hook_metabat_filtered_zlib_dep(self, *args, **kwargs): """ Pre-configure hook for MetaBAT: - take into account that zlib is a filtered dependency, @@ -201,7 +214,7 @@ def metabat_preconfigure(self, *args, **kwargs): raise EasyBuildError("MetaBAT-specific hook triggered for non-MetaBAT easyconfig?!") -def wrf_preconfigure(self, *args, **kwargs): +def pre_configure_hook_wrf_aarch64(self, *args, **kwargs): """ Pre-configure hook for WRF: - patch arch/configure_new.defaults so building WRF with foss toolchain works on aarch64 @@ -217,17 +230,18 @@ def wrf_preconfigure(self, *args, **kwargs): PARSE_HOOKS = { - 'CGAL': cgal_toolchainopts_precise, - 'fontconfig': fontconfig_add_fonts, - 'UCX': ucx_eprefix, + 'CGAL': parse_hook_cgal_toolchainopts_precise, + 'fontconfig': parse_hook_fontconfig_add_fonts, + 'UCX': parse_hook_ucx_eprefix, } POST_PREPARE_HOOKS = { - 'GCCcore': gcc_postprepare, + 'GCCcore': post_prepare_hook_gcc_prefixed_ld_rpath_wrapper, } PRE_CONFIGURE_HOOKS = { - 'libfabric': libfabric_disable_psm3_x86_64_generic, - 'MetaBAT': metabat_preconfigure, - 'WRF': wrf_preconfigure, + 'libfabric': pre_configure_hook_libfabric_disable_psm3_x86_64_generic, + 'MetaBAT': pre_configure_hook_metabat_filtered_zlib_dep, + 'OpenBLAS': pre_configure_hook_openblas_optarch_generic, + 'WRF': pre_configure_hook_wrf_aarch64, } diff --git a/eessi-2023.06-eb-4.7.2-2021a.yml b/eessi-2023.06-eb-4.7.2-2021a.yml new file mode 100644 index 0000000000..9b1312ba9a --- /dev/null +++ b/eessi-2023.06-eb-4.7.2-2021a.yml @@ -0,0 +1,6 @@ +easyconfigs: + - GCC-10.3.0 + - CMake-3.20.1-GCCcore-10.3.0.eb: + options: + include-easyblocks-from-pr: 2248 + - foss-2021a diff --git a/init/eessi_defaults b/init/eessi_defaults index f482cbc269..0143dc38ab 100644 --- a/init/eessi_defaults +++ b/init/eessi_defaults @@ -9,4 +9,4 @@ # export EESSI_CVMFS_REPO="${EESSI_CVMFS_REPO_OVERRIDE:=/cvmfs/pilot.eessi-hpc.org}" -export EESSI_PILOT_VERSION="${EESSI_PILOT_VERSION_OVERRIDE:=2021.12}" +export EESSI_PILOT_VERSION="${EESSI_PILOT_VERSION_OVERRIDE:=2023.06}"