CI lockfiles

.cirrus.yml

@@ -79,11 +79,10 @@ linux_task_template: &LINUX_TASK_TEMPLATE
       - echo "$(date +%Y).$(expr $(date +%U) / ${CACHE_PERIOD}):${CARTOPY_CACHE_BUILD}"
   nox_cache:
     folder: ${CIRRUS_WORKING_DIR}/.nox
+    reupload_on_changes: true
     fingerprint_script:
       - echo "${CIRRUS_TASK_NAME}"
-      - echo "$(date +%Y).$(expr $(date +%U) / ${CACHE_PERIOD}):${NOX_CACHE_BUILD}"
-      - sha256sum ${CIRRUS_WORKING_DIR}/requirements/ci/py$(echo ${PY_VER} | tr -d ".").yml
-
+      - echo "${NOX_CACHE_BUILD}"
 
 #
 # YAML alias for compute credits.

.github/workflows/refresh-lockfiles.yml

@@ -0,0 +1,78 @@
+# This workflow periodically creates new environment lock files based on the newest
+# available packages and dependencies.
+#
+# Environment specifications are given as conda environment.yml files found in
+# `requirements/ci/py**.yml`.  These state the pacakges required, the conda channels
+# that the packages will be pulled from, and any versions of packages that need to be
+# pinned at specific versions.
+#
+# For environments that have changed, a pull request will be made and submitted
+# to the master branch
+
+name: Refresh Lockfiles
+
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Run once a week on a Saturday night 
+    - cron: 1 0 * * 6
+
+
+jobs:
+
+  gen_lockfiles:
+    # this is a matrix job: it splits to create new lockfiles for each
+    # of the CI test python versions.
+    # this list below should be changed when covering more python versions
+    # TODO: generate this matrix automatically from the list of available py**.yml files
+    #       ref: https://tomasvotruba.com/blog/2020/11/16/how-to-make-dynamic-matrix-in-github-actions/
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        python: ['36', '37', '38']
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: install conda-lock
+        run: |
+          source $CONDA/bin/activate base
+          conda install -y -c conda-forge conda-lock
+      - name: generate lockfile
+        run: |
+          $CONDA/bin/conda-lock lock -p linux-64 -f requirements/ci/py${{matrix.python}}.yml
+          mv conda-linux-64.lock py${{matrix.python}}-linux-64.lock
+      - name: output lockfile
+        uses: actions/upload-artifact@v2
+        with:
+          path: py${{matrix.python}}-linux-64.lock
+
+  create_pr:
+    # once the matrix job has completed all the lock files will have been uploaded as artifacts.
+    # Download the artifacts, add them to the repo, and create a PR.
+    runs-on: ubuntu-latest
+    needs: gen_lockfiles
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: get artifacts
+        uses: actions/download-artifact@v2
+        with:
+          path: artifacts
+
+      - name: Update lock files in repo
+        run: | 
+          cp artifacts/artifact/*.lock requirements/ci/nox.lock
+          rm -r artifacts
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@052fc72b4198ba9fbc81b818c6e1859f747d49a8
+        with:
+          commit-message: Updated environment lockfiles
+          delete-branch: true
+          branch: lockfiles
+          branch-suffix: timestamp
+          title: Update CI environment lockfiles
+          body: |
+            Lockfiles updated to the latest resolvable environment.

Makefile

@@ -0,0 +1,2 @@
+lockfiles:
+	python tools/update_lockfiles.py -o requirements/ci/nox.lock requirements/ci/py*.yml

docs/src/developers_guide/contributing_ci_tests.rst

@@ -38,6 +38,36 @@ The above `cirrus-ci`_ tasks are run automatically against all `Iris`_ branches
 on GitHub whenever a pull-request is submitted, updated or merged. See the
 `Cirrus-CI Dashboard`_ for details of recent past and active Iris jobs.
 
+
+.. _cirrus_test_env:
+
+Cirrus CI Test environment
+--------------------------
+
+The test environment on the Cirrus-CI service is determined from the requirement files
+in ``requirements/ci/py**.yml``.  These are conda environment files that list the entire
+set of build, test and run requirements for Iris.
+
+For reproducible test results, these environments are resolved for all their dependencies
+and stored as lock files in ``requirements/ci/nox.lock``.  The test environments will not
+resolve the dependencies each time, instead they will use the lock file to reproduce the
+same exact environment each time.
+
+**If you have updated the requirement yaml files with new dependencies, you will need to
+generate new lock files.** To do this, run the command::
+
+   python tools/update_lockfiles.py -o requirements/ci/nox.lock requirements/ci/py*.yml
+
+or simply::
+
+   make lockfiles
+
+and add the changed lockfiles to your pull request.
+
+New lockfiles are generated automatically each week to ensure that Iris continues to be
+tested against the latest available version of its dependencies.
+
+
 .. _skipping Cirrus-CI tasks:
 
 Skipping Cirrus-CI Tasks
@@ -107,4 +137,3 @@ See the `pre-commit.ci dashboard`_ for details of recent past and active Iris jo
 .. _Cirrus-CI Documentation: https://cirrus-ci.org/guide/writing-tasks/
 .. _.pre-commit-config.yaml: https://github.com/SciTools/iris/blob/master/.pre-commit-config.yaml
 .. _pre-commit.ci dashboard: https://results.pre-commit.ci/repo/github/5312648
-

docs/src/developers_guide/contributing_pull_request_checklist.rst

@@ -29,6 +29,10 @@ is merged.  Before submitting a pull request please consider this list.
 #. **Check all modified and new source files conform to the required**
    :ref:`code_formatting`.
 
+#. **Check all new dependencies added to the ``requirements/ci/*.yml`` files.**  If
+   dependencies have been added then new nox testing lockfiles should be generated too,
+   see :ref:`cirrus_test_env`.
+
 #. **Check the source documentation been updated to explain all new or changed
    features**.  See :ref:`docstrings`.
 

docs/src/whatsnew/latest.rst

@@ -150,6 +150,8 @@ This document explains the changes made to Iris for this release
 #. `@bjlittle`_ updated the perceptual imagehash graphical test support for
    `matplotlib`_ 3.4.1. (:pull:`4087`)
 
+#. `@jamesp`_ switched `cirrus-ci`_ testing and `nox`_
+   testing to use `conda-lock`_ files for static test environments. (:pull:`4108`)
 
 .. comment
     Whatsnew author names (@github name) in alphabetical order. Note that,
@@ -175,3 +177,4 @@ This document explains the changes made to Iris for this release
 .. _Python 3.8: https://www.python.org/downloads/release/python-380/
 .. _README.md: https://github.com/SciTools/iris#-----
 .. _xxhash: http://cyan4973.github.io/xxHash/
+.. _conda-lock: https://github.com/conda-incubator/conda-lock