Merge pull request #281 from iiasa/ci/uv

Update CI and code quality

Author: khaeru

.github/workflows/pytest-snapshots.yaml

@@ -14,8 +14,8 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  gams: "43.4.1"
-  python: "3.12"
+  gams-version: 43.4.1
+  python-version: "3.13"
   upstream: main
 
 jobs:
@@ -35,15 +35,15 @@ jobs:
       with:
         lfs: true
 
-    - uses: actions/setup-python@v5
+    - name: Set up uv, Python
+      uses: astral-sh/setup-uv@v5
       with:
+        cache-dependency-glob: "**/pyproject.toml"
         python-version: ${{ env.python }}
-        cache: pip
-        cache-dependency-path: "**/pyproject.toml"
 
     - uses: iiasa/actions/setup-gams@main
       with:
-        version: ${{ env.gams }}
+        version: ${{ env.gams-version }}
         license: ${{ secrets.GAMS_LICENSE }}
 
     - uses: ts-graphviz/setup-graphviz@v2
@@ -59,12 +59,12 @@ jobs:
       # for open PRs), temporarily uncomment, add, or edit lines below
       # as needed. DO NOT merge such changes to `main`.
       run: |
-        # pip install --upgrade "genno @ git+https://github.com/khaeru/genno.git@main"
-        pip install --upgrade "ixmp @ git+https://github.com/iiasa/ixmp.git@${{ env.upstream }}"
-        # pip install --upgrade "message-ix @ git+https://github.com/iiasa/message_ix.git@${{ env.upstream }}"
-        pip install --upgrade "message-ix @ git+https://github.com/iiasa/message_ix.git@issue/723"
+        # uv pip install --upgrade "genno @ git+https://github.com/khaeru/genno.git@main"
+        uv pip install --upgrade "ixmp @ git+https://github.com/iiasa/ixmp.git@${{ env.upstream }}"
+        # uv pip install --upgrade "message-ix @ git+https://github.com/iiasa/message_ix.git@${{ env.upstream }}"
+        uv pip install --upgrade "message-ix @ git+https://github.com/iiasa/message_ix.git@issue/723"
 
-        pip install .[docs,tests] dask[dataframe]
+        uv pip install .[docs,tests] dask[dataframe]
 
     - name: Configure local data path
       run: |

.github/workflows/pytest.yaml

@@ -1,19 +1,47 @@
 name: Test
 
 on:
-  pull_request:
-    branches: [ main , "migrate**"]
+  # Uncomment this entry only to debug the workflow
+  # pull_request:
+  #   branches: [ main ]
+  pull_request_target:
+    branches: [ main, "migrate**" ]
+    types: [ labeled, opened, reopened, synchronize ]
   schedule:
-  # 05:00 UTC = 06:00 CET = 07:00 CEST
-  - cron: "0 5 * * *"
+  - cron: "0 5 * * *"  # = 06:00 CET = 07:00 CEST
 
 # Cancel previous runs that have not completed
 concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
+defaults:
+  run:
+    shell: bash
+
+env:
+  gams-version: 43.4.1
+  label: "safe to test"
+  python-version: "3.13"
+
 jobs:
+  check:
+    name: check permissions
+    runs-on: ubuntu-latest
+    steps:
+    - if: >
+        !(
+          github.event_name == 'schedule'
+          || github.repository == github.event.pull_request.head.repo.full_name
+          || contains(github.event.pull_request.labels.*.name, env.label)
+        )
+      run: |
+        echo "Pytest workflow will not run for branch in fork without label \`${{ env.label }}\`." >>$GITHUB_STEP_SUMMARY
+        exit 1
+
   warm-lfs-cache:
+    needs: check
+
     strategy:
       matrix:
         os: [ macos-13, macos-latest, ubuntu-latest, windows-latest ]
@@ -73,23 +101,23 @@ jobs:
     - name: Check out message-ix-models
       uses: francisbilham11/action-cached-lfs-checkout@v3
 
-    - uses: actions/setup-python@v5
+    - name: Set up uv, Python
+      uses: astral-sh/setup-uv@v5
       with:
+        cache-dependency-glob: "**/pyproject.toml"
         python-version: ${{ matrix.version.python }}
-        cache: pip
-        cache-dependency-path: "**/pyproject.toml"
 
     - uses: iiasa/actions/setup-gams@main
       with:
-        version: 43.4.1
+        version: ${{ env.gams-version }}
         license: ${{ secrets.GAMS_LICENSE }}
 
     - uses: ts-graphviz/setup-graphviz@v2
       # Work around ts-graphviz/setup-graphviz#630
-      if: ${{ matrix.os != 'macos-13' }}
+      if: matrix.os != 'macos-13'
 
     - name: Determine extra dependencies
-      id: extra-deps
+      id: dependencies
       run : |
         from os import environ
         from pathlib import Path
@@ -110,42 +138,36 @@ jobs:
     - name: Install packages and dependencies
       # By default, install:
       # - ixmp, message_ix: from GitHub branches/tags per matrix.version.upstream (above)
-      # - other dependencies including genno: from PyPI.
+      # - other dependencies: from PyPI.
       #
-      # To test against unreleased code (on `main`, or other branches
-      # for open PRs), temporarily uncomment, add, or edit lines below
-      # as needed. DO NOT merge such changes to `main`.
+      # To test against unreleased code (on `main`, or other branches for open PRs),
+      # temporarily edit or add lines below as needed. DO NOT merge such changes to `main`.
       run: |
-        pip install --upgrade "ixmp @ git+https://github.com/iiasa/ixmp.git@${{ matrix.version.upstream }}"
-        pip install --upgrade "message-ix @ git+https://github.com/iiasa/message_ix.git@${{ matrix.version.upstream }}"
-
-        pip install .[docs,tests] ${{ steps.extra-deps.outputs.value }}
-
-        # TEMPORARY With Python 3.13 pyam-iamc resolves to 1.3.1, which in turn
-        # limits pint < 0.17. Override.
-        pip install --upgrade pint
+        uv pip install --upgrade \
+          ${{ steps.dependencies.outputs.value }} \
+          "ixmp @ git+https://github.com/iiasa/ixmp.git@${{ matrix.version.upstream }}" \
+          "message-ix @ git+https://github.com/iiasa/message_ix.git@${{ matrix.version.upstream }}" \
+          .[docs,tests]
 
     - name: Configure local data path
       run: |
         mkdir -p message-local-data/cache
         mix-models config set "message local data" "$(realpath message-local-data)"
         mix-models config show
-      shell: bash
 
     - name: Run test suite using pytest
       run: |
         pytest message_ix_models \
           -m "not (ece_db or snapshot)" \
-          -rA --verbose --color=yes --durations=20 \
+          --color=yes --durations=20 -rA --verbose \
           --cov-report=term-missing --cov-report=xml \
           --numprocesses=auto \
           --local-cache --jvmargs="-Xmx6G"
-      shell: bash
 
     - name: Upload test coverage to Codecov.io
       uses: codecov/codecov-action@v5
       with:
-        token: ${{ secrets.CODECOV_TOKEN }} # required
+        token: ${{ secrets.CODECOV_TOKEN }}  # Required
 
   pre-commit:
     name: Code quality
@@ -154,12 +176,13 @@ jobs:
 
     steps:
     - uses: actions/checkout@v4
-    - uses: actions/setup-python@v5
-      with: { python-version: "3.13" }
-
-    - name: Force recreation of pre-commit virtual environment for mypy
-      if: github.event_name == 'schedule'  # Comment this line to run on a PR
-      run: gh cache list -L 999 | cut -f2 | grep pre-commit | xargs -I{} gh cache delete "{}" || true
-      env: { GH_TOKEN: "${{ github.token }}" }
-
-    - uses: pre-commit/[email protected]
+    - uses: astral-sh/setup-uv@v5
+      with:
+        cache-dependency-glob: "**/pyproject.toml"
+        python-version: ${{ env.python-version }}
+    - uses: actions/cache@v4
+      with:
+        path: ~/.cache/pre-commit
+        key: pre-commit|${{ env.UV_PYTHON }}|${{ hashFiles('.pre-commit-config.yaml') }}
+        lookup-only: ${{ github.event_name == 'schedule' }}  # Set 'true' to recreate cache
+    - run: uvx pre-commit run --all-files --color=always --show-diff-on-failure

.pre-commit-config.yaml

@@ -1,26 +1,19 @@
 repos:
-- repo: local
+- repo: https://github.com/pre-commit/mirrors-mypy
+  rev: v1.14.1
   hooks:
   - id: mypy
-    name: mypy
-    always_run: true
-    require_serial: true
     pass_filenames: false
-
-    language: python
-    entry: bash -c ". ${PRE_COMMIT_MYPY_VENV:-/dev/null}/bin/activate 2>/dev/null; mypy $0 $@; python -m pip list"
     additional_dependencies:
-    - mypy >= 1.13.0
+    - "ixmp @ git+https://github.com/iiasa/ixmp.git@main"
+    - "message-ix @ git+https://github.com/iiasa/message_ix.git@main"
     - plotnine
     - pytest
     - sdmx1
     - types-PyYAML
     - types-tqdm
-    - "ixmp @ git+https://github.com/iiasa/ixmp.git@main"
-    - "message-ix @ git+https://github.com/iiasa/message_ix.git@main"
-    args: ["."]
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.7.4
+  rev: v0.9.1
   hooks:
   - id: ruff
   - id: ruff-format

doc/conf.py

@@ -184,6 +184,8 @@ def local_inv(name: str, *parts: str) -> Optional[str]:
     spec = find_spec(name)
     if spec and spec.origin:
         return str(Path(spec.origin).parents[1].joinpath(*parts, "objects.inv"))
+    else:
+        return None
 
 
 # For message-data, see: https://docs.readthedocs.io/en/stable/guides/intersphinx.html#intersphinx-with-private-projects

message_ix_models/model/material/material_demand/material_demand_calc.py

@@ -100,7 +100,7 @@ def read_timer_pop(
     datapath: str | Path, material: Literal["cement", "steel", "aluminum"]
 ):
     df_population = pd.read_excel(
-        f'{datapath}/{material_data[material]["dir"]}{material_data[material]["file"]}',
+        f"{datapath}/{material_data[material]['dir']}{material_data[material]['file']}",
         sheet_name="Timer_POP",
         skiprows=[0, 1, 2, 30],
         nrows=26,
@@ -119,7 +119,7 @@ def read_timer_gdp(
 ):
     # Read GDP per capita data
     df_gdp = pd.read_excel(
-        f'{datapath}/{material_data[material]["dir"]}{material_data[material]["file"]}',
+        f"{datapath}/{material_data[material]['dir']}{material_data[material]['file']}",
         sheet_name="Timer_GDPCAP",
         skiprows=[0, 1, 2, 30],
         nrows=26,
@@ -195,7 +195,7 @@ def read_hist_mat_demand(material: Literal["cement", "steel", "aluminum"]):
     if material == "aluminum":
         df_raw_cons = (
             pd.read_excel(
-                f'{datapath}/{material_data[material]["dir"]}{material_data[material]["file"]}',
+                f"{datapath}/{material_data[material]['dir']}{material_data[material]['file']}",
                 sheet_name="final_table",
                 nrows=378,
             )
@@ -213,7 +213,7 @@ def read_hist_mat_demand(material: Literal["cement", "steel", "aluminum"]):
         )
     elif material == "steel":
         df_raw_cons = pd.read_excel(
-            f'{datapath}/{material_data[material]["dir"]}{material_data[material]["file"]}',
+            f"{datapath}/{material_data[material]['dir']}{material_data[material]['file']}",
             sheet_name="Consumption regions",
             nrows=26,
         )
@@ -246,7 +246,7 @@ def read_hist_mat_demand(material: Literal["cement", "steel", "aluminum"]):
         )
     elif material == "cement":
         df_raw_cons = pd.read_excel(
-            f'{datapath}/{material_data[material]["dir"]}{material_data[material]["file"]}',
+            f"{datapath}/{material_data[material]['dir']}{material_data[material]['file']}",
             sheet_name="Regions",
             skiprows=122,
             nrows=26,
@@ -358,7 +358,7 @@ def derive_demand(
 
     # get base year demand of material
     df_base_demand = read_base_demand(
-        f'{datapath}/{material_data[material]["dir"]}/demand_{material}.yaml'
+        f"{datapath}/{material_data[material]['dir']}/demand_{material}.yaml"
     )
 
     # get historical data (material consumption, pop, gdp)

message_ix_models/model/transport/base.py

@@ -373,7 +373,7 @@ def share_constraints(c: Computer, k_fe: "genno.Key", k_ue: "genno.Key") -> None
         )
 
     # Transform ue-share values to the expected format
-    base = c.full_key("ue::share")
+    base = Key(c.full_key("ue::share"))
     for kind, (label, groupby) in product(
         ("lo", "up"),
         (

message_ix_models/model/transport/config.py

@@ -441,7 +441,7 @@ def get_cl_scenario() -> "Codelist":
     as_ = read("IIASA_ECE:AGENCIES")
     cl_ssp_2024 = read("ICONICS:SSP(2024)")
 
-    cl = common.Codelist(
+    cl: "common.Codelist" = common.Codelist(
         id="CL_TRANSPORT_SCENARIO", maintainer=as_["IIASA_ECE"], version="1.0.0"
     )