[ENG-7598] 3.2.4 BE: MFR Update CI infrastructure

.github/workflows/test-build.yml

@@ -5,59 +5,75 @@ on: [push, pull_request, workflow_dispatch]
 jobs:
 
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     env:
-      GHA_DISTRO: ubuntu-20.04
+      GHA_DISTRO: ubuntu-24.04
     if: "!contains(github.event.head_commit.message, 'skip ci')"
     strategy:
       matrix:
-        python-version: [3.6]
+        python-version: [3.13]
     steps:
       - name: Git checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
+
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python-version }}
+
       - name: Cache Build Requirements
         id: pip-cache-step
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
-          path: ${{ env.pythonLocation }}
-          key: ${{ env.GHA_DISTRO }}-${{ env.pythonLocation }}-${{ hashFiles('requirements.txt', 'dev-requirements.txt') }}
-      - name: install dependencies
-        if: steps.pip-cache-step.outputs.cache-hit != 'true'
+          path: ~/.cache/pip
+          key: ${{ env.GHA_DISTRO }}-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'dev-requirements.txt') }}
+
+      - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
+          python -m pip install --upgrade pip==24.0
+          pip install setuptools==78.1.0
           pip install -r dev-requirements.txt
 
   runtests:
     name: Run unit tests
     needs: build
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-24.04
     env:
-      GHA_DISTRO: ubuntu-20.04
+      GHA_DISTRO: ubuntu-24.04
+    strategy:
+      matrix:
+        python-version: [3.13]
     steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.6
-        uses: actions/setup-python@v2
+      - uses: actions/checkout@v3
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
         with:
-          python-version: 3.6
+          python-version: ${{ matrix.python-version }}
+
       - name: Cache pip
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
-          path: ${{ env.pythonLocation }}
-          key: ${{ env.GHA_DISTRO }}-${{ env.pythonLocation }}-${{ hashFiles('requirements.txt', 'dev-requirements.txt') }}
-      - name: run syntax checks
-        run: |
-          flake8 .
-      - name: build plugins
+          path: ~/.cache/pip
+          key: ${{ env.GHA_DISTRO }}-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'dev-requirements.txt') }}
+
+      - name: Install test dependencies
         run: |
-          python setup.py develop
-      - name: run unit tests
+          python -m pip install --upgrade pip==24.0
+          pip install setuptools==78.1.0
+          pip install -r dev-requirements.txt
+
+      - name: Run flake8
+        run: flake8 .
+
+      - name: Build plugins
+        run: python setup.py develop
+
+      - name: Run unit tests
         run: |
-          py.test --cov-report term-missing --cov mfr tests
-      - name: Upload coverage data to coveralls.io
+          pytest --cov-report term-missing --cov modular-file-renderer tests
+
+      - name: Upload coverage to Coveralls
         run: coveralls --service=github
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

Dockerfile

@@ -1,7 +1,7 @@
-FROM python:3.6-slim-buster
+FROM python:3.13-slim
 
 # ensure unoconv can locate the uno library
-ENV PYTHONPATH /usr/lib/python3/dist-packages
+ENV PYTHONPATH=/usr/lib/python3/dist-packages
 
 RUN usermod -d /home www-data \
     && chown www-data:www-data /home \
@@ -43,9 +43,9 @@ RUN usermod -d /home www-data \
 RUN mkdir -p /code
 WORKDIR /code
 
-RUN pip install -U pip==18.1
-RUN pip install setuptools==37.0.0
-RUN pip install unoconv==0.8.2
+RUN pip install -U pip==24.0
+RUN pip install setuptools==69.5.1
+RUN pip install unoconv==0.9.0
 
 COPY ./requirements.txt /code/
 
@@ -55,7 +55,7 @@ RUN pip install --no-cache-dir -r ./requirements.txt
 COPY ./ /code/
 
 ARG GIT_COMMIT=
-ENV GIT_COMMIT ${GIT_COMMIT}
+ENV GIT_COMMIT=${GIT_COMMIT}
 
 RUN python setup.py develop
 

dev-requirements.txt

@@ -9,8 +9,8 @@ coveralls
 flake8==3.0.4
 ipdb
 mccabe
-pydevd==0.0.6
+pydevd==3.3.0
 pyflakes
 pytest==2.8.2
 pytest-cov==2.2.0
-pyzmq==14.4.1
+pyzmq==26.1.0

docs/conf.py

@@ -1,4 +1,3 @@
-# -*- coding: utf-8 -*-
 #
 # MFR documentation build configuration file.
 #
@@ -46,8 +45,8 @@
 master_doc = 'index'
 
 # General information about the project.
-project = u'mfr'
-copyright = u'2023, Center For Open Science'
+project = 'mfr'
+copyright = '2023, Center For Open Science'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the

mfr/__init__.py

@@ -1,4 +1,3 @@
 # This is a namespace package, don't put any functional code in here besides the
 # declare_namespace call, or it will disappear on install. See:
 # https://setuptools.readthedocs.io/en/latest/setuptools.html#namespace-packages
-__import__('pkg_resources').declare_namespace(__name__)

mfr/core/exceptions.py

@@ -26,7 +26,8 @@ def as_html(self):
               free, open source software? Check out our openings!</div>
         '''.format(self.message)
 
-    def _format_original_exception(self, exc):
+    @staticmethod
+    def _format_original_exception(exc):
         """Sometimes we catch an error from an external library, but would like to throw our own
         error instead.  This method will take in an external error class and format it for
         consistent representation in the error metrics.

mfr/core/extension.py

@@ -49,7 +49,7 @@ def __init__(self, metadata, file_path, url, assets_url, export_url):
         self.metadata = metadata
         self.file_path = file_path
         self.url = url
-        self.assets_url = '{}/{}'.format(assets_url, self._get_module_name())
+        self.assets_url = f'{assets_url}/{self._get_module_name()}'
         self.export_url = export_url
         self.renderer_metrics = MetricsRecord('renderer')
         if self._get_module_name():
@@ -77,15 +77,15 @@ def __init__(self, metadata, file_path, url, assets_url, export_url):
     def render(self):
         pass
 
-    @abc.abstractproperty
+    @abc.abstractmethod
     def file_required(self):
         """Does the rendering html need the raw file content to display correctly?
         Syntax-highlighted text files do.  Standard image formats do not, since an <img> tag
         only needs a url to the file.
         """
         pass
 
-    @abc.abstractproperty
+    @abc.abstractmethod
     def cache_result(self):
         pass
 

mfr/core/metrics.py

@@ -22,7 +22,7 @@ def _merge_dicts(a, b, path=None):
     return a
 
 
-class MetricsBase():
+class MetricsBase:
     """Lightweight wrapper around a dict to make keeping track of metrics a little easier.
 
     Current functionality is limited, but may be extended later.  To do:
@@ -77,7 +77,8 @@ def manifesto(self):
         """
         return {self.key: self.serialize()}
 
-    def _set_dotted_key(self, store, key, value):
+    @staticmethod
+    def _set_dotted_key(store, key, value):
         """Naive method to set nested dict values via dot-separated keys. e.g
         ``_set_dotted_keys(self._metrics, 'foo.bar', 'moo')`` is equivalent to
         ``self._metrics['foo']['bar'] = 'moo'``.  This method is neither resilient nor intelligent
@@ -138,7 +139,7 @@ def __init__(self, category, name):
     @property
     def key(self):
         """ID string for this subrecord: '{category}_{name}'"""
-        return '{}_{}'.format(self.category, self.name)
+        return f'{self.category}_{self.name}'
 
     def new_subrecord(self, name):
         """Creates and saves a new subrecord.  The new subrecord will have its category set to the

mfr/core/provider.py

@@ -34,8 +34,9 @@ def __init__(self, request, url, action=None):
             'url': str(self.url),
         })
 
-    @abc.abstractproperty
+    @abc.abstractmethod
     def NAME(self):
+        # Todo: not see Name implementation in child classes
         raise NotImplementedError
 
     @abc.abstractmethod

mfr/core/remote_logging.py

@@ -78,7 +78,7 @@ async def log_analytics(request, metrics, is_error=False):
 
     # send the private payload
     private_collection = 'mfr_errors' if is_error else 'mfr_action'
-    if ((is_error and settings.KEEN_PRIVATE_LOG_ERRORS) or settings.KEEN_PRIVATE_LOG_VIEWS):
+    if (is_error and settings.KEEN_PRIVATE_LOG_ERRORS) or settings.KEEN_PRIVATE_LOG_VIEWS:
         await _send_to_keen(keen_payload, private_collection, settings.KEEN_PRIVATE_PROJECT_ID,
                             settings.KEEN_PRIVATE_WRITE_KEY, keen_payload['handler']['type'],
                             domain='private')
@@ -104,18 +104,18 @@ async def _send_to_keen(payload, collection, project_id, write_key, action, doma
     Will raise an excpetion if the event cannot be sent."""
 
     serialized = json.dumps(payload).encode('UTF-8')
-    logger.debug("Serialized payload: {}".format(serialized))
+    logger.debug(f"Serialized payload: {serialized}")
     headers = {
         'Content-Type': 'application/json',
         'Authorization': write_key,
     }
-    url = '{0}/{1}/projects/{2}/events/{3}'.format(settings.KEEN_API_BASE_URL,
+    url = '{}/{}/projects/{}/events/{}'.format(settings.KEEN_API_BASE_URL,
                                                    settings.KEEN_API_VERSION,
                                                    project_id, collection)
 
     async with await aiohttp.request('POST', url, headers=headers, data=serialized) as resp:
         if resp.status == 201:
-            logger.info('Successfully logged {} to {} collection in {} Keen'.format(action, collection, domain))
+            logger.info(f'Successfully logged {action} to {collection} collection in {domain} Keen')
         else:
             raise Exception('Failed to log {} to {} collection in {} Keen. Status: {} Error: {}'.format(
                 action, collection, domain, str(int(resp.status)), await resp.read()
@@ -133,7 +133,7 @@ def _scrub_headers_for_keen(payload, MAX_ITERATIONS=10):
         # if our new scrubbed key is already in the payload, we need to increment it
         if scrubbed_key in scrubbed_payload:
             for i in range(1, MAX_ITERATIONS + 1):  # try MAX_ITERATION times, then give up & drop it
-                incremented_key = '{}-{}'.format(scrubbed_key, i)
+                incremented_key = f'{scrubbed_key}-{i}'
                 if incremented_key not in scrubbed_payload:  # we found an unused key!
                     scrubbed_payload[incremented_key] = payload[key]
                     break