diff --git a/.github/workflows/test-build.yml b/.github/workflows/test-build.yml index 2c77176e0..c34ead2b9 100644 --- a/.github/workflows/test-build.yml +++ b/.github/workflows/test-build.yml @@ -5,59 +5,75 @@ on: [push, pull_request, workflow_dispatch] jobs: build: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 env: - GHA_DISTRO: ubuntu-20.04 + GHA_DISTRO: ubuntu-24.04 if: "!contains(github.event.head_commit.message, 'skip ci')" strategy: matrix: - python-version: [3.6] + python-version: [3.13] steps: - name: Git checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} + - name: Cache Build Requirements id: pip-cache-step - uses: actions/cache@v2 + uses: actions/cache@v4 with: - path: ${{ env.pythonLocation }} - key: ${{ env.GHA_DISTRO }}-${{ env.pythonLocation }}-${{ hashFiles('requirements.txt', 'dev-requirements.txt') }} - - name: install dependencies - if: steps.pip-cache-step.outputs.cache-hit != 'true' + path: ~/.cache/pip + key: ${{ env.GHA_DISTRO }}-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'dev-requirements.txt') }} + + - name: Install dependencies run: | - python -m pip install --upgrade pip + python -m pip install --upgrade pip==24.0 + pip install setuptools==78.1.0 pip install -r dev-requirements.txt runtests: name: Run unit tests needs: build - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 env: - GHA_DISTRO: ubuntu-20.04 + GHA_DISTRO: ubuntu-24.04 + strategy: + matrix: + python-version: [3.13] steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.6 - uses: actions/setup-python@v2 + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 with: - python-version: 3.6 + python-version: ${{ matrix.python-version }} + - name: Cache pip - uses: actions/cache@v2 + uses: actions/cache@v4 with: - path: ${{ env.pythonLocation }} - key: ${{ env.GHA_DISTRO }}-${{ env.pythonLocation }}-${{ hashFiles('requirements.txt', 'dev-requirements.txt') }} - - name: run syntax checks - run: | - flake8 . - - name: build plugins + path: ~/.cache/pip + key: ${{ env.GHA_DISTRO }}-${{ matrix.python-version }}-${{ hashFiles('requirements.txt', 'dev-requirements.txt') }} + + - name: Install test dependencies run: | - python setup.py develop - - name: run unit tests + python -m pip install --upgrade pip==24.0 + pip install setuptools==78.1.0 + pip install -r dev-requirements.txt + + - name: Run flake8 + run: flake8 . + + - name: Build plugins + run: python setup.py develop + + - name: Run unit tests run: | - py.test --cov-report term-missing --cov mfr tests - - name: Upload coverage data to coveralls.io + pytest --cov-report term-missing --cov modular-file-renderer tests + + - name: Upload coverage to Coveralls run: coveralls --service=github env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/Dockerfile b/Dockerfile index f73f5c93e..8fd4097d7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ -FROM python:3.6-slim-buster +FROM python:3.13-slim # ensure unoconv can locate the uno library -ENV PYTHONPATH /usr/lib/python3/dist-packages +ENV PYTHONPATH=/usr/lib/python3/dist-packages RUN usermod -d /home www-data \ && chown www-data:www-data /home \ @@ -43,9 +43,9 @@ RUN usermod -d /home www-data \ RUN mkdir -p /code WORKDIR /code -RUN pip install -U pip==18.1 -RUN pip install setuptools==37.0.0 -RUN pip install unoconv==0.8.2 +RUN pip install -U pip==24.0 +RUN pip install setuptools==69.5.1 +RUN pip install unoconv==0.9.0 COPY ./requirements.txt /code/ @@ -55,7 +55,7 @@ RUN pip install --no-cache-dir -r ./requirements.txt COPY ./ /code/ ARG GIT_COMMIT= -ENV GIT_COMMIT ${GIT_COMMIT} +ENV GIT_COMMIT=${GIT_COMMIT} RUN python setup.py develop diff --git a/dev-requirements.txt b/dev-requirements.txt index 37036c257..c0757f842 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -9,8 +9,8 @@ coveralls flake8==3.0.4 ipdb mccabe -pydevd==0.0.6 +pydevd==3.3.0 pyflakes pytest==2.8.2 pytest-cov==2.2.0 -pyzmq==14.4.1 +pyzmq==26.1.0 diff --git a/docs/conf.py b/docs/conf.py index 42c804545..6d31f10bc 100755 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # MFR documentation build configuration file. # @@ -46,8 +45,8 @@ master_doc = 'index' # General information about the project. -project = u'mfr' -copyright = u'2023, Center For Open Science' +project = 'mfr' +copyright = '2023, Center For Open Science' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/mfr/__init__.py b/mfr/__init__.py index 202301093..5e2406e4c 100755 --- a/mfr/__init__.py +++ b/mfr/__init__.py @@ -1,4 +1,3 @@ # This is a namespace package, don't put any functional code in here besides the # declare_namespace call, or it will disappear on install. See: # https://setuptools.readthedocs.io/en/latest/setuptools.html#namespace-packages -__import__('pkg_resources').declare_namespace(__name__) diff --git a/mfr/core/exceptions.py b/mfr/core/exceptions.py index 8f3573b54..a551a50d0 100644 --- a/mfr/core/exceptions.py +++ b/mfr/core/exceptions.py @@ -26,7 +26,8 @@ def as_html(self): free, open source software? Check out our openings! '''.format(self.message) - def _format_original_exception(self, exc): + @staticmethod + def _format_original_exception(exc): """Sometimes we catch an error from an external library, but would like to throw our own error instead. This method will take in an external error class and format it for consistent representation in the error metrics. diff --git a/mfr/core/extension.py b/mfr/core/extension.py index 928cf2224..cec133e7c 100644 --- a/mfr/core/extension.py +++ b/mfr/core/extension.py @@ -49,7 +49,7 @@ def __init__(self, metadata, file_path, url, assets_url, export_url): self.metadata = metadata self.file_path = file_path self.url = url - self.assets_url = '{}/{}'.format(assets_url, self._get_module_name()) + self.assets_url = f'{assets_url}/{self._get_module_name()}' self.export_url = export_url self.renderer_metrics = MetricsRecord('renderer') if self._get_module_name(): @@ -77,7 +77,7 @@ def __init__(self, metadata, file_path, url, assets_url, export_url): def render(self): pass - @abc.abstractproperty + @abc.abstractmethod def file_required(self): """Does the rendering html need the raw file content to display correctly? Syntax-highlighted text files do. Standard image formats do not, since an tag @@ -85,7 +85,7 @@ def file_required(self): """ pass - @abc.abstractproperty + @abc.abstractmethod def cache_result(self): pass diff --git a/mfr/core/metrics.py b/mfr/core/metrics.py index bf47eee25..9dc29aefe 100644 --- a/mfr/core/metrics.py +++ b/mfr/core/metrics.py @@ -22,7 +22,7 @@ def _merge_dicts(a, b, path=None): return a -class MetricsBase(): +class MetricsBase: """Lightweight wrapper around a dict to make keeping track of metrics a little easier. Current functionality is limited, but may be extended later. To do: @@ -77,7 +77,8 @@ def manifesto(self): """ return {self.key: self.serialize()} - def _set_dotted_key(self, store, key, value): + @staticmethod + def _set_dotted_key(store, key, value): """Naive method to set nested dict values via dot-separated keys. e.g ``_set_dotted_keys(self._metrics, 'foo.bar', 'moo')`` is equivalent to ``self._metrics['foo']['bar'] = 'moo'``. This method is neither resilient nor intelligent @@ -138,7 +139,7 @@ def __init__(self, category, name): @property def key(self): """ID string for this subrecord: '{category}_{name}'""" - return '{}_{}'.format(self.category, self.name) + return f'{self.category}_{self.name}' def new_subrecord(self, name): """Creates and saves a new subrecord. The new subrecord will have its category set to the diff --git a/mfr/core/provider.py b/mfr/core/provider.py index dac7c0f62..fb7a2b722 100644 --- a/mfr/core/provider.py +++ b/mfr/core/provider.py @@ -34,8 +34,9 @@ def __init__(self, request, url, action=None): 'url': str(self.url), }) - @abc.abstractproperty + @abc.abstractmethod def NAME(self): + # Todo: not see Name implementation in child classes raise NotImplementedError @abc.abstractmethod diff --git a/mfr/core/remote_logging.py b/mfr/core/remote_logging.py index bdde5446f..e98dad7e8 100644 --- a/mfr/core/remote_logging.py +++ b/mfr/core/remote_logging.py @@ -78,7 +78,7 @@ async def log_analytics(request, metrics, is_error=False): # send the private payload private_collection = 'mfr_errors' if is_error else 'mfr_action' - if ((is_error and settings.KEEN_PRIVATE_LOG_ERRORS) or settings.KEEN_PRIVATE_LOG_VIEWS): + if (is_error and settings.KEEN_PRIVATE_LOG_ERRORS) or settings.KEEN_PRIVATE_LOG_VIEWS: await _send_to_keen(keen_payload, private_collection, settings.KEEN_PRIVATE_PROJECT_ID, settings.KEEN_PRIVATE_WRITE_KEY, keen_payload['handler']['type'], domain='private') @@ -104,18 +104,18 @@ async def _send_to_keen(payload, collection, project_id, write_key, action, doma Will raise an excpetion if the event cannot be sent.""" serialized = json.dumps(payload).encode('UTF-8') - logger.debug("Serialized payload: {}".format(serialized)) + logger.debug(f"Serialized payload: {serialized}") headers = { 'Content-Type': 'application/json', 'Authorization': write_key, } - url = '{0}/{1}/projects/{2}/events/{3}'.format(settings.KEEN_API_BASE_URL, + url = '{}/{}/projects/{}/events/{}'.format(settings.KEEN_API_BASE_URL, settings.KEEN_API_VERSION, project_id, collection) async with await aiohttp.request('POST', url, headers=headers, data=serialized) as resp: if resp.status == 201: - logger.info('Successfully logged {} to {} collection in {} Keen'.format(action, collection, domain)) + logger.info(f'Successfully logged {action} to {collection} collection in {domain} Keen') else: raise Exception('Failed to log {} to {} collection in {} Keen. Status: {} Error: {}'.format( action, collection, domain, str(int(resp.status)), await resp.read() @@ -133,7 +133,7 @@ def _scrub_headers_for_keen(payload, MAX_ITERATIONS=10): # if our new scrubbed key is already in the payload, we need to increment it if scrubbed_key in scrubbed_payload: for i in range(1, MAX_ITERATIONS + 1): # try MAX_ITERATION times, then give up & drop it - incremented_key = '{}-{}'.format(scrubbed_key, i) + incremented_key = f'{scrubbed_key}-{i}' if incremented_key not in scrubbed_payload: # we found an unused key! scrubbed_payload[incremented_key] = payload[key] break diff --git a/mfr/core/utils.py b/mfr/core/utils.py index 52ae1c2b8..71bcf18fb 100644 --- a/mfr/core/utils.py +++ b/mfr/core/utils.py @@ -1,4 +1,4 @@ -import pkg_resources +from importlib.metadata import entry_points from stevedore import driver from mfr.core import exceptions @@ -10,6 +10,7 @@ def make_provider(name, request, url, action=None): :param str name: The name of the provider to instantiate. (osf) :param request: :param dict url: + :param action: :rtype: :class:`mfr.core.provider.BaseProvider` """ @@ -23,7 +24,7 @@ def make_provider(name, request, url, action=None): ).driver except RuntimeError: raise exceptions.MakeProviderError( - '"{}" is not a supported provider'.format(name.lower()), + f'"{name.lower()}" is not a supported provider', namespace='mfr.providers', name=name.lower(), invoke_on_load=True, @@ -34,13 +35,14 @@ def make_provider(name, request, url, action=None): ) -def make_exporter(name, source_file_path, output_file_path, format, metadata): +def make_exporter(name, source_file_path, output_file_path, file_format, metadata): """Returns an instance of :class:`mfr.core.extension.BaseExporter` :param str name: The name of the extension to instantiate. (.jpg, .docx, etc) :param str source_file_path: :param str output_file_path: - :param str format: + :param str file_format: + :param metadata: :rtype: :class:`mfr.core.extension.BaseExporter` """ @@ -50,7 +52,7 @@ def make_exporter(name, source_file_path, output_file_path, format, metadata): namespace='mfr.exporters', name=normalized_name, invoke_on_load=True, - invoke_args=(normalized_name, source_file_path, output_file_path, format, metadata), + invoke_args=(normalized_name, source_file_path, output_file_path, file_format, metadata), ).driver except RuntimeError: raise exceptions.MakeExporterError( @@ -60,7 +62,7 @@ def make_exporter(name, source_file_path, output_file_path, format, metadata): invoke_args={ 'source_file_path': source_file_path, 'output_file_path': output_file_path, - 'format': format, + 'format': file_format, } ) @@ -70,6 +72,7 @@ def make_renderer(name, metadata, file_path, url, assets_url, export_url): :param str name: The name of the extension to instantiate. (.jpg, .docx, etc) :param: :class:`mfr.core.provider.ProviderMetadata` metadata: + :param metadata: :param str file_path: :param str url: :param str assets_url: @@ -110,16 +113,15 @@ def get_renderer_name(name: str) -> str: # `ep_iterator` is an iterable object. Must convert it to a `list` for access. # `list()` can only be called once because the iterator moves to the end after conversion. - ep_iterator = pkg_resources.iter_entry_points(group='mfr.renderers', name=name.lower()) - ep_list = list(ep_iterator) + ep = entry_points().select(group='mfr.renderers', name=name.lower()) # Empty list indicates unsupported file type. Return '' and let `make_renderer()` handle it. - if len(ep_list) == 0: + if len(ep) == 0: return '' # If the file type is supported, there must be only one element in the list. - assert len(ep_list) == 1 - return ep_list[0].attrs[0] + assert len(ep) == 1 + return ep[0].value.split(":")[1].split('.')[0] def get_exporter_name(name: str) -> str: @@ -132,24 +134,23 @@ def get_exporter_name(name: str) -> str: # `ep_iterator` is an iterable object. Must convert it to a `list` for access. # `list()` can only be called once because the iterator moves to the end after conversion. - ep_iterator = pkg_resources.iter_entry_points(group='mfr.exporters', name=name.lower()) - ep_list = list(ep_iterator) + ep = entry_points().select(group='mfr.exporters', name=name.lower()) # Empty list indicates unsupported export type. Return '' and let `make_exporter()` handle it. - if len(ep_list) == 0: + if len(ep) == 0: return '' # If the export type is supported, there must be only one element in the list. - assert len(ep_list) == 1 - return ep_list[0].attrs[0] + assert len(ep) == 1 + return ep[0].value.split(":")[1].split('.')[0] def sizeof_fmt(num, suffix='B'): if abs(num) < 1000: - return '%3.0f%s' % (num, suffix) + return '{:3.0f}{}'.format(num, suffix) for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']: if abs(num) < 1000.0: - return '%3.1f%s%s' % (num, unit, suffix) + return '{:3.1f}{}{}'.format(num, unit, suffix) num /= 1000.0 - return '%.1f%s%s' % (num, 'Y', suffix) + return '{:.1f}{}{}'.format(num, 'Y', suffix) diff --git a/mfr/extensions/__init__.py b/mfr/extensions/__init__.py index de40ea7ca..e69de29bb 100644 --- a/mfr/extensions/__init__.py +++ b/mfr/extensions/__init__.py @@ -1 +0,0 @@ -__import__('pkg_resources').declare_namespace(__name__) diff --git a/mfr/extensions/codepygments/render.py b/mfr/extensions/codepygments/render.py index a76b30ae2..d28739610 100644 --- a/mfr/extensions/codepygments/render.py +++ b/mfr/extensions/codepygments/render.py @@ -80,7 +80,7 @@ def _render_html(self, fp, ext, *args, **kwargs): content = data.decode(encoding) except UnicodeDecodeError as err: raise exceptions.FileDecodingError( - message='Unable to decode file as {}.'.format(encoding), + message=f'Unable to decode file as {encoding}.', extension=ext, category='undecodable', original_exception=err, @@ -89,7 +89,7 @@ def _render_html(self, fp, ext, *args, **kwargs): if content is None: raise exceptions.FileDecodingError( - message='File decoded to undefined using encoding "{}"'.format(encoding), + message=f'File decoded to undefined using encoding "{encoding}"', extension=ext, category='decoded_to_undefined', code=500, diff --git a/mfr/extensions/image/export.py b/mfr/extensions/image/export.py index f95d7b8cb..db4ff60f8 100644 --- a/mfr/extensions/image/export.py +++ b/mfr/extensions/image/export.py @@ -21,7 +21,7 @@ def export(self): image_type = parts[-1].lower() max_size = {'w': None, 'h': None} if len(parts) == 2: - max_size['w'], max_size['h'] = [int(size) for size in parts[0].split('x')] + max_size['w'], max_size['h'] = (int(size) for size in parts[0].split('x')) self.metrics.merge({ 'type': image_type, 'max_size_w': max_size['w'], @@ -66,7 +66,7 @@ def export(self): image.save(self.output_file_path, image_type) image.close() - except (UnicodeDecodeError, IOError, FileNotFoundError, OSError) as err: + except (UnicodeDecodeError, OSError, FileNotFoundError) as err: os.path.splitext(os.path.split(self.source_file_path)[-1]) raise exceptions.PillowImageError( 'Unable to export the file as a {}, please check that the ' diff --git a/mfr/extensions/image/render.py b/mfr/extensions/image/render.py index 473d11dcd..dd8469433 100644 --- a/mfr/extensions/image/render.py +++ b/mfr/extensions/image/render.py @@ -24,7 +24,7 @@ def render(self): exported_url = furl.furl(self.export_url) if settings.EXPORT_MAXIMUM_SIZE and settings.EXPORT_TYPE: - exported_url.args['format'] = '{}.{}'.format(settings.EXPORT_MAXIMUM_SIZE, settings.EXPORT_TYPE) + exported_url.args['format'] = f'{settings.EXPORT_MAXIMUM_SIZE}.{settings.EXPORT_TYPE}' elif settings.EXPORT_TYPE: exported_url.args['format'] = settings.EXPORT_TYPE else: diff --git a/mfr/extensions/ipynb/render.py b/mfr/extensions/ipynb/render.py index c93232b3f..1ee836ea5 100644 --- a/mfr/extensions/ipynb/render.py +++ b/mfr/extensions/ipynb/render.py @@ -24,11 +24,11 @@ def __init__(self, *args, **kwargs): def render(self): try: - with open(self.file_path, 'r') as file_pointer: + with open(self.file_path) as file_pointer: notebook = nbformat.reads(file_pointer.read(), as_version=4) except ValueError as err: raise exceptions.InvalidFormatError( - 'Could not read ipython notebook file. {}'.format(str(err)), + f'Could not read ipython notebook file. {str(err)}', extension=self.metadata.ext, download_url=str(self.metadata.download_url), original_exception=err, diff --git a/mfr/extensions/jamovi/render.py b/mfr/extensions/jamovi/render.py index 8e59a6052..d634e89b9 100644 --- a/mfr/extensions/jamovi/render.py +++ b/mfr/extensions/jamovi/render.py @@ -30,7 +30,7 @@ def render(self): return self.TEMPLATE.render(base=self.assets_url, body=body) except BadZipFile as err: raise jamovi_exceptions.JamoviRendererError( - '{} {}.'.format(self.MESSAGE_FILE_CORRUPT, str(err)), + f'{self.MESSAGE_FILE_CORRUPT} {str(err)}.', extension=self.metadata.ext, corruption_type='bad_zip', reason=str(err), @@ -76,7 +76,7 @@ def _check_file(self, zip_file): manifest = manifest_data.read().decode('utf-8') except KeyError: raise jamovi_exceptions.JamoviFileCorruptError( - '{} Missing manifest'.format(self.MESSAGE_FILE_CORRUPT), + f'{self.MESSAGE_FILE_CORRUPT} Missing manifest', extension=self.metadata.ext, corruption_type='key_error', reason='zip missing manifest', @@ -93,7 +93,7 @@ def _check_file(self, zip_file): break else: raise jamovi_exceptions.JamoviFileCorruptError( - '{} Data-Archive-Version not found.'.format(self.MESSAGE_FILE_CORRUPT), + f'{self.MESSAGE_FILE_CORRUPT} Data-Archive-Version not found.', extension=self.metadata.ext, corruption_type='manifest_parse_error', reason='Data-Archive-Version not found.', @@ -104,17 +104,17 @@ def _check_file(self, zip_file): try: if archive_version < self.MINIMUM_VERSION: raise jamovi_exceptions.JamoviFileCorruptError( - '{} Data-Archive-Version is too old.'.format(self.MESSAGE_FILE_CORRUPT), + f'{self.MESSAGE_FILE_CORRUPT} Data-Archive-Version is too old.', extension=self.metadata.ext, corruption_type='manifest_parse_error', reason='Data-Archive-Version not found.', ) except TypeError: raise jamovi_exceptions.JamoviFileCorruptError( - '{} Data-Archive-Version not parsable.'.format(self.MESSAGE_FILE_CORRUPT), + f'{self.MESSAGE_FILE_CORRUPT} Data-Archive-Version not parsable.', extension=self.metadata.ext, corruption_type='manifest_parse_error', - reason='Data-Archive-Version ({}) not parsable.'.format(version_str), + reason=f'Data-Archive-Version ({version_str}) not parsable.', ) return True diff --git a/mfr/extensions/jasp/html_processor.py b/mfr/extensions/jasp/html_processor.py index 1dbf4547f..26289cb2f 100644 --- a/mfr/extensions/jasp/html_processor.py +++ b/mfr/extensions/jasp/html_processor.py @@ -1,4 +1,3 @@ - from io import StringIO from html.parser import HTMLParser diff --git a/mfr/extensions/jasp/render.py b/mfr/extensions/jasp/render.py index abcece4b4..ae9bd0e65 100644 --- a/mfr/extensions/jasp/render.py +++ b/mfr/extensions/jasp/render.py @@ -29,7 +29,7 @@ def render(self): return self.TEMPLATE.render(base=self.assets_url, body=body) except BadZipFile as err: raise exceptions.JaspFileCorruptError( - '{} Failure to unzip. {}.'.format(self.MESSAGE_FILE_CORRUPT, str(err)), + f'{self.MESSAGE_FILE_CORRUPT} Failure to unzip. {str(err)}.', extension=self.metadata.ext, corruption_type='bad_zip', reason=str(err), @@ -50,7 +50,7 @@ def _render_html(self, zip_file, ext, *args, **kwargs): index = index_data.read().decode('utf-8') except KeyError: raise exceptions.JaspFileCorruptError( - '{} Missing index.html.'.format(self.MESSAGE_FILE_CORRUPT), + f'{self.MESSAGE_FILE_CORRUPT} Missing index.html.', extension=self.metadata.ext, corruption_type='key_error', reason='zip missing ./index.html', @@ -78,7 +78,7 @@ def _check_file(self, zip_file): manifest, flavor = manifest_data.read().decode('utf-8'), 'java' except KeyError: raise exceptions.JaspFileCorruptError( - '{} Missing manifest'.format(self.MESSAGE_FILE_CORRUPT), + f'{self.MESSAGE_FILE_CORRUPT} Missing manifest', extension=self.metadata.ext, corruption_type='key_error', reason='zip missing manifest', @@ -107,7 +107,7 @@ def _verify_java_manifest(self, manifest): createdBy = str(value) if not dataArchiveVersionStr: raise exceptions.JaspFileCorruptError( - '{} Data-Archive-Version not found.'.format(self.MESSAGE_FILE_CORRUPT), + f'{self.MESSAGE_FILE_CORRUPT} Data-Archive-Version not found.', extension=self.metadata.ext, corruption_type='manifest_parse_error', reason='Data-Archive-Version not found.', @@ -130,10 +130,10 @@ def _verify_java_manifest(self, manifest): ) except TypeError: raise exceptions.JaspFileCorruptError( - '{} Data-Archive-Version not parsable.'.format(self.MESSAGE_FILE_CORRUPT), + f'{self.MESSAGE_FILE_CORRUPT} Data-Archive-Version not parsable.', extension=self.metadata.ext, corruption_type='manifest_parse_error', - reason='Data-Archive-Version ({}) not parsable.'.format(dataArchiveVersionStr), + reason=f'Data-Archive-Version ({dataArchiveVersionStr}) not parsable.', ) return @@ -145,7 +145,7 @@ def _verify_json_manifest(self, manifest): jasp_archive_version_str = manifest_data.get('jaspArchiveVersion', None) if not jasp_archive_version_str: raise exceptions.JaspFileCorruptError( - '{} jaspArchiveVersion not found.'.format(self.MESSAGE_FILE_CORRUPT), + f'{self.MESSAGE_FILE_CORRUPT} jaspArchiveVersion not found.', extension=self.metadata.ext, corruption_type='manifest_parse_error', reason='jaspArchiveVersion not found.', @@ -167,10 +167,10 @@ def _verify_json_manifest(self, manifest): ) except TypeError: raise exceptions.JaspFileCorruptError( - '{} jaspArchiveVersion not parsable.'.format(self.MESSAGE_FILE_CORRUPT), + f'{self.MESSAGE_FILE_CORRUPT} jaspArchiveVersion not parsable.', extension=self.metadata.ext, corruption_type='manifest_parse_error', - reason='jaspArchiveVersion ({}) not parsable.'.format(jasp_archive_version_str), + reason=f'jaspArchiveVersion ({jasp_archive_version_str}) not parsable.', ) return diff --git a/mfr/extensions/jsc3d/freecad_converter.py b/mfr/extensions/jsc3d/freecad_converter.py index 45b26bfc6..c1798fccb 100644 --- a/mfr/extensions/jsc3d/freecad_converter.py +++ b/mfr/extensions/jsc3d/freecad_converter.py @@ -12,6 +12,7 @@ try: Part.open(in_fn) except: + # Todo: maybe it is needed to use logger and specific message for exception logging sys.exit(1) o = [FreeCAD.getDocument("Unnamed").findObjects()[0]] diff --git a/mfr/extensions/md/render.py b/mfr/extensions/md/render.py index 792f07256..45027cf17 100644 --- a/mfr/extensions/md/render.py +++ b/mfr/extensions/md/render.py @@ -10,6 +10,7 @@ class EscapeHtml(Extension): def extendMarkdown(self, md, md_globals): + # Todo: do not see extendMarkdown explicit call and what is passed as the method args, maybe it is ok del md.preprocessors['html_block'] del md.inlinePatterns['html'] @@ -27,7 +28,7 @@ def __init__(self, *args, **kwargs): def render(self): """Render a markdown file to html.""" - with open(self.file_path, 'r') as fp: + with open(self.file_path) as fp: body = markdown.markdown(fp.read(), extensions=[EscapeHtml()]) return self.TEMPLATE.render(base=self.assets_url, body=body) diff --git a/mfr/extensions/pdf/export.py b/mfr/extensions/pdf/export.py index 18f7a7f12..878cb5e64 100644 --- a/mfr/extensions/pdf/export.py +++ b/mfr/extensions/pdf/export.py @@ -66,7 +66,7 @@ def tiff_to_pdf(self, tiff_img, max_size): c.save() def export(self): - logger.debug('pdf-export: format::{}'.format(self.format)) + logger.debug(f'pdf-export: format::{self.format}') parts = self.format.split('.') export_type = parts[-1].lower() max_size = [int(x) for x in parts[0].split('x')] if len(parts) == 2 else None @@ -89,8 +89,7 @@ def export(self): self.tiff_to_pdf(image, max_size) image.close() - except (UnicodeDecodeError, IOError) as err: - name, extension = os.path.splitext(os.path.split(self.source_file_path)[-1]) + except (UnicodeDecodeError, OSError) as err: raise exceptions.PillowImageError( 'Unable to export the file as a {}, please check that the ' 'file is a valid tiff image.'.format(export_type), diff --git a/mfr/extensions/pdf/render.py b/mfr/extensions/pdf/render.py index 147d0cab4..c0a10e94e 100644 --- a/mfr/extensions/pdf/render.py +++ b/mfr/extensions/pdf/render.py @@ -23,7 +23,7 @@ def render(self): download_url = munge_url_for_localdev(self.metadata.download_url) escaped_name = escape_url_for_template( - '{}{}'.format(self.metadata.name, self.metadata.ext) + f'{self.metadata.name}{self.metadata.ext}' ) logger.debug('extension::{} supported-list::{}'.format(self.metadata.ext, settings.EXPORT_SUPPORTED)) diff --git a/mfr/extensions/rst/render.py b/mfr/extensions/rst/render.py index fd853814f..d0d37d8b9 100644 --- a/mfr/extensions/rst/render.py +++ b/mfr/extensions/rst/render.py @@ -20,7 +20,7 @@ def __init__(self, *args, **kwargs): self.metrics.add('docutils_version', docutils.__version__) def render(self): - with open(self.file_path, 'r') as fp: + with open(self.file_path) as fp: body = publish_parts(fp.read(), writer_name='html')['html_body'] return self.TEMPLATE.render(base=self.assets_url, body=body) diff --git a/mfr/extensions/tabular/compat.py b/mfr/extensions/tabular/compat.py index e4bca09ac..4bb6a1988 100644 --- a/mfr/extensions/tabular/compat.py +++ b/mfr/extensions/tabular/compat.py @@ -1,5 +1,3 @@ -from __future__ import division - range = range string_types = (str,) unicode = str diff --git a/mfr/extensions/tabular/libs/panda_tools.py b/mfr/extensions/tabular/libs/panda_tools.py index 69810245a..14918a4be 100644 --- a/mfr/extensions/tabular/libs/panda_tools.py +++ b/mfr/extensions/tabular/libs/panda_tools.py @@ -51,7 +51,7 @@ def sav_pandas(fp): def data_from_dataframe(dataframe): """Convert a dataframe object to a list of dictionaries - :param fp: File pointer object + :param dataframe: pandas dataframe :return: tuple of table headers and data """ diff --git a/mfr/extensions/tabular/libs/stdlib_tools.py b/mfr/extensions/tabular/libs/stdlib_tools.py index 3d3611984..ed30f837c 100644 --- a/mfr/extensions/tabular/libs/stdlib_tools.py +++ b/mfr/extensions/tabular/libs/stdlib_tools.py @@ -27,7 +27,7 @@ def csv_stdlib(fp): for idx, fieldname in enumerate(reader.fieldnames or []): column_count = sum(1 for column in columns if fieldname == column['name']) if column_count: - unique_fieldname = '{}-{}'.format(fieldname, column_count + 1) + unique_fieldname = f'{fieldname}-{column_count + 1}' reader.fieldnames[idx] = unique_fieldname else: unique_fieldname = fieldname @@ -49,7 +49,7 @@ def csv_stdlib(fp): extension='csv', ) from e else: - raise TabularRendererError('csv.Error: {}'.format(e), extension='csv') from e + raise TabularRendererError(f'csv.Error: {e}', extension='csv') from e if not columns and not rows: raise EmptyTableError('Table empty or corrupt.', extension='csv') @@ -66,7 +66,7 @@ def sav_stdlib(fp): :return: tuple of table headers and data """ csv_file = utilities.sav_to_csv(fp) - with open(csv_file.name, 'r') as file: + with open(csv_file.name) as file: csv_file.close() return csv_stdlib(file) diff --git a/mfr/extensions/tabular/libs/xlrd_tools.py b/mfr/extensions/tabular/libs/xlrd_tools.py index 7afbd145d..184c3be43 100644 --- a/mfr/extensions/tabular/libs/xlrd_tools.py +++ b/mfr/extensions/tabular/libs/xlrd_tools.py @@ -31,7 +31,7 @@ def xlsx_xlrd(fp): fields = [ str(value) if not isinstance(value, basestring) and value is not None - else value or 'Unnamed: {0}'.format(index + 1) + else value or f'Unnamed: {index + 1}' for index, value in enumerate(fields) ] diff --git a/mfr/extensions/tabular/utilities.py b/mfr/extensions/tabular/utilities.py index 3996c3bcd..7385343cb 100644 --- a/mfr/extensions/tabular/utilities.py +++ b/mfr/extensions/tabular/utilities.py @@ -22,15 +22,15 @@ def header_population(headers): def data_population(in_data, headers=None): """Convert a list of lists into a list of dicts associating each cell with its column header and row - :param data: two dimensional list of data - :param fields: column headers + :param in_data: two dimensional list of data + :param headers: column headers :return: JSON representation of rows """ headers = headers or in_data[0] return [ - dict([(header, row[cindex]) - for cindex, header in enumerate(headers)]) + {header: row[cindex] + for cindex, header in enumerate(headers)} for row in in_data ] diff --git a/mfr/extensions/unoconv/export.py b/mfr/extensions/unoconv/export.py index 2bff77d3a..1e5c8057d 100644 --- a/mfr/extensions/unoconv/export.py +++ b/mfr/extensions/unoconv/export.py @@ -17,7 +17,7 @@ def export(self): run([ UNOCONV_BIN, '-n', - '-c', 'socket,host={},port={};urp;StarOffice.ComponentContext'.format(ADDRESS, PORT), + '-c', f'socket,host={ADDRESS},port={PORT};urp;StarOffice.ComponentContext', '-f', self.format, '-o', self.output_file_path, '-vvv', diff --git a/mfr/extensions/unoconv/render.py b/mfr/extensions/unoconv/render.py index a872829db..6914f35ff 100644 --- a/mfr/extensions/unoconv/render.py +++ b/mfr/extensions/unoconv/render.py @@ -51,7 +51,8 @@ def render(self): self.metadata.ext, self.file_path, self.export_file_path, - self.map['format'] + self.map['format'], + self.metadata, ) exporter.export() diff --git a/mfr/extensions/utils.py b/mfr/extensions/utils.py index ab43f5548..fa3b9ca67 100644 --- a/mfr/extensions/utils.py +++ b/mfr/extensions/utils.py @@ -7,7 +7,7 @@ logger = logging.getLogger(__name__) -def munge_url_for_localdev(url: str) -> Tuple: +def munge_url_for_localdev(url: str) -> tuple: """If MFR is being run in a local development environment (i.e. LOCAL_DEVELOPMENT is True), we need to replace the internal host (the one the backend services communicate on, default: 192.168.168.167) with the external host (the one the user provides, default: "localhost") @@ -23,7 +23,7 @@ def munge_url_for_localdev(url: str) -> Tuple: url_obj = url_obj._replace( query=urlencode(query_dict, doseq=True), - netloc='{}:{}'.format(settings.LOCAL_HOST, url_obj.port) + netloc=f'{settings.LOCAL_HOST}:{url_obj.port}' ) return url_obj diff --git a/mfr/providers/__init__.py b/mfr/providers/__init__.py index 5284146eb..e69de29bb 100644 --- a/mfr/providers/__init__.py +++ b/mfr/providers/__init__.py @@ -1 +0,0 @@ -__import__("pkg_resources").declare_namespace(__name__) diff --git a/mfr/providers/osf/provider.py b/mfr/providers/osf/provider.py index 4745e8ecc..32fe2b485 100644 --- a/mfr/providers/osf/provider.py +++ b/mfr/providers/osf/provider.py @@ -59,7 +59,8 @@ async def metadata(self): differently. """ download_url = await self._fetch_download_url() - logger.debug('download_url::{}'.format(download_url)) + logger.debug(f'download_url::{download_url}') + metadata = {} if '/file?' in download_url: # URL is for WaterButler v0 API # TODO Remove this when API v0 is officially deprecated @@ -82,7 +83,7 @@ async def metadata(self): if response_code != 200: raise exceptions.MetadataError( 'Failed to fetch file metadata from WaterButler. Received response: ', - 'code {} {}'.format(str(response_code), str(response_reason)), + f'code {str(response_code)} {str(response_reason)}', metadata_url=download_url, response=response_reason, provider=self.NAME, @@ -127,7 +128,7 @@ async def metadata(self): unique_key = hashlib.sha256((meta['etag'] + cleaned_url.url).encode('utf-8')).hexdigest() stable_str = '/{}/{}{}'.format(meta['resource'], meta['provider'], meta['path']) stable_id = hashlib.sha256(stable_str.encode('utf-8')).hexdigest() - logger.debug('stable_identifier: str({}) hash({})'.format(stable_str, stable_id)) + logger.debug(f'stable_identifier: str({stable_str}) hash({stable_id})') return provider.ProviderMetadata(name, ext, content_type, unique_key, download_url, stable_id) @@ -139,7 +140,7 @@ async def download(self): if response.status >= 400: resp_text = await response.text() - logger.error('Unable to download file: ({}) {}'.format(response.status, resp_text)) + logger.error(f'Unable to download file: ({response.status}) {resp_text}') raise exceptions.DownloadError( 'Unable to download the requested file, please try again later.', download_url=download_url, @@ -180,7 +181,7 @@ async def _fetch_download_url(self): ) await request.release() - logger.debug('osf-download-resolver: request.status::{}'.format(request.status)) + logger.debug(f'osf-download-resolver: request.status::{request.status}') if request.status != 302: raise exceptions.MetadataError( request.reason, diff --git a/mfr/server/app.py b/mfr/server/app.py index d7f4b2abc..4fbecdac1 100644 --- a/mfr/server/app.py +++ b/mfr/server/app.py @@ -7,7 +7,10 @@ import tornado.web import tornado.httpserver import tornado.platform.asyncio -from raven.contrib.tornado import AsyncSentryClient + +import sentry_sdk +from sentry_sdk.integrations.tornado import TornadoIntegration +from sentry_sdk.integrations.logging import LoggingIntegration from mfr import settings from mfr.server import settings as server_settings @@ -24,19 +27,29 @@ def sig_handler(sig, frame): - io_loop = tornado.ioloop.IOLoop.instance() + """ + https://stackoverflow.com/questions/34554247/python-tornado-i-o-loop-current-vs-instance-method + https://www.tornadoweb.org/en/branch6.3/_modules/tornado/testing.html + """ + io_loop = tornado.ioloop.IOLoop.current() + loop = io_loop.asyncio_loop # Access the asyncio loop from Tornado def stop_loop(): - if len(asyncio.Task.all_tasks(io_loop)) == 0: - io_loop.stop() - else: + """ + Retrieve all tasks associated with tornado in asyncio loop + Todo: (maybe there is more explicit way to check than 'tornado' in repr(task)) + """ + exists_tornado_task = any(task for task in asyncio.all_tasks(loop) if 'tornado' in repr(task)) + if exists_tornado_task: io_loop.call_later(1, stop_loop) + else: + io_loop.stop() io_loop.add_callback_from_signal(stop_loop) def almost_apache_style_log(handler): - '''without status code and body length''' + """without status code and body length""" req = handler.request access_logger.info('%s - - [%s +0800] "%s %s %s" - - "%s" "%s"' % (req.remote_ip, time.strftime("%d/%b/%Y:%X"), req.method, @@ -46,6 +59,15 @@ def almost_apache_style_log(handler): def make_app(debug): + sentry_logging = LoggingIntegration( + level=logging.INFO, # Capture INFO level and above as breadcrumbs + event_level=None, # Do not send logs of any level as events + ) + sentry_sdk.init( + dsn=settings.SENTRY_DSN, + release=__version__, + integrations=[TornadoIntegration(), sentry_logging, ], + ) app = tornado.web.Application( [ (r'/static/(.*)', tornado.web.StaticFileHandler, {'path': server_settings.STATIC_PATH}), @@ -59,7 +81,6 @@ def make_app(debug): debug=debug, log_function=almost_apache_style_log, ) - app.sentry_client = AsyncSentryClient(settings.SENTRY_DSN, release=__version__) return app @@ -83,7 +104,7 @@ def serve(): ssl_options=ssl_options, ) - logger.info("Listening on {0}:{1}".format(server_settings.ADDRESS, server_settings.PORT)) + logger.info(f"Listening on {server_settings.ADDRESS}:{server_settings.PORT}") signal.signal(signal.SIGTERM, partial(sig_handler)) asyncio.get_event_loop().set_debug(server_settings.DEBUG) diff --git a/mfr/server/handlers/core.py b/mfr/server/handlers/core.py index 4c2bd067f..416a2e4ff 100644 --- a/mfr/server/handlers/core.py +++ b/mfr/server/handlers/core.py @@ -3,11 +3,11 @@ import uuid import asyncio import logging -import pkg_resources +from importlib.metadata import entry_points import tornado.web import tornado.iostream -from raven.contrib.tornado import SentryMixin +import sentry_sdk import waterbutler.core.utils import waterbutler.server.utils @@ -76,7 +76,7 @@ def options(self): self.set_header('Access-Control-Allow-Methods', 'GET, PUT, POST, DELETE'), -class BaseHandler(CorsMixin, tornado.web.RequestHandler, SentryMixin): +class BaseHandler(CorsMixin, tornado.web.RequestHandler): """Base class for the Render and Export handlers. Fetches the file metadata for the file indicated by the ``url`` query parameter and builds the provider caches. Also handles writing output and errors. @@ -93,9 +93,12 @@ def __init__(self, *args, **kwargs): self.metrics = self.handler_metrics.new_subrecord(self.NAME) self.extension_metrics = MetricsRecord('extension') + self.url = '' - @abc.abstractproperty + + @abc.abstractmethod def NAME(self): + # Todo: not see Name implementation in child classes raise NotImplementedError async def prepare(self): @@ -113,7 +116,7 @@ async def prepare(self): provider=settings.PROVIDER_NAME, code=400, ) - logging.debug('target_url::{}'.format(self.url)) + logging.debug(f'target_url::{self.url}') self.provider = utils.make_provider( settings.PROVIDER_NAME, @@ -124,7 +127,7 @@ async def prepare(self): self.metadata = await self.provider.metadata() self.extension_metrics.add('ext', self.metadata.ext) - logging.debug('extension::{}'.format(self.metadata.ext)) + logging.debug(f'extension::{self.metadata.ext}') self.cache_provider = waterbutler.core.utils.make_provider( settings.CACHE_PROVIDER_NAME, @@ -159,15 +162,26 @@ async def write_stream(self, stream): return def write_error(self, status_code, exc_info): - self.captureException(exc_info) # Log all non 2XX codes to sentry + + # TODO: verify that `exc_info` arg is compatible with tornado 6.4.2 sig etype, exc, _ = exc_info + scope = sentry_sdk.get_current_scope() + scope.set_tag('class', etype.__name_) + scope.set_tag('status_code', status_code) + sentry_sdk.capture_exception(exc) # Log all non 2XX codes to sentry if issubclass(etype, exceptions.PluginError): try: # clever errors shouldn't break other things current, child_type = {}, None for level in reversed(exc.attr_stack): if current: - current = {'{}_{}'.format(level[0], child_type): current} + """ + TODO: dictionary 'current' is reassigned in condition, Qodana code inspector may be + complaining because some data previously saved in the dictionary may be lost + (but maybe it is ok in terms of business logic), + As I understand the current accumulate previous current versions so it may be ok + """ + current = {f'{level[0]}_{child_type}': current} current['child_type'] = child_type current.update(level[1]) current['self_type'] = level[0] @@ -176,6 +190,7 @@ def write_error(self, status_code, exc_info): current['materialized_type'] = '.'.join([x[0] for x in exc.attr_stack]) self.error_metrics = current except Exception: + # Todo: maybe it is needed to use logger and specific message for exception logging pass self.set_status(exc.code) self.finish(exc.as_html()) @@ -204,10 +219,10 @@ def write_error(self, status_code, exc_info): def log_exception(self, typ, value, tb): if isinstance(value, tornado.web.HTTPError): if value.log_message: - format = "%d %s: " + value.log_message + log_message_format = "%d %s: " + value.log_message args = ([value.status_code, self._request_summary()] + list(value.args)) - tornado.web.gen_log.warning(format, *args) + tornado.web.gen_log.warning(log_message_format, *args) else: tornado.web.app_log.error("[User-Agent: %s] Uncaught exception %s\n", self.request.headers.get('User-Agent', '*none found*'), @@ -282,22 +297,31 @@ class ExtensionsStaticFileHandler(tornado.web.StaticFileHandler, CorsMixin): """ def initialize(self): + # Todo: the method args differ in comparison with StaticFileHandler namespace = 'mfr.renderers' module_path = 'mfr.extensions' - self.modules = { - ep.module_name.replace(module_path + '.', ''): os.path.join(ep.dist.location, 'mfr', 'extensions', ep.module_name.replace(module_path + '.', ''), 'static') - for ep in list(pkg_resources.iter_entry_points(namespace)) - } + + self.modules = {} + + for ep in entry_points().select(group=namespace): + module_name = ep.value.split(":")[0] # replacement for ep.module_name + module = module_name.replace(module_path + ".", "").split(".")[0] + dist_location = ep.dist.locate_file("") # replacement for ep.dist.location + static_path = os.path.join(dist_location, 'mfr', 'extensions', module, 'static') + self.modules[module] = static_path async def get(self, module_name, path): + # Todo: the method args differ in comparison with StaticFileHandler, maybe it is ok try: super().initialize(self.modules[module_name]) return await super().get(path) except Exception: + # Todo: maybe it is needed to use logger and specific message for exception logging self.set_status(404) try: super().initialize(settings.STATIC_PATH) return await super().get(path) except Exception: + # Todo: maybe it is needed to use logger and specific message for exception logging self.set_status(404) diff --git a/mfr/server/handlers/export.py b/mfr/server/handlers/export.py index 080499b8e..24add4e58 100644 --- a/mfr/server/handlers/export.py +++ b/mfr/server/handlers/export.py @@ -23,29 +23,29 @@ async def prepare(self): await super().prepare() - format = self.request.query_arguments.get('format', None) - if not format: + query_arguments_format = self.request.query_arguments.get('format', None) + if not query_arguments_format: raise InvalidParameters("Invalid Request: Url requires query parameter 'format' with" " appropriate extension") # TODO: do we need to catch exceptions for decoding? - self.format = format[0].decode('utf-8') + self.format = query_arguments_format[0].decode('utf-8') self.exporter_name = utils.get_exporter_name(self.metadata.ext) - self.cache_file_id = '{}.{}'.format(self.metadata.unique_key, self.format) + self.cache_file_id = f'{self.metadata.unique_key}.{self.format}' if self.exporter_name: - cache_file_path_str = '/export/{}.{}'.format(self.cache_file_id, self.exporter_name) + cache_file_path_str = f'/export/{self.cache_file_id}.{self.exporter_name}' else: - cache_file_path_str = '/export/{}'.format(self.cache_file_id) + cache_file_path_str = f'/export/{self.cache_file_id}' self.cache_file_path = await self.cache_provider.validate_path(cache_file_path_str) self.source_file_path = await self.local_cache_provider.validate_path( - '/export/{}'.format(self.source_file_id) + f'/export/{self.source_file_id}' ) - self.output_file_id = '{}.{}'.format(self.source_file_path.name, self.format) + self.output_file_id = f'{self.source_file_path.name}.{self.format}' self.output_file_path = await self.local_cache_provider.validate_path( - '/export/{}'.format(self.output_file_id) + f'/export/{self.output_file_id}' ) self.metrics.merge({ 'output_file': { @@ -59,9 +59,9 @@ async def get(self): """Export a file to the format specified via the associated extension library""" # File is already in the requested format - if self.metadata.ext.lower() == ".{}".format(self.format.lower()): + if self.metadata.ext.lower() == f".{self.format.lower()}": await self.write_stream(await self.provider.download()) - logger.info('Exported {} with no conversion.'.format(self.format)) + logger.info(f'Exported {self.format} with no conversion.') self.metrics.add('export.conversion', 'noop') return @@ -69,11 +69,11 @@ async def get(self): try: cached_stream = await self.cache_provider.download(self.cache_file_path) except DownloadError as e: - assert e.code == 404, 'Non-404 DownloadError {!r}'.format(e) - logger.info('No cached file found; Starting export [{}]'.format(self.cache_file_path)) + assert e.code == 404, f'Non-404 DownloadError {e!r}' + logger.info(f'No cached file found; Starting export [{self.cache_file_path}]') self.metrics.add('cache_file.result', 'miss') else: - logger.info('Cached file found; Sending downstream [{}]'.format(self.cache_file_path)) + logger.info(f'Cached file found; Sending downstream [{self.cache_file_path}]') self.metrics.add('cache_file.result', 'hit') self._set_headers() return await self.write_stream(cached_stream) diff --git a/mfr/server/handlers/exporters.py b/mfr/server/handlers/exporters.py index 1bea4694d..13cb2c471 100644 --- a/mfr/server/handlers/exporters.py +++ b/mfr/server/handlers/exporters.py @@ -1,4 +1,4 @@ -import pkg_resources +from importlib.metadata import entry_points import tornado.web @@ -8,7 +8,7 @@ def get(self): """List available exporters""" exporters = {} - for ep in pkg_resources.iter_entry_points(group='mfr.exporters'): + for ep in entry_points().select(group='mfr.exporters'): exporters.update({ep.name: ep.load().__name__}) self.write({ diff --git a/mfr/server/handlers/render.py b/mfr/server/handlers/render.py index a8c75419a..842828f7a 100644 --- a/mfr/server/handlers/render.py +++ b/mfr/server/handlers/render.py @@ -29,13 +29,13 @@ async def prepare(self): self.cache_file_id = self.metadata.unique_key if self.renderer_name: - cache_file_path_str = '/export/{}.{}'.format(self.cache_file_id, self.renderer_name) + cache_file_path_str = f'/export/{self.cache_file_id}.{self.renderer_name}' else: - cache_file_path_str = '/export/{}'.format(self.cache_file_id) + cache_file_path_str = f'/export/{self.cache_file_id}' self.cache_file_path = await self.cache_provider.validate_path(cache_file_path_str) self.source_file_path = await self.local_cache_provider.validate_path( - '/render/{}'.format(self.source_file_id) + f'/render/{self.source_file_id}' ) async def get(self): @@ -45,7 +45,7 @@ async def get(self): self.metadata, self.source_file_path.full_path, self.url, - '{}://{}/assets'.format(self.request.protocol, self.request.host), + f'{self.request.protocol}://{self.request.host}/assets', self.request.uri.replace('/render?', '/export?', 1) ) @@ -55,11 +55,11 @@ async def get(self): try: cached_stream = await self.cache_provider.download(self.cache_file_path) except waterbutler.core.exceptions.DownloadError as e: - assert e.code == 404, 'Non-404 DownloadError {!r}'.format(e) - logger.info('No cached file found; Starting render [{}]'.format(self.cache_file_path)) + assert e.code == 404, f'Non-404 DownloadError {e!r}' + logger.info(f'No cached file found; Starting render [{self.cache_file_path}]') self.metrics.add('cache_file.result', 'miss') else: - logger.info('Cached file found; Sending downstream [{}]'.format(self.cache_file_path)) + logger.info(f'Cached file found; Sending downstream [{self.cache_file_path}]') self.metrics.add('cache_file.result', 'hit') return await self.write_stream(cached_stream) diff --git a/mfr/server/handlers/renderers.py b/mfr/server/handlers/renderers.py index 572e08365..dedbf3d6c 100644 --- a/mfr/server/handlers/renderers.py +++ b/mfr/server/handlers/renderers.py @@ -1,4 +1,4 @@ -import pkg_resources +from importlib.metadata import entry_points import tornado.web @@ -8,7 +8,7 @@ def get(self): """List available renderers""" renderers = {} - for ep in pkg_resources.iter_entry_points(group='mfr.renderers'): + for ep in entry_points().select(group='mfr.renderers'): renderers.update({ep.name: ep.load().__name__}) self.write({ diff --git a/mfr/settings.py b/mfr/settings.py index 7dd613ad4..2d63c6b51 100644 --- a/mfr/settings.py +++ b/mfr/settings.py @@ -41,9 +41,9 @@ def __init__(self, *args, parent=None, **kwargs): def get(self, key, default=None): """Fetch a config value for ``key`` from the settings. First checks the env, then the on-disk config. If neither exists, returns ``default``.""" - env = self.full_key(key) - if env in os.environ: - return os.environ.get(env) + environ = self.full_key(key) + if environ in os.environ: + return os.environ.get(environ) return super().get(key, default) def get_bool(self, key, default=None): @@ -80,7 +80,7 @@ def get_object(self, key, default=None): def full_key(self, key): """The name of the envvar which corresponds to this key.""" - return '{}_{}'.format(self.parent, key) if self.parent else key + return f'{self.parent}_{key}' if self.parent else key def child(self, key): """Fetch a sub-dict of the current dict.""" @@ -146,16 +146,16 @@ def child(self, key): try: - config_path = os.environ['{}_CONFIG'.format(PROJECT_NAME.upper())] + config_path = os.environ[f'{PROJECT_NAME.upper()}_CONFIG'] except KeyError: env = os.environ.get('ENV', 'test') - config_path = '{}/{}-{}.json'.format(PROJECT_CONFIG_PATH, PROJECT_NAME, env) + config_path = f'{PROJECT_CONFIG_PATH}/{PROJECT_NAME}-{env}.json' config = SettingsDict() config_path = os.path.expanduser(config_path) if not os.path.exists(config_path): - logging.warning('No \'{}\' configuration file found'.format(config_path)) + logging.warning(f'No \'{config_path}\' configuration file found') else: with open(os.path.expanduser(config_path)) as fp: config = SettingsDict(json.load(fp)) diff --git a/requirements.txt b/requirements.txt index 575639416..a3f9207a5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,16 @@ -aiohttp==0.18.4 +aiohttp==3.10.6 chardet==2.3.0 furl==0.4.2 humanfriendly==2.1 -invoke==0.13.0 +invoke==2.2.0 mako==1.0.1 -raven==5.27.0 -setuptools==37.0.0 +sentry-sdk==2.22.0 +setuptools==78.1.0 stevedore==1.2.0 -tornado==4.3 +tornado==6.4.2 # WaterButler -git+https://github.com/CenterForOpenScience/waterbutler.git@0.38.6#egg=waterbutler +git+https://github.com/CenterForOpenScience/waterbutler.git@feature/buff-worms agent==0.1.2 google-auth==1.4.1 @@ -23,7 +23,7 @@ pydocx==0.7.0 # Image olefile==0.44 -Pillow==4.3.0 +Pillow==11.0.0 psd-tools==1.4 # IPython @@ -36,7 +36,7 @@ jinja2==2.10.1 mistune==0.8.1 # Pdf -reportlab==3.6.5 +reportlab==4.2.4 # Pptx # python-pptx==0.5.7 @@ -45,10 +45,10 @@ reportlab==3.6.5 docutils==0.12 # Tabular -pandas==0.25.1 +pandas==2.2.3 xlrd==1.0.0 -h5py==2.7.0 -scipy==0.19.1 +h5py==3.13 +scipy==1.14.1 # Md markdown==2.6.2 diff --git a/setup.py b/setup.py index 6feaa350b..45c81a55f 100755 --- a/setup.py +++ b/setup.py @@ -1,8 +1,8 @@ from setuptools import setup, find_packages -def parse_requirements(requirements): - with open(requirements) as f: +def parse_requirements(requirements_txt): + with open(requirements_txt) as f: return [l.strip('\n') for l in f if l.strip('\n') and not l.startswith('#')] diff --git a/tasks.py b/tasks.py index 8280a6f3b..1c8d265c2 100644 --- a/tasks.py +++ b/tasks.py @@ -9,7 +9,7 @@ @task def wheelhouse(ctx, develop=False): req_file = 'dev-requirements.txt' if develop else 'requirements.txt' - cmd = 'pip wheel --find-links={} -r {} --wheel-dir={} -c {}'.format(WHEELHOUSE_PATH, req_file, WHEELHOUSE_PATH, CONSTRAINTS_FILE) + cmd = f'pip wheel --find-links={WHEELHOUSE_PATH} -r {req_file} --wheel-dir={WHEELHOUSE_PATH} -c {CONSTRAINTS_FILE}' ctx.run(cmd, pty=True) @@ -17,10 +17,10 @@ def wheelhouse(ctx, develop=False): def install(ctx, develop=False): ctx.run('python setup.py develop') req_file = 'dev-requirements.txt' if develop else 'requirements.txt' - cmd = 'pip install --upgrade -r {} -c {}'.format(req_file, CONSTRAINTS_FILE) + cmd = f'pip install --upgrade -r {req_file} -c {CONSTRAINTS_FILE}' if WHEELHOUSE_PATH: - cmd += ' --no-index --find-links={}'.format(WHEELHOUSE_PATH) + cmd += f' --no-index --find-links={WHEELHOUSE_PATH}' ctx.run(cmd, pty=True) @@ -43,14 +43,14 @@ def test(ctx, verbose=False, nocov=False, extension=None, path=None): # `--extension=` and `--path=` are mutually exclusive options assert not (extension and path) if path: - path = '/{}'.format(path) if path else '' + path = f'/{path}' if path else '' elif extension: - path = '/extensions/{}/'.format(extension) if extension else '' + path = f'/extensions/{extension}/' if extension else '' else: path = '' coverage = ' --cov-report term-missing --cov mfr' if not nocov else '' verbose = '-v' if verbose else '' - cmd = 'py.test{} tests{} {}'.format(coverage, path, verbose) + cmd = f'py.test{coverage} tests{path} {verbose}' ctx.run(cmd, pty=True) diff --git a/tests/core/test_utils.py b/tests/core/test_utils.py index 7e908e353..7d249ec98 100644 --- a/tests/core/test_utils.py +++ b/tests/core/test_utils.py @@ -1,5 +1,5 @@ import pytest -import pkg_resources +from importlib.metadata import entry_points from mfr.core import utils as mfr_utils @@ -14,9 +14,8 @@ def test_get_renderer_name_explicit_assertions(self): assert mfr_utils.get_renderer_name('.pdf') == 'PdfRenderer' def test_get_renderer_name(self): - entry_points = pkg_resources.iter_entry_points(group='mfr.renderers') - for ep in entry_points: - expected = ep.attrs[0] + for ep in entry_points().select(group='mfr.renderers'): + expected = ep.value.split(":")[1].split('.')[0] assert mfr_utils.get_renderer_name(ep.name) == expected def test_get_renderer_name_no_entry_point(self): @@ -30,9 +29,8 @@ def test_get_exporter_name_explicit_assertions(self): assert mfr_utils.get_exporter_name('.odt') == 'UnoconvExporter' def test_get_exporter_name(self): - entry_points = pkg_resources.iter_entry_points(group='mfr.exporters') - for ep in entry_points: - expected = ep.attrs[0] + for ep in entry_points().select(group='mfr.exporters'): + expected = ep.value.split(":")[1].split('.')[0] assert mfr_utils.get_exporter_name(ep.name) == expected def test_get_exporter_name_no_entry_point(self): diff --git a/tests/documentation/test_entrypoints.py b/tests/documentation/test_entrypoints.py index 88f6bbb35..c9f02af9a 100644 --- a/tests/documentation/test_entrypoints.py +++ b/tests/documentation/test_entrypoints.py @@ -1,7 +1,7 @@ import os import pytest -import pkg_resources +from importlib.metadata import entry_points class TestEntryPoints: @@ -9,9 +9,9 @@ class TestEntryPoints: def test_entry_points(self): parent_dir = os.pardir - readme_path = os.path.join(os.path.dirname((parent_dir)), 'supportedextensions.md') - with open(readme_path, 'r') as file: + readme_path = os.path.join(os.path.dirname(parent_dir), 'supportedextensions.md') + with open(readme_path) as file: readme_ext = [line.strip()[2:] for line in file if '*' in line] - for ep in pkg_resources.iter_entry_points(group='mfr.renderers'): + for ep in entry_points().select(group='mfr.renderers'): if ep.name != 'none': assert ep.name in readme_ext diff --git a/tests/extensions/audio/test_renderer.py b/tests/extensions/audio/test_renderer.py index 6dc344142..4fa7a4652 100644 --- a/tests/extensions/audio/test_renderer.py +++ b/tests/extensions/audio/test_renderer.py @@ -40,7 +40,7 @@ class TestAudioRenderer: def test_render_audio(self, renderer, url): body = renderer.render() assert '