SQUASH COMMITS from other PR

Prune CSL Items to validate JSON schema

Refs manubot#47

CSL: replace arxiv_id with archive_location

Travis: install package using pip

Attempt to fix
python-jsonschema/jsonschema#449

--prune-csl option for manubot cite

Only remove a single additional property sub_error

Workaround the effect of
citation-style-language/schema#154

Switch to dhimmel/schema CSL JSON

Move validation to remove_jsonschema_errors

Test CSL pruning

Improve CSL pruning documentation

Default to pruning unless --bad-csl flag supplied

DOI CSL retriever: use shortDOI for URL

Switch CSL pruning logging to DEBUG

Update manubot cite help in README

arxiv citeproc: use int for date-parts

Author: dhimmel

.travis.yml

@@ -9,7 +9,7 @@ addons:
   apt_packages:
     - pandoc
 install:
-  - python setup.py install
+  - pip install .
   - python setup.py sdist bdist_wheel
 script: py.test
 deploy:

README.md

@@ -98,7 +98,7 @@ manubot cite doi:10.1098/rsif.2017.0387 pmid:29424689 pmcid:PMC5640425 arxiv:180
 Additional usage information is available from `manubot cite --help`:
 
 ```
-usage: manubot cite [-h] [--file FILE] citations [citations ...]
+usage: manubot cite [-h] [--file FILE] [--bad-csl] citations [citations ...]
 
 Retrieve bibliographic metadata for one or more citation identifiers.
 
@@ -108,6 +108,8 @@ positional arguments:
 optional arguments:
   -h, --help   show this help message and exit
   --file FILE  specify a file to write CSL output, otherwise default to stdout
+  --bad-csl    allow CSL Items that do not conform to the JSON Schema. Skips
+               CSL pruning.
 ```
 
 ## Installation

manubot/cite/__init__.py

@@ -99,7 +99,7 @@ def get_citation_id(standard_citation):
     return citation_id
 
 
-def citation_to_citeproc(citation):
+def citation_to_citeproc(citation, prune=True):
     """
     Return a dictionary with citation metadata
     """
@@ -113,7 +113,7 @@ def citation_to_citeproc(citation):
         raise ValueError(msg)
 
     citation_id = get_citation_id(citation)
-    citeproc = citeproc_passthrough(citeproc, set_id=citation_id)
+    citeproc = citeproc_passthrough(citeproc, set_id=citation_id, prune=prune)
 
     return citeproc
 
@@ -130,6 +130,11 @@ def add_subparser_cite(subparsers):
         default=sys.stdout,
         help='specify a file to write CSL output, otherwise default to stdout',
     )
+    parser.add_argument(
+        '--bad-csl',
+        action='store_true',
+        help='allow CSL Items that do not conform to the JSON Schema. Skips CSL pruning.',
+    )
     parser.add_argument(
         'citations',
         nargs='+',
@@ -143,7 +148,7 @@ def cli_cite(args):
     csl_list = list()
     for citation in args.citations:
         citation = standardize_citation(citation)
-        csl_list.append(citation_to_citeproc(citation))
+        csl_list.append(citation_to_citeproc(citation, prune=not args.bad_csl))
     with args.file as write_file:
         json.dump(csl_list, write_file, ensure_ascii=False, indent=2)
         write_file.write('\n')

manubot/cite/arxiv.py

@@ -46,7 +46,7 @@ def get_arxiv_citeproc(arxiv_id):
     pattern = re.compile(r'arxiv.org/abs/(.+)')
     match = pattern.search(url)
     versioned_id = match.group(1)
-    csl_item['arxiv_id'] = versioned_id
+    csl_item['archive_location'] = versioned_id
     _, csl_item['version'] = versioned_id.rsplit('v', 1)
     csl_item['URL'] = 'https://arxiv.org/abs/' + versioned_id
 
@@ -56,7 +56,7 @@ def get_arxiv_citeproc(arxiv_id):
     # Extract CSL date field
     published = entry.findtext(prefix + 'published')
     published, _ = published.split('T', 1)
-    csl_item['issued'] = {'date-parts': [published.split('-')]}
+    csl_item['issued'] = {'date-parts': [[int(x) for x in published.split('-')]]}
 
     # Extract authors
     authors = list()

manubot/cite/citeproc.py

@@ -1,42 +1,9 @@
-# Valid CSL (citeproc JSON) types as per
-# https://github.com/citation-style-language/schema/blob/4846e02f0a775a8272819204379a4f8d7f45c16c/csl-types.rnc#L5-L39
-citeproc_types = {
-    "article",
-    "article-journal",
-    "article-magazine",
-    "article-newspaper",
-    "bill",
-    "book",
-    "broadcast",
-    "chapter",
-    "dataset",
-    "entry",
-    "entry-dictionary",
-    "entry-encyclopedia",
-    "figure",
-    "graphic",
-    "interview",
-    "legal_case",
-    "legislation",
-    "manuscript",
-    "map",
-    "motion_picture",
-    "musical_score",
-    "pamphlet",
-    "paper-conference",
-    "patent",
-    "personal_communication",
-    "post",
-    "post-weblog",
-    "report",
-    "review",
-    "review-book",
-    "song",
-    "speech",
-    "thesis",
-    "treaty",
-    "webpage",
-}
+import copy
+import functools
+import logging
+
+import jsonref
+import jsonschema
 
 citeproc_type_fixer = {
     'journal-article': 'article-journal',
@@ -47,51 +14,133 @@
     'reference-entry': 'entry',
 }
 
-# Remove citeproc keys to fix pandoc-citeproc errors
-citeproc_remove_keys = [
-    # Error in $[0].ISSN[0]: failed to parse field ISSN: mempty
-    'ISSN',
-    # Error in $[2].ISBN[0]: failed to parse field ISBN: mempty
-    'ISBN',
-    # pandoc-citeproc expected Object not array for archive
-    'archive',
-    # failed to parse field event: Could not read as string
-    'event',
-    # remove the references of cited papers. Not neccessary and unwieldy.
-    'reference',
-    # Error in $[26].categories[0][0]: failed to parse field categories: mempty
-    'categories',
-]
-
-
-def citeproc_passthrough(csl_item, set_id=None):
+
+def citeproc_passthrough(csl_item, set_id=None, prune=True):
     """
-    Fix errors in a CSL item and optionally change its id.
+    Fix errors in a CSL item, according to the CSL JSON schema, and optionally
+    change its id.
+
+    http://docs.citationstyles.org/en/1.0.1/specification.html
     http://citeproc-js.readthedocs.io/en/latest/csl-json/markup.html
     https://github.com/citation-style-language/schema/blob/master/csl-data.json
     """
     if set_id is not None:
         csl_item['id'] = set_id
+    logging.debug(f"Starting citeproc_passthrough with{'' if prune else 'out'} CSL pruning for id: {csl_item.get('id', 'id not specified')}")
 
     # Correct invalid CSL item types
     # See https://github.com/CrossRef/rest-api-doc/issues/187
-    old_type = csl_item['type']
-    csl_type = citeproc_type_fixer.get(old_type, old_type)
-    if csl_type not in citeproc_types:
-        csl_type = 'entry'
-    csl_item['type'] = csl_type
-
-    # Remove problematic objects
-    for key in citeproc_remove_keys:
-        csl_item.pop(key, None)
-
-    # pandoc-citeproc error
-    # failed to parse field issued: Could not read as string: Null
+    csl_item['type'] = citeproc_type_fixer.get(csl_item['type'], csl_item['type'])
+
+    if prune:
+        # Remove fields that violate the CSL Item JSON Schema
+        csl_item, = remove_jsonschema_errors([csl_item])
+
+    # Default CSL type to entry
+    csl_item['type'] = csl_item.get('type', 'entry')
+
+    if prune:
+        # Confirm that corrected CSL validates
+        validator = get_jsonschema_csl_validator()
+        validator.validate([csl_item])
+    return csl_item
+
+
+@functools.lru_cache()
+def get_jsonschema_csl_validator():
+    """
+    Return a jsonschema validator for the CSL Item JSON Schema
+    """
+    url = 'https://github.com/dhimmel/schema/raw/manubot/csl-data.json'
+    # Use jsonref to workaround https://github.com/Julian/jsonschema/issues/447
+    schema = jsonref.load_uri(url, jsonschema=True)
+    Validator = jsonschema.validators.validator_for(schema)
+    Validator.check_schema(schema)
+    return Validator(schema)
+
+
+def remove_jsonschema_errors(instance):
+    """
+    Remove fields in CSL Items that produce JSON Schema errors. Note that this
+    method may not be work for all types of JSON Schema errors and users
+    looking to adapt it for other applications should write task-specific tests
+    to provide empirical evaluate that it works as intended.
+
+    See also:
+    https://github.com/Julian/jsonschema/issues/448
+    https://stackoverflow.com/questions/44694835
+    """
+    validator = get_jsonschema_csl_validator()
+    errors = list(validator.iter_errors(instance))
+    instance = copy.deepcopy(instance)
+    errors = sorted(errors, key=lambda e: e.path, reverse=True)
+    for error in errors:
+        _remove_error(instance, error)
+    return instance
+
+
+def _delete_elem(instance, path, absolute_path=None, message=''):
+    """
+    Helper function for remove_jsonschema_errors that deletes an element in the
+    JSON-like input instance at the specified path. absolute_path is relative
+    to the original validated instance for logging purposes. Defaults to path,
+    if not specified. message is an optional string with additional error
+    information to log.
+    """
+    if absolute_path is None:
+        absolute_path = path
+    logging.debug(
+        (f'{message}\n' if message else message) +
+        '_delete_elem deleting CSL element at: ' +
+        '/'.join(map(str, absolute_path))
+    )
+    *head, tail = path
     try:
-        value = csl_item['issued']['date-parts'][0][0]
-        if value is None:
-            del csl_item['issued']
+        del _deep_get(instance, head)[tail]
     except KeyError:
         pass
 
-    return csl_item
+
+def _deep_get(instance, path):
+    """
+    Descend path to return a deep element in the JSON object instance.
+    """
+    for key in path:
+        instance = instance[key]
+    return instance
+
+
+def _remove_error(instance, error):
+    """
+    Remove a jsonschema ValidationError from the JSON-like instance.
+
+    See ValidationError documentation at
+    http://python-jsonschema.readthedocs.io/en/latest/errors/#jsonschema.exceptions.ValidationError
+    """
+    sub_errors = error.context
+    if sub_errors:
+        # already_removed_additional was neccessary to workaround https://github.com/citation-style-language/schema/issues/154
+        already_removed_additional = False
+        for sub_error in sub_errors:
+            if sub_error.validator == 'additionalProperties':
+                if already_removed_additional:
+                    continue
+                already_removed_additional = True
+            sub_instance = _deep_get(instance, error.path)
+            _remove_error(sub_instance, sub_error)
+    elif error.validator == 'additionalProperties':
+        extras = set(error.instance) - set(error.schema['properties'])
+        logging.debug(
+            error.message +
+            f'\nWill now remove these {len(extras)} additional properties.'
+        )
+        for key in extras:
+            _delete_elem(
+                instance=instance,
+                path=list(error.path) + [key],
+                absolute_path=list(error.absolute_path) + [key]
+            )
+    elif error.validator in {'enum', 'type'}:
+        _delete_elem(instance, error.path, error.absolute_path, error.message)
+    else:
+        raise NotImplementedError(f'{error.validator} is not yet supported')

manubot/cite/doi.py

@@ -19,7 +19,7 @@ def get_short_doi_url(doi):
         short_url = 'https://doi.org/' + short_doi
         return short_url
     except Exception:
-        logging.exception(f'shortDOI lookup failed for {doi}')
+        logging.warning(f'shortDOI lookup failed for {doi}', exc_info=True)
         return None
 
 
@@ -42,7 +42,7 @@ def get_doi_citeproc(doi):
     citeproc['URL'] = f'https://doi.org/{doi}'
     short_doi_url = get_short_doi_url(doi)
     if short_doi_url:
-        citeproc['short_url'] = short_doi_url
+        citeproc['URL'] = short_doi_url
     try:
         citeproc.update(get_pubmed_ids_for_doi(doi))
     except Exception:

setup.py

@@ -50,6 +50,8 @@
     install_requires=[
         'errorhandler',
         'jinja2',
+        'jsonref',
+        'jsonschema',
         'pandas',
         'pybase62',
         'pyyaml',