Provide line numbers in error messages (#62)

* Line number reporting WIP

* Handle line number tracking $import expansion of lists and idmaps.

* Break long lines.

* Formatting bullets, indentation.

* Add tests of validation error reporting.

* type cleanups

* Better error reporting on link checking.

* Add missing sourceline.py

* Fix tests, typing, add tests for validation errors.

* Bump version.

* Print error validation messages for manual audit.

Author: tetron

schema_salad/add_dictlist.py

@@ -1,6 +1,7 @@
 import sys
 from typing import Any, Dict
 
+
 def add_dictlist(di, key, val):  # type: (Dict, Any, Any) -> None
     if key not in di:
         di[key] = []

schema_salad/aslist.py

@@ -1,6 +1,7 @@
 import sys
 from typing import Any, List
 
+
 def aslist(l):  # type: (Any) -> List
     """Convenience function to wrap single items and lists, and return lists unchanged."""
 

schema_salad/flatten.py

@@ -2,6 +2,8 @@
 from typing import Any, Tuple
 
 # http://rightfootin.blogspot.com/2006/09/more-on-python-flatten.html
+
+
 def flatten(l, ltypes=(list, tuple)):
     # type: (Any, Any) -> Any
     if l is None:

schema_salad/jsonld_context.py

@@ -20,13 +20,19 @@
 import logging
 from .aslist import aslist
 from typing import Any, cast, Dict, Iterable, Tuple, Union
-from .ref_resolver import Loader
+from .ref_resolver import Loader, ContextType
 
 _logger = logging.getLogger("salad")
 
 
-def pred(datatype, field, name, context, defaultBase, namespaces):
-    # type: (Dict[str, Union[Dict, str]], Dict, str, Loader.ContextType, str, Dict[str, rdflib.namespace.Namespace]) -> Union[Dict, str]
+def pred(datatype,      # type: Dict[str, Union[Dict, str]]
+         field,         # type: Dict
+         name,          # type: str
+         context,       # type: ContextType
+         defaultBase,   # type: str
+         namespaces     # type: Dict[str, rdflib.namespace.Namespace]
+         ):
+    # type: (...) -> Union[Dict, str]
     split = urlparse.urlsplit(name)
 
     vee = None  # type: Union[str, unicode]
@@ -84,8 +90,14 @@ def pred(datatype, field, name, context, defaultBase, namespaces):
     return ret
 
 
-def process_type(t, g, context, defaultBase, namespaces, defaultPrefix):
-    # type: (Dict[str, Any], Graph, Loader.ContextType, str, Dict[str, rdflib.namespace.Namespace], str) -> None
+def process_type(t,             # type: Dict[str, Any]
+                 g,             # type: Graph
+                 context,       # type: ContextType
+                 defaultBase,   # type: str
+                 namespaces,    # type: Dict[str, rdflib.namespace.Namespace]
+                 defaultPrefix  # type: str
+                 ):
+    # type: (...) -> None
     if t["type"] == "record":
         recordname = t["name"]
 
@@ -154,8 +166,8 @@ def process_type(t, g, context, defaultBase, namespaces, defaultPrefix):
 
 
 def salad_to_jsonld_context(j, schema_ctx):
-    # type: (Iterable, Dict[str, Any]) -> Tuple[Loader.ContextType, Graph]
-    context = {}  # type: Loader.ContextType
+    # type: (Iterable, Dict[str, Any]) -> Tuple[ContextType, Graph]
+    context = {}  # type: ContextType
     namespaces = {}
     g = Graph()
     defaultPrefix = ""
@@ -178,8 +190,11 @@ def salad_to_jsonld_context(j, schema_ctx):
 
     return (context, g)
 
-def fix_jsonld_ids(obj, ids):
-    # type: (Union[Dict[unicode, Any], List[Dict[unicode, Any]]], List[unicode]) -> None
+
+def fix_jsonld_ids(obj,     # type: Union[Dict[unicode, Any], List[Dict[unicode, Any]]]
+                   ids      # type: List[unicode]
+                   ):
+    # type: (...) -> None
     if isinstance(obj, dict):
         for i in ids:
             if i in obj:
@@ -190,8 +205,13 @@ def fix_jsonld_ids(obj, ids):
         for entry in obj:
             fix_jsonld_ids(entry, ids)
 
-def makerdf(workflow, wf, ctx, graph=None):
-    # type: (Union[str, unicode], Union[List[Dict[unicode, Any]], Dict[unicode, Any]], Loader.ContextType, Graph) -> Graph
+
+def makerdf(workflow,       # type: Union[str, unicode]
+            wf,             # type: Union[List[Dict[unicode, Any]], Dict[unicode, Any]]
+            ctx,            # type: ContextType
+            graph=None      # type: Graph
+            ):
+    # type: (...) -> Graph
     prefixes = {}
     idfields = []
     for k, v in ctx.iteritems():

schema_salad/main.py

@@ -3,28 +3,36 @@
 import logging
 import sys
 import traceback
-import pkg_resources  # part of setuptools
-from . import schema
-from . import jsonld_context
-from . import makedoc
 import json
-from rdflib import Graph, plugin
-from rdflib.serializer import Serializer
 import os
 import urlparse
 
-from .ref_resolver import Loader
-from . import validate
+import pkg_resources  # part of setuptools
+
 from typing import Any, Dict, List, Union
 
+from rdflib import Graph, plugin
+from rdflib.serializer import Serializer
+
+from . import schema
+from . import jsonld_context
+from . import makedoc
+from . import validate
+from .sourceline import strip_dup_lineno
+from .ref_resolver import Loader
+
 _logger = logging.getLogger("salad")
 
 from rdflib.plugin import register, Parser
 register('json-ld', Parser, 'rdflib_jsonld.parser', 'JsonLDParser')
 
 
-def printrdf(workflow, wf, ctx, sr):
-    # type: (str, Union[List[Dict[unicode, Any]], Dict[unicode, Any]], Dict[unicode, Any], str) -> None
+def printrdf(workflow,  # type: str
+             wf,        # type: Union[List[Dict[unicode, Any]], Dict[unicode, Any]]
+             ctx,       # type: Dict[unicode, Any]
+             sr         # type: str
+             ):
+    # type: (...) -> None
     g = jsonld_context.makerdf(workflow, wf, ctx)
     print(g.serialize(format=sr))
 
@@ -104,10 +112,14 @@ def main(argsl=None):  # type: (List[str]) -> int
             schema_raw_doc, schema_uri)
     except (validate.ValidationException) as e:
         _logger.error("Schema `%s` failed link checking:\n%s",
-                args.schema, e, exc_info=(True if args.debug else False))
+                      args.schema, e, exc_info=(True if args.debug else False))
         _logger.debug("Index is %s", metaschema_loader.idx.keys())
         _logger.debug("Vocabulary is %s", metaschema_loader.vocab.keys())
         return 1
+    except (RuntimeError) as e:
+        _logger.error("Schema `%s` read error:\n%s",
+                      args.schema, e, exc_info=(True if args.debug else False))
+        return 1
 
     # Optionally print the schema after ref resolution
     if not args.document and args.print_pre:
@@ -121,7 +133,8 @@ def main(argsl=None):  # type: (List[str]) -> int
     # Validate the schema document against the metaschema
     try:
         schema.validate_doc(metaschema_names, schema_doc,
-                            metaschema_loader, args.strict)
+                            metaschema_loader, args.strict,
+                            source=schema_metadata["name"])
     except validate.ValidationException as e:
         _logger.error("While validating schema `%s`:\n%s" %
                       (args.schema, str(e)))
@@ -149,8 +162,8 @@ def main(argsl=None):  # type: (List[str]) -> int
 
     if isinstance(avsc_names, Exception):
         _logger.error("Schema `%s` error:\n%s", args.schema,
-                avsc_names, exc_info=((type(avsc_names), avsc_names,
-                    None) if args.debug else None))
+                      avsc_names, exc_info=((type(avsc_names), avsc_names,
+                                             None) if args.debug else None))
         if args.print_avro:
             print(json.dumps(avsc_obj, indent=4))
         return 1
@@ -188,7 +201,7 @@ def main(argsl=None):  # type: (List[str]) -> int
         document, doc_metadata = document_loader.resolve_ref(uri)
     except (validate.ValidationException, RuntimeError) as e:
         _logger.error("Document `%s` failed validation:\n%s",
-                      args.document, e, exc_info=args.debug)
+                      args.document, strip_dup_lineno(unicode(e)), exc_info=args.debug)
         return 1
 
     # Optionally print the document after ref resolution

schema_salad/makedoc.py

@@ -110,6 +110,7 @@ def contents(self, idn):  # type: (str) -> str
         c += """</nav>"""
         return c
 
+
 basicTypes = ("https://w3id.org/cwl/salad#null",
               "http://www.w3.org/2001/XMLSchema#boolean",
               "http://www.w3.org/2001/XMLSchema#int",
@@ -219,8 +220,13 @@ def __init__(self, toc, j, renderlist, redirects):
                  ("docAfter" not in f))):
                 self.render_type(f, 1)
 
-    def typefmt(self, tp, redirects, nbsp=False, jsonldPredicate=None):
-        # type: (Any, Dict[str, str], bool, Dict[str, str]) -> Union[str, unicode]
+    def typefmt(self,
+                tp,                     # type: Any
+                redirects,              # type: Dict[str, str]
+                nbsp=False,             # type: bool
+                jsonldPredicate=None    # type: Dict[str, str]
+                ):
+        # type: (...) -> Union[str, unicode]
         global primitiveType
         if isinstance(tp, list):
             if nbsp and len(tp) <= 3:
@@ -229,16 +235,20 @@ def typefmt(self, tp, redirects, nbsp=False, jsonldPredicate=None):
                 return " | ".join([self.typefmt(n, redirects) for n in tp])
         if isinstance(tp, dict):
             if tp["type"] == "https://w3id.org/cwl/salad#array":
-                ar = "array&lt;%s&gt;" % (self.typefmt(tp["items"], redirects, nbsp=True))
+                ar = "array&lt;%s&gt;" % (self.typefmt(
+                    tp["items"], redirects, nbsp=True))
                 if jsonldPredicate and "mapSubject" in jsonldPredicate:
                     if "mapPredicate" in jsonldPredicate:
                         ar += " | map&lt;%s.%s,&nbsp;%s.%s&gt" % (self.typefmt(tp["items"], redirects),
-                                                           jsonldPredicate["mapSubject"],
-                                                           self.typefmt(tp["items"], redirects),
-                                                           jsonldPredicate["mapPredicate"])
+                                                                  jsonldPredicate[
+                                                                      "mapSubject"],
+                                                                  self.typefmt(
+                                                                      tp["items"], redirects),
+                                                                  jsonldPredicate["mapPredicate"])
                     ar += " | map&lt;%s.%s,&nbsp;%s&gt" % (self.typefmt(tp["items"], redirects),
-                                                          jsonldPredicate["mapSubject"],
-                                                          self.typefmt(tp["items"], redirects))
+                                                           jsonldPredicate[
+                                                               "mapSubject"],
+                                                           self.typefmt(tp["items"], redirects))
                 return ar
             if tp["type"] in ("https://w3id.org/cwl/salad#record", "https://w3id.org/cwl/salad#enum"):
                 frg = schema.avro_name(tp["name"])
@@ -481,6 +491,7 @@ def avrold_doc(j, outdoc, renderlist, redirects, brand, brandlink):
     </body>
     </html>""")
 
+
 if __name__ == "__main__":
 
     parser = argparse.ArgumentParser()