Merge branch 'master' into adjust-manifest

Author: mr-c

requirements.txt

@@ -1,7 +1,8 @@
 typing==3.5.2.2 ; python_version>="2.7"
 avro-python3 ; python_version>="3"
 avro==1.8.1 ; python_version<"3"
-ruamel.yaml==0.12.4
+pathlib2==2.1.0
+ruamel.yaml==0.13.7
 rdflib==4.2.1
 rdflib-jsonld==0.4.0
 html5lib==0.9999999

schema_salad/ref_resolver.py

@@ -4,11 +4,13 @@
 import hashlib
 import logging
 import collections
+import urllib
 import urlparse
 import re
 import copy
 import pprint
 from StringIO import StringIO
+import  pathlib2
 
 from . import validate
 from .aslist import aslist
@@ -108,7 +110,7 @@ def fetch_text(self, url):
             return resp.text
         elif scheme == 'file':
             try:
-                with open(path) as fp:
+                with open(urllib.url2pathname(str(urlparse.urlparse(url).path))) as fp:
                     read = fp.read()
                 if hasattr(read, "decode"):
                     return read.decode("utf-8")
@@ -137,7 +139,7 @@ def check_exists(self, url):  # type: (unicode) -> bool
                 return False
             return True
         elif scheme == 'file':
-            return os.path.exists(path)
+            return os.path.exists(urllib.url2pathname(str(urlparse.urlparse(url).path)))
         else:
             raise ValueError('Unsupported scheme in url: %s' % url)
 
@@ -371,13 +373,19 @@ def resolve_ref(self,
                     checklinks=True  # type: bool
                     ):
         # type: (...) -> Tuple[Union[CommentedMap, CommentedSeq, unicode], Dict[unicode, Any]]
-        base_url = base_url or u'file://%s/' % os.path.abspath('.')
 
         obj = None              # type: CommentedMap
         resolved_obj = None     # type: Union[CommentedMap, CommentedSeq, unicode]
         inc = False
         mixin = None            # type: Dict[unicode, Any]
 
+        if not base_url:
+            base_url = pathlib2.Path(os.getcwd()).as_uri() + '/'
+
+        if isinstance(ref, (str, unicode)) and os.sep == "\\":
+            # Convert Windows paths
+            ref = ref.replace("\\", "/")
+
         sl = SourceLine(obj, None, ValueError)
         # If `ref` is a dict, look for special directives.
         if isinstance(ref, CommentedMap):
@@ -418,7 +426,6 @@ def resolve_ref(self,
             raise ValueError(u"Expected CommentedMap or string, got %s: `%s`" % (type(ref), unicode(ref)))
 
         url = self.expand_url(ref, base_url, scoped_id=(obj is not None))
-
         # Has this reference been loaded already?
         if url in self.idx and (not mixin):
             return self.idx[url], {}

schema_salad/sourceline.py

@@ -103,7 +103,8 @@ def cmap(d, lc=None, fn=None):  # type: (Union[int, float, str, unicode, Dict, L
         return d
     if isinstance(d, dict):
         cm = CommentedMap()
-        for k,v in d.iteritems():
+        for k in sorted(d.keys()):
+            v = d[k]
             if isinstance(v, CommentedBase):
                 uselc = [v.lc.line, v.lc.col, v.lc.line, v.lc.col]
                 vfn = v.lc.filename if hasattr(v.lc, "filename") else fn

schema_salad/tests/frag.yml

@@ -0,0 +1,4 @@
+- id: foo1
+  bar: b1
+- id: foo2
+  bar: b2

schema_salad/tests/test_examples.py

@@ -243,7 +243,7 @@ def test_examples(self):
             self.assertEqual(proc, src)
 
     def test_yaml_float_test(self):
-        self.assertEqual(ruamel.yaml.load("float-test: 2e-10")["float-test"],
+        self.assertEqual(ruamel.yaml.safe_load("float-test: 2e-10")["float-test"],
                 2e-10)
 
     def test_typedsl_ref(self):
@@ -341,13 +341,13 @@ def test_scoped_id(self):
         print(g.serialize(format="n3"))
 
     def test_mixin(self):
+        base_url = "file://" + os.getcwd() + "/tests/"
         ldr = schema_salad.ref_resolver.Loader({})
         ra = ldr.resolve_ref(cmap({"$mixin": get_data("tests/mixin.yml"), "one": "five"}),
-                             base_url="file://" + os.getcwd() + "/tests/")
+                             base_url=base_url)
         self.assertEqual({'id': 'four', 'one': 'five'}, ra[0])
-
         ldr = schema_salad.ref_resolver.Loader({"id": "@id"})
-        base_url = "file://" + os.getcwd() + "/tests/"
+
         ra = ldr.resolve_all(cmap([{
             "id": "a",
             "m": {"$mixin": get_data("tests/mixin.yml")}
@@ -368,6 +368,11 @@ def test_mixin(self):
                 'one': 'two'}
         }], ra[0])
 
+    def test_fragment(self):
+        ldr = schema_salad.ref_resolver.Loader({"id": "@id"})
+        b, _ = ldr.resolve_ref("schema_salad/tests/frag.yml#foo2")
+        self.assertEquals({"id": b["id"], "bar":"b2"}, b)
+
 
 if __name__ == '__main__':
     unittest.main()

setup.py

@@ -30,14 +30,14 @@
 install_requires = [
     'setuptools',
     'requests >= 1.0',
-    'ruamel.yaml >= 0.12.4, < 0.12.5',
+    'ruamel.yaml >= 0.12.4',
     'rdflib >= 4.2.0, < 4.3.0',
     'rdflib-jsonld >= 0.3.0, < 0.5.0',
-    'html5lib >= 0.90, <= 0.9999999',
     'mistune >= 0.7.3, < 0.8',
     'typing >= 3.5.2, < 3.6',
     'CacheControl >= 0.11.7, < 0.12',
-    'lockfile >= 0.9']
+    'lockfile >= 0.9',
+    'pathlib2 >= 2.1.0']
 
 install_requires.append("avro")  # TODO: remove me once cwltool is
 # available in Debian Stable, Ubuntu 12.04 LTS

tox.ini

@@ -1,6 +1,6 @@
 [tox]
 #envlist = py35-lint,py34-lint,py33-lint,py27-lint,py35-unit,py34-unit,py33-unit,py27-unit
-envlist = py27-lint, py27-unit, py35-mypy
+envlist = py27-lint, py27-unit, py35-mypy, py27-pipconflictchecker
 skipsdist = True
 
 [tox:travis]
@@ -38,6 +38,12 @@ commands = flake8 schema_salad setup.py
 whitelist_externals = flake8
 deps = flake8
 
+[testenv:py27-pipconflictchecker]
+commands = pipconflictchecker
+whitelist_externals = pipconflictchecker
+deps = pip-conflict-checker
+
+
 [testenv:py35-unit]
 commands = python setup.py test
 

typeshed/2.7/pathlib2.pyi

@@ -0,0 +1,188 @@
+# Stubs for pathlib2 (Python 2)
+#
+# NOTE: This dynamically typed stub was automatically generated by stubgen.
+
+from typing import Any, AnyStr, Type, TypeVar, Optional, Union
+from collections import Sequence
+
+_P = TypeVar('_P', bound='PurePath')
+
+intern = ...  # type: Any
+basestring = ...  # type: Any
+supports_symlinks = ...  # type: bool
+nt = ...  # type: Any
+
+class _Flavour:
+    join = ...  # type: Any
+    def __init__(self) -> None: ...
+    def parse_parts(self, parts): ...
+    def join_parsed_parts(self, drv, root, parts, drv2, root2, parts2): ...
+
+class _WindowsFlavour(_Flavour):
+    sep = ...  # type: str
+    altsep = ...  # type: str
+    has_drv = ...  # type: bool
+    pathmod = ...  # type: Any
+    is_supported = ...  # type: Any
+    drive_letters = ...  # type: Any
+    ext_namespace_prefix = ...  # type: str
+    reserved_names = ...  # type: Any
+    def splitroot(self, part, sep: Any = ...): ...
+    def casefold(self, s): ...
+    def casefold_parts(self, parts): ...
+    def resolve(self, path): ...
+    def is_reserved(self, parts): ...
+    def make_uri(self, path): ...
+    def gethomedir(self, username): ...
+
+class _PosixFlavour(_Flavour):
+    sep = ...  # type: str
+    altsep = ...  # type: str
+    has_drv = ...  # type: bool
+    pathmod = ...  # type: Any
+    is_supported = ...  # type: Any
+    def splitroot(self, part, sep: Any = ...): ...
+    def casefold(self, s): ...
+    def casefold_parts(self, parts): ...
+    def resolve(self, path): ...
+    def is_reserved(self, parts): ...
+    def make_uri(self, path): ...
+    def gethomedir(self, username): ...
+
+class _Accessor: ...
+
+class _NormalAccessor(_Accessor):
+    stat = ...  # type: Any
+    lstat = ...  # type: Any
+    open = ...  # type: Any
+    listdir = ...  # type: Any
+    chmod = ...  # type: Any
+    lchmod = ...  # type: Any
+    #def lchmod(self, pathobj, mode): ...
+    mkdir = ...  # type: Any
+    unlink = ...  # type: Any
+    rmdir = ...  # type: Any
+    rename = ...  # type: Any
+    replace = ...  # type: Any
+    symlink = ...  # type: Any
+    #def symlink(a, b, target_is_directory): ...
+    #@staticmethod
+    #def symlink(a, b, target_is_directory): ...
+    utime = ...  # type: Any
+    def readlink(self, path): ...
+
+class _Selector:
+    child_parts = ...  # type: Any
+    successor = ...  # type: Any
+    def __init__(self, child_parts) -> None: ...
+    def select_from(self, parent_path): ...
+
+class _TerminatingSelector: ...
+
+class _PreciseSelector(_Selector):
+    name = ...  # type: Any
+    def __init__(self, name, child_parts) -> None: ...
+
+class _WildcardSelector(_Selector):
+    pat = ...  # type: Any
+    def __init__(self, pat, child_parts) -> None: ...
+
+class _RecursiveWildcardSelector(_Selector):
+    def __init__(self, pat, child_parts) -> None: ...
+
+class _PathParents(Sequence):
+    def __init__(self, path) -> None: ...
+    def __len__(self): ...
+    def __getitem__(self, idx): ...
+
+class PurePath:
+    def __new__(cls, *args): ...
+    def __reduce__(self): ...
+    def as_posix(self): ...
+    def __bytes__(self): ...
+    def as_uri(self) -> str: ...
+    def __eq__(self, other): ...
+    def __ne__(self, other): ...
+    def __hash__(self): ...
+    def __lt__(self, other): ...
+    def __le__(self, other): ...
+    def __gt__(self, other): ...
+    def __ge__(self, other): ...
+    drive = ...  # type: Any
+    root = ...  # type: Any
+    @property
+    def anchor(self): ...
+    @property
+    def name(self): ...
+    @property
+    def suffix(self): ...
+    @property
+    def suffixes(self): ...
+    @property
+    def stem(self): ...
+    def with_name(self, name): ...
+    def with_suffix(self, suffix): ...
+    def relative_to(self, *other): ...
+    @property
+    def parts(self): ...
+    def joinpath(self, *args): ...
+    def __truediv__(self, key): ...
+    def __rtruediv__(self, key): ...
+    __div__ = ...  # type: Any
+    __rdiv__ = ...  # type: Any
+    @property
+    def parent(self): ...
+    @property
+    def parents(self): ...
+    def is_absolute(self): ...
+    def is_reserved(self): ...
+    def match(self, path_pattern): ...
+
+class PurePosixPath(PurePath): ...
+class PureWindowsPath(PurePath): ...
+
+class Path(PurePath):
+    def __new__(cls: Type[_P], *args: Union[AnyStr, PurePath],
+	**kwargs: Any) -> _P: ...
+    def __enter__(self): ...
+    def __exit__(self, t, v, tb): ...
+    @classmethod
+    def cwd(cls): ...
+    @classmethod
+    def home(cls): ...
+    def samefile(self, other_path): ...
+    def iterdir(self): ...
+    def glob(self, pattern): ...
+    def rglob(self, pattern): ...
+    def absolute(self): ...
+    def resolve(self): ...
+    def stat(self): ...
+    def owner(self): ...
+    def group(self): ...
+    def open(self, mode: str = ..., buffering: int = ..., encoding: Optional[Any] = ..., errors: Optional[Any] = ..., newline: Optional[Any] = ...): ...
+    def read_bytes(self): ...
+    def read_text(self, encoding: Optional[Any] = ..., errors: Optional[Any] = ...): ...
+    def write_bytes(self, data): ...
+    def write_text(self, data, encoding: Optional[Any] = ..., errors: Optional[Any] = ...): ...
+    def touch(self, mode: int = ..., exist_ok: bool = ...): ...
+    def mkdir(self, mode: int = ..., parents: bool = ..., exist_ok: bool = ...): ...
+    def chmod(self, mode): ...
+    def lchmod(self, mode): ...
+    def unlink(self): ...
+    def rmdir(self): ...
+    def lstat(self): ...
+    def rename(self, target): ...
+    def replace(self, target): ...
+    def symlink_to(self, target, target_is_directory: bool = ...): ...
+    def exists(self): ...
+    def is_dir(self): ...
+    def is_file(self): ...
+    def is_symlink(self): ...
+    def is_block_device(self): ...
+    def is_char_device(self): ...
+    def is_fifo(self): ...
+    def is_socket(self): ...
+    def expanduser(self): ...
+
+class PosixPath(Path, PurePosixPath): ...
+class WindowsPath(Path, PureWindowsPath): ...