Use schema_salad functions file_uri and file_uri_path (#269)

* Use schema_salad functions file_uri and file_uri_path for more correct file URI
handling.
* Remove dependency on pathlib2.

Author: tetron

cwltool/draft2tool.py

@@ -13,6 +13,7 @@
 
 import avro.schema
 import schema_salad.validate as validate
+from schema_salad.ref_resolver import file_uri, uri_file_path
 import shellescape
 from typing import Any, Callable, cast, Generator, Text, Union
 
@@ -94,11 +95,11 @@ def revmap_file(builder, outdir, f):
 
     split = urlparse.urlsplit(outdir)
     if not split.scheme:
-        outdir = "file://" + outdir
+        outdir = file_uri(str(outdir))
 
     if "location" in f:
         if f["location"].startswith("file://"):
-            path = f["location"][7:]
+            path = uri_file_path(f["location"])
             revmap_f = builder.pathmapper.reversemap(path)
             if revmap_f:
                 f["location"] = revmap_f[1]

cwltool/load_tool.py

@@ -6,15 +6,13 @@
 import logging
 import re
 import urlparse
-import pathlib2
 
 from typing import Any, AnyStr, Callable, cast, Dict, Text, Tuple, Union
 from ruamel.yaml.comments import CommentedSeq, CommentedMap
 from avro.schema import Names
 import requests.sessions
 
-
-from schema_salad.ref_resolver import Loader, Fetcher
+from schema_salad.ref_resolver import Loader, Fetcher, file_uri
 import schema_salad.validate as validate
 from schema_salad.validate import ValidationException
 import schema_salad.schema as schema
@@ -44,7 +42,7 @@ def fetch_document(argsworkflow,   # type: Union[Text, dict[Text, Any]]
         if split.scheme:
             uri = argsworkflow
         elif os.path.exists(os.path.abspath(argsworkflow)):
-            uri = pathlib2.Path(os.path.abspath(argsworkflow)).as_uri()
+            uri = file_uri(str(os.path.abspath(argsworkflow)))
         elif resolver:
             uri = resolver(document_loader, argsworkflow)
 

cwltool/main.py

@@ -18,7 +18,7 @@
 from typing import (Union, Any, AnyStr, cast, Callable, Dict, Sequence, Text,
     Tuple, Type, IO)
 
-from schema_salad.ref_resolver import Loader, Fetcher
+from schema_salad.ref_resolver import Loader, Fetcher, file_uri, uri_file_path
 import schema_salad.validate as validate
 import schema_salad.jsonld_context
 import schema_salad.makedoc
@@ -258,7 +258,7 @@ def __call__(self, parser, namespace, values, option_string=None):
         setattr(namespace,
             self.dest,  # type: ignore
             {"class": self.objclass,
-             "location": "file://%s" % os.path.abspath(cast(AnyStr, values))})
+             "location": file_uri(str(os.path.abspath(cast(AnyStr, values))))})
 
 class FSAppendAction(argparse.Action):
     objclass = None  # type: Text
@@ -281,7 +281,7 @@ def __call__(self, parser, namespace, values, option_string=None):
                     g)
         g.append(
             {"class": self.objclass,
-             "location": "file://%s" % os.path.abspath(cast(AnyStr, values))})
+             "location": file_uri(str(os.path.abspath(cast(AnyStr, values))))})
 
 class FileAction(FSAction):
     objclass = "File"
@@ -471,7 +471,7 @@ def load_job_order(args, t, stdin, print_input_deps=False, relative_deps=False,
 
     if print_input_deps:
         printdeps(job_order_object, loader, stdout, relative_deps, "",
-                  basedir=u"file://%s/" % input_basedir)
+                  basedir=file_uri(input_basedir+"/"))
         return 0
 
     def pathToLoc(p):
@@ -498,7 +498,7 @@ def makeRelative(base, ob):
         pass
     else:
         if u.startswith("file://"):
-            u = u[7:]
+            u = uri_file_path(u)
         ob["location"] = os.path.relpath(u, base)
 
 def printdeps(obj, document_loader, stdout, relative_deps, uri, basedir=None):
@@ -519,7 +519,7 @@ def loadref(b, u):
         if relative_deps == "primary":
             base = basedir if basedir else os.path.dirname(uri)
         elif relative_deps == "cwd":
-            base = "file://" + os.getcwd()
+            base = file_uri(os.getcwd())
         else:
             raise Exception(u"Unknown relative_deps %s" % relative_deps)
 
@@ -720,7 +720,7 @@ def main(argsl=None,  # type: List[str]
             if out is not None:
                 def locToPath(p):
                     if p["location"].startswith("file://"):
-                        p["path"] = p["location"][7:]
+                        p["path"] = uri_file_path(p["location"])
 
                 adjustDirObjs(out, locToPath)
                 adjustFileObjs(out, locToPath)

cwltool/pathmapper.py

@@ -9,6 +9,7 @@
 from typing import Any, Callable, Set, Text, Tuple, Union
 import schema_salad.validate as validate
 from schema_salad.sourceline import SourceLine
+from schema_salad.ref_resolver import uri_file_path
 
 _logger = logging.getLogger("cwltool")
 
@@ -65,15 +66,15 @@ def addLocation(d):
 
         if "basename" not in d:
             parse = urlparse.urlparse(d["location"])
-            d["basename"] = os.path.basename(parse.path)
+            d["basename"] = os.path.basename(urllib.url2pathname(parse.path))
 
     adjustFileObjs(job, addLocation)
     adjustDirObjs(job, addLocation)
 
 
 def abspath(src, basedir):  # type: (Text, Text) -> Text
     if src.startswith(u"file://"):
-        ab = urllib.url2pathname(urlparse.urlparse(src).path)
+        ab = unicode(uri_file_path(str(src)))
     else:
         ab = src if os.path.isabs(src) else os.path.join(basedir, src)
     return ab

cwltool/process.py

@@ -16,7 +16,7 @@
 import abc
 import schema_salad.validate as validate
 import schema_salad.schema
-from schema_salad.ref_resolver import Loader
+from schema_salad.ref_resolver import Loader, file_uri
 from schema_salad.sourceline import SourceLine
 import avro.schema
 from typing import (Any, AnyStr, Callable, cast, Dict, List, Generator, IO, Text,
@@ -220,7 +220,7 @@ def moveIt(src, dst):
     stageFiles(pm, moveIt)
 
     def _check_adjust(f):
-        f["location"] = "file://" + pm.mapper(f["location"])[1]
+        f["location"] = file_uri(pm.mapper(f["location"])[1])
         if "contents" in f:
             del f["contents"]
         if f["class"] == "File":

cwltool/resolver.py

@@ -3,6 +3,8 @@
 import urllib
 import urlparse
 
+from schema_salad.ref_resolver import file_uri
+
 _logger = logging.getLogger("cwltool")
 
 def resolve_local(document_loader, uri):
@@ -17,14 +19,14 @@ def resolve_local(document_loader, uri):
 
     for s in shares:
         if os.path.exists(s):
-            return ("file://%s" % s)
+            return file_uri(s)
         if os.path.exists("%s.cwl" % s):
-            return ("file://%s.cwl" % s)
+            return file_uri(s)
     return None
 
 def tool_resolver(document_loader, uri):
     for r in [resolve_local]:
         ret = r(document_loader, uri)
         if ret is not None:
             return ret
-    return "file://" + os.path.abspath(uri)
+    return file_uri(os.path.abspath(uri))

cwltool/stdfsaccess.py

@@ -2,6 +2,7 @@
 from .pathmapper import abspath
 import glob
 import os
+from schema_salad.ref_resolver import file_uri
 
 class StdFsAccess(object):
 
@@ -12,7 +13,7 @@ def _abs(self, p):  # type: (Text) -> Text
         return abspath(p, self.basedir)
 
     def glob(self, pattern):  # type: (Text) -> List[Text]
-        return ["file://%s" % self._abs(l) for l in glob.glob(self._abs(pattern))]
+        return [file_uri(str(self._abs(l))) for l in glob.glob(self._abs(pattern))]
 
     def open(self, fn, mode):  # type: (Text, Text) -> BinaryIO
         return open(self._abs(fn), mode)

setup.py

@@ -47,10 +47,9 @@
           'ruamel.yaml >= 0.12.4',
           'rdflib >= 4.2.0, < 4.3.0',
           'shellescape >= 3.4.1, < 3.5',
-          'schema-salad >= 2.1.20161227191302, < 3',
+          'schema-salad >= 2.2.20170111180227, < 3',
           'typing >= 3.5.2, < 3.6',
-          'cwltest >= 1.0.20160907111242',
-          'pathlib2 >= 2.1.0'
+          'cwltest >= 1.0.20161227194859'
       ],
       test_suite='tests',
       tests_require=[],