http: allow put_file()ing file-like objects

Author: isidentical

fsspec/implementations/http.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import, division, print_function
 
 import asyncio
+import io
 import logging
 import re
 import weakref
@@ -14,7 +15,7 @@
 from fsspec.callbacks import _DEFAULT_CALLBACK
 from fsspec.exceptions import FSTimeoutError
 from fsspec.spec import AbstractBufferedFile
-from fsspec.utils import DEFAULT_BLOCK_SIZE, tokenize
+from fsspec.utils import DEFAULT_BLOCK_SIZE, nullcontext, tokenize
 
 from ..caching import AllBytes
 
@@ -262,9 +263,19 @@ async def _put_file(
         **kwargs,
     ):
         async def gen_chunks():
-            with open(rpath, "rb") as f:
-                callback.set_size(f.seek(0, 2))
-                f.seek(0)
+            # Support passing arbitrary file-like objects
+            # and use them instead of streams.
+            if isinstance(rpath, io.IOBase):
+                context = nullcontext(rpath)
+                use_seek = False  # might not support seeking
+            else:
+                context = open(rpath, "rb")
+                use_seek = True
+
+            with context as f:
+                if use_seek:
+                    callback.set_size(f.seek(0, 2))
+                    f.seek(0)
 
                 chunk = f.read(64 * 1024)
                 while chunk:

fsspec/implementations/tests/test_http.py

@@ -465,6 +465,11 @@ def test_put_file(server, tmp_path, method, reset_files):
     fs.get_file(server + "/hey", dwl_file)
     assert dwl_file.read_bytes() == data
 
+    src_file.write_bytes(b"xxx")
+    with open(src_file, "rb") as stream:
+        fs.put_file(stream, server + "/hey_2", method=method)
+    assert fs.cat(server + "/hey_2") == b"xxx"
+
 
 @pytest.mark.xfail(
     condition=sys.flags.optimize > 1, reason="no docstrings when optimised"

fsspec/utils.py

@@ -4,6 +4,7 @@
 import pathlib
 import re
 import sys
+from contextlib import contextmanager
 from functools import partial
 from hashlib import md5
 from urllib.parse import urlsplit
@@ -466,3 +467,12 @@ def wrapper(cls):
         return cls
 
     return wrapper
+
+
+try:
+    from contextlib import nullcontext
+except ImportError:
+
+    @contextmanager
+    def nullcontext(obj=None):
+        yield obj