requests: HTTP/1.1 with Content-Length and streaming raw

python-ecosys/requests/README.md

@@ -4,7 +4,8 @@ This module provides a lightweight version of the Python
 [requests](https://requests.readthedocs.io/en/latest/) library.
 
 It includes support for all HTTP verbs, https, json decoding of responses,
-redirects, basic authentication.
+redirects, basic authentication, HTTP/1.1 requests, and reading response
+bodies with Content-Length via streaming ``.raw`` or lazy ``.content``.
 
 ### Limitations
 
@@ -14,3 +15,10 @@ redirects, basic authentication.
 * Compressed requests/responses are not currently supported.
 * File upload is not supported.
 * Chunked encoding in responses is not supported.
+* HTTP keep-alive connection reuse is not supported (Connection: close by default).
+
+### Follow-up work
+
+* Chunked response bodies (separate PR).
+* TLS certificate verification (see micropython-lib issue #838).
+* ``stream=True`` incremental body reads (see issue #777).

python-ecosys/requests/manifest.py

@@ -1,3 +1,3 @@
-metadata(version="0.10.2", pypi="requests")
+metadata(version="0.10.3", pypi="requests")
 
 package("requests")

python-ecosys/requests/requests/__init__.py

@@ -1,6 +1,81 @@
 import socket
 
 
+def _header_get(headers, name):
+    name_lower = name.lower()
+    for k, v in headers.items():
+        if k.lower() == name_lower:
+            return v
+    return None
+
+
+def read_status_line(stream):
+    line = stream.readline()
+    if not line:
+        raise ValueError("HTTP error: empty status line")
+    parts = line.split(None, 2)
+    if len(parts) < 2:
+        raise ValueError("HTTP error: BadStatusLine:\n%s" % parts)
+    status = int(parts[1])
+    reason = parts[2].rstrip() if len(parts) > 2 else ""
+    return status, reason
+
+
+def read_headers(stream, parse_headers=True):
+    resp_d = {}
+    while True:
+        line = stream.readline()
+        if not line or line == b"\r\n":
+            break
+        if parse_headers is False:
+            continue
+        if parse_headers is True:
+            text = str(line, "utf-8")
+            key, value = text.split(":", 1)
+            resp_d[key] = value.strip()
+        else:
+            parse_headers(line, resp_d)
+    return resp_d
+
+
+class BodyStream:
+    def __init__(self, sock, remaining=None):
+        self._sock = sock
+        self._remaining = remaining
+
+    def read(self, n=-1):
+        if self._remaining == 0:
+            return b""
+        if n < 0:
+            if self._remaining is not None:
+                n = self._remaining
+            data = self._sock.read(n)
+        else:
+            if self._remaining is not None and n > self._remaining:
+                n = self._remaining
+            data = self._sock.read(n)
+        if self._remaining is not None:
+            self._remaining -= len(data)
+            if self._remaining > 0 and not data:
+                raise ValueError("Connection closed before Content-Length satisfied")
+        return data
+
+    def close(self):
+        self._sock.close()
+
+
+def open_body(stream, headers):
+    encoding = _header_get(headers, "transfer-encoding")
+    if encoding and "chunked" in encoding.lower():
+        raise ValueError("Unsupported Transfer-Encoding: %s" % encoding)
+    content_length = _header_get(headers, "content-length")
+    if content_length is not None:
+        remaining = int(content_length)
+    else:
+        remaining = None
+    return BodyStream(stream, remaining)
+
+
 class Response:
     def __init__(self, f):
         self.raw = f
@@ -81,10 +156,6 @@ def request(
     ai = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)
     ai = ai[0]
 
-    resp_d = None
-    if parse_headers is not False:
-        resp_d = {}
-
     s = socket.socket(ai[0], socket.SOCK_STREAM, ai[2])
 
     if timeout is not None:
@@ -98,7 +169,7 @@ def request(
             context = tls.SSLContext(tls.PROTOCOL_TLS_CLIENT)
             context.verify_mode = tls.CERT_NONE
             s = context.wrap_socket(s, server_hostname=host)
-        s.write(b"%s /%s HTTP/1.0\r\n" % (method, path))
+        s.write(b"%s /%s HTTP/1.1\r\n" % (method, path))
 
         if "Host" not in headers:
             headers["Host"] = host
@@ -145,37 +216,21 @@ def request(
             else:
                 s.write(data)
 
-        l = s.readline()
-        # print(l)
-        l = l.split(None, 2)
-        if len(l) < 2:
-            # Invalid response
-            raise ValueError("HTTP error: BadStatusLine:\n%s" % l)
-        status = int(l[1])
-        reason = ""
-        if len(l) > 2:
-            reason = l[2].rstrip()
-        while True:
-            l = s.readline()
-            if not l or l == b"\r\n":
-                break
-            # print(l)
-            if l.startswith(b"Transfer-Encoding:"):
-                if b"chunked" in l:
-                    raise ValueError("Unsupported " + str(l, "utf-8"))
-            elif l.startswith(b"Location:") and not 200 <= status <= 299:
+        status, reason = read_status_line(s)
+        resp_d = read_headers(s, parse_headers)
+
+        if not 200 <= status <= 299:
+            location = None
+            if resp_d:
+                for k in resp_d:
+                    if k.lower() == "location":
+                        location = resp_d[k]
+                        break
+            if location:
                 if status in [301, 302, 303, 307, 308]:
-                    redirect = str(l[10:-2], "utf-8")
+                    redirect = location
                 else:
                     raise NotImplementedError("Redirect %d not yet supported" % status)
-            if parse_headers is False:
-                pass
-            elif parse_headers is True:
-                l = str(l, "utf-8")
-                k, v = l.split(":", 1)
-                resp_d[k] = v.strip()
-            else:
-                parse_headers(l, resp_d)
     except OSError:
         s.close()
         raise
@@ -189,10 +244,10 @@ def request(
         else:
             return request(method, redirect, data, json, headers, stream)
     else:
-        resp = Response(s)
+        resp = Response(open_body(s, resp_d))
         resp.status_code = status
         resp.reason = reason
-        if resp_d is not None:
+        if parse_headers is not False:
             resp.headers = resp_d
         return resp