Compare the actual etags instead of comparing md5sums

The server side is currently using a shortcut in calculating the values
of the `md5sum` field - it uses the Object Storage (S3) ETag value.

ETag is a MD5. But for the multipart uploaded files, the MD5 is computed
from the concatenation of the MD5s of each uploaded part.

Say you uploaded a 14MB file and your part size is 5MB.
Calculate 3 MD5 checksums corresponding to each part, i.e. the checksum
of the first 5MB, the second 5MB, and the last 4MB.
Then take the checksum of their concatenation.
Since MD5 checksums are hex representations of binary data, just make
sure you take the MD5 of the decoded binary concatenation, not of the
ASCII or UTF-8 encoded concatenation.
When that's done, add a hyphen and the number of parts to get the ETag.

Author: suricactus

qfieldcloud_sdk/sdk.py

@@ -12,7 +12,7 @@
 from requests.adapters import HTTPAdapter, Retry
 
 from .interfaces import QfcException, QfcRequest, QfcRequestException
-from .utils import get_md5sum, log
+from .utils import calc_etag, log
 
 logger = logging.getLogger(__file__)
 
@@ -153,7 +153,11 @@ def list_remote_files(
             params["skip_metadata"] = "1"
 
         resp = self._request("GET", f"files/{project_id}", params=params)
-        return resp.json()
+        remote_files = resp.json()
+        # TODO remove this temporary decoration with `etag` key
+        remote_files = list(map(lambda f: {"etag": f["md5sum"], **f}, remote_files))
+
+        return remote_files
 
     def create_project(
         self,
@@ -217,8 +221,8 @@ def upload_files(
                             remote_file = f
                             break
 
-                    md5sum = get_md5sum(local_file["absolute_filename"])
-                    if remote_file and remote_file.get("md5sum", None) == md5sum:
+                    etag = calc_etag(local_file["absolute_filename"])
+                    if remote_file and remote_file.get("etag", None) == etag:
                         continue
 
                     files_to_upload.append(local_file)
@@ -537,9 +541,9 @@ def download_files(
 
         for file in files_to_download:
             local_filename = Path(f'{local_dir}/{file["name"]}')
-            md5sum = None
+            etag = None
             if not force_download:
-                md5sum = file.get("md5sum", None)
+                etag = file.get("etag", None)
 
             try:
                 self.download_file(
@@ -548,7 +552,7 @@ def download_files(
                     local_filename,
                     file["name"],
                     show_progress,
-                    md5sum,
+                    etag,
                 )
                 file["status"] = FileTransferStatus.SUCCESS
             except QfcRequestException as err:
@@ -575,7 +579,7 @@ def download_file(
         local_filename: Path,
         remote_filename: Path,
         show_progress: bool,
-        remote_md5sum: str = None,
+        remote_etag: str = None,
     ) -> requests.Response:
         """Download a single project file.
 
@@ -585,7 +589,7 @@ def download_file(
             local_filename (Path): Local filename
             remote_filename (Path): Remote filename
             show_progress (bool): Show progressbar in the console
-            remote_md5sum (str, optional): The md5sum of the remote file. If is None, the download of the file happens even if it already exists locally. Defaults to None.
+            remote_etag (str, optional): The ETag of the remote file. If is None, the download of the file happens even if it already exists locally. Defaults to `None`.
 
         Raises:
             NotImplementedError: Raised if unknown `download_type` is passed
@@ -594,8 +598,8 @@ def download_file(
             requests.Response: the response object
         """
 
-        if remote_md5sum and local_filename.exists():
-            if get_md5sum(str(local_filename)) == remote_md5sum:
+        if remote_etag and local_filename.exists():
+            if calc_etag(str(local_filename)) == remote_etag:
                 if show_progress:
                     print(
                         f"{remote_filename}: Already present locally. Download skipped."

qfieldcloud_sdk/utils.py

@@ -1,5 +1,6 @@
 import hashlib
 import json
+import os
 import sys
 
 
@@ -14,7 +15,10 @@ def log(*msgs):
 
 
 def get_md5sum(filename: str) -> str:
-    """Calculate sha256sum of a file"""
+    """Calculate md5sum of a file.
+
+    Currently unused but will be revived in the upcoming versions.
+    """
     BLOCKSIZE = 65536
     hasher = hashlib.md5()
     with open(filename, "rb") as f:
@@ -23,3 +27,45 @@ def get_md5sum(filename: str) -> str:
             hasher.update(buf)
             buf = f.read(BLOCKSIZE)
     return hasher.hexdigest()
+
+
+def calc_etag(filename: str, part_size: int = 8 * 1024 * 1024) -> str:
+    """Calculate ETag as in Object Storage (S3) of a local file.
+
+    ETag is a MD5. But for the multipart uploaded files, the MD5 is computed from the concatenation of the MD5s of each uploaded part.
+
+    See the inspiration of this implementation here: https://stackoverflow.com/a/58239738/1226137
+
+    Args:
+        filename (str): the local filename
+        part_size (int): the size of the Object Storage part. Most Object Storages use 8MB. Defaults to 8*1024*1024.
+
+    Returns:
+        str: the calculated ETag value
+    """
+    with open(filename, "rb") as f:
+        file_size = os.fstat(f.fileno()).st_size
+
+        if file_size <= part_size:
+            BLOCKSIZE = 65536
+            hasher = hashlib.md5()
+
+            buf = f.read(BLOCKSIZE)
+            while len(buf) > 0:
+                hasher.update(buf)
+                buf = f.read(BLOCKSIZE)
+
+            return hasher.hexdigest()
+        else:
+            # Say you uploaded a 14MB file and your part size is 5MB.
+            # Calculate 3 MD5 checksums corresponding to each part, i.e. the checksum of the first 5MB, the second 5MB, and the last 4MB.
+            # Then take the checksum of their concatenation.
+            # Since MD5 checksums are hex representations of binary data, just make sure you take the MD5 of the decoded binary concatenation, not of the ASCII or UTF-8 encoded concatenation.
+            # When that's done, add a hyphen and the number of parts to get the ETag.
+            md5sums = []
+            for data in iter(lambda: f.read(part_size), b""):
+                md5sums.append(hashlib.md5(data).digest())
+
+            final_md5sum = hashlib.md5(b"".join(md5sums))
+
+            return "{}-{}".format(final_md5sum.hexdigest(), len(md5sums))