fix(storage): phase out unauthenticated upload

src/aleph/config.py

@@ -141,9 +141,9 @@ def get_defaults():
             # Whether to store files on the node.
             "store_files": True,
             # Interval between garbage collector runs, expressed in hours.
-            "garbage_collector_period": 24,
+            "garbage_collector_period": 4,
             # Grace period for files, expressed in hours.
-            "grace_period": 24,
+            "grace_period": 6,
             # Maximum file size for authenticated uploads, in bytes.
             "max_file_size": DEFAULT_MAX_FILE_SIZE,
             # Maximum file size for unauthenticated uploads, in bytes.

src/aleph/web/controllers/ipfs.py

@@ -29,6 +29,7 @@
     add_grace_period_for_file,
     broadcast_and_process_message,
     broadcast_status_to_http_status,
+    warn_deprecated_unauthenticated_upload,
 )
 
 logger = logging.getLogger(__name__)
@@ -123,11 +124,10 @@ async def ipfs_add_file(request: web.Request):
                 reason="Missing 'file' in multipart form."
             )
 
-        # Narrow the effective cap for unauthenticated requests.
-        if (
-            metadata is None
-            and uploaded_file.size > max_unauthenticated_upload_file_size
-        ):
+        # Narrow the effective cap for unauthenticated requests. `not metadata`
+        # (rather than `is None`) so an empty metadata part counts as
+        # unauthenticated, matching the message-parsing gate below.
+        if not metadata and uploaded_file.size > max_unauthenticated_upload_file_size:
             raise web.HTTPRequestEntityTooLarge(
                 actual_size=uploaded_file.size,
                 max_size=max_unauthenticated_upload_file_size,
@@ -189,7 +189,7 @@ async def ipfs_add_file(request: web.Request):
 
         # Post-pin: stat, CID match, persist.
         # Failures from this point on must leave the pin covered by the
-        # 24 h grace period so the GC doesn't strand it.
+        # grace period so the GC doesn't strand it.
         try:
             try:
                 stats = await asyncio.wait_for(
@@ -215,10 +215,11 @@ async def ipfs_add_file(request: web.Request):
                     size=size,
                     file_type=FileType.FILE,
                 )
-                if message_content is None:
-                    add_grace_period_for_file(
-                        session=session, file_hash=cid, hours=grace_period
-                    )
+                # Grace pin for anonymous uploads and as a bridge until the
+                # STORE message creates the permanent pin (see storage.py).
+                add_grace_period_for_file(
+                    session=session, file_hash=cid, hours=grace_period
+                )
                 session.commit()
         except Exception:
             # Bare `Exception` is intentional: any post-pin failure must
@@ -260,6 +261,9 @@ async def ipfs_add_file(request: web.Request):
             )
             status_code = broadcast_status_to_http_status(broadcast_status)
 
+        headers = (
+            warn_deprecated_unauthenticated_upload(request) if not metadata else None
+        )
         return web.json_response(
             data={
                 "status": "success",
@@ -268,6 +272,7 @@ async def ipfs_add_file(request: web.Request):
                 "size": size,
             },
             status=status_code,
+            headers=headers,
         )
 
     finally:
@@ -463,6 +468,11 @@ async def ipfs_add_car(request: web.Request):
                     size=size,
                     file_type=FileType.DIRECTORY,
                 )
+                # Grace pin bridging the gap until the STORE message creates
+                # the permanent pin (see storage.py _check_and_add_file).
+                add_grace_period_for_file(
+                    session=session, file_hash=cid, hours=grace_period
+                )
                 session.commit()
         except Exception:
             if cid is None:

src/aleph/web/controllers/storage.py

@@ -1,11 +1,12 @@
 import base64
 import hashlib
+import json
 import logging
 import math
 import os
 import tempfile
 from pathlib import Path
-from typing import Optional
+from typing import Any, Optional
 
 import aio_pika
 import aiofiles
@@ -54,11 +55,41 @@
     broadcast_and_process_message,
     broadcast_status_to_http_status,
     mq_make_aleph_message_topic_queue,
+    warn_deprecated_unauthenticated_upload,
 )
 
 logger = logging.getLogger(__name__)
 
 
+async def _read_json_body_with_limit(request: web.Request, max_size: int) -> Any:
+    """Read a JSON request body, aborting as soon as it exceeds max_size.
+
+    Unlike request.json(), this does not buffer up to client_max_size
+    (100 MiB) before checking; anonymous endpoints must reject at the
+    unauthenticated upload limit (25 MiB by default).
+    """
+    content_length = request.content_length
+    if content_length is not None and content_length > max_size:
+        raise web.HTTPRequestEntityTooLarge(
+            actual_size=content_length, max_size=max_size
+        )
+
+    buffer = bytearray()
+    async for chunk in request.content.iter_chunked(8192):
+        buffer.extend(chunk)
+        if len(buffer) > max_size:
+            raise web.HTTPRequestEntityTooLarge(
+                actual_size=len(buffer), max_size=max_size
+            )
+
+    try:
+        return json.loads(bytes(buffer))
+    except ValueError:
+        # Covers both JSONDecodeError and UnicodeDecodeError: any
+        # undecodable anonymous body is a client error, not a server error.
+        raise web.HTTPUnprocessableEntity(reason="Invalid JSON body")
+
+
 async def add_ipfs_json_controller(request: web.Request):
     """
     Forward the JSON content to IPFS server and return a hash.
@@ -86,7 +117,8 @@ async def add_ipfs_json_controller(request: web.Request):
     config = get_config_from_request(request)
     grace_period = config.storage.grace_period.value
 
-    data = await request.json()
+    max_size = config.storage.max_unauthenticated_upload_file_size.value
+    data = await _read_json_body_with_limit(request, max_size)
     with session_factory() as session:
         output = {
             "status": "success",
@@ -99,7 +131,9 @@ async def add_ipfs_json_controller(request: web.Request):
         )
         session.commit()
 
-    return web.json_response(output)
+    return web.json_response(
+        output, headers=warn_deprecated_unauthenticated_upload(request)
+    )
 
 
 async def add_storage_json_controller(request: web.Request):
@@ -129,7 +163,8 @@ async def add_storage_json_controller(request: web.Request):
     config = get_config_from_request(request)
     grace_period = config.storage.grace_period.value
 
-    data = await request.json()
+    max_size = config.storage.max_unauthenticated_upload_file_size.value
+    data = await _read_json_body_with_limit(request, max_size)
     with session_factory() as session:
         output = {
             "status": "success",
@@ -142,7 +177,9 @@ async def add_storage_json_controller(request: web.Request):
         )
         session.commit()
 
-    return web.json_response(output)
+    return web.json_response(
+        output, headers=warn_deprecated_unauthenticated_upload(request)
+    )
 
 
 async def _verify_message_signature(
@@ -350,11 +387,15 @@ async def _check_and_add_file(
             file_type=FileType.FILE,
         )
 
-        # For files uploaded without authenticated upload, add a grace period of 1 day.
-        if message_content is None:
-            add_grace_period_for_file(
-                session=session, file_hash=file_hash, hours=grace_period
-            )
+        # Pin the file for the grace period (storage.grace_period config).
+        # For anonymous uploads this is the only pin and bounds the file's
+        # lifetime. For authenticated uploads it bridges the gap until the
+        # STORE message is processed and creates the permanent pin; without
+        # it, a garbage collector sweep during a pending-queue backlog could
+        # delete the file before its message is processed.
+        add_grace_period_for_file(
+            session=session, file_hash=file_hash, hours=grace_period
+        )
 
         session.commit()
 
@@ -497,7 +538,10 @@ async def storage_add_file(request: web.Request):
             status_code = broadcast_status_to_http_status(broadcast_status)
 
         output = {"status": "success", "hash": file_hash}
-        return web.json_response(data=output, status=status_code)
+        headers = (
+            warn_deprecated_unauthenticated_upload(request) if message is None else None
+        )
+        return web.json_response(data=output, status=status_code, headers=headers)
 
     finally:
         if uploaded_file is not None:

src/aleph/web/controllers/utils.py

@@ -447,6 +447,24 @@ def get_item_hash_from_request(request: web.Request) -> ItemHash:
     return item_hash
 
 
+UPLOAD_DEPRECATION_HEADERS: Dict[str, str] = {"Deprecation": "true"}
+
+
+def warn_deprecated_unauthenticated_upload(request: web.Request) -> Dict[str, str]:
+    """Log and return deprecation headers for anonymous upload requests.
+
+    Unauthenticated uploads are deprecated and will be removed; the log
+    line lets node operators spot remaining anonymous traffic.
+    """
+    logging.getLogger(__name__).warning(
+        "Deprecated unauthenticated upload on %s from %s. This path will be "
+        "removed in a future release; uploads will require a signed message.",
+        request.path,
+        request.remote,
+    )
+    return dict(UPLOAD_DEPRECATION_HEADERS)
+
+
 CURSOR_MAX_PAGINATION = 200
 
 

tests/api/test_ipfs.py

@@ -123,6 +123,7 @@ async def test_unauth_upload_happy_path(api_client, session_factory: DbSessionFa
     response = await api_client.post(IPFS_ADD_FILE_URI, data=form_data)
     body = await response.text()
     assert response.status == 200, body
+    assert response.headers.get("Deprecation") == "true"
     payload = await response.json()
     assert payload["status"] == "success"
     assert payload["hash"] == EXPECTED_FILE_CID
@@ -193,14 +194,16 @@ async def test_auth_upload_happy_path(
     response = await api_client.post(IPFS_ADD_FILE_URI, data=form_data)
     body = await response.text()
     assert response.status == 200, body
+    assert "Deprecation" not in response.headers
     payload = await response.json()
     assert payload["hash"] == EXPECTED_FILE_CID
 
     with session_factory() as session:
         file = get_file(session=session, file_hash=EXPECTED_FILE_CID)
         assert file is not None
-        # Authenticated uploads do NOT get a grace period (message anchors).
-        assert not _has_grace_period(session, EXPECTED_FILE_CID)
+        # Authenticated uploads must get a grace pin to bridge the gap until
+        # the STORE message is processed and creates the permanent pin.
+        assert _has_grace_period(session, EXPECTED_FILE_CID)
 
 
 @pytest.mark.asyncio
@@ -700,8 +703,9 @@ async def test_add_car_success(
         assert file is not None
         assert file.type == FileType.DIRECTORY
         assert file.size == MOCK_DIR_SIZE
-        # Authenticated success: no grace period.
-        assert not _has_grace_period(session, DIR_ROOT_CID)
+        # Bridge grace pin: written on success so the GC cannot reap the pin
+        # before the STORE message is processed and creates the permanent pin.
+        assert _has_grace_period(session, DIR_ROOT_CID)
 
     ipfs_service = _get_ipfs_service_mock(api_client_with_dag_import)
     ipfs_service.dag_import.assert_called_once()
@@ -1151,3 +1155,25 @@ async def test_add_car_dag_import_failure(
 
     with session_factory() as session:
         assert get_file(session=session, file_hash=DIR_ROOT_CID) is None
+
+
+@pytest.mark.asyncio
+async def test_empty_metadata_treated_as_unauthenticated(
+    api_client, session_factory: DbSessionFactory
+):
+    """A multipart request with an empty metadata field (b"") must be treated
+    as unauthenticated: the Deprecation header must be present and a grace pin
+    must be written, identical to a request with no metadata field at all."""
+    form_data = aiohttp.FormData()
+    form_data.add_field("file", BytesIO(FILE_CONTENT))
+    form_data.add_field("metadata", b"")
+
+    response = await api_client.post(IPFS_ADD_FILE_URI, data=form_data)
+    body = await response.text()
+    assert response.status == 200, body
+    assert response.headers.get("Deprecation") == "true"
+
+    with session_factory() as session:
+        file = get_file(session=session, file_hash=EXPECTED_FILE_CID)
+        assert file is not None
+        assert _has_grace_period(session, EXPECTED_FILE_CID)