feat: make storage upload limits configurable (#1090)

Make MAX_FILE_SIZE and MAX_UNAUTHENTICATED_UPLOAD_FILE_SIZE configurable
via the storage config section so testnets can raise them.

- Add storage.max_file_size (default 100 MiB) and
  storage.max_unauthenticated_upload_file_size (default 25 MiB) config entries
- Inject limits via constructor into StoreMessageHandler, BalanceCronJob,
  and CreditBalanceCronJob
- Read limits from config in storage controller route handlers
- Fix client_max_size bug (was 64 MiB, below the 100 MiB upload limit)
- Rename constants to DEFAULT_MAX_FILE_SIZE / DEFAULT_MAX_UNAUTHENTICATED_UPLOAD_FILE_SIZE

Author: odesenfans

src/aleph/api_entrypoint.py

@@ -62,7 +62,9 @@ async def configure_aiohttp_app(
         )
         signature_verifier = SignatureVerifier()
 
-        app = create_aiohttp_app()
+        app = create_aiohttp_app(
+            max_file_size=config.storage.max_file_size.value,
+        )
 
         # Reuse the connection of the P2P client to avoid opening two connections
         mq_conn = p2p_client.mq_client.connection

src/aleph/commands.py

@@ -153,8 +153,14 @@ async def main(args: List[str]) -> None:
         cron_job = CronJob(
             session_factory=session_factory,
             jobs={
-                "balance": BalanceCronJob(session_factory=session_factory),
-                "credit_balance": CreditBalanceCronJob(session_factory=session_factory),
+                "balance": BalanceCronJob(
+                    session_factory=session_factory,
+                    max_unauthenticated_upload_file_size=config.storage.max_unauthenticated_upload_file_size.value,
+                ),
+                "credit_balance": CreditBalanceCronJob(
+                    session_factory=session_factory,
+                    max_unauthenticated_upload_file_size=config.storage.max_unauthenticated_upload_file_size.value,
+                ),
             },
         )
         chain_data_service = ChainDataService(

src/aleph/config.py

@@ -2,6 +2,11 @@
 
 from configmanager import Config
 
+from aleph.toolkit.constants import (
+    DEFAULT_MAX_FILE_SIZE,
+    DEFAULT_MAX_UNAUTHENTICATED_UPLOAD_FILE_SIZE,
+)
+
 
 def get_defaults():
     return {
@@ -117,8 +122,12 @@ def get_defaults():
             "store_files": True,
             # Interval between garbage collector runs, expressed in hours.
             "garbage_collector_period": 24,
-            # Grapce period for files, expressed in hours.
+            # Grace period for files, expressed in hours.
             "grace_period": 24,
+            # Maximum file size for authenticated uploads, in bytes.
+            "max_file_size": DEFAULT_MAX_FILE_SIZE,
+            # Maximum file size for unauthenticated uploads, in bytes.
+            "max_unauthenticated_upload_file_size": DEFAULT_MAX_UNAUTHENTICATED_UPLOAD_FILE_SIZE,
         },
         "nuls2": {
             # NULS2 chain ID.

src/aleph/handlers/content/store.py

@@ -40,7 +40,7 @@
 from aleph.services.cost_validation import validate_balance_for_payment
 from aleph.services.ipfs import IpfsService
 from aleph.storage import StorageService
-from aleph.toolkit.constants import MAX_UNAUTHENTICATED_UPLOAD_FILE_SIZE, MiB
+from aleph.toolkit.constants import MiB
 from aleph.toolkit.costs import are_store_and_program_free, is_credit_only_required
 from aleph.toolkit.timestamp import timestamp_to_datetime, utc_now
 from aleph.types.db_session import DbSession
@@ -135,9 +135,15 @@ def _should_pin_on_ipfs(
 
 
 class StoreMessageHandler(ContentHandler):
-    def __init__(self, storage_service: StorageService, grace_period: int):
+    def __init__(
+        self,
+        storage_service: StorageService,
+        grace_period: int,
+        max_unauthenticated_upload_file_size: int,
+    ):
         self.storage_service = storage_service
         self.grace_period = grace_period
+        self.max_unauthenticated_upload_file_size = max_unauthenticated_upload_file_size
 
     async def is_related_content_fetched(
         self, session: DbSession, message: MessageDb
@@ -245,7 +251,7 @@ async def pre_check_balance(self, session: DbSession, message: MessageDb):
 
                 # Allow users to pin small files (only for hold payment type, before cutoff)
                 if payment_type == PaymentType.hold and storage_mib <= (
-                    MAX_UNAUTHENTICATED_UPLOAD_FILE_SIZE / MiB
+                    self.max_unauthenticated_upload_file_size / MiB
                 ):
                     return None
 
@@ -298,7 +304,7 @@ async def check_balance(
         if (
             payment_type == PaymentType.hold
             and storage_size_mib
-            and storage_size_mib <= (MAX_UNAUTHENTICATED_UPLOAD_FILE_SIZE / MiB)
+            and storage_size_mib <= (self.max_unauthenticated_upload_file_size / MiB)
         ):
             return costs
 

src/aleph/handlers/message_handler.py

@@ -78,6 +78,7 @@ def __init__(
             MessageType.store: StoreMessageHandler(
                 storage_service=storage_service,
                 grace_period=config.storage.grace_period.value,
+                max_unauthenticated_upload_file_size=config.storage.max_unauthenticated_upload_file_size.value,
             ),
         }
 

src/aleph/jobs/cron/balance_job.py

@@ -17,11 +17,7 @@
 from aleph.db.models.messages import MessageDb, MessageStatusDb
 from aleph.jobs.cron.cron_job import BaseCronJob
 from aleph.services.cost import calculate_storage_size
-from aleph.toolkit.constants import (
-    MAX_UNAUTHENTICATED_UPLOAD_FILE_SIZE,
-    STORE_AND_PROGRAM_COST_CUTOFF_HEIGHT,
-    MiB,
-)
+from aleph.toolkit.constants import STORE_AND_PROGRAM_COST_CUTOFF_HEIGHT, MiB
 from aleph.toolkit.timestamp import utc_now
 from aleph.types.db_session import DbSession, DbSessionFactory
 from aleph.types.message_status import MessageStatus
@@ -30,8 +26,13 @@
 
 
 class BalanceCronJob(BaseCronJob):
-    def __init__(self, session_factory: DbSessionFactory):
+    def __init__(
+        self,
+        session_factory: DbSessionFactory,
+        max_unauthenticated_upload_file_size: int,
+    ):
         self.session_factory = session_factory
+        self.max_unauthenticated_upload_file_size = max_unauthenticated_upload_file_size
 
     async def run(self, now: dt.datetime, job: CronJobDb):
         with self.session_factory() as session:
@@ -101,7 +102,7 @@ async def delete_messages(self, session: DbSession, messages: List[ItemHash]):
                 )
 
                 if storage_size_mib and storage_size_mib <= (
-                    MAX_UNAUTHENTICATED_UPLOAD_FILE_SIZE / MiB
+                    self.max_unauthenticated_upload_file_size / MiB
                 ):
                     continue
 

src/aleph/jobs/cron/credit_balance_job.py

@@ -20,11 +20,7 @@
 from aleph.db.models.messages import MessageDb, MessageStatusDb
 from aleph.jobs.cron.cron_job import BaseCronJob
 from aleph.services.cost import calculate_storage_size
-from aleph.toolkit.constants import (
-    CREDIT_ONLY_CUTOFF_TIMESTAMP,
-    MAX_UNAUTHENTICATED_UPLOAD_FILE_SIZE,
-    MiB,
-)
+from aleph.toolkit.constants import CREDIT_ONLY_CUTOFF_TIMESTAMP, MiB
 from aleph.toolkit.timestamp import utc_now
 from aleph.types.db_session import DbSession, DbSessionFactory
 from aleph.types.message_status import MessageStatus
@@ -33,8 +29,13 @@
 
 
 class CreditBalanceCronJob(BaseCronJob):
-    def __init__(self, session_factory: DbSessionFactory):
+    def __init__(
+        self,
+        session_factory: DbSessionFactory,
+        max_unauthenticated_upload_file_size: int,
+    ):
         self.session_factory = session_factory
+        self.max_unauthenticated_upload_file_size = max_unauthenticated_upload_file_size
 
     async def run(self, now: dt.datetime, job: CronJobDb):
         with self.session_factory() as session:
@@ -114,7 +115,8 @@ async def delete_messages(self, session: DbSession, messages: List[ItemHash]):
                 if (
                     is_legacy_message
                     and storage_size_mib
-                    and storage_size_mib <= (MAX_UNAUTHENTICATED_UPLOAD_FILE_SIZE / MiB)
+                    and storage_size_mib
+                    <= (self.max_unauthenticated_upload_file_size / MiB)
                 ):
                     continue
 

src/aleph/toolkit/constants.py

@@ -332,9 +332,8 @@ class ProductPriceType(str, Enum):
 CREDIT_PRECISION_CUTOFF_TIMESTAMP = 1769990400  # 2026-02-02 00:00:00 UTC
 CREDIT_PRECISION_MULTIPLIER = 10000
 
-MAX_FILE_SIZE = 100 * MiB
-MAX_UNAUTHENTICATED_UPLOAD_FILE_SIZE = 25 * MiB
-# MAX_UPLOAD_FILE_SIZE = 1000 * MiB (not used?)
+DEFAULT_MAX_FILE_SIZE = 100 * MiB
+DEFAULT_MAX_UNAUTHENTICATED_UPLOAD_FILE_SIZE = 25 * MiB
 MIN_STORE_COST_MIB = 25  # Minimum MiB cost for pure STORE messages
 MIN_CREDIT_COST_PER_HOUR = (
     1  # Minimum cost per hour in credits for instances and volumes

src/aleph/web/__init__.py

@@ -9,6 +9,7 @@
 from aiohttp import web
 from aiohttp_swagger3 import SwaggerDocs, SwaggerInfo, SwaggerUiSettings
 
+from aleph.toolkit.constants import DEFAULT_MAX_FILE_SIZE
 from aleph.version import get_version
 from aleph.web.controllers.routes import register_routes
 
@@ -32,8 +33,11 @@ def init_cors(app: web.Application):
             cors.add(route)
 
 
-def create_aiohttp_app(with_swagger: bool = True) -> web.Application:
-    app = web.Application(client_max_size=1024**2 * 64)
+def create_aiohttp_app(
+    with_swagger: bool = True,
+    max_file_size: int = DEFAULT_MAX_FILE_SIZE,
+) -> web.Application:
+    app = web.Application(client_max_size=max_file_size)
 
     swagger = None
     if with_swagger:

src/aleph/web/controllers/storage.py

@@ -33,11 +33,7 @@
 )
 from aleph.services.cost import get_total_and_detailed_costs
 from aleph.storage import StorageService
-from aleph.toolkit.constants import (
-    MAX_FILE_SIZE,
-    MAX_UNAUTHENTICATED_UPLOAD_FILE_SIZE,
-    MiB,
-)
+from aleph.toolkit.constants import MiB
 from aleph.types.db_session import DbSession, DbSessionFactory
 from aleph.types.files import FileTag, FileType
 from aleph.types.message_status import InvalidSignature
@@ -155,10 +151,12 @@ async def _verify_message_signature(
 
 
 def _verify_user_balance(
-    session: DbSession, content: CostEstimationStoreContent
+    session: DbSession,
+    content: CostEstimationStoreContent,
+    max_unauthenticated_upload_file_size: int,
 ) -> None:
     if content.estimated_size_mib and content.estimated_size_mib > (
-        MAX_UNAUTHENTICATED_UPLOAD_FILE_SIZE / MiB
+        max_unauthenticated_upload_file_size / MiB
     ):
         current_balance = get_total_balance(session=session, address=content.address)
         current_cost = get_total_cost_for_address(
@@ -271,6 +269,7 @@ async def _check_and_add_file(
     message: Optional[PendingStoreMessage],
     uploaded_file: UploadedFile,
     grace_period: int,
+    max_unauthenticated_upload_file_size: int,
 ) -> str:
     file_hash = uploaded_file.get_hash()
     # Perform authentication and balance checks
@@ -297,7 +296,11 @@ async def _check_and_add_file(
             )
 
         with session_factory() as session:
-            _verify_user_balance(session=session, content=message_content)
+            _verify_user_balance(
+                session=session,
+                content=message_content,
+                max_unauthenticated_upload_file_size=max_unauthenticated_upload_file_size,
+            )
     else:
         message_content = None
 
@@ -394,6 +397,10 @@ async def storage_add_file(request: web.Request):
     signature_verifier = get_signature_verifier_from_request(request)
     config = get_config_from_request(request)
     grace_period = config.storage.grace_period.value
+    max_file_size = config.storage.max_file_size.value
+    max_unauthenticated_upload_file_size = (
+        config.storage.max_unauthenticated_upload_file_size.value
+    )
     metadata = None
     uploaded_file: Optional[UploadedFile] = None
 
@@ -409,13 +416,13 @@ async def storage_add_file(request: web.Request):
                     raise web.HTTPBadRequest(text="Invalid multipart structure")
 
                 if part.name == "file":
-                    uploaded_file = MultipartUploadedFile(part, MAX_FILE_SIZE)
+                    uploaded_file = MultipartUploadedFile(part, max_file_size)
                     await uploaded_file.read_and_validate()
                 elif part.name == "metadata":
                     metadata = await part.read(decode=True)
         else:
             uploaded_file = RawUploadedFile(
-                request=request, max_size=MAX_UNAUTHENTICATED_UPLOAD_FILE_SIZE
+                request=request, max_size=max_unauthenticated_upload_file_size
             )
             await uploaded_file.read_and_validate()
 
@@ -425,7 +432,7 @@ async def storage_add_file(request: web.Request):
             )
 
         max_upload_size = (
-            MAX_UNAUTHENTICATED_UPLOAD_FILE_SIZE if not metadata else MAX_FILE_SIZE
+            max_unauthenticated_upload_file_size if not metadata else max_file_size
         )
         if uploaded_file.size > max_upload_size:
             raise web.HTTPRequestEntityTooLarge(
@@ -460,6 +467,7 @@ async def storage_add_file(request: web.Request):
             message=message,
             uploaded_file=uploaded_file,
             grace_period=grace_period,
+            max_unauthenticated_upload_file_size=max_unauthenticated_upload_file_size,
         )
         if message:
             broadcast_status = await broadcast_and_process_message(
@@ -475,7 +483,9 @@ async def storage_add_file(request: web.Request):
             await uploaded_file.cleanup()
 
 
-def assert_file_is_downloadable(session: DbSession, file_hash: str) -> None:
+def assert_file_is_downloadable(
+    session: DbSession, file_hash: str, max_file_size: int
+) -> None:
     """
     Check if the file is on the aleph.im network and can be downloaded from the API.
     This filters out requests for files outside the network / nonexistent files.
@@ -484,9 +494,9 @@ def assert_file_is_downloadable(session: DbSession, file_hash: str) -> None:
     if not file_metadata:
         raise web.HTTPNotFound(text="Not found")
 
-    if file_metadata.size > MAX_FILE_SIZE:
+    if file_metadata.size > max_file_size:
         raise web.HTTPRequestEntityTooLarge(
-            max_size=MAX_FILE_SIZE, actual_size=file_metadata.size
+            max_size=max_file_size, actual_size=file_metadata.size
         )
 
 
@@ -529,9 +539,14 @@ async def get_hash(request):
         logger.warning(e.args[0])
         raise web.HTTPBadRequest(text="Invalid hash provided")
 
+    config = get_config_from_request(request)
+    max_file_size = config.storage.max_file_size.value
+
     session_factory = get_session_factory_from_request(request)
     with session_factory() as session:
-        assert_file_is_downloadable(session=session, file_hash=file_hash)
+        assert_file_is_downloadable(
+            session=session, file_hash=file_hash, max_file_size=max_file_size
+        )
 
     storage_service = get_storage_service_from_request(request)