[6.x] Chunked uploads

config/assets.php

@@ -252,6 +252,25 @@
 
     'svg_sanitization_on_upload' => true,
 
+    /*
+    |--------------------------------------------------------------------------
+    | Chunked Uploads
+    |--------------------------------------------------------------------------
+    |
+    | Large Control Panel uploads can be split into chunks to get past PHP and
+    | proxy upload size limits. Chunks are staged on the local "disk" and
+    | assembled server-side; "max_chunk_size" caps the per-request size.
+    |
+    */
+
+    'chunked_upload' => [
+        'enabled' => true,
+        'max_chunk_size' => 10 * 1024 * 1024,
+        'chunk_overhead' => 64 * 1024,
+        'disk' => 'local',
+        'directory' => 'statamic/chunks',
+    ],
+
     /*
     |--------------------------------------------------------------------------
     | FFmpeg

lang/en/messages.php

@@ -6,6 +6,7 @@
     'addon_has_more_releases_beyond_license_body' => 'You may update, but will need to upgrade or purchase a new license.',
     'addon_has_more_releases_beyond_license_heading' => 'This addon has more releases beyond your licensed limit.',
     'asset_container_blueprint_instructions' => 'Blueprints define custom fields available when editing assets.',
+    'asset_container_chunked_uploads_instructions' => 'Split large uploads into chunks to get past PHP and proxy upload size limits. The maximum size is set by your validation rules.',
     'asset_container_disk_instructions' => 'Filesystem disks specify where files are stored (locally or remotely like Amazon S3). Configure in `config/filesystems.php`',
     'asset_container_handle_instructions' => 'Used to reference this container on the frontend. This cannot be easily changed later.',
     'asset_container_intro' => 'Media and document files are stored in folders on the server or other storage services. Each storage location is called a container.',

resources/js/components/assets/Browser/Browser.vue

@@ -3,6 +3,10 @@
         <Uploader
             ref="internalUploader"
             :container="container.id"
+            :chunked-uploads="container.chunked_uploads"
+            :chunk-size="container.chunk_size"
+            :max-filesize="container.max_filesize"
+            :chunk-upload-url="container.chunk_upload_url"
             :path="path"
             :enabled="!uploader && !preventDragging && canUpload"
             @updated="uploadsUpdated"

resources/js/components/assets/ChunkedUpload.js

@@ -0,0 +1,124 @@
+import axios from 'axios';
+import { nanoid } from 'nanoid';
+
+/**
+ * Uploads a single file in sequential chunks to the chunk endpoint, then resolves
+ * with the final assembly response. Duck-typed to the `upload` package so the
+ * surrounding Uploader component treats it identically to a single-request upload.
+ */
+export default class ChunkedUpload {
+    constructor({ url, file, data = {}, chunkSize, http = axios, wait, maxRetries = 3, baseDelay = 500 }) {
+        this.url = url;
+        this.file = file;
+        this.data = data;
+        this.chunkSize = chunkSize;
+        this.http = http;
+        this.wait = wait ?? ((ms) => new Promise((resolve) => setTimeout(resolve, ms)));
+        this.maxRetries = maxRetries;
+        this.baseDelay = baseDelay;
+        this.uploadId = nanoid();
+        this.state = 'new';
+        this.uploadedBytes = 0;
+        this.progressHandlers = [];
+        this.form = { get: (key) => (key === 'file' ? this.file : undefined) };
+    }
+
+    on(event, handler) {
+        if (event === 'progress') this.progressHandlers.push(handler);
+    }
+
+    emitProgress(value) {
+        this.progressHandlers.forEach((handler) => handler(value));
+    }
+
+    async upload() {
+        this.state = 'started';
+
+        const totalChunks = Math.ceil(this.file.size / this.chunkSize);
+
+        for (let index = 0; index < totalChunks; index++) {
+            const start = index * this.chunkSize;
+            const end = Math.min(start + this.chunkSize, this.file.size);
+            const chunk = this.file.slice(start, end);
+
+            let response;
+
+            try {
+                response = await this.sendChunk(chunk, index, totalChunks);
+            } catch (error) {
+                this.state = 'failed';
+
+                return this.toResponse(error.response);
+            }
+
+            this.uploadedBytes = end;
+
+            // The final chunk's response carries the assembled asset, or a validation error.
+            if (index === totalChunks - 1) {
+                this.state = 'finished';
+                this.emitProgress(1);
+
+                return this.toResponse(response);
+            }
+
+            this.emitProgress(this.uploadedBytes / this.file.size);
+        }
+    }
+
+    async sendChunk(chunk, index, totalChunks) {
+        for (let attempt = 0; ; attempt++) {
+            try {
+                return await this.http.post(this.url, this.formData(chunk, index, totalChunks), {
+                    headers: { Accept: 'application/json' },
+                    onUploadProgress: (e) => {
+                        const loaded = Math.min(e.loaded, chunk.size);
+                        this.emitProgress((this.uploadedBytes + loaded) / this.file.size);
+                    },
+                });
+            } catch (error) {
+                if (attempt >= this.maxRetries || !this.isRetryable(error, index, totalChunks)) {
+                    throw error;
+                }
+
+                await this.wait(this.baseDelay * 2 ** attempt);
+            }
+        }
+    }
+
+    isRetryable(error, index, totalChunks) {
+        const status = error.response?.status;
+
+        // A network error has no response.
+        if (!status) return true;
+
+        // Validation and conflict responses are definitive.
+        if ([422, 409].includes(status)) return false;
+
+        // A proxy rejecting the assembly request is not recoverable by retrying.
+        if (status === 413 && index === totalChunks - 1) return false;
+
+        return [408, 413, 429].includes(status) || status >= 500;
+    }
+
+    formData(chunk, index, totalChunks) {
+        const form = new FormData();
+
+        form.append('chunk', chunk, this.file.name);
+        form.append('uploadId', this.uploadId);
+        form.append('chunkIndex', index);
+        form.append('totalChunks', totalChunks);
+
+        for (const key in this.data) {
+            form.append(key, this.data[key]);
+        }
+
+        return form;
+    }
+
+    toResponse(response) {
+        return {
+            status: response?.status ?? 0,
+            data: JSON.stringify(response?.data ?? {}),
+        };
+    }
+}

resources/js/components/assets/Editor/CropEditor.vue

@@ -375,6 +375,7 @@ function unbindKeyboardShortcuts() {
     isAdjustingCropBox.value = false;
 }
 
+// Cropped output is small, so it uses the single-request endpoint rather than chunked uploads.
 async function upload(replaceOriginal) {
     if (!pendingBlob.value) return;
 

resources/js/components/assets/Selector.vue

@@ -2,6 +2,10 @@
     <Uploader
         ref="uploader"
         :container="container.id"
+        :chunked-uploads="container.chunked_uploads"
+        :chunk-size="container.chunk_size"
+        :max-filesize="container.max_filesize"
+        :chunk-upload-url="container.chunk_upload_url"
         :path="currentPath"
         :enabled="container.can_upload"
         @updated="(uploads) => $refs.browser?.uploadsUpdated(uploads)"

resources/js/components/assets/Uploader.vue

@@ -2,6 +2,8 @@
 import { Upload } from 'upload';
 import { nanoid as uniqid } from 'nanoid';
 import { h } from 'vue';
+import ChunkedUpload from './ChunkedUpload';
+import { useUploadsStore } from '../../stores/uploads';
 
 export default {
     emits: ['updated', 'upload-complete', 'error'],
@@ -50,6 +52,10 @@ export default {
             type: Object,
             default: () => ({}),
         },
+        chunkedUploads: { type: Boolean, default: false },
+        chunkSize: { type: Number, default: 0 },
+        maxFilesize: { type: Number, default: null },
+        chunkUploadUrl: { type: String, default: null },
     },
 
     data() {
@@ -59,6 +65,10 @@ export default {
         };
     },
 
+    created() {
+        this.uploadsStore = useUploadsStore();
+    },
+
     mounted() {
         this.$refs.nativeFileField.addEventListener('change', this.addNativeFileFieldSelections);
     },
@@ -178,18 +188,24 @@ export default {
             if (!this.enabled) return;
 
             const id = uniqid();
-            const upload = this.makeUpload(id, file, data);
+            const tooLarge = this.maxFilesize && file.size > this.maxFilesize;
 
-            this.uploads.push({
+            const upload = {
                 id,
                 basename: file.name,
                 extension: file.name.split('.').pop(),
                 percent: 0,
-                errorMessage: null,
-                errorStatus: null,
-                instance: upload,
+                errorMessage: tooLarge ? __('Upload failed. The file is larger than is allowed.') : null,
+                errorStatus: tooLarge ? 413 : null,
+            };
+
+            this.uploads.push({
+                ...upload,
+                instance: tooLarge ? { state: 'failed', form: { get: () => file } } : this.makeUpload(id, file, data),
                 retry: (opts) => this.retry(id, opts),
             });
+
+            this.uploadsStore.add(this.container, { ...upload });
         },
 
         findUpload(id) {
@@ -201,44 +217,63 @@ export default {
         },
 
         makeUpload(id, file, data = {}) {
-            const upload = new Upload({
-                url: this.url,
-                form: this.makeFormData(file, data),
-                headers: {
-                    Accept: 'application/json',
-                },
-            });
+            const useChunked =
+                this.chunkedUploads &&
+                this.chunkSize > 0 &&
+                this.chunkUploadUrl &&
+                file.size >= this.chunkSize &&
+                typeof file.slice === 'function';
+
+            const upload = useChunked
+                ? new ChunkedUpload({
+                      url: this.chunkUploadUrl,
+                      file,
+                      data: this.uploadParams(file, data),
+                      chunkSize: this.chunkSize,
+                  })
+                : new Upload({
+                      url: this.url,
+                      form: this.makeFormData(file, data),
+                      headers: {
+                          Accept: 'application/json',
+                      },
+                  });
 
             upload.on('progress', (progress) => {
-                this.findUpload(id).percent = progress * 100;
+                const percent = progress * 100;
+                this.findUpload(id).percent = percent;
+                this.uploadsStore.update(this.container, id, { percent });
             });
 
             return upload;
         },
 
-        makeFormData(file, data = {}) {
-            const form = new FormData();
-
-            form.append('file', file);
-
-            // Pass along the relative path of files uploaded as a directory
-            if (file.relativePath) {
-                form.append('relativePath', file.relativePath);
-            }
-
-            let parameters = {
+        uploadParams(file, data = {}) {
+            const params = {
                 ...this.extraData,
                 container: this.container,
                 folder: this.path,
                 _token: Statamic.$config.get('csrfToken'),
+                ...data,
             };
 
-            for (let key in parameters) {
-                form.append(key, parameters[key]);
+            // Pass along the relative path of files uploaded as a directory
+            if (file.relativePath) {
+                params.relativePath = file.relativePath;
             }
 
-            for (let key in data) {
-                form.append(key, data[key]);
+            return params;
+        },
+
+        makeFormData(file, data = {}) {
+            const form = new FormData();
+
+            form.append('file', file);
+
+            const params = this.uploadParams(file, data);
+
+            for (let key in params) {
+                form.append(key, params[key]);
             }
 
             return form;
@@ -274,6 +309,7 @@ export default {
         handleUploadSuccess(id, response) {
             this.$emit('upload-complete', response.data, this.uploads);
             this.uploads.splice(this.findUploadIndex(id), 1);
+            this.uploadsStore.remove(this.container, id);
 
             this.handleToasts(response._toasts ?? []);
         },
@@ -297,6 +333,7 @@ export default {
 
             upload.errorMessage = msg;
             upload.errorStatus = status;
+            this.uploadsStore.update(this.container, id, { errorMessage: msg, errorStatus: status });
             this.$emit('error', upload, this.uploads);
             this.processUploadQueue();
         },
@@ -309,6 +346,7 @@ export default {
             let file = this.findUpload(id).instance.form.get('file');
             this.addFile(file, args);
             this.uploads.splice(this.findUploadIndex(id), 1);
+            this.uploadsStore.remove(this.container, id);
         },
     },
 };

resources/js/components/fieldtypes/assets/AssetsFieldtype.vue

@@ -9,6 +9,10 @@
         <uploader
             ref="uploader"
             :container="container.id"
+            :chunked-uploads="container.chunked_uploads"
+            :chunk-size="container.chunk_size"
+            :max-filesize="container.max_filesize"
+            :chunk-upload-url="container.chunk_upload_url"
             :enabled="canUpload"
             :path="folder"
             @updated="uploadsUpdated"