Support byte range downloads (#436)

* chore: update dependencies

- Update uuid to ^11.1.0 and add @types/uuid
- Update @autonomys/file-caching to ^1.5.1
- Update related dependencies across packages

* feat: add HTTP range request utility functions

- Add parseByteRange function for parsing Range headers
- Add createPartialStream function for creating range-aware streams
- Add utility functions for handling HTTP 206 partial content responses
- Support for byte-range requests in readable streams

* feat: implement range request support in download services

- Add ByteRange interface for range request parameters
- Implement partial download support in memory cache
- Add range-aware download methods in download service
- Support for caching partial content with byte ranges
- Add sync download functionality with range support

* feat: add HTTP range request support to download controllers

- Update download controller to handle Range headers
- Implement HTTP 206 Partial Content responses
- Add proper Content-Range and Accept-Ranges headers
- Support partial file downloads in object controllers
- Update use cases to handle byte range parameters
- Add range request support to object download endpoints

* test: add comprehensive tests for partial retrieval functionality

- Add tests for byte range parsing and validation
- Test partial content download scenarios
- Verify HTTP 206 response handling
- Add test cases for range request edge cases
- Test memory cache integration with range requests

* refactor: move to separate files related logic

* refactor: enhance download response handling with byte range support

- Introduced isExpectedDocument function to streamline document expectation checks
- Refactored handleDownloadResponseHeaders to utilize setFileResponseHeaders and setFolderResponseHeaders for improved clarity
- Added byte range handling in setFileResponseHeaders for partial content responses
- Updated content disposition and content type settings based on encryption and document expectations

* refactor: reorganize file use cases

* fix(testing): cache populated in async setting

* feat: update auto files gateway version

* update: bump auto-sdk version

* refactor: update FilesUseCases tests and imports

- Refactored tests for FilesUseCases to use getObjectInformation instead of getMetadata.
- Updated mock implementations to reflect changes in the download blocking logic.
- Cleaned up imports in files.spec.ts and index.ts for better organization and clarity.

* refactor: clean up imports in files.spec.ts

- Removed unused dbMigration import to streamline the test file and improve clarity.

* fix: update byte range header handling in getByteRange function

- Changed the header used to retrieve the byte range from 'content-range' to 'range' for improved accuracy in handling byte range requests.

* fix: enhance byte range validation in getByteRange function

- Added validation checks to ensure start and end byte values are non-negative and that the start value does not exceed the end value, improving the robustness of byte range handling.

* chore: update package dependencies to version 1.5.11

* chore: update yarn.lock to reflect package versions 1.5.11 for auto-dag-data and file-caching

Author: clostao

.gitmodules

@@ -1,4 +1,4 @@
 [submodule "files-gateway"]
 	path = submodules/files-gateway
 	url = https://github.com/autonomys/auto-files-gateway.git
-	branch = v1.0.3
+	branch = v1.0.4

auth/package.json

@@ -32,7 +32,7 @@
     "serverless-http": "^3.2.0",
     "siwe": "^3.0.0",
     "typescript": "^5.7.3",
-    "uuid": "^11.0.5"
+    "uuid": "^11.1.0"
   },
   "devDependencies": {
     "@testcontainers/postgresql": "^10.14.0",
@@ -43,7 +43,7 @@
     "@types/express": "^5.0.0",
     "@types/jest": "^29.5.14",
     "@types/multer": "^1.4.12",
-    "@types/uuid": "^9.0.0",
+    "@types/uuid": "^9.0.1",
     "db-migrate-plugin-typescript": "^2.0.0",
     "eslint": "^8.57.1",
     "eslint-config-prettier": "^9.1.0",

backend/__tests__/e2e/uploads/files.spec.ts

@@ -29,7 +29,6 @@ import { ObjectUseCases } from '../../../src/useCases/objects/object.js'
 import { uploadsRepository } from '../../../src/repositories/uploads/uploads.js'
 import { asyncIterableToPromiseOfArray } from '@autonomys/asynchronous'
 import {
-  FilesUseCases,
   NodesUseCases,
   SubscriptionsUseCases,
 } from '../../../src/useCases/index.js'
@@ -38,19 +37,19 @@ import {
   metadataRepository,
   nodesRepository,
 } from '../../../src/repositories/index.js'
-import { memoryDownloadCache } from '../../../src/services/download/memoryDownloadCache/index.js'
 import { jest } from '@jest/globals'
-import { downloadService } from '../../../src/services/download/index.js'
 import { BlockstoreUseCases } from '../../../src/useCases/uploads/blockstore.js'
 import { Rabbit } from '../../../src/drivers/rabbit.js'
 import { EventRouter } from '../../../src/services/eventRouter/index.js'
 import { MAX_RETRIES } from '../../../src/services/eventRouter/tasks.js'
+import { DownloadUseCase } from '../../../src/useCases/objects/downloads.js'
+import { downloadService } from '../../../src/services/download/index.js'
 
 const files = [
   {
     filename: 'test.pdf',
     mimeType: 'application/pdf',
-    rndBuffer: Buffer.alloc(1024 ** 2).fill(0),
+    rndBuffer: Buffer.alloc(64000).fill(0),
   },
   {
     filename: 'test.txt',
@@ -246,15 +245,28 @@ files.map((file, index) => {
     })
 
     describe('Downloading the file', () => {
+      let handleCacheMock: jest.SpiedFunction<
+        typeof downloadService.handleCache
+      >
+
       it('should be able to retrieve the file', async () => {
-        const { startDownload } = await FilesUseCases.downloadObjectByUser(
+        handleCacheMock = jest.spyOn(downloadService, 'handleCache')
+
+        const { startDownload } = await DownloadUseCase.downloadObjectByUser(
           user,
           cid,
         )
         const file = await startDownload()
         const fileArray = await asyncIterableToPromiseOfArray(file)
         const fileBuffer = Buffer.concat(fileArray)
         expect(fileBuffer).toEqual(rndBuffer)
+
+        expect(handleCacheMock).toHaveBeenCalledWith(
+          cid,
+          expect.any(Object),
+          expect.any(Object),
+          expect.any(BigInt),
+        )
       })
 
       it('should have been added an interaction', async () => {
@@ -269,26 +281,6 @@ files.map((file, index) => {
 
         expect(interactions).toHaveLength(1)
       })
-
-      it('download cache should be updated', async () => {
-        // Wait for the async context to finish
-        await new Promise((resolve) => setTimeout(resolve, 300))
-        const asyncFromDatabase = await downloadService.fsCache.get(cid)
-        expect(asyncFromDatabase).not.toBeNull()
-        const fileArrayFromDatabase = await asyncIterableToPromiseOfArray(
-          asyncFromDatabase!.data,
-        )
-        const fileBufferFromDatabase = Buffer.concat(fileArrayFromDatabase)
-        expect(fileBufferFromDatabase).toEqual(rndBuffer)
-
-        const asyncFromMemory = memoryDownloadCache.get(cid)
-        expect(asyncFromMemory).not.toBeNull()
-        const fileArrayFromMemory = await asyncIterableToPromiseOfArray(
-          asyncFromMemory!,
-        )
-        const fileBufferFromMemory = Buffer.concat(fileArrayFromMemory)
-        expect(fileBufferFromMemory).toEqual(rndBuffer)
-      })
     })
 
     describe('Object Information', () => {
@@ -358,7 +350,7 @@ files.map((file, index) => {
         )
       })
 
-      it('metadata should be updated as archived', async () => {
+      it('metadata should be updated as archived and cache should be updated', async () => {
         const processArchivalSpy = jest
           .spyOn(EventRouter, 'publish')
           .mockReturnValue()
@@ -368,9 +360,6 @@ files.map((file, index) => {
         const metadata = await metadataRepository.getMetadata(cid)
         expect(metadata).not.toBeNull()
 
-        expect(memoryDownloadCache.has(cid)).toBe(true)
-        expect(downloadService.fsCache.get(cid)).not.toBeNull()
-
         expect(processArchivalSpy).toHaveBeenCalledWith({
           id: 'object-archived',
           params: {

backend/__tests__/e2e/uploads/folder.spec.ts

@@ -14,11 +14,7 @@ import {
   UploadType,
   UserWithOrganization,
 } from '@auto-drive/models'
-import {
-  FilesUseCases,
-  NodesUseCases,
-  ObjectUseCases,
-} from '../../../src/useCases/index.js'
+import { NodesUseCases, ObjectUseCases } from '../../../src/useCases/index.js'
 import { UploadsUseCases } from '../../../src/useCases/uploads/uploads.js'
 import { dbMigration } from '../../utils/dbMigrate.js'
 import { PreconditionError } from '../../utils/error.js'
@@ -37,6 +33,7 @@ import { asyncIterableToPromiseOfArray } from '@autonomys/asynchronous'
 import PizZip from 'pizzip'
 import { BlockstoreUseCases } from '../../../src/useCases/uploads/blockstore.js'
 import { Rabbit } from '../../../src/drivers/rabbit.js'
+import { DownloadUseCase } from '../../../src/useCases/objects/downloads.js'
 
 describe('Folder Upload', () => {
   let user: UserWithOrganization
@@ -365,7 +362,7 @@ describe('Folder Upload', () => {
     })
 
     it('should be able to download folder as zip', async () => {
-      const zip = await FilesUseCases.downloadObjectByUser(user, folderCID)
+      const zip = await DownloadUseCase.downloadObjectByUser(user, folderCID)
       const dataStream = await zip.startDownload()
       const zipArray = await asyncIterableToPromiseOfArray(dataStream)
       const zipBuffer = Buffer.concat(zipArray)

backend/__tests__/unit/useCases/files.spec.ts

@@ -1,7 +1,10 @@
 import { jest } from '@jest/globals'
 import { ObjectUseCases } from '../../../src/useCases/objects/object.js'
-import { FilesUseCases } from '../../../src/useCases/objects/files.js'
+import { ObjectStatus } from '@auto-drive/models'
+import { ByteRange } from '@autonomys/file-caching'
+import { DownloadUseCase } from '../../../src/useCases/objects/downloads.js'
 import { OffchainMetadata } from '@autonomys/auto-dag-data'
+import { FilesUseCases } from '../../../src/useCases/index.js'
 
 jest.unstable_mockModule('../../../src/useCases/objects/object.js', () => ({
   ObjectUseCases: {
@@ -27,10 +30,25 @@ describe('FilesUseCases', () => {
       chunks: [],
     }
 
-    jest.spyOn(ObjectUseCases, 'getMetadata').mockResolvedValue(metadata)
+    jest.spyOn(ObjectUseCases, 'getObjectInformation').mockResolvedValue({
+      metadata,
+      tags: [],
+      cid: '<cid>',
+      createdAt: '',
+      status: ObjectStatus.Processing,
+      uploadState: {
+        uploadedNodes: 0,
+        totalNodes: 0,
+        archivedNodes: 0,
+        minimumBlockDepth: 0,
+        maximumBlockDepth: 0,
+      },
+      owners: [],
+      publishedObjectId: null,
+    })
     jest.spyOn(ObjectUseCases, 'shouldBlockDownload').mockResolvedValue(false)
 
-    const result = await FilesUseCases.downloadObjectByAnonymous(
+    const result = await DownloadUseCase.downloadObjectByAnonymous(
       metadata.dataCid,
     )
 
@@ -44,21 +62,108 @@ describe('FilesUseCases', () => {
       type: 'file',
     }
 
-    const metadata: OffchainMetadata = {
-      totalSize: 100n,
-      type: 'file',
-      dataCid: 'test-cid',
-      totalChunks: 1,
-      chunks: [],
-    }
-
-    jest.spyOn(ObjectUseCases, 'getMetadata').mockResolvedValue(metadata)
+    jest.spyOn(ObjectUseCases, 'getObjectInformation').mockResolvedValue({
+      metadata: {
+        totalSize: 100n,
+        type: 'file',
+        dataCid: 'test-cid',
+        totalChunks: 1,
+        chunks: [],
+      },
+      tags: ['insecure'],
+      cid: '',
+      createdAt: '',
+      status: ObjectStatus.Processing,
+      uploadState: {
+        uploadedNodes: 0,
+        totalNodes: 0,
+        archivedNodes: 0,
+        minimumBlockDepth: 0,
+        maximumBlockDepth: 0,
+      },
+      owners: [],
+      publishedObjectId: null,
+    })
     jest.spyOn(ObjectUseCases, 'shouldBlockDownload').mockResolvedValue(true)
 
     await expect(
-      FilesUseCases.downloadObjectByAnonymous(mockFile.cid, ['insecure']),
-    ).rejects.toThrow(
-      new Error('File download is blocked by blocking tags or is banned'),
-    )
+      DownloadUseCase.downloadObjectByAnonymous(mockFile.cid, {
+        blockingTags: ['insecure'],
+      }),
+    ).rejects.toThrow(new Error('File is blocked'))
+  })
+
+  describe('getNodesForPartialRetrieval', () => {
+    const tests = [
+      {
+        name: 'should return the nodes for a partial retrieval',
+        nodes: [
+          { cid: 'test-cid', size: 100n },
+          { cid: 'test-cid-2', size: 100n },
+          { cid: 'test-cid-3', size: 50n },
+        ],
+        byteRange: [0, 99] as ByteRange,
+        expectedNodes: ['test-cid'],
+        expectedFirstNodeFileOffset: 0,
+      },
+      {
+        name: 'should return the nodes for a partial retrieval',
+        nodes: [
+          { cid: 'test-cid', size: 100n },
+          { cid: 'test-cid-2', size: 100n },
+          { cid: 'test-cid-3', size: 50n },
+        ],
+        byteRange: [0, 100] as ByteRange,
+        expectedNodes: ['test-cid', 'test-cid-2'],
+        expectedFirstNodeFileOffset: 0,
+      },
+      {
+        name: 'should return the nodes for a partial retrieval',
+        nodes: [
+          { cid: 'test-cid', size: 100n },
+          { cid: 'test-cid-2', size: 100n },
+          { cid: 'test-cid-3', size: 50n },
+        ],
+        byteRange: [1, 100] as ByteRange,
+        expectedNodes: ['test-cid', 'test-cid-2'],
+        expectedFirstNodeFileOffset: 0,
+      },
+      {
+        name: 'should return the nodes for a partial retrieval',
+        nodes: [
+          { cid: 'test-cid', size: 100n },
+          { cid: 'test-cid-2', size: 100n },
+          { cid: 'test-cid-3', size: 50n },
+        ],
+        byteRange: [0, undefined] as ByteRange,
+        expectedNodes: ['test-cid', 'test-cid-2', 'test-cid-3'],
+        expectedFirstNodeFileOffset: 0,
+      },
+      {
+        name: 'should return the nodes for a partial retrieval',
+        nodes: [
+          { cid: 'test-cid', size: 100n },
+          { cid: 'test-cid-2', size: 100n },
+          { cid: 'test-cid-3', size: 50n },
+        ],
+        byteRange: [100, undefined] as ByteRange,
+        expectedNodes: ['test-cid-2', 'test-cid-3'],
+        expectedFirstNodeFileOffset: 100,
+      },
+    ]
+
+    for (const test of tests) {
+      it(`${test.name} (byteRange=[${test.byteRange[0]},${test.byteRange[1]}])`, async () => {
+        const result = await FilesUseCases.getNodesForPartialRetrieval(
+          test.nodes,
+          test.byteRange,
+        )
+
+        expect(result.nodes).toEqual(test.expectedNodes)
+        expect(result.firstNodeFileOffset).toEqual(
+          test.expectedFirstNodeFileOffset,
+        )
+      })
+    }
   })
 })

backend/package.json

@@ -21,12 +21,12 @@
   "dependencies": {
     "@auto-drive/models": "workspace:*",
     "@auto-files/rpc-apis": "workspace:*",
-    "@autonomys/asynchronous": "^1.5.1",
-    "@autonomys/auto-dag-data": "^1.5.1",
-    "@autonomys/auto-drive": "^1.5.1",
-    "@autonomys/auto-files": "^1.5.2",
-    "@autonomys/file-caching": "^1.5.1",
-    "@autonomys/rpc": "^1.5.1",
+    "@autonomys/asynchronous": "^1.5.11",
+    "@autonomys/auto-dag-data": "^1.5.11",
+    "@autonomys/auto-drive": "^1.5.11",
+    "@autonomys/auto-files": "^1.5.11",
+    "@autonomys/file-caching": "^1.5.11",
+    "@autonomys/rpc": "^1.5.11",
     "@keyvhq/sqlite": "^2.1.6",
     "@polkadot/api": "^15.8.1",
     "@polkadot/types": "^15.8.1",
@@ -52,7 +52,7 @@
     "pg-format": "^1.0.4",
     "pizzip": "^3.1.7",
     "tar": "^7.4.3",
-    "uuid": "^10.0.0",
+    "uuid": "^11.1.0",
     "websocket": "^1.0.35",
     "zod": "^3.23.8"
   },