diff --git a/src/@types/fileObject.ts b/src/@types/fileObject.ts index a662b51ec..4ddba81f7 100644 --- a/src/@types/fileObject.ts +++ b/src/@types/fileObject.ts @@ -20,10 +20,12 @@ export interface UrlFileObject extends BaseFileObject { url: string method: string headers?: [HeadersObject] + fileHash?: string } export interface IpfsFileObject extends BaseFileObject { hash: string + fileHash?: string } export interface S3Object { endpoint: string @@ -39,6 +41,7 @@ export interface S3FileObject extends BaseFileObject { export interface ArweaveFileObject extends BaseFileObject { transactionId: string + fileHash?: string } export interface StorageReadable { diff --git a/src/components/storage/index.ts b/src/components/storage/index.ts index 509766e1d..2242191bf 100644 --- a/src/components/storage/index.ts +++ b/src/components/storage/index.ts @@ -32,7 +32,7 @@ export abstract class Storage { } abstract validate(): [boolean, string] - abstract getDownloadUrl(): string + abstract getDownloadUrl(): Promise abstract fetchSpecificFileMetadata( fileObject: any, @@ -48,7 +48,7 @@ export abstract class Storage { // similar to all subclasses async getReadableStream(): Promise { - const input = this.getDownloadUrl() + const input = await this.getDownloadUrl() const response = await axios({ method: 'get', url: input, @@ -183,6 +183,14 @@ export abstract class Storage { return false } } + + async decryptUrl(hashedUrl: string, encryptedMethod: EncryptMethod): Promise { + const decodedUrlBuffer = Buffer.from(hashedUrl, 'base64') + const decryptedBuffer = await decryptData(decodedUrlBuffer, encryptedMethod) + const decoder = new TextDecoder() + const decryptedUrl = decoder.decode(decryptedBuffer) + return decryptedUrl + } } export class UrlStorage extends Storage { @@ -222,16 +230,26 @@ export class UrlStorage extends Storage { isFilePath(): boolean { const regex: RegExp = /^(.+)\/([^/]*)$/ // The URL should not represent a path - const { url } = this.getFile() - if (url.startsWith('http://') || url.startsWith('https://')) { + const file = this.getFile() + if ( + file.url.startsWith('http://') || + file.url.startsWith('https://') || + (file.encryptedBy && file.encryptedMethod) + ) { return false } - return regex.test(url) + return regex.test(file.url) } - getDownloadUrl(): string { + async getDownloadUrl(): Promise { if (this.validate()[0] === true) { - return this.getFile().url + const file = this.getFile() + if (file?.encryptedBy && file?.encryptedMethod) { + const decryptedUrl = await this.decryptUrl(file.url, file.encryptedMethod) + return decryptedUrl + } else { + return file.url + } } return null } @@ -240,12 +258,16 @@ export class UrlStorage extends Storage { fileObject: UrlFileObject, forceChecksum: boolean ): Promise { - const { url, method } = fileObject + const url = await this.getDownloadUrl() + const { method, fileHash } = fileObject const { contentLength, contentType, contentChecksum } = await fetchFileMetadata( url, method, forceChecksum ) + if (forceChecksum && fileHash && contentChecksum !== fileHash) { + throw new Error(`Error checksum`) + } return { valid: true, contentLength, @@ -311,20 +333,26 @@ export class ArweaveStorage extends Storage { return regex.test(transactionId) } - getDownloadUrl(): string { - return urlJoin(process.env.ARWEAVE_GATEWAY, this.getFile().transactionId) + getDownloadUrl(): Promise { + return Promise.resolve( + urlJoin(process.env.ARWEAVE_GATEWAY, this.getFile().transactionId) + ) } async fetchSpecificFileMetadata( fileObject: ArweaveFileObject, forceChecksum: boolean ): Promise { + const { fileHash } = fileObject const url = urlJoin(process.env.ARWEAVE_GATEWAY, fileObject.transactionId) const { contentLength, contentType, contentChecksum } = await fetchFileMetadata( url, 'get', forceChecksum ) + if (forceChecksum && fileHash && contentChecksum !== fileHash) { + throw new Error(`Error checksum`) + } return { valid: true, contentLength, @@ -378,25 +406,36 @@ export class IpfsStorage extends Storage { isFilePath(): boolean { const regex: RegExp = /^(.+)\/([^/]*)$/ // The CID should not represent a path - const { hash } = this.getFile() - - return regex.test(hash) + const file = this.getFile() + if (file.encryptedBy && file.encryptedMethod) { + return false + } + return regex.test(file.hash) } - getDownloadUrl(): string { - return urlJoin(process.env.IPFS_GATEWAY, urlJoin('/ipfs', this.getFile().hash)) + async getDownloadUrl(): Promise { + const file = this.getFile() + if (file?.encryptedBy && file?.encryptedMethod) { + const decryptedUrl = await this.decryptUrl(file.hash, file.encryptedMethod) + return urlJoin(process.env.IPFS_GATEWAY, urlJoin('/ipfs', decryptedUrl)) + } + return urlJoin(process.env.IPFS_GATEWAY, urlJoin('/ipfs', file.hash)) } async fetchSpecificFileMetadata( fileObject: IpfsFileObject, forceChecksum: boolean ): Promise { - const url = urlJoin(process.env.IPFS_GATEWAY, urlJoin('/ipfs', fileObject.hash)) + const url = await this.getDownloadUrl() + const { fileHash } = fileObject const { contentLength, contentType, contentChecksum } = await fetchFileMetadata( url, 'get', forceChecksum ) + if (forceChecksum && fileHash && contentChecksum !== fileHash) { + throw new Error(`Error checksum`) + } return { valid: true, contentLength, @@ -463,9 +502,9 @@ export class S3Storage extends Storage { return endpoint.includes('.') } - getDownloadUrl(): string { + getDownloadUrl(): Promise { const { s3Access } = this.getFile() - return JSON.stringify(s3Access) + return Promise.resolve(JSON.stringify(s3Access)) } async fetchDataContent(): Promise { diff --git a/src/test/data/ddo.ts b/src/test/data/ddo.ts index cc624b7f4..9a7d59f5d 100644 --- a/src/test/data/ddo.ts +++ b/src/test/data/ddo.ts @@ -489,6 +489,16 @@ export const remoteDDOTypeURLNotEncrypted = { } } +export const remoteDDOTypeURLEncrypted = { + remote: { + type: 'url', + url: '', + method: 'GET', + encryptedBy: '16Uiu2HAmN211yBiE6dF5xu8GFXV1jqZQzK5MbzBuQDspfa6qNgXF', + encryptedMethod: 'ECIES' + } +} + export const remoteDDOTypeIPFSNotEncrypted = { remote: { type: 'ipfs', @@ -499,7 +509,7 @@ export const remoteDDOTypeIPFSNotEncrypted = { export const remoteDDOTypeIPFSEncrypted = { remote: { type: 'ipfs', - hash: 'QmaD5S7TakPs3a4fijatbfqhmhhrEbCvbqGTTAp7VrZ91T', + hash: 'BK/WRmZCK4dN58E9E5ilUsmSP7q11P4ri9Y0A4WL4ealbr4crSrACw4Q7xbiYymjYw/noHErKVuOytGx9tzR8ThilK0cFodlQctQKaFewtBeYj4hhErIJkn+4MAV+dGsEnlZKT0IrmLI12MhnfRBLJ606AI0HGnOndGAiYJMhieNSfWMbvk8pYCIQ9P95OE=', encryptedBy: '16Uiu2HAmN211yBiE6dF5xu8GFXV1jqZQzK5MbzBuQDspfa6qNgXF', encryptedMethod: 'ECIES' } diff --git a/src/test/data/organizations-100.aes b/src/test/data/organizations-100.aes index 403c48d8b..9ed369728 100644 Binary files a/src/test/data/organizations-100.aes and b/src/test/data/organizations-100.aes differ diff --git a/src/test/unit/s3.storage.test.ts b/src/test/unit/s3.storage.test.ts index ea423bded..c446d9f39 100644 --- a/src/test/unit/s3.storage.test.ts +++ b/src/test/unit/s3.storage.test.ts @@ -46,8 +46,8 @@ describe('S3 Storage tests', () => { expect(parsedData).to.deep.equal({ key: 'value' }) }) - it('should fetch data from s3', () => { - const result = s3Storage.getDownloadUrl() + it('should fetch data from s3', async () => { + const result = await s3Storage.getDownloadUrl() expect(result).to.be.equal(JSON.stringify(s3Object)) }) diff --git a/src/test/unit/storage.test.ts b/src/test/unit/storage.test.ts index 4ef9fa174..4bc34fbb1 100644 --- a/src/test/unit/storage.test.ts +++ b/src/test/unit/storage.test.ts @@ -22,6 +22,8 @@ import { getConfiguration } from '../../utils/index.js' import { Readable } from 'stream' import fs from 'fs' import { expectedTimeoutFailure } from '../integration/testUtils.js' +import { encrypt } from '../../utils/crypt.js' +import urlJoin from 'url-join' let nodeId: string @@ -150,7 +152,7 @@ describe('URL Storage tests', () => { 'Error validationg the URL file: URL looks like a file path' ) }) - it('Gets download URL', () => { + it('Gets download URL', async () => { file = { type: 'url', url: 'http://someUrl.com/file.json', @@ -163,7 +165,7 @@ describe('URL Storage tests', () => { ] } storage = Storage.getStorageClass(file, config) - expect(storage.getDownloadUrl()).to.eql('http://someUrl.com/file.json') + expect(await storage.getDownloadUrl()).to.eql('http://someUrl.com/file.json') }) it('Gets readable stream', async () => { @@ -209,14 +211,14 @@ describe('Unsafe URL tests', () => { 'Error validationg the URL file: URL is marked as unsafe' ) }) - it('Should allow safe URL', () => { + it('Should allow safe URL', async () => { file = { type: 'url', url: 'https://oceanprotocol.com', method: 'get' } const storage = Storage.getStorageClass(file, config) - expect(storage.getDownloadUrl()).to.eql('https://oceanprotocol.com') + expect(await storage.getDownloadUrl()).to.eql('https://oceanprotocol.com') }) after(() => { tearDownEnvironment(previousConfiguration) @@ -239,6 +241,33 @@ describe('IPFS Storage tests', () => { config = await getConfiguration() }) + it('Gets download encrypted IPFS', async () => { + const storage = Storage.getStorageClass(file, config) + const downloadUrl = await storage.getDownloadUrl() + expect(downloadUrl).to.eql( + urlJoin( + process.env.IPFS_GATEWAY, + urlJoin('/ipfs', 'Qxchjkflsejdfklgjhfkgjkdjoiderj') + ) + ) + }) + + it('Gets download encrypted IPFS', async () => { + const hash = 'QmaD5S7TakPs3a4fijatbfqhmhhrEbCvbqGTTAp7VrZ91T' + const uint8Array = Uint8Array.from(Buffer.from(hash)) + const encryptedHash = await encrypt(uint8Array, EncryptMethod.ECIES) + const encodedHash = encryptedHash.toString('base64') + const fileDummy = { + type: 'ipfs', + hash: encodedHash, + encryptedBy: '16Uiu2HAmN211yBiE6dF5xu8GFXV1jqZQzK5MbzBuQDspfa6qNgXF', + encryptedMethod: 'ECIES' + } + const storage = Storage.getStorageClass(fileDummy, config) + const downloadUrl = await storage.getDownloadUrl() + expect(downloadUrl).to.eql(urlJoin(process.env.IPFS_GATEWAY, urlJoin('/ipfs', hash))) + }) + it('Storage instance', () => { expect(Storage.getStorageClass(file, config)).to.be.instanceOf(IpfsStorage) }) @@ -335,8 +364,7 @@ describe('URL Storage getFileInfo tests', () => { const fileInfoRequest: FileInfoRequest = { type: FileObjectType.URL } - const fileInfo = await storage.getFileInfo(fileInfoRequest) - + const fileInfo = await storage.getFileInfo(fileInfoRequest, true) assert(fileInfo[0].valid, 'File info is valid') expect(fileInfo[0].contentLength).to.equal('319520') expect(fileInfo[0].contentType).to.equal('text/plain; charset=utf-8') @@ -344,6 +372,28 @@ describe('URL Storage getFileInfo tests', () => { expect(fileInfo[0].type).to.equal('url') }) + it('Throws error when checksum fails', async () => { + const fileInfoRequest: FileInfoRequest = { + type: FileObjectType.URL + } + const config = await getConfiguration() + const storage2 = new UrlStorage( + { + type: 'url', + url: 'https://raw.githubusercontent.com/tbertinmahieux/MSongsDB/master/Tasks_Demos/CoverSongs/shs_dataset_test.txt', + method: 'get', + fileHash: 'wrong' + }, + config + ) + try { + await storage2.getFileInfo(fileInfoRequest, true) + } catch (err) { + // get log because in getFileInfo there is a console + expect(err.message).to.equal('Error checksum') + } + }) + it('Throws error when URL is missing in request', async () => { const fileInfoRequest: FileInfoRequest = { type: FileObjectType.URL } try { @@ -400,15 +450,30 @@ describe('Arweave Storage getFileInfo tests', function () { const fileInfoRequest: FileInfoRequest = { type: FileObjectType.ARWEAVE } - const fileInfo = await storage.getFileInfo(fileInfoRequest) - + const fileInfo = await storage.getFileInfo(fileInfoRequest, true) assert(fileInfo[0].valid, 'File info is valid') assert(fileInfo[0].type === FileObjectType.ARWEAVE, 'Type is incorrect') - assert( - fileInfo[0].contentType === 'text/csv; charset=utf-8', - 'Content type is incorrect' + }) + + it('Throws error when checksum fails', async () => { + const fileInfoRequest: FileInfoRequest = { + type: FileObjectType.ARWEAVE + } + const config = await getConfiguration() + const storage2 = new ArweaveStorage( + { + type: FileObjectType.ARWEAVE, + transactionId: 'gPPDyusRh2ZyFl-sQ2ODK6hAwCRBAOwp0OFKr0n23QE', + fileHash: 'wrong' + }, + config ) - assert(fileInfo[0].contentLength === '680782', 'Content length is incorrect') + try { + await storage2.getFileInfo(fileInfoRequest, true) + } catch (err) { + // get log because in getFileInfo there is a console + expect(err.message).to.equal('Error checksum') + } }) it('Throws error when transaction ID is missing in request', async () => { @@ -523,7 +588,8 @@ describe('IPFS Storage getFileInfo tests', function () { storage = new IpfsStorage( { type: FileObjectType.IPFS, - hash: 'QmRhsp7eghZtW4PktPC2wAHdKoy2LiF1n6UXMKmAhqQJUA' + hash: 'QmRhsp7eghZtW4PktPC2wAHdKoy2LiF1n6UXMKmAhqQJUA', + fileHash: '40f90cef24cf570149f27c3054752333b75081f6efc4e90ba1a2496b7adc9e48' }, config ) @@ -538,7 +604,7 @@ describe('IPFS Storage getFileInfo tests', function () { } // and only fire the test half way setTimeout(async () => { - const fileInfo = await storage.getFileInfo(fileInfoRequest) + const fileInfo = await storage.getFileInfo(fileInfoRequest, true) if (fileInfo && fileInfo.length > 0) { assert(fileInfo[0].valid, 'File info is valid') assert(fileInfo[0].type === 'ipfs', 'Type is incorrect') @@ -551,6 +617,24 @@ describe('IPFS Storage getFileInfo tests', function () { }, DEFAULT_TEST_TIMEOUT) }) + it('Throws error when checksum fails', async () => { + const storage2 = new IpfsStorage( + { + type: FileObjectType.IPFS, + hash: 'QmRhsp7eghZtW4PktPC2wAHdKoy2LiF1n6UXMKmAhqQJUA', + fileHash: 'wrong' + }, + config + ) + const fileInfoRequest: FileInfoRequest = { type: FileObjectType.IPFS } + try { + await storage2.getFileInfo(fileInfoRequest, true) + } catch (err) { + // get log because in getFileInfo there is a console + expect(err.message).to.equal('Error checksum') + } + }) + it('Throws error when hash is missing in request', async () => { const fileInfoRequest: FileInfoRequest = { type: FileObjectType.IPFS } try {