Skip to content

Commit ddc5163

Browse files
fenosfenos
authored
fix: allow copying objects to same destination and overwrite metadata (#622)
1 parent cf23e7d commit ddc5163

File tree

8 files changed

+158
-31
lines changed

8 files changed

+158
-31
lines changed

‎src/http/routes/object/copyObject.ts

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ import { FromSchema } from 'json-schema-to-ts'
33
import { createDefaultSchema } from '../../routes-helper'
44
import { AuthenticatedRequest } from '../../types'
55
import { ROUTE_OPERATIONS } from '../operations'
6+
import { parseUserMetadata } from '@storage/uploader'
7+
import { objectSchema } from '@storage/schemas'
68

79
const copyRequestBodySchema = {
810
type: 'object',
@@ -11,14 +13,23 @@ const copyRequestBodySchema = {
1113
sourceKey: { type: 'string', examples: ['folder/source.png'] },
1214
destinationBucket: { type: 'string', examples: ['users'] },
1315
destinationKey: { type: 'string', examples: ['folder/destination.png'] },
16+
metadata: {
17+
type: 'object',
18+
properties: {
19+
cacheControl: { type: 'string' },
20+
mimetype: { type: 'string' },
21+
},
22+
},
1423
copyMetadata: { type: 'boolean', examples: [true] },
1524
},
1625
required: ['sourceKey', 'bucketId', 'destinationKey'],
1726
} as const
1827
const successResponseSchema = {
1928
type: 'object',
2029
properties: {
30+
Id: { type: 'string' },
2131
Key: { type: 'string', examples: ['folder/destination.png'] },
32+
...objectSchema.properties,
2233
},
2334
required: ['Key'],
2435
}
@@ -48,21 +59,29 @@ export default async function routes(fastify: FastifyInstance) {
4859
},
4960
},
5061
async (request, response) => {
51-
const { sourceKey, destinationKey, bucketId, destinationBucket } = request.body
62+
const { sourceKey, destinationKey, bucketId, destinationBucket, metadata } = request.body
5263

5364
const destinationBucketId = destinationBucket || bucketId
65+
const userMetadata = request.headers['x-metadata']
5466

5567
const result = await request.storage.from(bucketId).copyObject({
5668
sourceKey,
5769
destinationBucket: destinationBucketId,
5870
destinationKey,
5971
owner: request.owner,
72+
userMetadata:
73+
typeof userMetadata === 'string' ? parseUserMetadata(userMetadata) : undefined,
74+
metadata: metadata,
6075
copyMetadata: request.body.copyMetadata ?? true,
76+
upsert: request.headers['x-upsert'] === 'true',
6177
})
6278

6379
return response.status(result.httpStatusCode ?? 200).send({
80+
// Deprecated, remove in next major
6481
Id: result.destObject.id,
6582
Key: `${destinationBucketId}/${destinationKey}`,
83+
84+
...result.destObject,
6685
})
6786
}
6887
)

‎src/http/routes/s3/commands/copy-object.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { S3ProtocolHandler } from '@storage/protocols/s3/s3-handler'
22
import { S3Router } from '../router'
33
import { ROUTE_OPERATIONS } from '../../operations'
4+
import { MetadataDirective } from '@aws-sdk/client-s3'
45

56
const CopyObjectInput = {
67
summary: 'Copy Object',
@@ -20,6 +21,7 @@ const CopyObjectInput = {
2021
'x-amz-copy-source-if-modified-since': { type: 'string' },
2122
'x-amz-copy-source-if-none-match': { type: 'string' },
2223
'x-amz-copy-source-if-unmodified-since': { type: 'string' },
24+
'x-amz-metadata-directive': { type: 'string' },
2325
'content-encoding': { type: 'string' },
2426
'content-type': { type: 'string' },
2527
'cache-control': { type: 'string' },
@@ -42,6 +44,7 @@ export default function CopyObject(s3Router: S3Router) {
4244
CopySource: req.Headers['x-amz-copy-source'],
4345
ContentType: req.Headers['content-type'],
4446
CacheControl: req.Headers['cache-control'],
47+
MetadataDirective: req.Headers['x-amz-metadata-directive'] as MetadataDirective | undefined,
4548
Expires: req.Headers.expires ? new Date(req.Headers.expires) : undefined,
4649
ContentEncoding: req.Headers['content-encoding'],
4750
CopySourceIfMatch: req.Headers['x-amz-copy-source-if-match'],

‎src/storage/backend/s3/adapter.ts

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -231,7 +231,7 @@ export class S3Backend implements StorageBackendAdapter {
231231
try {
232232
const command = new CopyObjectCommand({
233233
Bucket: bucket,
234-
CopySource: `${bucket}/${withOptionalVersion(source, version)}`,
234+
CopySource: encodeURIComponent(`${bucket}/${withOptionalVersion(source, version)}`),
235235
Key: withOptionalVersion(destination, destinationVersion),
236236
CopySourceIfMatch: conditions?.ifMatch,
237237
CopySourceIfNoneMatch: conditions?.ifNoneMatch,

‎src/storage/object.ts

Lines changed: 19 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -263,6 +263,9 @@ export class ObjectStorage {
263263
* @param owner
264264
* @param conditions
265265
* @param copyMetadata
266+
* @param upsert
267+
* @param fileMetadata
268+
* @param userMetadata
266269
*/
267270
async copyObject({
268271
sourceKey,
@@ -288,16 +291,14 @@ export class ObjectStorage {
288291
'bucket_id,metadata,user_metadata,version'
289292
)
290293

291-
if (s3SourceKey === s3DestinationKey) {
292-
return {
293-
destObject: originObject,
294-
httpStatusCode: 200,
295-
eTag: originObject.metadata?.eTag,
296-
lastModified: originObject.metadata?.lastModified
297-
? new Date(originObject.metadata.lastModified as string)
298-
: undefined,
299-
}
300-
}
294+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
295+
const baseMetadata = originObject.metadata || {}
296+
const destinationMetadata = copyMetadata
297+
? baseMetadata
298+
: {
299+
...baseMetadata,
300+
...(fileMetadata || {}),
301+
}
301302

302303
await this.uploader.canUpload({
303304
bucketId: destinationBucket,
@@ -313,7 +314,7 @@ export class ObjectStorage {
313314
originObject.version,
314315
s3DestinationKey,
315316
newVersion,
316-
fileMetadata,
317+
destinationMetadata,
317318
conditions
318319
)
319320

@@ -334,14 +335,16 @@ export class ObjectStorage {
334335
}
335336
)
336337

337-
const destinationMetadata = copyMetadata ? originObject.metadata : fileMetadata || {}
338-
339338
const destinationObject = await db.upsertObject({
340339
...originObject,
341340
bucket_id: destinationBucket,
342341
name: destinationKey,
343342
owner,
344-
metadata: destinationMetadata,
343+
metadata: {
344+
...destinationMetadata,
345+
lastModified: copyResult.lastModified,
346+
eTag: copyResult.eTag,
347+
},
345348
user_metadata: copyMetadata ? originObject.user_metadata : userMetadata,
346349
version: newVersion,
347350
})
@@ -402,9 +405,8 @@ export class ObjectStorage {
402405
mustBeValidKey(destinationObjectName)
403406

404407
const newVersion = randomUUID()
405-
const s3SourceKey = encodeURIComponent(
406-
`${this.db.tenantId}/${this.bucketId}/${sourceObjectName}`
407-
)
408+
const s3SourceKey = `${this.db.tenantId}/${this.bucketId}/${sourceObjectName}`
409+
408410
const s3DestinationKey = `${this.db.tenantId}/${destinationBucket}/${destinationObjectName}`
409411

410412
await this.db.testPermission((db) => {

‎src/storage/protocols/s3/s3-handler.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1012,6 +1012,11 @@ export class S3ProtocolHandler {
10121012
throw ERRORS.MissingParameter('CopySource')
10131013
}
10141014

1015+
if (!command.MetadataDirective) {
1016+
// default metadata directive is copy
1017+
command.MetadataDirective = 'COPY'
1018+
}
1019+
10151020
const copyResult = await this.storage.from(sourceBucket).copyObject({
10161021
sourceKey,
10171022
destinationBucket: Bucket,

‎src/storage/uploader.ts

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -354,8 +354,6 @@ export async function fileUploadFromRequest(
354354
mimeType = request.headers['content-type'] || 'application/octet-stream'
355355
cacheControl = request.headers['cache-control'] ?? 'no-cache'
356356

357-
const customMd = request.headers['x-metadata']
358-
359357
if (
360358
options.allowedMimeTypes &&
361359
options.allowedMimeTypes.length > 0 &&
@@ -364,13 +362,10 @@ export async function fileUploadFromRequest(
364362
validateMimeType(mimeType, options.allowedMimeTypes)
365363
}
366364

365+
const customMd = request.headers['x-metadata']
366+
367367
if (typeof customMd === 'string') {
368-
try {
369-
const json = Buffer.from(customMd, 'base64').toString('utf8')
370-
userMetadata = JSON.parse(json)
371-
} catch (e) {
372-
// no-op
373-
}
368+
userMetadata = parseUserMetadata(customMd)
374369
}
375370
isTruncated = () => {
376371
// @todo more secure to get this from the stream or from s3 in the next step
@@ -387,6 +382,16 @@ export async function fileUploadFromRequest(
387382
}
388383
}
389384

385+
export function parseUserMetadata(metadata: string) {
386+
try {
387+
const json = Buffer.from(metadata, 'base64').toString('utf8')
388+
return JSON.parse(json) as Record<string, string>
389+
} catch (e) {
390+
// no-op
391+
return undefined
392+
}
393+
}
394+
390395
export async function getStandardMaxFileSizeLimit(
391396
tenantId: string,
392397
bucketSizeLimit?: number | null

‎src/test/object.test.ts

Lines changed: 68 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1216,7 +1216,8 @@ describe('testing copy object', () => {
12161216
})
12171217
expect(response.statusCode).toBe(200)
12181218
expect(S3Backend.prototype.copyObject).toBeCalled()
1219-
expect(response.body).toBe(`{"Key":"bucket2/authenticated/casestudy11.png"}`)
1219+
const jsonResponse = await response.json()
1220+
expect(jsonResponse.Key).toBe(`bucket2/authenticated/casestudy11.png`)
12201221
})
12211222

12221223
test('can copy objects across buckets', async () => {
@@ -1235,7 +1236,9 @@ describe('testing copy object', () => {
12351236
})
12361237
expect(response.statusCode).toBe(200)
12371238
expect(S3Backend.prototype.copyObject).toBeCalled()
1238-
expect(response.body).toBe(`{"Key":"bucket3/authenticated/casestudy11.png"}`)
1239+
const jsonResponse = await response.json()
1240+
1241+
expect(jsonResponse.Key).toBe(`bucket3/authenticated/casestudy11.png`)
12391242
})
12401243

12411244
test('can copy objects keeping their metadata', async () => {
@@ -1255,7 +1258,8 @@ describe('testing copy object', () => {
12551258
})
12561259
expect(response.statusCode).toBe(200)
12571260
expect(S3Backend.prototype.copyObject).toBeCalled()
1258-
expect(response.body).toBe(`{"Key":"bucket2/authenticated/${copiedKey}"}`)
1261+
const jsonResponse = response.json()
1262+
expect(jsonResponse.Key).toBe(`bucket2/authenticated/${copiedKey}`)
12591263

12601264
const conn = await getSuperuserPostgrestClient()
12611265
const object = await conn
@@ -1271,6 +1275,65 @@ describe('testing copy object', () => {
12711275
})
12721276
})
12731277

1278+
test('can copy objects to itself overwriting their metadata', async () => {
1279+
const copiedKey = 'casestudy-2349.png'
1280+
const response = await app().inject({
1281+
method: 'POST',
1282+
url: '/object/copy',
1283+
headers: {
1284+
authorization: `Bearer ${process.env.AUTHENTICATED_KEY}`,
1285+
'x-upsert': 'true',
1286+
'x-metadata': Buffer.from(
1287+
JSON.stringify({
1288+
newMetadata: 'test1',
1289+
})
1290+
).toString('base64'),
1291+
},
1292+
payload: {
1293+
bucketId: 'bucket2',
1294+
sourceKey: `authenticated/${copiedKey}`,
1295+
destinationKey: `authenticated/${copiedKey}`,
1296+
metadata: {
1297+
cacheControl: 'max-age=999',
1298+
mimetype: 'image/gif',
1299+
},
1300+
copyMetadata: false,
1301+
},
1302+
})
1303+
expect(response.statusCode).toBe(200)
1304+
expect(S3Backend.prototype.copyObject).toBeCalled()
1305+
const parsedBody = JSON.parse(response.body)
1306+
1307+
expect(parsedBody.Key).toBe(`bucket2/authenticated/${copiedKey}`)
1308+
expect(parsedBody.name).toBe(`authenticated/${copiedKey}`)
1309+
expect(parsedBody.bucket_id).toBe(`bucket2`)
1310+
expect(parsedBody.metadata).toEqual(
1311+
expect.objectContaining({
1312+
cacheControl: 'max-age=999',
1313+
mimetype: 'image/gif',
1314+
})
1315+
)
1316+
1317+
const conn = await getSuperuserPostgrestClient()
1318+
const object = await conn
1319+
.table('objects')
1320+
.select('*')
1321+
.where('bucket_id', 'bucket2')
1322+
.where('name', `authenticated/${copiedKey}`)
1323+
.first()
1324+
1325+
expect(object).not.toBeFalsy()
1326+
expect(object.user_metadata).toEqual({
1327+
newMetadata: 'test1',
1328+
})
1329+
expect(object.metadata).toEqual(
1330+
expect.objectContaining({
1331+
cacheControl: 'max-age=999',
1332+
mimetype: 'image/gif',
1333+
})
1334+
)
1335+
})
1336+
12741337
test('can copy objects excluding their metadata', async () => {
12751338
const copiedKey = 'casestudy-2450.png'
12761339
const response = await app().inject({
@@ -1288,7 +1351,8 @@ describe('testing copy object', () => {
12881351
})
12891352
expect(response.statusCode).toBe(200)
12901353
expect(S3Backend.prototype.copyObject).toBeCalled()
1291-
expect(response.body).toBe(`{"Key":"bucket2/authenticated/${copiedKey}"}`)
1354+
const jsonResponse = response.json()
1355+
expect(jsonResponse.Key).toBe(`bucket2/authenticated/${copiedKey}`)
12921356

12931357
const conn = await getSuperuserPostgrestClient()
12941358
const object = await conn

‎src/test/s3-protocol.test.ts

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -976,6 +976,7 @@ describe('S3 Protocol', () => {
976976
CopySource: `${bucketName}/test-copy-1.jpg`,
977977
ContentType: 'image/png',
978978
CacheControl: 'max-age=2009',
979+
MetadataDirective: 'REPLACE',
979980
})
980981

981982
const resp = await client.send(copyObjectCommand)
@@ -991,6 +992,34 @@ describe('S3 Protocol', () => {
991992
expect(headObj.CacheControl).toBe('max-age=2009')
992993
})
993994

995+
it('will allow copying an object in the same path, just altering its metadata', async () => {
996+
const bucketName = await createBucket(client)
997+
const fileName = 'test-copy-1.jpg'
998+
999+
await uploadFile(client, bucketName, fileName, 1)
1000+
1001+
const copyObjectCommand = new CopyObjectCommand({
1002+
Bucket: bucketName,
1003+
Key: fileName,
1004+
CopySource: `${bucketName}/${fileName}`,
1005+
ContentType: 'image/png',
1006+
CacheControl: 'max-age=2009',
1007+
MetadataDirective: 'REPLACE',
1008+
})
1009+
1010+
const resp = await client.send(copyObjectCommand)
1011+
expect(resp.CopyObjectResult?.ETag).toBeTruthy()
1012+
1013+
const headObjectCommand = new HeadObjectCommand({
1014+
Bucket: bucketName,
1015+
Key: fileName,
1016+
})
1017+
1018+
const headObj = await client.send(headObjectCommand)
1019+
expect(headObj.ContentType).toBe('image/png')
1020+
expect(headObj.CacheControl).toBe('max-age=2009')
1021+
})
1022+
9941023
it('will not be able to copy an object that doesnt exists', async () => {
9951024
const bucketName1 = await createBucket(client)
9961025
await uploadFile(client, bucketName1, 'test-copy-1.jpg', 1)

0 commit comments

Comments
 (0)