From 84b0f91b44f796a60568a5bd2071c651cbd07556 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20H=E1=BA=A3i=20Quang?= Date: Sat, 9 Nov 2024 18:49:26 +0700 Subject: [PATCH 1/2] support attachments s3 --- docker-compose.yml | 10 +- server/.env.sample | 7 + server/api/helpers/attachments/delete-one.js | 13 +- .../get-simple-storage-service-client.js | 45 ++++ .../attachments/process-uploaded-file.js | 64 ++++++ server/api/models/Attachment.js | 27 ++- server/config/custom.js | 10 + ...41109164629_add_url_to_attachment_table.js | 15 ++ server/package-lock.json | 214 +++++++++++++++++- server/package.json | 1 + 10 files changed, 392 insertions(+), 14 deletions(-) create mode 100644 server/api/helpers/attachments/get-simple-storage-service-client.js create mode 100644 server/db/migrations/20241109164629_add_url_to_attachment_table.js diff --git a/docker-compose.yml b/docker-compose.yml index ed6156972..1f586c442 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -80,6 +80,14 @@ services: # - TELEGRAM_BOT_TOKEN= # - TELEGRAM_CHAT_ID= # - TELEGRAM_THREAD_ID= + + # Attachments S3 + # - ATTACHMENTS_S3=true + # - ATTACHMENTS_S3_REGION= + # - ATTACHMENTS_S3_ENDPOINT= + # - ATTACHMENTS_S3_BUCKET= + # - ATTACHMENTS_S3_ACCESS_KEY= + # - ATTACHMENTS_S3_SECRET_KEY= depends_on: postgres: condition: service_healthy @@ -93,7 +101,7 @@ services: - POSTGRES_DB=planka - POSTGRES_HOST_AUTH_METHOD=trust healthcheck: - test: ["CMD-SHELL", "pg_isready -U postgres -d planka"] + test: ['CMD-SHELL', 'pg_isready -U postgres -d planka'] interval: 10s timeout: 5s retries: 5 diff --git a/server/.env.sample b/server/.env.sample index b03819bb2..3d30a14bf 100644 --- a/server/.env.sample +++ b/server/.env.sample @@ -75,3 +75,10 @@ SECRET_KEY=notsecretkey ## Do not edit this TZ=UTC + +# ATTACHMENTS_S3=true +# ATTACHMENTS_S3_REGION= +# ATTACHMENTS_S3_ENDPOINT= +# ATTACHMENTS_S3_BUCKET= +# ATTACHMENTS_S3_ACCESS_KEY= +# ATTACHMENTS_S3_SECRET_KEY= diff --git a/server/api/helpers/attachments/delete-one.js b/server/api/helpers/attachments/delete-one.js index 9d08cb4f4..afba75937 100644 --- a/server/api/helpers/attachments/delete-one.js +++ b/server/api/helpers/attachments/delete-one.js @@ -51,7 +51,18 @@ module.exports = { if (attachment) { try { - rimraf.sync(path.join(sails.config.custom.attachmentsPath, attachment.dirname)); + const type = attachment.type || 'local'; + if (type === 's3') { + const client = await sails.helpers.attachments.getSimpleStorageServiceClient(); + if (client) { + const file1 = `${attachment.dirname}/${attachment.filename}`; + const file2 = `${attachment.dirname}/thumbnails/cover-256.png`; + await client.delete({ Key: file1 }); + await client.delete({ Key: file2 }); + } + } else { + rimraf.sync(path.join(sails.config.custom.attachmentsPath, attachment.dirname)); + } } catch (error) { console.warn(error.stack); // eslint-disable-line no-console } diff --git a/server/api/helpers/attachments/get-simple-storage-service-client.js b/server/api/helpers/attachments/get-simple-storage-service-client.js new file mode 100644 index 000000000..b65020443 --- /dev/null +++ b/server/api/helpers/attachments/get-simple-storage-service-client.js @@ -0,0 +1,45 @@ +const AWS = require('aws-sdk'); + +class S3Client { + constructor(options) { + AWS.config.update({ + accessKeyId: options.accessKeyId, + secretAccessKey: options.secretAccessKey, + region: options.region, + }); + this.bucket = options.bucket; + this.client = new AWS.S3({ + endpoint: options.endpoint, + }); + } + + upload({ Key, Body, ContentType }) { + return this.client + .upload({ + Bucket: this.bucket, + Key, + Body, + ContentType, + ACL: 'public-read', + }) + .promise(); + } + + delete({ Key }) { + return this.client + .deleteObject({ + Bucket: this.bucket, + Key, + }) + .promise(); + } +} + +module.exports = { + fn() { + if (sails.config.custom.attachmentsS3) { + return new S3Client(sails.config.custom.attachmentsS3); + } + return null; + }, +}; diff --git a/server/api/helpers/attachments/process-uploaded-file.js b/server/api/helpers/attachments/process-uploaded-file.js index 786c77973..93e443f0f 100644 --- a/server/api/helpers/attachments/process-uploaded-file.js +++ b/server/api/helpers/attachments/process-uploaded-file.js @@ -23,6 +23,69 @@ module.exports = { const rootPath = path.join(sails.config.custom.attachmentsPath, dirname); const filePath = path.join(rootPath, filename); + if (sails.config.custom.attachmentsS3) { + const client = await sails.helpers.attachments.getSimpleStorageServiceClient(); + const s3Image = await client.upload({ + Body: fs.createReadStream(inputs.file.fd), + Key: `${dirname}/${filename}`, + ContentType: inputs.file.type, + }); + + let image = sharp(inputs.file.fd, { + animated: true, + }); + + let metadata; + try { + metadata = await image.metadata(); + } catch (error) {} // eslint-disable-line no-empty + + const fileData = { + type: 's3', + dirname, + filename, + thumb: null, + image: null, + url: s3Image.Location, + name: inputs.file.filename, + }; + + if (metadata && !['svg', 'pdf'].includes(metadata.format)) { + let { width, pageHeight: height = metadata.height } = metadata; + if (metadata.orientation && metadata.orientation > 4) { + [image, width, height] = [image.rotate(), height, width]; + } + + const isPortrait = height > width; + const thumbnailsExtension = metadata.format === 'jpeg' ? 'jpg' : metadata.format; + + try { + const resizeBuffer = await image + .resize( + 256, + isPortrait ? 320 : undefined, + width < 256 || (isPortrait && height < 320) + ? { + kernel: sharp.kernel.nearest, + } + : undefined, + ) + .toBuffer(); + const s3Thumb = await client.upload({ + Key: `${dirname}/thumbnails/cover-256.${thumbnailsExtension}`, + Body: resizeBuffer, + ContentType: inputs.file.type, + }); + fileData.thumb = s3Thumb.Location; + fileData.image = { width, height }; + } catch (error1) { + console.warn(error2.stack); // eslint-disable-line no-console + } + } + + return fileData; + } + fs.mkdirSync(rootPath); await moveFile(inputs.file.fd, filePath); @@ -36,6 +99,7 @@ module.exports = { } catch (error) {} // eslint-disable-line no-empty const fileData = { + type: 'local', dirname, filename, image: null, diff --git a/server/api/models/Attachment.js b/server/api/models/Attachment.js index 508272cbe..fb09f2202 100644 --- a/server/api/models/Attachment.js +++ b/server/api/models/Attachment.js @@ -26,6 +26,15 @@ module.exports = { type: 'string', required: true, }, + type: { + type: 'string', + }, + url: { + type: 'string', + }, + thumb: { + type: 'string', + }, // ╔═╗╔╦╗╔╗ ╔═╗╔╦╗╔═╗ // ║╣ ║║║╠╩╗║╣ ║║╚═╗ @@ -48,12 +57,20 @@ module.exports = { }, customToJSON() { - return { - ..._.omit(this, ['dirname', 'filename', 'image.thumbnailsExtension']), - url: `${sails.config.custom.attachmentsUrl}/${this.id}/download/${this.filename}`, - coverUrl: this.image + let { url, thumb } = this; + if (!url) { + url = `${sails.config.custom.attachmentsUrl}/${this.id}/download/${this.filename}`; + } + if (!thumb) { + thumb = this.image ? `${sails.config.custom.attachmentsUrl}/${this.id}/download/thumbnails/cover-256.${this.image.thumbnailsExtension}` - : null, + : null; + } + + return { + ..._.omit(this, ['type', 'dirname', 'filename', 'image.thumbnailsExtension']), + url, + coverUrl: thumb, }; }, }; diff --git a/server/config/custom.js b/server/config/custom.js index 164996dba..fd5e6b2a2 100644 --- a/server/config/custom.js +++ b/server/config/custom.js @@ -36,6 +36,16 @@ module.exports.custom = { projectBackgroundImagesPath: path.join(sails.config.paths.public, 'project-background-images'), projectBackgroundImagesUrl: `${process.env.BASE_URL}/project-background-images`, + attachmentsS3: + process.env.ATTACHMENTS_S3 === 'true' + ? { + accessKeyId: process.env.ATTACHMENTS_S3_ACCESS_KEY, + secretAccessKey: process.env.ATTACHMENTS_S3_SECRET_KEY, + region: process.env.ATTACHMENTS_S3_REGION, + endpoint: process.env.ATTACHMENTS_S3_ENDPOINT, + bucket: process.env.ATTACHMENTS_S3_BUCKET, + } + : null, attachmentsPath: path.join(sails.config.appPath, 'private', 'attachments'), attachmentsUrl: `${process.env.BASE_URL}/attachments`, diff --git a/server/db/migrations/20241109164629_add_url_to_attachment_table.js b/server/db/migrations/20241109164629_add_url_to_attachment_table.js new file mode 100644 index 000000000..14d5fdcdb --- /dev/null +++ b/server/db/migrations/20241109164629_add_url_to_attachment_table.js @@ -0,0 +1,15 @@ +module.exports.up = async (knex) => { + return knex.schema.table('attachment', (table) => { + table.text('type'); + table.text('url'); + table.text('thumb'); + }); +}; + +module.exports.down = async (knex) => { + return knex.schema.table('attachment', (table) => { + table.dropColumn('type'); + table.dropColumn('url'); + table.dropColumn('thumb'); + }); +}; diff --git a/server/package-lock.json b/server/package-lock.json index 0ae84bfb0..9ee11aa17 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -6,6 +6,7 @@ "": { "name": "planka-server", "dependencies": { + "aws-sdk": "^2.1692.0", "bcrypt": "^5.1.1", "dotenv": "^16.4.5", "dotenv-cli": "^7.4.2", @@ -754,6 +755,11 @@ "ms": "2.0.0" } }, + "node_modules/@sailshq/router/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + }, "node_modules/@sailshq/router/node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -1110,7 +1116,6 @@ "version": "1.0.7", "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.7.tgz", "integrity": "sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ==", - "dev": true, "dependencies": { "possible-typed-array-names": "^1.0.0" }, @@ -1121,11 +1126,59 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/aws-sdk": { + "version": "2.1692.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1692.0.tgz", + "integrity": "sha512-x511uiJ/57FIsbgUe5csJ13k3uzu25uWQE+XqfBis/sB0SFoiElJWXRkgEAUh0U6n40eT3ay5Ue4oPkRMu1LYw==", + "hasInstallScript": true, + "dependencies": { + "buffer": "4.9.2", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.16.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "util": "^0.12.4", + "uuid": "8.0.0", + "xml2js": "0.6.2" + }, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/aws-sdk/node_modules/uuid": { + "version": "8.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.0.0.tgz", + "integrity": "sha512-jOXGuXZAWdsTH7eZLtyXMqUb9EcWMGZNbL9YcGBJl4MH4nrxHmZJhEHvyLFrkxo+28uLb/NYRcStH48fnD0Vzw==", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==" }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, "node_modules/base64id": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/base64id/-/base64id-2.0.0.tgz", @@ -1258,6 +1311,16 @@ "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==", "dev": true }, + "node_modules/buffer": { + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", + "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", + "dependencies": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, "node_modules/buffer-equal-constant-time": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", @@ -1271,6 +1334,11 @@ "node": ">=4" } }, + "node_modules/buffer/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha512-VLghIWNM6ELQzo7zwmcg0NmTVyWKYjvIeM83yjp0wRDTmUnrM678fQbcKBo6n2CJEF0szoG//ytg+TKla89ALQ==" + }, "node_modules/bytes": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz", @@ -1645,6 +1713,11 @@ "ms": "2.0.0" } }, + "node_modules/compression/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + }, "node_modules/compression/node_modules/safe-buffer": { "version": "5.1.1", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz", @@ -2603,6 +2676,14 @@ "node": ">= 0.6" } }, + "node_modules/events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha512-kEcvvCBByWXGnZy6JUlgAp2gBIUjfCAV6P6TgT1/aaQKcmuAEC4OZTV1I4EWQLz2gxZw76atuVyvHhTxvi0Flw==", + "engines": { + "node": ">=0.4.x" + } + }, "node_modules/express": { "version": "4.21.0", "resolved": "https://registry.npmjs.org/express/-/express-4.21.0.tgz", @@ -2675,6 +2756,11 @@ "ms": "2.0.0" } }, + "node_modules/express-session/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + }, "node_modules/express-session/node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -2749,6 +2835,11 @@ "node": ">= 0.8" } }, + "node_modules/express/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + }, "node_modules/express/node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -3031,7 +3122,6 @@ "version": "0.3.3", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", "integrity": "sha512-jqYfLp7mo9vIyQf8ykW2v7A+2N4QjeCeI5+Dz9XraiO1ign81wjiH7Fb9vSOWvQfNtmSa4H2RoQTrrXivdUZmw==", - "dev": true, "dependencies": { "is-callable": "^1.1.3" } @@ -3449,7 +3539,6 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", - "dev": true, "dependencies": { "has-symbols": "^1.0.3" }, @@ -3564,6 +3653,11 @@ "node": ">=0.10.0" } }, + "node_modules/ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", @@ -3671,6 +3765,21 @@ "node": ">= 0.10" } }, + "node_modules/is-arguments": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/is-arguments/-/is-arguments-1.1.1.tgz", + "integrity": "sha512-8Q7EARjzEnKpt/PCD7e1cgUS0a6X8u5tdSiMqXhojOdoV9TsMsiO+9VLC5vAmO8N7/GmXn7yjR8qnA6bVAEzfA==", + "dependencies": { + "call-bind": "^1.0.2", + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-array-buffer": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/is-array-buffer/-/is-array-buffer-3.0.4.tgz", @@ -3736,7 +3845,6 @@ "version": "1.2.7", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", "integrity": "sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA==", - "dev": true, "engines": { "node": ">= 0.4" }, @@ -3805,6 +3913,20 @@ "node": ">=8" } }, + "node_modules/is-generator-function": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz", + "integrity": "sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==", + "dependencies": { + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-glob": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", @@ -3947,7 +4069,6 @@ "version": "1.1.13", "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.13.tgz", "integrity": "sha512-uZ25/bUAlUY5fR4OKT4rZQEBrzQWYV9ZJYGGsUmEJ6thodVJ1HX64ePQ6Z0qPWP+m+Uq6e9UugrE38jeYsDSMw==", - "dev": true, "dependencies": { "which-typed-array": "^1.1.14" }, @@ -4042,6 +4163,14 @@ "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==" }, + "node_modules/jmespath": { + "version": "0.16.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.16.0.tgz", + "integrity": "sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw==", + "engines": { + "node": ">= 0.6.0" + } + }, "node_modules/jose": { "version": "4.15.9", "resolved": "https://registry.npmjs.org/jose/-/jose-4.15.9.tgz", @@ -5346,7 +5475,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.0.0.tgz", "integrity": "sha512-d7Uw+eZoloe0EHDIYoe+bQ5WXnGMOpmiZFTuMWCwpjzzkL2nTjcKiAk4hh8TjnGye2TwWOk3UXucZ+3rbmBa8Q==", - "dev": true, "engines": { "node": ">= 0.4" } @@ -5501,6 +5629,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha512-X/xY82scca2tau62i9mDyU9K+I+djTMUsvwf7xnUX5GLvVzgJybOJf4Y6o9Zx3oJK/LSXg5tTZBjwzqVPaPO2g==", + "deprecated": "The querystring API is considered Legacy. new code should use the URLSearchParams API instead.", + "engines": { + "node": ">=0.4.x" + } + }, "node_modules/queue-microtask": { "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", @@ -6407,6 +6544,11 @@ "node": ">=4" } }, + "node_modules/sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha512-8I2a3LovHTOpm7NV5yOyO8IHqgVsfK4+UuySrXU8YXkSRX7k6hCV9b3HrkKCr3nMpgj+0bmocaJJWpvp1oc7ZA==" + }, "node_modules/semver": { "version": "7.6.3", "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", @@ -6469,6 +6611,14 @@ "node": ">= 0.8" } }, + "node_modules/send/node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "engines": { + "node": ">=0.6" + } + }, "node_modules/serialize-javascript": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.2.tgz", @@ -7313,6 +7463,32 @@ "punycode": "^2.1.0" } }, + "node_modules/url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha512-hzSUW2q06EqL1gKM/a+obYHLIO6ct2hwPuviqTTOcfFVc61UbfJ2Q32+uGL/HCPxKqrdGB5QUwIe7UqlDgwsOQ==", + "dependencies": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } + }, + "node_modules/url/node_modules/punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw==" + }, + "node_modules/util": { + "version": "0.12.5", + "resolved": "https://registry.npmjs.org/util/-/util-0.12.5.tgz", + "integrity": "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA==", + "dependencies": { + "inherits": "^2.0.3", + "is-arguments": "^1.0.4", + "is-generator-function": "^1.0.7", + "is-typed-array": "^1.1.3", + "which-typed-array": "^1.1.2" + } + }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -7582,6 +7758,11 @@ "color-name": "1.1.3" } }, + "node_modules/whelk/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==" + }, "node_modules/whelk/node_modules/commander": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/commander/-/commander-2.8.1.tgz", @@ -7673,7 +7854,6 @@ "version": "1.1.15", "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.15.tgz", "integrity": "sha512-oV0jmFtUky6CXfkqehVvBP/LSWJ2sy4vWMioiENyJLePrBO/yKyV9OyJySfAKosh+RYkIl5zJCNZ8/4JncrpdA==", - "dev": true, "dependencies": { "available-typed-arrays": "^1.0.7", "call-bind": "^1.0.7", @@ -7825,6 +8005,26 @@ } } }, + "node_modules/xml2js": { + "version": "0.6.2", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.6.2.tgz", + "integrity": "sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA==", + "dependencies": { + "sax": ">=0.6.0", + "xmlbuilder": "~11.0.0" + }, + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/xmlbuilder": { + "version": "11.0.1", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-11.0.1.tgz", + "integrity": "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA==", + "engines": { + "node": ">=4.0" + } + }, "node_modules/xtend": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", diff --git a/server/package.json b/server/package.json index 0b22212c0..8f2751e0b 100644 --- a/server/package.json +++ b/server/package.json @@ -27,6 +27,7 @@ } }, "dependencies": { + "aws-sdk": "^2.1692.0", "bcrypt": "^5.1.1", "dotenv": "^16.4.5", "dotenv-cli": "^7.4.2", From e84a2837ab804942bbadb909ba339d8601c4606e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nguy=E1=BB=85n=20H=E1=BA=A3i=20Quang?= Date: Sun, 10 Nov 2024 00:26:49 +0700 Subject: [PATCH 2/2] support project background + user avatar --- docker-compose.yml | 12 ++-- server/.env.sample | 12 ++-- server/api/helpers/attachments/delete-one.js | 21 ++++--- .../attachments/process-uploaded-file.js | 14 +++-- .../process-uploaded-background-image-file.js | 61 ++++++++++++++++++- server/api/helpers/projects/update-one.js | 15 +++++ .../users/process-uploaded-avatar-file.js | 61 ++++++++++++++++++- server/api/helpers/users/update-one.js | 15 +++++ .../get-simple-storage-service-client.js | 4 +- server/api/models/Attachment.js | 2 + server/api/models/Project.js | 19 +++++- server/api/models/User.js | 12 +++- server/config/custom.js | 19 +++--- 13 files changed, 222 insertions(+), 45 deletions(-) rename server/api/helpers/{attachments => utils}/get-simple-storage-service-client.js (87%) diff --git a/docker-compose.yml b/docker-compose.yml index 1f586c442..eacc9f3dd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -82,12 +82,12 @@ services: # - TELEGRAM_THREAD_ID= # Attachments S3 - # - ATTACHMENTS_S3=true - # - ATTACHMENTS_S3_REGION= - # - ATTACHMENTS_S3_ENDPOINT= - # - ATTACHMENTS_S3_BUCKET= - # - ATTACHMENTS_S3_ACCESS_KEY= - # - ATTACHMENTS_S3_SECRET_KEY= + # - S3_ENABLE=true + # - S3_REGION= + # - S3_ENDPOINT= + # - S3_BUCKET= + # - S3_ACCESS_KEY= + # - S3_SECRET_KEY= depends_on: postgres: condition: service_healthy diff --git a/server/.env.sample b/server/.env.sample index 3d30a14bf..e9f3ec221 100644 --- a/server/.env.sample +++ b/server/.env.sample @@ -76,9 +76,9 @@ SECRET_KEY=notsecretkey TZ=UTC -# ATTACHMENTS_S3=true -# ATTACHMENTS_S3_REGION= -# ATTACHMENTS_S3_ENDPOINT= -# ATTACHMENTS_S3_BUCKET= -# ATTACHMENTS_S3_ACCESS_KEY= -# ATTACHMENTS_S3_SECRET_KEY= +# S3_ENABLE=true +# S3_REGION= +# S3_ENDPOINT= +# S3_BUCKET= +# S3_ACCESS_KEY= +# S3_SECRET_KEY= diff --git a/server/api/helpers/attachments/delete-one.js b/server/api/helpers/attachments/delete-one.js index afba75937..39eb40130 100644 --- a/server/api/helpers/attachments/delete-one.js +++ b/server/api/helpers/attachments/delete-one.js @@ -53,19 +53,26 @@ module.exports = { try { const type = attachment.type || 'local'; if (type === 's3') { - const client = await sails.helpers.attachments.getSimpleStorageServiceClient(); + const client = await sails.helpers.utils.getSimpleStorageServiceClient(); if (client) { - const file1 = `${attachment.dirname}/${attachment.filename}`; - const file2 = `${attachment.dirname}/thumbnails/cover-256.png`; - await client.delete({ Key: file1 }); - await client.delete({ Key: file2 }); + if (attachment.url) { + const parsedUrl = new URL(attachment.url); + await client.delete({ Key: parsedUrl.pathname.replace(/^\/+/, '') }); + } + if (attachment.thumb) { + const parsedUrl = new URL(attachment.thumb); + await client.delete({ Key: parsedUrl.pathname.replace(/^\/+/, '') }); + } } - } else { - rimraf.sync(path.join(sails.config.custom.attachmentsPath, attachment.dirname)); } } catch (error) { console.warn(error.stack); // eslint-disable-line no-console } + try { + rimraf.sync(path.join(sails.config.custom.attachmentsPath, attachment.dirname)); + } catch (error) { + console.warn(error.stack); // eslint-disable-line no-console + } sails.sockets.broadcast( `board:${inputs.board.id}`, diff --git a/server/api/helpers/attachments/process-uploaded-file.js b/server/api/helpers/attachments/process-uploaded-file.js index 93e443f0f..92b6b4430 100644 --- a/server/api/helpers/attachments/process-uploaded-file.js +++ b/server/api/helpers/attachments/process-uploaded-file.js @@ -23,11 +23,11 @@ module.exports = { const rootPath = path.join(sails.config.custom.attachmentsPath, dirname); const filePath = path.join(rootPath, filename); - if (sails.config.custom.attachmentsS3) { - const client = await sails.helpers.attachments.getSimpleStorageServiceClient(); + if (sails.config.custom.s3Config) { + const client = await sails.helpers.utils.getSimpleStorageServiceClient(); const s3Image = await client.upload({ Body: fs.createReadStream(inputs.file.fd), - Key: `${dirname}/${filename}`, + Key: `attachments/${dirname}/${filename}`, ContentType: inputs.file.type, }); @@ -72,7 +72,7 @@ module.exports = { ) .toBuffer(); const s3Thumb = await client.upload({ - Key: `${dirname}/thumbnails/cover-256.${thumbnailsExtension}`, + Key: `attachments/${dirname}/thumbnails/cover-256.${thumbnailsExtension}`, Body: resizeBuffer, ContentType: inputs.file.type, }); @@ -83,6 +83,12 @@ module.exports = { } } + try { + rimraf.sync(inputs.file.fd); + } catch (error) { + console.warn(error.stack); // eslint-disable-line no-console + } + return fileData; } diff --git a/server/api/helpers/projects/process-uploaded-background-image-file.js b/server/api/helpers/projects/process-uploaded-background-image-file.js index 70792a037..daecaee19 100644 --- a/server/api/helpers/projects/process-uploaded-background-image-file.js +++ b/server/api/helpers/projects/process-uploaded-background-image-file.js @@ -33,9 +33,6 @@ module.exports = { } const dirname = uuid(); - const rootPath = path.join(sails.config.custom.projectBackgroundImagesPath, dirname); - - fs.mkdirSync(rootPath); let { width, pageHeight: height = metadata.height } = metadata; if (metadata.orientation && metadata.orientation > 4) { @@ -44,6 +41,64 @@ module.exports = { const extension = metadata.format === 'jpeg' ? 'jpg' : metadata.format; + if (sails.config.custom.s3Config) { + const client = await sails.helpers.utils.getSimpleStorageServiceClient(); + let originalUrl = ''; + let thumbUrl = ''; + + try { + const s3Original = await client.upload({ + Body: await image.toBuffer(), + Key: `project-background-images/${dirname}/original.${extension}`, + ContentType: inputs.file.type, + }); + originalUrl = s3Original.Location; + + const resizeBuffer = await image + .resize( + 336, + 200, + width < 336 || height < 200 + ? { + kernel: sharp.kernel.nearest, + } + : undefined, + ) + .toBuffer(); + const s3Thumb = await client.upload({ + Body: resizeBuffer, + Key: `project-background-images/${dirname}/cover-336.${extension}`, + ContentType: inputs.file.type, + }); + thumbUrl = s3Thumb.Location; + } catch (error1) { + try { + client.delete({ Key: `project-background-images/${dirname}/original.${extension}` }); + } catch (error2) { + console.warn(error2.stack); // eslint-disable-line no-console + } + + throw 'fileIsNotImage'; + } + + try { + rimraf.sync(inputs.file.fd); + } catch (error) { + console.warn(error.stack); // eslint-disable-line no-console + } + + return { + dirname, + extension, + original: originalUrl, + thumb: thumbUrl, + }; + } + + const rootPath = path.join(sails.config.custom.projectBackgroundImagesPath, dirname); + + fs.mkdirSync(rootPath); + try { await image.toFile(path.join(rootPath, `original.${extension}`)); diff --git a/server/api/helpers/projects/update-one.js b/server/api/helpers/projects/update-one.js index db3e94187..b8f277b59 100644 --- a/server/api/helpers/projects/update-one.js +++ b/server/api/helpers/projects/update-one.js @@ -86,6 +86,21 @@ module.exports = { (!project.backgroundImage || project.backgroundImage.dirname !== inputs.record.backgroundImage.dirname) ) { + try { + if (sails.config.custom.s3Config) { + const client = await sails.helpers.utils.getSimpleStorageServiceClient(); + if (client && inputs.record.backgroundImage && inputs.record.backgroundImage.original) { + const parsedUrl = new URL(inputs.record.backgroundImage.original); + await client.delete({ Key: parsedUrl.pathname.replace(/^\/+/, '') }); + } + if (client && inputs.record.backgroundImage && inputs.record.backgroundImage.thumb) { + const parsedUrl = new URL(inputs.record.backgroundImage.thumb); + await client.delete({ Key: parsedUrl.pathname.replace(/^\/+/, '') }); + } + } + } catch (error) { + console.warn(error.stack); // eslint-disable-line no-console + } try { rimraf.sync( path.join( diff --git a/server/api/helpers/users/process-uploaded-avatar-file.js b/server/api/helpers/users/process-uploaded-avatar-file.js index 23058412c..fc1b4a719 100644 --- a/server/api/helpers/users/process-uploaded-avatar-file.js +++ b/server/api/helpers/users/process-uploaded-avatar-file.js @@ -33,9 +33,6 @@ module.exports = { } const dirname = uuid(); - const rootPath = path.join(sails.config.custom.userAvatarsPath, dirname); - - fs.mkdirSync(rootPath); let { width, pageHeight: height = metadata.height } = metadata; if (metadata.orientation && metadata.orientation > 4) { @@ -44,6 +41,64 @@ module.exports = { const extension = metadata.format === 'jpeg' ? 'jpg' : metadata.format; + if (sails.config.custom.s3Config) { + const client = await sails.helpers.utils.getSimpleStorageServiceClient(); + let originalUrl = ''; + let squareUrl = ''; + + try { + const s3Original = await client.upload({ + Body: await image.toBuffer(), + Key: `user-avatars/${dirname}/original.${extension}`, + ContentType: inputs.file.type, + }); + originalUrl = s3Original.Location; + + const resizeBuffer = await image + .resize( + 100, + 100, + width < 100 || height < 100 + ? { + kernel: sharp.kernel.nearest, + } + : undefined, + ) + .toBuffer(); + const s3Square = await client.upload({ + Body: resizeBuffer, + Key: `user-avatars/${dirname}/square-100.${extension}`, + ContentType: inputs.file.type, + }); + squareUrl = s3Square.Location; + } catch (error1) { + try { + client.delete({ Key: `user-avatars/${dirname}/original.${extension}` }); + } catch (error2) { + console.warn(error2.stack); // eslint-disable-line no-console + } + + throw 'fileIsNotImage'; + } + + try { + rimraf.sync(inputs.file.fd); + } catch (error) { + console.warn(error.stack); // eslint-disable-line no-console + } + + return { + dirname, + extension, + original: originalUrl, + square: squareUrl, + }; + } + + const rootPath = path.join(sails.config.custom.userAvatarsPath, dirname); + + fs.mkdirSync(rootPath); + try { await image.toFile(path.join(rootPath, `original.${extension}`)); diff --git a/server/api/helpers/users/update-one.js b/server/api/helpers/users/update-one.js index 05f025f7a..01aa819f4 100644 --- a/server/api/helpers/users/update-one.js +++ b/server/api/helpers/users/update-one.js @@ -101,6 +101,21 @@ module.exports = { inputs.record.avatar && (!user.avatar || user.avatar.dirname !== inputs.record.avatar.dirname) ) { + try { + if (sails.config.custom.s3Config) { + const client = await sails.helpers.utils.getSimpleStorageServiceClient(); + if (client && inputs.record.avatar && inputs.record.avatar.original) { + const parsedUrl = new URL(inputs.record.avatar.original); + await client.delete({ Key: parsedUrl.pathname.replace(/^\/+/, '') }); + } + if (client && inputs.record.avatar && inputs.record.avatar.square) { + const parsedUrl = new URL(inputs.record.avatar.square); + await client.delete({ Key: parsedUrl.pathname.replace(/^\/+/, '') }); + } + } + } catch (error) { + console.warn(error.stack); // eslint-disable-line no-console + } try { rimraf.sync(path.join(sails.config.custom.userAvatarsPath, inputs.record.avatar.dirname)); } catch (error) { diff --git a/server/api/helpers/attachments/get-simple-storage-service-client.js b/server/api/helpers/utils/get-simple-storage-service-client.js similarity index 87% rename from server/api/helpers/attachments/get-simple-storage-service-client.js rename to server/api/helpers/utils/get-simple-storage-service-client.js index b65020443..649fcd4ef 100644 --- a/server/api/helpers/attachments/get-simple-storage-service-client.js +++ b/server/api/helpers/utils/get-simple-storage-service-client.js @@ -37,8 +37,8 @@ class S3Client { module.exports = { fn() { - if (sails.config.custom.attachmentsS3) { - return new S3Client(sails.config.custom.attachmentsS3); + if (sails.config.custom.s3Config) { + return new S3Client(sails.config.custom.s3Config); } return null; }, diff --git a/server/api/models/Attachment.js b/server/api/models/Attachment.js index fb09f2202..0beeca0a2 100644 --- a/server/api/models/Attachment.js +++ b/server/api/models/Attachment.js @@ -31,9 +31,11 @@ module.exports = { }, url: { type: 'string', + allowNull: true, }, thumb: { type: 'string', + allowNull: true, }, // ╔═╗╔╦╗╔╗ ╔═╗╔╦╗╔═╗ diff --git a/server/api/models/Project.js b/server/api/models/Project.js index 586463705..ad13385e1 100755 --- a/server/api/models/Project.js +++ b/server/api/models/Project.js @@ -79,11 +79,26 @@ module.exports = { }, customToJSON() { + let url = ''; + let coverUrl = ''; + if (this.backgroundImage) { + if (this.backgroundImage.original) { + url = this.backgroundImage.original; + } else { + url = `${sails.config.custom.projectBackgroundImagesUrl}/${this.backgroundImage.dirname}/original.${this.backgroundImage.extension}`; + } + if (this.backgroundImage.thumb) { + coverUrl = this.backgroundImage.thumb; + } else { + coverUrl = `${sails.config.custom.projectBackgroundImagesUrl}/${this.backgroundImage.dirname}/cover-336.${this.backgroundImage.extension}`; + } + } + return { ..._.omit(this, ['backgroundImage']), backgroundImage: this.backgroundImage && { - url: `${sails.config.custom.projectBackgroundImagesUrl}/${this.backgroundImage.dirname}/original.${this.backgroundImage.extension}`, - coverUrl: `${sails.config.custom.projectBackgroundImagesUrl}/${this.backgroundImage.dirname}/cover-336.${this.backgroundImage.extension}`, + url, + coverUrl, }, }; }, diff --git a/server/api/models/User.js b/server/api/models/User.js index a3336d398..4421d1ec9 100755 --- a/server/api/models/User.js +++ b/server/api/models/User.js @@ -148,6 +148,14 @@ module.exports = { customToJSON() { const isDefaultAdmin = this.email === sails.config.custom.defaultAdminEmail; + let avatarUrl = ''; + if (this.avatar) { + if (this.avatar.square) { + avatarUrl = this.avatar.square; + } else { + avatarUrl = `${sails.config.custom.userAvatarsUrl}/${this.avatar.dirname}/square-100.${this.avatar.extension}`; + } + } return { ..._.omit(this, ['password', 'isSso', 'avatar', 'passwordChangedAt']), @@ -155,9 +163,7 @@ module.exports = { isRoleLocked: (this.isSso && !sails.config.custom.oidcIgnoreRoles) || isDefaultAdmin, isUsernameLocked: (this.isSso && !sails.config.custom.oidcIgnoreUsername) || isDefaultAdmin, isDeletionLocked: isDefaultAdmin, - avatarUrl: - this.avatar && - `${sails.config.custom.userAvatarsUrl}/${this.avatar.dirname}/square-100.${this.avatar.extension}`, + avatarUrl, }; }, }; diff --git a/server/config/custom.js b/server/config/custom.js index fd5e6b2a2..4c4ad1e5c 100644 --- a/server/config/custom.js +++ b/server/config/custom.js @@ -36,18 +36,19 @@ module.exports.custom = { projectBackgroundImagesPath: path.join(sails.config.paths.public, 'project-background-images'), projectBackgroundImagesUrl: `${process.env.BASE_URL}/project-background-images`, - attachmentsS3: - process.env.ATTACHMENTS_S3 === 'true' + attachmentsPath: path.join(sails.config.appPath, 'private', 'attachments'), + attachmentsUrl: `${process.env.BASE_URL}/attachments`, + + s3Config: + process.env.S3_ENABLE === 'true' ? { - accessKeyId: process.env.ATTACHMENTS_S3_ACCESS_KEY, - secretAccessKey: process.env.ATTACHMENTS_S3_SECRET_KEY, - region: process.env.ATTACHMENTS_S3_REGION, - endpoint: process.env.ATTACHMENTS_S3_ENDPOINT, - bucket: process.env.ATTACHMENTS_S3_BUCKET, + accessKeyId: process.env.S3_ACCESS_KEY, + secretAccessKey: process.env.S3_SECRET_KEY, + region: process.env.S3_REGION, + endpoint: process.env.S3_ENDPOINT, + bucket: process.env.S3_BUCKET, } : null, - attachmentsPath: path.join(sails.config.appPath, 'private', 'attachments'), - attachmentsUrl: `${process.env.BASE_URL}/attachments`, defaultAdminEmail: process.env.DEFAULT_ADMIN_EMAIL && process.env.DEFAULT_ADMIN_EMAIL.toLowerCase(),