diff --git a/lib/network/index.ts b/lib/network/index.ts index 96e9a41ec..d97640a03 100644 --- a/lib/network/index.ts +++ b/lib/network/index.ts @@ -11,5 +11,6 @@ export const probe = { ProbeServer }; export { default as RoundRobin } from './RoundRobin'; export { default as kmip } from './kmip'; export { default as kmipClient } from './kmip/Client'; +export { default as awsClient } from './kmsAWS/Client'; export * as rpc from './rpc/rpc'; export * as level from './rpc/level-net'; diff --git a/lib/network/kmsAWS/Client.ts b/lib/network/kmsAWS/Client.ts new file mode 100644 index 000000000..6d2719138 --- /dev/null +++ b/lib/network/kmsAWS/Client.ts @@ -0,0 +1,276 @@ +'use strict'; // eslint-disable-line +/* eslint new-cap: "off" */ + +import errors from '../../errors'; +import { Agent } from "https"; +import { SecureVersion } from "tls"; +import * as werelogs from 'werelogs'; +import { KMSClient, CreateKeyCommand, ScheduleKeyDeletionCommand, EncryptCommand, DecryptCommand, GenerateDataKeyCommand, DataKeySpec } from "@aws-sdk/client-kms"; +import { NodeHttpHandler } from "@smithy/node-http-handler"; +import { AwsCredentialIdentity } from "@smithy/types"; +import assert from 'assert'; + +/** + * Normalize errors according to arsenal definitions + * @param err - an Error instance or a message string + * @returns - arsenal error + * + * @note Copied from the KMIP implementation + */ +function _arsenalError(err: string | Error) { + const messagePrefix = 'AWS_KMS:'; + if (typeof err === 'string') { + return errors.InternalError + .customizeDescription(`${messagePrefix} ${err}`); + } else if ( + err instanceof Error || + // INFO: The second part is here only for Jest, to remove when we'll be + // fully migrated to TS + // @ts-expect-error + (err && typeof err.message === 'string') + ) { + return errors.InternalError + .customizeDescription(`${messagePrefix} ${err.message}`); + } + return errors.InternalError + .customizeDescription(`${messagePrefix} Unspecified error`); +} + +export default class Client { + client: KMSClient; + options: any; + + /** + * Construct a high level KMIP driver suitable for cloudserver + * @param options - Instance options + * @param options.kmsAWS - AWS client options + * @param options.kmsAWS.region - KMS region + * @param options.kmsAWS.endpoint - Endpoint URL of the KMS service + * @param options.kmsAWS.ak - Application Key + * @param options.kmsAWS.sk - Secret Key + * @param options.kmsAWS.tls.rejectUnauthorized - default to true, reject unauthenticated TLS connections (set to false to accept auto-signed certificates, useful in development ONLY) + * @param options.kmsAWS.tls.ca - override CA definition(s) + * @param options.kmsAWS.tls.cert - certificate or list of certificates + * @param options.kmsAWS.tls.minVersion - min TLS version accepted, One of 'TLSv1.3', 'TLSv1.2', 'TLSv1.1', or 'TLSv1' (see https://nodejs.org/api/tls.html#tlscreatesecurecontextoptions) + * @param options.kmsAWS.tls.maxVersion - max TLS version accepted, One of 'TLSv1.3', 'TLSv1.2', 'TLSv1.1', or 'TLSv1' (see https://nodejs.org/api/tls.html#tlscreatesecurecontextoptions) + * @param options.kmsAWS.tls.key - private key or list of private keys + * + * This client also looks in the standard AWS configuration files (https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-files.html). + * If no option is passed to this constructor, the client will try to get it from the configuration file. + * + * TLS configuration options are those of nodejs, you can refere to https://nodejs.org/api/tls.html#tlsconnectoptions and https://nodejs.org/api/tls.html#tlscreatesecurecontextoptions + */ + constructor( + options: { + kmsAWS: { + region?: string, + endpoint?: string, + ak?: string, + sk?: string, + tls?: { + rejectUnauthorized?: boolean, + ca?: [Buffer] | Buffer, + cert?: [Buffer] | Buffer, + minVersion?: string, + maxVersion?: string, + key?: [Buffer] | Buffer, + } + } + }, + ) { + let requestHandler: {requestHandler: NodeHttpHandler} | null = null; + const tlsOpts = options.kmsAWS.tls; + if (tlsOpts) { + const agent = new Agent({ + rejectUnauthorized: tlsOpts?.rejectUnauthorized, + ca: tlsOpts?.ca, + cert: tlsOpts?.cert, + minVersion: tlsOpts?.minVersion, + maxVersion: tlsOpts?.maxVersion, + key: tlsOpts?.key, + }); + + requestHandler = {requestHandler: new NodeHttpHandler({ + httpAgent: agent, + httpsAgent: agent, + })} + } + + let credentials: {credentials: AwsCredentialIdentity} | null = null; + if (options.kmsAWS.ak && options.kmsAWS.sk) { + credentials = {credentials: { + accessKeyId: options.kmsAWS.ak, + secretAccessKey: options.kmsAWS.sk, + }}; + } + + this.client = new KMSClient({ + region: options.kmsAWS.region, + endpoint: options.kmsAWS.endpoint, + ...credentials, + ...requestHandler + }); + } + + /** + * Create a new cryptographic key managed by the server, + * for a specific bucket + * @param bucketName - The bucket name + * @param logger - Werelog logger object + * @param cb - The callback(err: Error, bucketKeyId: String) + */ + createBucketKey(bucketName: string, logger: werelogs.Logger, cb: any) { + logger.debug("AWS KMS: createBucketKey", {bucketName}); + + const command = new CreateKeyCommand({}); + this.client.send(command, (err, data) => { + if (err) { + const error = _arsenalError(err); + logger.error("AWS_KMS::createBucketKey", {err, bucketName}); + cb (error); + } else { + logger.debug("AWS KMS: createBucketKey", {bucketName, KeyMetadata: data?.KeyMetadata}); + cb(null, data?.KeyMetadata?.KeyId); + } + }); + } + + /** + * Destroy a cryptographic key managed by the server, for a specific bucket. + * @param bucketKeyId - The bucket key Id + * @param logger - Werelog logger object + * @param cb - The callback(err: Error) + */ + destroyBucketKey(bucketKeyId: string, logger: werelogs.Logger, cb: any) { + logger.debug("AWS KMS: destroyBucketKey", {bucketKeyId: bucketKeyId}); + + // Schedule a deletion in 7 days (the minimum value on this API) + const command = new ScheduleKeyDeletionCommand({KeyId: bucketKeyId, PendingWindowInDays: 7}); + this.client.send(command, (err, data) => { + if (err) { + const error = _arsenalError(err); + logger.error("AWS_KMS::destroyBucketKey", {err}); + cb (error); + } else { + // Sanity check + if (data?.KeyState != "PendingDeletion") { + const error = _arsenalError("Key is not in PendingDeletion state") + logger.error("AWS_KMS::destroyBucketKey", {err, data}); + cb(error); + } else { + cb(); + } + } + }); + } + + /** + * @param cryptoScheme - crypto scheme version number + * @param masterKeyId - key to retrieve master key + * @param logger - werelog logger object + * @param cb - callback + * @callback called with (err, plainTextDataKey: Buffer, cipheredDataKey: Buffer) + */ + generateDataKey( + cryptoScheme: number, + masterKeyId: string, + logger: werelogs.Logger, + cb: any, + ) { + logger.debug("AWS KMS: generateDataKey", {cryptoScheme, masterKeyId}); + + // Only support cryptoScheme v1 + assert.strictEqual (cryptoScheme, 1); + + const command = new GenerateDataKeyCommand({KeyId: masterKeyId, KeySpec: DataKeySpec.AES_256}); + this.client.send(command, (err, data) => { + if (err) { + const error = _arsenalError(err); + logger.error("AWS_KMS::generateDataKey", {err}); + cb (error); + } else if (!data) { + const error = _arsenalError("generateDataKey: empty response"); + logger.error("AWS_KMS::generateDataKey empty reponse"); + cb (error); + } else { + // Convert to a buffer. This allows the wrapper to use .toString("base64") + cb(null, Buffer.from(data.Plaintext!), Buffer.from(data.CiphertextBlob!)); + } + }); + } + + /** + * + * @param cryptoScheme - crypto scheme version number + * @param masterKeyId - key to retrieve master key + * @param plainTextDataKey - data key + * @param logger - werelog logger object + * @param cb - callback + * @callback called with (err, cipheredDataKey: Buffer) + */ + cipherDataKey( + cryptoScheme: number, + masterKeyId: string, + plainTextDataKey: Buffer, + logger: werelogs.Logger, + cb: any, + ) { + logger.debug("AWS KMS: cipherDataKey", {cryptoScheme, masterKeyId}); + + // Only support cryptoScheme v1 + assert.strictEqual (cryptoScheme, 1); + + const command = new EncryptCommand({KeyId: masterKeyId, Plaintext: plainTextDataKey}); + this.client.send(command, (err, data) => { + if (err) { + const error = _arsenalError(err); + logger.error("AWS_KMS::cipherDataKey", {err}); + cb (error); + } else if (!data) { + const error = _arsenalError("cipherDataKey: empty response"); + logger.error("AWS_KMS::cipherDataKey empty reponse"); + cb (error); + } else { + // Convert to a buffer. This allows the wrapper to use .toString("base64") + cb(null, Buffer.from(data.CiphertextBlob!)); + } + }); + } + + /** + * + * @param cryptoScheme - crypto scheme version number + * @param masterKeyId - key to retrieve master key + * @param cipheredDataKey - data key + * @param logger - werelog logger object + * @param cb - callback + * @callback called with (err, plainTextDataKey: Buffer) + */ + decipherDataKey( + cryptoScheme: number, + masterKeyId: string, + cipheredDataKey: Buffer, + logger: werelogs.Logger, + cb: any, + ) { + logger.debug("AWS KMS: decipherDataKey", {cryptoScheme, masterKeyId}); + + // Only support cryptoScheme v1 + assert.strictEqual (cryptoScheme, 1); + + const command = new DecryptCommand({CiphertextBlob: cipheredDataKey}); + this.client.send(command, (err, data) => { + if (err) { + const error = _arsenalError(err); + logger.error("AWS_KMS::decipherDataKey", {err}); + cb (error); + } else if (!data) { + const error = _arsenalError("decipherDataKey: empty response"); + logger.error("AWS_KMS::decipherDataKey empty reponse"); + cb (error); + } else { + cb(null, Buffer.from(data?.Plaintext!)); + } + }); + } +} diff --git a/lib/network/kmsAWS/README.md b/lib/network/kmsAWS/README.md new file mode 100644 index 000000000..e7ec7e79c --- /dev/null +++ b/lib/network/kmsAWS/README.md @@ -0,0 +1,58 @@ +# AWS KMS connector + +Allow to use AWS KMS backend for encryption of objects. It currently only support AK+SK for authentication. +mTLS can also be used to add extra security. + +## Configuration + +Configuration is done using the configuration file or environment variables. A Mix of both can be used, the configuration file takes precedence over environment variables. +Environment variables are the same as the ones used by the AWS CLI prefixed with "KMS_" (in order to scope them to the KMS module). + +The following parameters are supported: + +| config file | env variable | Description +|---------------------|--------------------------------------------------|------------ +| kmsAWS.region | KMS_AWS_REGION or KMS_AWS_DEFAULT_REGION | AWS region tu use +| kmsAWS.endpoint | KMS_AWS_ENDPOINT_URL_KMS or KMS_AWS_ENDPOINT_URL | Endpoint URL +| kmsAWS.ak | KMS_AWS_ACCESS_KEY_ID | Credentials, Access Key +| kmsAWS.sk | KMS_AWS_SECRET_ACCESS_KEY | Credentials, Secret Key +| kmsAWS.tls | | TLS configuration (Object, see below) + +The TLS configuration is can contain the following attributes: + +| config file | Description +|---------------------|-------------------------------------------------- +| rejectUnauthorized | false to disable TLS certificates checks (useful in development, DON'T disable in production) +| minVersion | min TLS version, One of 'TLSv1.3', 'TLSv1.2', 'TLSv1.1', or 'TLSv1' (see https://nodejs.org/api/tls.html#tlscreatesecurecontextoptions) +| maxVersion | max TLS version, One of 'TLSv1.3', 'TLSv1.2', 'TLSv1.1', or 'TLSv1' (see https://nodejs.org/api/tls.html#tlscreatesecurecontextoptions) +| ca | filename or array of filenames containing CA(s) +| cert | filename or array of filenames containing certificate(s) +| key | filename or array of filenames containing private key(s) + +All TLS attributes conform to their nodejs definition. See https://nodejs.org/api/tls.html#tlsconnectoptions and https://nodejs.org/api/tls.html#tlscreatesecurecontextoptions. + +Configuration example: +```json + "kmsAWS": { + "region": "us-east-1", + "endpoint": "https://kms.us-east-1.amazonaws.com", + "ak": "xxxxxxx", + "sk": "xxxxxxx" + }, +``` + +With TLS configuration: +```json + "kmsAWS": { + "region": "us-east-1", + "endpoint": "https://kms.us-east-1.amazonaws.com", + "ak": "xxxxxxx", + "sk": "xxxxxxx", + "tls": { + "rejectUnauthorized": false, + "cert": "mtls.crt.pem", + "key": "mtls.key.pem", + "minVersion": "TLSv1.3" + } + }, +``` diff --git a/package.json b/package.json index a4c3c2787..483de22d1 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,7 @@ }, "homepage": "https://github.com/scality/Arsenal#readme", "dependencies": { + "@aws-sdk/client-kms": "^3.485.0", "@js-sdsl/ordered-set": "^4.4.2", "@types/async": "^3.2.12", "@types/utf8": "^3.0.1", @@ -63,6 +64,7 @@ "@types/jest": "^27.4.1", "@types/node": "^17.0.21", "@types/xml2js": "^0.4.11", + "aws-sdk-client-mock": "^3.0.1", "eslint": "^8.12.0", "eslint-config-airbnb": "6.2.0", "eslint-config-scality": "scality/Guidelines#7.10.2", diff --git a/tests/functional/kmsAWS/highlevel.spec.js b/tests/functional/kmsAWS/highlevel.spec.js new file mode 100644 index 000000000..7b33f0fe2 --- /dev/null +++ b/tests/functional/kmsAWS/highlevel.spec.js @@ -0,0 +1,264 @@ +'use strict'; // eslint-disable-line strict + +// Mocking official nodejs aws client +const { mockClient } = require('aws-sdk-client-mock'); +const { KMSClient, CreateKeyCommand, ScheduleKeyDeletionCommand, GenerateDataKeyCommand, + EncryptCommand, DecryptCommand } = require('@aws-sdk/client-kms'); + +const KmsAWSClient = require('../../../lib/network/kmsAWS/Client').default; + +const logger = { + info: () => {}, + debug: () => {}, + warn: () => {}, + error: () => {}, +}; + +describe('KMS AWS Client', () => { + const options = { + kmsAWS: { + region: 'test-region-1', + endpoint: 'http://mocked.doesnt.matter', + }, + }; + + const kmsClient = new KmsAWSClient(options); + + // Mock the AWS client + const mockedAwsClient = mockClient(KMSClient); + + // Mock the send method to allow using callbacks + // Note: we cannot replace the whole aws client with its mocked version + // because the current mocking implementation doesn't support callbacks + // See https://github.com/m-radzikowski/aws-sdk-client-mock/issues/6 + kmsClient.client.send = function mockedSend(cmd, cbOrOption, cb = null) { + let callback = cb; + if (! cb && typeof(cbOrOption) === 'function') { + callback = cbOrOption; + } + + const mockedCall = mockedAwsClient.send(cmd); + + // mockedCall is undefined when parameters doesn't match those expected. + expect(mockedCall).toBeDefined(); + + let gotError = false; + mockedCall.catch((err) => { + gotError = true; + callback(err); + }) + // Order is important here: the catch is done before + // so it doesn't catch exceptions raised by Jest in case of + // test failure + .then((data) => { + // Looks like this "then" statement is alway run after + // a catch. We check for errors handled in the catch to + // avoid a double invocation of the callback. + if (!gotError) { + callback(null, data); + } + }); + }; + + beforeEach(() => { + mockedAwsClient.reset(); + }); + + it('should create a new key on bucket creation', done => { + mockedAwsClient.on(CreateKeyCommand).resolves({ + KeyMetadata: { + KeyId: 'mocked-kms-key-id', + }, + }); + + kmsClient.createBucketKey('plop', logger, (err, bucketKeyId) => { + // Check the result + expect(err).toBeNull(); + expect(bucketKeyId).toEqual('mocked-kms-key-id'); + + // Check that the CreateKey of the aws client have been used to create the key + expect(mockedAwsClient.commandCalls(CreateKeyCommand).length).toEqual(1); + + done(); + }); + }); + + it('should handle errors creating the key on bucket creation', done => { + mockedAwsClient.on(CreateKeyCommand).rejects('Error'); + + kmsClient.createBucketKey('plop', logger, (err, bucketKeyId) => { + // Check the result + expect(bucketKeyId).toBeUndefined(); + expect(err).toEqual(Error('InternalError')); + + // Check that the CreateKey of the aws client have been used to create the key + expect(mockedAwsClient.commandCalls(CreateKeyCommand).length).toEqual(1); + + done(); + }); + }); + + it('should delete an existing key on bucket deletion', done => { + mockedAwsClient.on(ScheduleKeyDeletionCommand, { + KeyId: 'mocked-kms-key-id', + PendingWindowInDays: 7, // Should be set to 7 (the minimum accepted value on this operation) + }).resolves({ + KeyId: 'mocked-kms-key-id', + KeyState: 'PendingDeletion', + PendingWindowInDays: 7, + }); + + kmsClient.destroyBucketKey('mocked-kms-key-id', logger, (err) => { + // Check the result + expect(err).toBeUndefined(); + + // Check that the ScheduleKeyDeletion of the aws client have been invoked + expect(mockedAwsClient.commandCalls(ScheduleKeyDeletionCommand).length).toEqual(1); + + done(); + }); + }); + + it('should handle errors deleting an existing key on bucket deletion', done => { + mockedAwsClient.on(ScheduleKeyDeletionCommand, { + KeyId: 'mocked-kms-key-id', + PendingWindowInDays: 7, // Should be set to 7 (the minimum accepted value on this operation) + }).rejects('Error'); + + kmsClient.destroyBucketKey('mocked-kms-key-id', logger, (err) => { + // Check the result + expect(err).toEqual(Error('InternalError')); + + // Check that the ScheduleKeyDeletion of the aws client have been invoked + expect(mockedAwsClient.commandCalls(ScheduleKeyDeletionCommand).length).toEqual(1); + + done(); + }); + }); + + it('should generate a datakey for ciphering', done => { + mockedAwsClient.on(GenerateDataKeyCommand).resolves({ + CiphertextBlob: 'encryptedDataKey', + Plaintext: 'dataKey', + KeyId: 'mocked-kms-key-id', + }); + + kmsClient.generateDataKey(1, 'mocked-kms-key-id', logger, (err, plaintextDataKey, cipheredDataKey) => { + // Check the result + expect(err).toBeNull(); + expect(plaintextDataKey).toEqual(Buffer.from('dataKey')); + expect(cipheredDataKey).toEqual(Buffer.from('encryptedDataKey')); + + // Check that the the aws client have been used to create the data key + expect(mockedAwsClient.commandCalls(GenerateDataKeyCommand).length).toEqual(1); + + done(); + }); + }); + + it('should handle errors generating a datakey', done => { + mockedAwsClient.on(GenerateDataKeyCommand).rejects('Error'); + + kmsClient.generateDataKey(1, 'mocked-kms-key-id', logger, (err, plaintextDataKey, cipheredDataKey) => { + // Check the result + expect(plaintextDataKey).toBeUndefined(); + expect(cipheredDataKey).toBeUndefined(); + expect(err).toEqual(Error('InternalError')); + + // Check that the the aws client have been used to create the data key + expect(mockedAwsClient.commandCalls(GenerateDataKeyCommand).length).toEqual(1); + + done(); + }); + }); + + it('should allow ciphering a datakey', done => { + mockedAwsClient.on(EncryptCommand, { + KeyId: 'mocked-kms-key-id', + Plaintext: 'dataKey-value', + EncryptionContext: undefined, + GrantTokens: undefined, + EncryptionAlgorithm: undefined, + }).resolves({ + CiphertextBlob: 'encryptedDataKey-value', + KeyId: 'mocked-kms-key-id', + }); + + kmsClient.cipherDataKey(1, 'mocked-kms-key-id', 'dataKey-value', logger, (err, cipheredDataKey) => { + // Check the result + expect(err).toBeNull(); + expect(cipheredDataKey).toEqual(Buffer.from('encryptedDataKey-value')); + + // Check that the Encrypt of the aws client has been used + expect(mockedAwsClient.commandCalls(EncryptCommand).length).toEqual(1); + + done(); + }); + }); + + it('should handle errors ciphering a datakey', done => { + mockedAwsClient.on(EncryptCommand, { + KeyId: 'mocked-kms-key-id', + Plaintext: 'dataKey-value', + EncryptionContext: undefined, + GrantTokens: undefined, + EncryptionAlgorithm: undefined, + }).rejects('Error'); + + kmsClient.cipherDataKey(1, 'mocked-kms-key-id', 'dataKey-value', logger, (err, cipheredDataKey) => { + // Check the result + expect(cipheredDataKey).toBeUndefined(); + expect(err).toEqual(Error('InternalError')); + + // Check that the Encrypt of the aws client have been used + expect(mockedAwsClient.commandCalls(EncryptCommand).length).toEqual(1); + + done(); + }); + }); + + it('should allow deciphering a datakey', done => { + mockedAwsClient.on(DecryptCommand, { + KeyId: undefined, // Key id is embedded in the CiphertextBlob + CiphertextBlob: 'encryptedDataKey-value', + EncryptionContext: undefined, + GrantTokens: undefined, + EncryptionAlgorithm: undefined, + }).resolves({ + Plaintext: 'dataKey-value', + KeyId: 'mocked-kms-key-id', + }); + + kmsClient.decipherDataKey(1, 'mocked-kms-key-id', 'encryptedDataKey-value', logger, (err, plainTextDataKey) => { + // Check the result + expect(err).toBeNull(); + expect(plainTextDataKey).toEqual(Buffer.from('dataKey-value')); + + // Check that the Decrypt of the aws client have been used + expect(mockedAwsClient.commandCalls(DecryptCommand).length).toEqual(1); + + done(); + }); + }); + + it('should handle errors deciphering a datakey', done => { + mockedAwsClient.on(DecryptCommand, { + KeyId: undefined, // Key id is embedded in the CiphertextBlob + CiphertextBlob: 'encryptedDataKey-value', + EncryptionContext: undefined, + GrantTokens: undefined, + EncryptionAlgorithm: undefined, + }).rejects('Error'); + + kmsClient.decipherDataKey(1, 'mocked-kms-key-id', 'encryptedDataKey-value', logger, (err, plainTextDataKey) => { + // Check the result + expect(plainTextDataKey).toBeUndefined(); + expect(err).toEqual(Error('InternalError')); + + // Check that the Decrypt of the aws client have been used + expect(mockedAwsClient.commandCalls(DecryptCommand).length).toEqual(1); + + done(); + }); + }); +});