Skip to content

Commit 5b87ad9

Browse files
utayutay
committed
Add support for SSE-C encryption on S3
1 parent 9497e1b commit 5b87ad9

File tree

4 files changed

+44
-0
lines changed

4 files changed

+44
-0
lines changed

‎extension/httpfs/create_secret_functions.cpp

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,10 @@ unique_ptr<BaseSecret> CreateS3SecretFunctions::CreateSecretFunctionInternal(Cli
8282
secret->secret_map["use_ssl"] = Value::BOOLEAN(named_param.second.GetValue<bool>());
8383
} else if (lower_name == "kms_key_id") {
8484
secret->secret_map["kms_key_id"] = named_param.second.ToString();
85+
} else if (lower_name == "sse_c_key") {
86+
secret->secret_map["sse_c_key"] = named_param.second.ToString();
87+
} else if (lower_name == "sse_c_key_md5") {
88+
secret->secret_map["sse_c_key_md5"] = named_param.second.ToString();
8589
} else if (lower_name == "url_compatibility_mode") {
8690
if (named_param.second.type() != LogicalType::BOOLEAN) {
8791
throw InvalidInputException("Invalid type past to secret option: '%s', found '%s', expected: 'BOOLEAN'",
@@ -114,6 +118,16 @@ unique_ptr<BaseSecret> CreateS3SecretFunctions::CreateSecretFunctionInternal(Cli
114118
}
115119
}
116120

121+
bool has_sse_c_key = secret->secret_map.find("sse_c_key") != secret->secret_map.end();
122+
bool has_sse_c_key_md5 = secret->secret_map.find("sse_c_key_md5") != secret->secret_map.end();
123+
if (has_sse_c_key != has_sse_c_key_md5) {
124+
throw InvalidInputException("Both `sse_c_key` and `sse_c_key_md5` must be set together, or neither should be set");
125+
}
126+
bool has_kms_key_id = secret->secret_map.find("kms_key_id") != secret->secret_map.end();
127+
if (has_kms_key_id && has_sse_c_key) {
128+
throw InvalidInputException("Cannot set `kms_key_id` and `sse_c_key` at the same time");
129+
}
130+
117131
return std::move(secret);
118132
}
119133

@@ -184,6 +198,8 @@ void CreateS3SecretFunctions::SetBaseNamedParams(CreateSecretFunction &function,
184198
function.named_parameters["url_style"] = LogicalType::VARCHAR;
185199
function.named_parameters["use_ssl"] = LogicalType::BOOLEAN;
186200
function.named_parameters["kms_key_id"] = LogicalType::VARCHAR;
201+
function.named_parameters["sse_c_key"] = LogicalType::VARCHAR;
202+
function.named_parameters["sse_c_key_md5"] = LogicalType::VARCHAR;
187203
function.named_parameters["url_compatibility_mode"] = LogicalType::BOOLEAN;
188204

189205
// Whether a secret refresh attempt should be made when the secret appears to be incorrect

‎extension/httpfs/httpfs_extension.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,8 @@ static void LoadInternal(DatabaseInstance &instance) {
6868
config.AddExtensionOption("s3_url_style", "S3 URL style", LogicalType::VARCHAR, Value("vhost"));
6969
config.AddExtensionOption("s3_use_ssl", "S3 use SSL", LogicalType::BOOLEAN, Value(true));
7070
config.AddExtensionOption("s3_kms_key_id", "S3 KMS Key ID", LogicalType::VARCHAR);
71+
config.AddExtensionOption("s3_sse_c_key", "S3 SSE-C Key", LogicalType::VARCHAR);
72+
config.AddExtensionOption("s3_sse_c_key_md5", "S3 SSE-C Key MD5", LogicalType::VARCHAR);
7173
config.AddExtensionOption("s3_url_compatibility_mode", "Disable Globs and Query Parameters on S3 URLs",
7274
LogicalType::BOOLEAN, Value(false));
7375

‎extension/httpfs/include/s3fs.hpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ struct S3AuthParams {
2727
string session_token;
2828
string endpoint;
2929
string kms_key_id;
30+
string sse_c_key;
31+
string sse_c_key_md5;
3032
string url_style;
3133
bool use_ssl = true;
3234
bool s3_url_compatibility_mode = false;
@@ -43,6 +45,8 @@ struct AWSEnvironmentCredentialsProvider {
4345
static constexpr const char *DUCKDB_ENDPOINT_ENV_VAR = "DUCKDB_S3_ENDPOINT";
4446
static constexpr const char *DUCKDB_USE_SSL_ENV_VAR = "DUCKDB_S3_USE_SSL";
4547
static constexpr const char *DUCKDB_KMS_KEY_ID_ENV_VAR = "DUCKDB_S3_KMS_KEY_ID";
48+
static constexpr const char *DUCKDB_SSE_C_KEY_ENV_VAR = "DUCKDB_S3_SSE_C_KEY";
49+
static constexpr const char *DUCKDB_SSE_C_KEY_MD5_ENV_VAR = "DUCKDB_S3_SSE_C_KEY_MD5";
4650

4751
explicit AWSEnvironmentCredentialsProvider(DBConfig &config) : config(config) {};
4852

‎extension/httpfs/s3fs.cpp

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ static HTTPHeaders create_s3_header(string url, string query, string host, strin
5151
// https://docs.aws.amazon.com/AmazonS3/latest/userguide/specifying-kms-encryption.html#sse-request-headers-kms
5252
bool use_sse_kms = auth_params.kms_key_id.length() > 0 && (method == "POST" || method == "PUT") &&
5353
query.find("uploadId") == std::string::npos;
54+
bool use_sse_c = auth_params.sse_c_key.length() > 0 && auth_params.sse_c_key_md5.length() > 0;
5455

5556
res["x-amz-date"] = datetime_now;
5657
res["x-amz-content-sha256"] = payload_hash;
@@ -61,6 +62,11 @@ static HTTPHeaders create_s3_header(string url, string query, string host, strin
6162
res["x-amz-server-side-encryption"] = "aws:kms";
6263
res["x-amz-server-side-encryption-aws-kms-key-id"] = auth_params.kms_key_id;
6364
}
65+
if (use_sse_c) {
66+
res["x-amz-server-side-encryption-customer-algorithm"] = "AES256";
67+
res["x-amz-server-side-encryption-customer-key"] = auth_params.sse_c_key;
68+
res["x-amz-server-side-encryption-customer-key-md5"] = auth_params.sse_c_key_md5;
69+
}
6470

6571
string signed_headers = "";
6672
hash_bytes canonical_request_hash;
@@ -75,6 +81,9 @@ static HTTPHeaders create_s3_header(string url, string query, string host, strin
7581
if (use_sse_kms) {
7682
signed_headers += ";x-amz-server-side-encryption;x-amz-server-side-encryption-aws-kms-key-id";
7783
}
84+
if (use_sse_c) {
85+
signed_headers += ";x-amz-server-side-encryption-customer-algorithm;x-amz-server-side-encryption-customer-key;x-amz-server-side-encryption-customer-key-md5";
86+
}
7887
auto canonical_request = method + "\n" + S3FileSystem::UrlEncode(url) + "\n" + query;
7988
if (content_type.length() > 0) {
8089
canonical_request += "\ncontent-type:" + content_type;
@@ -87,6 +96,11 @@ static HTTPHeaders create_s3_header(string url, string query, string host, strin
8796
canonical_request += "\nx-amz-server-side-encryption:aws:kms";
8897
canonical_request += "\nx-amz-server-side-encryption-aws-kms-key-id:" + auth_params.kms_key_id;
8998
}
99+
if (use_sse_c) {
100+
canonical_request += "\nx-amz-server-side-encryption-customer-algorithm:AES256";
101+
canonical_request += "\nx-amz-server-side-encryption-customer-key:" + auth_params.sse_c_key;
102+
canonical_request += "\nx-amz-server-side-encryption-customer-key-md5:" + auth_params.sse_c_key_md5;
103+
}
90104

91105
canonical_request += "\n\n" + signed_headers + "\n" + payload_hash;
92106
sha256(canonical_request.c_str(), canonical_request.length(), canonical_request_hash);
@@ -143,6 +157,8 @@ void AWSEnvironmentCredentialsProvider::SetAll() {
143157
this->SetExtensionOptionValue("s3_endpoint", DUCKDB_ENDPOINT_ENV_VAR);
144158
this->SetExtensionOptionValue("s3_use_ssl", DUCKDB_USE_SSL_ENV_VAR);
145159
this->SetExtensionOptionValue("s3_kms_key_id", DUCKDB_KMS_KEY_ID_ENV_VAR);
160+
this->SetExtensionOptionValue("s3_sse_c_key", DUCKDB_SSE_C_KEY_ENV_VAR);
161+
this->SetExtensionOptionValue("s3_sse_c_key_md5", DUCKDB_SSE_C_KEY_MD5_ENV_VAR);
146162
}
147163

148164
S3AuthParams AWSEnvironmentCredentialsProvider::CreateParams() {
@@ -155,6 +171,8 @@ S3AuthParams AWSEnvironmentCredentialsProvider::CreateParams() {
155171
params.session_token = SESSION_TOKEN_ENV_VAR;
156172
params.endpoint = DUCKDB_ENDPOINT_ENV_VAR;
157173
params.kms_key_id = DUCKDB_KMS_KEY_ID_ENV_VAR;
174+
params.sse_c_key = DUCKDB_SSE_C_KEY_ENV_VAR;
175+
params.sse_c_key_md5 = DUCKDB_SSE_C_KEY_MD5_ENV_VAR;
158176
params.use_ssl = DUCKDB_USE_SSL_ENV_VAR;
159177

160178
return params;
@@ -179,6 +197,8 @@ S3AuthParams S3AuthParams::ReadFrom(optional_ptr<FileOpener> opener, FileOpenerI
179197
secret_reader.TryGetSecretKeyOrSetting("region", "s3_region", result.region);
180198
secret_reader.TryGetSecretKeyOrSetting("use_ssl", "s3_use_ssl", result.use_ssl);
181199
secret_reader.TryGetSecretKeyOrSetting("kms_key_id", "s3_kms_key_id", result.kms_key_id);
200+
secret_reader.TryGetSecretKeyOrSetting("sse_c_key", "s3_sse_c_key", result.sse_c_key);
201+
secret_reader.TryGetSecretKeyOrSetting("sse_c_key_md5", "s3_sse_c_key_md5", result.sse_c_key_md5);
182202
secret_reader.TryGetSecretKeyOrSetting("s3_url_compatibility_mode", "s3_url_compatibility_mode",
183203
result.s3_url_compatibility_mode);
184204

@@ -216,6 +236,8 @@ unique_ptr<KeyValueSecret> CreateSecret(vector<string> &prefix_paths_p, string &
216236
return_value->secret_map["url_style"] = params.url_style;
217237
return_value->secret_map["use_ssl"] = params.use_ssl;
218238
return_value->secret_map["kms_key_id"] = params.kms_key_id;
239+
return_value->secret_map["sse_c_key"] = params.sse_c_key;
240+
return_value->secret_map["sse_c_key_md5"] = params.sse_c_key_md5;
219241
return_value->secret_map["s3_url_compatibility_mode"] = params.s3_url_compatibility_mode;
220242

221243
//! Set redact keys

0 commit comments

Comments
 (0)