Skip to content

Commit f0a655d

Browse files
authored
Merge pull request #63 from Tmonster/update_v1.3_to_current_main
Update v1.3 to current main
2 parents c225324 + 7ce5308 commit f0a655d

23 files changed

+1003
-957
lines changed

.github/workflows/MainDistributionPipeline.yml

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -14,19 +14,19 @@ concurrency:
1414
jobs:
1515
duckdb-stable-build:
1616
name: Build extension binaries
17-
uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@v1.2.1
17+
uses: duckdb/extension-ci-tools/.github/workflows/_extension_distribution.yml@main
1818
with:
1919
extension_name: httpfs
20-
duckdb_version: v1.2.1
21-
ci_tools_version: v1.2.1
20+
duckdb_version: v1.3.0
21+
ci_tools_version: main
2222

2323
duckdb-stable-deploy:
2424
name: Deploy extension binaries
2525
needs: duckdb-stable-build
26-
uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@v1.2.1
26+
uses: duckdb/extension-ci-tools/.github/workflows/_extension_deploy.yml@main
2727
secrets: inherit
2828
with:
2929
extension_name: httpfs
30-
duckdb_version: v1.2.1
31-
ci_tools_version: v1.2.1
32-
deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}
30+
duckdb_version: v1.3.0
31+
ci_tools_version: main
32+
deploy_latest: ${{ startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' }}

CMakeLists.txt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ build_static_extension(
1414
extension/httpfs/hffs.cpp
1515
extension/httpfs/s3fs.cpp
1616
extension/httpfs/httpfs.cpp
17+
extension/httpfs/httpfs_client.cpp
1718
extension/httpfs/http_state.cpp
1819
extension/httpfs/crypto.cpp
1920
extension/httpfs/create_secret_functions.cpp
@@ -23,10 +24,10 @@ set(PARAMETERS "-warnings")
2324
build_loadable_extension(
2425
httpfs
2526
${PARAMETERS}
26-
extension/httpfs/httpfs
2727
extension/httpfs/hffs.cpp
2828
extension/httpfs/s3fs.cpp
2929
extension/httpfs/httpfs.cpp
30+
extension/httpfs/httpfs_client.cpp
3031
extension/httpfs/http_state.cpp
3132
extension/httpfs/crypto.cpp
3233
extension/httpfs/create_secret_functions.cpp

duckdb

Submodule duckdb updated 2460 files

extension/httpfs/create_secret_functions.cpp

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ void CreateS3SecretFunctions::Register(DatabaseInstance &instance) {
1212
RegisterCreateSecretFunction(instance, "gcs");
1313
}
1414

15-
static Value MapToStruct(const Value &map){
15+
static Value MapToStruct(const Value &map) {
1616
auto children = MapValue::GetChildren(map);
1717

1818
child_list_t<Value> struct_fields;
@@ -109,17 +109,21 @@ unique_ptr<BaseSecret> CreateS3SecretFunctions::CreateSecretFunctionInternal(Cli
109109
refresh = true;
110110
secret->secret_map["refresh_info"] = MapToStruct(named_param.second);
111111
} else {
112-
throw InvalidInputException("Unknown named parameter passed to CreateSecretFunctionInternal: " + lower_name);
112+
throw InvalidInputException("Unknown named parameter passed to CreateSecretFunctionInternal: " +
113+
lower_name);
113114
}
114115
}
115116

116117
return std::move(secret);
117118
}
118119

119-
CreateSecretInfo CreateS3SecretFunctions::GenerateRefreshSecretInfo(const SecretEntry &secret_entry, Value &refresh_info) {
120-
const auto &kv_secret = dynamic_cast<const KeyValueSecret&>(*secret_entry.secret);
120+
CreateSecretInput CreateS3SecretFunctions::GenerateRefreshSecretInfo(const SecretEntry &secret_entry,
121+
Value &refresh_info) {
122+
const auto &kv_secret = dynamic_cast<const KeyValueSecret &>(*secret_entry.secret);
121123

122-
CreateSecretInfo result(OnCreateConflict::REPLACE_ON_CONFLICT, secret_entry.persist_type);
124+
CreateSecretInput result;
125+
result.on_conflict = OnCreateConflict::REPLACE_ON_CONFLICT;
126+
result.persist_type = SecretPersistType::TEMPORARY;
123127

124128
result.type = kv_secret.GetType();
125129
result.name = kv_secret.GetName();
@@ -141,7 +145,7 @@ CreateSecretInfo CreateS3SecretFunctions::GenerateRefreshSecretInfo(const Secret
141145

142146
//! Function that will automatically try to refresh a secret
143147
bool CreateS3SecretFunctions::TryRefreshS3Secret(ClientContext &context, const SecretEntry &secret_to_refresh) {
144-
const auto &kv_secret = dynamic_cast<const KeyValueSecret&>(*secret_to_refresh.secret);
148+
const auto &kv_secret = dynamic_cast<const KeyValueSecret &>(*secret_to_refresh.secret);
145149

146150
Value refresh_info;
147151
if (!kv_secret.TryGetValue("refresh_info", refresh_info)) {
@@ -153,12 +157,15 @@ bool CreateS3SecretFunctions::TryRefreshS3Secret(ClientContext &context, const S
153157
// TODO: change SecretManager API to avoid requiring catching this exception
154158
try {
155159
auto res = secret_manager.CreateSecret(context, refresh_input);
156-
auto &new_secret = dynamic_cast<const KeyValueSecret&>(*res->secret);
157-
DUCKDB_LOG_INFO(context, "httpfs.SecretRefresh", "Successfully refreshed secret: %s, new key_id: %s", secret_to_refresh.secret->GetName(), new_secret.TryGetValue("key_id").ToString());
160+
auto &new_secret = dynamic_cast<const KeyValueSecret &>(*res->secret);
161+
DUCKDB_LOG_INFO(context, "Successfully refreshed secret: %s, new key_id: %s",
162+
secret_to_refresh.secret->GetName(), new_secret.TryGetValue("key_id").ToString());
158163
return true;
159164
} catch (std::exception &ex) {
160165
ErrorData error(ex);
161-
string new_message = StringUtil::Format("Exception thrown while trying to refresh secret %s. To fix this, please recreate or remove the secret and try again. Error: '%s'", secret_to_refresh.secret->GetName(), error.Message());
166+
string new_message = StringUtil::Format("Exception thrown while trying to refresh secret %s. To fix this, "
167+
"please recreate or remove the secret and try again. Error: '%s'",
168+
secret_to_refresh.secret->GetName(), error.Message());
162169
throw Exception(error.Type(), new_message);
163170
}
164171
}
@@ -204,6 +211,7 @@ void CreateS3SecretFunctions::RegisterCreateSecretFunction(DatabaseInstance &ins
204211
secret_type.name = type;
205212
secret_type.deserializer = KeyValueSecret::Deserialize<KeyValueSecret>;
206213
secret_type.default_provider = "config";
214+
secret_type.extension = "httpfs";
207215

208216
ExtensionUtil::RegisterSecretType(instance, secret_type);
209217

@@ -218,6 +226,7 @@ void CreateBearerTokenFunctions::Register(DatabaseInstance &instance) {
218226
secret_type_hf.name = HUGGINGFACE_TYPE;
219227
secret_type_hf.deserializer = KeyValueSecret::Deserialize<KeyValueSecret>;
220228
secret_type_hf.default_provider = "config";
229+
secret_type_hf.extension = "httpfs";
221230
ExtensionUtil::RegisterSecretType(instance, secret_type_hf);
222231

223232
// Huggingface config provider

extension/httpfs/crypto.cpp

Lines changed: 86 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -31,68 +31,81 @@ void hex256(hash_bytes &in, hash_str &out) {
3131
}
3232
}
3333

34-
const EVP_CIPHER *GetCipher(const string &key) {
35-
// For now, we only support GCM ciphers
36-
switch (key.size()) {
37-
case 16:
38-
return EVP_aes_128_gcm();
39-
case 24:
40-
return EVP_aes_192_gcm();
41-
case 32:
42-
return EVP_aes_256_gcm();
43-
default:
44-
throw InternalException("Invalid AES key length");
45-
}
46-
}
47-
48-
AESGCMStateSSL::AESGCMStateSSL() : gcm_context(EVP_CIPHER_CTX_new()) {
49-
if (!(gcm_context)) {
34+
AESStateSSL::AESStateSSL(const std::string *key) : context(EVP_CIPHER_CTX_new()) {
35+
if (!(context)) {
5036
throw InternalException("AES GCM failed with initializing context");
5137
}
5238
}
5339

54-
AESGCMStateSSL::~AESGCMStateSSL() {
40+
AESStateSSL::~AESStateSSL() {
5541
// Clean up
56-
EVP_CIPHER_CTX_free(gcm_context);
42+
EVP_CIPHER_CTX_free(context);
5743
}
5844

59-
bool AESGCMStateSSL::IsOpenSSL() {
60-
return ssl;
45+
const EVP_CIPHER *AESStateSSL::GetCipher(const string &key) {
46+
47+
switch (cipher) {
48+
case GCM:
49+
switch (key.size()) {
50+
case 16:
51+
return EVP_aes_128_gcm();
52+
case 24:
53+
return EVP_aes_192_gcm();
54+
case 32:
55+
return EVP_aes_256_gcm();
56+
default:
57+
throw InternalException("Invalid AES key length");
58+
}
59+
case CTR:
60+
switch (key.size()) {
61+
case 16:
62+
return EVP_aes_128_ctr();
63+
case 24:
64+
return EVP_aes_192_ctr();
65+
case 32:
66+
return EVP_aes_256_ctr();
67+
default:
68+
throw InternalException("Invalid AES key length");
69+
}
70+
71+
default:
72+
throw duckdb::InternalException("Invalid Encryption/Decryption Cipher: %d", static_cast<int>(cipher));
73+
}
6174
}
6275

63-
void AESGCMStateSSL::GenerateRandomData(data_ptr_t data, idx_t len) {
76+
void AESStateSSL::GenerateRandomData(data_ptr_t data, idx_t len) {
6477
// generate random bytes for nonce
6578
RAND_bytes(data, len);
6679
}
6780

68-
void AESGCMStateSSL::InitializeEncryption(const_data_ptr_t iv, idx_t iv_len, const string *key) {
81+
void AESStateSSL::InitializeEncryption(const_data_ptr_t iv, idx_t iv_len, const string *key) {
6982
mode = ENCRYPT;
7083

71-
if (1 != EVP_EncryptInit_ex(gcm_context, GetCipher(*key), NULL, const_data_ptr_cast(key->data()), iv)) {
84+
if (1 != EVP_EncryptInit_ex(context, GetCipher(*key), NULL, const_data_ptr_cast(key->data()), iv)) {
7285
throw InternalException("EncryptInit failed");
7386
}
7487
}
7588

76-
void AESGCMStateSSL::InitializeDecryption(const_data_ptr_t iv, idx_t iv_len, const string *key) {
89+
void AESStateSSL::InitializeDecryption(const_data_ptr_t iv, idx_t iv_len, const string *key) {
7790
mode = DECRYPT;
7891

79-
if (1 != EVP_DecryptInit_ex(gcm_context, GetCipher(*key), NULL, const_data_ptr_cast(key->data()), iv)) {
92+
if (1 != EVP_DecryptInit_ex(context, GetCipher(*key), NULL, const_data_ptr_cast(key->data()), iv)) {
8093
throw InternalException("DecryptInit failed");
8194
}
8295
}
8396

84-
size_t AESGCMStateSSL::Process(const_data_ptr_t in, idx_t in_len, data_ptr_t out, idx_t out_len) {
97+
size_t AESStateSSL::Process(const_data_ptr_t in, idx_t in_len, data_ptr_t out, idx_t out_len) {
8598

8699
switch (mode) {
87100
case ENCRYPT:
88-
if (1 != EVP_EncryptUpdate(gcm_context, data_ptr_cast(out), reinterpret_cast<int *>(&out_len),
101+
if (1 != EVP_EncryptUpdate(context, data_ptr_cast(out), reinterpret_cast<int *>(&out_len),
89102
const_data_ptr_cast(in), (int)in_len)) {
90103
throw InternalException("EncryptUpdate failed");
91104
}
92105
break;
93106

94107
case DECRYPT:
95-
if (1 != EVP_DecryptUpdate(gcm_context, data_ptr_cast(out), reinterpret_cast<int *>(&out_len),
108+
if (1 != EVP_DecryptUpdate(context, data_ptr_cast(out), reinterpret_cast<int *>(&out_len),
96109
const_data_ptr_cast(in), (int)in_len)) {
97110

98111
throw InternalException("DecryptUpdate failed");
@@ -107,30 +120,30 @@ size_t AESGCMStateSSL::Process(const_data_ptr_t in, idx_t in_len, data_ptr_t out
107120
return out_len;
108121
}
109122

110-
size_t AESGCMStateSSL::Finalize(data_ptr_t out, idx_t out_len, data_ptr_t tag, idx_t tag_len) {
123+
size_t AESStateSSL::FinalizeGCM(data_ptr_t out, idx_t out_len, data_ptr_t tag, idx_t tag_len) {
111124
auto text_len = out_len;
112125

113126
switch (mode) {
114-
case ENCRYPT:
115-
{
116-
if (1 != EVP_EncryptFinal_ex(gcm_context, data_ptr_cast(out) + out_len, reinterpret_cast<int *>(&out_len))) {
127+
case ENCRYPT: {
128+
if (1 != EVP_EncryptFinal_ex(context, data_ptr_cast(out) + out_len, reinterpret_cast<int *>(&out_len))) {
117129
throw InternalException("EncryptFinal failed");
118130
}
119131
text_len += out_len;
132+
120133
// The computed tag is written at the end of a chunk
121-
if (1 != EVP_CIPHER_CTX_ctrl(gcm_context, EVP_CTRL_GCM_GET_TAG, tag_len, tag)) {
134+
if (1 != EVP_CIPHER_CTX_ctrl(context, EVP_CTRL_GCM_GET_TAG, tag_len, tag)) {
122135
throw InternalException("Calculating the tag failed");
123136
}
124137
return text_len;
125138
}
126-
case DECRYPT:
127-
{
139+
case DECRYPT: {
128140
// Set expected tag value
129-
if (!EVP_CIPHER_CTX_ctrl(gcm_context, EVP_CTRL_GCM_SET_TAG, tag_len, tag)) {
141+
if (!EVP_CIPHER_CTX_ctrl(context, EVP_CTRL_GCM_SET_TAG, tag_len, tag)) {
130142
throw InternalException("Finalizing tag failed");
131143
}
144+
132145
// EVP_DecryptFinal() will return an error code if final block is not correctly formatted.
133-
int ret = EVP_DecryptFinal_ex(gcm_context, data_ptr_cast(out) + out_len, reinterpret_cast<int *>(&out_len));
146+
int ret = EVP_DecryptFinal_ex(context, data_ptr_cast(out) + out_len, reinterpret_cast<int *>(&out_len));
134147
text_len += out_len;
135148

136149
if (ret > 0) {
@@ -144,12 +157,46 @@ size_t AESGCMStateSSL::Finalize(data_ptr_t out, idx_t out_len, data_ptr_t tag, i
144157
}
145158
}
146159

160+
size_t AESStateSSL::Finalize(data_ptr_t out, idx_t out_len, data_ptr_t tag, idx_t tag_len) {
161+
162+
if (cipher == GCM) {
163+
return FinalizeGCM(out, out_len, tag, tag_len);
164+
}
165+
166+
auto text_len = out_len;
167+
switch (mode) {
168+
169+
case ENCRYPT: {
170+
if (1 != EVP_EncryptFinal_ex(context, data_ptr_cast(out) + out_len, reinterpret_cast<int *>(&out_len))) {
171+
throw InternalException("EncryptFinal failed");
172+
}
173+
174+
return text_len += out_len;
175+
}
176+
177+
case DECRYPT: {
178+
// EVP_DecryptFinal() will return an error code if final block is not correctly formatted.
179+
int ret = EVP_DecryptFinal_ex(context, data_ptr_cast(out) + out_len, reinterpret_cast<int *>(&out_len));
180+
text_len += out_len;
181+
182+
if (ret > 0) {
183+
// success
184+
return text_len;
185+
}
186+
187+
throw InvalidInputException("Computed AES tag differs from read AES tag, are you using the right key?");
188+
}
189+
default:
190+
throw InternalException("Unhandled encryption mode %d", static_cast<int>(mode));
191+
}
192+
}
193+
147194
} // namespace duckdb
148195

149196
extern "C" {
150197

151198
// Call the member function through the factory object
152-
DUCKDB_EXTENSION_API AESGCMStateSSLFactory *CreateSSLFactory() {
153-
return new AESGCMStateSSLFactory();
199+
DUCKDB_EXTENSION_API AESStateSSLFactory *CreateSSLFactory() {
200+
return new AESStateSSLFactory();
154201
};
155202
}

0 commit comments

Comments
 (0)