Add the key management tools API for encryption (#426)

G-Research · Mar 4, 2024 · e767413 · e767413
1 parent d531836
commit e767413
Show file tree

Hide file tree

Showing 35 changed files with 3,065 additions and 40 deletions.
diff --git a/README.md b/README.md
@@ -98,6 +98,7 @@ For more detailed information on how to use ParquetSharp, see the following docu
 * [Row-oriented API](docs/RowOriented.md) &mdash; a higher level API that abstracts away the column-oriented nature of Parquet files
 * [Custom types](docs/TypeFactories.md) &mdash; how to customize the mapping between .NET and Parquet types,
     including using the `DateOnly` and `TimeOnly` types added in .NET 6.
+* [Encryption](docs/Encryption.md) &mdash; using Parquet Modular Encryption to read and write encrypted data
 * [Writing TimeSpan data](docs/TimeSpan.md) &mdash; interoperability with other libraries when writing TimeSpan data
 * [Use from PowerShell](docs/PowerShell.md)
 

diff --git a/build_windows.ps1 b/build_windows.ps1
@@ -34,6 +34,18 @@ if ($Env:GITHUB_ACTIONS -eq "true") {
   $customTripletFile = "$customTripletsDir/$triplet.cmake"
   Copy-Item -Path $sourceTripletFile -Destination $customTripletFile
   Add-Content -Path $customTripletFile -Value "set(VCPKG_BUILD_TYPE release)"
+
+  # Ensure vcpkg uses the same MSVC version to build dependencies as we use to build the ParquetSharp library.
+  # By default, vcpkg uses the most recent version it can find, which might not be the same as what msbuild uses.
+  $vsInstPath = & "${env:ProgramFiles(x86)}/Microsoft Visual Studio/Installer/vswhere.exe" -latest -property installationPath
+  Import-Module "$vsInstPath/Common7/Tools/Microsoft.VisualStudio.DevShell.dll"
+  Enter-VsDevShell -VsInstallPath $vsInstPath -SkipAutomaticLocation
+  $clPath = Get-Command cl.exe | Select -ExpandProperty "Source"
+  $toolsetVersion = $clPath.Split("\")[8]
+  if (-not $toolsetVersion.StartsWith("14.")) { throw "Couldn't get toolset version from path '$clPath'" }
+  Write-Output "Using platform toolset version = $toolsetVersion"
+  Add-Content -Path $customTripletFile -Value "set(VCPKG_PLATFORM_TOOLSET_VERSION $toolsetVersion)"
+
   $options += "-D"
   $options += "VCPKG_OVERLAY_TRIPLETS=$customTripletsDir"
 }

diff --git a/cpp/Buffer.cpp b/cpp/Buffer.cpp
@@ -26,6 +26,11 @@ extern "C"
 		TRYCATCH(*data = (*buffer)->data();)
 	}
 
+	PARQUETSHARP_EXPORT ExceptionInfo* Buffer_MutableData(const std::shared_ptr<arrow::Buffer>* buffer, uint8_t** data)
+	{
+		TRYCATCH(*data = (*buffer)->mutable_data();)
+	}
+
 	PARQUETSHARP_EXPORT ExceptionInfo* Buffer_Size(const std::shared_ptr<arrow::Buffer>* buffer, int64_t* size)
 	{
 		TRYCATCH(*size = (*buffer)->size();)

diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
@@ -67,6 +67,12 @@ add_library(ParquetSharpNative SHARED
 	arrow/ArrowWriterPropertiesBuilder.cpp
 	arrow/FileReader.cpp
 	arrow/FileWriter.cpp
+	encryption/CryptoFactory.cpp
+	encryption/DecryptionConfiguration.cpp
+	encryption/EncryptionConfiguration.cpp
+	encryption/KmsConnectionConfig.cpp
+	encryption/ManagedKmsClient.h
+	encryption/ManagedKmsClientFactory.h
 )
 
 generate_export_header(ParquetSharpNative 

diff --git a/cpp/FileDecryptionProperties.cpp b/cpp/FileDecryptionProperties.cpp
@@ -15,7 +15,7 @@ extern "C"
     {
         TRYCATCH(*clone = new std::shared_ptr((*properties)->DeepClone());)
     }
-	
+
     PARQUETSHARP_EXPORT void FileDecryptionProperties_Free(const std::shared_ptr<const FileDecryptionProperties>* properties)
     {
         delete properties;
@@ -41,12 +41,14 @@ extern "C"
         FreeCString(aad_prefix);
     }
 
-	PARQUETSHARP_EXPORT ExceptionInfo* FileDecryptionProperties_Key_Retriever(const std::shared_ptr<const FileDecryptionProperties>* properties, void** key_retriever)
+    PARQUETSHARP_EXPORT ExceptionInfo* FileDecryptionProperties_Key_Retriever(const std::shared_ptr<const FileDecryptionProperties>* properties, void** key_retriever)
     {
         TRYCATCH
         (
-            const auto r = (*properties)->key_retriever();
-            *key_retriever = r ? dynamic_cast<ManagedDecryptionKeyRetriever&>(*r).Handle : nullptr;
+            // This only returns a KeyRetriever handle when a ManagedDecryptionKeyRetriever is used.
+            // If the key retriever is set using the Key Management Tools API (CryptoFactory) then this will return null.
+            const auto retriever = std::dynamic_pointer_cast<ManagedDecryptionKeyRetriever>((*properties)->key_retriever());
+            *key_retriever = retriever ? retriever->Handle : nullptr;
         )
     }
 

diff --git a/cpp/FileEncryptionProperties.cpp b/cpp/FileEncryptionProperties.cpp
@@ -13,7 +13,7 @@ extern "C"
     {
         TRYCATCH(*clone = new std::shared_ptr((*properties)->DeepClone());)
     }
-	
+
     PARQUETSHARP_EXPORT void FileEncryptionProperties_Free(const std::shared_ptr<const FileEncryptionProperties>* properties)
     {
         delete properties;
@@ -54,14 +54,15 @@ extern "C"
         FreeCString(file_aad);
     }
 
-    PARQUETSHARP_EXPORT ExceptionInfo* FileEncryptionProperties_Column_Encryption_Properties(const std::shared_ptr<FileEncryptionProperties>* properties, const char* column_path, std::shared_ptr<ColumnEncryptionProperties>* column_encryption_properties)
+    PARQUETSHARP_EXPORT ExceptionInfo* FileEncryptionProperties_Column_Encryption_Properties(const std::shared_ptr<FileEncryptionProperties>* properties, const char* column_path, std::shared_ptr<const ColumnEncryptionProperties>** column_encryption_properties)
     {
-        TRYCATCH(*column_encryption_properties = (*properties)->column_encryption_properties(column_path);)
+        TRYCATCH(
+            std::shared_ptr<const ColumnEncryptionProperties> column_properties = (*properties)->column_encryption_properties(column_path);
+            if (column_properties != nullptr) {
+              *column_encryption_properties = new std::shared_ptr<const ColumnEncryptionProperties>(column_properties);
+            } else {
+              *column_encryption_properties = nullptr;
+            }
+        )
     }
-
-	// TODO: do we really need this?
-    //PARQUETSHARP_EXPORT ExceptionInfo* FileEncryptionProperties_Encrypted_Columns(const std::shared_ptr<const FileEncryptionProperties>* properties, bool* is_encrypted_with_footer_key)
-    //{
-    //    TRYCATCH(*encrypted_columns = (*properties)->encrypted_columns();)
-    //}
 }
diff --git a/cpp/ResizableBuffer.cpp b/cpp/ResizableBuffer.cpp
@@ -4,6 +4,7 @@
 
 #include <arrow/buffer.h>
 #include <arrow/result.h>
+#include <parquet/exception.h>
 
 extern "C"
 {
@@ -14,4 +15,11 @@ extern "C"
 			*buffer = new std::shared_ptr<arrow::ResizableBuffer>(pBuffer.ValueOrDie().release());
 		)
 	}
+
+	PARQUETSHARP_EXPORT ExceptionInfo* ResizableBuffer_Resize(std::shared_ptr<arrow::ResizableBuffer>* buffer, int64_t new_size)
+	{
+		TRYCATCH(
+			PARQUET_THROW_NOT_OK((*buffer)->Resize(new_size));
+		)
+	}
 }
diff --git a/cpp/encryption/CryptoFactory.cpp b/cpp/encryption/CryptoFactory.cpp
@@ -0,0 +1,85 @@
+#include <parquet/encryption/encryption.h>
+#include <parquet/encryption/crypto_factory.h>
+#include <arrow/filesystem/localfs.h>
+
+#include "cpp/ParquetSharpExport.h"
+#include "../ExceptionInfo.h"
+#include "ManagedKmsClientFactory.h"
+
+using namespace parquet::encryption;
+
+extern "C"
+{
+  PARQUETSHARP_EXPORT ExceptionInfo* CryptoFactory_Create(CryptoFactory** crypto_factory)
+  {
+    TRYCATCH(*crypto_factory = new CryptoFactory();)
+  }
+
+  PARQUETSHARP_EXPORT void CryptoFactory_Free(CryptoFactory* crypto_factory)
+  {
+    delete crypto_factory;
+  }
+
+  PARQUETSHARP_EXPORT ExceptionInfo* CryptoFactory_RegisterKmsClientFactory(
+      CryptoFactory* crypto_factory,
+      void* const client_factory_handle,
+      const ManagedKmsClient::FreeGcHandleFunc free_gc_handle,
+      const ManagedKmsClientFactory::CreateClientFunc create_client,
+      const ManagedKmsClient::WrapFunc wrap,
+      const ManagedKmsClient::UnwrapFunc unwrap)
+  {
+    TRYCATCH(
+      crypto_factory->RegisterKmsClientFactory(
+          std::make_shared<ManagedKmsClientFactory>(client_factory_handle, free_gc_handle, create_client, wrap, unwrap));
+    )
+  }
+
+  PARQUETSHARP_EXPORT ExceptionInfo* CryptoFactory_GetFileEncryptionProperties(
+      CryptoFactory* crypto_factory,
+      const KmsConnectionConfig* kms_connection_config,
+      const EncryptionConfiguration* encryption_configuration,
+      const char* file_path,
+      std::shared_ptr<parquet::FileEncryptionProperties>** file_encryption_properties)
+  {
+    TRYCATCH(
+        std::string file_path_str = file_path == nullptr ? "" : file_path;
+        std::shared_ptr<::arrow::fs::FileSystem> file_system = file_path_str.empty() ?
+            nullptr : std::make_shared<::arrow::fs::LocalFileSystem>();
+        (*file_encryption_properties) = new std::shared_ptr<parquet::FileEncryptionProperties>(
+            crypto_factory->GetFileEncryptionProperties(
+                *kms_connection_config, *encryption_configuration, file_path_str, file_system));
+    )
+  }
+
+  PARQUETSHARP_EXPORT ExceptionInfo* CryptoFactory_GetFileDecryptionProperties(
+      CryptoFactory* crypto_factory,
+      const KmsConnectionConfig* kms_connection_config,
+      const DecryptionConfiguration* decryption_configuration,
+      const char* file_path,
+      std::shared_ptr<parquet::FileDecryptionProperties>** file_decryption_properties)
+  {
+    TRYCATCH(
+        std::string file_path_str = file_path == nullptr ? "" : file_path;
+        std::shared_ptr<::arrow::fs::FileSystem> file_system = file_path_str.empty() ?
+            nullptr : std::make_shared<::arrow::fs::LocalFileSystem>();
+        (*file_decryption_properties) = new std::shared_ptr<parquet::FileDecryptionProperties>(
+            crypto_factory->GetFileDecryptionProperties(
+                *kms_connection_config, *decryption_configuration, file_path_str, file_system));
+    )
+  }
+
+  PARQUETSHARP_EXPORT ExceptionInfo* CryptoFactory_RotateMasterKeys(
+      CryptoFactory* crypto_factory,
+      const KmsConnectionConfig* kms_connection_config,
+      const char* file_path,
+      bool double_wrapping,
+      double cache_lifetime_seconds)
+  {
+    TRYCATCH(
+        std::string file_path_str = file_path == nullptr ? "" : file_path;
+        std::shared_ptr<::arrow::fs::FileSystem> file_system = std::make_shared<::arrow::fs::LocalFileSystem>();
+        crypto_factory->RotateMasterKeys(
+            *kms_connection_config, file_path_str, file_system, double_wrapping, cache_lifetime_seconds);
+    )
+  }
+}
diff --git a/cpp/encryption/DecryptionConfiguration.cpp b/cpp/encryption/DecryptionConfiguration.cpp
@@ -0,0 +1,29 @@
+#include <parquet/encryption/crypto_factory.h>
+
+#include "cpp/ParquetSharpExport.h"
+#include "../ExceptionInfo.h"
+
+using namespace parquet::encryption;
+
+extern "C"
+{
+  PARQUETSHARP_EXPORT ExceptionInfo* DecryptionConfiguration_Create(DecryptionConfiguration** configuration)
+  {
+    TRYCATCH(*configuration = new DecryptionConfiguration();)
+  }
+
+  PARQUETSHARP_EXPORT void DecryptionConfiguration_Free(DecryptionConfiguration* configuration)
+  {
+    delete configuration;
+  }
+
+  PARQUETSHARP_EXPORT ExceptionInfo* DecryptionConfiguration_GetCacheLifetimeSeconds(const DecryptionConfiguration* configuration, double* cache_lifetime_seconds)
+  {
+    TRYCATCH(*cache_lifetime_seconds = configuration->cache_lifetime_seconds;)
+  }
+
+  PARQUETSHARP_EXPORT ExceptionInfo* DecryptionConfiguration_SetCacheLifetimeSeconds(DecryptionConfiguration* configuration, double cache_lifetime_seconds)
+  {
+    TRYCATCH(configuration->cache_lifetime_seconds = cache_lifetime_seconds;)
+  }
+}
diff --git a/cpp/encryption/EncryptionConfiguration.cpp b/cpp/encryption/EncryptionConfiguration.cpp
@@ -0,0 +1,109 @@
+#include <parquet/encryption/crypto_factory.h>
+
+#include "cpp/ParquetSharpExport.h"
+#include "../ExceptionInfo.h"
+
+using namespace parquet::encryption;
+
+extern "C"
+{
+  PARQUETSHARP_EXPORT ExceptionInfo* EncryptionConfiguration_Create(const char* footer_key, EncryptionConfiguration** configuration)
+  {
+    TRYCATCH(*configuration = new EncryptionConfiguration(footer_key == nullptr ? "" : footer_key);)
+  }
+
+  PARQUETSHARP_EXPORT void EncryptionConfiguration_Free(EncryptionConfiguration* configuration)
+  {
+    delete configuration;
+  }
+
+  PARQUETSHARP_EXPORT ExceptionInfo* EncryptionConfiguration_GetFooterKey(const EncryptionConfiguration* configuration, const char** footer_key)
+  {
+    TRYCATCH(*footer_key = configuration->footer_key.c_str();)
+  }
+
+  PARQUETSHARP_EXPORT ExceptionInfo* EncryptionConfiguration_SetFooterKey(EncryptionConfiguration* configuration, const char* footer_key)
+  {
+    TRYCATCH(configuration->footer_key = footer_key;)
+  }
+
+  PARQUETSHARP_EXPORT ExceptionInfo* EncryptionConfiguration_GetColumnKeys(const EncryptionConfiguration* configuration, const char** column_keys)
+  {
+    TRYCATCH(*column_keys = configuration->column_keys.c_str();)
+  }
+
+  PARQUETSHARP_EXPORT ExceptionInfo* EncryptionConfiguration_SetColumnKeys(EncryptionConfiguration* configuration, const char* column_keys)
+  {
+    TRYCATCH(configuration->column_keys = column_keys;)
+  }
+
+  PARQUETSHARP_EXPORT ExceptionInfo* EncryptionConfiguration_GetUniformEncryption(const EncryptionConfiguration* configuration, bool* uniform_encryption)
+  {
+    TRYCATCH(*uniform_encryption = configuration->uniform_encryption;)
+  }
+
+  PARQUETSHARP_EXPORT ExceptionInfo* EncryptionConfiguration_SetUniformEncryption(EncryptionConfiguration* configuration, bool uniform_encryption)
+  {
+    TRYCATCH(configuration->uniform_encryption = uniform_encryption;)
+  }
+
+  PARQUETSHARP_EXPORT ExceptionInfo* EncryptionConfiguration_GetEncryptionAlgorithm(const EncryptionConfiguration* configuration, parquet::ParquetCipher::type* encryption_algorithm)
+  {
+    TRYCATCH(*encryption_algorithm = configuration->encryption_algorithm;)
+  }
+
+  PARQUETSHARP_EXPORT ExceptionInfo* EncryptionConfiguration_SetEncryptionAlgorithm(EncryptionConfiguration* configuration, parquet::ParquetCipher::type encryption_algorithm)
+  {
+    TRYCATCH(configuration->encryption_algorithm = encryption_algorithm;)
+  }
+
+  PARQUETSHARP_EXPORT ExceptionInfo* EncryptionConfiguration_GetPlaintextFooter(const EncryptionConfiguration* configuration, bool* plaintext_footer)
+  {
+    TRYCATCH(*plaintext_footer = configuration->plaintext_footer;)
+  }
+
+  PARQUETSHARP_EXPORT ExceptionInfo* EncryptionConfiguration_SetPlaintextFooter(EncryptionConfiguration* configuration, bool plaintext_footer)
+  {
+    TRYCATCH(configuration->plaintext_footer = plaintext_footer;)
+  }
+
+  PARQUETSHARP_EXPORT ExceptionInfo* EncryptionConfiguration_GetDoubleWrapping(const EncryptionConfiguration* configuration, bool* double_wrapping)
+  {
+    TRYCATCH(*double_wrapping = configuration->double_wrapping;)
+  }
+
+  PARQUETSHARP_EXPORT ExceptionInfo* EncryptionConfiguration_SetDoubleWrapping(EncryptionConfiguration* configuration, bool double_wrapping)
+  {
+    TRYCATCH(configuration->double_wrapping = double_wrapping;)
+  }
+
+  PARQUETSHARP_EXPORT ExceptionInfo* EncryptionConfiguration_GetCacheLifetimeSeconds(const EncryptionConfiguration* configuration, double* cache_lifetime_seconds)
+  {
+    TRYCATCH(*cache_lifetime_seconds = configuration->cache_lifetime_seconds;)
+  }
+
+  PARQUETSHARP_EXPORT ExceptionInfo* EncryptionConfiguration_SetCacheLifetimeSeconds(EncryptionConfiguration* configuration, double cache_lifetime_seconds)
+  {
+    TRYCATCH(configuration->cache_lifetime_seconds = cache_lifetime_seconds;)
+  }
+
+  PARQUETSHARP_EXPORT ExceptionInfo* EncryptionConfiguration_GetInternalKeyMaterial(const EncryptionConfiguration* configuration, bool* internal_key_material)
+  {
+    TRYCATCH(*internal_key_material = configuration->internal_key_material;)
+  }
+
+  PARQUETSHARP_EXPORT ExceptionInfo* EncryptionConfiguration_SetInternalKeyMaterial(EncryptionConfiguration* configuration, bool internal_key_material)
+  {
+    TRYCATCH(configuration->internal_key_material = internal_key_material;)
+  }
+
+  PARQUETSHARP_EXPORT ExceptionInfo* EncryptionConfiguration_GetDataKeyLengthBits(const EncryptionConfiguration* configuration, int32_t* data_key_length_bits)
+  {
+    TRYCATCH(*data_key_length_bits = configuration->data_key_length_bits;)
+  }
+
+  PARQUETSHARP_EXPORT ExceptionInfo* EncryptionConfiguration_SetDataKeyLengthBits(EncryptionConfiguration* configuration, int32_t data_key_length_bits)
+  {
+    TRYCATCH(configuration->data_key_length_bits = data_key_length_bits;)
+  }
+}