Upgrade to arrow-1.0.1 (#146)

* Upgrade to arrow-1.0.0. * Add Findutf8proc. Revert back to Snappy as LZ4 is not supported due to ongoing compatibility issues. * Remove -DUTF8PROC_STATIC, only relevant when compiling Arrow itself. * Point to vcpkg master with Arrow 1.0.0 PR merged in * Specific commit didn't work, point to master instead * Add pkg-config dependency on macOS * Point at proposed vcpkg arrow-1.0.1 update. * Point vcpkg to master right after Arrow 1.0.1 PR. * Bump version to 2.3.0-beta1
G-Research · Sep 1, 2020 · 5b349ea · 5b349ea
1 parent 1acd5c1
commit 5b349ea
Show file tree

Hide file tree

Showing 12 changed files with 33 additions and 13 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -48,7 +48,7 @@ jobs:
         sudo apt-get --yes install flex bison
     - name: Install vcpkg build dependencies (macOS)
       if: steps.get-cached-vcpkg.outputs.cache-hit != 'true' && runner.os == 'macOS'
-      run: brew install bison
+      run: brew install bison pkg-config
     - name: Compile vcpkg dependencies (Unix)
       if: steps.get-cached-vcpkg.outputs.cache-hit != 'true' && (runner.os == 'Linux' || runner.os == 'macOS')
       run: ./vcpkg_unix.sh

diff --git a/cmake/Findutf8proc.cmake b/cmake/Findutf8proc.cmake
@@ -0,0 +1,18 @@
+
+include(SelectLibraryConfigurations)
+include(FindPackageHandleStandardArgs)
+
+find_path(utf8proc_INCLUDE_DIR utf8proc.h)
+
+if (NOT utf8proc_LIBRARIES)
+    find_library(utf8proc_LIBRARY_RELEASE NAMES utf8proc utf8proc_static PATHS ${CMAKE_PREFIX_PATH}/lib NO_DEFAULT_PATH)
+    find_library(utf8proc_LIBRARY_DEBUG NAMES utf8proc utf8proc_static PATHS ${CMAKE_PREFIX_PATH}/debug/lib NO_DEFAULT_PATH)
+    SELECT_LIBRARY_CONFIGURATIONS(utf8proc)
+endif()
+
+mark_as_advanced(utf8proc_FOUND utf8proc_INCLUDE_DIR utf8proc_LIBRARY_RELEASE utf8proc_LIBRARY_DEBUG)
+find_package_handle_standard_args(utf8proc REQUIRED_VARS utf8proc_INCLUDE_DIR utf8proc_LIBRARIES)
+
+if(utf8proc_FOUND)
+    set(utf8proc_INCLUDE_DIRS ${utf8proc_INCLUDE_DIR})
+endif()
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
@@ -11,6 +11,7 @@ find_package(lz4 CONFIG REQUIRED)
 find_package(OpenSSL REQUIRED)
 find_package(Snappy CONFIG REQUIRED)
 find_package(Thrift CONFIG REQUIRED)
+find_package(utf8proc REQUIRED)
 find_package(ZLIB REQUIRED)
 find_package(Zstd REQUIRED)
 find_package(ParquetCpp REQUIRED)
@@ -96,6 +97,7 @@ target_link_libraries(ParquetSharpNative
 	${SSL_LIBRARIES}
 	Snappy::snappy
 	thrift::thrift
+	${utf8proc_LIBRARIES}
 	ZLIB::ZLIB
 	${Zstd_LIBRARIES} 
 	${Threads_LIBRARIES}

diff --git a/csharp.test/TestByteArrayReaderCache.cs b/csharp.test/TestByteArrayReaderCache.cs
@@ -61,7 +61,7 @@ public static void TestReadingDuplicateStrings([Values(true, false)] bool enable
         private static WriterProperties CreateWriterProperties(bool enableDictionary)
         {
             using var builder = new WriterPropertiesBuilder();
-            builder.Compression(Compression.Lz4);
+            builder.Compression(Compression.Snappy);
             return (enableDictionary ? builder : builder.DisableDictionary("value")).Build();
         }
 

diff --git a/csharp.test/TestEncryption.cs b/csharp.test/TestEncryption.cs
@@ -297,7 +297,7 @@ private static WriterProperties CreateWriterProperties(FileEncryptionProperties
             using var builder = new WriterPropertiesBuilder();
 
             return builder
-                .Compression(Compression.Lz4)
+                .Compression(Compression.Snappy)
                 .Encryption(fileEncryptionProperties)
                 .Build();
         }

diff --git a/csharp.test/TestLogicalTypeRoundtrip.cs b/csharp.test/TestLogicalTypeRoundtrip.cs
@@ -96,7 +96,7 @@ private static WriterProperties CreateWriterProperties(ExpectedColumn[] expected
         {
             var builder = new WriterPropertiesBuilder();
 
-            builder.Compression(Compression.Lz4);
+            builder.Compression(Compression.Snappy);
 
             if (!useDictionaryEncoding)
             {

diff --git a/csharp.test/TestPhysicalTypeRoundtrip.cs b/csharp.test/TestPhysicalTypeRoundtrip.cs
@@ -174,7 +174,7 @@ private static WriterProperties CreateWriterProperties(ExpectedColumn[] expected
         {
             var builder = new WriterPropertiesBuilder();
 
-            builder.Compression(Compression.Lz4);
+            builder.Compression(Compression.Snappy);
 
             if (!useDictionaryEncoding)
             {
@@ -248,7 +248,7 @@ private sealed class ExpectedColumn
             public int TypeScale = -1;
 
             public Encoding[] Encodings = {Encoding.PlainDictionary, Encoding.Plain, Encoding.Rle};
-            public Compression Compression = Compression.Lz4;
+            public Compression Compression = Compression.Snappy;
         }
     }
 }
diff --git a/csharp.test/TestWriterProperties.cs b/csharp.test/TestWriterProperties.cs
@@ -73,7 +73,7 @@ public static void TestByteStreamSplitEncoding()
                 };
 
                 var p = new WriterPropertiesBuilder()
-                    .Compression(Compression.Lz4)
+                    .Compression(Compression.Snappy)
                     .DisableDictionary("value")
                     .Encoding("value", Encoding.ByteStreamSplit)
                     .Build();

diff --git a/csharp/ParquetFileWriter.cs b/csharp/ParquetFileWriter.cs
@@ -11,7 +11,7 @@ public sealed class ParquetFileWriter : IDisposable
         public ParquetFileWriter(
             string path, 
             Column[] columns, 
-            Compression compression = Compression.Lz4, 
+            Compression compression = Compression.Snappy, 
             IReadOnlyDictionary<string, string> keyValueMetadata = null)
         {
             using (var schema = Column.CreateSchemaNode(columns))
@@ -24,7 +24,7 @@ public ParquetFileWriter(
         public ParquetFileWriter(
             OutputStream outputStream, 
             Column[] columns, 
-            Compression compression = Compression.Lz4, 
+            Compression compression = Compression.Snappy, 
             IReadOnlyDictionary<string, string> keyValueMetadata = null)
         {
             using (var schema = Column.CreateSchemaNode(columns))

diff --git a/csharp/ParquetSharp.csproj b/csharp/ParquetSharp.csproj
@@ -13,7 +13,7 @@
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>
     <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
     <NoWarn>1591;</NoWarn>
-    <Version>2.2.0</Version>
+    <Version>2.3.0-beta1</Version>
     <Company>G-Research</Company>
     <Authors>G-Research</Authors>
     <Product>ParquetSharp</Product>

diff --git a/csharp/RowOriented/ParquetFile.cs b/csharp/RowOriented/ParquetFile.cs
@@ -43,7 +43,7 @@ public static ParquetRowReader<TTuple> CreateRowReader<TTuple>(RandomAccessFile
         public static ParquetRowWriter<TTuple> CreateRowWriter<TTuple>(
             string path,
             string[] columnNames = null,
-            Compression compression = Compression.Lz4,
+            Compression compression = Compression.Snappy,
             IReadOnlyDictionary<string, string> keyValueMetadata = null)
         {
             var (columns, writeDelegate) = GetOrCreateWriteDelegate<TTuple>(columnNames);
@@ -56,7 +56,7 @@ public static ParquetRowWriter<TTuple> CreateRowWriter<TTuple>(
         public static ParquetRowWriter<TTuple> CreateRowWriter<TTuple>(
             OutputStream outputStream,
             string[] columnNames = null,
-            Compression compression = Compression.Lz4,
+            Compression compression = Compression.Snappy,
             IReadOnlyDictionary<string, string> keyValueMetadata = null)
         {
             var (columns, writeDelegate) = GetOrCreateWriteDelegate<TTuple>(columnNames);

diff --git a/vcpkg_version.txt b/vcpkg_version.txt
@@ -1 +1 @@
-https://github.com/microsoft/vcpkg.git 2020.06
+https://github.com/Microsoft/vcpkg.git aea911913663ee417391e64c74814b34c7da0345
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		https://github.com/microsoft/vcpkg.git 2020.06
		https://github.com/Microsoft/vcpkg.git aea911913663ee417391e64c74814b34c7da0345