From 24f2aa9e7ce22259b6e2014237d82196a1781595 Mon Sep 17 00:00:00 2001 From: Adam Reeve Date: Wed, 13 Sep 2023 09:17:56 +1200 Subject: [PATCH] Upgrade the NUnit test adapter (#372) --- csharp.test/ParquetSharp.Test.csproj | 2 +- csharp.test/TestColumn.cs | 44 +++++++----- csharp.test/TestDecimal128.cs | 6 +- csharp.test/TestLogicalTypeRoundtrip.cs | 80 +++++++++++---------- csharp.test/TestPhysicalTypeRoundtrip.cs | 78 ++++++++++---------- fsharp.test/ParquetSharp.Test.FSharp.fsproj | 2 +- 6 files changed, 115 insertions(+), 97 deletions(-) diff --git a/csharp.test/ParquetSharp.Test.csproj b/csharp.test/ParquetSharp.Test.csproj index a286bc71..24fd1400 100644 --- a/csharp.test/ParquetSharp.Test.csproj +++ b/csharp.test/ParquetSharp.Test.csproj @@ -19,7 +19,7 @@ - + diff --git a/csharp.test/TestColumn.cs b/csharp.test/TestColumn.cs index 9437bb36..464a328e 100644 --- a/csharp.test/TestColumn.cs +++ b/csharp.test/TestColumn.cs @@ -16,29 +16,35 @@ public static void TestPrimitives() { foreach (var expected in expectedPrimitives) { - Console.WriteLine("Testing primitive type {0}", expected.Type); + try + { + Assert.True(LogicalTypeFactory.Default.IsSupported(expected.Type)); + var type = expected.Type; + var column = new Column(type, expected.Name, expected.LogicalTypeOverride); - Assert.True(LogicalTypeFactory.Default.IsSupported(expected.Type)); - var type = expected.Type; - var column = new Column(type, expected.Name, expected.LogicalTypeOverride); + using var node = column.CreateSchemaNode(); - using var node = column.CreateSchemaNode(); + using var nodeLogicalType = node.LogicalType; + Assert.AreEqual(expected.LogicalType, nodeLogicalType); + Assert.AreEqual(-1, node.FieldId); + Assert.AreEqual(expected.Name, node.Name); + Assert.AreEqual(NodeType.Primitive, node.NodeType); + Assert.AreEqual(null, node.Parent); + Assert.AreEqual(expected.Repetition, node.Repetition); - using var nodeLogicalType = node.LogicalType; - Assert.AreEqual(expected.LogicalType, nodeLogicalType); - Assert.AreEqual(-1, node.FieldId); - Assert.AreEqual(expected.Name, node.Name); - Assert.AreEqual(NodeType.Primitive, node.NodeType); - Assert.AreEqual(null, node.Parent); - Assert.AreEqual(expected.Repetition, node.Repetition); + var primitive = (PrimitiveNode) node; - var primitive = (PrimitiveNode) node; - - Assert.AreEqual(expected.ColumnOrder, primitive.ColumnOrder); - Assert.AreEqual(expected.PhysicalType, primitive.PhysicalType); - Assert.AreEqual(expected.Length, primitive.TypeLength); - using var logicalType = primitive.LogicalType; - Assert.AreEqual(expected.LogicalType, logicalType); + Assert.AreEqual(expected.ColumnOrder, primitive.ColumnOrder); + Assert.AreEqual(expected.PhysicalType, primitive.PhysicalType); + Assert.AreEqual(expected.Length, primitive.TypeLength); + using var logicalType = primitive.LogicalType; + Assert.AreEqual(expected.LogicalType, logicalType); + } + catch (Exception) + { + TestContext.Out.WriteLine("Failure testing primitive type {0}", expected.Type); + throw; + } } } finally diff --git a/csharp.test/TestDecimal128.cs b/csharp.test/TestDecimal128.cs index 506740a7..9bfb785e 100644 --- a/csharp.test/TestDecimal128.cs +++ b/csharp.test/TestDecimal128.cs @@ -30,11 +30,9 @@ public static void TestRoundTrip(int scale) foreach (var value in decimals) { - Console.WriteLine($"{value:E}"); - Assert.AreEqual(value, new Decimal128(value, multiplier).ToDecimal(multiplier)); + Assert.That(value, Is.EqualTo(new Decimal128(value, multiplier).ToDecimal(multiplier))); - Console.WriteLine($"{-value:E}"); - Assert.AreEqual(-value, new Decimal128(-value, multiplier).ToDecimal(multiplier)); + Assert.That(-value, Is.EqualTo(new Decimal128(-value, multiplier).ToDecimal(multiplier))); } } diff --git a/csharp.test/TestLogicalTypeRoundtrip.cs b/csharp.test/TestLogicalTypeRoundtrip.cs index 922a6144..2a6c46f7 100644 --- a/csharp.test/TestLogicalTypeRoundtrip.cs +++ b/csharp.test/TestLogicalTypeRoundtrip.cs @@ -37,17 +37,21 @@ public static void TestRoundTrip( foreach (var column in expectedColumns) { - Console.WriteLine("Writing '{0}' ({1})", column.Name, column.Values.GetType().GetElementType()); - - using var columnWriter = rowGroupWriter.NextColumn().LogicalWriter(writeBufferLength); - columnWriter.Apply(new LogicalValueSetter(column.Values, rowsPerBatch)); + try + { + using var columnWriter = rowGroupWriter.NextColumn().LogicalWriter(writeBufferLength); + columnWriter.Apply(new LogicalValueSetter(column.Values, rowsPerBatch)); + } + catch (Exception) + { + TestContext.Out.WriteLine("Failure writing '{0}' ({1})", column.Name, column.Values.GetType().GetElementType()); + throw; + } } fileWriter.Close(); } - Console.WriteLine(); - // Read back the columns and make sure they match. AssertReadRoundtrip(rowsPerBatch, readBufferLength, buffer, expectedColumns); } @@ -94,8 +98,6 @@ public static void TestRoundTripBuffered( var column = expectedColumns[i]; var range = (r, Math.Min(r + rangeLength, NumRows)); - Console.WriteLine("Writing '{0}' (element type: {1}) (range: {2})", column.Name, column.Values.GetType().GetElementType(), range); - using var columnWriter = rowGroupWriter.Column(i).LogicalWriter(writeBufferLength); columnWriter.Apply(new LogicalValueSetter(column.Values, rowsPerBatch, range)); } @@ -104,8 +106,6 @@ public static void TestRoundTripBuffered( fileWriter.Close(); } - Console.WriteLine(); - // Read back the columns and make sure they match. AssertReadRoundtrip(rowsPerBatch, readBufferLength, buffer, expectedColumns); } @@ -295,38 +295,46 @@ private static void AssertReadRoundtrip(int rowsPerBatch, int readBufferLength, using var chunkMetaData = rowGroupMetaData.GetColumnChunkMetaData(c); using var statistics = chunkMetaData.Statistics; - Console.WriteLine("Reading '{0}'", expected.Name); - - using var root = fileMetaData.Schema.ColumnRoot(c); - Assert.AreEqual(expected.Name, root.Name); - using var path = descr.Path; - Assert.AreEqual(expected.Name, path.ToDotVector().First()); - Assert.AreEqual(c, fileMetaData.Schema.ColumnIndex(path.ToDotString())); - Assert.AreEqual(expected.PhysicalType, descr.PhysicalType); - using var logicalType = descr.LogicalType; - Assert.AreEqual(expected.LogicalType, logicalType); - Assert.AreEqual(expected.Values, columnReader.Apply(new LogicalValueGetter(checked((int) numRows), rowsPerBatch))); - Assert.AreEqual(expected.Length, descr.TypeLength); - Assert.AreEqual((expected.LogicalType as DecimalLogicalType)?.Precision ?? -1, descr.TypePrecision); - Assert.AreEqual((expected.LogicalType as DecimalLogicalType)?.Scale ?? -1, descr.TypeScale); - Assert.AreEqual(expected.HasStatistics, chunkMetaData.IsStatsSet); - - if (expected.HasStatistics) + try { - Assert.AreEqual(expected.HasMinMax, statistics?.HasMinMax); - Assert.AreEqual(expected.NullCount, statistics?.NullCount); - Assert.AreEqual(expected.NumValues, statistics?.NumValues); - Assert.AreEqual(expected.PhysicalType, statistics?.PhysicalType); + using var root = fileMetaData.Schema.ColumnRoot(c); + Assert.AreEqual(expected.Name, root.Name); + using var path = descr.Path; + Assert.AreEqual(expected.Name, path.ToDotVector().First()); + Assert.AreEqual(c, fileMetaData.Schema.ColumnIndex(path.ToDotString())); + Assert.AreEqual(expected.PhysicalType, descr.PhysicalType); + using var logicalType = descr.LogicalType; + Assert.AreEqual(expected.LogicalType, logicalType); + Assert.AreEqual(expected.Values, + columnReader.Apply(new LogicalValueGetter(checked((int) numRows), rowsPerBatch))); + Assert.AreEqual(expected.Length, descr.TypeLength); + Assert.AreEqual((expected.LogicalType as DecimalLogicalType)?.Precision ?? -1, + descr.TypePrecision); + Assert.AreEqual((expected.LogicalType as DecimalLogicalType)?.Scale ?? -1, descr.TypeScale); + Assert.AreEqual(expected.HasStatistics, chunkMetaData.IsStatsSet); + + if (expected.HasStatistics) + { + Assert.AreEqual(expected.HasMinMax, statistics?.HasMinMax); + Assert.AreEqual(expected.NullCount, statistics?.NullCount); + Assert.AreEqual(expected.NumValues, statistics?.NumValues); + Assert.AreEqual(expected.PhysicalType, statistics?.PhysicalType); - if (expected.HasMinMax) + if (expected.HasMinMax) + { + Assert.AreEqual(expected.Min, expected.Converter(statistics!.MinUntyped, descr)); + Assert.AreEqual(expected.Max, expected.Converter(statistics!.MaxUntyped, descr)); + } + } + else { - Assert.AreEqual(expected.Min, expected.Converter(statistics!.MinUntyped, descr)); - Assert.AreEqual(expected.Max, expected.Converter(statistics!.MaxUntyped, descr)); + Assert.IsNull(statistics); } } - else + catch (Exception) { - Assert.IsNull(statistics); + TestContext.Out.WriteLine("Failure reading '{0}'", expected.Name); + throw; } } diff --git a/csharp.test/TestPhysicalTypeRoundtrip.cs b/csharp.test/TestPhysicalTypeRoundtrip.cs index 5616178a..c2f00f66 100644 --- a/csharp.test/TestPhysicalTypeRoundtrip.cs +++ b/csharp.test/TestPhysicalTypeRoundtrip.cs @@ -98,10 +98,16 @@ private static void TestRoundTrip(ExpectedColumn[] expectedColumns, bool useDict foreach (var column in expectedColumns) { - Console.WriteLine("Writing '{0}'", column.Name); - - using var columnWriter = rowGroupWriter.NextColumn(); - columnWriter.Apply(new ValueSetter(column.Values)); + try + { + using var columnWriter = rowGroupWriter.NextColumn(); + columnWriter.Apply(new ValueSetter(column.Values)); + } + catch (Exception) + { + TestContext.WriteLine("Failure writing '{0}'", column.Name); + throw; + } } fileWriter.Close(); @@ -137,11 +143,6 @@ private static void TestRoundTripBuffered(ExpectedColumn[] expectedColumns, bool var column = expectedColumns[i]; var range = (r, Math.Min(r + rangeLength, numRows)); - if (range.Item1 == 0 || range.Item2 == numRows) - { - Console.WriteLine("Writing '{0}' (range: {1})", column.Name, range); - } - using var columnWriter = rowGroupWriter.Column(i); columnWriter.Apply(new ValueSetter(column.Values, range)); } @@ -181,34 +182,39 @@ private static void AssertReadRoundtrip(ResizableBuffer buffer, ExpectedColumn[] for (int c = 0; c != fileMetaData.NumColumns; ++c) { using var columnReader = rowGroupReader.Column(c); - var expected = expectedColumns[c]; - - Console.WriteLine("Reading '{0}'", expected.Name); - - var descr = columnReader.ColumnDescriptor; - using var chunkMetaData = rowGroupMetaData.GetColumnChunkMetaData(c); - - Assert.AreEqual(expected.MaxDefinitionlevel, descr.MaxDefinitionLevel); - Assert.AreEqual(expected.MaxRepetitionLevel, descr.MaxRepetitionLevel); - Assert.AreEqual(expected.PhysicalType, descr.PhysicalType); - using var logicalType = descr.LogicalType; - Assert.AreEqual(expected.LogicalType, logicalType); - Assert.AreEqual(expected.ColumnOrder, descr.ColumnOrder); - Assert.AreEqual(expected.SortOrder, descr.SortOrder); - Assert.AreEqual(expected.Name, descr.Name); - Assert.AreEqual(expected.TypeLength, descr.TypeLength); - Assert.AreEqual(expected.TypePrecision, descr.TypePrecision); - Assert.AreEqual(expected.TypeScale, descr.TypeScale); - - var expectedEncodings = expected.Encodings - .Where(e => useDictionaryEncoding || e != Encoding.RleDictionary).ToArray(); - var actualEncodings = chunkMetaData.Encodings.Distinct().ToArray(); - // Encoding ordering is not important - Assert.That(actualEncodings, Is.EquivalentTo(expectedEncodings)); - - Assert.AreEqual(expected.Compression, chunkMetaData.Compression); - Assert.AreEqual(expected.Values, columnReader.Apply(new PhysicalValueGetter(chunkMetaData.NumValues)).values); + try + { + var descr = columnReader.ColumnDescriptor; + using var chunkMetaData = rowGroupMetaData.GetColumnChunkMetaData(c); + + Assert.AreEqual(expected.MaxDefinitionlevel, descr.MaxDefinitionLevel); + Assert.AreEqual(expected.MaxRepetitionLevel, descr.MaxRepetitionLevel); + Assert.AreEqual(expected.PhysicalType, descr.PhysicalType); + using var logicalType = descr.LogicalType; + Assert.AreEqual(expected.LogicalType, logicalType); + Assert.AreEqual(expected.ColumnOrder, descr.ColumnOrder); + Assert.AreEqual(expected.SortOrder, descr.SortOrder); + Assert.AreEqual(expected.Name, descr.Name); + Assert.AreEqual(expected.TypeLength, descr.TypeLength); + Assert.AreEqual(expected.TypePrecision, descr.TypePrecision); + Assert.AreEqual(expected.TypeScale, descr.TypeScale); + + var expectedEncodings = expected.Encodings + .Where(e => useDictionaryEncoding || e != Encoding.RleDictionary).ToArray(); + var actualEncodings = chunkMetaData.Encodings.Distinct().ToArray(); + // Encoding ordering is not important + Assert.That(actualEncodings, Is.EquivalentTo(expectedEncodings)); + + Assert.AreEqual(expected.Compression, chunkMetaData.Compression); + Assert.AreEqual(expected.Values, + columnReader.Apply(new PhysicalValueGetter(chunkMetaData.NumValues)).values); + } + catch (Exception) + { + TestContext.Out.WriteLine("Failure reading '{0}'", expected.Name); + throw; + } } } diff --git a/fsharp.test/ParquetSharp.Test.FSharp.fsproj b/fsharp.test/ParquetSharp.Test.FSharp.fsproj index b93641ec..d3bc9942 100644 --- a/fsharp.test/ParquetSharp.Test.FSharp.fsproj +++ b/fsharp.test/ParquetSharp.Test.FSharp.fsproj @@ -14,7 +14,7 @@ - +