diff --git a/cadc-tap-schema/build.gradle b/cadc-tap-schema/build.gradle index 69980d38..771140f7 100644 --- a/cadc-tap-schema/build.gradle +++ b/cadc-tap-schema/build.gradle @@ -15,7 +15,7 @@ sourceCompatibility = 11 group = 'org.opencadc' -version = '1.2.10' +version = '1.2.11' description = 'OpenCADC TAP-1.1 tap schema server library' def git_url = 'https://github.com/opencadc/tap' @@ -29,7 +29,9 @@ dependencies { implementation 'org.opencadc:cadc-cdp:[1.2.3,2.0)' implementation 'org.opencadc:cadc-gms:[1.0,2.0)' implementation 'org.opencadc:cadc-rest:[1.3.1,2.0)' + implementation 'org.apache.parquet:parquet-common:[1.13.0,)' api 'org.opencadc:cadc-dali:[1.1,2.0)' + api 'org.opencadc:cadc-dali-parquet:[0.5.2,)' api 'org.opencadc:cadc-tap:[1.1.21,2.0)' implementation 'uk.ac.starlink:jcdf:[1.2.3,2.0)' diff --git a/cadc-tap-schema/src/main/java/ca/nrc/cadc/tap/db/TableLoader.java b/cadc-tap-schema/src/main/java/ca/nrc/cadc/tap/db/TableLoader.java index 5eac3317..720c635d 100644 --- a/cadc-tap-schema/src/main/java/ca/nrc/cadc/tap/db/TableLoader.java +++ b/cadc-tap-schema/src/main/java/ca/nrc/cadc/tap/db/TableLoader.java @@ -106,7 +106,7 @@ public class TableLoader { private final DatabaseDataType ddType; private final DataSource dataSource; - private final int batchSize; + public Integer batchSize; private long totalInserts = 0; /** @@ -115,7 +115,7 @@ public class TableLoader { * @param dataSource destination database connection pool * @param batchSize number of rows per commit transaction */ - public TableLoader(DataSource dataSource, int batchSize) { + public TableLoader(DataSource dataSource, Integer batchSize) { this.dataSource = dataSource; this.batchSize = batchSize; PluginFactory pf = new PluginFactory(); @@ -131,9 +131,15 @@ public TableLoader(DataSource dataSource, int batchSize) { */ public void load(TableDesc destTable, TableDataInputStream data) { TableDesc reorgTable = data.acceptTargetTableDesc(destTable); - + + boolean manageTxn = true; + if (batchSize == null || batchSize <= 0) { + manageTxn = false; + batchSize = Integer.MAX_VALUE; // no batching, just one transaction + } + Profiler prof = new Profiler(TableLoader.class); - DatabaseTransactionManager tm = new DatabaseTransactionManager(dataSource); + DatabaseTransactionManager tm = manageTxn ? new DatabaseTransactionManager(dataSource) : null; JdbcTemplate jdbc = new JdbcTemplate(dataSource); // Loop over rows, start/commit txn every batchSize rows @@ -142,13 +148,15 @@ public void load(TableDesc destTable, TableDataInputStream data) { Iterator> dataIterator = data.iterator(); List nextRow = null; - List> batch = new ArrayList<>(batchSize); + List> batch = manageTxn ? new ArrayList<>(batchSize) : new ArrayList<>(); int count = 0; try { while (!done) { count = 0; - tm.startTransaction(); - prof.checkpoint("start-transaction"); + if (manageTxn) { + tm.startTransaction(); + prof.checkpoint("start-transaction"); + } BulkInsertStatement bulkInsertStatement = new BulkInsertStatement(reorgTable); while (batch.size() < batchSize && dataIterator.hasNext()) { @@ -160,9 +168,11 @@ public void load(TableDesc destTable, TableDataInputStream data) { log.debug("Inserting " + batch.size() + " rows in this batch."); jdbc.batchUpdate(sql, batch, batchSize, bulkInsertStatement); prof.checkpoint("batch-of-inserts"); - - tm.commitTransaction(); - prof.checkpoint("commit-transaction"); + + if (manageTxn) { + tm.commitTransaction(); + prof.checkpoint("commit-transaction"); + } totalInserts += batch.size(); batch.clear(); done = !dataIterator.hasNext(); @@ -175,7 +185,7 @@ public void load(TableDesc destTable, TableDataInputStream data) { log.error("unexpected exception trying to close input stream", oops); } try { - if (tm.isOpen()) { + if (manageTxn && tm.isOpen()) { tm.rollbackTransaction(); prof.checkpoint("rollback-transaction"); } @@ -192,7 +202,7 @@ public void load(TableDesc destTable, TableDataInputStream data) { log.error("unexpected exception trying to close input stream", oops); } try { - if (tm.isOpen()) { + if (manageTxn && tm.isOpen()) { tm.rollbackTransaction(); prof.checkpoint("rollback-transaction"); } @@ -205,7 +215,7 @@ public void load(TableDesc destTable, TableDataInputStream data) { + " Current batch of " + batchSize + " failed with: " + t.getMessage(), t); } finally { - if (tm.isOpen()) { + if (manageTxn && tm.isOpen()) { log.error("BUG: Transaction manager unexpectedly open, rolling back."); try { tm.rollbackTransaction(); diff --git a/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/PutAction.java b/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/PutAction.java index bf5f1d19..65a1a4cc 100644 --- a/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/PutAction.java +++ b/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/PutAction.java @@ -70,19 +70,27 @@ import ca.nrc.cadc.auth.AuthenticationUtil; import ca.nrc.cadc.auth.HttpPrincipal; import ca.nrc.cadc.auth.IdentityManager; +import ca.nrc.cadc.dali.tables.TableData; +import ca.nrc.cadc.dali.tables.votable.VOTableDocument; import ca.nrc.cadc.db.DatabaseTransactionManager; import ca.nrc.cadc.net.ResourceAlreadyExistsException; import ca.nrc.cadc.profiler.Profiler; import ca.nrc.cadc.rest.InlineContentHandler; import ca.nrc.cadc.tap.db.TableCreator; +import ca.nrc.cadc.tap.db.TableLoader; import ca.nrc.cadc.tap.schema.ColumnDesc; import ca.nrc.cadc.tap.schema.SchemaDesc; import ca.nrc.cadc.tap.schema.TableDesc; import ca.nrc.cadc.tap.schema.TapPermissions; import ca.nrc.cadc.tap.schema.TapSchemaDAO; + +import java.util.Iterator; +import java.util.List; + import javax.security.auth.Subject; import javax.sql.DataSource; import org.apache.log4j.Logger; +import org.opencadc.tap.io.TableDataInputStream; import org.springframework.jdbc.core.JdbcTemplate; /** @@ -250,7 +258,35 @@ private void createTable(TapSchemaDAO ts, String schemaName, String tableName) t TableCreator tc = new TableCreator(ds); tc.createTable(inputTable); prof.checkpoint("create-table"); - + + // load the table data if available + Object in = syncInput.getContent(INPUT_TAG); + if (in instanceof VOTableDocument) { + VOTableDocument voTableDocument = (VOTableDocument) in; + TableData tableData = voTableDocument.getResourceByType("results").getTable().getTableData(); + + if (tableData != null) { + TableLoader tableLoader = new TableLoader(ds, null); + tableLoader.load(inputTable, new TableDataInputStream() { + @Override + public void close() { + //no-op: fully read already + } + + @Override + public Iterator> iterator() { + return tableData.iterator(); + } + + @Override + public TableDesc acceptTargetTableDesc(TableDesc td) { + return td; + } + }); + tableData.close(); + } + } + // add to tap_schema // flag table as created using the API to allow table deletion in the DeleteAction inputTable.apiCreated = true; diff --git a/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/TablesAction.java b/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/TablesAction.java index d730e586..019db169 100644 --- a/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/TablesAction.java +++ b/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/TablesAction.java @@ -69,6 +69,9 @@ import ca.nrc.cadc.auth.AuthenticationUtil; import ca.nrc.cadc.auth.HttpPrincipal; +import ca.nrc.cadc.dali.tables.votable.VOTableDocument; +import ca.nrc.cadc.dali.tables.votable.VOTableResource; +import ca.nrc.cadc.dali.tables.votable.VOTableTable; import ca.nrc.cadc.log.WebServiceLogInfo; import ca.nrc.cadc.net.ResourceNotFoundException; import ca.nrc.cadc.net.TransientException; @@ -80,6 +83,7 @@ import ca.nrc.cadc.tap.schema.TableDesc; import ca.nrc.cadc.tap.schema.TapPermissions; import ca.nrc.cadc.tap.schema.TapSchemaDAO; +import ca.nrc.cadc.tap.schema.TapSchemaUtil; import java.io.IOException; import java.security.AccessControlException; import java.security.Principal; @@ -183,25 +187,49 @@ protected final TapSchemaDAO getTapSchemaDAO() { dao.setOrdered(true); return dao; } - + + private TableDesc toTableDesc(VOTableDocument doc) { + for (VOTableResource vr : doc.getResources()) { + VOTableTable vtab = vr.getTable(); + if (vtab != null) { + TableDesc ret = TapSchemaUtil.createTableDesc("default", "default", vtab); + log.debug("create from VOtable: " + ret); + // strip out some incoming table metadata + // - ID attr (should be transient usage only) + for (ColumnDesc cd : ret.getColumnDescs()) { + cd.columnID = null; + } + return ret; + } + } + throw new IllegalArgumentException("no table description found in VOTable document"); + } + protected TableDesc getInputTable(String schemaName, String tableName) { Object in = syncInput.getContent(INPUT_TAG); if (in == null) { throw new IllegalArgumentException("no input: expected a document describing the table to create/update"); } + + TableDesc input; + if (in instanceof TableDesc) { - TableDesc input = (TableDesc) in; - input.setSchemaName(schemaName); - input.setTableName(tableName); - // TODO: move this to PutAction (create only) - int c = 0; - for (ColumnDesc cd : input.getColumnDescs()) { - cd.setTableName(tableName); - cd.columnIndex = c++; - } - return input; + input = (TableDesc) in; + } else if (in instanceof VOTableDocument) { + input = toTableDesc((VOTableDocument) in); + } else { + throw new RuntimeException("BUG: no input table"); } - throw new RuntimeException("BUG: no input table"); + + input.setSchemaName(schemaName); + input.setTableName(tableName); + // TODO: move this to PutAction (create only) + int c = 0; + for (ColumnDesc cd : input.getColumnDescs()) { + cd.setTableName(tableName); + cd.columnIndex = c++; + } + return input; } protected SchemaDesc getInputSchema(String schemaName) { diff --git a/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/TablesInputHandler.java b/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/TablesInputHandler.java index 13a17560..f9bed6ad 100644 --- a/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/TablesInputHandler.java +++ b/cadc-tap-schema/src/main/java/ca/nrc/cadc/vosi/actions/TablesInputHandler.java @@ -67,18 +67,14 @@ package ca.nrc.cadc.vosi.actions; +import ca.nrc.cadc.dali.tables.parquet.ParquetReader; import ca.nrc.cadc.dali.tables.votable.VOTableDocument; import ca.nrc.cadc.dali.tables.votable.VOTableReader; -import ca.nrc.cadc.dali.tables.votable.VOTableResource; -import ca.nrc.cadc.dali.tables.votable.VOTableTable; import ca.nrc.cadc.io.ByteCountInputStream; import ca.nrc.cadc.rest.InlineContentException; import ca.nrc.cadc.rest.InlineContentHandler; -import ca.nrc.cadc.tap.schema.ColumnDesc; import ca.nrc.cadc.tap.schema.SchemaDesc; -import ca.nrc.cadc.tap.schema.TableDesc; import ca.nrc.cadc.tap.schema.TapSchema; -import ca.nrc.cadc.tap.schema.TapSchemaUtil; import ca.nrc.cadc.util.StringUtil; import ca.nrc.cadc.vosi.InvalidTableSetException; import ca.nrc.cadc.vosi.TableReader; @@ -99,6 +95,7 @@ public class TablesInputHandler implements InlineContentHandler { public static final String VOSI_TABLE_TYPE = "text/xml"; public static final String VOTABLE_TYPE = "application/x-votable+xml"; public static final String VOSI_SCHEMA_TYPE = "application/x-vosi-schema"; + public static final String PARQUET_TYPE = "application/vnd.apache.parquet"; // VOSI tableset schema cannot carry owner information //public static final String VOSI_SCHEMA_TYPE = "text/plain"; // key = value @@ -114,7 +111,7 @@ public Content accept(String name, String contentType, InputStream in) throws In try { String schemaOwner = null; SchemaDesc sch = null; - TableDesc tab = null; + Object tab = null; if (VOSI_SCHEMA_TYPE.equalsIgnoreCase(contentType)) { ByteCountInputStream istream = new ByteCountInputStream(in, BYTE_LIMIT); String str = StringUtil.readFromInputStream(istream, "UTF-8"); @@ -136,8 +133,11 @@ public Content accept(String name, String contentType, InputStream in) throws In } else if (VOTABLE_TYPE.equalsIgnoreCase(contentType)) { VOTableReader tr = new VOTableReader(); ByteCountInputStream istream = new ByteCountInputStream(in, BYTE_LIMIT); - VOTableDocument doc = tr.read(istream); - tab = toTableDesc(doc); + tab = tr.read(istream); + } else if (PARQUET_TYPE.equalsIgnoreCase(contentType)) { + ByteCountInputStream istream = new ByteCountInputStream(in, BYTE_LIMIT); + ParquetReader parquetReader = new ParquetReader(); + tab = parquetReader.read(istream); } InlineContentHandler.Content ret = new InlineContentHandler.Content(); ret.name = objectTag; @@ -151,21 +151,4 @@ public Content accept(String name, String contentType, InputStream in) throws In } } - private TableDesc toTableDesc(VOTableDocument doc) { - // TODO: reject if the table has any rows? try to insert them if it is small enough? - for (VOTableResource vr : doc.getResources()) { - VOTableTable vtab = vr.getTable(); - if (vtab != null) { - TableDesc ret = TapSchemaUtil.createTableDesc("default", "default", vtab); - log.debug("create from VOtable: " + ret); - // strip out some incoming table metadata - // - ID attr (should be transient usage only) - for (ColumnDesc cd : ret.getColumnDescs()) { - cd.columnID = null; - } - return ret; - } - } - throw new IllegalArgumentException("no table description found in VOTable document"); - } } diff --git a/cadc-tap-server/build.gradle b/cadc-tap-server/build.gradle index 91d87c94..83767ac8 100644 --- a/cadc-tap-server/build.gradle +++ b/cadc-tap-server/build.gradle @@ -27,7 +27,7 @@ dependencies { implementation 'org.opencadc:cadc-registry:[1.4,)' implementation 'org.opencadc:cadc-rest:[1.4,)' api 'org.opencadc:cadc-dali:[1.2.23,)' - api 'org.opencadc:cadc-tap-schema:[1.2.6,)' + api 'org.opencadc:cadc-tap-schema:[1.2.11,)' api 'org.opencadc:cadc-uws:[1.0,)' api 'org.opencadc:cadc-uws-server:[1.2.23,)' diff --git a/youcat/build.gradle b/youcat/build.gradle index a3dc43e6..ace9f2e4 100644 --- a/youcat/build.gradle +++ b/youcat/build.gradle @@ -25,7 +25,7 @@ dependencies { implementation 'org.opencadc:cadc-util:[1.12.0,2.0)' implementation 'org.opencadc:cadc-rest:[1.4.5,)' implementation 'org.opencadc:cadc-dali:[1.2.23,)' - implementation 'org.opencadc:cadc-dali-parquet:[0.5.0,)' + implementation 'org.opencadc:cadc-dali-parquet:[0.5.2,)' implementation 'org.opencadc:cadc-datalink:[1.1.5,)' implementation 'org.opencadc:cadc-adql:[1.1.14,)' implementation 'org.opencadc:cadc-uws:[1.0.2,)' @@ -36,7 +36,7 @@ dependencies { implementation 'org.opencadc:cadc-tap-server-pg:[1.1.1,)' implementation 'org.opencadc:cadc-adql:[1.1.4,)' implementation 'org.opencadc:cadc-vosi:[1.4.3,2.0)' - implementation 'org.opencadc:cadc-registry:[1.7.2,)' + implementation 'org.opencadc:cadc-registry:[1.8.0,)' implementation 'org.opencadc:cadc-gms:[1.0.4,2.0)' implementation 'org.opencadc:cadc-tap-tmp:[1.1.5,)' implementation 'org.opencadc:cadc-cdp:[1.4.0,)' @@ -51,6 +51,7 @@ dependencies { intTestImplementation 'org.opencadc:cadc-test-vosi:[1.0.11,)' intTestImplementation 'org.opencadc:cadc-test-tap:[1.1,)' intTestImplementation 'uk.ac.starlink:stil:[4.0,5.0)' + intTestImplementation 'org.apache.parquet:parquet-common:[1.13.0,)' intTestRuntimeOnly 'org.postgresql:postgresql:[42.2,43.0)' } diff --git a/youcat/src/intTest/java/org/opencadc/youcat/AbstractTablesTest.java b/youcat/src/intTest/java/org/opencadc/youcat/AbstractTablesTest.java index 5c98b233..227d728c 100644 --- a/youcat/src/intTest/java/org/opencadc/youcat/AbstractTablesTest.java +++ b/youcat/src/intTest/java/org/opencadc/youcat/AbstractTablesTest.java @@ -72,6 +72,10 @@ import ca.nrc.cadc.auth.HttpPrincipal; import ca.nrc.cadc.auth.RunnableAction; import ca.nrc.cadc.auth.SSLUtil; +import ca.nrc.cadc.dali.Circle; +import ca.nrc.cadc.dali.DoubleInterval; +import ca.nrc.cadc.dali.Point; +import ca.nrc.cadc.dali.Polygon; import ca.nrc.cadc.dali.tables.votable.VOTableField; import ca.nrc.cadc.dali.tables.votable.VOTableTable; import ca.nrc.cadc.net.FileContent; @@ -254,7 +258,10 @@ TableDesc doCreateTable(Subject subject, String tableName) throws Exception { orig.getColumnDescs().add(new ColumnDesc(tableName, "a13", new TapDataType("long", "*", null))); orig.getColumnDescs().add(new ColumnDesc(tableName, "a14", new TapDataType("float", "*", null))); orig.getColumnDescs().add(new ColumnDesc(tableName, "a15", new TapDataType("double", "*", null))); - + + orig.getColumnDescs().add(new ColumnDesc(tableName, "e16", TapDataType.URI)); + orig.getColumnDescs().add(new ColumnDesc(tableName, "e17", new TapDataType("char","36", "uuid"))); + // create URL tableURL = new URL(certTablesURL.toExternalForm() + "/" + tableName); TableWriter w = new TableWriter(); @@ -469,4 +476,28 @@ protected TableDesc doVosiCheck(String testTable) throws Exception { return td; } + public void assertEquals(DoubleInterval expected, DoubleInterval actual) { + Assert.assertEquals(expected.getLower(), actual.getLower(), 1.0e-9); + Assert.assertEquals(expected.getUpper(), actual.getUpper(), 1.0e-9); + + } + + public void assertEquals(Point expected, Point actual) { + Assert.assertEquals(expected.getLongitude(), actual.getLongitude(), 1.0e-9); + Assert.assertEquals(expected.getLatitude(), actual.getLatitude(), 1.0e-9); + } + + public void assertEquals(Circle expected, Circle actual) { + assertEquals(expected.getCenter(), actual.getCenter()); + Assert.assertEquals(expected.getRadius(), actual.getRadius(), 1.0e-9); + } + + public void assertEquals(Polygon expected, Polygon actual) { + Assert.assertEquals("num vertices", expected.getVertices().size(), actual.getVertices().size()); + for (int i=0; i < expected.getVertices().size(); i++) { + Point ep = expected.getVertices().get(i); + Point ap = actual.getVertices().get(i); + assertEquals(ep, ap); + } + } } diff --git a/youcat/src/intTest/java/org/opencadc/youcat/CreateTableTest.java b/youcat/src/intTest/java/org/opencadc/youcat/CreateTableTest.java index b9195d7b..89531eed 100644 --- a/youcat/src/intTest/java/org/opencadc/youcat/CreateTableTest.java +++ b/youcat/src/intTest/java/org/opencadc/youcat/CreateTableTest.java @@ -68,6 +68,11 @@ package org.opencadc.youcat; import ca.nrc.cadc.auth.RunnableAction; +import ca.nrc.cadc.dali.Circle; +import ca.nrc.cadc.dali.Interval; +import ca.nrc.cadc.dali.Point; +import ca.nrc.cadc.dali.Polygon; +import ca.nrc.cadc.dali.tables.ListTableData; import ca.nrc.cadc.dali.tables.TableData; import ca.nrc.cadc.dali.tables.votable.VOTableDocument; import ca.nrc.cadc.dali.tables.votable.VOTableField; @@ -75,10 +80,7 @@ import ca.nrc.cadc.dali.tables.votable.VOTableResource; import ca.nrc.cadc.dali.tables.votable.VOTableTable; import ca.nrc.cadc.dali.tables.votable.VOTableWriter; -import ca.nrc.cadc.net.FileContent; -import ca.nrc.cadc.net.HttpPost; -import ca.nrc.cadc.net.HttpUpload; -import ca.nrc.cadc.net.OutputStreamWrapper; +import ca.nrc.cadc.net.*; import ca.nrc.cadc.tap.schema.ColumnDesc; import ca.nrc.cadc.tap.schema.SchemaDesc; import ca.nrc.cadc.tap.schema.TableDesc; @@ -89,15 +91,13 @@ import ca.nrc.cadc.vosi.TableSetWriter; import ca.nrc.cadc.vosi.TableWriter; import ca.nrc.cadc.vosi.actions.TablesInputHandler; -import java.io.IOException; -import java.io.OutputStream; -import java.io.StringWriter; + +import java.io.*; +import java.net.MalformedURLException; import java.net.URL; import java.nio.charset.Charset; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; +import java.security.PrivilegedActionException; +import java.util.*; import javax.security.auth.Subject; import org.apache.log4j.Level; import org.apache.log4j.Logger; @@ -252,6 +252,11 @@ public void testCreateQueryUpdateDropVOSI() { @Test public void testCreateQueryDropVOTable() { + testCreateQueryDropVOTable(true); + testCreateQueryDropVOTable(false); + } + + public void testCreateQueryDropVOTable(boolean uploadTableData) { try { clearSchemaPerms(); TapPermissions tp = new TapPermissions(null, true, null, null); @@ -262,63 +267,9 @@ public void testCreateQueryDropVOTable() { // cleanup just in case doDelete(schemaOwner, testTable, true); - VOTableTable vtab = new VOTableTable(); - vtab.getFields().add(new VOTableField("c0", TapDataType.STRING.getDatatype(), TapDataType.STRING.arraysize)); - vtab.getFields().add(new VOTableField("c1", TapDataType.SHORT.getDatatype())); - vtab.getFields().add(new VOTableField("c2", TapDataType.INTEGER.getDatatype())); - vtab.getFields().add(new VOTableField("c3", TapDataType.LONG.getDatatype())); - vtab.getFields().add(new VOTableField("c4", TapDataType.FLOAT.getDatatype())); - vtab.getFields().add(new VOTableField("c5", TapDataType.DOUBLE.getDatatype())); - VOTableField f0 = vtab.getFields().get(0); - f0.id = "bogus_id"; - - // extended types - VOTableField tf; - tf = new VOTableField("e6", TapDataType.TIMESTAMP.getDatatype(), TapDataType.TIMESTAMP.arraysize); - tf.xtype = TapDataType.TIMESTAMP.xtype; - vtab.getFields().add(tf); - - tf = new VOTableField("e7", TapDataType.INTERVAL.getDatatype(), TapDataType.INTERVAL.arraysize); - tf.xtype = TapDataType.INTERVAL.xtype; - vtab.getFields().add(tf); - tf = new VOTableField("e8", TapDataType.POINT.getDatatype(), TapDataType.POINT.arraysize); - tf.xtype = TapDataType.POINT.xtype; - vtab.getFields().add(tf); - tf = new VOTableField("e9", TapDataType.CIRCLE.getDatatype(), TapDataType.CIRCLE.arraysize); - tf.xtype = TapDataType.CIRCLE.xtype; - vtab.getFields().add(tf); - tf = new VOTableField("e10", TapDataType.POLYGON.getDatatype(), TapDataType.POLYGON.arraysize); - tf.xtype = TapDataType.POLYGON.xtype; - vtab.getFields().add(tf); - - // arrays - vtab.getFields().add(new VOTableField("a11", "short", "*")); - vtab.getFields().add(new VOTableField("a12", "int", "*")); - vtab.getFields().add(new VOTableField("a13", "long", "*")); - vtab.getFields().add(new VOTableField("a14", "float", "*")); - vtab.getFields().add(new VOTableField("a15", "double", "*")); - - VOTableResource vres = new VOTableResource("meta"); - vres.setTable(vtab); - final VOTableDocument doc = new VOTableDocument(); - doc.getResources().add(vres); - - // create - URL tableURL = new URL(certTablesURL.toExternalForm() + "/" + testTable); - OutputStreamWrapper src = new OutputStreamWrapper() { - @Override - public void write(OutputStream out) throws IOException { - VOTableWriter w = new VOTableWriter(TablesInputHandler.VOTABLE_TYPE); - w.write(doc, out); - } - }; - HttpUpload put = new HttpUpload(src, tableURL); - put.setContentType(TablesInputHandler.VOTABLE_TYPE); - Subject.doAs(schemaOwner, new RunnableAction(put)); - Assert.assertNull("throwable", put.getThrowable()); - Assert.assertEquals("response code", 200, put.getResponseCode()); - put = null; - + VOTableTable actualVOTableTable = prepareVOTableTable(uploadTableData); + createTableFromVOTable(actualVOTableTable, testTable); + TableDesc td = doVosiCheck(testTable); super.setPerms(schemaOwner, testTable, tp, 200); @@ -328,8 +279,12 @@ public void write(OutputStream out) throws IOException { Assert.assertNull("field ID attr ignored by create", field0.id); TableData tdata = vt.getTableData(); Iterator> iter = tdata.iterator(); - Assert.assertFalse("no result rows", iter.hasNext()); - + Assert.assertEquals("Table rows are not as per expectation.", uploadTableData, iter.hasNext()); + + if (uploadTableData) { + verifyUploadedVOTableTableData(iter, actualVOTableTable.getTableData().iterator()); + } + // cleanup on success doDelete(schemaOwner, testTable, false); } catch (Exception unexpected) { @@ -338,6 +293,205 @@ public void write(OutputStream out) throws IOException { } } + @Test + public void testCreateQueryDropParquet() throws Exception { + String testTable = testSchemaName + ".testCreateQueryDropParquet"; + + // Permission updates + clearSchemaPerms(); + TapPermissions tp = new TapPermissions(null, true, null, null); + super.setPerms(schemaOwner, testSchemaName, tp, 200); + + // delete table if it exists + doDelete(schemaOwner, testTable, true); + + // Create table from VOTable data + VOTableTable actualVOTableTable = prepareVOTableTable(true); + VOTableDocument voTableDocument = createTableFromVOTable(actualVOTableTable, testTable); + + // Create table from Parquet data + createTableFromParquet(testTable); + + super.setPerms(schemaOwner, testTable, tp, 200); + + // Verify the table created from Parquet data + VOTableTable voTableTable = doQueryCheck(testTable); + TableData tdata = voTableTable.getTableData(); + Assert.assertEquals(voTableDocument.getResourceByType("results").getTable().getFields().size(), voTableTable.getFields().size()); + + Iterator> iter = tdata.iterator(); + Assert.assertTrue("no result rows", iter.hasNext()); + + verifyUploadedVOTableTableData(iter, actualVOTableTable.getTableData().iterator()); + } + + /* + * Steps: + * - get the data from testTable in Parquet format + * - delete the testTable table + * - create a new testTable table using Parquet format + * */ + private void createTableFromParquet(String testTable) throws Exception { + // get data in parquet format + ByteArrayOutputStream parquetData = getParquetData(testTable); + + doDelete(schemaOwner, testTable, false); + + // Create table using Parquet data + URL tableURL = new URL(certTablesURL.toExternalForm() + "/" + testTable); + ByteArrayInputStream inputStream = new ByteArrayInputStream(parquetData.toByteArray()); + + HttpUpload put = new HttpUpload(inputStream, tableURL); + put.setRequestProperty(HttpTransfer.CONTENT_TYPE, TablesInputHandler.PARQUET_TYPE); + Subject.doAs(schemaOwner, new RunnableAction(put)); + Assert.assertNull("throwable", put.getThrowable()); + Assert.assertEquals("response code", 200, put.getResponseCode()); + } + + private ByteArrayOutputStream getParquetData(String testTable) throws PrivilegedActionException { + String adql = "SELECT * from " + testTable; + + Map params = new TreeMap<>(); + params.put("LANG", "ADQL"); + params.put("QUERY", adql); + params.put("RESPONSEFORMAT", "parquet"); + + ByteArrayOutputStream out = new ByteArrayOutputStream(); + + String result = Subject.doAs(schemaOwner, new AuthQueryTest.SyncQueryAction(anonQueryURL, params, out, "application/vnd.apache.parquet")); + Assert.assertNotNull(result); + + return out; + } + + private static VOTableTable prepareVOTableTable(boolean testUploadTableData) { + VOTableTable vtab = new VOTableTable(); + vtab.getFields().add(new VOTableField("c0", TapDataType.STRING.getDatatype(), TapDataType.STRING.arraysize)); + vtab.getFields().add(new VOTableField("c1", TapDataType.SHORT.getDatatype())); + vtab.getFields().add(new VOTableField("c2", TapDataType.INTEGER.getDatatype())); + vtab.getFields().add(new VOTableField("c3", TapDataType.LONG.getDatatype())); + vtab.getFields().add(new VOTableField("c4", TapDataType.FLOAT.getDatatype())); + vtab.getFields().add(new VOTableField("c5", TapDataType.DOUBLE.getDatatype())); + VOTableField f0 = vtab.getFields().get(0); + f0.id = "bogus_id"; + + // extended types + VOTableField tf; + tf = new VOTableField("e6", TapDataType.TIMESTAMP.getDatatype(), TapDataType.TIMESTAMP.arraysize); + tf.xtype = TapDataType.TIMESTAMP.xtype; + vtab.getFields().add(tf); + + tf = new VOTableField("e7", TapDataType.INTERVAL.getDatatype(), TapDataType.INTERVAL.arraysize); + tf.xtype = TapDataType.INTERVAL.xtype; + vtab.getFields().add(tf); + tf = new VOTableField("e8", TapDataType.POINT.getDatatype(), TapDataType.POINT.arraysize); + tf.xtype = TapDataType.POINT.xtype; + vtab.getFields().add(tf); + tf = new VOTableField("e9", TapDataType.CIRCLE.getDatatype(), TapDataType.CIRCLE.arraysize); + tf.xtype = TapDataType.CIRCLE.xtype; + vtab.getFields().add(tf); + tf = new VOTableField("e10", TapDataType.POLYGON.getDatatype(), TapDataType.POLYGON.arraysize); + tf.xtype = TapDataType.POLYGON.xtype; + vtab.getFields().add(tf); + + // arrays + vtab.getFields().add(new VOTableField("a11", "short", "*")); + vtab.getFields().add(new VOTableField("a12", "int", "*")); + vtab.getFields().add(new VOTableField("a13", "long", "*")); + vtab.getFields().add(new VOTableField("a14", "float", "*")); + vtab.getFields().add(new VOTableField("a15", "double", "*")); + + if (testUploadTableData) { + addTableDataToVOTable(vtab); + } + + return vtab; + } + + private static void addTableDataToVOTable(VOTableTable vtab) { + ListTableData tableData = new ListTableData(); + vtab.setTableData(tableData); + + // prepare iterator with dummy data + for(int i=0; i<10; i++) { + List row = new ArrayList<>(); + row.add("string" + i); // c0 + row.add(Short.MAX_VALUE); // c1 + row.add(Integer.MAX_VALUE); // c2 + row.add(Long.MAX_VALUE); // c3 + row.add(Float.MAX_VALUE); // c4 + row.add(Double.MAX_VALUE); // c5 + + row.add(new Date()); // e6 + row.add(new Interval(1.0, 2.0)); // e7 + row.add(new Point(1.0, 2.0)); // e8 + row.add(new Circle(new Point(1, 2), 3)); // e9 + + Polygon p = new Polygon(); + p.getVertices().add(new Point(1.0, 2.0)); + p.getVertices().add(new Point(3.0, 4.0)); + p.getVertices().add(new Point(5.0, 6.0)); + row.add(p); // e10 + + row.add(new short[] { (short) i, (short) (i + 1) }); // a11 + row.add(new int[] { i, i + 1 }); // a12 + row.add(new long[] { i, i + 1 }); // a13 + row.add(new float[] { i, i + 1 }); // a14 + row.add(new double[] { i, i + 1 }); // a15 + tableData.getArrayList().add(row); + } + } + + private VOTableDocument createTableFromVOTable(VOTableTable vtab, String testTable) throws MalformedURLException { + VOTableResource vres = new VOTableResource("results"); + vres.setTable(vtab); + final VOTableDocument doc = new VOTableDocument(); + doc.getResources().add(vres); + + // create + URL tableURL = new URL(certTablesURL.toExternalForm() + "/" + testTable); + OutputStreamWrapper src = new OutputStreamWrapper() { + @Override + public void write(OutputStream out) throws IOException { + VOTableWriter w = new VOTableWriter(TablesInputHandler.VOTABLE_TYPE); + w.write(doc, out); + } + }; + HttpUpload put = new HttpUpload(src, tableURL); + put.setContentType(TablesInputHandler.VOTABLE_TYPE); + Subject.doAs(schemaOwner, new RunnableAction(put)); + Assert.assertNull("throwable", put.getThrowable()); + Assert.assertEquals("response code", 200, put.getResponseCode()); + + return doc; + } + + private void verifyUploadedVOTableTableData(Iterator> retrievedDataIter, Iterator> actualDataIter) { + int count = 0; + while (retrievedDataIter.hasNext() && actualDataIter.hasNext()) { + List retrievedRow = retrievedDataIter.next(); + List actualRow = actualDataIter.next(); + Assert.assertEquals("string" + count, retrievedRow.get(0)); + Assert.assertEquals(actualRow.get(2), retrievedRow.get(2)); + Assert.assertEquals(actualRow.get(3), retrievedRow.get(3)); + Assert.assertEquals(actualRow.get(4), retrievedRow.get(4)); + Assert.assertEquals(actualRow.get(5), retrievedRow.get(5)); + Assert.assertTrue(retrievedRow.get(6) instanceof Date); + + Assert.assertEquals(actualRow.get(7), retrievedRow.get(7)); + assertEquals((Point) actualRow.get(8), (Point) retrievedRow.get(8)); + assertEquals((Circle) actualRow.get(9), (Circle) retrievedRow.get(9)); + Polygon p = new Polygon(); + p.getVertices().add(new Point(1.0, 2.0)); + p.getVertices().add(new Point(3.0, 4.0)); + p.getVertices().add(new Point(5.0, 6.0)); + assertEquals(p, (Polygon) retrievedRow.get(10)); + + count++; + } + Assert.assertEquals(10, count); + } + @Test public void testCreateUpdateDropSchema() { diff --git a/youcat/src/intTest/java/org/opencadc/youcat/LoadTableDataTest.java b/youcat/src/intTest/java/org/opencadc/youcat/LoadTableDataTest.java index bd75e683..54147cd3 100644 --- a/youcat/src/intTest/java/org/opencadc/youcat/LoadTableDataTest.java +++ b/youcat/src/intTest/java/org/opencadc/youcat/LoadTableDataTest.java @@ -682,29 +682,4 @@ public Object run() throws Exception { Assert.fail("unexpected exception: " + unexpected); } } - - private void assertEquals(DoubleInterval expected, DoubleInterval actual) { - Assert.assertEquals(expected.getLower(), actual.getLower(), 1.0e-9); - Assert.assertEquals(expected.getUpper(), actual.getUpper(), 1.0e-9); - - } - - private void assertEquals(Point expected, Point actual) { - Assert.assertEquals(expected.getLongitude(), actual.getLongitude(), 1.0e-9); - Assert.assertEquals(expected.getLatitude(), actual.getLatitude(), 1.0e-9); - } - - private void assertEquals(Circle expected, Circle actual) { - assertEquals(expected.getCenter(), actual.getCenter()); - Assert.assertEquals(expected.getRadius(), actual.getRadius(), 1.0e-9); - } - - private void assertEquals(Polygon expected, Polygon actual) { - Assert.assertEquals("num vertices", expected.getVertices().size(), actual.getVertices().size()); - for (int i=0; i < expected.getVertices().size(); i++) { - Point ep = expected.getVertices().get(i); - Point ap = actual.getVertices().get(i); - assertEquals(ep, ap); - } - } } diff --git a/youcat/src/intTest/java/org/opencadc/youcat/ParquetReaderTest.java b/youcat/src/intTest/java/org/opencadc/youcat/ParquetReaderTest.java new file mode 100644 index 00000000..0900e144 --- /dev/null +++ b/youcat/src/intTest/java/org/opencadc/youcat/ParquetReaderTest.java @@ -0,0 +1,216 @@ +package org.opencadc.youcat; + +import ca.nrc.cadc.dali.Circle; +import ca.nrc.cadc.dali.DoubleInterval; +import ca.nrc.cadc.dali.Point; +import ca.nrc.cadc.dali.Polygon; +import ca.nrc.cadc.dali.tables.parquet.ParquetReader; +import ca.nrc.cadc.dali.tables.parquet.ParquetWriter; +import ca.nrc.cadc.dali.tables.parquet.io.FileRandomAccessSource; +import ca.nrc.cadc.dali.tables.parquet.io.HttpRandomAccessSource; +import ca.nrc.cadc.dali.tables.parquet.io.InMemoryRandomAccessSource; +import ca.nrc.cadc.dali.tables.parquet.io.RandomAccessSource; +import ca.nrc.cadc.dali.tables.votable.VOTableDocument; +import ca.nrc.cadc.dali.tables.votable.VOTableReader; +import ca.nrc.cadc.dali.tables.votable.VOTableTable; +import ca.nrc.cadc.net.HttpPost; +import ca.nrc.cadc.net.ResourceNotFoundException; +import ca.nrc.cadc.vosi.actions.TableContentHandler; +import org.apache.log4j.Logger; +import org.junit.Assert; +import org.junit.Test; + +import javax.security.auth.Subject; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.InputStream; +import java.io.IOException; +import java.net.URI; +import java.net.URL; +import java.security.PrivilegedActionException; +import java.security.PrivilegedExceptionAction; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.UUID; + +public class ParquetReaderTest extends AbstractTablesTest { + + private static final Logger log = Logger.getLogger(ParquetReaderTest.class); + + public ParquetReaderTest() { + super(); + } + + @Test + public void testParquetReader() throws Exception { + log.info("Running ParquetReaderTest"); + testParquetReader(true); + testParquetReader(false); + testHttpRemoteAccessSource(true); + testHttpRemoteAccessSource(false); + } + + private void testParquetReader(boolean withMetadata) throws Exception { + String testTable = testSchemaName + ".testReadParquet"; + + // create table + doCreateTable(schemaOwner, testTable); + + // load table data + loadTableData(testTable); + + // Write out the table into Parquet format + ByteArrayOutputStream out = getParquetStreamingData(withMetadata, testTable); + + // Read the Parquet data into a VOTableDocument + ParquetReader reader = new ParquetReader(); + InputStream inputStream = new ByteArrayInputStream(out.toByteArray()); + VOTableDocument voDocFromParquetReader = reader.read(inputStream); + Assert.assertNotNull(voDocFromParquetReader); + + verifyTableData(voDocFromParquetReader.getResourceByType("results").getTable(), withMetadata); + + // Test In memory parquet content via InMemoryRandomAccessSource + RandomAccessSource randomAccessSource = new InMemoryRandomAccessSource(out.toByteArray()); + voDocFromParquetReader = reader.read(randomAccessSource); + Assert.assertNotNull(voDocFromParquetReader); + verifyTableData(voDocFromParquetReader.getResourceByType("results").getTable(), withMetadata); + + // Test local parquet file via FileRandomAccessSource + File tempFile = File.createTempFile("parquetTest", ".parquet"); + tempFile.deleteOnExit(); + try (FileOutputStream fos = new FileOutputStream(tempFile)) { + fos.write(out.toByteArray()); + } + + randomAccessSource = new FileRandomAccessSource(tempFile); + voDocFromParquetReader = reader.read(randomAccessSource); + Assert.assertNotNull(voDocFromParquetReader); + verifyTableData(voDocFromParquetReader.getResourceByType("results").getTable(), withMetadata); + } + + private void testHttpRemoteAccessSource(boolean withMetadata) throws IOException, ResourceNotFoundException { + ParquetReader reader = new ParquetReader(); + String remoteFileName = withMetadata ? "parquet-with-metadata" : "parquet-without-metadata"; + URL artifactURL = new URL("https://ws-cadc.canfar.net/vault/files/CADC/test-data/tap-upload/" + remoteFileName + ".parquet"); + + RandomAccessSource randomAccessSource = new HttpRandomAccessSource(artifactURL); + VOTableDocument voDocFromParquetReader = reader.read(randomAccessSource); + Assert.assertNotNull(voDocFromParquetReader); + verifyTableData(voDocFromParquetReader.getResourceByType("results").getTable(), withMetadata); + } + + // Write out the table into Parquet format - Use writer directly to test it with and without metadata + private ByteArrayOutputStream getParquetStreamingData(boolean withMetadata, String testTable) throws PrivilegedActionException, IOException { + // get data in votable format + String adql = "SELECT * from " + testTable; + + Map params = new TreeMap<>(); + params.put("LANG", "ADQL"); + params.put("QUERY", adql); + params.put("RESPONSEFORMAT", "votable"); + + String result = Subject.doAs(schemaOwner, new AuthQueryTest.SyncQueryAction(anonQueryURL, params)); + Assert.assertNotNull(result); + + VOTableReader voTableReader = new VOTableReader(); + VOTableDocument voTableDocument = voTableReader.read(result); + + // write the votable to parquet format + ByteArrayOutputStream out = new ByteArrayOutputStream(); + ParquetWriter parquetWriter = new ParquetWriter(withMetadata); + parquetWriter.write(voTableDocument, out); + return out; + } + + private void verifyTableData(VOTableTable voTableFromParquet, boolean withMetadata) { + Iterator> parquetDataIterator = voTableFromParquet.getTableData().iterator(); + int count = 0; + while (parquetDataIterator.hasNext()) { + List next = parquetDataIterator.next(); + Assert.assertEquals("string" + count, next.get(0)); + Assert.assertEquals((int) Short.MAX_VALUE, next.get(1)); + Assert.assertEquals(Integer.MAX_VALUE, next.get(2)); + Assert.assertEquals(Long.MAX_VALUE, next.get(3)); + Assert.assertEquals(Float.MAX_VALUE, next.get(4)); + Assert.assertEquals(Double.MAX_VALUE, next.get(5)); + Assert.assertTrue(next.get(6) instanceof Date); + if (withMetadata) { + assertEquals(new DoubleInterval(1.0, 2.0), (DoubleInterval) next.get(7)); + assertEquals(new Point(1.0, 2.0), (Point) next.get(8)); + assertEquals(new Circle(new Point(1, 2), 3), (Circle) next.get(9)); + Polygon p = new Polygon(); + p.getVertices().add(new Point(1.0, 2.0)); + p.getVertices().add(new Point(3.0, 4.0)); + p.getVertices().add(new Point(5.0, 6.0)); + assertEquals(p, (Polygon) next.get(10)); + } else { + Assert.assertArrayEquals(new double[]{1.0, 2.0}, (double[]) next.get(7), 0.0001); + Assert.assertArrayEquals(new double[]{1.0, 2.0}, (double[]) next.get(8), 0.0001); + Assert.assertArrayEquals(new double[]{1.0, 2.0, 3.0}, (double[]) next.get(9), 0.0001); + Assert.assertArrayEquals(new double[]{1.0, 2.0, 3.0, 4.0, 5.0, 6.0}, (double[]) next.get(10), 0.0001); + } + + Assert.assertArrayEquals(new int[]{Short.MIN_VALUE, Short.MAX_VALUE}, (int[]) next.get(11)); + Assert.assertArrayEquals(new int[]{Integer.MIN_VALUE, Integer.MAX_VALUE}, (int[]) next.get(12)); + Assert.assertArrayEquals(new long[]{Long.MIN_VALUE, Long.MAX_VALUE}, (long[]) next.get(13)); + Assert.assertArrayEquals(new float[]{Float.MIN_VALUE, Float.MAX_VALUE}, (float[]) next.get(14), 0.0001f); + Assert.assertArrayEquals(new double[]{Double.MIN_VALUE, Double.MAX_VALUE}, (double[]) next.get(15), 0.0001); + + Assert.assertEquals("ivo://opencadc.org/youcat", next.get(16).toString()); + Assert.assertTrue(next.get(17) instanceof UUID); + count++; + } + Assert.assertEquals(10, count); + } + + private void loadTableData(String testTable) throws Exception { + StringBuilder data = prepareData(); + + URL postURL = new URL(certLoadURL.toString() + "/" + testTable); + final HttpPost post = new HttpPost(postURL, data.toString(), TableContentHandler.CONTENT_TYPE_TSV, false); + Subject.doAs(schemaOwner, (PrivilegedExceptionAction) () -> { + post.run(); + return null; + }); + + Assert.assertNull(post.getThrowable()); + Assert.assertEquals(200, post.getResponseCode()); + } + + private static StringBuilder prepareData() { + StringBuilder data = new StringBuilder(); + data.append("c0\tc1\tc2\tc3\tc4\tc5\tc6\te7\te8\te9\te10\ta11\ta12\ta13\ta14\ta15\te16\te17\n"); + for (int i = 0; i < 10; i++) { + data.append("string").append(i).append("\t"); + data.append(Short.MAX_VALUE).append("\t"); + data.append(Integer.MAX_VALUE).append("\t"); + data.append(Long.MAX_VALUE).append("\t"); + data.append(Float.MAX_VALUE).append("\t"); + data.append(Double.MAX_VALUE).append("\t"); + + data.append("2018-11-05T22:12:33.111").append("\t"); + + data.append("1.0 2.0").append("\t"); // interval + data.append("1.0 2.0").append("\t"); // point + data.append("1.0 2.0 3.0").append("\t"); // circle + data.append("1.0 2.0 3.0 4.0 5.0 6.0").append("\t"); // polygon + + data.append(Short.MIN_VALUE + " " + Short.MAX_VALUE).append("\t"); + data.append(Integer.MIN_VALUE + " " + Integer.MAX_VALUE).append("\t"); + data.append(Long.MIN_VALUE + " " + Long.MAX_VALUE).append("\t"); + data.append(Float.MIN_VALUE + " " + Float.MAX_VALUE).append("\t"); + data.append(Double.MIN_VALUE + " " + Double.MAX_VALUE).append("\t"); + + data.append(URI.create("ivo://opencadc.org/youcat")).append("\t"); + data.append(UUID.randomUUID()).append("\n"); + } + return data; + } + +} \ No newline at end of file diff --git a/youcat/src/intTest/java/org/opencadc/youcat/ParquetWriterRoundTripTest.java b/youcat/src/intTest/java/org/opencadc/youcat/ParquetWriterTest.java similarity index 78% rename from youcat/src/intTest/java/org/opencadc/youcat/ParquetWriterRoundTripTest.java rename to youcat/src/intTest/java/org/opencadc/youcat/ParquetWriterTest.java index 3377ef9c..cb7c3ed3 100644 --- a/youcat/src/intTest/java/org/opencadc/youcat/ParquetWriterRoundTripTest.java +++ b/youcat/src/intTest/java/org/opencadc/youcat/ParquetWriterTest.java @@ -1,7 +1,6 @@ package org.opencadc.youcat; import ca.nrc.cadc.dali.tables.parquet.ParquetReader; -import ca.nrc.cadc.dali.tables.parquet.ParquetReader.TableShape; import ca.nrc.cadc.dali.tables.votable.VOTableDocument; import ca.nrc.cadc.dali.tables.votable.VOTableField; import ca.nrc.cadc.dali.tables.votable.VOTableInfo; @@ -20,8 +19,12 @@ import java.net.MalformedURLException; import java.net.URL; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.security.PrivilegedActionException; import java.security.PrivilegedExceptionAction; +import java.util.Date; +import java.util.Iterator; +import java.util.List; import java.util.Map; import java.util.TreeMap; import javax.security.auth.Subject; @@ -30,10 +33,10 @@ import org.junit.Test; import org.opencadc.tap.TapClient; -public class ParquetWriterRoundTripTest extends AbstractTablesTest { - private static final Logger log = Logger.getLogger(ParquetWriterRoundTripTest.class); +public class ParquetWriterTest extends AbstractTablesTest { + private static final Logger log = Logger.getLogger(ParquetWriterTest.class); private static URL url; - private static final Charset UTF8 = Charset.forName("utf-8"); + private static final Charset UTF8 = StandardCharsets.UTF_8; @Test public void testWriteParquetWithExistingTable() throws Exception { @@ -93,19 +96,13 @@ public void testWriteParquetWithCustomTable() throws Exception { private static VOTableTable extractVOTableFromOutputStream(ByteArrayOutputStream out, String adql) throws IOException { ParquetReader reader = new ParquetReader(); InputStream inputStream = new ByteArrayInputStream(out.toByteArray()); - TableShape readerResponse = reader.read(inputStream); - - log.info(readerResponse.getColumnCount() + " columns, " + readerResponse.getRecordCount() + " records"); - - Assert.assertTrue(readerResponse.getRecordCount() > 0); - Assert.assertTrue(readerResponse.getColumnCount() > 0); - - VOTableDocument voTableDocument = readerResponse.getVoTableDocument(); - + VOTableDocument voTableDocument = reader.read(inputStream); Assert.assertNotNull(voTableDocument.getResources()); VOTableResource results = voTableDocument.getResourceByType("results"); Assert.assertNotNull(results); + log.info(results.getTable().getFields().size() + " columns found."); + Assert.assertFalse(results.getTable().getFields().isEmpty()); boolean queryFound = false; boolean queryStatusFound = false; @@ -124,7 +121,6 @@ private static VOTableTable extractVOTableFromOutputStream(ByteArrayOutputStream Assert.assertTrue(queryStatusFound); Assert.assertNotNull(results.getTable()); - Assert.assertEquals(readerResponse.getColumnCount(), results.getTable().getFields().size()); return results.getTable(); } @@ -164,18 +160,51 @@ private static void compareVOTables(VOTableTable voTableFromVOTableWriter, VOTab Assert.assertEquals(field1.getName(), field2.getName()); - if (field1.xtype != null && field1.xtype.equals("timestamp")) { - Assert.assertEquals("long", field2.getDatatype()); - - Assert.assertNull(field2.xtype); - Assert.assertNull(field2.getArraysize()); - } else if (field1.getDatatype().equals("short")) { + if (field1.getDatatype().equals("short")) { Assert.assertEquals("int", field2.getDatatype()); } else { Assert.assertEquals(field1.getDatatype(), field2.getDatatype()); Assert.assertEquals(field1.getArraysize(), field2.getArraysize()); } } + + compareTableData(voTableFromVOTableWriter, voTableFromParquet); + } + + private static void compareTableData(VOTableTable voTableFromVOTableWriter, VOTableTable voTableFromParquet) { + Iterator> parquetDataIterator = voTableFromParquet.getTableData().iterator(); + Iterator> votableDataIterator = voTableFromVOTableWriter.getTableData().iterator(); + int count = 0; + while (parquetDataIterator.hasNext() && votableDataIterator.hasNext()) { + List pqRow = parquetDataIterator.next(); + List vtRow = votableDataIterator.next(); + Assert.assertEquals(vtRow.get(0), pqRow.get(0)); + Assert.assertEquals((int) Short.MAX_VALUE, pqRow.get(1)); + Assert.assertEquals(vtRow.get(2), pqRow.get(2)); + Assert.assertEquals(vtRow.get(3), pqRow.get(3)); + Assert.assertEquals(vtRow.get(4), pqRow.get(4)); + Assert.assertEquals(vtRow.get(5), pqRow.get(5)); + Assert.assertTrue(pqRow.get(6) instanceof Date); + Assert.assertEquals(vtRow.get(7), pqRow.get(7)); + Assert.assertEquals(vtRow.get(8), pqRow.get(8)); + Assert.assertEquals(vtRow.get(9), pqRow.get(9)); + Assert.assertEquals(vtRow.get(10), pqRow.get(10)); + + short[] shortArray = (short[]) vtRow.get(11); + int[] convertedShortArray = new int[shortArray.length]; + for (int i = 0; i < shortArray.length; i++) { + convertedShortArray[i] = shortArray[i]; + } + Assert.assertArrayEquals((int[]) pqRow.get(11), convertedShortArray); + + Assert.assertArrayEquals((int[]) vtRow.get(12), (int[]) pqRow.get(12)); + Assert.assertArrayEquals((long[]) vtRow.get(13), (long[]) pqRow.get(13)); + Assert.assertArrayEquals((float[]) vtRow.get(14), (float[]) pqRow.get(14), 0.0001f); + Assert.assertArrayEquals((double[]) vtRow.get(15), (double[]) pqRow.get(15), 0.0001); + + count++; + } + Assert.assertEquals(10, count); } private String queryVOTableWriter(String testTable) throws Exception { diff --git a/youcat/src/main/webapp/service.json b/youcat/src/main/webapp/service.json index 23e7099e..880e052d 100644 --- a/youcat/src/main/webapp/service.json +++ b/youcat/src/main/webapp/service.json @@ -427,7 +427,8 @@ ], "consumes": [ "text/xml", - "application/x-votable+xml" + "application/x-votable+xml", + "application/vnd.apache.parquet" ], "responses": { "201": {