diff --git a/dataframe-jdbc/build.gradle.kts b/dataframe-jdbc/build.gradle.kts index ea2160035a..dd7e4cfea1 100644 --- a/dataframe-jdbc/build.gradle.kts +++ b/dataframe-jdbc/build.gradle.kts @@ -25,6 +25,7 @@ dependencies { testImplementation(libs.postgresql) testImplementation(libs.mysql) testImplementation(libs.h2db) + testImplementation(libs.vertica) testImplementation(libs.junit) testImplementation(libs.sl4j) testImplementation(libs.kotestAssertions) { diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Vertica.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Vertica.kt new file mode 100644 index 0000000000..39045b45a0 --- /dev/null +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/Vertica.kt @@ -0,0 +1,48 @@ +package org.jetbrains.kotlinx.dataframe.io.db + +import org.jetbrains.kotlinx.dataframe.io.TableColumnMetadata +import org.jetbrains.kotlinx.dataframe.io.TableMetadata +import org.jetbrains.kotlinx.dataframe.schema.ColumnSchema +import java.sql.ResultSet +import kotlin.reflect.KType +import kotlin.reflect.full.createType + +/** + * Represents the Vertica database type. + * + * This class provides methods to convert data from a ResultSet to the appropriate type for Vertica, + * and to generate the corresponding column schema. + */ +public object Vertica : DbType("vertica") { + override val driverClassName: String + get() = "com.vertica.jdbc.Driver" + + override fun convertSqlTypeToColumnSchemaValue(tableColumnMetadata: TableColumnMetadata): ColumnSchema? = + when(tableColumnMetadata.sqlTypeName.uppercase()) { + "UUID" -> ColumnSchema.Value(String::class.createType(nullable = tableColumnMetadata.isNullable)) + "ARRAY" -> ColumnSchema.Value(String::class.createType(nullable = tableColumnMetadata.isNullable)) + "UNKNOWN" -> ColumnSchema.Value(String::class.createType(nullable = tableColumnMetadata.isNullable)) + else -> null + } + + override fun isSystemTable(tableMetadata: TableMetadata): Boolean { + val schemaName = tableMetadata.schemaName + + return schemaName?.startsWith("v_", true) ?: false + } + + override fun buildTableMetadata(tables: ResultSet): TableMetadata { + return TableMetadata( + tables.getString("table_name"), + tables.getString("table_schem"), + tables.getString("table_cat")) + } + + override fun convertSqlTypeToKType(tableColumnMetadata: TableColumnMetadata): KType? = + when(tableColumnMetadata.sqlTypeName.uppercase()) { + "UUID" -> String::class.createType(nullable = tableColumnMetadata.isNullable) + "ARRAY" -> String::class.createType(nullable = tableColumnMetadata.isNullable) + "UNKNOWN" -> String::class.createType(nullable = tableColumnMetadata.isNullable) + else -> null + } +} diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/util.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/util.kt index cfe59732f4..d8975a745e 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/util.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/db/util.kt @@ -17,8 +17,9 @@ public fun extractDBTypeFromUrl(url: String?): DbType { MySql.dbTypeInJdbcUrl in url -> MySql Sqlite.dbTypeInJdbcUrl in url -> Sqlite PostgreSql.dbTypeInJdbcUrl in url -> PostgreSql + Vertica.dbTypeInJdbcUrl in url -> Vertica else -> throw IllegalArgumentException("Unsupported database type in the url: $url. " + - "Only H2, MariaDB, MySQL, SQLite and PostgreSQL are supported!") + "Only H2, MariaDB, MySQL, SQLite, PostgreSQL and Vertica are supported!") } } else { throw SQLException("Database URL could not be null. The existing value is $url") diff --git a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt index c695c9bfb5..858d857bac 100644 --- a/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt +++ b/dataframe-jdbc/src/main/kotlin/org/jetbrains/kotlinx/dataframe/io/readJdbc.kt @@ -1,31 +1,32 @@ package org.jetbrains.kotlinx.dataframe.io import io.github.oshai.kotlinlogging.KotlinLogging +import org.jetbrains.kotlinx.dataframe.AnyFrame +import org.jetbrains.kotlinx.dataframe.DataColumn +import org.jetbrains.kotlinx.dataframe.DataFrame +import org.jetbrains.kotlinx.dataframe.api.toDataFrame +import org.jetbrains.kotlinx.dataframe.impl.schema.DataFrameSchemaImpl +import org.jetbrains.kotlinx.dataframe.io.db.DbType +import org.jetbrains.kotlinx.dataframe.io.db.Vertica +import org.jetbrains.kotlinx.dataframe.io.db.extractDBTypeFromUrl +import org.jetbrains.kotlinx.dataframe.schema.ColumnSchema +import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema import java.math.BigDecimal +import java.sql.Blob +import java.sql.Clob import java.sql.Connection import java.sql.DatabaseMetaData import java.sql.DriverManager +import java.sql.NClob +import java.sql.Ref import java.sql.ResultSet import java.sql.ResultSetMetaData +import java.sql.RowId +import java.sql.SQLXML import java.sql.Time import java.sql.Timestamp import java.sql.Types -import java.sql.RowId -import java.sql.Ref -import java.sql.Clob -import java.sql.Blob -import java.sql.NClob -import java.sql.SQLXML import java.util.Date -import org.jetbrains.kotlinx.dataframe.AnyFrame -import org.jetbrains.kotlinx.dataframe.DataColumn -import org.jetbrains.kotlinx.dataframe.DataFrame -import org.jetbrains.kotlinx.dataframe.api.toDataFrame -import org.jetbrains.kotlinx.dataframe.impl.schema.DataFrameSchemaImpl -import org.jetbrains.kotlinx.dataframe.io.db.DbType -import org.jetbrains.kotlinx.dataframe.io.db.extractDBTypeFromUrl -import org.jetbrains.kotlinx.dataframe.schema.ColumnSchema -import org.jetbrains.kotlinx.dataframe.schema.DataFrameSchema import kotlin.reflect.KType import kotlin.reflect.full.createType import kotlin.reflect.full.isSupertypeOf @@ -296,7 +297,7 @@ public fun DataFrame.Companion.readAllSqlTables( if (!dbType.isSystemTable(table)) { // we filter her second time because of specific logic with SQLite and possible issues with future databases // val tableName = if (table.catalogue != null) table.catalogue + "." + table.name else table.name - val tableName = if (catalogue != null) catalogue + "." + table.name else table.name + val tableName = getTableName(catalogue, table, dbType) // TODO: both cases is schema specified or not in URL // in h2 database name is recognized as a schema name https://www.h2database.com/html/features.html#database_url @@ -305,6 +306,7 @@ public fun DataFrame.Companion.readAllSqlTables( logger.debug { "Reading table: $tableName" } val dataFrame = readSqlTable(connection, tableName, limit) + dataFrames += dataFrame logger.debug { "Finished reading table: $tableName" } } @@ -313,6 +315,14 @@ public fun DataFrame.Companion.readAllSqlTables( return dataFrames } +private fun getTableName(catalogue: String?, table: TableMetadata, dbType: DbType) = + catalogue + ?.let { catalogue + "." + table.name } + ?: when (dbType) { + Vertica -> "${table.schemaName}.${table.name}" // Vertica needs schema name + else -> table.name + } + /** * Retrieves the schema for an SQL table using the provided database configuration. * @@ -642,7 +652,7 @@ private fun generateKType(dbType: DbType, tableColumnMetadata: TableColumnMetada * @param tableColumnMetadata The metadata of the table column. * @return The KType associated with the SQL type, or a default type if no mapping is found. */ -private fun makeCommonSqlToKTypeMapping(tableColumnMetadata: TableColumnMetadata): KType { +public fun makeCommonSqlToKTypeMapping(tableColumnMetadata: TableColumnMetadata): KType { val jdbcTypeToKTypeMapping = mapOf( Types.BIT to Boolean::class, Types.TINYINT to Int::class, diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/mysqlTest.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/mysqlTest.kt index 929a248b9e..8366ced6b5 100644 --- a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/mysqlTest.kt +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/mysqlTest.kt @@ -229,10 +229,38 @@ class MySqlTest { @Language("SQL") val insertData2 = """ INSERT INTO table2 ( - bitCol, tinyintCol, smallintCol, mediumintCol, mediumintUnsignedCol, integerCol, intCol, - integerUnsignedCol, bigintCol, floatCol, doubleCol, decimalCol, dateCol, datetimeCol, timestampCol, - timeCol, yearCol, varcharCol, charCol, binaryCol, varbinaryCol, tinyblobCol, blobCol, - mediumblobCol, longblobCol, textCol, mediumtextCol, longtextCol, enumCol, setCol, location, data + bitCol, + tinyintCol, + smallintCol, + mediumintCol, + mediumintUnsignedCol, + integerCol, + intCol, + integerUnsignedCol, + bigintCol, + floatCol, + doubleCol, + decimalCol, + dateCol, + datetimeCol, + timestampCol, + timeCol, + yearCol, + varcharCol, + charCol, + binaryCol, + varbinaryCol, + tinyblobCol, + blobCol, + mediumblobCol, + longblobCol, + textCol, + mediumtextCol, + longtextCol, + enumCol, + setCol, + location, + data ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ST_GeomFromText('POINT(1 1)'), ?) """.trimIndent() diff --git a/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/verticaTest.kt b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/verticaTest.kt new file mode 100644 index 0000000000..1c6b3b2b54 --- /dev/null +++ b/dataframe-jdbc/src/test/kotlin/org/jetbrains/kotlinx/dataframe/io/verticaTest.kt @@ -0,0 +1,340 @@ +package org.jetbrains.kotlinx.dataframe.io + +import com.vertica.dsi.dataengine.utilities.TimeTz +import com.vertica.dsi.dataengine.utilities.TimestampTz +import com.vertica.jdbc.VerticaDayTimeInterval +import com.vertica.jdbc.jdbc42.S42Array +import com.vertica.util.VerticaStruct +import io.kotest.matchers.shouldBe +import org.intellij.lang.annotations.Language +import org.jetbrains.kotlinx.dataframe.DataFrame +import org.jetbrains.kotlinx.dataframe.annotations.DataSchema +import org.jetbrains.kotlinx.dataframe.api.cast +import org.jetbrains.kotlinx.dataframe.api.filter +import org.junit.AfterClass +import org.junit.BeforeClass +import org.junit.Ignore +import org.junit.Test +import java.math.BigDecimal +import java.sql.Connection +import java.sql.DriverManager +import java.sql.SQLException +import java.sql.Time +import java.sql.Timestamp +import java.time.LocalDate +import java.time.LocalDateTime +import java.time.LocalTime +import java.util.Calendar +import java.util.Date +import java.util.UUID +import kotlin.reflect.typeOf + +// Run with https://hub.docker.com/r/vertica/vertica-ce +private const val URL = "jdbc:vertica://localhost:5433" +private const val USER_NAME = "dbadmin" +private const val PASSWORD = "" +private const val TEST_SCHEMA_NAME = "testschema" + +@DataSchema +interface Table1Vertica { + val id: Int + val varcharCol: String +} + +@DataSchema +interface Table2Vertica { + val id: Int + val boolCol: Boolean +} + +@DataSchema +interface Table3Vertica { + val id: Int + val varcharCol: String + val boolCol: Boolean +} + +@Ignore +class VerticaTest { + companion object { + private lateinit var connection: Connection + + @BeforeClass + @JvmStatic + fun setUpClass() { + connection = DriverManager.getConnection(URL, USER_NAME, PASSWORD) + + connection.createStatement().use { st -> + // Drop the test schema if it exists + val dropSchemaQuery = "DROP SCHEMA IF EXISTS $TEST_SCHEMA_NAME" + st.executeUpdate(dropSchemaQuery) + + // Create the test schema + val createSchemaQuery = "CREATE SCHEMA $TEST_SCHEMA_NAME" + st.executeUpdate(createSchemaQuery) + +// Set the schema as the default schema + val setDefaultSchemaQuery = "SET SEARCH_PATH TO $TEST_SCHEMA_NAME" + st.execute(setDefaultSchemaQuery) + } + + connection.createStatement().use { st -> + st.execute("DROP TABLE IF EXISTS table1") + st.execute("DROP TABLE IF EXISTS table2") + } + + @Language("SQL") + val createTableQuery = """ + CREATE TABLE IF NOT EXISTS table1 ( + id INT NOT NULL PRIMARY KEY, + boolCol BOOLEAN, + charCol CHAR(10), + varcharCol VARCHAR, + longvarcharCol LONG VARCHAR, + binaryCol BINARY(11), + varbinaryCol VARBINARY, + longvarbinaryCol LONG VARBINARY, + dateCol DATE, + timeCol TIME, + timestampCol TIMESTAMP, + doubleprecisionCol DOUBLE PRECISION, + floatCol FLOAT, + float8Col FLOAT8, + realCol REAL, + integerCol INTEGER, + intCol INT, + bigintCol BIGINT, + int8Col INT8, + smallintCol SMALLINT, + tinyintCol TINYINT, + decimalCol DECIMAL, + numericCol NUMERIC, + numberCol NUMBER, + moneyCol MONEY, + geometryCol GEOMETRY, + geographyCol GEOGRAPHY, + timewithtimezoneCol TIMETZ, + timestampwithtimezoneCol TIMESTAMPTZ, + uuidCol UUID, + arrayCol ARRAY[VARCHAR(50)], + rowCol ROW(street VARCHAR, city VARCHAR), + setCol SET[VARCHAR], + intervalCol INTERVAL + ) + """ + + @Language("SQL") + val createTable2Query = """ + CREATE TABLE IF NOT EXISTS table2 ( + id INT NOT NULL PRIMARY KEY, + boolCol BOOLEAN + ) + """ + + connection.createStatement().execute( + createTableQuery.trimIndent() + ) + + connection.createStatement().execute( + createTable2Query.trimIndent() + ) + + @Language("SQL") + val insertData1 = """ + INSERT INTO table1 VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, public.ST_GeomFromText('POINT(1 1)'), public.ST_GeographyFromText('POLYGON((1 2,3 4,2 3,1 2))'), ?, ?, ?, ARRAY['Test', 'Test1'], ROW('aStreet', 'aCity'), SET['aStreet', 'aCity'], INTERVAL '1 12:59:10:05') + """.trimIndent() + + connection.prepareStatement(insertData1).use { st -> + // Insert data into table1 + for (i in 1..3) { + st.setInt(1, i) + st.setBoolean(2, true) + st.setString(3, "charValue$i") + st.setString(4, "varcharValue$i") + st.setString(5, "longvarcharValue$i") + st.setBytes(6, "binaryValue".toByteArray()) + st.setBytes(7, "varbinaryValue".toByteArray()) + st.setBytes(8, "longvarbinaryValue".toByteArray()) + st.setDate(9, java.sql.Date.valueOf(LocalDate.of(2024, 1, 1))) + st.setTime(10, Time.valueOf(LocalTime.of(10, 0, 0))) + st.setTimestamp(11, Timestamp.valueOf(LocalDateTime.of(2024, 1, 1, 10, 0, 0))) + st.setFloat(12, i * 10.0f) + st.setFloat(13, i * 10.0f) + st.setFloat(14, i * 10.0f) + st.setFloat(15, i * 10.0f) + st.setInt(16, i * 100) + st.setInt(17, i * 100) + st.setInt(18, i * 100) + st.setInt(19, i * 100) + st.setInt(20, i * 100) + st.setInt(21, i * 100) + st.setBigDecimal(22, BigDecimal(i * 10)) + st.setBigDecimal(23, BigDecimal(i * 10)) + st.setBigDecimal(24, BigDecimal(i * 10)) + st.setBigDecimal(25, BigDecimal(i * 10)) + st.setTime(26, TimeTz(Time.valueOf(LocalTime.of(10, 0, 0, 0)), Calendar.getInstance())) + st.setTimestamp( + 27, + TimestampTz(Timestamp.valueOf(LocalDateTime.of(2024, 1, 1, 10, 0, 0)), Calendar.getInstance()) + ) + st.setString(28, "4a866db2-baa6-442a-a371-1f4b5ee627ba") + st.executeUpdate() + } + } + + @Language("SQL") + val insertData2 = """ + INSERT INTO table2 VALUES (?, ?) + """.trimIndent() + + connection.prepareStatement(insertData2).use { st -> + // Insert data into table2 + for (i in 1..3) { + st.setInt(1, i) + st.setBoolean(2, true) + st.executeUpdate() + } + } + } + + @AfterClass + @JvmStatic + fun tearDownClass() { + try { + connection.createStatement().use { st -> + st.execute("DROP TABLE IF EXISTS table1") + st.execute("DROP TABLE IF EXISTS table2") + } + connection.createStatement().use { st -> st.execute("DROP SCHEMA IF EXISTS $TEST_SCHEMA_NAME") } + } catch (e: SQLException) { + e.printStackTrace() + } finally { + connection.close() + } + } + } + + @Test + fun `basic test for reading sql tables`() { + connection.createStatement().use { st -> +// Set the schema as the default schema + val setDefaultSchemaQuery = "SET SEARCH_PATH TO $TEST_SCHEMA_NAME" + st.execute(setDefaultSchemaQuery) + } + + val df = DataFrame.readSqlTable(connection, "table1").cast() + df.rowsCount() shouldBe 3 + val result = df.filter { it[Table1Vertica::id] == 1 } + result[0][0] shouldBe 1L + result[0][1] shouldBe true + result[0][2] shouldBe "charValue1" + result[0][3] shouldBe "varcharValue1" + result[0][4] shouldBe "longvarcharValue1" + result[0][5] shouldBe "binaryValue".toByteArray() + result[0][6] shouldBe "varbinaryValue".toByteArray() + result[0][7] shouldBe "longvarbinaryValue".toByteArray() + result[0][8] shouldBe java.sql.Date.valueOf(LocalDate.of(2024, 1, 1)) + result[0][9] shouldBe Time.valueOf(LocalTime.of(10, 0, 0)) + result[0][10] shouldBe Timestamp.valueOf(LocalDateTime.of(2024, 1, 1, 10, 0, 0)) + result[0][11] shouldBe 1 * 10.0f + result[0][12] shouldBe 1 * 10.0f + result[0][13] shouldBe 1 * 10.0f + result[0][14] shouldBe 1 * 10.0f + result[0][15] shouldBe 1 * 100 + result[0][16] shouldBe 1 * 100 + result[0][17] shouldBe 1 * 100 + result[0][18] shouldBe 1 * 100 + result[0][19] shouldBe 1 * 100 + result[0][20] shouldBe 1 * 100 + result[0][21] shouldBe BigDecimal("10.000000000000000") + result[0][22] shouldBe BigDecimal("10.000000000000000") + result[0][23] shouldBe BigDecimal("10") + result[0][24] shouldBe BigDecimal("10.0000") + result[0][27] shouldBe TimeTz(Time.valueOf(LocalTime.of(10, 0, 0, 0)), Calendar.getInstance()) + (result[0][28] as TimestampTz).time shouldBe TimestampTz(Timestamp.valueOf(LocalDateTime.of(2024, 1,1, 10,0,0)), Calendar.getInstance()).time + result[0][29] shouldBe UUID.fromString("4a866db2-baa6-442a-a371-1f4b5ee627ba") + (result[0][30] as S42Array).toString() shouldBe "[\"Test\",\"Test1\"]" + (result[0][31] as VerticaStruct).toString() shouldBe "{\"street\":\"aStreet\",\"city\":\"aCity\"}" + (result[0][32] as S42Array).toString() shouldBe "[\"aCity\",\"aStreet\"]" + (result[0][33] as VerticaDayTimeInterval).toString() shouldBe "1 12:59:10.005000" + + val schema = DataFrame.getSchemaForSqlTable(connection, "table1") + schema.columns["id"]!!.type shouldBe typeOf() + schema.columns["boolCol"]!!.type shouldBe typeOf() + schema.columns["charCol"]!!.type shouldBe typeOf() + schema.columns["varcharCol"]!!.type shouldBe typeOf() + schema.columns["longvarcharCol"]!!.type shouldBe typeOf() + schema.columns["binaryCol"]!!.type shouldBe typeOf() + schema.columns["varbinaryCol"]!!.type shouldBe typeOf() + schema.columns["longvarbinaryCol"]!!.type shouldBe typeOf() + schema.columns["dateCol"]!!.type shouldBe typeOf() + schema.columns["timeCol"]!!.type shouldBe typeOf() + schema.columns["timestampCol"]!!.type shouldBe typeOf() + schema.columns["doubleprecisionCol"]!!.type shouldBe typeOf() + schema.columns["floatCol"]!!.type shouldBe typeOf() + schema.columns["float8Col"]!!.type shouldBe typeOf() + schema.columns["realCol"]!!.type shouldBe typeOf() + schema.columns["integerCol"]!!.type shouldBe typeOf() + schema.columns["intCol"]!!.type shouldBe typeOf() + schema.columns["bigintCol"]!!.type shouldBe typeOf() + schema.columns["int8Col"]!!.type shouldBe typeOf() + schema.columns["smallintCol"]!!.type shouldBe typeOf() + schema.columns["tinyintCol"]!!.type shouldBe typeOf() + schema.columns["decimalCol"]!!.type shouldBe typeOf() + schema.columns["numericCol"]!!.type shouldBe typeOf() + schema.columns["numberCol"]!!.type shouldBe typeOf() + schema.columns["moneyCol"]!!.type shouldBe typeOf() + schema.columns["geometryCol"]!!.type shouldBe typeOf() + schema.columns["geographyCol"]!!.type shouldBe typeOf() + schema.columns["timewithtimezoneCol"]!!.type shouldBe typeOf() + schema.columns["timestampwithtimezoneCol"]!!.type shouldBe typeOf() + schema.columns["uuidCol"]!!.type shouldBe typeOf() + schema.columns["arrayCol"]!!.type shouldBe typeOf() + schema.columns["rowCol"]!!.type shouldBe typeOf() + schema.columns["setCol"]!!.type shouldBe typeOf() + schema.columns["intervalCol"]!!.type shouldBe typeOf() + } + + @Test + fun `read from sql query`() { + connection.createStatement().use { st -> +// Set the schema as the default schema + val setDefaultSchemaQuery = "SET SEARCH_PATH TO $TEST_SCHEMA_NAME" + st.execute(setDefaultSchemaQuery) + } + + @Language("SQL") + val sqlQuery = """ + SELECT + t1.id, + t1.varcharCol, + t2.boolCol + FROM table1 t1 + JOIN table2 t2 ON t1.id = t2.id + """.trimIndent() + + val df = DataFrame.readSqlQuery(connection, sqlQuery = sqlQuery).cast() + val result = df.filter { it[Table3Vertica::id] == 1 } + result[0][2] shouldBe true + + val schema = DataFrame.getSchemaForSqlQuery(connection, sqlQuery = sqlQuery) + schema.columns["id"]!!.type shouldBe typeOf() + schema.columns["varcharCol"]!!.type shouldBe typeOf() + schema.columns["boolCol"]!!.type shouldBe typeOf() + } + + @Test + fun `read from all tables`() { + connection.createStatement().use { st -> +// Set the schema as the default schema + val setDefaultSchemaQuery = "SET SEARCH_PATH TO DEFAULT" + st.execute(setDefaultSchemaQuery) + } + + val dataframes = DataFrame.readAllSqlTables(connection, limit = 1) + + val table1Df = dataframes.first { it.columnNames().any {column -> column == "geometryCol" } } + + table1Df.columnsCount() shouldBe 34 + } +} diff --git a/docs/StardustDocs/topics/gradleReference.md b/docs/StardustDocs/topics/gradleReference.md index 5c848e261a..dc049a83e3 100644 --- a/docs/StardustDocs/topics/gradleReference.md +++ b/docs/StardustDocs/topics/gradleReference.md @@ -156,7 +156,7 @@ dataframes { Find full example code [here](https://github.com/zaleslaw/KotlinDataFrame-SQL-Examples/blob/master/src/main/kotlin/Example_3_Import_schema_via_Gradle.kt). **NOTE:** This is an experimental functionality and, for now, -we only support four databases: MariaDB, MySQL, PostgreSQL, and SQLite. +we only support four databases: MariaDB, MySQL, PostgreSQL, SQLite and Vertica. Additionally, support for JSON and date-time types is limited. Please take this into consideration when using these functions. diff --git a/docs/StardustDocs/topics/readSqlDatabases.md b/docs/StardustDocs/topics/readSqlDatabases.md index fda5388cb6..982d6f2047 100644 --- a/docs/StardustDocs/topics/readSqlDatabases.md +++ b/docs/StardustDocs/topics/readSqlDatabases.md @@ -15,7 +15,7 @@ There are two main blocks of available functionality: * ```getSchemaForAllSqlTables``` for all non-system tables **NOTE:** This is an experimental module and for now, -we only support four databases: MariaDB, MySQL, PostgreSQL, and SQLite. +we only support four databases: MariaDB, MySQL, PostgreSQL, SQLite and Vertica. Additionally, support for JSON and date-time types is limited. Please take this into consideration when using these functions. @@ -55,6 +55,12 @@ For SQLite: implementation("org.xerial:sqlite-jdbc:$version") ``` +For Vertica: + +```kotlin +implementation("com.vertica.jdbc:vertica-jdbc:$version") +``` + In the second, be sure that you can establish a connection to the database. For this, usually, you need to have three things: a URL to a database, a username and a password. @@ -240,7 +246,7 @@ and transforms it into an AnyFrame object. The `dbType: DbType` parameter specifies the type of our database (e.g., PostgreSQL, MySQL, etc.), supported by a library. -Currently, the following classes are available: `H2, MariaDb, MySql, PostgreSql, Sqlite`. +Currently, the following classes are available: `H2, MariaDb, MySql, PostgreSql, Sqlite, Vertica`. ```kotlin @@ -449,7 +455,7 @@ This function reads the schema from a ResultSet object provided by the user. The `dbType: DbType` parameter specifies the type of our database (e.g., PostgreSQL, MySQL, etc.), supported by a library. -Currently, the following classes are available: `H2, MariaDb, MySql, PostgreSql, Sqlite`. +Currently, the following classes are available: `H2, MariaDb, MySql, PostgreSql, Sqlite, Vertica`. ```kotlin import org.jetbrains.kotlinx.dataframe.io.db.PostgreSql diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 897492bf7f..7bddabfadd 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -34,6 +34,7 @@ h2db = "2.2.224" mysql = "8.0.33" postgresql = "42.7.1" sqlite = "3.45.1.0" +vertica = "24.1.0-0" kotlinDatetime = "0.5.0" openapi = "2.1.20" kotlinLogging = "6.0.3" @@ -77,6 +78,7 @@ h2db = { group = "com.h2database", name = "h2", version.ref = "h2db" } mysql = { group = "mysql", name = "mysql-connector-java", version.ref = "mysql" } postgresql = { group = "org.postgresql", name = "postgresql", version.ref = "postgresql" } sqlite = { group = "org.xerial", name = "sqlite-jdbc", version.ref = "sqlite" } +vertica = { group = "com.vertica.jdbc", name = "vertica-jdbc", version.ref = "vertica" } poi-ooxml = { group = "org.apache.poi", name = "poi-ooxml", version.ref = "poi" } kotlin-datetimeJvm = { group = "org.jetbrains.kotlinx", name = "kotlinx-datetime-jvm", version.ref = "kotlinDatetime" } diff --git a/gradle/scripts/pre-commit b/gradle/scripts/pre-commit old mode 100644 new mode 100755