From ef9da7d6c41a1e6dea12a02470cc71b4377d0578 Mon Sep 17 00:00:00 2001 From: Antonio171003 Date: Tue, 3 Sep 2024 21:56:24 -0700 Subject: [PATCH 1/3] Handle the bytea data received to be stored correctly in the database. --- .../wiringbits/spra/admin/models/FieldValue.scala | 8 ++++++++ .../repositories/DatabaseTablesRepository.scala | 12 ++++++++++-- .../admin/repositories/daos/DatabaseTablesDAO.scala | 12 ++++++------ .../wiringbits/spra/admin/utils/QueryBuilder.scala | 8 ++++---- .../wiringbits/spra/admin/utils/StringParse.scala | 12 ++++++++++++ 5 files changed, 40 insertions(+), 12 deletions(-) create mode 100644 spra-play-server/src/main/scala/net/wiringbits/spra/admin/models/FieldValue.scala create mode 100644 spra-play-server/src/main/scala/net/wiringbits/spra/admin/utils/StringParse.scala diff --git a/spra-play-server/src/main/scala/net/wiringbits/spra/admin/models/FieldValue.scala b/spra-play-server/src/main/scala/net/wiringbits/spra/admin/models/FieldValue.scala new file mode 100644 index 0000000..5781e02 --- /dev/null +++ b/spra-play-server/src/main/scala/net/wiringbits/spra/admin/models/FieldValue.scala @@ -0,0 +1,8 @@ +package net.wiringbits.spra.admin.models + +trait FieldValue[T] extends Serializable { + val value: T +} + +case class StringValue(value: String) extends FieldValue[String] +case class ByteArrayValue(value: Array[Byte]) extends FieldValue[Array[Byte]] diff --git a/spra-play-server/src/main/scala/net/wiringbits/spra/admin/repositories/DatabaseTablesRepository.scala b/spra-play-server/src/main/scala/net/wiringbits/spra/admin/repositories/DatabaseTablesRepository.scala index 4319c31..aa694eb 100644 --- a/spra-play-server/src/main/scala/net/wiringbits/spra/admin/repositories/DatabaseTablesRepository.scala +++ b/spra-play-server/src/main/scala/net/wiringbits/spra/admin/repositories/DatabaseTablesRepository.scala @@ -6,6 +6,8 @@ import net.wiringbits.spra.admin.repositories.daos.DatabaseTablesDAO import net.wiringbits.spra.admin.repositories.models.{DatabaseTable, ForeignKey, TableColumn, TableData} import net.wiringbits.spra.admin.utils.models.QueryParameters import play.api.db.Database +import net.wiringbits.spra.admin.models.{ByteArrayValue, StringValue} +import net.wiringbits.spra.admin.utils.StringParse import javax.inject.Inject import scala.concurrent.Future @@ -75,7 +77,10 @@ class DatabaseTablesRepository @Inject() (database: Database)(implicit val fieldsAndValues = body.map { case (key, value) => val field = columns.find(_.name == key).getOrElse(throw new RuntimeException(s"Invalid property in body request: $key")) - (field, value) + if (field.`type` == "bytea") + val byteaValue = StringParse.stringToByteArray(value) + (field, ByteArrayValue(byteaValue)) + else (field, StringValue(value)) } DatabaseTablesDAO.create( tableName = tableName, @@ -100,7 +105,10 @@ class DatabaseTablesRepository @Inject() (database: Database)(implicit val fieldsAndValues = bodyWithoutNonEditableColumns.map { case (key, value) => val field = columns.find(_.name == key).getOrElse(throw new RuntimeException(s"Invalid property in body request: $key")) - (field, value) + if (field.`type` == "bytea") + val byteaValue = StringParse.stringToByteArray(value) + (field, ByteArrayValue(byteaValue)) + else (field, StringValue(value)) } val primaryKeyType = settings.primaryKeyDataType DatabaseTablesDAO.update( diff --git a/spra-play-server/src/main/scala/net/wiringbits/spra/admin/repositories/daos/DatabaseTablesDAO.scala b/spra-play-server/src/main/scala/net/wiringbits/spra/admin/repositories/daos/DatabaseTablesDAO.scala index 2aed493..bf9794c 100644 --- a/spra-play-server/src/main/scala/net/wiringbits/spra/admin/repositories/daos/DatabaseTablesDAO.scala +++ b/spra-play-server/src/main/scala/net/wiringbits/spra/admin/repositories/daos/DatabaseTablesDAO.scala @@ -5,7 +5,7 @@ import net.wiringbits.spra.admin.config.{CustomDataType, PrimaryKeyDataType, Tab import net.wiringbits.spra.admin.repositories.models.* import net.wiringbits.spra.admin.utils.models.{FilterParameter, QueryParameters} import net.wiringbits.spra.admin.utils.{QueryBuilder, StringRegex} - +import net.wiringbits.spra.admin.models.{ByteArrayValue, FieldValue, StringValue} import java.sql.{Connection, Date, PreparedStatement, ResultSet} import java.time.LocalDate import java.util.UUID @@ -230,7 +230,7 @@ object DatabaseTablesDAO { } def create( tableName: String, - fieldsAndValues: Map[TableColumn, String], + fieldsAndValues: Map[TableColumn, FieldValue[_]], primaryKeyField: String, primaryKeyType: PrimaryKeyDataType = PrimaryKeyDataType.UUID )(implicit @@ -250,7 +250,7 @@ object DatabaseTablesDAO { // Postgres: INSERT INTO test_serial (id) VALUES(DEFAULT); MySQL: INSERT INTO table (id) VALUES(NULL) for (j <- i + 1 to fieldsAndValues.size + i) { - val value = fieldsAndValues(fieldsAndValues.keys.toList(j - i - 1)) + val value = fieldsAndValues(fieldsAndValues.keys.toList(j - i - 1)).value preparedStatement.setObject(j, value) } val result = preparedStatement.executeQuery() @@ -260,7 +260,7 @@ object DatabaseTablesDAO { def update( tableName: String, - fieldsAndValues: Map[TableColumn, String], + fieldsAndValues: Map[TableColumn, FieldValue[_]], primaryKeyField: String, primaryKeyValue: String, primaryKeyType: PrimaryKeyDataType = PrimaryKeyDataType.UUID @@ -268,9 +268,9 @@ object DatabaseTablesDAO { val sql = QueryBuilder.update(tableName, fieldsAndValues, primaryKeyField) val preparedStatement = conn.prepareStatement(sql) - val notNullData = fieldsAndValues.filterNot { case (_, value) => value == "null" } + val notNullData = fieldsAndValues.filterNot { case (_, value) => value.value == "null" } notNullData.zipWithIndex.foreach { case ((_, value), i) => - preparedStatement.setObject(i + 1, value) + preparedStatement.setObject(i + 1, value.value) } // where ... = ? setPreparedStatementKey(preparedStatement, primaryKeyValue, primaryKeyType, notNullData.size + 1) diff --git a/spra-play-server/src/main/scala/net/wiringbits/spra/admin/utils/QueryBuilder.scala b/spra-play-server/src/main/scala/net/wiringbits/spra/admin/utils/QueryBuilder.scala index 8193eba..2af6119 100644 --- a/spra-play-server/src/main/scala/net/wiringbits/spra/admin/utils/QueryBuilder.scala +++ b/spra-play-server/src/main/scala/net/wiringbits/spra/admin/utils/QueryBuilder.scala @@ -2,13 +2,13 @@ package net.wiringbits.spra.admin.utils import net.wiringbits.spra.admin.config.PrimaryKeyDataType import net.wiringbits.spra.admin.repositories.models.TableColumn - +import net.wiringbits.spra.admin.models.FieldValue import scala.collection.mutable object QueryBuilder { def create( tableName: String, - fieldsAndValues: Map[TableColumn, String], + fieldsAndValues: Map[TableColumn, FieldValue[_]], primaryKeyField: String, primaryKeyType: PrimaryKeyDataType = PrimaryKeyDataType.UUID ): String = { @@ -33,10 +33,10 @@ object QueryBuilder { |""".stripMargin } - def update(tableName: String, body: Map[TableColumn, String], primaryKeyField: String): String = { + def update(tableName: String, body: Map[TableColumn, FieldValue[_]], primaryKeyField: String): String = { val updateStatement = new mutable.StringBuilder("SET") for ((tableField, value) <- body) { - val resultStatement = if (value == "null") "NULL" else s"?::${tableField.`type`}" + val resultStatement = if (value.value == "null") "NULL" else s"?::${tableField.`type`}" val statement = s" ${tableField.name} = $resultStatement," updateStatement.append(statement) } diff --git a/spra-play-server/src/main/scala/net/wiringbits/spra/admin/utils/StringParse.scala b/spra-play-server/src/main/scala/net/wiringbits/spra/admin/utils/StringParse.scala new file mode 100644 index 0000000..c49d786 --- /dev/null +++ b/spra-play-server/src/main/scala/net/wiringbits/spra/admin/utils/StringParse.scala @@ -0,0 +1,12 @@ +package net.wiringbits.spra.admin.utils + +import scala.util.{Failure, Success, Try} + +object StringParse { + def stringToByteArray(value: String): Array[Byte] = { + // Removes whitespace characters (\\s) and brackets ([, ]) to prepare the string for byte array conversion + Try(value.replaceAll("[\\[\\]\\s]", "").split(",").map(_.toByte)) match + case Success(value) => value + case Failure(_) => Array.emptyByteArray + } +} From 87ebdf4ac211f5340a84f12a9a0037f378677d80 Mon Sep 17 00:00:00 2001 From: Antonio171003 Date: Wed, 4 Sep 2024 23:04:49 -0700 Subject: [PATCH 2/3] Added tests and request corrections --- .../spra/admin/models/FieldValue.scala | 2 +- .../repositories/daos/DatabaseTablesDAO.scala | 6 +-- .../spra/admin/utils/QueryBuilder.scala | 4 +- .../test/resources/evolutions/default/3.sql | 7 +++ .../controllers/AdminControllerSpec.scala | 49 ++++++++++++++++++- .../spra/admin/QueryBuilderSpec.scala | 18 ++++--- .../spra/admin/StringParseSpec.scala | 22 +++++++++ 7 files changed, 93 insertions(+), 15 deletions(-) create mode 100644 spra-play-server/src/test/resources/evolutions/default/3.sql create mode 100644 spra-play-server/src/test/scala/net/wiringbits/spra/admin/StringParseSpec.scala diff --git a/spra-play-server/src/main/scala/net/wiringbits/spra/admin/models/FieldValue.scala b/spra-play-server/src/main/scala/net/wiringbits/spra/admin/models/FieldValue.scala index 5781e02..2729df4 100644 --- a/spra-play-server/src/main/scala/net/wiringbits/spra/admin/models/FieldValue.scala +++ b/spra-play-server/src/main/scala/net/wiringbits/spra/admin/models/FieldValue.scala @@ -1,6 +1,6 @@ package net.wiringbits.spra.admin.models -trait FieldValue[T] extends Serializable { +sealed trait FieldValue[T] extends Serializable { val value: T } diff --git a/spra-play-server/src/main/scala/net/wiringbits/spra/admin/repositories/daos/DatabaseTablesDAO.scala b/spra-play-server/src/main/scala/net/wiringbits/spra/admin/repositories/daos/DatabaseTablesDAO.scala index bf9794c..23eef66 100644 --- a/spra-play-server/src/main/scala/net/wiringbits/spra/admin/repositories/daos/DatabaseTablesDAO.scala +++ b/spra-play-server/src/main/scala/net/wiringbits/spra/admin/repositories/daos/DatabaseTablesDAO.scala @@ -268,9 +268,9 @@ object DatabaseTablesDAO { val sql = QueryBuilder.update(tableName, fieldsAndValues, primaryKeyField) val preparedStatement = conn.prepareStatement(sql) - val notNullData = fieldsAndValues.filterNot { case (_, value) => value.value == "null" } - notNullData.zipWithIndex.foreach { case ((_, value), i) => - preparedStatement.setObject(i + 1, value.value) + val notNullData = fieldsAndValues.filterNot { case (_, field) => field.value == "null" } + notNullData.zipWithIndex.foreach { case ((_, field), i) => + preparedStatement.setObject(i + 1, field.value) } // where ... = ? setPreparedStatementKey(preparedStatement, primaryKeyValue, primaryKeyType, notNullData.size + 1) diff --git a/spra-play-server/src/main/scala/net/wiringbits/spra/admin/utils/QueryBuilder.scala b/spra-play-server/src/main/scala/net/wiringbits/spra/admin/utils/QueryBuilder.scala index 2af6119..ec07676 100644 --- a/spra-play-server/src/main/scala/net/wiringbits/spra/admin/utils/QueryBuilder.scala +++ b/spra-play-server/src/main/scala/net/wiringbits/spra/admin/utils/QueryBuilder.scala @@ -35,8 +35,8 @@ object QueryBuilder { def update(tableName: String, body: Map[TableColumn, FieldValue[_]], primaryKeyField: String): String = { val updateStatement = new mutable.StringBuilder("SET") - for ((tableField, value) <- body) { - val resultStatement = if (value.value == "null") "NULL" else s"?::${tableField.`type`}" + for ((tableField, field) <- body) { + val resultStatement = if (field.value == "null") "NULL" else s"?::${tableField.`type`}" val statement = s" ${tableField.name} = $resultStatement," updateStatement.append(statement) } diff --git a/spra-play-server/src/test/resources/evolutions/default/3.sql b/spra-play-server/src/test/resources/evolutions/default/3.sql new file mode 100644 index 0000000..bcb435a --- /dev/null +++ b/spra-play-server/src/test/resources/evolutions/default/3.sql @@ -0,0 +1,7 @@ + +-- !Ups + +CREATE TABLE bytea_table ( + id UUID NOT NULL PRIMARY KEY, + data BYTEA NOT NULL +); diff --git a/spra-play-server/src/test/scala/controllers/AdminControllerSpec.scala b/spra-play-server/src/test/scala/controllers/AdminControllerSpec.scala index 7194e23..f84c748 100644 --- a/spra-play-server/src/test/scala/controllers/AdminControllerSpec.scala +++ b/spra-play-server/src/test/scala/controllers/AdminControllerSpec.scala @@ -39,6 +39,10 @@ class AdminControllerSpec extends PlayPostgresSpec with AdminUtils { tableName = "big_serial_table_overflow", primaryKeyField = "id", primaryKeyDataType = PrimaryKeyDataType.BigSerial + ), + TableSettings( + tableName = "bytea_table", + primaryKeyField = "id" ) ) val dataExplorerConfig: DataExplorerConfig = DataExplorerConfig("http://localhost:9000", dataExplorerConfigTables) @@ -50,6 +54,7 @@ class AdminControllerSpec extends PlayPostgresSpec with AdminUtils { def bigSerialSettings: TableSettings = dataExplorerConfig.tablesSettings(4) def serialOverflowSettings: TableSettings = dataExplorerConfig.tablesSettings(5) def bigSerialOverflowSettings: TableSettings = dataExplorerConfig.tablesSettings(6) + def byteaSettings: TableSettings = dataExplorerConfig.tablesSettings(7) def isValidUUID(str: String): Boolean = { if (str == null) return false @@ -70,12 +75,13 @@ class AdminControllerSpec extends PlayPostgresSpec with AdminUtils { "return tables from modules" in withApiClient { client => val response = client.getTables.futureValue response.data.map(_.name) match - case List(users, userLogs, uuidTable, serialTable, bigSerialTable, _, _) => + case List(users, userLogs, uuidTable, serialTable, bigSerialTable, _, _, byteaTable) => users must be(usersSettings.tableName) userLogs must be(userLogsSettings.tableName) uuidTable must be(uuidSettings.tableName) serialTable must be(serialSettings.tableName) bigSerialTable must be(bigSerialSettings.tableName) + byteaTable must be(byteaSettings.tableName) case list => fail(s"Unexpected response: ${list.mkString(", ")}") } @@ -135,6 +141,17 @@ class AdminControllerSpec extends PlayPostgresSpec with AdminUtils { bigSerialSettings.filterableColumns must be(List.empty) bigSerialSettings.createSettings.nonRequiredColumns must be(List.empty) bigSerialSettings.createSettings.requiredColumns must be(List.empty) + + val head6 = response.data(5) + head6.primaryKeyName must be(byteaSettings.primaryKeyField) + byteaSettings.referenceField must be(None) + byteaSettings.hiddenColumns must be(List.empty) + byteaSettings.nonEditableColumns must be(List.empty) + byteaSettings.canBeDeleted must be(true) + byteaSettings.columnTypeOverrides must be(Map.empty) + byteaSettings.filterableColumns must be(List.empty) + byteaSettings.createSettings.nonRequiredColumns must be(List.empty) + byteaSettings.createSettings.requiredColumns must be(List.empty) } } @@ -680,7 +697,7 @@ class AdminControllerSpec extends PlayPostgresSpec with AdminUtils { responseMetadata.head.nonEmpty mustBe true } - + "return new user id" in withApiClient { implicit client => val user = createUser.futureValue val response = client.getTableMetadata(usersSettings.tableName, List("name", "ASC"), List(0, 9), "{}").futureValue @@ -737,6 +754,18 @@ class AdminControllerSpec extends PlayPostgresSpec with AdminUtils { s"ERROR: nextval: reached maximum value of sequence \"big_serial_table_overflow_seq\" (9223372036854775807)" ) } + + "create a new bytea" in withApiClient { implicit client => + val stringBytea = "[0, 10, 20, 30]" + // The response returns the bytea as Hex; this would be its equivalent in Hex. + val correctValue = "\\x000a141e" + val request = AdminCreateTable.Request(Map("data" -> stringBytea)) + val byteaId = client.createItem(byteaSettings.tableName, request).futureValue.id + + val response = client.viewItem(byteaSettings.tableName, byteaId).futureValue + val dataResponse = response.find(_._1 == "data").value._2 + dataResponse must be(correctValue) + } } "fail when field in request doesn't exists" in withApiClient { client => @@ -779,6 +808,22 @@ class AdminControllerSpec extends PlayPostgresSpec with AdminUtils { emailResponse must be(email) } + "update a new bytea" in withApiClient { client => + val request = AdminCreateTable.Request(Map("data" -> "[10, 10, 10, 10]")) + val byteaId = client.createItem(byteaSettings.tableName, request).futureValue.id + + val stringBytea = "[0, 10, 20, 30]" + // The response returns the bytea as Hex; this would be its equivalent in Hex. + val correctValue = "\\x000a141e" + val updateRequest = AdminUpdateTable.Request(Map("data" -> stringBytea)) + val updateResponse = client.updateItem(byteaSettings.tableName, byteaId, updateRequest).futureValue + + val newResponse = client.viewItem(byteaSettings.tableName, byteaId).futureValue + val dataResponse = newResponse.find(_._1 == "data").value._2 + updateResponse.id must be(byteaId) + dataResponse must be(correctValue) + } + "update a new row for all tables" in withApiClient { client => val tables = List(serialSettings, bigSerialSettings) for (table <- tables) { diff --git a/spra-play-server/src/test/scala/net/wiringbits/spra/admin/QueryBuilderSpec.scala b/spra-play-server/src/test/scala/net/wiringbits/spra/admin/QueryBuilderSpec.scala index 7f53bfc..0b33eb5 100644 --- a/spra-play-server/src/test/scala/net/wiringbits/spra/admin/QueryBuilderSpec.scala +++ b/spra-play-server/src/test/scala/net/wiringbits/spra/admin/QueryBuilderSpec.scala @@ -1,5 +1,6 @@ package net.wiringbits.spra.admin +import net.wiringbits.spra.admin.models.{FieldValue, StringValue} import net.wiringbits.spra.admin.repositories.models.TableColumn import net.wiringbits.spra.admin.utils.QueryBuilder import org.scalatest.matchers.must.Matchers.{be, must} @@ -20,7 +21,10 @@ class QueryBuilderSpec extends AnyWordSpec { |""".stripMargin val tableName = "users" val body = - Map(TableColumn("email", "citext") -> "wiringbits@wiringbits.net", TableColumn("name", "text") -> "wiringbits") + Map( + TableColumn("email", "citext") -> StringValue("wiringbits@wiringbits.net"), + TableColumn("name", "text") -> StringValue("wiringbits") + ) val primaryKeyField = "user_id" val response = QueryBuilder.create(tableName, body, primaryKeyField) @@ -40,7 +44,7 @@ class QueryBuilderSpec extends AnyWordSpec { |RETURNING user_id::TEXT |""".stripMargin val tableName = "users" - val body = Map.empty[TableColumn, String] + val body = Map.empty[TableColumn, FieldValue[_]] val primaryKeyField = "user_id" val response = QueryBuilder.create(tableName, body, primaryKeyField) @@ -58,8 +62,8 @@ class QueryBuilderSpec extends AnyWordSpec { |""".stripMargin val tableName = "users" val body = Map( - TableColumn("email", "citext") -> "wiringbits@wiringbits.net", - TableColumn("name", "text") -> "wiringbits@wiringbits.net" + TableColumn("email", "citext") -> StringValue("wiringbits@wiringbits.net"), + TableColumn("name", "text") -> StringValue("wiringbits@wiringbits.net") ) val primaryKeyField = "user_id" @@ -76,9 +80,9 @@ class QueryBuilderSpec extends AnyWordSpec { |""".stripMargin val tableName = "users" val body = Map( - TableColumn("email", "citext") -> "wiringbits@wiringbits.net", - TableColumn("name", "text") -> "wiringbits@wiringbits.net", - TableColumn("phone_number", "text") -> "null" + TableColumn("email", "citext") -> StringValue("wiringbits@wiringbits.net"), + TableColumn("name", "text") -> StringValue("wiringbits@wiringbits.net"), + TableColumn("phone_number", "text") -> StringValue("null") ) val primaryKeyField = "user_id" diff --git a/spra-play-server/src/test/scala/net/wiringbits/spra/admin/StringParseSpec.scala b/spra-play-server/src/test/scala/net/wiringbits/spra/admin/StringParseSpec.scala new file mode 100644 index 0000000..2f41a21 --- /dev/null +++ b/spra-play-server/src/test/scala/net/wiringbits/spra/admin/StringParseSpec.scala @@ -0,0 +1,22 @@ +package net.wiringbits.spra.admin + +import net.wiringbits.spra.admin.utils.StringParse.stringToByteArray +import org.scalatest.matchers.must.Matchers.{be, must} +import org.scalatest.wordspec.AnyWordSpec + +class StringParseSpec extends AnyWordSpec { + "dataParse" should { + val data = List( + ("[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]", Array[Byte](0, 1, 2, 3, 4, 5, 6, 7, 8, 9)), + ("[-128, -64, 0, 64, 127]", Array[Byte](-128, -64, 0, 64, 127)), + ("[10, 20, 30, 40, 50]", Array[Byte](10, 20, 30, 40, 50)), + ("[127, -127, 127, -127]", Array[Byte](127, -127, 127, -127)) + ) + + data.foreach { (data, valid) => + s"accept valid conversion: $data" in { + stringToByteArray(data) must be(valid) + } + } + } +} From ea05bf3b46928a0b8eb4ad44dcca8d39cfeafbc1 Mon Sep 17 00:00:00 2001 From: Antonio171003 Date: Thu, 5 Sep 2024 22:07:00 -0700 Subject: [PATCH 3/3] Removed extends serializable of FieldValue trait --- .../scala/net/wiringbits/spra/admin/models/FieldValue.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spra-play-server/src/main/scala/net/wiringbits/spra/admin/models/FieldValue.scala b/spra-play-server/src/main/scala/net/wiringbits/spra/admin/models/FieldValue.scala index 2729df4..4b61c0a 100644 --- a/spra-play-server/src/main/scala/net/wiringbits/spra/admin/models/FieldValue.scala +++ b/spra-play-server/src/main/scala/net/wiringbits/spra/admin/models/FieldValue.scala @@ -1,6 +1,6 @@ package net.wiringbits.spra.admin.models -sealed trait FieldValue[T] extends Serializable { +sealed trait FieldValue[T] { val value: T }