From ac813f0a6c2ec14caf7df14a5f80d0e807984f37 Mon Sep 17 00:00:00 2001 From: psainics Date: Mon, 19 May 2025 14:31:12 +0530 Subject: [PATCH 01/28] PLUGIN-1823: Retrying all SQLTransientExceptions --- .../widgets/Redshift-batchsource.json | 33 ++++++- .../widgets/Redshift-connector.json | 31 +++++++ .../widgets/AuroraMysql-action.json | 31 +++++++ .../widgets/AuroraMysql-batchsink.json | 31 +++++++ .../widgets/AuroraMysql-batchsource.json | 31 +++++++ .../widgets/AuroraMysql-postaction.json | 31 +++++++ .../widgets/AuroraPostgres-action.json | 31 +++++++ .../widgets/AuroraPostgres-batchsink.json | 31 +++++++ .../widgets/AuroraPostgres-batchsource.json | 31 +++++++ .../widgets/AuroraPostgres-postaction.json | 31 +++++++ .../widgets/CloudSQLMySQL-action.json | 31 +++++++ .../widgets/CloudSQLMySQL-batchsink.json | 31 +++++++ .../widgets/CloudSQLMySQL-batchsource.json | 31 +++++++ .../widgets/CloudSQLMySQL-connector.json | 31 +++++++ .../widgets/CloudSQLPostgreSQL-action.json | 31 +++++++ .../widgets/CloudSQLPostgreSQL-batchsink.json | 31 +++++++ .../CloudSQLPostgreSQL-batchsource.json | 31 +++++++ .../widgets/CloudSQLPostgreSQL-connector.json | 31 +++++++ database-commons/pom.xml | 6 ++ .../io/cdap/plugin/db/ConnectionConfig.java | 38 ++++++++ .../io/cdap/plugin/db/RetryExceptions.java | 40 ++++++++ .../plugin/db/action/AbstractDBAction.java | 7 +- .../db/action/AbstractDBArgumentSetter.java | 70 ++++++++------ .../plugin/db/action/AbstractQueryAction.java | 5 +- .../java/io/cdap/plugin/db/action/DBRun.java | 41 +++++--- .../config/AbstractDBSpecificSinkConfig.java | 12 +++ .../AbstractDBSpecificSourceConfig.java | 12 +++ .../db/config/DatabaseConnectionConfig.java | 6 ++ .../connector/AbstractDBConnectorConfig.java | 38 ++++++++ .../AbstractDBSpecificConnector.java | 19 ++-- .../cdap/plugin/db/sink/AbstractDBSink.java | 82 +++++++++------- .../plugin/db/source/AbstractDBSource.java | 93 +++++++++++-------- .../io/cdap/plugin/util/RetryPolicyUtil.java | 54 +++++++++++ .../cdap/plugin/db/RetryPolicyUtilTest.java | 81 ++++++++++++++++ .../db/sink/CommonFieldsValidatorTest.java | 14 ++- db2-plugin/widgets/Db2-action.json | 31 +++++++ db2-plugin/widgets/Db2-batchsink.json | 31 +++++++ db2-plugin/widgets/Db2-batchsource.json | 31 +++++++ db2-plugin/widgets/Db2-postaction.json | 31 +++++++ .../widgets/Database-action.json | 31 +++++++ .../widgets/Database-batchsink.json | 31 +++++++ .../widgets/Database-batchsource.json | 31 +++++++ .../widgets/Database-postaction.json | 31 +++++++ .../DatabaseArgumentSetter-action.json | 31 +++++++ mariadb-plugin/widgets/Mariadb-action.json | 31 +++++++ memsql-plugin/widgets/Memsql-action.json | 31 +++++++ memsql-plugin/widgets/Memsql-batchsink.json | 31 +++++++ memsql-plugin/widgets/Memsql-batchsource.json | 31 +++++++ memsql-plugin/widgets/Memsql-postaction.json | 31 +++++++ .../widgets/SQL Server-connector.json | 31 +++++++ mssql-plugin/widgets/SqlServer-action.json | 31 +++++++ mssql-plugin/widgets/SqlServer-batchsink.json | 31 +++++++ .../widgets/SqlServer-batchsource.json | 31 +++++++ .../widgets/SqlServer-postaction.json | 31 +++++++ mysql-plugin/widgets/MySQL-connector.json | 31 +++++++ mysql-plugin/widgets/Mysql-action.json | 31 +++++++ mysql-plugin/widgets/Mysql-batchsink.json | 31 +++++++ mysql-plugin/widgets/Mysql-batchsource.json | 31 +++++++ mysql-plugin/widgets/Mysql-postaction.json | 31 +++++++ netezza-plugin/widgets/Netezza-action.json | 31 +++++++ netezza-plugin/widgets/Netezza-batchsink.json | 31 +++++++ .../widgets/Netezza-batchsource.json | 31 +++++++ .../widgets/Netezza-postaction.json | 31 +++++++ oracle-plugin/widgets/Oracle-action.json | 31 +++++++ oracle-plugin/widgets/Oracle-batchsink.json | 31 +++++++ oracle-plugin/widgets/Oracle-batchsource.json | 31 +++++++ oracle-plugin/widgets/Oracle-connector.json | 31 +++++++ oracle-plugin/widgets/Oracle-postaction.json | 31 +++++++ pom.xml | 6 ++ .../widgets/PostgreSQL-connector.json | 31 +++++++ .../widgets/Postgres-action.json | 31 +++++++ .../widgets/Postgres-batchsink.json | 31 +++++++ .../widgets/Postgres-batchsource.json | 31 +++++++ .../widgets/Postgres-postaction.json | 31 +++++++ saphana-plugin/widgets/SapHana-action.json | 31 +++++++ saphana-plugin/widgets/SapHana-batchsink.json | 31 +++++++ .../widgets/SapHana-batchsource.json | 31 +++++++ .../widgets/SapHana-postaction.json | 31 +++++++ teradata-plugin/widgets/Teradata-action.json | 31 +++++++ .../widgets/Teradata-batchsink.json | 31 +++++++ .../widgets/Teradata-batchsource.json | 31 +++++++ .../widgets/Teradata-postaction.json | 31 +++++++ 82 files changed, 2478 insertions(+), 132 deletions(-) create mode 100644 database-commons/src/main/java/io/cdap/plugin/db/RetryExceptions.java create mode 100644 database-commons/src/main/java/io/cdap/plugin/util/RetryPolicyUtil.java create mode 100644 database-commons/src/test/java/io/cdap/plugin/db/RetryPolicyUtilTest.java diff --git a/amazon-redshift-plugin/widgets/Redshift-batchsource.json b/amazon-redshift-plugin/widgets/Redshift-batchsource.json index 943e2d24e..586a8993b 100644 --- a/amazon-redshift-plugin/widgets/Redshift-batchsource.json +++ b/amazon-redshift-plugin/widgets/Redshift-batchsource.json @@ -156,6 +156,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ @@ -228,7 +259,7 @@ "name": "connection" } ] - }, + } ], "jump-config": { "datasets": [ diff --git a/amazon-redshift-plugin/widgets/Redshift-connector.json b/amazon-redshift-plugin/widgets/Redshift-connector.json index 3a2af8e01..f392e3a78 100644 --- a/amazon-redshift-plugin/widgets/Redshift-connector.json +++ b/amazon-redshift-plugin/widgets/Redshift-connector.json @@ -69,6 +69,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [] diff --git a/aurora-mysql-plugin/widgets/AuroraMysql-action.json b/aurora-mysql-plugin/widgets/AuroraMysql-action.json index efc5f98ff..bd2bac558 100644 --- a/aurora-mysql-plugin/widgets/AuroraMysql-action.json +++ b/aurora-mysql-plugin/widgets/AuroraMysql-action.json @@ -90,6 +90,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/aurora-mysql-plugin/widgets/AuroraMysql-batchsink.json b/aurora-mysql-plugin/widgets/AuroraMysql-batchsink.json index a435e4e4f..6663be7ce 100644 --- a/aurora-mysql-plugin/widgets/AuroraMysql-batchsink.json +++ b/aurora-mysql-plugin/widgets/AuroraMysql-batchsink.json @@ -116,6 +116,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/aurora-mysql-plugin/widgets/AuroraMysql-batchsource.json b/aurora-mysql-plugin/widgets/AuroraMysql-batchsource.json index 50b435645..bd2bb88a9 100644 --- a/aurora-mysql-plugin/widgets/AuroraMysql-batchsource.json +++ b/aurora-mysql-plugin/widgets/AuroraMysql-batchsource.json @@ -135,6 +135,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/aurora-mysql-plugin/widgets/AuroraMysql-postaction.json b/aurora-mysql-plugin/widgets/AuroraMysql-postaction.json index cc33cf0a1..64da4f1bc 100644 --- a/aurora-mysql-plugin/widgets/AuroraMysql-postaction.json +++ b/aurora-mysql-plugin/widgets/AuroraMysql-postaction.json @@ -105,6 +105,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/aurora-postgresql-plugin/widgets/AuroraPostgres-action.json b/aurora-postgresql-plugin/widgets/AuroraPostgres-action.json index 1f3bca862..e012f65eb 100644 --- a/aurora-postgresql-plugin/widgets/AuroraPostgres-action.json +++ b/aurora-postgresql-plugin/widgets/AuroraPostgres-action.json @@ -79,6 +79,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/aurora-postgresql-plugin/widgets/AuroraPostgres-batchsink.json b/aurora-postgresql-plugin/widgets/AuroraPostgres-batchsink.json index 53979d6d4..bfc83bd4e 100644 --- a/aurora-postgresql-plugin/widgets/AuroraPostgres-batchsink.json +++ b/aurora-postgresql-plugin/widgets/AuroraPostgres-batchsink.json @@ -121,6 +121,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/aurora-postgresql-plugin/widgets/AuroraPostgres-batchsource.json b/aurora-postgresql-plugin/widgets/AuroraPostgres-batchsource.json index 14b00b974..fc2503c67 100644 --- a/aurora-postgresql-plugin/widgets/AuroraPostgres-batchsource.json +++ b/aurora-postgresql-plugin/widgets/AuroraPostgres-batchsource.json @@ -124,6 +124,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/aurora-postgresql-plugin/widgets/AuroraPostgres-postaction.json b/aurora-postgresql-plugin/widgets/AuroraPostgres-postaction.json index 3fdb1a14b..8b328160d 100644 --- a/aurora-postgresql-plugin/widgets/AuroraPostgres-postaction.json +++ b/aurora-postgresql-plugin/widgets/AuroraPostgres-postaction.json @@ -94,6 +94,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-action.json b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-action.json index 66d6ebb85..0dd6f8f41 100644 --- a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-action.json +++ b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-action.json @@ -112,6 +112,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "filters": [ diff --git a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsink.json b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsink.json index 89a7d7736..3a3277ed8 100644 --- a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsink.json +++ b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsink.json @@ -176,6 +176,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "filters": [ diff --git a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsource.json b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsource.json index 4ac7747f4..a90154670 100644 --- a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsource.json +++ b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-batchsource.json @@ -175,6 +175,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-connector.json b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-connector.json index b5c2c9993..1cebc7850 100644 --- a/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-connector.json +++ b/cloudsql-mysql-plugin/widgets/CloudSQLMySQL-connector.json @@ -94,6 +94,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-action.json b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-action.json index eab240679..e14646154 100644 --- a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-action.json +++ b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-action.json @@ -112,6 +112,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "filters": [ diff --git a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsink.json b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsink.json index 2fda594dd..8d6578413 100644 --- a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsink.json +++ b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsink.json @@ -192,6 +192,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsource.json b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsource.json index 96ea97ac2..ea449120d 100644 --- a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsource.json +++ b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-batchsource.json @@ -175,6 +175,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-connector.json b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-connector.json index 9824f91bd..36013ac40 100644 --- a/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-connector.json +++ b/cloudsql-postgresql-plugin/widgets/CloudSQLPostgreSQL-connector.json @@ -94,6 +94,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/database-commons/pom.xml b/database-commons/pom.xml index 67dc8e82e..1d49676be 100644 --- a/database-commons/pom.xml +++ b/database-commons/pom.xml @@ -41,6 +41,12 @@ guava + + + dev.failsafe + failsafe + + io.cdap.cdap diff --git a/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java index c5320e25e..f28d2c450 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java @@ -45,6 +45,12 @@ public abstract class ConnectionConfig extends PluginConfig implements DatabaseC public static final String CONNECTION_ARGUMENTS = "connectionArguments"; public static final String JDBC_PLUGIN_NAME = "jdbcPluginName"; public static final String JDBC_PLUGIN_TYPE = "jdbc"; + private static final String NAME_INITIAL_RETRY_DURATION = "initialRetryDuration"; + private static final String NAME_MAX_RETRY_DURATION = "maxRetryDuration"; + private static final String NAME_MAX_RETRY_COUNT = "maxRetryCount"; + public static final int DEFAULT_INITIAL_RETRY_DURATION_SECONDS = 5; + public static final int DEFAULT_MAX_RETRY_COUNT = 5; + public static final int DEFAULT_MAX_RETRY_DURATION_SECONDS = 80; public static final String TRANSACTION_ISOLATION_LEVEL = "transactionIsolationLevel"; @Name(JDBC_PLUGIN_NAME) @@ -72,6 +78,38 @@ public abstract class ConnectionConfig extends PluginConfig implements DatabaseC @Macro public String connectionArguments; + @Name(NAME_INITIAL_RETRY_DURATION) + @Description("Time taken for the first retry. Default is 5 seconds.") + @Nullable + @Macro + private Integer initialRetryDuration; + + @Name(NAME_MAX_RETRY_DURATION) + @Description("Maximum time in seconds retries can take. Default is 80 seconds.") + @Nullable + @Macro + private Integer maxRetryDuration; + + @Name(NAME_MAX_RETRY_COUNT) + @Description("Maximum number of retries allowed. Default is 5.") + @Nullable + @Macro + private Integer maxRetryCount; + + + public Integer getInitialRetryDuration() { + return initialRetryDuration == null ? DEFAULT_INITIAL_RETRY_DURATION_SECONDS : initialRetryDuration; + } + + public Integer getMaxRetryDuration() { + return maxRetryDuration == null ? DEFAULT_MAX_RETRY_DURATION_SECONDS : maxRetryDuration; + } + + public Integer getMaxRetryCount() { + return maxRetryCount == null ? DEFAULT_MAX_RETRY_COUNT : maxRetryCount; + } + + public ConnectionConfig() { } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/RetryExceptions.java b/database-commons/src/main/java/io/cdap/plugin/db/RetryExceptions.java new file mode 100644 index 000000000..e16531e33 --- /dev/null +++ b/database-commons/src/main/java/io/cdap/plugin/db/RetryExceptions.java @@ -0,0 +1,40 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + + +package io.cdap.plugin.db; + +import java.sql.SQLTransientException; +import java.util.HashSet; +import java.util.Set; + + +/** + * Checks whether the given exception or one of its causes is a known retryable SQLException. + */ +public class RetryExceptions { + public static boolean isRetryable(Throwable t) { + Set seen = new HashSet<>(); + while (t != null && seen.add(t)) { + if (t instanceof SQLTransientException) { + return true; + } + t = t.getCause(); + } + return false; + } +} + diff --git a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBAction.java b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBAction.java index 0eaac3148..bef93a0de 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBAction.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBAction.java @@ -16,12 +16,15 @@ package io.cdap.plugin.db.action; +import dev.failsafe.Failsafe; +import dev.failsafe.RetryPolicy; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.PipelineConfigurer; import io.cdap.cdap.etl.api.action.Action; import io.cdap.cdap.etl.api.action.ActionContext; import io.cdap.plugin.common.db.DBErrorDetailsProvider; import io.cdap.plugin.util.DBUtils; +import io.cdap.plugin.util.RetryPolicyUtil; import java.sql.Driver; import java.sql.SQLException; @@ -33,7 +36,6 @@ public abstract class AbstractDBAction extends Action { private static final String JDBC_PLUGIN_ID = "driver"; private final QueryConfig config; private final Boolean enableAutoCommit; - public AbstractDBAction(QueryConfig config, Boolean enableAutoCommit) { this.config = config; this.enableAutoCommit = enableAutoCommit; @@ -44,7 +46,8 @@ public void run(ActionContext context) throws Exception { Class driverClass = context.loadPluginClass(JDBC_PLUGIN_ID); DBRun executeQuery = new DBRun(config, driverClass, enableAutoCommit); try { - executeQuery.run(); + Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(config.getInitialRetryDuration(), + config.getMaxRetryDuration(), config.getMaxRetryCount())).run(()-> executeQuery.run()); } catch (Exception e) { if (e instanceof SQLException) { DBErrorDetailsProvider dbe = new DBErrorDetailsProvider(); diff --git a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java index 5e22abf85..540d2e7f0 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java @@ -16,6 +16,7 @@ package io.cdap.plugin.db.action; +import dev.failsafe.Failsafe; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.PipelineConfigurer; import io.cdap.cdap.etl.api.StageConfigurer; @@ -25,6 +26,7 @@ import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.util.DBUtils; import io.cdap.plugin.util.DriverCleanup; +import io.cdap.plugin.util.RetryPolicyUtil; import java.sql.Connection; import java.sql.Driver; @@ -92,50 +94,60 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) */ private void processArguments(Class driverClass, FailureCollector failureCollector, SettableArguments settableArguments) - throws SQLException, IllegalAccessException, InstantiationException { + throws SQLException, IllegalAccessException, InstantiationException { DriverCleanup driverCleanup; - driverCleanup = DBUtils.ensureJDBCDriverIsAvailable(driverClass, config.getConnectionString(), - config.getJdbcPluginName()); + config.getJdbcPluginName()); Properties connectionProperties = new Properties(); connectionProperties.putAll(config.getConnectionArguments()); try { - Connection connection = DriverManager - .getConnection(config.getConnectionString(), connectionProperties); - Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery(config.getQuery()); - boolean hasRecord = resultSet.next(); - if (!hasRecord) { - failureCollector.addFailure("No record found.", - "The argument selection conditions must match only one record."); - return; - } - if (settableArguments != null) { - setArguments(resultSet, failureCollector, settableArguments); - } - if (resultSet.next()) { - failureCollector - .addFailure("More than one records found.", - "The argument selection conditions must match only one record."); - } + executeWithRetry(failureCollector, settableArguments, connectionProperties); } finally { driverCleanup.destroy(); } } + private void executeWithRetry(FailureCollector failureCollector, SettableArguments settableArguments, + Properties connectionProperties) { + Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(config.getInitialRetryDuration(), + config.getMaxRetryDuration(), config.getMaxRetryCount())).run(() -> { + try (Connection connection = DriverManager + .getConnection(config.getConnectionString(), connectionProperties)) { + ResultSet resultSet; + try (Statement statement = connection.createStatement()) { + resultSet = statement.executeQuery(config.getQuery()); + } + boolean hasRecord = resultSet.next(); + if (!hasRecord) { + failureCollector.addFailure("No record found.", + "The argument selection conditions must match only one record."); + return; + } + if (settableArguments != null) { + setArguments(resultSet, settableArguments); + } + if (resultSet.next()) { + failureCollector + .addFailure("More than one records found.", + "The argument selection conditions must match only one record."); + } + } + }); + } + /** * Converts column from jdbc results set into pipeline arguments * * @param resultSet - result set from db {@link ResultSet} - * @param failureCollector - context failure collector @{link FailureCollector} * @param arguments - context argument setter {@link SettableArguments} - * @throws SQLException - raises {@link SQLException} when configuration is not valid */ - private void setArguments(ResultSet resultSet, FailureCollector failureCollector, - SettableArguments arguments) throws SQLException { - String[] columns = config.getArgumentsColumns().split(","); - for (String column : columns) { - arguments.set(column, resultSet.getString(column)); - } + private void setArguments(ResultSet resultSet, SettableArguments arguments) { + Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(config.getInitialRetryDuration(), + config.getMaxRetryDuration(), config.getMaxRetryCount())).run(() -> { + String[] columns = config.getArgumentsColumns().split(","); + for (String column : columns) { + arguments.set(column, resultSet.getString(column)); + } + }); } } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractQueryAction.java b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractQueryAction.java index e4b91adbd..a9ce100cf 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractQueryAction.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractQueryAction.java @@ -16,11 +16,13 @@ package io.cdap.plugin.db.action; +import dev.failsafe.Failsafe; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.PipelineConfigurer; import io.cdap.cdap.etl.api.batch.BatchActionContext; import io.cdap.cdap.etl.api.batch.PostAction; import io.cdap.plugin.util.DBUtils; +import io.cdap.plugin.util.RetryPolicyUtil; import java.sql.Driver; @@ -51,7 +53,8 @@ public void run(BatchActionContext batchContext) throws Exception { Class driverClass = batchContext.loadPluginClass(JDBC_PLUGIN_ID); DBRun executeQuery = new DBRun(config, driverClass, enableAutoCommit); - executeQuery.run(); + Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(config.getInitialRetryDuration(), + config.getMaxRetryDuration(), config.getMaxRetryCount())).run(() -> executeQuery.run()); } @Override diff --git a/database-commons/src/main/java/io/cdap/plugin/db/action/DBRun.java b/database-commons/src/main/java/io/cdap/plugin/db/action/DBRun.java index e2ccfc57e..1d99974e0 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/action/DBRun.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/action/DBRun.java @@ -16,8 +16,10 @@ package io.cdap.plugin.db.action; +import dev.failsafe.Failsafe; import io.cdap.plugin.util.DBUtils; import io.cdap.plugin.util.DriverCleanup; +import io.cdap.plugin.util.RetryPolicyUtil; import java.sql.Connection; import java.sql.Driver; @@ -48,6 +50,7 @@ public DBRun(QueryConfig config, Class driverClass, Boolean en * to use and which connection string to use come from the plugin configuration. */ public void run() throws SQLException, InstantiationException, IllegalAccessException { + DriverCleanup driverCleanup = null; try { driverCleanup = DBUtils.ensureJDBCDriverIsAvailable(driverClass, config.getConnectionString(), @@ -55,18 +58,21 @@ public void run() throws SQLException, InstantiationException, IllegalAccessExce Properties connectionProperties = new Properties(); connectionProperties.putAll(config.getConnectionArguments()); - try (Connection connection = DriverManager.getConnection(config.getConnectionString(), connectionProperties)) { - executeInitQueries(connection, config.getInitQueries()); - if (!enableAutoCommit) { - connection.setAutoCommit(false); - } - try (Statement statement = connection.createStatement()) { - statement.execute(config.query); + Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(config.getInitialRetryDuration(), + config.getMaxRetryDuration(), config.getMaxRetryCount())).run(() -> { + try (Connection connection = DriverManager.getConnection(config.getConnectionString(), connectionProperties)) { + executeInitQueries(connection, config.getInitQueries()); if (!enableAutoCommit) { - connection.commit(); + connection.setAutoCommit(false); + } + try (Statement statement = connection.createStatement()) { + statement.execute(config.query); + if (!enableAutoCommit) { + connection.commit(); + } } } - } + }); } finally { if (driverCleanup != null) { driverCleanup.destroy(); @@ -74,11 +80,16 @@ public void run() throws SQLException, InstantiationException, IllegalAccessExce } } - private void executeInitQueries(Connection connection, List initQueries) throws SQLException { - for (String query : initQueries) { - try (Statement statement = connection.createStatement()) { - statement.execute(query); - } - } + private void executeInitQueries(Connection connection, List initQueries) { + + Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(config.getInitialRetryDuration(), + config.getMaxRetryDuration(), config.getMaxRetryCount())) + .run(() -> { + for (String query : initQueries) { + try (Statement statement = connection.createStatement()) { + statement.execute(query); + } + } + }); } } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSinkConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSinkConfig.java index 5b92a85f7..3d9ed16ff 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSinkConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSinkConfig.java @@ -155,4 +155,16 @@ public Operation getOperationName() { public String getRelationTableKey() { return relationTableKey; } + + public Integer getInitialRetryDuration() { + return getConnection().getInitialRetryDuration(); + } + + public Integer getMaxRetryDuration() { + return getConnection().getMaxRetryDuration(); + } + + public Integer getMaxRetryCount() { + return getConnection().getMaxRetryCount(); + } } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSourceConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSourceConfig.java index 41c577397..f15939ab7 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSourceConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/config/AbstractDBSpecificSourceConfig.java @@ -268,4 +268,16 @@ public Integer getFetchSize() { return fetchSize; } + public Integer getInitialRetryDuration() { + return getConnection().getInitialRetryDuration(); + } + + public Integer getMaxRetryDuration() { + return getConnection().getMaxRetryDuration(); + } + + public Integer getMaxRetryCount() { + return getConnection().getMaxRetryCount(); + } + } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/config/DatabaseConnectionConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/config/DatabaseConnectionConfig.java index 55cfe363f..e1fde69a1 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/config/DatabaseConnectionConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/config/DatabaseConnectionConfig.java @@ -50,4 +50,10 @@ public interface DatabaseConnectionConfig { */ String getPassword(); + Integer getInitialRetryDuration(); + + Integer getMaxRetryDuration(); + + Integer getMaxRetryCount(); + } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBConnectorConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBConnectorConfig.java index 4bee056f8..3423440d0 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBConnectorConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBConnectorConfig.java @@ -36,6 +36,12 @@ * */ public abstract class AbstractDBConnectorConfig extends PluginConfig implements DBConnectorProperties { + private static final String NAME_INITIAL_RETRY_DURATION = "initialRetryDuration"; + private static final String NAME_MAX_RETRY_DURATION = "maxRetryDuration"; + private static final String NAME_MAX_RETRY_COUNT = "maxRetryCount"; + public static final int DEFAULT_INITIAL_RETRY_DURATION_SECONDS = 5; + public static final int DEFAULT_MAX_RETRY_COUNT = 5; + public static final int DEFAULT_MAX_RETRY_DURATION_SECONDS = 80; @Name(ConnectionConfig.JDBC_PLUGIN_NAME) @Description("Name of the JDBC driver to use. This is the value of the 'jdbcPluginName' key defined in the JSON " + @@ -63,6 +69,26 @@ public abstract class AbstractDBConnectorConfig extends PluginConfig implements @Macro protected String connectionArguments; + + @Name(NAME_INITIAL_RETRY_DURATION) + @Description("Time taken for the first retry. Default is 5 seconds.") + @Nullable + @Macro + private Integer initialRetryDuration; + + @Name(NAME_MAX_RETRY_DURATION) + @Description("Maximum time in seconds retries can take. Default is 80 seconds.") + @Nullable + @Macro + private Integer maxRetryDuration; + + @Name(NAME_MAX_RETRY_COUNT) + @Description("Maximum number of retries allowed. Default is 5.") + @Nullable + @Macro + private Integer maxRetryCount; + + @Nullable @Override public String getUser() { @@ -74,6 +100,18 @@ public String getUser() { public String getPassword() { return password; } + + public Integer getInitialRetryDuration() { + return initialRetryDuration == null ? DEFAULT_INITIAL_RETRY_DURATION_SECONDS : initialRetryDuration; + } + + public Integer getMaxRetryDuration() { + return maxRetryDuration == null ? DEFAULT_MAX_RETRY_DURATION_SECONDS : maxRetryDuration; + } + + public Integer getMaxRetryCount() { + return maxRetryCount == null ? DEFAULT_MAX_RETRY_COUNT : maxRetryCount; + } @Override public Properties getConnectionArgumentsProperties() { diff --git a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java index 8a9b7b6e4..f49cbae19 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java @@ -17,6 +17,7 @@ package io.cdap.plugin.db.connector; import com.google.common.collect.Maps; +import dev.failsafe.Failsafe; import io.cdap.cdap.api.data.batch.InputFormatProvider; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.etl.api.batch.BatchConnector; @@ -33,6 +34,7 @@ import io.cdap.plugin.db.ConnectionConfigAccessor; import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.source.DataDrivenETLDBInputFormat; +import io.cdap.plugin.util.RetryPolicyUtil; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.lib.db.DBConfiguration; @@ -172,13 +174,16 @@ protected String getStratifiedQuery(String tableName, int limit, String strata, protected Schema loadTableSchema(Connection connection, String query, @Nullable Integer timeoutSec, String sessionID) throws SQLException { - Statement statement = connection.createStatement(); - statement.setMaxRows(1); - if (timeoutSec != null) { - statement.setQueryTimeout(timeoutSec); - } - ResultSet resultSet = statement.executeQuery(query); - return Schema.recordOf("outputSchema", getSchemaReader(sessionID).getSchemaFields(resultSet)); + return Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(config.getInitialRetryDuration(), + config.getMaxRetryDuration(), config.getMaxRetryCount())).get(() -> { + Statement statement = connection.createStatement(); + statement.setMaxRows(1); + if (timeoutSec != null) { + statement.setQueryTimeout(timeoutSec); + } + ResultSet resultSet = statement.executeQuery(query); + return Schema.recordOf("outputSchema", getSchemaReader(sessionID).getSchemaFields(resultSet)); + }); } protected void setConnectionProperties(Map properties, ConnectorSpecRequest request) { diff --git a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java index 0bb4bf123..f1d5e6d4c 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java @@ -18,6 +18,8 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import dev.failsafe.Failsafe; +import dev.failsafe.FailsafeException; import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Macro; import io.cdap.cdap.api.annotation.Name; @@ -53,6 +55,7 @@ import io.cdap.plugin.db.config.DatabaseSinkConfig; import io.cdap.plugin.util.DBUtils; import io.cdap.plugin.util.DriverCleanup; +import io.cdap.plugin.util.RetryPolicyUtil; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapred.lib.db.DBConfiguration; @@ -302,6 +305,8 @@ private Schema inferSchema(Class driverClass) { dbSinkConfig.getJdbcPluginName()); Properties connectionProperties = new Properties(); connectionProperties.putAll(dbSinkConfig.getConnectionArguments()); + Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(dbSinkConfig.getInitialRetryDuration(), + dbSinkConfig.getMaxRetryDuration(), dbSinkConfig.getMaxRetryCount())).run(() -> { try (Connection connection = DriverManager.getConnection(dbSinkConfig.getConnectionString(), connectionProperties)) { executeInitQueries(connection, dbSinkConfig.getInitQueries()); @@ -330,6 +335,7 @@ private Schema inferSchema(Class driverClass) { e.getSQLState(), externalDocumentationLink, new SQLException(e.getMessage(), e.getSQLState(), e.getErrorCode())); } + }); } catch (IllegalAccessException | InstantiationException | SQLException e) { throw new InvalidStageException("JDBC Driver unavailable: " + dbSinkConfig.getJdbcPluginName(), e); } @@ -369,18 +375,20 @@ private void setResultSetMetadata() throws Exception { Properties connectionProperties = new Properties(); connectionProperties.putAll(dbSinkConfig.getConnectionArguments()); - try (Connection connection = DriverManager.getConnection(connectionString, connectionProperties)) { - executeInitQueries(connection, dbSinkConfig.getInitQueries()); - try (Statement statement = connection.createStatement(); - // Run a query against the DB table that returns 0 records, but returns valid ResultSetMetadata - // that can be used to construct DBRecord objects to sink to the database table. - ResultSet rs = statement.executeQuery(String.format("SELECT %s FROM %s WHERE 1 = 0", - dbColumns, fullyQualifiedTableName)) - ) { - columnTypes.addAll(getMatchedColumnTypeList(rs, columns)); + Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(dbSinkConfig.getInitialRetryDuration(), + dbSinkConfig.getMaxRetryDuration(), dbSinkConfig.getMaxRetryCount())).run(() -> { + try (Connection connection = DriverManager.getConnection(connectionString, connectionProperties)) { + executeInitQueries(connection, dbSinkConfig.getInitQueries()); + try (Statement statement = connection.createStatement(); + // Run a query against the DB table that returns 0 records, but returns valid ResultSetMetadata + // that can be used to construct DBRecord objects to sink to the database table. + ResultSet rs = statement.executeQuery(String.format("SELECT %s FROM %s WHERE 1 = 0", + dbColumns, fullyQualifiedTableName)) + ) { + columnTypes.addAll(getMatchedColumnTypeList(rs, columns)); + } } - } - + }); this.columnTypes = Collections.unmodifiableList(columnTypes); } @@ -438,26 +446,31 @@ private void validateSchema(FailureCollector collector, Class Properties connectionProperties = new Properties(); connectionProperties.putAll(dbSinkConfig.getConnectionArguments()); - try (Connection connection = DriverManager.getConnection(connectionString, connectionProperties)) { - executeInitQueries(connection, dbSinkConfig.getInitQueries()); - try (ResultSet tables = connection.getMetaData().getTables(null, dbSchemaName, tableName, null)) { - if (!tables.next()) { - collector.addFailure( - String.format("Table '%s' does not exist.", tableName), - String.format("Ensure table '%s' is set correctly and that the connection string '%s' " + - "points to a valid database.", fullyQualifiedTableName, connectionString)) - .withConfigProperty(DBSinkConfig.TABLE_NAME); - return; + try { + Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(dbSinkConfig.getInitialRetryDuration(), + dbSinkConfig.getMaxRetryDuration(), dbSinkConfig.getMaxRetryCount())).run(() -> { + try (Connection connection = DriverManager.getConnection(connectionString, connectionProperties)) { + executeInitQueries(connection, dbSinkConfig.getInitQueries()); + try (ResultSet tables = connection.getMetaData().getTables(null, dbSchemaName, tableName, null)) { + if (!tables.next()) { + collector.addFailure( + String.format("Table '%s' does not exist.", tableName), + String.format("Ensure table '%s' is set correctly and that the connection string '%s' " + + "points to a valid database.", fullyQualifiedTableName, connectionString)) + .withConfigProperty(DBSinkConfig.TABLE_NAME); + return; + } + } + setColumnsInfo(inputSchema.getFields()); + try (PreparedStatement pStmt = connection.prepareStatement(String.format("SELECT %s FROM %s WHERE 1 = 0", + dbColumns, + fullyQualifiedTableName)); + ResultSet rs = pStmt.executeQuery()) { + getFieldsValidator().validateFields(inputSchema, rs, collector); + } } - } - setColumnsInfo(inputSchema.getFields()); - try (PreparedStatement pStmt = connection.prepareStatement(String.format("SELECT %s FROM %s WHERE 1 = 0", - dbColumns, - fullyQualifiedTableName)); - ResultSet rs = pStmt.executeQuery()) { - getFieldsValidator().validateFields(inputSchema, rs, collector); - } - } catch (SQLException e) { + }); + } catch (FailsafeException e) { LOG.error("Exception while trying to validate schema of database table {} for connection {}.", fullyQualifiedTableName, connectionString, e); collector.addFailure( @@ -486,9 +499,12 @@ protected LineageRecorder getLineageRecorder(BatchSinkContext context) { private void executeInitQueries(Connection connection, List initQueries) throws SQLException { for (String query : initQueries) { - try (Statement statement = connection.createStatement()) { - statement.execute(query); - } + Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(dbSinkConfig.getInitialRetryDuration(), + dbSinkConfig.getMaxRetryDuration(), dbSinkConfig.getMaxRetryCount())).run(() -> { + try (Statement statement = connection.createStatement()) { + statement.execute(query); + } + }); } } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java index 54d1e2ab6..470e3c0c3 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java @@ -18,6 +18,7 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; +import dev.failsafe.Failsafe; import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Macro; import io.cdap.cdap.api.annotation.Name; @@ -52,13 +53,13 @@ import io.cdap.plugin.db.config.DatabaseSourceConfig; import io.cdap.plugin.util.DBUtils; import io.cdap.plugin.util.DriverCleanup; +import io.cdap.plugin.util.RetryPolicyUtil; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.lib.db.DBConfiguration; import org.apache.hadoop.mapreduce.lib.db.DBWritable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; - import java.io.IOException; import java.sql.Connection; import java.sql.Driver; @@ -137,7 +138,6 @@ public Schema getSchema(Class driverClass) throws IllegalAcces SQLException, InstantiationException { DriverCleanup driverCleanup; try { - driverCleanup = loadPluginClassAndGetDriver(driverClass); try { return getSchema(); @@ -168,13 +168,15 @@ public Schema getSchema() throws SQLException { } private Schema loadSchemaFromDB(Connection connection, String query) throws SQLException { - Statement statement = connection.createStatement(); - statement.setMaxRows(1); - if (query.contains("$CONDITIONS")) { - query = removeConditionsClause(query); - } - ResultSet resultSet = statement.executeQuery(query); - return Schema.recordOf("outputSchema", getSchemaReader().getSchemaFields(resultSet)); + return Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(sourceConfig.getInitialRetryDuration(), + sourceConfig.getMaxRetryDuration(), sourceConfig.getMaxRetryCount())) + .get(() -> { + Statement statement = connection.createStatement(); + statement.setMaxRows(1); + String finalQuery = query.contains("$CONDITIONS") ? removeConditionsClause(query) : query; + ResultSet resultSet = statement.executeQuery(finalQuery); + return Schema.recordOf("outputSchema", getSchemaReader().getSchemaFields(resultSet)); + }); } @VisibleForTesting @@ -191,41 +193,52 @@ private Schema loadSchemaFromDB(Class driverClass) String connectionString = sourceConfig.getConnectionString(); DriverCleanup driverCleanup = DBUtils.ensureJDBCDriverIsAvailable(driverClass, connectionString, sourceConfig.getJdbcPluginName()); - Properties connectionProperties = new Properties(); connectionProperties.putAll(sourceConfig.getConnectionArguments()); - try (Connection connection = DriverManager.getConnection(connectionString, connectionProperties)) { - executeInitQueries(connection, sourceConfig.getInitQueries()); - return loadSchemaFromDB(connection, sourceConfig.getImportQuery()); - - } catch (SQLException e) { - // wrap exception to ensure SQLException-child instances not exposed to contexts without jdbc driver in classpath - String errorMessage = - String.format("SQL Exception occurred: [Message='%s', SQLState='%s', ErrorCode='%s'].", e.getMessage(), - e.getSQLState(), e.getErrorCode()); - String errorMessageWithDetails = String.format("Error occurred while trying to get schema from database." + - "Error message: '%s'. Error code: '%s'. SQLState: '%s'", e.getMessage(), e.getErrorCode(), e.getSQLState()); - String externalDocumentationLink = getExternalDocumentationLink(); - if (!Strings.isNullOrEmpty(externalDocumentationLink)) { - if (!errorMessage.endsWith(".")) { - errorMessage = errorMessage + "."; - } - errorMessage = String.format("%s For more details, see %s", errorMessage, externalDocumentationLink); - } - throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), - errorMessage, errorMessageWithDetails, ErrorType.USER, false, ErrorCodeType.SQLSTATE, - e.getSQLState(), externalDocumentationLink, new SQLException(e.getMessage(), - e.getSQLState(), e.getErrorCode())); - } finally { - driverCleanup.destroy(); + return Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(sourceConfig.getInitialRetryDuration(), + sourceConfig.getMaxRetryDuration(), sourceConfig.getMaxRetryCount())) + .get(() -> { + try (Connection connection = DriverManager.getConnection(connectionString, connectionProperties)) { + executeInitQueries(connection, sourceConfig.getInitQueries()); + return loadSchemaFromDB(connection, sourceConfig.getImportQuery()); + } + catch (SQLException e) { + // wrap exception to ensure SQLException-child instances not exposed to contexts without jdbc + // driver in classpath + String errorMessage = + String.format("SQL Exception occurred: [Message='%s', SQLState='%s', ErrorCode='%s'].", + e.getMessage(), + e.getSQLState(), e.getErrorCode()); + String errorMessageWithDetails = String.format("Error occurred while trying to" + + " get schema from database." + + "Error message: '%s'. Error code: '%s'. SQLState: '%s'", e.getMessage(), e.getErrorCode(), + e.getSQLState()); + String externalDocumentationLink = getExternalDocumentationLink(); + if (!Strings.isNullOrEmpty(externalDocumentationLink)) { + if (!errorMessage.endsWith(".")) { + errorMessage = errorMessage + "."; + } + errorMessage = String.format("%s For more details, see %s", errorMessage, + externalDocumentationLink); + } + throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, errorMessageWithDetails, ErrorType.USER, false, ErrorCodeType.SQLSTATE, + e.getSQLState(), externalDocumentationLink, new SQLException(e.getMessage(), + e.getSQLState(), e.getErrorCode())); + } finally { + driverCleanup.destroy(); + } + }); } - } private void executeInitQueries(Connection connection, List initQueries) throws SQLException { for (String query : initQueries) { - try (Statement statement = connection.createStatement()) { - statement.execute(query); - } + Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(sourceConfig.getInitialRetryDuration(), + sourceConfig.getMaxRetryDuration(), sourceConfig.getMaxRetryCount())).run(() -> { + try (Statement statement = connection.createStatement()) { + statement.execute(query); + } + }); } } @@ -266,7 +279,9 @@ private Connection getConnection() throws SQLException { String connectionString = createConnectionString(); Properties connectionProperties = new Properties(); connectionProperties.putAll(sourceConfig.getConnectionArguments()); - return DriverManager.getConnection(connectionString, connectionProperties); + return Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(sourceConfig.getInitialRetryDuration(), + sourceConfig.getMaxRetryDuration(), sourceConfig.getMaxRetryCount())) + .get(() -> DriverManager.getConnection(connectionString, connectionProperties)); } @Override diff --git a/database-commons/src/main/java/io/cdap/plugin/util/RetryPolicyUtil.java b/database-commons/src/main/java/io/cdap/plugin/util/RetryPolicyUtil.java new file mode 100644 index 000000000..1aca23ec7 --- /dev/null +++ b/database-commons/src/main/java/io/cdap/plugin/util/RetryPolicyUtil.java @@ -0,0 +1,54 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + + +package io.cdap.plugin.util; + +import dev.failsafe.RetryPolicy; +import io.cdap.cdap.api.Config; +import io.cdap.plugin.db.RetryExceptions; +import io.cdap.plugin.db.connector.AbstractDBConnectorConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.SQLTransientConnectionException; +import java.time.Duration; + +/** + * Utility class for creating standardized {@link dev.failsafe.RetryPolicy} configurations + * to handle transient SQL exceptions using the Failsafe library. + */ +public class RetryPolicyUtil extends Config { + public static final Logger LOG = LoggerFactory.getLogger(RetryPolicyUtil.class); + + /** + * Create a RetryPolicy using custom config values. + */ + public static RetryPolicy createConnectionRetryPolicy(Integer initialRetryDuration, + Integer maxRetryDuration, Integer maxRetryCount) { + return RetryPolicy.builder() + .handleIf((failure) -> RetryExceptions.isRetryable(failure)) + .withBackoff(Duration.ofSeconds(initialRetryDuration), Duration.ofSeconds(maxRetryDuration)) + .withMaxRetries(maxRetryCount) + .onRetry(e -> LOG.debug("Retrying... Attempt {}", + e.getAttemptCount())) + .onFailedAttempt(e -> LOG.debug("Failed Attempt : {}", e.getLastException())) + .onFailure(e -> LOG.debug("Failed after retries." + + " Reason: {}", + e.getException() != null ? e.getException().getMessage() : "Unknown error")) + .build(); + } +} diff --git a/database-commons/src/test/java/io/cdap/plugin/db/RetryPolicyUtilTest.java b/database-commons/src/test/java/io/cdap/plugin/db/RetryPolicyUtilTest.java new file mode 100644 index 000000000..e85e50e8b --- /dev/null +++ b/database-commons/src/test/java/io/cdap/plugin/db/RetryPolicyUtilTest.java @@ -0,0 +1,81 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.db; + +import dev.failsafe.Failsafe; +import dev.failsafe.FailsafeException; +import dev.failsafe.RetryPolicy; +import io.cdap.plugin.db.connector.AbstractDBConnectorConfig; +import io.cdap.plugin.util.RetryPolicyUtil; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.sql.SQLSyntaxErrorException; +import java.sql.SQLTransientConnectionException; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class RetryPolicyUtilTest { + + private AbstractDBConnectorConfig mockConfig; + + @Before + public void setup() { + mockConfig = mock(AbstractDBConnectorConfig.class); + when(mockConfig.getInitialRetryDuration()).thenReturn(5); + when(mockConfig.getMaxRetryDuration()).thenReturn(10); + when(mockConfig.getMaxRetryCount()).thenReturn(2); + } + + @Test + public void testCreateConnectionRetryPolicy_Retryable() { + RetryPolicy retryPolicy = RetryPolicyUtil + .createConnectionRetryPolicy(mockConfig.getInitialRetryDuration(), mockConfig.getMaxRetryDuration(), + mockConfig.getMaxRetryCount()); + + AtomicInteger attemptCounter = new AtomicInteger(); + + FailsafeException ex = Assert.assertThrows(FailsafeException.class, () -> Failsafe.with(retryPolicy).run(() -> { + attemptCounter.incrementAndGet(); + throw new SQLTransientConnectionException("Temporary issue"); + })); + + Assert.assertTrue(ex.getCause() instanceof SQLTransientConnectionException); + Assert.assertEquals(3, attemptCounter.get()); + } + + @Test + public void testCreateConnectionRetryPolicy_NonRetryable() { + RetryPolicy retryPolicy = RetryPolicyUtil + .createConnectionRetryPolicy(mockConfig.getInitialRetryDuration(), mockConfig.getMaxRetryDuration(), + mockConfig.getMaxRetryCount()); + + AtomicInteger attemptCounter = new AtomicInteger(); + + FailsafeException ex = Assert.assertThrows(FailsafeException.class, () -> + Failsafe.with(retryPolicy).run(() -> { + attemptCounter.incrementAndGet(); + throw new SQLSyntaxErrorException("Bad SQL syntax"); + })); + Assert.assertTrue(ex.getCause() instanceof SQLSyntaxErrorException); + Assert.assertEquals(1, attemptCounter.get()); + } +} + diff --git a/database-commons/src/test/java/io/cdap/plugin/db/sink/CommonFieldsValidatorTest.java b/database-commons/src/test/java/io/cdap/plugin/db/sink/CommonFieldsValidatorTest.java index fa9e371da..23abe08a9 100644 --- a/database-commons/src/test/java/io/cdap/plugin/db/sink/CommonFieldsValidatorTest.java +++ b/database-commons/src/test/java/io/cdap/plugin/db/sink/CommonFieldsValidatorTest.java @@ -216,10 +216,14 @@ public void testValidateFieldsWithNullable() throws Exception { public void validateFieldCompatible(Schema.Type fieldType, Schema.LogicalType fieldLogicalType, int sqlType, boolean isCompatible, int precision, boolean isSigned) { String errorMessage = String.format("Expected type '%s' is %s with sql type '%d'", - fieldType, - isCompatible ? "compatible" : "not compatible", - sqlType); - Assert.assertEquals(errorMessage, isCompatible, VALIDATOR.isFieldCompatible(fieldType, fieldLogicalType, sqlType, - precision, isSigned)); + fieldType, + isCompatible ? "compatible" : "not compatible", + sqlType); + try { + boolean actualCompatible = VALIDATOR.isFieldCompatible(fieldType, fieldLogicalType, sqlType, precision, isSigned); + Assert.assertEquals(errorMessage, isCompatible, actualCompatible); + } catch (Exception e) { + throw new AssertionError("Unexpected exception during compatibility check: " + e.getMessage(), e); + } } } diff --git a/db2-plugin/widgets/Db2-action.json b/db2-plugin/widgets/Db2-action.json index 3e9159c0d..c144f8605 100644 --- a/db2-plugin/widgets/Db2-action.json +++ b/db2-plugin/widgets/Db2-action.json @@ -74,6 +74,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/db2-plugin/widgets/Db2-batchsink.json b/db2-plugin/widgets/Db2-batchsink.json index 5345f03d5..d7c97199b 100644 --- a/db2-plugin/widgets/Db2-batchsink.json +++ b/db2-plugin/widgets/Db2-batchsink.json @@ -100,6 +100,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/db2-plugin/widgets/Db2-batchsource.json b/db2-plugin/widgets/Db2-batchsource.json index 1c221606d..df8e93bdf 100644 --- a/db2-plugin/widgets/Db2-batchsource.json +++ b/db2-plugin/widgets/Db2-batchsource.json @@ -119,6 +119,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/db2-plugin/widgets/Db2-postaction.json b/db2-plugin/widgets/Db2-postaction.json index cd75dec04..1a13f10a9 100644 --- a/db2-plugin/widgets/Db2-postaction.json +++ b/db2-plugin/widgets/Db2-postaction.json @@ -89,6 +89,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/generic-database-plugin/widgets/Database-action.json b/generic-database-plugin/widgets/Database-action.json index c849c4cba..3fde94085 100644 --- a/generic-database-plugin/widgets/Database-action.json +++ b/generic-database-plugin/widgets/Database-action.json @@ -74,6 +74,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/generic-database-plugin/widgets/Database-batchsink.json b/generic-database-plugin/widgets/Database-batchsink.json index 90332c5c5..9b95ef80a 100644 --- a/generic-database-plugin/widgets/Database-batchsink.json +++ b/generic-database-plugin/widgets/Database-batchsink.json @@ -115,6 +115,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/generic-database-plugin/widgets/Database-batchsource.json b/generic-database-plugin/widgets/Database-batchsource.json index 579b87bd9..bf363027d 100644 --- a/generic-database-plugin/widgets/Database-batchsource.json +++ b/generic-database-plugin/widgets/Database-batchsource.json @@ -134,6 +134,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/generic-database-plugin/widgets/Database-postaction.json b/generic-database-plugin/widgets/Database-postaction.json index abf02ef22..2d07b6055 100644 --- a/generic-database-plugin/widgets/Database-postaction.json +++ b/generic-database-plugin/widgets/Database-postaction.json @@ -89,6 +89,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/generic-db-argument-setter/widgets/DatabaseArgumentSetter-action.json b/generic-db-argument-setter/widgets/DatabaseArgumentSetter-action.json index 423792391..49d3ddb91 100644 --- a/generic-db-argument-setter/widgets/DatabaseArgumentSetter-action.json +++ b/generic-db-argument-setter/widgets/DatabaseArgumentSetter-action.json @@ -93,6 +93,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/mariadb-plugin/widgets/Mariadb-action.json b/mariadb-plugin/widgets/Mariadb-action.json index bb78abb27..7588b6016 100644 --- a/mariadb-plugin/widgets/Mariadb-action.json +++ b/mariadb-plugin/widgets/Mariadb-action.json @@ -156,6 +156,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/memsql-plugin/widgets/Memsql-action.json b/memsql-plugin/widgets/Memsql-action.json index 61cbb1e47..67888e80f 100644 --- a/memsql-plugin/widgets/Memsql-action.json +++ b/memsql-plugin/widgets/Memsql-action.json @@ -156,6 +156,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/memsql-plugin/widgets/Memsql-batchsink.json b/memsql-plugin/widgets/Memsql-batchsink.json index 98c2c1f8e..7f060289c 100644 --- a/memsql-plugin/widgets/Memsql-batchsink.json +++ b/memsql-plugin/widgets/Memsql-batchsink.json @@ -170,6 +170,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/memsql-plugin/widgets/Memsql-batchsource.json b/memsql-plugin/widgets/Memsql-batchsource.json index ef5d17b39..81644861b 100644 --- a/memsql-plugin/widgets/Memsql-batchsource.json +++ b/memsql-plugin/widgets/Memsql-batchsource.json @@ -205,6 +205,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/memsql-plugin/widgets/Memsql-postaction.json b/memsql-plugin/widgets/Memsql-postaction.json index 72e67abd9..661b946c8 100644 --- a/memsql-plugin/widgets/Memsql-postaction.json +++ b/memsql-plugin/widgets/Memsql-postaction.json @@ -172,6 +172,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/mssql-plugin/widgets/SQL Server-connector.json b/mssql-plugin/widgets/SQL Server-connector.json index c326cd81d..198ac1fd4 100644 --- a/mssql-plugin/widgets/SQL Server-connector.json +++ b/mssql-plugin/widgets/SQL Server-connector.json @@ -97,6 +97,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [] diff --git a/mssql-plugin/widgets/SqlServer-action.json b/mssql-plugin/widgets/SqlServer-action.json index 303944d71..511793545 100644 --- a/mssql-plugin/widgets/SqlServer-action.json +++ b/mssql-plugin/widgets/SqlServer-action.json @@ -204,6 +204,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/mssql-plugin/widgets/SqlServer-batchsink.json b/mssql-plugin/widgets/SqlServer-batchsink.json index fb20cad9d..3f3ad5887 100644 --- a/mssql-plugin/widgets/SqlServer-batchsink.json +++ b/mssql-plugin/widgets/SqlServer-batchsink.json @@ -257,6 +257,37 @@ "name": "currentLanguage" } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/mssql-plugin/widgets/SqlServer-batchsource.json b/mssql-plugin/widgets/SqlServer-batchsource.json index b3494e485..d35db27ae 100644 --- a/mssql-plugin/widgets/SqlServer-batchsource.json +++ b/mssql-plugin/widgets/SqlServer-batchsource.json @@ -276,6 +276,37 @@ "name": "currentLanguage" } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/mssql-plugin/widgets/SqlServer-postaction.json b/mssql-plugin/widgets/SqlServer-postaction.json index 5cec14b89..5f191000e 100644 --- a/mssql-plugin/widgets/SqlServer-postaction.json +++ b/mssql-plugin/widgets/SqlServer-postaction.json @@ -219,6 +219,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/mysql-plugin/widgets/MySQL-connector.json b/mysql-plugin/widgets/MySQL-connector.json index f60f5526f..52982aa32 100644 --- a/mysql-plugin/widgets/MySQL-connector.json +++ b/mysql-plugin/widgets/MySQL-connector.json @@ -78,6 +78,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [] diff --git a/mysql-plugin/widgets/Mysql-action.json b/mysql-plugin/widgets/Mysql-action.json index ae5d0b555..45cbed4ba 100644 --- a/mysql-plugin/widgets/Mysql-action.json +++ b/mysql-plugin/widgets/Mysql-action.json @@ -161,6 +161,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/mysql-plugin/widgets/Mysql-batchsink.json b/mysql-plugin/widgets/Mysql-batchsink.json index 58596aae2..ca23f71c8 100644 --- a/mysql-plugin/widgets/Mysql-batchsink.json +++ b/mysql-plugin/widgets/Mysql-batchsink.json @@ -217,6 +217,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/mysql-plugin/widgets/Mysql-batchsource.json b/mysql-plugin/widgets/Mysql-batchsource.json index 506e837f7..ce6cd047e 100644 --- a/mysql-plugin/widgets/Mysql-batchsource.json +++ b/mysql-plugin/widgets/Mysql-batchsource.json @@ -252,6 +252,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/mysql-plugin/widgets/Mysql-postaction.json b/mysql-plugin/widgets/Mysql-postaction.json index e34a40928..498f6f064 100644 --- a/mysql-plugin/widgets/Mysql-postaction.json +++ b/mysql-plugin/widgets/Mysql-postaction.json @@ -176,6 +176,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/netezza-plugin/widgets/Netezza-action.json b/netezza-plugin/widgets/Netezza-action.json index de523e860..e84555c95 100644 --- a/netezza-plugin/widgets/Netezza-action.json +++ b/netezza-plugin/widgets/Netezza-action.json @@ -74,6 +74,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/netezza-plugin/widgets/Netezza-batchsink.json b/netezza-plugin/widgets/Netezza-batchsink.json index c8634a58b..73884e782 100644 --- a/netezza-plugin/widgets/Netezza-batchsink.json +++ b/netezza-plugin/widgets/Netezza-batchsink.json @@ -95,6 +95,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/netezza-plugin/widgets/Netezza-batchsource.json b/netezza-plugin/widgets/Netezza-batchsource.json index c1e0f26c3..196546791 100644 --- a/netezza-plugin/widgets/Netezza-batchsource.json +++ b/netezza-plugin/widgets/Netezza-batchsource.json @@ -110,6 +110,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/netezza-plugin/widgets/Netezza-postaction.json b/netezza-plugin/widgets/Netezza-postaction.json index 85ea43ad9..5de3780d3 100644 --- a/netezza-plugin/widgets/Netezza-postaction.json +++ b/netezza-plugin/widgets/Netezza-postaction.json @@ -89,6 +89,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/oracle-plugin/widgets/Oracle-action.json b/oracle-plugin/widgets/Oracle-action.json index 815aa4b0f..15d4f81c3 100644 --- a/oracle-plugin/widgets/Oracle-action.json +++ b/oracle-plugin/widgets/Oracle-action.json @@ -107,6 +107,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/oracle-plugin/widgets/Oracle-batchsink.json b/oracle-plugin/widgets/Oracle-batchsink.json index 8d6168780..8d8fc79a2 100644 --- a/oracle-plugin/widgets/Oracle-batchsink.json +++ b/oracle-plugin/widgets/Oracle-batchsink.json @@ -220,6 +220,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/oracle-plugin/widgets/Oracle-batchsource.json b/oracle-plugin/widgets/Oracle-batchsource.json index 5eca20cc4..5adf07834 100644 --- a/oracle-plugin/widgets/Oracle-batchsource.json +++ b/oracle-plugin/widgets/Oracle-batchsource.json @@ -248,6 +248,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/oracle-plugin/widgets/Oracle-connector.json b/oracle-plugin/widgets/Oracle-connector.json index 628027caf..a72ecd203 100644 --- a/oracle-plugin/widgets/Oracle-connector.json +++ b/oracle-plugin/widgets/Oracle-connector.json @@ -148,6 +148,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "filters" : [ diff --git a/oracle-plugin/widgets/Oracle-postaction.json b/oracle-plugin/widgets/Oracle-postaction.json index 9a18077c4..3e913ae9b 100644 --- a/oracle-plugin/widgets/Oracle-postaction.json +++ b/oracle-plugin/widgets/Oracle-postaction.json @@ -122,6 +122,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/pom.xml b/pom.xml index 384a50374..c496330fd 100644 --- a/pom.xml +++ b/pom.xml @@ -298,6 +298,12 @@ ${junit.version} test + + + dev.failsafe + failsafe + 3.3.2 + org.hsqldb hsqldb diff --git a/postgresql-plugin/widgets/PostgreSQL-connector.json b/postgresql-plugin/widgets/PostgreSQL-connector.json index 9a7a02e14..88a1714fa 100644 --- a/postgresql-plugin/widgets/PostgreSQL-connector.json +++ b/postgresql-plugin/widgets/PostgreSQL-connector.json @@ -82,6 +82,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [] diff --git a/postgresql-plugin/widgets/Postgres-action.json b/postgresql-plugin/widgets/Postgres-action.json index 351c023f1..afed87295 100644 --- a/postgresql-plugin/widgets/Postgres-action.json +++ b/postgresql-plugin/widgets/Postgres-action.json @@ -82,6 +82,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/postgresql-plugin/widgets/Postgres-batchsink.json b/postgresql-plugin/widgets/Postgres-batchsink.json index 14e6f8154..f58cb1995 100644 --- a/postgresql-plugin/widgets/Postgres-batchsink.json +++ b/postgresql-plugin/widgets/Postgres-batchsink.json @@ -169,6 +169,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/postgresql-plugin/widgets/Postgres-batchsource.json b/postgresql-plugin/widgets/Postgres-batchsource.json index 60de4725f..6d0656e39 100644 --- a/postgresql-plugin/widgets/Postgres-batchsource.json +++ b/postgresql-plugin/widgets/Postgres-batchsource.json @@ -172,6 +172,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/postgresql-plugin/widgets/Postgres-postaction.json b/postgresql-plugin/widgets/Postgres-postaction.json index 5a0daf595..6b3ebe1f3 100644 --- a/postgresql-plugin/widgets/Postgres-postaction.json +++ b/postgresql-plugin/widgets/Postgres-postaction.json @@ -97,6 +97,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/saphana-plugin/widgets/SapHana-action.json b/saphana-plugin/widgets/SapHana-action.json index 7e60ac35d..5fb59faab 100644 --- a/saphana-plugin/widgets/SapHana-action.json +++ b/saphana-plugin/widgets/SapHana-action.json @@ -82,6 +82,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/saphana-plugin/widgets/SapHana-batchsink.json b/saphana-plugin/widgets/SapHana-batchsink.json index a9d8c6343..56958358f 100644 --- a/saphana-plugin/widgets/SapHana-batchsink.json +++ b/saphana-plugin/widgets/SapHana-batchsink.json @@ -103,6 +103,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/saphana-plugin/widgets/SapHana-batchsource.json b/saphana-plugin/widgets/SapHana-batchsource.json index 9352b02f7..7df341a7e 100644 --- a/saphana-plugin/widgets/SapHana-batchsource.json +++ b/saphana-plugin/widgets/SapHana-batchsource.json @@ -127,6 +127,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/saphana-plugin/widgets/SapHana-postaction.json b/saphana-plugin/widgets/SapHana-postaction.json index ad2c8b938..e260ebf0e 100644 --- a/saphana-plugin/widgets/SapHana-postaction.json +++ b/saphana-plugin/widgets/SapHana-postaction.json @@ -97,6 +97,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/teradata-plugin/widgets/Teradata-action.json b/teradata-plugin/widgets/Teradata-action.json index 2ffba361c..0662ed778 100644 --- a/teradata-plugin/widgets/Teradata-action.json +++ b/teradata-plugin/widgets/Teradata-action.json @@ -74,6 +74,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } diff --git a/teradata-plugin/widgets/Teradata-batchsink.json b/teradata-plugin/widgets/Teradata-batchsink.json index f455991d4..861bbbaa2 100644 --- a/teradata-plugin/widgets/Teradata-batchsink.json +++ b/teradata-plugin/widgets/Teradata-batchsink.json @@ -95,6 +95,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [], diff --git a/teradata-plugin/widgets/Teradata-batchsource.json b/teradata-plugin/widgets/Teradata-batchsource.json index 94f5314e5..2d10020a5 100644 --- a/teradata-plugin/widgets/Teradata-batchsource.json +++ b/teradata-plugin/widgets/Teradata-batchsource.json @@ -115,6 +115,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ], "outputs": [ diff --git a/teradata-plugin/widgets/Teradata-postaction.json b/teradata-plugin/widgets/Teradata-postaction.json index 35ead0013..deeccbb69 100644 --- a/teradata-plugin/widgets/Teradata-postaction.json +++ b/teradata-plugin/widgets/Teradata-postaction.json @@ -90,6 +90,37 @@ } } ] + }, + { + "properties": [ + { + "widget-type": "hidden", + "label": "Initial Retry Duration (sec)", + "name": "initialRetryDuration", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Duration (sec)", + "name": "maxRetryDuration", + "widget-attributes": { + "default": 80, + "minimum": 0 + } + }, + { + "widget-type": "hidden", + "label": "Maximum Retry Count", + "name": "maxRetryCount", + "widget-attributes": { + "default": 5, + "minimum": 0 + } + } + ] } ] } From 8b1f12c55417ce9a77f92a49545222650df072c7 Mon Sep 17 00:00:00 2001 From: Sanchit Garg Date: Fri, 30 May 2025 21:47:28 +0530 Subject: [PATCH 02/28] PLUGIN-1823: Rework --- .../plugin/db/action/AbstractDBAction.java | 7 +- .../db/action/AbstractDBArgumentSetter.java | 78 +++++---- .../plugin/db/action/AbstractQueryAction.java | 5 +- .../java/io/cdap/plugin/db/action/DBRun.java | 51 +++--- .../AbstractDBSpecificConnector.java | 7 +- .../cdap/plugin/db/sink/AbstractDBSink.java | 155 ++++++++++-------- .../plugin/db/source/AbstractDBSource.java | 117 +++++++------ .../io/cdap/plugin/util/RetryPolicyUtil.java | 2 +- .../cdap/plugin/db/RetryPolicyUtilTest.java | 8 +- 9 files changed, 236 insertions(+), 194 deletions(-) diff --git a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBAction.java b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBAction.java index bef93a0de..0eaac3148 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBAction.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBAction.java @@ -16,15 +16,12 @@ package io.cdap.plugin.db.action; -import dev.failsafe.Failsafe; -import dev.failsafe.RetryPolicy; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.PipelineConfigurer; import io.cdap.cdap.etl.api.action.Action; import io.cdap.cdap.etl.api.action.ActionContext; import io.cdap.plugin.common.db.DBErrorDetailsProvider; import io.cdap.plugin.util.DBUtils; -import io.cdap.plugin.util.RetryPolicyUtil; import java.sql.Driver; import java.sql.SQLException; @@ -36,6 +33,7 @@ public abstract class AbstractDBAction extends Action { private static final String JDBC_PLUGIN_ID = "driver"; private final QueryConfig config; private final Boolean enableAutoCommit; + public AbstractDBAction(QueryConfig config, Boolean enableAutoCommit) { this.config = config; this.enableAutoCommit = enableAutoCommit; @@ -46,8 +44,7 @@ public void run(ActionContext context) throws Exception { Class driverClass = context.loadPluginClass(JDBC_PLUGIN_ID); DBRun executeQuery = new DBRun(config, driverClass, enableAutoCommit); try { - Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(config.getInitialRetryDuration(), - config.getMaxRetryDuration(), config.getMaxRetryCount())).run(()-> executeQuery.run()); + executeQuery.run(); } catch (Exception e) { if (e instanceof SQLException) { DBErrorDetailsProvider dbe = new DBErrorDetailsProvider(); diff --git a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java index 540d2e7f0..d26cdc2d0 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java @@ -17,6 +17,7 @@ package io.cdap.plugin.db.action; import dev.failsafe.Failsafe; +import dev.failsafe.RetryPolicy; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.PipelineConfigurer; import io.cdap.cdap.etl.api.StageConfigurer; @@ -43,9 +44,12 @@ public class AbstractDBArgumentSetter extends Action { private static final String JDBC_PLUGIN_ID = "driver"; private final ArgumentSetterConfig config; + private final RetryPolicy retryPolicy; public AbstractDBArgumentSetter(ArgumentSetterConfig config) { this.config = config; + this.retryPolicy = RetryPolicyUtil.getRetryPolicy(config.getInitialRetryDuration(), config.getMaxRetryDuration(), + config.getMaxRetryCount()); } @Override @@ -94,10 +98,11 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) */ private void processArguments(Class driverClass, FailureCollector failureCollector, SettableArguments settableArguments) - throws SQLException, IllegalAccessException, InstantiationException { + throws SQLException, IllegalAccessException, InstantiationException { DriverCleanup driverCleanup; + driverCleanup = DBUtils.ensureJDBCDriverIsAvailable(driverClass, config.getConnectionString(), - config.getJdbcPluginName()); + config.getJdbcPluginName()); Properties connectionProperties = new Properties(); connectionProperties.putAll(config.getConnectionArguments()); try { @@ -108,46 +113,51 @@ private void processArguments(Class driverClass, } private void executeWithRetry(FailureCollector failureCollector, SettableArguments settableArguments, - Properties connectionProperties) { - Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(config.getInitialRetryDuration(), - config.getMaxRetryDuration(), config.getMaxRetryCount())).run(() -> { - try (Connection connection = DriverManager - .getConnection(config.getConnectionString(), connectionProperties)) { - ResultSet resultSet; - try (Statement statement = connection.createStatement()) { - resultSet = statement.executeQuery(config.getQuery()); - } - boolean hasRecord = resultSet.next(); - if (!hasRecord) { - failureCollector.addFailure("No record found.", - "The argument selection conditions must match only one record."); - return; - } - if (settableArguments != null) { - setArguments(resultSet, settableArguments); - } - if (resultSet.next()) { - failureCollector - .addFailure("More than one records found.", - "The argument selection conditions must match only one record."); - } + Properties connectionProperties) throws SQLException { + try (Connection connection = createConnectionWithRetry(connectionProperties)) { + ResultSet resultSet; + try (Statement statement = createStatementWithRetry(connection)) { + resultSet = Failsafe.with(retryPolicy).get(() -> statement.executeQuery(config.getQuery())); + } + boolean hasRecord = resultSet.next(); + if (!hasRecord) { + failureCollector.addFailure("No record found.", + "The argument selection conditions must match only one record."); + return; + } + if (settableArguments != null) { + setArguments(resultSet, failureCollector, settableArguments); } - }); + if (resultSet.next()) { + failureCollector + .addFailure("More than one records found.", + "The argument selection conditions must match only one record."); + } + } + } + + private Connection createConnectionWithRetry(Properties connectionProperties) { + return Failsafe.with(retryPolicy).get(() -> DriverManager + .getConnection(config.getConnectionString(), connectionProperties)); + } + + private Statement createStatementWithRetry(Connection connection) { + return Failsafe.with(retryPolicy).get(() -> connection.createStatement()); } /** * Converts column from jdbc results set into pipeline arguments * * @param resultSet - result set from db {@link ResultSet} + * @param failureCollector - context failure collector @{link FailureCollector} * @param arguments - context argument setter {@link SettableArguments} + * @throws SQLException - raises {@link SQLException} when configuration is not valid */ - private void setArguments(ResultSet resultSet, SettableArguments arguments) { - Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(config.getInitialRetryDuration(), - config.getMaxRetryDuration(), config.getMaxRetryCount())).run(() -> { - String[] columns = config.getArgumentsColumns().split(","); - for (String column : columns) { - arguments.set(column, resultSet.getString(column)); - } - }); + private void setArguments(ResultSet resultSet, FailureCollector failureCollector, + SettableArguments arguments) throws SQLException { + String[] columns = config.getArgumentsColumns().split(","); + for (String column : columns) { + arguments.set(column, resultSet.getString(column)); + } } } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractQueryAction.java b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractQueryAction.java index a9ce100cf..e4b91adbd 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractQueryAction.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractQueryAction.java @@ -16,13 +16,11 @@ package io.cdap.plugin.db.action; -import dev.failsafe.Failsafe; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.PipelineConfigurer; import io.cdap.cdap.etl.api.batch.BatchActionContext; import io.cdap.cdap.etl.api.batch.PostAction; import io.cdap.plugin.util.DBUtils; -import io.cdap.plugin.util.RetryPolicyUtil; import java.sql.Driver; @@ -53,8 +51,7 @@ public void run(BatchActionContext batchContext) throws Exception { Class driverClass = batchContext.loadPluginClass(JDBC_PLUGIN_ID); DBRun executeQuery = new DBRun(config, driverClass, enableAutoCommit); - Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(config.getInitialRetryDuration(), - config.getMaxRetryDuration(), config.getMaxRetryCount())).run(() -> executeQuery.run()); + executeQuery.run(); } @Override diff --git a/database-commons/src/main/java/io/cdap/plugin/db/action/DBRun.java b/database-commons/src/main/java/io/cdap/plugin/db/action/DBRun.java index 1d99974e0..8f2cb0e2f 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/action/DBRun.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/action/DBRun.java @@ -17,6 +17,7 @@ package io.cdap.plugin.db.action; import dev.failsafe.Failsafe; +import dev.failsafe.RetryPolicy; import io.cdap.plugin.util.DBUtils; import io.cdap.plugin.util.DriverCleanup; import io.cdap.plugin.util.RetryPolicyUtil; @@ -36,6 +37,7 @@ public class DBRun { private final QueryConfig config; private final Class driverClass; private boolean enableAutoCommit; + private final RetryPolicy retryPolicy; public DBRun(QueryConfig config, Class driverClass, Boolean enableAutocommit) { this.config = config; @@ -43,6 +45,8 @@ public DBRun(QueryConfig config, Class driverClass, Boolean en if (enableAutocommit != null) { this.enableAutoCommit = enableAutocommit; } + this.retryPolicy = RetryPolicyUtil.getRetryPolicy(config.getInitialRetryDuration(), config.getMaxRetryDuration(), + config.getMaxRetryCount()); } /** @@ -50,7 +54,6 @@ public DBRun(QueryConfig config, Class driverClass, Boolean en * to use and which connection string to use come from the plugin configuration. */ public void run() throws SQLException, InstantiationException, IllegalAccessException { - DriverCleanup driverCleanup = null; try { driverCleanup = DBUtils.ensureJDBCDriverIsAvailable(driverClass, config.getConnectionString(), @@ -58,21 +61,19 @@ public void run() throws SQLException, InstantiationException, IllegalAccessExce Properties connectionProperties = new Properties(); connectionProperties.putAll(config.getConnectionArguments()); - Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(config.getInitialRetryDuration(), - config.getMaxRetryDuration(), config.getMaxRetryCount())).run(() -> { - try (Connection connection = DriverManager.getConnection(config.getConnectionString(), connectionProperties)) { - executeInitQueries(connection, config.getInitQueries()); + try (Connection connection = createConnectionWithRetry(connectionProperties)) { + executeInitQueries(connection, config.getInitQueries()); + if (!enableAutoCommit) { + connection.setAutoCommit(false); + } + try (Statement statement = createStatementWithRetry(connection)) { + Failsafe.with(retryPolicy).run(() -> statement.execute(config.query)); + if (!enableAutoCommit) { - connection.setAutoCommit(false); - } - try (Statement statement = connection.createStatement()) { - statement.execute(config.query); - if (!enableAutoCommit) { - connection.commit(); - } + connection.commit(); } } - }); + } } finally { if (driverCleanup != null) { driverCleanup.destroy(); @@ -80,16 +81,20 @@ public void run() throws SQLException, InstantiationException, IllegalAccessExce } } - private void executeInitQueries(Connection connection, List initQueries) { + private void executeInitQueries(Connection connection, List initQueries) throws SQLException { + for (String query : initQueries) { + try (Statement statement = createStatementWithRetry(connection)) { + Failsafe.with(retryPolicy).run(() -> statement.execute(query)); + } + } + } - Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(config.getInitialRetryDuration(), - config.getMaxRetryDuration(), config.getMaxRetryCount())) - .run(() -> { - for (String query : initQueries) { - try (Statement statement = connection.createStatement()) { - statement.execute(query); - } - } - }); + private Connection createConnectionWithRetry(Properties connectionProperties) { + return Failsafe.with(retryPolicy).get(() -> DriverManager + .getConnection(config.getConnectionString(), connectionProperties)); + } + + private Statement createStatementWithRetry(Connection connection) { + return Failsafe.with(retryPolicy).get(() -> connection.createStatement()); } } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java index f49cbae19..caeeac568 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java @@ -174,16 +174,15 @@ protected String getStratifiedQuery(String tableName, int limit, String strata, protected Schema loadTableSchema(Connection connection, String query, @Nullable Integer timeoutSec, String sessionID) throws SQLException { - return Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(config.getInitialRetryDuration(), - config.getMaxRetryDuration(), config.getMaxRetryCount())).get(() -> { Statement statement = connection.createStatement(); statement.setMaxRows(1); if (timeoutSec != null) { statement.setQueryTimeout(timeoutSec); } - ResultSet resultSet = statement.executeQuery(query); + ResultSet resultSet = Failsafe.with(RetryPolicyUtil.getRetryPolicy(config.getInitialRetryDuration(), + config.getMaxRetryDuration(), config.getMaxRetryCount())) + .get(() -> statement.executeQuery(query)); return Schema.recordOf("outputSchema", getSchemaReader(sessionID).getSchemaFields(resultSet)); - }); } protected void setConnectionProperties(Map properties, ConnectorSpecRequest request) { diff --git a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java index f1d5e6d4c..543110237 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java @@ -20,6 +20,7 @@ import com.google.common.base.Strings; import dev.failsafe.Failsafe; import dev.failsafe.FailsafeException; +import dev.failsafe.RetryPolicy; import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Macro; import io.cdap.cdap.api.annotation.Name; @@ -31,6 +32,7 @@ import io.cdap.cdap.api.exception.ErrorCodeType; import io.cdap.cdap.api.exception.ErrorType; import io.cdap.cdap.api.exception.ErrorUtils; +import io.cdap.cdap.api.exception.ProgramFailureException; import io.cdap.cdap.api.plugin.PluginConfig; import io.cdap.cdap.etl.api.Emitter; import io.cdap.cdap.etl.api.FailureCollector; @@ -101,12 +103,15 @@ public abstract class AbstractDBSink retryPolicy; public AbstractDBSink(T dbSinkConfig) { super(new ReferencePluginConfig(dbSinkConfig.getReferenceName())); this.dbSinkConfig = dbSinkConfig; this.configAccessor = new ConnectionConfigAccessor(); this.configuration = configAccessor.getConfiguration(); + this.retryPolicy = RetryPolicyUtil.getRetryPolicy(dbSinkConfig.getInitialRetryDuration(), + dbSinkConfig.getMaxRetryDuration(), dbSinkConfig.getMaxRetryCount()); } private String getJDBCPluginId() { @@ -305,43 +310,60 @@ private Schema inferSchema(Class driverClass) { dbSinkConfig.getJdbcPluginName()); Properties connectionProperties = new Properties(); connectionProperties.putAll(dbSinkConfig.getConnectionArguments()); - Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(dbSinkConfig.getInitialRetryDuration(), - dbSinkConfig.getMaxRetryDuration(), dbSinkConfig.getMaxRetryCount())).run(() -> { - try (Connection connection = DriverManager.getConnection(dbSinkConfig.getConnectionString(), - connectionProperties)) { + try (Connection connection = createConnectionWithRetry(connectionProperties)) { executeInitQueries(connection, dbSinkConfig.getInitQueries()); - - try (Statement statement = connection.createStatement(); - ResultSet rs = statement.executeQuery("SELECT * FROM " + fullyQualifiedTableName - + " WHERE 1 = 0")) { + try (Statement statement = createStatementWithRetry(connection); + ResultSet rs = Failsafe.with(retryPolicy).get(() -> statement + .executeQuery("SELECT * FROM " + fullyQualifiedTableName + " WHERE 1 = 0"))) { inferredFields.addAll(getSchemaReader().getSchemaFields(rs)); } - } catch (SQLException e) { - // wrap exception to ensure SQLException-child instances not exposed to contexts w/o jdbc driver in classpath - String errorMessage = - String.format("SQL Exception occurred: [Message='%s', SQLState='%s', ErrorCode='%s'].", e.getMessage(), - e.getSQLState(), e.getErrorCode()); - String errorMessageWithDetails = String.format("Error while reading table metadata." + - "Error message: '%s'. Error code: '%s'. SQLState: '%s'", e.getMessage(), e.getErrorCode(), e.getSQLState()); - String externalDocumentationLink = getExternalDocumentationLink(); - if (!Strings.isNullOrEmpty(externalDocumentationLink)) { - if (!errorMessage.endsWith(".")) { - errorMessage = errorMessage + "."; - } - errorMessage = String.format("%s For more details, see %s", errorMessageWithDetails, errorMessage); + } catch (FailsafeException e) { + Throwable cause = e.getCause(); + if (cause instanceof SQLException) { + throw wrapException((SQLException) cause); } - throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), - errorMessage, errorMessageWithDetails, ErrorType.USER, false, ErrorCodeType.SQLSTATE, - e.getSQLState(), externalDocumentationLink, new SQLException(e.getMessage(), - e.getSQLState(), e.getErrorCode())); + throw e; + } catch (SQLException e) { + throw wrapException(e); } - }); } catch (IllegalAccessException | InstantiationException | SQLException e) { throw new InvalidStageException("JDBC Driver unavailable: " + dbSinkConfig.getJdbcPluginName(), e); } return Schema.recordOf("inferredSchema", inferredFields); } + private ProgramFailureException wrapException(SQLException e) { + // wrap exception to ensure SQLException-child instances not exposed to contexts w/o jdbc driver in classpath + String errorMessage = + String.format("SQL Exception occurred: [Message='%s', SQLState='%s', ErrorCode='%s'].", e.getMessage(), + e.getSQLState(), e.getErrorCode()); + String errorMessageWithDetails = String.format("Error while reading table metadata." + + "Error message: '%s'. Error code: '%s'. SQLState: '%s'", e.getMessage(), e.getErrorCode(), e.getSQLState()); + String externalDocumentationLink = getExternalDocumentationLink(); + if (!Strings.isNullOrEmpty(externalDocumentationLink)) { + if (!errorMessage.endsWith(".")) { + errorMessage = errorMessage + "."; + } + errorMessage = String.format("%s For more details, see %s", errorMessageWithDetails, errorMessage); + } + return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, errorMessageWithDetails, ErrorType.USER, false, ErrorCodeType.SQLSTATE, e.getSQLState(), + externalDocumentationLink, new SQLException(e.getMessage(), e.getSQLState(), e.getErrorCode())); + } + + private Connection createConnectionWithRetry(Properties connectionProperties) { + return Failsafe.with(retryPolicy).get(() -> DriverManager + .getConnection(dbSinkConfig.getConnectionString(), connectionProperties)); + } + + private Statement createStatementWithRetry(Connection connection) { + return Failsafe.with(retryPolicy).get(() -> connection.createStatement()); + } + + private PreparedStatement prepareStatementWithRetry(Connection connection, String sqlQuery) { + return Failsafe.with(retryPolicy).get(() -> connection.prepareStatement(sqlQuery)); + } + @Override public void transform(StructuredRecord input, Emitter> emitter) { emitter.emit(new KeyValue<>(getDBRecord(input), null)); @@ -375,20 +397,17 @@ private void setResultSetMetadata() throws Exception { Properties connectionProperties = new Properties(); connectionProperties.putAll(dbSinkConfig.getConnectionArguments()); - Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(dbSinkConfig.getInitialRetryDuration(), - dbSinkConfig.getMaxRetryDuration(), dbSinkConfig.getMaxRetryCount())).run(() -> { - try (Connection connection = DriverManager.getConnection(connectionString, connectionProperties)) { - executeInitQueries(connection, dbSinkConfig.getInitQueries()); - try (Statement statement = connection.createStatement(); - // Run a query against the DB table that returns 0 records, but returns valid ResultSetMetadata - // that can be used to construct DBRecord objects to sink to the database table. - ResultSet rs = statement.executeQuery(String.format("SELECT %s FROM %s WHERE 1 = 0", - dbColumns, fullyQualifiedTableName)) - ) { - columnTypes.addAll(getMatchedColumnTypeList(rs, columns)); - } + try (Connection connection = createConnectionWithRetry(connectionProperties)) { + executeInitQueries(connection, dbSinkConfig.getInitQueries()); + try (Statement statement = createStatementWithRetry(connection); + // Run a query against the DB table that returns 0 records, but returns valid ResultSetMetadata + // that can be used to construct DBRecord objects to sink to the database table. + ResultSet rs = Failsafe.with(retryPolicy).get(() -> statement + .executeQuery(String.format("SELECT %s FROM %s WHERE 1 = 0", dbColumns, fullyQualifiedTableName)))) { + columnTypes.addAll(getMatchedColumnTypeList(rs, columns)); } - }); + } + this.columnTypes = Collections.unmodifiableList(columnTypes); } @@ -447,30 +466,37 @@ private void validateSchema(FailureCollector collector, Class Properties connectionProperties = new Properties(); connectionProperties.putAll(dbSinkConfig.getConnectionArguments()); try { - Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(dbSinkConfig.getInitialRetryDuration(), - dbSinkConfig.getMaxRetryDuration(), dbSinkConfig.getMaxRetryCount())).run(() -> { - try (Connection connection = DriverManager.getConnection(connectionString, connectionProperties)) { - executeInitQueries(connection, dbSinkConfig.getInitQueries()); - try (ResultSet tables = connection.getMetaData().getTables(null, dbSchemaName, tableName, null)) { - if (!tables.next()) { - collector.addFailure( - String.format("Table '%s' does not exist.", tableName), - String.format("Ensure table '%s' is set correctly and that the connection string '%s' " + - "points to a valid database.", fullyQualifiedTableName, connectionString)) - .withConfigProperty(DBSinkConfig.TABLE_NAME); - return; - } - } - setColumnsInfo(inputSchema.getFields()); - try (PreparedStatement pStmt = connection.prepareStatement(String.format("SELECT %s FROM %s WHERE 1 = 0", - dbColumns, - fullyQualifiedTableName)); - ResultSet rs = pStmt.executeQuery()) { - getFieldsValidator().validateFields(inputSchema, rs, collector); + try (Connection connection = createConnectionWithRetry(connectionProperties)) { + executeInitQueries(connection, dbSinkConfig.getInitQueries()); + try (ResultSet tables = connection.getMetaData().getTables(null, dbSchemaName, tableName, null)) { + if (!tables.next()) { + collector.addFailure( + String.format("Table '%s' does not exist.", tableName), + String.format("Ensure table '%s' is set correctly and that the connection string '%s' " + + "points to a valid database.", fullyQualifiedTableName, connectionString)) + .withConfigProperty(DBSinkConfig.TABLE_NAME); + return; } } - }); + setColumnsInfo(inputSchema.getFields()); + try (PreparedStatement pStmt = prepareStatementWithRetry(connection, + String.format("SELECT %s FROM %s WHERE 1 = 0", dbColumns, fullyQualifiedTableName)); + ResultSet rs = Failsafe.with(retryPolicy).get(() -> pStmt.executeQuery())) { + getFieldsValidator().validateFields(inputSchema, rs, collector); + } + } } catch (FailsafeException e) { + Throwable cause = e.getCause(); + if (cause instanceof SQLException) { + LOG.error("Exception while trying to validate schema of database table {} for connection {}.", + fullyQualifiedTableName, connectionString, e); + collector.addFailure( + String.format("Exception while trying to validate schema of database table '%s' for connection '%s' with %s", + fullyQualifiedTableName, connectionString, e.getMessage()), + null).withStacktrace(e.getStackTrace()); + } + throw e; + } catch (SQLException e) { LOG.error("Exception while trying to validate schema of database table {} for connection {}.", fullyQualifiedTableName, connectionString, e); collector.addFailure( @@ -499,12 +525,9 @@ protected LineageRecorder getLineageRecorder(BatchSinkContext context) { private void executeInitQueries(Connection connection, List initQueries) throws SQLException { for (String query : initQueries) { - Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(dbSinkConfig.getInitialRetryDuration(), - dbSinkConfig.getMaxRetryDuration(), dbSinkConfig.getMaxRetryCount())).run(() -> { - try (Statement statement = connection.createStatement()) { - statement.execute(query); - } - }); + try (Statement statement = createStatementWithRetry(connection)) { + Failsafe.with(retryPolicy).run(() -> statement.execute(query)); + } } } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java index 470e3c0c3..c209986ab 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java @@ -19,6 +19,8 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; import dev.failsafe.Failsafe; +import dev.failsafe.FailsafeException; +import dev.failsafe.RetryPolicy; import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Macro; import io.cdap.cdap.api.annotation.Name; @@ -30,6 +32,7 @@ import io.cdap.cdap.api.exception.ErrorCodeType; import io.cdap.cdap.api.exception.ErrorType; import io.cdap.cdap.api.exception.ErrorUtils; +import io.cdap.cdap.api.exception.ProgramFailureException; import io.cdap.cdap.api.plugin.PluginConfig; import io.cdap.cdap.etl.api.Emitter; import io.cdap.cdap.etl.api.FailureCollector; @@ -60,6 +63,7 @@ import org.apache.hadoop.mapreduce.lib.db.DBWritable; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + import java.io.IOException; import java.sql.Connection; import java.sql.Driver; @@ -88,13 +92,17 @@ public abstract class AbstractDBSource retryPolicy; protected final T sourceConfig; protected Class driverClass; + public AbstractDBSource(T sourceConfig) { super(new ReferencePluginConfig(sourceConfig.getReferenceName())); this.sourceConfig = sourceConfig; + this.retryPolicy = RetryPolicyUtil.getRetryPolicy(sourceConfig.getInitialRetryDuration(), + sourceConfig.getMaxRetryDuration(), sourceConfig.getMaxRetryCount()); } @Override @@ -168,15 +176,11 @@ public Schema getSchema() throws SQLException { } private Schema loadSchemaFromDB(Connection connection, String query) throws SQLException { - return Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(sourceConfig.getInitialRetryDuration(), - sourceConfig.getMaxRetryDuration(), sourceConfig.getMaxRetryCount())) - .get(() -> { - Statement statement = connection.createStatement(); - statement.setMaxRows(1); - String finalQuery = query.contains("$CONDITIONS") ? removeConditionsClause(query) : query; - ResultSet resultSet = statement.executeQuery(finalQuery); - return Schema.recordOf("outputSchema", getSchemaReader().getSchemaFields(resultSet)); - }); + Statement statement = createStatementWithRetry(connection); + statement.setMaxRows(1); + String finalQuery = query.contains("$CONDITIONS") ? removeConditionsClause(query) : query; + ResultSet resultSet = Failsafe.with(retryPolicy).get(() -> statement.executeQuery(finalQuery)); + return Schema.recordOf("outputSchema", getSchemaReader().getSchemaFields(resultSet)); } @VisibleForTesting @@ -193,55 +197,55 @@ private Schema loadSchemaFromDB(Class driverClass) String connectionString = sourceConfig.getConnectionString(); DriverCleanup driverCleanup = DBUtils.ensureJDBCDriverIsAvailable(driverClass, connectionString, sourceConfig.getJdbcPluginName()); + Properties connectionProperties = new Properties(); connectionProperties.putAll(sourceConfig.getConnectionArguments()); - return Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(sourceConfig.getInitialRetryDuration(), - sourceConfig.getMaxRetryDuration(), sourceConfig.getMaxRetryCount())) - .get(() -> { - try (Connection connection = DriverManager.getConnection(connectionString, connectionProperties)) { - executeInitQueries(connection, sourceConfig.getInitQueries()); - return loadSchemaFromDB(connection, sourceConfig.getImportQuery()); - } - catch (SQLException e) { - // wrap exception to ensure SQLException-child instances not exposed to contexts without jdbc - // driver in classpath - String errorMessage = - String.format("SQL Exception occurred: [Message='%s', SQLState='%s', ErrorCode='%s'].", - e.getMessage(), - e.getSQLState(), e.getErrorCode()); - String errorMessageWithDetails = String.format("Error occurred while trying to" + - " get schema from database." + - "Error message: '%s'. Error code: '%s'. SQLState: '%s'", e.getMessage(), e.getErrorCode(), - e.getSQLState()); - String externalDocumentationLink = getExternalDocumentationLink(); - if (!Strings.isNullOrEmpty(externalDocumentationLink)) { - if (!errorMessage.endsWith(".")) { - errorMessage = errorMessage + "."; - } - errorMessage = String.format("%s For more details, see %s", errorMessage, - externalDocumentationLink); - } - throw ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), - errorMessage, errorMessageWithDetails, ErrorType.USER, false, ErrorCodeType.SQLSTATE, - e.getSQLState(), externalDocumentationLink, new SQLException(e.getMessage(), - e.getSQLState(), e.getErrorCode())); - } finally { - driverCleanup.destroy(); - } - }); + try (Connection connection = createConnectionWithRetry(connectionString, connectionProperties)) { + executeInitQueries(connection, sourceConfig.getInitQueries()); + return loadSchemaFromDB(connection, sourceConfig.getImportQuery()); + } catch (FailsafeException e) { + Throwable cause = e.getCause(); + if (cause instanceof SQLException) { + throw wrapException((SQLException) cause); + } + throw e; + } catch (SQLException e) { + throw wrapException(e); + } finally { + driverCleanup.destroy(); } + } private void executeInitQueries(Connection connection, List initQueries) throws SQLException { for (String query : initQueries) { - Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(sourceConfig.getInitialRetryDuration(), - sourceConfig.getMaxRetryDuration(), sourceConfig.getMaxRetryCount())).run(() -> { - try (Statement statement = connection.createStatement()) { - statement.execute(query); - } - }); + try (Statement statement = createStatementWithRetry(connection)) { + Failsafe.with(retryPolicy).run(() -> statement.execute(query)); + } } } + private ProgramFailureException wrapException(SQLException e) { + // wrap exception to ensure SQLException-child instances not exposed to contexts without jdbc + // driver in classpath + String errorMessage = + String.format("SQL Exception occurred: [Message='%s', SQLState='%s', ErrorCode='%s'].", + e.getMessage(), e.getSQLState(), e.getErrorCode()); + String errorMessageWithDetails = String.format("Error occurred while trying to" + + " get schema from database." + "Error message: '%s'. Error code: '%s'. SQLState: '%s'", e.getMessage(), + e.getErrorCode(), e.getSQLState()); + String externalDocumentationLink = getExternalDocumentationLink(); + if (!Strings.isNullOrEmpty(externalDocumentationLink)) { + if (!errorMessage.endsWith(".")) { + errorMessage = errorMessage + "."; + } + errorMessage = String.format("%s For more details, see %s", errorMessage, externalDocumentationLink); + } + return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, errorMessageWithDetails, ErrorType.USER, false, ErrorCodeType.SQLSTATE, e.getSQLState(), + externalDocumentationLink, new SQLException(e.getMessage(), e.getSQLState(), e.getErrorCode())); + + } + protected SchemaReader getSchemaReader() { return new CommonSchemaReader(); } @@ -275,13 +279,20 @@ private DriverCleanup loadPluginClassAndGetDriver(Class driver } } - private Connection getConnection() throws SQLException { + private Connection getConnection() { String connectionString = createConnectionString(); Properties connectionProperties = new Properties(); connectionProperties.putAll(sourceConfig.getConnectionArguments()); - return Failsafe.with(RetryPolicyUtil.createConnectionRetryPolicy(sourceConfig.getInitialRetryDuration(), - sourceConfig.getMaxRetryDuration(), sourceConfig.getMaxRetryCount())) - .get(() -> DriverManager.getConnection(connectionString, connectionProperties)); + return createConnectionWithRetry(connectionString, connectionProperties); + } + + private Connection createConnectionWithRetry(String connectionString, Properties connectionProperties) { + return Failsafe.with(retryPolicy).get(() -> DriverManager + .getConnection(connectionString, connectionProperties)); + } + + private Statement createStatementWithRetry(Connection connection) { + return Failsafe.with(retryPolicy).get(() -> connection.createStatement()); } @Override diff --git a/database-commons/src/main/java/io/cdap/plugin/util/RetryPolicyUtil.java b/database-commons/src/main/java/io/cdap/plugin/util/RetryPolicyUtil.java index 1aca23ec7..40f75f74e 100644 --- a/database-commons/src/main/java/io/cdap/plugin/util/RetryPolicyUtil.java +++ b/database-commons/src/main/java/io/cdap/plugin/util/RetryPolicyUtil.java @@ -37,7 +37,7 @@ public class RetryPolicyUtil extends Config { /** * Create a RetryPolicy using custom config values. */ - public static RetryPolicy createConnectionRetryPolicy(Integer initialRetryDuration, + public static RetryPolicy getRetryPolicy(Integer initialRetryDuration, Integer maxRetryDuration, Integer maxRetryCount) { return RetryPolicy.builder() .handleIf((failure) -> RetryExceptions.isRetryable(failure)) diff --git a/database-commons/src/test/java/io/cdap/plugin/db/RetryPolicyUtilTest.java b/database-commons/src/test/java/io/cdap/plugin/db/RetryPolicyUtilTest.java index e85e50e8b..fa330de4e 100644 --- a/database-commons/src/test/java/io/cdap/plugin/db/RetryPolicyUtilTest.java +++ b/database-commons/src/test/java/io/cdap/plugin/db/RetryPolicyUtilTest.java @@ -47,8 +47,8 @@ public void setup() { @Test public void testCreateConnectionRetryPolicy_Retryable() { RetryPolicy retryPolicy = RetryPolicyUtil - .createConnectionRetryPolicy(mockConfig.getInitialRetryDuration(), mockConfig.getMaxRetryDuration(), - mockConfig.getMaxRetryCount()); + .getRetryPolicy(mockConfig.getInitialRetryDuration(), mockConfig.getMaxRetryDuration(), + mockConfig.getMaxRetryCount()); AtomicInteger attemptCounter = new AtomicInteger(); @@ -64,8 +64,8 @@ public void testCreateConnectionRetryPolicy_Retryable() { @Test public void testCreateConnectionRetryPolicy_NonRetryable() { RetryPolicy retryPolicy = RetryPolicyUtil - .createConnectionRetryPolicy(mockConfig.getInitialRetryDuration(), mockConfig.getMaxRetryDuration(), - mockConfig.getMaxRetryCount()); + .getRetryPolicy(mockConfig.getInitialRetryDuration(), mockConfig.getMaxRetryDuration(), + mockConfig.getMaxRetryCount()); AtomicInteger attemptCounter = new AtomicInteger(); From d0c15c00691572f9769e12c61c61f3a2d922d245 Mon Sep 17 00:00:00 2001 From: Sanchit Garg Date: Mon, 2 Jun 2025 16:46:05 +0530 Subject: [PATCH 03/28] Fix unit test for MySQLSinkTest --- .../test/java/io/cdap/plugin/mysql/MysqlSinkTest.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlSinkTest.java b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlSinkTest.java index 1dd4e809e..42f5df18c 100644 --- a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlSinkTest.java +++ b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlSinkTest.java @@ -20,6 +20,9 @@ import org.junit.Assert; import org.junit.Test; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + public class MysqlSinkTest { @Test public void testSetColumnsInfo() { @@ -27,7 +30,13 @@ public void testSetColumnsInfo() { Schema.Field.of("id", Schema.of(Schema.Type.INT)), Schema.Field.of("name", Schema.of(Schema.Type.STRING)), Schema.Field.of("insert", Schema.of(Schema.Type.STRING))); - MysqlSink mySQLSink = new MysqlSink(new MysqlSink.MysqlSinkConfig()); + + MysqlSink.MysqlSinkConfig mockConfig = mock(MysqlSink.MysqlSinkConfig.class); + when(mockConfig.getInitialRetryDuration()).thenReturn(5); // or appropriate value + when(mockConfig.getMaxRetryDuration()).thenReturn(80); // or appropriate value + when(mockConfig.getMaxRetryCount()).thenReturn(5); // or appropriate value + + MysqlSink mySQLSink = new MysqlSink(mockConfig); Assert.assertNotNull(outputSchema.getFields()); mySQLSink.setColumnsInfo(outputSchema.getFields()); Assert.assertEquals("`id`,`name`,`insert`", mySQLSink.getDbColumns()); From ad6b3fe3506e3b6a3234a1ece2f2187666402739 Mon Sep 17 00:00:00 2001 From: Sanchit Garg Date: Mon, 2 Jun 2025 16:57:40 +0530 Subject: [PATCH 04/28] Fix failing unit test for CloudSQLMySQLTest --- .../plugin/cloudsql/mysql/CloudSQLMySQLSinkTest.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cloudsql-mysql-plugin/src/test/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSinkTest.java b/cloudsql-mysql-plugin/src/test/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSinkTest.java index 65a14502e..93c06981a 100644 --- a/cloudsql-mysql-plugin/src/test/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSinkTest.java +++ b/cloudsql-mysql-plugin/src/test/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSinkTest.java @@ -20,6 +20,9 @@ import org.junit.Assert; import org.junit.Test; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + public class CloudSQLMySQLSinkTest { @Test public void testSetColumnsInfo() { @@ -27,7 +30,13 @@ public void testSetColumnsInfo() { Schema.Field.of("id", Schema.of(Schema.Type.INT)), Schema.Field.of("name", Schema.of(Schema.Type.STRING)), Schema.Field.of("insert", Schema.of(Schema.Type.STRING))); - CloudSQLMySQLSink cloudSQLMySQLSink = new CloudSQLMySQLSink(new CloudSQLMySQLSink.CloudSQLMySQLSinkConfig()); + + CloudSQLMySQLSink.CloudSQLMySQLSinkConfig mockConfig = mock(CloudSQLMySQLSink.CloudSQLMySQLSinkConfig.class); + when(mockConfig.getInitialRetryDuration()).thenReturn(5); // or appropriate value + when(mockConfig.getMaxRetryDuration()).thenReturn(80); // or appropriate value + when(mockConfig.getMaxRetryCount()).thenReturn(5); // or appropriate value + + CloudSQLMySQLSink cloudSQLMySQLSink = new CloudSQLMySQLSink(mockConfig); Assert.assertNotNull(outputSchema.getFields()); cloudSQLMySQLSink.setColumnsInfo(outputSchema.getFields()); Assert.assertEquals("`id`,`name`,`insert`", cloudSQLMySQLSink.getDbColumns()); From ff0cd0ba832e5e1dacd54000f4705e38961c5d1f Mon Sep 17 00:00:00 2001 From: Sanchit Garg Date: Tue, 3 Jun 2025 19:09:57 +0530 Subject: [PATCH 05/28] PLUGIN-1823: Rework Move retry logic into a separate class: RetryUtils and add exception handling --- .../io/cdap/plugin/db/ConnectionConfig.java | 1 - .../db/action/AbstractDBArgumentSetter.java | 23 +-- .../java/io/cdap/plugin/db/action/DBRun.java | 26 ++- .../AbstractDBSpecificConnector.java | 13 +- .../cdap/plugin/db/sink/AbstractDBSink.java | 151 +++++++----------- .../plugin/db/source/AbstractDBSource.java | 39 ++--- .../io/cdap/plugin/util/RetryPolicyUtil.java | 2 - .../java/io/cdap/plugin/util/RetryUtils.java | 141 ++++++++++++++++ .../plugin/mysql/MysqlConnectorUnitTest.java | 5 +- 9 files changed, 238 insertions(+), 163 deletions(-) create mode 100644 database-commons/src/main/java/io/cdap/plugin/util/RetryUtils.java diff --git a/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java index f28d2c450..53ac81237 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java @@ -109,7 +109,6 @@ public Integer getMaxRetryCount() { return maxRetryCount == null ? DEFAULT_MAX_RETRY_COUNT : maxRetryCount; } - public ConnectionConfig() { } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java index d26cdc2d0..6d6c9d5ec 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java @@ -16,7 +16,6 @@ package io.cdap.plugin.db.action; -import dev.failsafe.Failsafe; import dev.failsafe.RetryPolicy; import io.cdap.cdap.etl.api.FailureCollector; import io.cdap.cdap.etl.api.PipelineConfigurer; @@ -28,10 +27,10 @@ import io.cdap.plugin.util.DBUtils; import io.cdap.plugin.util.DriverCleanup; import io.cdap.plugin.util.RetryPolicyUtil; +import io.cdap.plugin.util.RetryUtils; import java.sql.Connection; import java.sql.Driver; -import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -44,7 +43,7 @@ public class AbstractDBArgumentSetter extends Action { private static final String JDBC_PLUGIN_ID = "driver"; private final ArgumentSetterConfig config; - private final RetryPolicy retryPolicy; + private final RetryPolicy retryPolicy; public AbstractDBArgumentSetter(ArgumentSetterConfig config) { this.config = config; @@ -114,10 +113,13 @@ private void processArguments(Class driverClass, private void executeWithRetry(FailureCollector failureCollector, SettableArguments settableArguments, Properties connectionProperties) throws SQLException { - try (Connection connection = createConnectionWithRetry(connectionProperties)) { + try (Connection connection = RetryUtils.createConnectionWithRetry((RetryPolicy) retryPolicy, + config.getConnectionString(), connectionProperties, null)) { ResultSet resultSet; - try (Statement statement = createStatementWithRetry(connection)) { - resultSet = Failsafe.with(retryPolicy).get(() -> statement.executeQuery(config.getQuery())); + try (Statement statement = RetryUtils.createStatementWithRetry((RetryPolicy) retryPolicy, connection, + null)) { + resultSet = RetryUtils.executeQueryWithRetry((RetryPolicy) retryPolicy, statement, + config.getQuery(), null); } boolean hasRecord = resultSet.next(); if (!hasRecord) { @@ -136,15 +138,6 @@ private void executeWithRetry(FailureCollector failureCollector, SettableArgumen } } - private Connection createConnectionWithRetry(Properties connectionProperties) { - return Failsafe.with(retryPolicy).get(() -> DriverManager - .getConnection(config.getConnectionString(), connectionProperties)); - } - - private Statement createStatementWithRetry(Connection connection) { - return Failsafe.with(retryPolicy).get(() -> connection.createStatement()); - } - /** * Converts column from jdbc results set into pipeline arguments * diff --git a/database-commons/src/main/java/io/cdap/plugin/db/action/DBRun.java b/database-commons/src/main/java/io/cdap/plugin/db/action/DBRun.java index 8f2cb0e2f..ab3392a9b 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/action/DBRun.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/action/DBRun.java @@ -21,6 +21,7 @@ import io.cdap.plugin.util.DBUtils; import io.cdap.plugin.util.DriverCleanup; import io.cdap.plugin.util.RetryPolicyUtil; +import io.cdap.plugin.util.RetryUtils; import java.sql.Connection; import java.sql.Driver; @@ -37,7 +38,7 @@ public class DBRun { private final QueryConfig config; private final Class driverClass; private boolean enableAutoCommit; - private final RetryPolicy retryPolicy; + private final RetryPolicy retryPolicy; public DBRun(QueryConfig config, Class driverClass, Boolean enableAutocommit) { this.config = config; @@ -61,14 +62,15 @@ public void run() throws SQLException, InstantiationException, IllegalAccessExce Properties connectionProperties = new Properties(); connectionProperties.putAll(config.getConnectionArguments()); - try (Connection connection = createConnectionWithRetry(connectionProperties)) { + try (Connection connection = RetryUtils.createConnectionWithRetry((RetryPolicy) retryPolicy, + config.getConnectionString(), connectionProperties, null)) { executeInitQueries(connection, config.getInitQueries()); if (!enableAutoCommit) { connection.setAutoCommit(false); } - try (Statement statement = createStatementWithRetry(connection)) { - Failsafe.with(retryPolicy).run(() -> statement.execute(config.query)); - + try (Statement statement = RetryUtils.createStatementWithRetry((RetryPolicy) retryPolicy, connection, + null)) { + RetryUtils.executeInitQueryWithRetry(retryPolicy, statement, config.query, null); if (!enableAutoCommit) { connection.commit(); } @@ -83,18 +85,10 @@ public void run() throws SQLException, InstantiationException, IllegalAccessExce private void executeInitQueries(Connection connection, List initQueries) throws SQLException { for (String query : initQueries) { - try (Statement statement = createStatementWithRetry(connection)) { - Failsafe.with(retryPolicy).run(() -> statement.execute(query)); + try (Statement statement = RetryUtils.createStatementWithRetry((RetryPolicy) retryPolicy, connection, + null)) { + RetryUtils.executeInitQueryWithRetry(retryPolicy, statement, query, null); } } } - - private Connection createConnectionWithRetry(Properties connectionProperties) { - return Failsafe.with(retryPolicy).get(() -> DriverManager - .getConnection(config.getConnectionString(), connectionProperties)); - } - - private Statement createStatementWithRetry(Connection connection) { - return Failsafe.with(retryPolicy).get(() -> connection.createStatement()); - } } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java index caeeac568..973574db0 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java @@ -17,7 +17,7 @@ package io.cdap.plugin.db.connector; import com.google.common.collect.Maps; -import dev.failsafe.Failsafe; +import dev.failsafe.RetryPolicy; import io.cdap.cdap.api.data.batch.InputFormatProvider; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.etl.api.batch.BatchConnector; @@ -35,6 +35,7 @@ import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.source.DataDrivenETLDBInputFormat; import io.cdap.plugin.util.RetryPolicyUtil; +import io.cdap.plugin.util.RetryUtils; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.lib.db.DBConfiguration; @@ -58,10 +59,13 @@ public abstract class AbstractDBSpecificConnector extends implements BatchConnector { private final AbstractDBConnectorConfig config; + private final RetryPolicy retryPolicy; protected AbstractDBSpecificConnector(AbstractDBConnectorConfig config) { super(config); this.config = config; + this.retryPolicy = RetryPolicyUtil.getRetryPolicy(config.getInitialRetryDuration(), config.getMaxRetryDuration(), + config.getMaxRetryCount()); } public abstract boolean supportSchema(); @@ -174,14 +178,13 @@ protected String getStratifiedQuery(String tableName, int limit, String strata, protected Schema loadTableSchema(Connection connection, String query, @Nullable Integer timeoutSec, String sessionID) throws SQLException { - Statement statement = connection.createStatement(); + Statement statement = RetryUtils.createStatementWithRetry((RetryPolicy) retryPolicy, connection, null); statement.setMaxRows(1); if (timeoutSec != null) { statement.setQueryTimeout(timeoutSec); } - ResultSet resultSet = Failsafe.with(RetryPolicyUtil.getRetryPolicy(config.getInitialRetryDuration(), - config.getMaxRetryDuration(), config.getMaxRetryCount())) - .get(() -> statement.executeQuery(query)); + ResultSet resultSet = RetryUtils.executeQueryWithRetry((RetryPolicy) retryPolicy, statement, query, + null); return Schema.recordOf("outputSchema", getSchemaReader(sessionID).getSchemaFields(resultSet)); } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java index 543110237..624542d0b 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java @@ -18,8 +18,6 @@ import com.google.common.base.Preconditions; import com.google.common.base.Strings; -import dev.failsafe.Failsafe; -import dev.failsafe.FailsafeException; import dev.failsafe.RetryPolicy; import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Macro; @@ -28,11 +26,6 @@ import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.api.dataset.lib.KeyValue; -import io.cdap.cdap.api.exception.ErrorCategory; -import io.cdap.cdap.api.exception.ErrorCodeType; -import io.cdap.cdap.api.exception.ErrorType; -import io.cdap.cdap.api.exception.ErrorUtils; -import io.cdap.cdap.api.exception.ProgramFailureException; import io.cdap.cdap.api.plugin.PluginConfig; import io.cdap.cdap.etl.api.Emitter; import io.cdap.cdap.etl.api.FailureCollector; @@ -46,6 +39,7 @@ import io.cdap.plugin.common.ReferenceBatchSink; import io.cdap.plugin.common.ReferencePluginConfig; import io.cdap.plugin.common.batch.sink.SinkOutputFormatProvider; +import io.cdap.plugin.common.util.ExceptionUtils; import io.cdap.plugin.db.ColumnType; import io.cdap.plugin.db.CommonSchemaReader; import io.cdap.plugin.db.ConnectionConfig; @@ -58,15 +52,16 @@ import io.cdap.plugin.util.DBUtils; import io.cdap.plugin.util.DriverCleanup; import io.cdap.plugin.util.RetryPolicyUtil; +import io.cdap.plugin.util.RetryUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapred.lib.db.DBConfiguration; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.sql.Connection; import java.sql.Driver; -import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.ResultSetMetaData; @@ -103,7 +98,7 @@ public abstract class AbstractDBSink retryPolicy; + private final RetryPolicy retryPolicy; public AbstractDBSink(T dbSinkConfig) { super(new ReferencePluginConfig(dbSinkConfig.getReferenceName())); @@ -134,7 +129,17 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { Class driverClass = DBUtils.getDriverClass( pipelineConfigurer, dbSinkConfig, ConnectionConfig.JDBC_PLUGIN_TYPE); if (driverClass != null && dbSinkConfig.canConnect()) { - validateSchema(collector, driverClass, dbSinkConfig.getTableName(), inputSchema, dbSinkConfig.getDBSchemaName()); + try { + validateSchema(collector, driverClass, dbSinkConfig.getTableName(), inputSchema, + dbSinkConfig.getDBSchemaName()); + } catch (SQLException e) { + String details = String.format("SQL error while validating the schema: Error: %s, SQLState: %s, ErrorCode: %s", + e.getMessage(), e.getSQLState(), e.getErrorCode()); + collector.addFailure(details, + "Ensure the table exists and the connection string points to a valid database.") + .withConfigProperty(DBSinkConfig.TABLE_NAME) + .withStacktrace(e.getStackTrace()); + } } } public void validateOperations(FailureCollector collector, T dbSinkConfig, @Nullable Schema inputSchema) { @@ -224,6 +229,9 @@ public void prepareRun(BatchSinkContext context) { } else { outputSchema = inferSchema(driverClass); } + } catch (SQLException e) { + throw new RuntimeException(String.format("Unable to validate schema due to: %s.", + ExceptionUtils.getRootCauseMessage(e)), e); } finally { DBUtils.cleanup(driverClass); } @@ -310,21 +318,16 @@ private Schema inferSchema(Class driverClass) { dbSinkConfig.getJdbcPluginName()); Properties connectionProperties = new Properties(); connectionProperties.putAll(dbSinkConfig.getConnectionArguments()); - try (Connection connection = createConnectionWithRetry(connectionProperties)) { + try (Connection connection = RetryUtils.createConnectionWithRetry((RetryPolicy) retryPolicy, + dbSinkConfig.getConnectionString(), connectionProperties, getExternalDocumentationLink())) { executeInitQueries(connection, dbSinkConfig.getInitQueries()); - try (Statement statement = createStatementWithRetry(connection); - ResultSet rs = Failsafe.with(retryPolicy).get(() -> statement - .executeQuery("SELECT * FROM " + fullyQualifiedTableName + " WHERE 1 = 0"))) { + try (Statement statement = RetryUtils.createStatementWithRetry((RetryPolicy) retryPolicy, + connection, getExternalDocumentationLink()); + ResultSet rs = RetryUtils.executeQueryWithRetry((RetryPolicy) retryPolicy, statement, + String.format("SELECT * FROM %s WHERE 1 = 0", fullyQualifiedTableName), + getExternalDocumentationLink())) { inferredFields.addAll(getSchemaReader().getSchemaFields(rs)); } - } catch (FailsafeException e) { - Throwable cause = e.getCause(); - if (cause instanceof SQLException) { - throw wrapException((SQLException) cause); - } - throw e; - } catch (SQLException e) { - throw wrapException(e); } } catch (IllegalAccessException | InstantiationException | SQLException e) { throw new InvalidStageException("JDBC Driver unavailable: " + dbSinkConfig.getJdbcPluginName(), e); @@ -332,38 +335,6 @@ private Schema inferSchema(Class driverClass) { return Schema.recordOf("inferredSchema", inferredFields); } - private ProgramFailureException wrapException(SQLException e) { - // wrap exception to ensure SQLException-child instances not exposed to contexts w/o jdbc driver in classpath - String errorMessage = - String.format("SQL Exception occurred: [Message='%s', SQLState='%s', ErrorCode='%s'].", e.getMessage(), - e.getSQLState(), e.getErrorCode()); - String errorMessageWithDetails = String.format("Error while reading table metadata." + - "Error message: '%s'. Error code: '%s'. SQLState: '%s'", e.getMessage(), e.getErrorCode(), e.getSQLState()); - String externalDocumentationLink = getExternalDocumentationLink(); - if (!Strings.isNullOrEmpty(externalDocumentationLink)) { - if (!errorMessage.endsWith(".")) { - errorMessage = errorMessage + "."; - } - errorMessage = String.format("%s For more details, see %s", errorMessageWithDetails, errorMessage); - } - return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), - errorMessage, errorMessageWithDetails, ErrorType.USER, false, ErrorCodeType.SQLSTATE, e.getSQLState(), - externalDocumentationLink, new SQLException(e.getMessage(), e.getSQLState(), e.getErrorCode())); - } - - private Connection createConnectionWithRetry(Properties connectionProperties) { - return Failsafe.with(retryPolicy).get(() -> DriverManager - .getConnection(dbSinkConfig.getConnectionString(), connectionProperties)); - } - - private Statement createStatementWithRetry(Connection connection) { - return Failsafe.with(retryPolicy).get(() -> connection.createStatement()); - } - - private PreparedStatement prepareStatementWithRetry(Connection connection, String sqlQuery) { - return Failsafe.with(retryPolicy).get(() -> connection.prepareStatement(sqlQuery)); - } - @Override public void transform(StructuredRecord input, Emitter> emitter) { emitter.emit(new KeyValue<>(getDBRecord(input), null)); @@ -385,7 +356,7 @@ public void destroy() { } } - private void setResultSetMetadata() throws Exception { + private void setResultSetMetadata() throws SQLException, IllegalAccessException, InstantiationException { List columnTypes = new ArrayList<>(columns.size()); String connectionString = dbSinkConfig.getConnectionString(); String dbSchemaName = dbSinkConfig.getDBSchemaName(); @@ -397,13 +368,16 @@ private void setResultSetMetadata() throws Exception { Properties connectionProperties = new Properties(); connectionProperties.putAll(dbSinkConfig.getConnectionArguments()); - try (Connection connection = createConnectionWithRetry(connectionProperties)) { + try (Connection connection = RetryUtils.createConnectionWithRetry((RetryPolicy) retryPolicy, + connectionString, connectionProperties, getExternalDocumentationLink())) { executeInitQueries(connection, dbSinkConfig.getInitQueries()); - try (Statement statement = createStatementWithRetry(connection); + try (Statement statement = RetryUtils.createStatementWithRetry((RetryPolicy) retryPolicy, connection, + getExternalDocumentationLink()); // Run a query against the DB table that returns 0 records, but returns valid ResultSetMetadata // that can be used to construct DBRecord objects to sink to the database table. - ResultSet rs = Failsafe.with(retryPolicy).get(() -> statement - .executeQuery(String.format("SELECT %s FROM %s WHERE 1 = 0", dbColumns, fullyQualifiedTableName)))) { + ResultSet rs = RetryUtils.executeQueryWithRetry((RetryPolicy) retryPolicy, statement, + String.format("SELECT %s FROM %s WHERE 1 = 0", dbColumns, fullyQualifiedTableName), + getExternalDocumentationLink())) { columnTypes.addAll(getMatchedColumnTypeList(rs, columns)); } } @@ -449,7 +423,7 @@ static List getMatchedColumnTypeList(ResultSet resultSet, List jdbcDriverClass, String tableName, - Schema inputSchema, String dbSchemaName) { + Schema inputSchema, String dbSchemaName) throws SQLException { String connectionString = dbSinkConfig.getConnectionString(); String fullyQualifiedTableName = dbSchemaName == null ? dbSinkConfig.getEscapedTableName() : dbSinkConfig.getEscapedDbSchemaName() + "." + dbSinkConfig.getEscapedTableName(); @@ -465,44 +439,26 @@ private void validateSchema(FailureCollector collector, Class Properties connectionProperties = new Properties(); connectionProperties.putAll(dbSinkConfig.getConnectionArguments()); - try { - try (Connection connection = createConnectionWithRetry(connectionProperties)) { - executeInitQueries(connection, dbSinkConfig.getInitQueries()); - try (ResultSet tables = connection.getMetaData().getTables(null, dbSchemaName, tableName, null)) { - if (!tables.next()) { - collector.addFailure( - String.format("Table '%s' does not exist.", tableName), - String.format("Ensure table '%s' is set correctly and that the connection string '%s' " + - "points to a valid database.", fullyQualifiedTableName, connectionString)) - .withConfigProperty(DBSinkConfig.TABLE_NAME); - return; - } - } - setColumnsInfo(inputSchema.getFields()); - try (PreparedStatement pStmt = prepareStatementWithRetry(connection, - String.format("SELECT %s FROM %s WHERE 1 = 0", dbColumns, fullyQualifiedTableName)); - ResultSet rs = Failsafe.with(retryPolicy).get(() -> pStmt.executeQuery())) { - getFieldsValidator().validateFields(inputSchema, rs, collector); + try (Connection connection = RetryUtils.createConnectionWithRetry((RetryPolicy) retryPolicy, + connectionString, connectionProperties, getExternalDocumentationLink())) { + executeInitQueries(connection, dbSinkConfig.getInitQueries()); + try (ResultSet tables = connection.getMetaData().getTables(null, dbSchemaName, tableName, null)) { + if (!tables.next()) { + collector.addFailure(String.format("Table '%s' does not exist.", tableName), + String.format("Ensure table '%s' is set correctly and that the connection string '%s' " + + "points to a valid database.", fullyQualifiedTableName, connectionString)) + .withConfigProperty(DBSinkConfig.TABLE_NAME); + return; } } - } catch (FailsafeException e) { - Throwable cause = e.getCause(); - if (cause instanceof SQLException) { - LOG.error("Exception while trying to validate schema of database table {} for connection {}.", - fullyQualifiedTableName, connectionString, e); - collector.addFailure( - String.format("Exception while trying to validate schema of database table '%s' for connection '%s' with %s", - fullyQualifiedTableName, connectionString, e.getMessage()), - null).withStacktrace(e.getStackTrace()); + setColumnsInfo(inputSchema.getFields()); + try (PreparedStatement pStmt = RetryUtils.prepareStatementWithRetry((RetryPolicy) retryPolicy, + connection, String.format("SELECT %s FROM %s WHERE 1 = 0", dbColumns, fullyQualifiedTableName), + getExternalDocumentationLink()); + ResultSet rs = RetryUtils.executeQueryWithRetry((RetryPolicy) retryPolicy, pStmt, + getExternalDocumentationLink())) { + getFieldsValidator().validateFields(inputSchema, rs, collector); } - throw e; - } catch (SQLException e) { - LOG.error("Exception while trying to validate schema of database table {} for connection {}.", - fullyQualifiedTableName, connectionString, e); - collector.addFailure( - String.format("Exception while trying to validate schema of database table '%s' for connection '%s' with %s", - fullyQualifiedTableName, connectionString, e.getMessage()), - null).withStacktrace(e.getStackTrace()); } } @@ -525,8 +481,9 @@ protected LineageRecorder getLineageRecorder(BatchSinkContext context) { private void executeInitQueries(Connection connection, List initQueries) throws SQLException { for (String query : initQueries) { - try (Statement statement = createStatementWithRetry(connection)) { - Failsafe.with(retryPolicy).run(() -> statement.execute(query)); + try (Statement statement = RetryUtils.createStatementWithRetry((RetryPolicy) retryPolicy, connection, + getExternalDocumentationLink())) { + RetryUtils.executeInitQueryWithRetry(retryPolicy, statement, query, getExternalDocumentationLink()); } } } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java index c209986ab..a87e597e2 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java @@ -57,6 +57,7 @@ import io.cdap.plugin.util.DBUtils; import io.cdap.plugin.util.DriverCleanup; import io.cdap.plugin.util.RetryPolicyUtil; +import io.cdap.plugin.util.RetryUtils; import org.apache.hadoop.io.LongWritable; import org.apache.hadoop.mapreduce.MRJobConfig; import org.apache.hadoop.mapreduce.lib.db.DBConfiguration; @@ -67,7 +68,6 @@ import java.io.IOException; import java.sql.Connection; import java.sql.Driver; -import java.sql.DriverManager; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; @@ -92,12 +92,11 @@ public abstract class AbstractDBSource retryPolicy; + private final RetryPolicy retryPolicy; protected final T sourceConfig; protected Class driverClass; - public AbstractDBSource(T sourceConfig) { super(new ReferencePluginConfig(sourceConfig.getReferenceName())); this.sourceConfig = sourceConfig; @@ -176,10 +175,12 @@ public Schema getSchema() throws SQLException { } private Schema loadSchemaFromDB(Connection connection, String query) throws SQLException { - Statement statement = createStatementWithRetry(connection); + Statement statement = RetryUtils.createStatementWithRetry((RetryPolicy) retryPolicy, connection, + getExternalDocumentationLink()); statement.setMaxRows(1); String finalQuery = query.contains("$CONDITIONS") ? removeConditionsClause(query) : query; - ResultSet resultSet = Failsafe.with(retryPolicy).get(() -> statement.executeQuery(finalQuery)); + ResultSet resultSet = RetryUtils.executeQueryWithRetry((RetryPolicy) retryPolicy, statement, + finalQuery, getExternalDocumentationLink()); return Schema.recordOf("outputSchema", getSchemaReader().getSchemaFields(resultSet)); } @@ -200,17 +201,10 @@ private Schema loadSchemaFromDB(Class driverClass) Properties connectionProperties = new Properties(); connectionProperties.putAll(sourceConfig.getConnectionArguments()); - try (Connection connection = createConnectionWithRetry(connectionString, connectionProperties)) { + try (Connection connection = RetryUtils.createConnectionWithRetry((RetryPolicy) retryPolicy, + connectionString, connectionProperties, getExternalDocumentationLink())) { executeInitQueries(connection, sourceConfig.getInitQueries()); return loadSchemaFromDB(connection, sourceConfig.getImportQuery()); - } catch (FailsafeException e) { - Throwable cause = e.getCause(); - if (cause instanceof SQLException) { - throw wrapException((SQLException) cause); - } - throw e; - } catch (SQLException e) { - throw wrapException(e); } finally { driverCleanup.destroy(); } @@ -218,8 +212,9 @@ private Schema loadSchemaFromDB(Class driverClass) private void executeInitQueries(Connection connection, List initQueries) throws SQLException { for (String query : initQueries) { - try (Statement statement = createStatementWithRetry(connection)) { - Failsafe.with(retryPolicy).run(() -> statement.execute(query)); + try (Statement statement = RetryUtils.createStatementWithRetry((RetryPolicy) retryPolicy, connection, + getExternalDocumentationLink())) { + RetryUtils.executeInitQueryWithRetry(retryPolicy, statement, query, getExternalDocumentationLink()); } } } @@ -283,16 +278,8 @@ private Connection getConnection() { String connectionString = createConnectionString(); Properties connectionProperties = new Properties(); connectionProperties.putAll(sourceConfig.getConnectionArguments()); - return createConnectionWithRetry(connectionString, connectionProperties); - } - - private Connection createConnectionWithRetry(String connectionString, Properties connectionProperties) { - return Failsafe.with(retryPolicy).get(() -> DriverManager - .getConnection(connectionString, connectionProperties)); - } - - private Statement createStatementWithRetry(Connection connection) { - return Failsafe.with(retryPolicy).get(() -> connection.createStatement()); + return RetryUtils.createConnectionWithRetry((RetryPolicy) retryPolicy, + connectionString, connectionProperties, getExternalDocumentationLink()); } @Override diff --git a/database-commons/src/main/java/io/cdap/plugin/util/RetryPolicyUtil.java b/database-commons/src/main/java/io/cdap/plugin/util/RetryPolicyUtil.java index 40f75f74e..91d6bd6ba 100644 --- a/database-commons/src/main/java/io/cdap/plugin/util/RetryPolicyUtil.java +++ b/database-commons/src/main/java/io/cdap/plugin/util/RetryPolicyUtil.java @@ -20,11 +20,9 @@ import dev.failsafe.RetryPolicy; import io.cdap.cdap.api.Config; import io.cdap.plugin.db.RetryExceptions; -import io.cdap.plugin.db.connector.AbstractDBConnectorConfig; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.sql.SQLTransientConnectionException; import java.time.Duration; /** diff --git a/database-commons/src/main/java/io/cdap/plugin/util/RetryUtils.java b/database-commons/src/main/java/io/cdap/plugin/util/RetryUtils.java new file mode 100644 index 000000000..93170ddd9 --- /dev/null +++ b/database-commons/src/main/java/io/cdap/plugin/util/RetryUtils.java @@ -0,0 +1,141 @@ +/* + * Copyright © 2025 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + + +package io.cdap.plugin.util; + +import com.google.common.base.Strings; +import dev.failsafe.Failsafe; +import dev.failsafe.FailsafeException; +import dev.failsafe.RetryPolicy; +import io.cdap.cdap.api.exception.ErrorCategory; +import io.cdap.cdap.api.exception.ErrorCodeType; +import io.cdap.cdap.api.exception.ErrorType; +import io.cdap.cdap.api.exception.ErrorUtils; +import io.cdap.cdap.api.exception.ProgramFailureException; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Properties; + +/** + * Utility class for retrieving common methods using {@link dev.failsafe.RetryPolicy} + */ +public final class RetryUtils { + + public static Connection createConnectionWithRetry(RetryPolicy retryPolicy, String connectionString, + Properties connectionProperties, String externalDocumentationLink) { + try { + return Failsafe.with(retryPolicy).get(() -> DriverManager + .getConnection(connectionString, connectionProperties) + ); + } catch (Exception e) { + throw unwrapFailsafeException(e, externalDocumentationLink); + } + } + + public static Statement createStatementWithRetry(RetryPolicy retryPolicy, Connection connection, + String externalDocumentationLink) { + try { + return Failsafe.with(retryPolicy).get(connection::createStatement); + } catch (Exception e) { + throw unwrapFailsafeException(e, externalDocumentationLink); + } + } + + public static PreparedStatement prepareStatementWithRetry(RetryPolicy retryPolicy, + Connection connection, String sqlQuery, String externalDocumentationLink) { + try { + return Failsafe.with(retryPolicy).get(() -> connection.prepareStatement(sqlQuery)); + } catch (Exception e) { + throw unwrapFailsafeException(e, externalDocumentationLink); + } + } + + public static ResultSet executeQueryWithRetry(RetryPolicy retryPolicy, + PreparedStatement preparedStatement, String externalDocumentationLink) { + try { + return Failsafe.with(retryPolicy).get(() -> preparedStatement.executeQuery()); + } catch (Exception e) { + throw unwrapFailsafeException(e, externalDocumentationLink); + } + } + + public static ResultSet executeQueryWithRetry(RetryPolicy retryPolicy, Statement statement, + String query, String externalDocumentationLink) { + try { + return Failsafe.with(retryPolicy).get(() -> statement.executeQuery(query)); + } catch (Exception e) { + throw unwrapFailsafeException(e, externalDocumentationLink); + } + } + + public static void executeInitQueryWithRetry(RetryPolicy retryPolicy, Statement statement, String query, + String externalDocumentationLink) { + try { + Failsafe.with(retryPolicy).run(() -> statement.execute(query)); + } catch (Exception e) { + throw unwrapFailsafeException(e, externalDocumentationLink); + } + } + + private static RuntimeException unwrapFailsafeException(Exception e, String externalDocumentationLink) { + if (e instanceof FailsafeException) { + Throwable cause = e.getCause(); + if (cause instanceof SQLException) { + return programFailureException((SQLException) cause, externalDocumentationLink); + } else if (cause instanceof RuntimeException) { + return (RuntimeException) cause; + } else if (cause instanceof Error) { + return new RuntimeException("Failsafe wrapped an Error", cause); + } else { + return new RuntimeException("Failsafe wrapped a non-runtime exception", cause); + } + } + if (e instanceof SQLException) { + return programFailureException((SQLException) e, externalDocumentationLink); + } + if (e instanceof RuntimeException) { + return (RuntimeException) e; + } + return new RuntimeException("Unexpected checked exception", e); + } + + private static ProgramFailureException programFailureException(SQLException e, String externalDocumentationLink) { + // wrap exception to ensure SQLException-child instances not exposed to contexts without jdbc + // driver in classpath + String errorMessage = + String.format("SQL Exception occurred: [Message='%s', SQLState='%s', ErrorCode='%s'].", e.getMessage(), + e.getSQLState(), e.getErrorCode()); + String errorMessageWithDetails = String.format("Error occurred while trying to" + + " get schema from database." + "Error message: '%s'. Error code: '%s'. SQLState: '%s'", e.getMessage(), + e.getErrorCode(), e.getSQLState()); + + if (!Strings.isNullOrEmpty(externalDocumentationLink)) { + if (!errorMessage.endsWith(".")) { + errorMessage = errorMessage + "."; + } + errorMessage = String.format("%s For more details, see %s", errorMessage, externalDocumentationLink); + } + return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), + errorMessage, errorMessageWithDetails, ErrorType.USER, false, ErrorCodeType.SQLSTATE, e.getSQLState(), + externalDocumentationLink, e); + } +} diff --git a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlConnectorUnitTest.java b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlConnectorUnitTest.java index 80ca78d51..df731d194 100644 --- a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlConnectorUnitTest.java +++ b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlConnectorUnitTest.java @@ -28,7 +28,10 @@ public class MysqlConnectorUnitTest { @Rule public ExpectedException expectedEx = ExpectedException.none(); - private static final MysqlConnector CONNECTOR = new MysqlConnector(null); + private static final MysqlConnectorConfig CONFIG = + new MysqlConnectorConfig("host", 3306, "user", "password", "mysql", null); + + private static final MysqlConnector CONNECTOR = new MysqlConnector(CONFIG); /** * Unit test for getTableName() From fc78d9c6acca9e40c5f0665c26314a28e653cac5 Mon Sep 17 00:00:00 2001 From: Sanchit Garg Date: Tue, 3 Jun 2025 20:50:27 +0530 Subject: [PATCH 06/28] Removed unused method and extra line --- .../plugin/db/source/AbstractDBSource.java | 29 ------------------- .../java/io/cdap/plugin/util/RetryUtils.java | 1 - 2 files changed, 30 deletions(-) diff --git a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java index a87e597e2..834058124 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java @@ -18,8 +18,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Strings; -import dev.failsafe.Failsafe; -import dev.failsafe.FailsafeException; import dev.failsafe.RetryPolicy; import io.cdap.cdap.api.annotation.Description; import io.cdap.cdap.api.annotation.Macro; @@ -28,11 +26,6 @@ import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; import io.cdap.cdap.api.dataset.lib.KeyValue; -import io.cdap.cdap.api.exception.ErrorCategory; -import io.cdap.cdap.api.exception.ErrorCodeType; -import io.cdap.cdap.api.exception.ErrorType; -import io.cdap.cdap.api.exception.ErrorUtils; -import io.cdap.cdap.api.exception.ProgramFailureException; import io.cdap.cdap.api.plugin.PluginConfig; import io.cdap.cdap.etl.api.Emitter; import io.cdap.cdap.etl.api.FailureCollector; @@ -219,28 +212,6 @@ private void executeInitQueries(Connection connection, List initQueries) } } - private ProgramFailureException wrapException(SQLException e) { - // wrap exception to ensure SQLException-child instances not exposed to contexts without jdbc - // driver in classpath - String errorMessage = - String.format("SQL Exception occurred: [Message='%s', SQLState='%s', ErrorCode='%s'].", - e.getMessage(), e.getSQLState(), e.getErrorCode()); - String errorMessageWithDetails = String.format("Error occurred while trying to" + - " get schema from database." + "Error message: '%s'. Error code: '%s'. SQLState: '%s'", e.getMessage(), - e.getErrorCode(), e.getSQLState()); - String externalDocumentationLink = getExternalDocumentationLink(); - if (!Strings.isNullOrEmpty(externalDocumentationLink)) { - if (!errorMessage.endsWith(".")) { - errorMessage = errorMessage + "."; - } - errorMessage = String.format("%s For more details, see %s", errorMessage, externalDocumentationLink); - } - return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), - errorMessage, errorMessageWithDetails, ErrorType.USER, false, ErrorCodeType.SQLSTATE, e.getSQLState(), - externalDocumentationLink, new SQLException(e.getMessage(), e.getSQLState(), e.getErrorCode())); - - } - protected SchemaReader getSchemaReader() { return new CommonSchemaReader(); } diff --git a/database-commons/src/main/java/io/cdap/plugin/util/RetryUtils.java b/database-commons/src/main/java/io/cdap/plugin/util/RetryUtils.java index 93170ddd9..c7371673a 100644 --- a/database-commons/src/main/java/io/cdap/plugin/util/RetryUtils.java +++ b/database-commons/src/main/java/io/cdap/plugin/util/RetryUtils.java @@ -14,7 +14,6 @@ * the License. */ - package io.cdap.plugin.util; import com.google.common.base.Strings; From 61232a45ae2325976fe3a9a6dc3228db8c7567ba Mon Sep 17 00:00:00 2001 From: Sanchit Garg Date: Wed, 4 Jun 2025 20:45:06 +0530 Subject: [PATCH 07/28] PLUGIN-1823: Fix failing unit tests --- .../plugin/amazon/redshift/RedshiftConnectorUnitTest.java | 4 +++- .../amazon/redshift/RedshiftFailedConnectionTest.java | 3 +-- .../io/cdap/plugin/mssql/SqlServerConnectorUnitTest.java | 6 +++++- .../io/cdap/plugin/mysql/MysqlFailedConnectionTest.java | 4 ++-- .../java/io/cdap/plugin/oracle/OracleConnectorUnitTest.java | 3 ++- .../io/cdap/plugin/postgres/PostgresConnectorUnitTest.java | 4 +++- .../cdap/plugin/postgres/PostgresFailedConnectionTest.java | 4 ++-- 7 files changed, 18 insertions(+), 10 deletions(-) diff --git a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorUnitTest.java b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorUnitTest.java index 47e8b0a52..94ea6004b 100644 --- a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorUnitTest.java +++ b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftConnectorUnitTest.java @@ -28,7 +28,9 @@ public class RedshiftConnectorUnitTest { @Rule public ExpectedException expectedEx = ExpectedException.none(); - private static final RedshiftConnector CONNECTOR = new RedshiftConnector(null); + private static final RedshiftConnector CONNECTOR = new RedshiftConnector(new RedshiftConnectorConfig( + "username", "password", "jdbc", "", "localhost", + "db", 5432)); /** * Unit test for getTableName() diff --git a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftFailedConnectionTest.java b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftFailedConnectionTest.java index 2d21c4478..93b5d082e 100644 --- a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftFailedConnectionTest.java +++ b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftFailedConnectionTest.java @@ -32,7 +32,6 @@ public void test() throws ClassNotFoundException, IOException { super.test(JDBC_DRIVER_CLASS_NAME, connector, "Failed to create connection to database via connection string: " + "jdbc:redshift://localhost:5432/db and arguments: " + - "{user=username}. Error: ConnectException: Connection refused " + - "(Connection refused)."); + "{user=username}. Error: SQLException: The server does not support SSL.."); } } diff --git a/mssql-plugin/src/test/java/io/cdap/plugin/mssql/SqlServerConnectorUnitTest.java b/mssql-plugin/src/test/java/io/cdap/plugin/mssql/SqlServerConnectorUnitTest.java index 2952e0518..0163cd55e 100644 --- a/mssql-plugin/src/test/java/io/cdap/plugin/mssql/SqlServerConnectorUnitTest.java +++ b/mssql-plugin/src/test/java/io/cdap/plugin/mssql/SqlServerConnectorUnitTest.java @@ -28,7 +28,11 @@ public class SqlServerConnectorUnitTest { @Rule public ExpectedException expectedEx = ExpectedException.none(); - private static final SqlServerConnector CONNECTOR = new SqlServerConnector(null); + private static final SqlServerConnector CONNECTOR = new SqlServerConnector( + new SqlServerConnectorConfig("localhost", 1433, "user", "password", + "sqlserver", "")); + + // private static final SqlServerConnector CONNECTOR = new SqlServerConnector(new SqlServerConnectorConfig()); /** * Unit tests for getTableQuery() diff --git a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlFailedConnectionTest.java b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlFailedConnectionTest.java index 5c4f35828..22d2b0ff7 100644 --- a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlFailedConnectionTest.java +++ b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlFailedConnectionTest.java @@ -35,7 +35,7 @@ public void test() throws ClassNotFoundException, IOException { "rewriteBatchedStatements=true, " + "connectTimeout=20000, tinyInt1isBit=false, " + "socketTimeout=20000}. Error: " + - "ConnectException: Connection refused (Connection refused)."); + "SQLException: Access denied for user 'username'@'localhost' (using password: YES)."); } @Test @@ -50,7 +50,7 @@ public void testWithUpdatedConnectionArguments() throws ClassNotFoundException, "rewriteBatchedStatements=true, " + "connectTimeout=30000, tinyInt1isBit=false, " + "socketTimeout=30000}. Error: " + - "ConnectException: Connection refused (Connection refused)."); + "SQLException: Access denied for user 'username'@'localhost' (using password: YES)."); } } diff --git a/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleConnectorUnitTest.java b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleConnectorUnitTest.java index 9ac8602bd..f89a6c1d7 100644 --- a/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleConnectorUnitTest.java +++ b/oracle-plugin/src/test/java/io/cdap/plugin/oracle/OracleConnectorUnitTest.java @@ -25,7 +25,8 @@ public class OracleConnectorUnitTest { @Rule public ExpectedException expectedEx = ExpectedException.none(); - private static final OracleConnector CONNECTOR = new OracleConnector(null); + private static final OracleConnector CONNECTOR = new OracleConnector(new OracleConnectorConfig("localhost", + 1521, "user", "password", "oracle", "", "testdb")); /** * Unit test for getTableName() diff --git a/postgresql-plugin/src/test/java/io/cdap/plugin/postgres/PostgresConnectorUnitTest.java b/postgresql-plugin/src/test/java/io/cdap/plugin/postgres/PostgresConnectorUnitTest.java index 4f1f53964..f286cf574 100644 --- a/postgresql-plugin/src/test/java/io/cdap/plugin/postgres/PostgresConnectorUnitTest.java +++ b/postgresql-plugin/src/test/java/io/cdap/plugin/postgres/PostgresConnectorUnitTest.java @@ -28,7 +28,9 @@ public class PostgresConnectorUnitTest { @Rule public ExpectedException expectedEx = ExpectedException.none(); - private static final PostgresConnector CONNECTOR = new PostgresConnector(null); + private static final PostgresConnector CONNECTOR = new PostgresConnector(new PostgresConnectorConfig( + "localhost", 5432, "user", "password", "postgresql", + "")); /** * Unit test for getTableName() diff --git a/postgresql-plugin/src/test/java/io/cdap/plugin/postgres/PostgresFailedConnectionTest.java b/postgresql-plugin/src/test/java/io/cdap/plugin/postgres/PostgresFailedConnectionTest.java index a9cc4ab80..ea2bbfd0d 100644 --- a/postgresql-plugin/src/test/java/io/cdap/plugin/postgres/PostgresFailedConnectionTest.java +++ b/postgresql-plugin/src/test/java/io/cdap/plugin/postgres/PostgresFailedConnectionTest.java @@ -32,7 +32,7 @@ public void test() throws ClassNotFoundException, IOException { super.test(JDBC_DRIVER_CLASS_NAME, connector, "Failed to create connection to database via connection string: " + "jdbc:postgresql://localhost:5432/null and arguments: " + - "{user=username}. Error: ConnectException: Connection refused " + - "(Connection refused)."); + "{user=username}. Error: PSQLException: FATAL: " + + "password authentication failed for user \"username\"."); } } From 6d4f62dd1a5a2c95f146f6fc0de15287a8336938 Mon Sep 17 00:00:00 2001 From: Sanchit Garg Date: Wed, 4 Jun 2025 21:06:50 +0530 Subject: [PATCH 08/28] PLUGIN-1823: Revert changes done to FailedConnectionTest.java --- .../plugin/amazon/redshift/RedshiftFailedConnectionTest.java | 3 ++- .../java/io/cdap/plugin/mysql/MysqlFailedConnectionTest.java | 4 ++-- .../io/cdap/plugin/postgres/PostgresFailedConnectionTest.java | 4 ++-- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftFailedConnectionTest.java b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftFailedConnectionTest.java index 93b5d082e..2d21c4478 100644 --- a/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftFailedConnectionTest.java +++ b/amazon-redshift-plugin/src/test/java/io/cdap/plugin/amazon/redshift/RedshiftFailedConnectionTest.java @@ -32,6 +32,7 @@ public void test() throws ClassNotFoundException, IOException { super.test(JDBC_DRIVER_CLASS_NAME, connector, "Failed to create connection to database via connection string: " + "jdbc:redshift://localhost:5432/db and arguments: " + - "{user=username}. Error: SQLException: The server does not support SSL.."); + "{user=username}. Error: ConnectException: Connection refused " + + "(Connection refused)."); } } diff --git a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlFailedConnectionTest.java b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlFailedConnectionTest.java index 22d2b0ff7..5c4f35828 100644 --- a/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlFailedConnectionTest.java +++ b/mysql-plugin/src/test/java/io/cdap/plugin/mysql/MysqlFailedConnectionTest.java @@ -35,7 +35,7 @@ public void test() throws ClassNotFoundException, IOException { "rewriteBatchedStatements=true, " + "connectTimeout=20000, tinyInt1isBit=false, " + "socketTimeout=20000}. Error: " + - "SQLException: Access denied for user 'username'@'localhost' (using password: YES)."); + "ConnectException: Connection refused (Connection refused)."); } @Test @@ -50,7 +50,7 @@ public void testWithUpdatedConnectionArguments() throws ClassNotFoundException, "rewriteBatchedStatements=true, " + "connectTimeout=30000, tinyInt1isBit=false, " + "socketTimeout=30000}. Error: " + - "SQLException: Access denied for user 'username'@'localhost' (using password: YES)."); + "ConnectException: Connection refused (Connection refused)."); } } diff --git a/postgresql-plugin/src/test/java/io/cdap/plugin/postgres/PostgresFailedConnectionTest.java b/postgresql-plugin/src/test/java/io/cdap/plugin/postgres/PostgresFailedConnectionTest.java index ea2bbfd0d..a9cc4ab80 100644 --- a/postgresql-plugin/src/test/java/io/cdap/plugin/postgres/PostgresFailedConnectionTest.java +++ b/postgresql-plugin/src/test/java/io/cdap/plugin/postgres/PostgresFailedConnectionTest.java @@ -32,7 +32,7 @@ public void test() throws ClassNotFoundException, IOException { super.test(JDBC_DRIVER_CLASS_NAME, connector, "Failed to create connection to database via connection string: " + "jdbc:postgresql://localhost:5432/null and arguments: " + - "{user=username}. Error: PSQLException: FATAL: " + - "password authentication failed for user \"username\"."); + "{user=username}. Error: ConnectException: Connection refused " + + "(Connection refused)."); } } From 037a0d4fc724a108d41966303ef96f14940b661d Mon Sep 17 00:00:00 2001 From: Sanchit Garg Date: Thu, 5 Jun 2025 12:51:15 +0530 Subject: [PATCH 09/28] PLUGIN-1823: Rework Moved all the retry constants to RetryUtils class, Added the methods to determine error type and error category methods from Error code and SQL State --- .../io/cdap/plugin/db/ConnectionConfig.java | 19 +- .../connector/AbstractDBConnectorConfig.java | 19 +- .../plugin/db/source/AbstractDBSource.java | 15 +- .../java/io/cdap/plugin/util/RetryUtils.java | 177 +++++++++++++++++- 4 files changed, 195 insertions(+), 35 deletions(-) diff --git a/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java index 53ac81237..2a5aaa988 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/ConnectionConfig.java @@ -24,6 +24,7 @@ import io.cdap.cdap.api.plugin.PluginConfig; import io.cdap.plugin.common.KeyValueListParser; import io.cdap.plugin.db.config.DatabaseConnectionConfig; +import io.cdap.plugin.util.RetryUtils; import java.util.Collections; import java.util.HashMap; @@ -45,12 +46,6 @@ public abstract class ConnectionConfig extends PluginConfig implements DatabaseC public static final String CONNECTION_ARGUMENTS = "connectionArguments"; public static final String JDBC_PLUGIN_NAME = "jdbcPluginName"; public static final String JDBC_PLUGIN_TYPE = "jdbc"; - private static final String NAME_INITIAL_RETRY_DURATION = "initialRetryDuration"; - private static final String NAME_MAX_RETRY_DURATION = "maxRetryDuration"; - private static final String NAME_MAX_RETRY_COUNT = "maxRetryCount"; - public static final int DEFAULT_INITIAL_RETRY_DURATION_SECONDS = 5; - public static final int DEFAULT_MAX_RETRY_COUNT = 5; - public static final int DEFAULT_MAX_RETRY_DURATION_SECONDS = 80; public static final String TRANSACTION_ISOLATION_LEVEL = "transactionIsolationLevel"; @Name(JDBC_PLUGIN_NAME) @@ -78,19 +73,19 @@ public abstract class ConnectionConfig extends PluginConfig implements DatabaseC @Macro public String connectionArguments; - @Name(NAME_INITIAL_RETRY_DURATION) + @Name(RetryUtils.NAME_INITIAL_RETRY_DURATION) @Description("Time taken for the first retry. Default is 5 seconds.") @Nullable @Macro private Integer initialRetryDuration; - @Name(NAME_MAX_RETRY_DURATION) + @Name(RetryUtils.NAME_MAX_RETRY_DURATION) @Description("Maximum time in seconds retries can take. Default is 80 seconds.") @Nullable @Macro private Integer maxRetryDuration; - @Name(NAME_MAX_RETRY_COUNT) + @Name(RetryUtils.NAME_MAX_RETRY_COUNT) @Description("Maximum number of retries allowed. Default is 5.") @Nullable @Macro @@ -98,15 +93,15 @@ public abstract class ConnectionConfig extends PluginConfig implements DatabaseC public Integer getInitialRetryDuration() { - return initialRetryDuration == null ? DEFAULT_INITIAL_RETRY_DURATION_SECONDS : initialRetryDuration; + return initialRetryDuration == null ? RetryUtils.DEFAULT_INITIAL_RETRY_DURATION_SECONDS : initialRetryDuration; } public Integer getMaxRetryDuration() { - return maxRetryDuration == null ? DEFAULT_MAX_RETRY_DURATION_SECONDS : maxRetryDuration; + return maxRetryDuration == null ? RetryUtils.DEFAULT_MAX_RETRY_DURATION_SECONDS : maxRetryDuration; } public Integer getMaxRetryCount() { - return maxRetryCount == null ? DEFAULT_MAX_RETRY_COUNT : maxRetryCount; + return maxRetryCount == null ? RetryUtils.DEFAULT_MAX_RETRY_COUNT : maxRetryCount; } public ConnectionConfig() { diff --git a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBConnectorConfig.java b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBConnectorConfig.java index 3423440d0..646c5e388 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBConnectorConfig.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBConnectorConfig.java @@ -25,6 +25,7 @@ import io.cdap.plugin.common.KeyValueListParser; import io.cdap.plugin.common.db.DBConnectorProperties; import io.cdap.plugin.db.ConnectionConfig; +import io.cdap.plugin.util.RetryUtils; import java.util.Collections; import java.util.HashMap; @@ -36,12 +37,6 @@ * */ public abstract class AbstractDBConnectorConfig extends PluginConfig implements DBConnectorProperties { - private static final String NAME_INITIAL_RETRY_DURATION = "initialRetryDuration"; - private static final String NAME_MAX_RETRY_DURATION = "maxRetryDuration"; - private static final String NAME_MAX_RETRY_COUNT = "maxRetryCount"; - public static final int DEFAULT_INITIAL_RETRY_DURATION_SECONDS = 5; - public static final int DEFAULT_MAX_RETRY_COUNT = 5; - public static final int DEFAULT_MAX_RETRY_DURATION_SECONDS = 80; @Name(ConnectionConfig.JDBC_PLUGIN_NAME) @Description("Name of the JDBC driver to use. This is the value of the 'jdbcPluginName' key defined in the JSON " + @@ -70,19 +65,19 @@ public abstract class AbstractDBConnectorConfig extends PluginConfig implements protected String connectionArguments; - @Name(NAME_INITIAL_RETRY_DURATION) + @Name(RetryUtils.NAME_INITIAL_RETRY_DURATION) @Description("Time taken for the first retry. Default is 5 seconds.") @Nullable @Macro private Integer initialRetryDuration; - @Name(NAME_MAX_RETRY_DURATION) + @Name(RetryUtils.NAME_MAX_RETRY_DURATION) @Description("Maximum time in seconds retries can take. Default is 80 seconds.") @Nullable @Macro private Integer maxRetryDuration; - @Name(NAME_MAX_RETRY_COUNT) + @Name(RetryUtils.NAME_MAX_RETRY_COUNT) @Description("Maximum number of retries allowed. Default is 5.") @Nullable @Macro @@ -102,15 +97,15 @@ public String getPassword() { } public Integer getInitialRetryDuration() { - return initialRetryDuration == null ? DEFAULT_INITIAL_RETRY_DURATION_SECONDS : initialRetryDuration; + return initialRetryDuration == null ? RetryUtils.DEFAULT_INITIAL_RETRY_DURATION_SECONDS : initialRetryDuration; } public Integer getMaxRetryDuration() { - return maxRetryDuration == null ? DEFAULT_MAX_RETRY_DURATION_SECONDS : maxRetryDuration; + return maxRetryDuration == null ? RetryUtils.DEFAULT_MAX_RETRY_DURATION_SECONDS : maxRetryDuration; } public Integer getMaxRetryCount() { - return maxRetryCount == null ? DEFAULT_MAX_RETRY_COUNT : maxRetryCount; + return maxRetryCount == null ? RetryUtils.DEFAULT_MAX_RETRY_COUNT : maxRetryCount; } @Override diff --git a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java index 834058124..3e413f078 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java @@ -168,13 +168,14 @@ public Schema getSchema() throws SQLException { } private Schema loadSchemaFromDB(Connection connection, String query) throws SQLException { - Statement statement = RetryUtils.createStatementWithRetry((RetryPolicy) retryPolicy, connection, - getExternalDocumentationLink()); - statement.setMaxRows(1); - String finalQuery = query.contains("$CONDITIONS") ? removeConditionsClause(query) : query; - ResultSet resultSet = RetryUtils.executeQueryWithRetry((RetryPolicy) retryPolicy, statement, - finalQuery, getExternalDocumentationLink()); - return Schema.recordOf("outputSchema", getSchemaReader().getSchemaFields(resultSet)); + try (Statement statement = RetryUtils.createStatementWithRetry((RetryPolicy) retryPolicy, connection, + getExternalDocumentationLink())) { + statement.setMaxRows(1); + String finalQuery = query.contains("$CONDITIONS") ? removeConditionsClause(query) : query; + ResultSet resultSet = RetryUtils.executeQueryWithRetry((RetryPolicy) retryPolicy, statement, + finalQuery, getExternalDocumentationLink()); + return Schema.recordOf("outputSchema", getSchemaReader().getSchemaFields(resultSet)); + } } @VisibleForTesting diff --git a/database-commons/src/main/java/io/cdap/plugin/util/RetryUtils.java b/database-commons/src/main/java/io/cdap/plugin/util/RetryUtils.java index c7371673a..4a1b6902f 100644 --- a/database-commons/src/main/java/io/cdap/plugin/util/RetryUtils.java +++ b/database-commons/src/main/java/io/cdap/plugin/util/RetryUtils.java @@ -32,6 +32,8 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; +import java.util.HashMap; +import java.util.Map; import java.util.Properties; /** @@ -39,6 +41,137 @@ */ public final class RetryUtils { + private static final Map ERROR_CODE_TO_ERROR_TYPE; + private static final Map ERROR_CODE_TO_ERROR_CATEGORY; + public static final String NAME_INITIAL_RETRY_DURATION = "initialRetryDuration"; + public static final String NAME_MAX_RETRY_DURATION = "maxRetryDuration"; + public static final String NAME_MAX_RETRY_COUNT = "maxRetryCount"; + public static final int DEFAULT_INITIAL_RETRY_DURATION_SECONDS = 5; + public static final int DEFAULT_MAX_RETRY_COUNT = 5; + public static final int DEFAULT_MAX_RETRY_DURATION_SECONDS = 80; + + static { + // https://en.wikipedia.org/wiki/SQLSTATE + ERROR_CODE_TO_ERROR_TYPE = new HashMap<>(); + ERROR_CODE_TO_ERROR_TYPE.put("01", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("02", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("07", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("08", ErrorType.SYSTEM); + ERROR_CODE_TO_ERROR_TYPE.put("09", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("0A", ErrorType.SYSTEM); + ERROR_CODE_TO_ERROR_TYPE.put("0D", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("0E", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("0F", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("0K", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("0L", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("0M", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("0N", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("0P", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("0S", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("0T", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("0U", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("0V", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("0W", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("0X", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("0Y", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("0Z", ErrorType.SYSTEM); + ERROR_CODE_TO_ERROR_TYPE.put("10", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("20", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("21", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("22", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("23", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("24", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("25", ErrorType.SYSTEM); + ERROR_CODE_TO_ERROR_TYPE.put("26", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("27", ErrorType.SYSTEM); + ERROR_CODE_TO_ERROR_TYPE.put("28", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("2B", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("2C", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("2D", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("2E", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("2F", ErrorType.SYSTEM); + ERROR_CODE_TO_ERROR_TYPE.put("2H", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("30", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("33", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("34", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("35", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("36", ErrorType.SYSTEM); + ERROR_CODE_TO_ERROR_TYPE.put("38", ErrorType.SYSTEM); + ERROR_CODE_TO_ERROR_TYPE.put("39", ErrorType.SYSTEM); + ERROR_CODE_TO_ERROR_TYPE.put("3B", ErrorType.SYSTEM); + ERROR_CODE_TO_ERROR_TYPE.put("3C", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("3D", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("3F", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("40", ErrorType.SYSTEM); + ERROR_CODE_TO_ERROR_TYPE.put("42", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("44", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("45", ErrorType.USER); + ERROR_CODE_TO_ERROR_TYPE.put("46", ErrorType.SYSTEM); + ERROR_CODE_TO_ERROR_TYPE.put("HW", ErrorType.SYSTEM); + + ERROR_CODE_TO_ERROR_CATEGORY = new HashMap<>(); + ErrorCategory.ErrorCategoryEnum plugin = ErrorCategory.ErrorCategoryEnum.PLUGIN; + ERROR_CODE_TO_ERROR_CATEGORY.put("01", new ErrorCategory(plugin, "DB Warning")); + ERROR_CODE_TO_ERROR_CATEGORY.put("02", new ErrorCategory(plugin, "DB No Data")); + ERROR_CODE_TO_ERROR_CATEGORY.put("07", new ErrorCategory(plugin, "DB Dynamic SQL error")); + ERROR_CODE_TO_ERROR_CATEGORY.put("08", new ErrorCategory(plugin, "DB Connection Exception")); + ERROR_CODE_TO_ERROR_CATEGORY.put("09", new ErrorCategory(plugin, "DB Triggered Action Exception")); + ERROR_CODE_TO_ERROR_CATEGORY.put("0A", new ErrorCategory(plugin, "DB Feature Not Supported")); + ERROR_CODE_TO_ERROR_CATEGORY.put("0D", new ErrorCategory(plugin, "DB Invalid Target Type Specification")); + ERROR_CODE_TO_ERROR_CATEGORY.put("0E", new ErrorCategory(plugin, "DB Invalid Schema Name List Specification")); + ERROR_CODE_TO_ERROR_CATEGORY.put("0F", new ErrorCategory(plugin, "DB Locator Exception")); + ERROR_CODE_TO_ERROR_CATEGORY.put("0K", new ErrorCategory(plugin, "DB Resignal When Handler Not Active")); + ERROR_CODE_TO_ERROR_CATEGORY.put("0L", new ErrorCategory(plugin, "DB Invalid Grantor")); + ERROR_CODE_TO_ERROR_CATEGORY.put("0M", new ErrorCategory(plugin, "DB Invalid SQL-Invoked Procedure Reference")); + ERROR_CODE_TO_ERROR_CATEGORY.put("0N", new ErrorCategory(plugin, "DB SQL/XML Mapping Error")); + ERROR_CODE_TO_ERROR_CATEGORY.put("0P", new ErrorCategory(plugin, "DB Invalid Role Specification")); + ERROR_CODE_TO_ERROR_CATEGORY.put("0S", + new ErrorCategory(plugin, "DB Invalid Transform Group Name Specification")); + ERROR_CODE_TO_ERROR_CATEGORY.put("0T", + new ErrorCategory(plugin, "DB Target Table Disagrees With Cursor Specification")); + ERROR_CODE_TO_ERROR_CATEGORY.put("0U", + new ErrorCategory(plugin, "DB Attempt To Assign To Non-Updatable Column")); + ERROR_CODE_TO_ERROR_CATEGORY.put("0V", new ErrorCategory(plugin, "DB Attempt To Assign To Ordering Column")); + ERROR_CODE_TO_ERROR_CATEGORY.put("0W", new ErrorCategory(plugin, "DB Prohibited Statement Encountered")); + ERROR_CODE_TO_ERROR_CATEGORY.put("0X", new ErrorCategory(plugin, "DB Invalid Foreign Server Specification")); + ERROR_CODE_TO_ERROR_CATEGORY.put("0Y", new ErrorCategory(plugin, "DB Pass-Through Specific Condition")); + ERROR_CODE_TO_ERROR_CATEGORY.put("0Z", new ErrorCategory(plugin, "DB Diagnostics Exception")); + ERROR_CODE_TO_ERROR_CATEGORY.put("10", new ErrorCategory(plugin, "DB XQuery Error")); + ERROR_CODE_TO_ERROR_CATEGORY.put("20", new ErrorCategory(plugin, "DB Case Not Found For Case Statement")); + ERROR_CODE_TO_ERROR_CATEGORY.put("21", new ErrorCategory(plugin, "DB Cardinality Violation")); + ERROR_CODE_TO_ERROR_CATEGORY.put("22", new ErrorCategory(plugin, "DB Data Exception")); + ERROR_CODE_TO_ERROR_CATEGORY.put("23", new ErrorCategory(plugin, "DB Integrity Constraint Violation")); + ERROR_CODE_TO_ERROR_CATEGORY.put("24", new ErrorCategory(plugin, "DB Invalid Cursor State")); + ERROR_CODE_TO_ERROR_CATEGORY.put("25", new ErrorCategory(plugin, "DB Invalid Transaction State")); + ERROR_CODE_TO_ERROR_CATEGORY.put("26", new ErrorCategory(plugin, "DB Invalid SQL Statement Name")); + ERROR_CODE_TO_ERROR_CATEGORY.put("27", new ErrorCategory(plugin, "DB Triggered Data Change Violation")); + ERROR_CODE_TO_ERROR_CATEGORY.put("28", new ErrorCategory(plugin, "DB Invalid Authorization Specification")); + ERROR_CODE_TO_ERROR_CATEGORY.put("2B", + new ErrorCategory(plugin, "DB Dependent Privilege Descriptors Still Exist")); + ERROR_CODE_TO_ERROR_CATEGORY.put("2C", new ErrorCategory(plugin, "DB Invalid Character Set Name")); + ERROR_CODE_TO_ERROR_CATEGORY.put("2D", new ErrorCategory(plugin, "DB Invalid Transaction Termination")); + ERROR_CODE_TO_ERROR_CATEGORY.put("2E", new ErrorCategory(plugin, "DB Invalid Connection Name")); + ERROR_CODE_TO_ERROR_CATEGORY.put("2F", new ErrorCategory(plugin, "DB SQL Routine Exception")); + ERROR_CODE_TO_ERROR_CATEGORY.put("2H", new ErrorCategory(plugin, "DB Invalid Collation Name")); + ERROR_CODE_TO_ERROR_CATEGORY.put("30", new ErrorCategory(plugin, "DB Invalid SQL Statement Identifier")); + ERROR_CODE_TO_ERROR_CATEGORY.put("33", new ErrorCategory(plugin, "DB Invalid SQL Descriptor Name")); + ERROR_CODE_TO_ERROR_CATEGORY.put("34", new ErrorCategory(plugin, "DB Invalid Cursor Name")); + ERROR_CODE_TO_ERROR_CATEGORY.put("35", new ErrorCategory(plugin, "DB Invalid Condition Number")); + ERROR_CODE_TO_ERROR_CATEGORY.put("36", new ErrorCategory(plugin, "DB Cursor Sensitivity Exception")); + ERROR_CODE_TO_ERROR_CATEGORY.put("38", new ErrorCategory(plugin, "DB External Routine Exception")); + ERROR_CODE_TO_ERROR_CATEGORY.put("39", new ErrorCategory(plugin, "DB External Routine Invocation Exception")); + ERROR_CODE_TO_ERROR_CATEGORY.put("3B", new ErrorCategory(plugin, "DB Savepoint Exception")); + ERROR_CODE_TO_ERROR_CATEGORY.put("3C", new ErrorCategory(plugin, "DB Ambiguous Cursor Name")); + ERROR_CODE_TO_ERROR_CATEGORY.put("3D", new ErrorCategory(plugin, "DB Invalid Catalog Name")); + ERROR_CODE_TO_ERROR_CATEGORY.put("3F", new ErrorCategory(plugin, "DB Invalid Schema Name")); + ERROR_CODE_TO_ERROR_CATEGORY.put("40", new ErrorCategory(plugin, "DB Transaction Rollback")); + ERROR_CODE_TO_ERROR_CATEGORY.put("42", new ErrorCategory(plugin, "DB Syntax Error or Access Rule Violation")); + ERROR_CODE_TO_ERROR_CATEGORY.put("44", new ErrorCategory(plugin, "DB With Check Option Violation")); + ERROR_CODE_TO_ERROR_CATEGORY.put("45", new ErrorCategory(plugin, "DB Unhandled User-Defined Exception")); + ERROR_CODE_TO_ERROR_CATEGORY.put("46", new ErrorCategory(plugin, "DB JAVA DDL")); + ERROR_CODE_TO_ERROR_CATEGORY.put("HW", new ErrorCategory(plugin, "DB Datalink Exception")); + } + public static Connection createConnectionWithRetry(RetryPolicy retryPolicy, String connectionString, Properties connectionProperties, String externalDocumentationLink) { try { @@ -123,9 +256,11 @@ private static ProgramFailureException programFailureException(SQLException e, S String errorMessage = String.format("SQL Exception occurred: [Message='%s', SQLState='%s', ErrorCode='%s'].", e.getMessage(), e.getSQLState(), e.getErrorCode()); + String sqlState = e.getSQLState(); + int errorCode = e.getErrorCode(); String errorMessageWithDetails = String.format("Error occurred while trying to" + " get schema from database." + "Error message: '%s'. Error code: '%s'. SQLState: '%s'", e.getMessage(), - e.getErrorCode(), e.getSQLState()); + errorCode, sqlState); if (!Strings.isNullOrEmpty(externalDocumentationLink)) { if (!errorMessage.endsWith(".")) { @@ -133,8 +268,42 @@ private static ProgramFailureException programFailureException(SQLException e, S } errorMessage = String.format("%s For more details, see %s", errorMessage, externalDocumentationLink); } - return ErrorUtils.getProgramFailureException(new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN), - errorMessage, errorMessageWithDetails, ErrorType.USER, false, ErrorCodeType.SQLSTATE, e.getSQLState(), - externalDocumentationLink, e); + return ErrorUtils.getProgramFailureException(Strings.isNullOrEmpty(sqlState) ? + new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN) : getErrorCategoryFromSqlState(sqlState), + errorMessage, errorMessageWithDetails, getErrorTypeFromErrorCodeAndSqlState(errorCode, sqlState), true, + ErrorCodeType.SQLSTATE, e.getSQLState(), externalDocumentationLink, e); + } + + /** + * Get the {@link ErrorCategory} for the given SQL state. + * Implements generic error categories based on the SQL state. + * See SQLSTATE for more information. + * Override this method to provide custom error categories based on the SQL state. + * + * @param sqlState The SQL state. + * @return The {@link ErrorCategory} for the given SQL state. + */ + private static ErrorCategory getErrorCategoryFromSqlState(String sqlState) { + if (!Strings.isNullOrEmpty(sqlState) && sqlState.length() >= 2 && + ERROR_CODE_TO_ERROR_CATEGORY.containsKey(sqlState.substring(0, 2))) { + return ERROR_CODE_TO_ERROR_CATEGORY.get(sqlState.substring(0, 2)); + } + return new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN); + } + + /** + * Get the {@link ErrorType} for the given error code and SQL state. + * Override this method to provide custom error types based on the error code and SQL state. + * + * @param errorCode The error code. + * @param sqlState The SQL state. + * @return The {@link ErrorType} for the given error code and SQL state. + */ + private static ErrorType getErrorTypeFromErrorCodeAndSqlState(int errorCode, String sqlState) { + if (!Strings.isNullOrEmpty(sqlState) && sqlState.length() >= 2 && + ERROR_CODE_TO_ERROR_TYPE.containsKey(sqlState.substring(0, 2))) { + return ERROR_CODE_TO_ERROR_TYPE.get(sqlState.substring(0, 2)); + } + return ErrorType.UNKNOWN; } } From a2bd3a902d0c4575f44a19abd02947b3756b5787 Mon Sep 17 00:00:00 2001 From: Sanchit Garg Date: Thu, 5 Jun 2025 18:04:29 +0530 Subject: [PATCH 10/28] PLUGIN-1823: Revert SQLException handling changes for validateSchema --- .../cdap/plugin/db/sink/AbstractDBSink.java | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java index 624542d0b..acff4b282 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java @@ -39,7 +39,6 @@ import io.cdap.plugin.common.ReferenceBatchSink; import io.cdap.plugin.common.ReferencePluginConfig; import io.cdap.plugin.common.batch.sink.SinkOutputFormatProvider; -import io.cdap.plugin.common.util.ExceptionUtils; import io.cdap.plugin.db.ColumnType; import io.cdap.plugin.db.CommonSchemaReader; import io.cdap.plugin.db.ConnectionConfig; @@ -59,7 +58,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.sql.Connection; import java.sql.Driver; import java.sql.PreparedStatement; @@ -129,17 +127,8 @@ public void configurePipeline(PipelineConfigurer pipelineConfigurer) { Class driverClass = DBUtils.getDriverClass( pipelineConfigurer, dbSinkConfig, ConnectionConfig.JDBC_PLUGIN_TYPE); if (driverClass != null && dbSinkConfig.canConnect()) { - try { validateSchema(collector, driverClass, dbSinkConfig.getTableName(), inputSchema, dbSinkConfig.getDBSchemaName()); - } catch (SQLException e) { - String details = String.format("SQL error while validating the schema: Error: %s, SQLState: %s, ErrorCode: %s", - e.getMessage(), e.getSQLState(), e.getErrorCode()); - collector.addFailure(details, - "Ensure the table exists and the connection string points to a valid database.") - .withConfigProperty(DBSinkConfig.TABLE_NAME) - .withStacktrace(e.getStackTrace()); - } } } public void validateOperations(FailureCollector collector, T dbSinkConfig, @Nullable Schema inputSchema) { @@ -229,9 +218,6 @@ public void prepareRun(BatchSinkContext context) { } else { outputSchema = inferSchema(driverClass); } - } catch (SQLException e) { - throw new RuntimeException(String.format("Unable to validate schema due to: %s.", - ExceptionUtils.getRootCauseMessage(e)), e); } finally { DBUtils.cleanup(driverClass); } @@ -423,7 +409,7 @@ static List getMatchedColumnTypeList(ResultSet resultSet, List jdbcDriverClass, String tableName, - Schema inputSchema, String dbSchemaName) throws SQLException { + Schema inputSchema, String dbSchemaName) { String connectionString = dbSinkConfig.getConnectionString(); String fullyQualifiedTableName = dbSchemaName == null ? dbSinkConfig.getEscapedTableName() : dbSinkConfig.getEscapedDbSchemaName() + "." + dbSinkConfig.getEscapedTableName(); @@ -459,6 +445,13 @@ connectionString, connectionProperties, getExternalDocumentationLink())) { getExternalDocumentationLink())) { getFieldsValidator().validateFields(inputSchema, rs, collector); } + } catch (SQLException e) { + LOG.error("Exception while trying to validate schema of database table {} for connection {}.", + fullyQualifiedTableName, connectionString, e); + collector.addFailure( + String.format("Exception while trying to validate schema of database table '%s' for connection '%s' with %s", + fullyQualifiedTableName, connectionString, e.getMessage()), + null).withStacktrace(e.getStackTrace()); } } From 35537d702af0175647970a21ec764956694cbd94 Mon Sep 17 00:00:00 2001 From: Sanchit Garg Date: Thu, 5 Jun 2025 20:26:22 +0530 Subject: [PATCH 11/28] PLUGIN-1823: Add default value for external documentation --- .../plugin/db/action/AbstractDBArgumentSetter.java | 14 ++++++++++++-- .../io/cdap/plugin/db/sink/AbstractDBSink.java | 2 +- .../io/cdap/plugin/db/source/AbstractDBSource.java | 2 +- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java index 6d6c9d5ec..e20774dbb 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java @@ -114,10 +114,10 @@ private void processArguments(Class driverClass, private void executeWithRetry(FailureCollector failureCollector, SettableArguments settableArguments, Properties connectionProperties) throws SQLException { try (Connection connection = RetryUtils.createConnectionWithRetry((RetryPolicy) retryPolicy, - config.getConnectionString(), connectionProperties, null)) { + config.getConnectionString(), connectionProperties, getExternalDocumentationLink())) { ResultSet resultSet; try (Statement statement = RetryUtils.createStatementWithRetry((RetryPolicy) retryPolicy, connection, - null)) { + getExternalDocumentationLink())) { resultSet = RetryUtils.executeQueryWithRetry((RetryPolicy) retryPolicy, statement, config.getQuery(), null); } @@ -153,4 +153,14 @@ private void setArguments(ResultSet resultSet, FailureCollector failureCollector arguments.set(column, resultSet.getString(column)); } } + + /** + * Returns the external documentation link. + * Override this method to provide a custom external documentation link. + * + * @return external documentation link + */ + protected String getExternalDocumentationLink() { + return "https://en.wikipedia.org/wiki/SQLSTATE"; + } } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java index acff4b282..0f10fe579 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java @@ -187,7 +187,7 @@ protected String getErrorDetailsProviderClassName() { * @return external documentation link */ protected String getExternalDocumentationLink() { - return null; + return "https://en.wikipedia.org/wiki/SQLSTATE"; } @Override diff --git a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java index 3e413f078..c964b8fdb 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java @@ -368,7 +368,7 @@ protected Class getDBRecordType() { * @return external documentation link */ protected String getExternalDocumentationLink() { - return null; + return "https://en.wikipedia.org/wiki/SQLSTATE"; } @Override From b41551c9408a86f8539f07416394350c9452f4e7 Mon Sep 17 00:00:00 2001 From: suryakumari Date: Thu, 19 Jun 2025 22:28:38 +0530 Subject: [PATCH 12/28] error messages changes --- .../src/e2e-test/resources/errorMessage.properties | 4 ++-- .../features/mysqlsink/DesignTimeValidation.feature | 2 +- .../src/e2e-test/resources/errorMessage.properties | 7 ++++--- .../features/sink/OracleDesignTimeValidation.feature | 2 +- .../src/e2e-test/resources/errorMessage.properties | 12 ++++++------ .../src/e2e-test/resources/errorMessage.properties | 4 ++-- 6 files changed, 16 insertions(+), 15 deletions(-) diff --git a/cloudsql-postgresql-plugin/src/e2e-test/resources/errorMessage.properties b/cloudsql-postgresql-plugin/src/e2e-test/resources/errorMessage.properties index 7e9cd2337..b82f546e3 100644 --- a/cloudsql-postgresql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/cloudsql-postgresql-plugin/src/e2e-test/resources/errorMessage.properties @@ -15,11 +15,11 @@ errorMessageBlankPassword=SQL error while getting query schema: errorMessageInvalidPassword=SQL error while getting query schema: Error: FATAL: password authentication failed for user errorMessageInvalidSourceHost=SQL error while getting query schema: The connection attempt failed. errorMessageInvalidTableName=Table 'table' does not exist. Ensure table '"table"' is set correctly and that the -errorMessageInvalidSinkDatabase=Exception while trying to validate schema of database table +errorMessageInvalidSinkDatabase=Error occurred while trying to get schema from database.Error message: 'FATAL: database "invalidDB" does not exist'. Error code: '0'. SQLState: '3D000'' errorLogsMessageInvalidBoundingQuery=The column index is out of range: 1, number of columns: 0.. errorMessageConnectionName=Connection Name must be in the format :: to connect to \ a public CloudSQL PostgreSQL instance. errorMessagePrivateConnectionName=Enter the internal IP address of the Compute Engine VM cloudsql proxy is running on, \ to connect to a private CloudSQL PostgreSQL instance. -errorMessageWithBlankPassword=Exception while trying to validate schema of database table +errorMessageWithBlankPassword=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'Something unusual has occurred to cause the driver to fail. Please report this exception.'. Error code: '0'. SQLState: '99999'' errorMessageUpdateUpsertOperationName=Table key must be set if the operation is 'Update' or 'Upsert'. diff --git a/mysql-plugin/src/e2e-test/features/mysqlsink/DesignTimeValidation.feature b/mysql-plugin/src/e2e-test/features/mysqlsink/DesignTimeValidation.feature index 2c7050d08..effa33496 100644 --- a/mysql-plugin/src/e2e-test/features/mysqlsink/DesignTimeValidation.feature +++ b/mysql-plugin/src/e2e-test/features/mysqlsink/DesignTimeValidation.feature @@ -254,4 +254,4 @@ Feature: MySQL Sink - Design time validation scenarios Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields Then Enter input plugin property: "referenceName" with value: "targetRef" Then Click on the Validate button - Then Verify that the Plugin is displaying an error message: "blank.connection.message" on the header + Then Verify that the Plugin is displaying an error message: "blank.HostConnection.message" on the header diff --git a/mysql-plugin/src/e2e-test/resources/errorMessage.properties b/mysql-plugin/src/e2e-test/resources/errorMessage.properties index 4a7188ff8..0cf1dd2b5 100644 --- a/mysql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/mysql-plugin/src/e2e-test/resources/errorMessage.properties @@ -2,17 +2,18 @@ validationSuccessMessage=No errors found. invalid.username.message=SQL error while getting query schema: invalidtableName.error.message=Table '123#' does not exist. Ensure table '123#' is set correctly invalidreferenceName.error.message=Invalid reference name -invalid.host.message=Exception while trying to validate schema of database table +invalid.host.message=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'Communications link failure The last packet sent successfully to the server was 0 milliseconds ago. The driver has not received any packets from the server.'. Error code: '0'. SQLState: '08S01'' invalid.password.message=SQL error while getting query schema: invalid.databasename.message=SQL error while getting query schema: invalid.query.message=SQL error while getting query schema: numberofsplits.error.message=Unable to create config for batchsource Mysql 'numSplits' is invalid boundingQuery.error.message=Bounding Query must be specified if Number of Splits is not set to 1. splitfield.error.message=Split-By Field Name must be specified if Number of Splits is not set to 1 -invalid.sink.database.message=Exception while trying to validate schema +invalid.sink.database.message=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'Unknown database 'test123''. Error code: '1049'. SQLState: '42000'' blank.username.message=Username is required when password is given. errorMessageInvalidFetchSize=Invalid fetch size. Fetch size must be a positive integer. errorMessageInvalidNumberOfSplits=Invalid value for Number of Splits '0'. Must be at least 1. errorMessageInvalidImportQuery=Import Query select must contain the string '$CONDITIONS'. -blank.connection.message=Exception while trying to validate schema of database table +blank.connection.message=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'Access denied for user +blank.HostConnection.message=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'Communications link failure The last packet sent successfully to the server was 0 milliseconds ago. The driver has not received any packets from the server.'. Error code: '0'. SQLState: '08S01'' blank.jdbcPluginName.message=Required property 'jdbcPluginName' has no value. diff --git a/oracle-plugin/src/e2e-test/features/sink/OracleDesignTimeValidation.feature b/oracle-plugin/src/e2e-test/features/sink/OracleDesignTimeValidation.feature index 6ba3f92dd..5b69fbbb2 100644 --- a/oracle-plugin/src/e2e-test/features/sink/OracleDesignTimeValidation.feature +++ b/oracle-plugin/src/e2e-test/features/sink/OracleDesignTimeValidation.feature @@ -260,5 +260,5 @@ Feature: Oracle sink- Verify Oracle sink plugin design time validation scenarios Then Select radio button plugin property: "connectionType" with value: "service" Then Select radio button plugin property: "role" with value: "normal" Then Click on the Validate button - Then Verify that the Plugin is displaying an error message: "blank.connection.message" on the header + Then Verify that the Plugin is displaying an error message: "blank.HostBlank.message" on the header diff --git a/oracle-plugin/src/e2e-test/resources/errorMessage.properties b/oracle-plugin/src/e2e-test/resources/errorMessage.properties index 620b91af8..14db938d2 100644 --- a/oracle-plugin/src/e2e-test/resources/errorMessage.properties +++ b/oracle-plugin/src/e2e-test/resources/errorMessage.properties @@ -7,15 +7,15 @@ errorMessageInvalidNumberOfSplits=Invalid value for Number of Splits '0'. Must b errorMessageNumberOfSplitNotNumber=Unable to create config for batchsource Oracle 'numSplits' is invalid: Value of field\ \ class io.cdap.plugin.db.config.AbstractDBSpecificSourceConfig.numSplits is expected to be a number. errorMessageInvalidFetchSize=Invalid fetch size. Fetch size must be a positive integer. -errorMessageInvalidSourceDatabase=SQL error while getting query schema: Error: Listener refused the connection with the following error: ORA-12514, \ - TNS:listener does not currently know of service requested in connect descriptor , SQLState: 08006, ErrorCode: 12514 +errorMessageInvalidSourceDatabase=Error occurred while trying to get schema from database.Error message: 'Listener refused the connection with the following error: ORA-12514, TNS:listener does not currently know of service requested in connect descriptor '. Error code: '12514'. SQLState: '08006' errorMessageInvalidImportQuery=Import Query select must contain the string '$CONDITIONS'. if Number of Splits is not set\ \ to 1. Include '$CONDITIONS' in the Import Query errorMessageBlankUsername=Username is required when password is given. -errorMessageInvalidTableName=Exception while trying to validate schema of database table '"table"' for connection -errorMessageInvalidSinkDatabase=Exception while trying to validate schema of database table '"TARGETTABLE_ -errorMessageInvalidHost=Exception while trying to validate schema of database table '"table"' for connection +errorMessageInvalidTableName=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'IO Error: Unknown host specified '. Error code: '17002'. SQLState: '08006'' +errorMessageInvalidSinkDatabase=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'IO Error: Unknown host specified '. Error code: '17002'. SQLState: '08006'' +errorMessageInvalidHost=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'IO Error: Unknown host specified '. Error code: '17002'. SQLState: '08006'' errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: Stage 'Oracle' encountered : \ java.io.IOException: ORA-00936: missing expression . Please check the system logs for more details. blank.database.message=Required property 'database' has no value. -blank.connection.message=Exception while trying to validate schema of database table +blank.connection.message=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'ORA-01005: null password given; logon denied '. Error code: '1005'. SQLState: '72000'' +blank.HostBlank.message=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'IO Error: The Network Adapter could not establish the connection'. Error code: '17002'. SQLState: '08006'' \ No newline at end of file diff --git a/postgresql-plugin/src/e2e-test/resources/errorMessage.properties b/postgresql-plugin/src/e2e-test/resources/errorMessage.properties index f793e3be7..3d6f0484b 100644 --- a/postgresql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/postgresql-plugin/src/e2e-test/resources/errorMessage.properties @@ -16,7 +16,7 @@ errorMessageBlankPassword=SQL error while getting query schema: The server reque errorMessageInvalidPassword=SQL error while getting query schema: FATAL: password authentication failed for user errorMessageInvalidSourceHost=SQL error while getting query schema: The connection attempt failed. errorMessageInvalidTableName=Table 'table' does not exist. Ensure table '"table"' is set correctly and that the -errorMessageInvalidSinkDatabase=Exception while trying to validate schema of database table '"targettable_ -errorMessageInvalidHost=Exception while trying to validate schema of database table '"table"' for connection +errorMessageInvalidSinkDatabase='Error occurred while trying to get schema from database.Error message: 'FATAL: database "invalidDB" does not exist'. Error code: '0'. SQLState: '3D000'' +errorMessageInvalidHost=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'The connection attempt failed.'. Error code: '0'. SQLState: '08001'' errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: Stage 'PostgreSQL' encountered : \ java.io.IOException: The column index is out of range: 1, number of columns: 0.. Please check the system logs for more details. From ba524f862baf874ccc70939a422f62b473d13a08 Mon Sep 17 00:00:00 2001 From: suryakumari Date: Fri, 20 Jun 2025 12:04:39 +0530 Subject: [PATCH 13/28] cloudsqlmysql changes --- .../src/e2e-test/resources/errorMessage.properties | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties b/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties index 89a1b23df..60a8df927 100644 --- a/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties @@ -1,9 +1,9 @@ -errorMessageInvalidSourceDatabase=SQL error while getting query schema: Error: Unknown database 'invalidDatabase', SQLState: 42000, ErrorCode: 1049 +errorMessageInvalidSourceDatabase=Error occurred while trying to get schema from database.Error message: 'Access denied for user errorMessageInvalidImportQuery=Import Query select must contain the string '$CONDITIONS'. if Number of Splits is not set\ \ to 1. Include '$CONDITIONS' in the Import Query errorMessageCloudMySqlInvalidReferenceName=Invalid reference name errorMessageBlankUsername=Username is required when password is given. -errorMessageBlankPassword=SQL error while getting query schema: Error: Access denied for user +errorMessageBlankPassword=Error occurred while trying to get schema from database.Error message: 'Access denied for user errorMessageInvalidFetchSize=Invalid fetch size. Fetch size must be a positive integer. errorMessageBlankSplitBy=Split-By Field Name must be specified if Number of Splits is not set to 1. Specify the Split-by Field Name. errorMessageInvalidNumberOfSplits=Invalid value for Number of Splits '0'. Must be at least 1. Specify a Number of Splits no less than 1. @@ -17,10 +17,9 @@ validationSuccessMessage=No errors found. validationErrorMessage=COUNT ERROR found errorLogsMessageInvalidTableName=Spark program 'phase-1' failed with error: Stage 'CloudSQL MySQL' encountered : io.cdap.cdap.etl.api.validation.ValidationException: Errors were encountered during validation. \ Table 'Table123' does not exist.. Please check the system logs for more details. -errorLogsMessageInvalidCredentials =Spark program 'phase-1' failed with error: Stage 'CloudSQL MySQL' encountered : io.cdap.cdap.etl.api.validation.ValidationException: Errors were encountered during validation. \ - Exception while trying to validate schema of database table +errorLogsMessageInvalidCredentials =Spark program 'phase-1' failed with error: Stage 'CloudSQL MySQL' encountered : io.cdap.cdap.api.exception.ProgramFailureException: Error occurred while trying to get schema from database.Error message: 'Access denied for user 'testUser' errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: Stage 'CloudSQL MySQL' encountered : java.io.IOException: You have an error in your SQL syntax; \ check the manual that corresponds to your MySQL server version for the right syntax to use near 'table' at line 1. Please check the system logs for more details. -errorMessageInvalidPassword=SQL error while getting query schema: Error: Access denied for user +errorMessageInvalidPassword=Error occurred while trying to get schema from database.Error message: 'Access denied for user errorMessagePrivateConnectionName=Enter the internal IP address of the Compute Engine VM cloudsql proxy is running on, to connect to a private -errorMessageWithBlankPassword=Exception while trying to validate schema of database table +errorMessageWithBlankPassword=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'Access denied for user From 2d59b51c918b5d5a753908a2019a39a7204384f3 Mon Sep 17 00:00:00 2001 From: suryakumari Date: Fri, 20 Jun 2025 16:20:22 +0530 Subject: [PATCH 14/28] mysql changes --- .../src/e2e-test/resources/errorMessage.properties | 8 ++++---- .../src/e2e-test/resources/errorMessage.properties | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/mysql-plugin/src/e2e-test/resources/errorMessage.properties b/mysql-plugin/src/e2e-test/resources/errorMessage.properties index 0cf1dd2b5..fd0869726 100644 --- a/mysql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/mysql-plugin/src/e2e-test/resources/errorMessage.properties @@ -1,11 +1,11 @@ validationSuccessMessage=No errors found. -invalid.username.message=SQL error while getting query schema: +invalid.username.message=Error occurred while trying to get schema from database.Error message: 'Access denied for user ' invalidtableName.error.message=Table '123#' does not exist. Ensure table '123#' is set correctly invalidreferenceName.error.message=Invalid reference name invalid.host.message=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'Communications link failure The last packet sent successfully to the server was 0 milliseconds ago. The driver has not received any packets from the server.'. Error code: '0'. SQLState: '08S01'' -invalid.password.message=SQL error while getting query schema: -invalid.databasename.message=SQL error while getting query schema: -invalid.query.message=SQL error while getting query schema: +invalid.password.message=Error occurred while trying to get schema from database.Error message: 'Access denied for user ' +invalid.databasename.message=Error occurred while trying to get schema from database.Error message: 'Access denied for user ' +invalid.query.message=Error occurred while trying to get schema from database.Error message: 'No tables used'. numberofsplits.error.message=Unable to create config for batchsource Mysql 'numSplits' is invalid boundingQuery.error.message=Bounding Query must be specified if Number of Splits is not set to 1. splitfield.error.message=Split-By Field Name must be specified if Number of Splits is not set to 1 diff --git a/oracle-plugin/src/e2e-test/resources/errorMessage.properties b/oracle-plugin/src/e2e-test/resources/errorMessage.properties index 14db938d2..c7d9fd7f6 100644 --- a/oracle-plugin/src/e2e-test/resources/errorMessage.properties +++ b/oracle-plugin/src/e2e-test/resources/errorMessage.properties @@ -18,4 +18,4 @@ errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: java.io.IOException: ORA-00936: missing expression . Please check the system logs for more details. blank.database.message=Required property 'database' has no value. blank.connection.message=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'ORA-01005: null password given; logon denied '. Error code: '1005'. SQLState: '72000'' -blank.HostBlank.message=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'IO Error: The Network Adapter could not establish the connection'. Error code: '17002'. SQLState: '08006'' \ No newline at end of file +blank.HostBlank.message=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'IO Error: The Network Adapter could not establish the connection'. Error code: '17002'. SQLState: '08006'' From c31c021748cb9f8d4f55511d0e1d8e95b47aa6bf Mon Sep 17 00:00:00 2001 From: suryakumari Date: Mon, 23 Jun 2025 13:41:35 +0530 Subject: [PATCH 15/28] mssql changes --- .../src/e2e-test/resources/errorMessage.properties | 2 +- .../mssql/mssql sink/DesignTimeValidation.feature | 2 +- .../src/e2e-test/resources/errorMessage.properties | 9 +++++---- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties b/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties index 60a8df927..68a0e0484 100644 --- a/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties @@ -1,4 +1,4 @@ -errorMessageInvalidSourceDatabase=Error occurred while trying to get schema from database.Error message: 'Access denied for user +errorMessageInvalidSourceDatabase=Error occurred while trying to get schema from database.Error message: 'Access denied for user ' errorMessageInvalidImportQuery=Import Query select must contain the string '$CONDITIONS'. if Number of Splits is not set\ \ to 1. Include '$CONDITIONS' in the Import Query errorMessageCloudMySqlInvalidReferenceName=Invalid reference name diff --git a/mssql-plugin/src/e2e-test/features/mssql/mssql sink/DesignTimeValidation.feature b/mssql-plugin/src/e2e-test/features/mssql/mssql sink/DesignTimeValidation.feature index a3b751371..a14bcba5a 100644 --- a/mssql-plugin/src/e2e-test/features/mssql/mssql sink/DesignTimeValidation.feature +++ b/mssql-plugin/src/e2e-test/features/mssql/mssql sink/DesignTimeValidation.feature @@ -260,4 +260,4 @@ Feature: Mssql source- Verify Mssql source plugin design time validation scenari Then Replace input plugin property: "password" with value: "password" for Credentials and Authorization related fields Then Enter input plugin property: "referenceName" with value: "targetRef" Then Click on the Validate button - Then Verify that the Plugin is displaying an error message: "blank.connection.message" on the header + Then Verify that the Plugin is displaying an error message: "errormessageBlankHost" on the header diff --git a/mssql-plugin/src/e2e-test/resources/errorMessage.properties b/mssql-plugin/src/e2e-test/resources/errorMessage.properties index 4eb83c386..ceb700c57 100644 --- a/mssql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/mssql-plugin/src/e2e-test/resources/errorMessage.properties @@ -1,9 +1,9 @@ validationSuccessMessage=No errors found. -errorMessageInvalidSourceDatabase=SQL error while getting query schema: Error: Cannot open database "test123" requested by the login. The login failed. +errorMessageInvalidSourceDatabase=Error occurred while trying to get schema from database.Error message: 'Cannot open database "test123" requested by the login. The login failed. errorMessageInvalidImportQuery=Import Query select must contain the string '$CONDITIONS'. errorMessageMssqlInvalidReferenceName=Invalid reference name 'invalidRef&^*&&*'. errorMessageBlankUsername=Username is required when password is given. -errorMessageBlankPassword=SQL error while getting query schema: Error: Login failed for user +errorMessageBlankPassword=Error occurred while trying to get schema from database.Error message: 'Login failed for user errorMessageInvalidFetchSize=Invalid fetch size. Fetch size must be a positive integer. errorMessageBlankSplitBy=Split-By Field Name must be specified if Number of Splits is not set to 1. errorMessageInvalidNumberOfSplits=Invalid value for Number of Splits '0'. Must be at least 1. @@ -12,7 +12,8 @@ errorMessageBoundingQuery=Bounding Query must be specified if Number of Splits i errorMessagenumofSplit=Split-By Field Name must be specified if Number of Splits is not set to 1. errorMessageInvalidSinkDatabase=Exception while trying to validate schema of database table errorMessageInvalidSinkTableName=Table 'Table123@' does not exist. -errormessageBlankHost=Exception while trying to validate schema of database table +errormessageBlankHost=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'The TCP/IP connection to the host localhost, port 1433 has failed. Error: "Connection refused (Connection refused). \ + Verify the connection properties. Make sure that an instance of SQL Server is running on the host and accepting TCP/IP connections at the port. Make sure that TCP connections to the port are not blocked by a firewall.".'. Error code: '0'. SQLState: '08S01'' errorMessageInvalidTableName=Spark program 'phase-1' failed with error: Stage 'SQL Server2' encountered : io.cdap.cdap.etl.api.validation.ValidationException: \ Errors were encountered during validation. Table 'Table123@' does not exist.. Please check the system logs for more details. errorMessageInvalidCredentials=Spark program 'phase-1' failed with error: Unable to create config for batchsink SqlServer \ @@ -23,4 +24,4 @@ errorMessageInvalidCredentialSource=Spark program 'phase-1' failed with error: S Plugin with id SQL Server:source.jdbc.sqlserver does not exist in program phase-1 of application errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: Stage 'SQL Server' encountered : java.io.IOException: Could not find stored procedure blank.jdbcPluginName.message=Required property 'jdbcPluginName' has no value. -blank.connection.message=Exception while trying to validate schema of database table +blank.connection.message=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'Login failed for user ' From 852bcbaff06b4f1b2bd2e002a4c3874c2f8243cd Mon Sep 17 00:00:00 2001 From: suryakumari Date: Tue, 24 Jun 2025 14:14:55 +0530 Subject: [PATCH 16/28] cloudsqlpostgresql changes --- .../src/e2e-test/resources/errorMessage.properties | 2 +- .../src/e2e-test/resources/errorMessage.properties | 6 +++--- mssql-plugin/src/e2e-test/resources/errorMessage.properties | 2 +- mysql-plugin/src/e2e-test/resources/errorMessage.properties | 2 +- .../src/e2e-test/resources/errorMessage.properties | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties b/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties index 68a0e0484..4556e268c 100644 --- a/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties @@ -1,4 +1,4 @@ -errorMessageInvalidSourceDatabase=Error occurred while trying to get schema from database.Error message: 'Access denied for user ' +errorMessageInvalidSourceDatabase=Error occurred while trying to get schema from database. errorMessageInvalidImportQuery=Import Query select must contain the string '$CONDITIONS'. if Number of Splits is not set\ \ to 1. Include '$CONDITIONS' in the Import Query errorMessageCloudMySqlInvalidReferenceName=Invalid reference name diff --git a/cloudsql-postgresql-plugin/src/e2e-test/resources/errorMessage.properties b/cloudsql-postgresql-plugin/src/e2e-test/resources/errorMessage.properties index b82f546e3..1d7e23e01 100644 --- a/cloudsql-postgresql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/cloudsql-postgresql-plugin/src/e2e-test/resources/errorMessage.properties @@ -7,12 +7,12 @@ errorMessageInvalidNumberOfSplits=Invalid value for Number of Splits '0'. Must b errorMessageNumberOfSplitNotNumber=Unable to create config for batchsource CloudSQLPostgreSQL 'numSplits' is invalid: Value of \ field class io.cdap.plugin.db.config.AbstractDBSpecificSourceConfig.numSplits is expected to be a number. errorMessageInvalidFetchSize=Invalid fetch size. Fetch size must be a positive integer. -errorMessageInvalidSourceDatabase=SQL error while getting query schema: Error: FATAL: database "invalidDatabase" does not exist, +errorMessageInvalidSourceDatabase=Error occurred while trying to get schema from database.Error message: 'FATAL: database "invalidDatabase" does not exist'. errorMessageInvalidImportQuery=Import Query select must contain the string '$CONDITIONS'. if Number of Splits is not set\ \ to 1. Include '$CONDITIONS' in the Import Query errorMessageBlankUsername=Username is required when password is given. -errorMessageBlankPassword=SQL error while getting query schema: -errorMessageInvalidPassword=SQL error while getting query schema: Error: FATAL: password authentication failed for user +errorMessageBlankPassword=Error occurred while trying to get schema from database.Error message: 'Something unusual has occurred to cause the driver to fail. +errorMessageInvalidPassword=Error occurred while trying to get schema from database.Error message: 'FATAL: password authentication failed for user errorMessageInvalidSourceHost=SQL error while getting query schema: The connection attempt failed. errorMessageInvalidTableName=Table 'table' does not exist. Ensure table '"table"' is set correctly and that the errorMessageInvalidSinkDatabase=Error occurred while trying to get schema from database.Error message: 'FATAL: database "invalidDB" does not exist'. Error code: '0'. SQLState: '3D000'' diff --git a/mssql-plugin/src/e2e-test/resources/errorMessage.properties b/mssql-plugin/src/e2e-test/resources/errorMessage.properties index ceb700c57..f33ae97db 100644 --- a/mssql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/mssql-plugin/src/e2e-test/resources/errorMessage.properties @@ -10,7 +10,7 @@ errorMessageInvalidNumberOfSplits=Invalid value for Number of Splits '0'. Must b errorMessageNumberOfSplitNotNumber=Unable to create config for batchsource errorMessageBoundingQuery=Bounding Query must be specified if Number of Splits is not set to 1. errorMessagenumofSplit=Split-By Field Name must be specified if Number of Splits is not set to 1. -errorMessageInvalidSinkDatabase=Exception while trying to validate schema of database table +errorMessageInvalidSinkDatabase=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'Cannot open database "test123" requested by the login. errorMessageInvalidSinkTableName=Table 'Table123@' does not exist. errormessageBlankHost=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'The TCP/IP connection to the host localhost, port 1433 has failed. Error: "Connection refused (Connection refused). \ Verify the connection properties. Make sure that an instance of SQL Server is running on the host and accepting TCP/IP connections at the port. Make sure that TCP connections to the port are not blocked by a firewall.".'. Error code: '0'. SQLState: '08S01'' diff --git a/mysql-plugin/src/e2e-test/resources/errorMessage.properties b/mysql-plugin/src/e2e-test/resources/errorMessage.properties index fd0869726..72f7a5501 100644 --- a/mysql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/mysql-plugin/src/e2e-test/resources/errorMessage.properties @@ -4,7 +4,7 @@ invalidtableName.error.message=Table '123#' does not exist. Ensure table '123#' invalidreferenceName.error.message=Invalid reference name invalid.host.message=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'Communications link failure The last packet sent successfully to the server was 0 milliseconds ago. The driver has not received any packets from the server.'. Error code: '0'. SQLState: '08S01'' invalid.password.message=Error occurred while trying to get schema from database.Error message: 'Access denied for user ' -invalid.databasename.message=Error occurred while trying to get schema from database.Error message: 'Access denied for user ' +invalid.databasename.message=Error occurred while trying to get schema from database. invalid.query.message=Error occurred while trying to get schema from database.Error message: 'No tables used'. numberofsplits.error.message=Unable to create config for batchsource Mysql 'numSplits' is invalid boundingQuery.error.message=Bounding Query must be specified if Number of Splits is not set to 1. diff --git a/oracle-plugin/src/e2e-test/resources/errorMessage.properties b/oracle-plugin/src/e2e-test/resources/errorMessage.properties index c7d9fd7f6..ba94fd8ed 100644 --- a/oracle-plugin/src/e2e-test/resources/errorMessage.properties +++ b/oracle-plugin/src/e2e-test/resources/errorMessage.properties @@ -12,7 +12,7 @@ errorMessageInvalidImportQuery=Import Query select must contain the string '$CON \ to 1. Include '$CONDITIONS' in the Import Query errorMessageBlankUsername=Username is required when password is given. errorMessageInvalidTableName=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'IO Error: Unknown host specified '. Error code: '17002'. SQLState: '08006'' -errorMessageInvalidSinkDatabase=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'IO Error: Unknown host specified '. Error code: '17002'. SQLState: '08006'' +errorMessageInvalidSinkDatabase=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database. errorMessageInvalidHost=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'IO Error: Unknown host specified '. Error code: '17002'. SQLState: '08006'' errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: Stage 'Oracle' encountered : \ java.io.IOException: ORA-00936: missing expression . Please check the system logs for more details. From 1eba47c2ed1b87282ed95b8f4974e49b7b515f79 Mon Sep 17 00:00:00 2001 From: Sanchit Garg Date: Tue, 24 Jun 2025 15:05:08 +0530 Subject: [PATCH 17/28] PLUGIN-1823: Rework --- .../mysql/CloudSQLMySQLConnector.java | 5 + .../cloudsql/mysql/CloudSQLMySQLSink.java | 5 + .../cloudsql/mysql/CloudSQLMySQLSource.java | 5 + .../postgres/CloudSQLPostgreSQLConnector.java | 5 + .../postgres/CloudSQLPostgreSQLSink.java | 5 + .../postgres/CloudSQLPostgreSQLSource.java | 5 + .../db/action/AbstractDBArgumentSetter.java | 17 +- .../java/io/cdap/plugin/db/action/DBRun.java | 22 +- .../AbstractDBSpecificConnector.java | 29 ++- .../cdap/plugin/db/sink/AbstractDBSink.java | 34 ++- .../plugin/db/source/AbstractDBSource.java | 23 +- .../java/io/cdap/plugin/util/RetryUtils.java | 221 ++---------------- .../io/cdap/plugin/mariadb/MariadbSink.java | 5 + .../io/cdap/plugin/mariadb/MariadbSource.java | 5 + .../cdap/plugin/mssql/SqlServerConnector.java | 5 + .../io/cdap/plugin/mssql/SqlServerSink.java | 5 + .../io/cdap/plugin/mssql/SqlServerSource.java | 5 + .../io/cdap/plugin/mysql/MysqlConnector.java | 5 + .../java/io/cdap/plugin/mysql/MysqlSink.java | 5 + .../io/cdap/plugin/mysql/MysqlSource.java | 5 + .../cdap/plugin/oracle/OracleConnector.java | 5 + .../io/cdap/plugin/oracle/OracleSink.java | 5 + .../io/cdap/plugin/oracle/OracleSource.java | 5 + .../plugin/postgres/PostgresConnector.java | 5 + .../io/cdap/plugin/postgres/PostgresSink.java | 5 + .../cdap/plugin/postgres/PostgresSource.java | 5 + 26 files changed, 206 insertions(+), 240 deletions(-) diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java index b4b87c81b..b8045b5cf 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java @@ -108,4 +108,9 @@ protected void setConnectorSpec(ConnectorSpecRequest request, DBConnectorPath pa properties.put(Constants.Reference.REFERENCE_NAME, ReferenceNames.cleanseReferenceName(table)); properties.put(CloudSQLMySQLSink.CloudSQLMySQLSinkConfig.TABLE_NAME, table); } + + @Override + protected CloudSQLMySQLErrorDetailsProvider getErrorDetailsProvider() { + return new CloudSQLMySQLErrorDetailsProvider(); + } } diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java index 6cd1b0031..767842924 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java @@ -109,6 +109,11 @@ protected String getErrorDetailsProviderClassName() { return CloudSQLMySQLErrorDetailsProvider.class.getName(); } + @Override + protected CloudSQLMySQLErrorDetailsProvider getErrorDetailsProvider() { + return new CloudSQLMySQLErrorDetailsProvider(); + } + @Override protected String getExternalDocumentationLink() { return DBUtils.CLOUDSQLMYSQL_SUPPORTED_DOC_URL; diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java index 201360c67..75bbe6c8b 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java @@ -138,6 +138,11 @@ protected String getErrorDetailsProviderClassName() { return CloudSQLMySQLErrorDetailsProvider.class.getName(); } + @Override + protected CloudSQLMySQLErrorDetailsProvider getErrorDetailsProvider() { + return new CloudSQLMySQLErrorDetailsProvider(); + } + /** CloudSQL MySQL source config. */ public static class CloudSQLMySQLSourceConfig extends AbstractDBSpecificSourceConfig { diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnector.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnector.java index 07c83ebbe..131f09b7b 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnector.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnector.java @@ -118,4 +118,9 @@ protected void setConnectorSpec(ConnectorSpecRequest request, DBConnectorPath pa sourceProperties.put(Constants.Reference.REFERENCE_NAME, ReferenceNames.cleanseReferenceName(table)); sinkProperties.put(Constants.Reference.REFERENCE_NAME, ReferenceNames.cleanseReferenceName(table)); } + + @Override + protected CloudSQLPostgreSQLErrorDetailsProvider getErrorDetailsProvider() { + return new CloudSQLPostgreSQLErrorDetailsProvider(); + } } diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java index 060b67f82..302903d87 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java @@ -153,6 +153,11 @@ protected String getErrorDetailsProviderClassName() { return CloudSQLPostgreSQLErrorDetailsProvider.class.getName(); } + @Override + protected CloudSQLPostgreSQLErrorDetailsProvider getErrorDetailsProvider() { + return new CloudSQLPostgreSQLErrorDetailsProvider(); + } + @Override protected String getExternalDocumentationLink() { return DBUtils.CLOUDSQLPOSTGRES_SUPPORTED_DOC_URL; diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java index db3f2d708..b5e0922a8 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java @@ -97,6 +97,11 @@ protected String getErrorDetailsProviderClassName() { return CloudSQLPostgreSQLErrorDetailsProvider.class.getName(); } + @Override + protected CloudSQLPostgreSQLErrorDetailsProvider getErrorDetailsProvider() { + return new CloudSQLPostgreSQLErrorDetailsProvider(); + } + @Override protected String createConnectionString() { if (CloudSQLUtil.PRIVATE_INSTANCE.equalsIgnoreCase( diff --git a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java index e20774dbb..e5788ea94 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java @@ -23,6 +23,7 @@ import io.cdap.cdap.etl.api.action.Action; import io.cdap.cdap.etl.api.action.ActionContext; import io.cdap.cdap.etl.api.action.SettableArguments; +import io.cdap.plugin.common.db.DBErrorDetailsProvider; import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.util.DBUtils; import io.cdap.plugin.util.DriverCleanup; @@ -114,12 +115,12 @@ private void processArguments(Class driverClass, private void executeWithRetry(FailureCollector failureCollector, SettableArguments settableArguments, Properties connectionProperties) throws SQLException { try (Connection connection = RetryUtils.createConnectionWithRetry((RetryPolicy) retryPolicy, - config.getConnectionString(), connectionProperties, getExternalDocumentationLink())) { + config.getConnectionString(), connectionProperties, getErrorDetailsProvider())) { ResultSet resultSet; try (Statement statement = RetryUtils.createStatementWithRetry((RetryPolicy) retryPolicy, connection, - getExternalDocumentationLink())) { + getErrorDetailsProvider())) { resultSet = RetryUtils.executeQueryWithRetry((RetryPolicy) retryPolicy, statement, - config.getQuery(), null); + config.getQuery(), getErrorDetailsProvider()); } boolean hasRecord = resultSet.next(); if (!hasRecord) { @@ -163,4 +164,14 @@ private void setArguments(ResultSet resultSet, FailureCollector failureCollector protected String getExternalDocumentationLink() { return "https://en.wikipedia.org/wiki/SQLSTATE"; } + + /** + * Returns the DBErrorDetailsProvider instance. + * Override this method to provide a custom DBErrorDetailsProvider instance. + * + * @return DBErrorDetailsProvider instance + */ + protected DBErrorDetailsProvider getErrorDetailsProvider() { + return new DBErrorDetailsProvider(); + } } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/action/DBRun.java b/database-commons/src/main/java/io/cdap/plugin/db/action/DBRun.java index ab3392a9b..9cf303d5e 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/action/DBRun.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/action/DBRun.java @@ -16,8 +16,8 @@ package io.cdap.plugin.db.action; -import dev.failsafe.Failsafe; import dev.failsafe.RetryPolicy; +import io.cdap.plugin.common.db.DBErrorDetailsProvider; import io.cdap.plugin.util.DBUtils; import io.cdap.plugin.util.DriverCleanup; import io.cdap.plugin.util.RetryPolicyUtil; @@ -25,7 +25,6 @@ import java.sql.Connection; import java.sql.Driver; -import java.sql.DriverManager; import java.sql.SQLException; import java.sql.Statement; import java.util.List; @@ -50,6 +49,15 @@ public DBRun(QueryConfig config, Class driverClass, Boolean en config.getMaxRetryCount()); } + /** + * Returns the DBErrorDetailsProvider instance. + * + * @return DBErrorDetailsProvider instance + */ + protected DBErrorDetailsProvider getErrorDetailsProvider() { + return new DBErrorDetailsProvider(); + } + /** * Uses a configured JDBC driver to execute a SQL statement. The configurations of which JDBC driver * to use and which connection string to use come from the plugin configuration. @@ -63,14 +71,14 @@ public void run() throws SQLException, InstantiationException, IllegalAccessExce Properties connectionProperties = new Properties(); connectionProperties.putAll(config.getConnectionArguments()); try (Connection connection = RetryUtils.createConnectionWithRetry((RetryPolicy) retryPolicy, - config.getConnectionString(), connectionProperties, null)) { + config.getConnectionString(), connectionProperties, getErrorDetailsProvider())) { executeInitQueries(connection, config.getInitQueries()); if (!enableAutoCommit) { connection.setAutoCommit(false); } try (Statement statement = RetryUtils.createStatementWithRetry((RetryPolicy) retryPolicy, connection, - null)) { - RetryUtils.executeInitQueryWithRetry(retryPolicy, statement, config.query, null); + getErrorDetailsProvider())) { + RetryUtils.executeInitQueryWithRetry(retryPolicy, statement, config.query, getErrorDetailsProvider()); if (!enableAutoCommit) { connection.commit(); } @@ -86,8 +94,8 @@ public void run() throws SQLException, InstantiationException, IllegalAccessExce private void executeInitQueries(Connection connection, List initQueries) throws SQLException { for (String query : initQueries) { try (Statement statement = RetryUtils.createStatementWithRetry((RetryPolicy) retryPolicy, connection, - null)) { - RetryUtils.executeInitQueryWithRetry(retryPolicy, statement, query, null); + getErrorDetailsProvider())) { + RetryUtils.executeInitQueryWithRetry(retryPolicy, statement, query, getErrorDetailsProvider()); } } } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java index 973574db0..368d04581 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java @@ -29,6 +29,7 @@ import io.cdap.plugin.common.SourceInputFormatProvider; import io.cdap.plugin.common.db.AbstractDBConnector; import io.cdap.plugin.common.db.DBConnectorPath; +import io.cdap.plugin.common.db.DBErrorDetailsProvider; import io.cdap.plugin.common.util.ExceptionUtils; import io.cdap.plugin.db.CommonSchemaReader; import io.cdap.plugin.db.ConnectionConfigAccessor; @@ -122,6 +123,16 @@ public InputFormatProvider getInputFormatProvider(ConnectorContext context, Samp return new SourceInputFormatProvider(DataDrivenETLDBInputFormat.class, connectionConfigAccessor.getConfiguration()); } + /** + * Returns the DBErrorDetailsProvider instance. + * Override this method to provide a custom DBErrorDetailsProvider instance. + * + * @return DBErrorDetailsProvider instance + */ + protected DBErrorDetailsProvider getErrorDetailsProvider() { + return new DBErrorDetailsProvider(); + } + protected Connection getConnection(DBConnectorPath path) { return getConnection(getConnectionString(path.getDatabase()), config.getConnectionArgumentsProperties()); } @@ -178,14 +189,16 @@ protected String getStratifiedQuery(String tableName, int limit, String strata, protected Schema loadTableSchema(Connection connection, String query, @Nullable Integer timeoutSec, String sessionID) throws SQLException { - Statement statement = RetryUtils.createStatementWithRetry((RetryPolicy) retryPolicy, connection, null); - statement.setMaxRows(1); - if (timeoutSec != null) { - statement.setQueryTimeout(timeoutSec); - } - ResultSet resultSet = RetryUtils.executeQueryWithRetry((RetryPolicy) retryPolicy, statement, query, - null); - return Schema.recordOf("outputSchema", getSchemaReader(sessionID).getSchemaFields(resultSet)); + try (Statement statement = RetryUtils.createStatementWithRetry((RetryPolicy) retryPolicy, connection, + getErrorDetailsProvider())) { + statement.setMaxRows(1); + if (timeoutSec != null) { + statement.setQueryTimeout(timeoutSec); + } + ResultSet resultSet = RetryUtils.executeQueryWithRetry((RetryPolicy) retryPolicy, statement, query, + getErrorDetailsProvider()); + return Schema.recordOf("outputSchema", getSchemaReader(sessionID).getSchemaFields(resultSet)); + } } protected void setConnectionProperties(Map properties, ConnectorSpecRequest request) { diff --git a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java index 0f10fe579..d87ba3b89 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java @@ -39,6 +39,7 @@ import io.cdap.plugin.common.ReferenceBatchSink; import io.cdap.plugin.common.ReferencePluginConfig; import io.cdap.plugin.common.batch.sink.SinkOutputFormatProvider; +import io.cdap.plugin.common.db.DBErrorDetailsProvider; import io.cdap.plugin.db.ColumnType; import io.cdap.plugin.db.CommonSchemaReader; import io.cdap.plugin.db.ConnectionConfig; @@ -180,6 +181,16 @@ protected String getErrorDetailsProviderClassName() { return null; } + /** + * Returns the DBErrorDetailsProvider instance. + * Override this method to provide a custom DBErrorDetailsProvider instance. + * + * @return DBErrorDetailsProvider instance + */ + protected DBErrorDetailsProvider getErrorDetailsProvider() { + return new DBErrorDetailsProvider(); + } + /** * Returns the external documentation link. * Override this method to provide a custom external documentation link. @@ -305,13 +316,12 @@ private Schema inferSchema(Class driverClass) { Properties connectionProperties = new Properties(); connectionProperties.putAll(dbSinkConfig.getConnectionArguments()); try (Connection connection = RetryUtils.createConnectionWithRetry((RetryPolicy) retryPolicy, - dbSinkConfig.getConnectionString(), connectionProperties, getExternalDocumentationLink())) { + dbSinkConfig.getConnectionString(), connectionProperties, getErrorDetailsProvider())) { executeInitQueries(connection, dbSinkConfig.getInitQueries()); try (Statement statement = RetryUtils.createStatementWithRetry((RetryPolicy) retryPolicy, - connection, getExternalDocumentationLink()); + connection, getErrorDetailsProvider()); ResultSet rs = RetryUtils.executeQueryWithRetry((RetryPolicy) retryPolicy, statement, - String.format("SELECT * FROM %s WHERE 1 = 0", fullyQualifiedTableName), - getExternalDocumentationLink())) { + String.format("SELECT * FROM %s WHERE 1 = 0", fullyQualifiedTableName), getErrorDetailsProvider())) { inferredFields.addAll(getSchemaReader().getSchemaFields(rs)); } } @@ -355,15 +365,15 @@ private void setResultSetMetadata() throws SQLException, IllegalAccessException, Properties connectionProperties = new Properties(); connectionProperties.putAll(dbSinkConfig.getConnectionArguments()); try (Connection connection = RetryUtils.createConnectionWithRetry((RetryPolicy) retryPolicy, - connectionString, connectionProperties, getExternalDocumentationLink())) { + connectionString, connectionProperties, getErrorDetailsProvider())) { executeInitQueries(connection, dbSinkConfig.getInitQueries()); try (Statement statement = RetryUtils.createStatementWithRetry((RetryPolicy) retryPolicy, connection, - getExternalDocumentationLink()); + getErrorDetailsProvider()); // Run a query against the DB table that returns 0 records, but returns valid ResultSetMetadata // that can be used to construct DBRecord objects to sink to the database table. ResultSet rs = RetryUtils.executeQueryWithRetry((RetryPolicy) retryPolicy, statement, String.format("SELECT %s FROM %s WHERE 1 = 0", dbColumns, fullyQualifiedTableName), - getExternalDocumentationLink())) { + getErrorDetailsProvider())) { columnTypes.addAll(getMatchedColumnTypeList(rs, columns)); } } @@ -426,7 +436,7 @@ private void validateSchema(FailureCollector collector, Class Properties connectionProperties = new Properties(); connectionProperties.putAll(dbSinkConfig.getConnectionArguments()); try (Connection connection = RetryUtils.createConnectionWithRetry((RetryPolicy) retryPolicy, - connectionString, connectionProperties, getExternalDocumentationLink())) { + connectionString, connectionProperties, getErrorDetailsProvider())) { executeInitQueries(connection, dbSinkConfig.getInitQueries()); try (ResultSet tables = connection.getMetaData().getTables(null, dbSchemaName, tableName, null)) { if (!tables.next()) { @@ -440,9 +450,9 @@ connectionString, connectionProperties, getExternalDocumentationLink())) { setColumnsInfo(inputSchema.getFields()); try (PreparedStatement pStmt = RetryUtils.prepareStatementWithRetry((RetryPolicy) retryPolicy, connection, String.format("SELECT %s FROM %s WHERE 1 = 0", dbColumns, fullyQualifiedTableName), - getExternalDocumentationLink()); + getErrorDetailsProvider()); ResultSet rs = RetryUtils.executeQueryWithRetry((RetryPolicy) retryPolicy, pStmt, - getExternalDocumentationLink())) { + getErrorDetailsProvider())) { getFieldsValidator().validateFields(inputSchema, rs, collector); } } catch (SQLException e) { @@ -475,8 +485,8 @@ protected LineageRecorder getLineageRecorder(BatchSinkContext context) { private void executeInitQueries(Connection connection, List initQueries) throws SQLException { for (String query : initQueries) { try (Statement statement = RetryUtils.createStatementWithRetry((RetryPolicy) retryPolicy, connection, - getExternalDocumentationLink())) { - RetryUtils.executeInitQueryWithRetry(retryPolicy, statement, query, getExternalDocumentationLink()); + getErrorDetailsProvider())) { + RetryUtils.executeInitQueryWithRetry(retryPolicy, statement, query, getErrorDetailsProvider()); } } } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java index c964b8fdb..99369b72a 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java @@ -39,6 +39,7 @@ import io.cdap.plugin.common.ReferenceBatchSource; import io.cdap.plugin.common.ReferencePluginConfig; import io.cdap.plugin.common.SourceInputFormatProvider; +import io.cdap.plugin.common.db.DBErrorDetailsProvider; import io.cdap.plugin.db.CommonSchemaReader; import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.db.ConnectionConfigAccessor; @@ -169,11 +170,11 @@ public Schema getSchema() throws SQLException { private Schema loadSchemaFromDB(Connection connection, String query) throws SQLException { try (Statement statement = RetryUtils.createStatementWithRetry((RetryPolicy) retryPolicy, connection, - getExternalDocumentationLink())) { + getErrorDetailsProvider())) { statement.setMaxRows(1); String finalQuery = query.contains("$CONDITIONS") ? removeConditionsClause(query) : query; ResultSet resultSet = RetryUtils.executeQueryWithRetry((RetryPolicy) retryPolicy, statement, - finalQuery, getExternalDocumentationLink()); + finalQuery, getErrorDetailsProvider()); return Schema.recordOf("outputSchema", getSchemaReader().getSchemaFields(resultSet)); } } @@ -196,7 +197,7 @@ private Schema loadSchemaFromDB(Class driverClass) Properties connectionProperties = new Properties(); connectionProperties.putAll(sourceConfig.getConnectionArguments()); try (Connection connection = RetryUtils.createConnectionWithRetry((RetryPolicy) retryPolicy, - connectionString, connectionProperties, getExternalDocumentationLink())) { + connectionString, connectionProperties, getErrorDetailsProvider())) { executeInitQueries(connection, sourceConfig.getInitQueries()); return loadSchemaFromDB(connection, sourceConfig.getImportQuery()); } finally { @@ -207,8 +208,8 @@ connectionString, connectionProperties, getExternalDocumentationLink())) { private void executeInitQueries(Connection connection, List initQueries) throws SQLException { for (String query : initQueries) { try (Statement statement = RetryUtils.createStatementWithRetry((RetryPolicy) retryPolicy, connection, - getExternalDocumentationLink())) { - RetryUtils.executeInitQueryWithRetry(retryPolicy, statement, query, getExternalDocumentationLink()); + getErrorDetailsProvider())) { + RetryUtils.executeInitQueryWithRetry(retryPolicy, statement, query, getErrorDetailsProvider()); } } } @@ -227,6 +228,16 @@ protected String getErrorDetailsProviderClassName() { return null; } + /** + * Returns the DBErrorDetailsProvider instance. + * Override this method to provide a custom DBErrorDetailsProvider instance. + * + * @return DBErrorDetailsProvider instance + */ + protected DBErrorDetailsProvider getErrorDetailsProvider() { + return new DBErrorDetailsProvider(); + } + private DriverCleanup loadPluginClassAndGetDriver(Class driverClass) throws IllegalAccessException, InstantiationException, SQLException { @@ -251,7 +262,7 @@ private Connection getConnection() { Properties connectionProperties = new Properties(); connectionProperties.putAll(sourceConfig.getConnectionArguments()); return RetryUtils.createConnectionWithRetry((RetryPolicy) retryPolicy, - connectionString, connectionProperties, getExternalDocumentationLink()); + connectionString, connectionProperties, getErrorDetailsProvider()); } @Override diff --git a/database-commons/src/main/java/io/cdap/plugin/util/RetryUtils.java b/database-commons/src/main/java/io/cdap/plugin/util/RetryUtils.java index 4a1b6902f..be11daaa9 100644 --- a/database-commons/src/main/java/io/cdap/plugin/util/RetryUtils.java +++ b/database-commons/src/main/java/io/cdap/plugin/util/RetryUtils.java @@ -16,15 +16,10 @@ package io.cdap.plugin.util; -import com.google.common.base.Strings; import dev.failsafe.Failsafe; import dev.failsafe.FailsafeException; import dev.failsafe.RetryPolicy; -import io.cdap.cdap.api.exception.ErrorCategory; -import io.cdap.cdap.api.exception.ErrorCodeType; -import io.cdap.cdap.api.exception.ErrorType; -import io.cdap.cdap.api.exception.ErrorUtils; -import io.cdap.cdap.api.exception.ProgramFailureException; +import io.cdap.plugin.common.db.DBErrorDetailsProvider; import java.sql.Connection; import java.sql.DriverManager; @@ -32,8 +27,6 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; -import java.util.HashMap; -import java.util.Map; import java.util.Properties; /** @@ -41,8 +34,6 @@ */ public final class RetryUtils { - private static final Map ERROR_CODE_TO_ERROR_TYPE; - private static final Map ERROR_CODE_TO_ERROR_CATEGORY; public static final String NAME_INITIAL_RETRY_DURATION = "initialRetryDuration"; public static final String NAME_MAX_RETRY_DURATION = "maxRetryDuration"; public static final String NAME_MAX_RETRY_COUNT = "maxRetryCount"; @@ -50,189 +41,68 @@ public final class RetryUtils { public static final int DEFAULT_MAX_RETRY_COUNT = 5; public static final int DEFAULT_MAX_RETRY_DURATION_SECONDS = 80; - static { - // https://en.wikipedia.org/wiki/SQLSTATE - ERROR_CODE_TO_ERROR_TYPE = new HashMap<>(); - ERROR_CODE_TO_ERROR_TYPE.put("01", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("02", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("07", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("08", ErrorType.SYSTEM); - ERROR_CODE_TO_ERROR_TYPE.put("09", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("0A", ErrorType.SYSTEM); - ERROR_CODE_TO_ERROR_TYPE.put("0D", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("0E", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("0F", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("0K", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("0L", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("0M", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("0N", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("0P", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("0S", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("0T", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("0U", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("0V", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("0W", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("0X", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("0Y", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("0Z", ErrorType.SYSTEM); - ERROR_CODE_TO_ERROR_TYPE.put("10", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("20", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("21", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("22", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("23", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("24", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("25", ErrorType.SYSTEM); - ERROR_CODE_TO_ERROR_TYPE.put("26", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("27", ErrorType.SYSTEM); - ERROR_CODE_TO_ERROR_TYPE.put("28", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("2B", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("2C", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("2D", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("2E", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("2F", ErrorType.SYSTEM); - ERROR_CODE_TO_ERROR_TYPE.put("2H", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("30", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("33", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("34", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("35", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("36", ErrorType.SYSTEM); - ERROR_CODE_TO_ERROR_TYPE.put("38", ErrorType.SYSTEM); - ERROR_CODE_TO_ERROR_TYPE.put("39", ErrorType.SYSTEM); - ERROR_CODE_TO_ERROR_TYPE.put("3B", ErrorType.SYSTEM); - ERROR_CODE_TO_ERROR_TYPE.put("3C", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("3D", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("3F", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("40", ErrorType.SYSTEM); - ERROR_CODE_TO_ERROR_TYPE.put("42", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("44", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("45", ErrorType.USER); - ERROR_CODE_TO_ERROR_TYPE.put("46", ErrorType.SYSTEM); - ERROR_CODE_TO_ERROR_TYPE.put("HW", ErrorType.SYSTEM); - - ERROR_CODE_TO_ERROR_CATEGORY = new HashMap<>(); - ErrorCategory.ErrorCategoryEnum plugin = ErrorCategory.ErrorCategoryEnum.PLUGIN; - ERROR_CODE_TO_ERROR_CATEGORY.put("01", new ErrorCategory(plugin, "DB Warning")); - ERROR_CODE_TO_ERROR_CATEGORY.put("02", new ErrorCategory(plugin, "DB No Data")); - ERROR_CODE_TO_ERROR_CATEGORY.put("07", new ErrorCategory(plugin, "DB Dynamic SQL error")); - ERROR_CODE_TO_ERROR_CATEGORY.put("08", new ErrorCategory(plugin, "DB Connection Exception")); - ERROR_CODE_TO_ERROR_CATEGORY.put("09", new ErrorCategory(plugin, "DB Triggered Action Exception")); - ERROR_CODE_TO_ERROR_CATEGORY.put("0A", new ErrorCategory(plugin, "DB Feature Not Supported")); - ERROR_CODE_TO_ERROR_CATEGORY.put("0D", new ErrorCategory(plugin, "DB Invalid Target Type Specification")); - ERROR_CODE_TO_ERROR_CATEGORY.put("0E", new ErrorCategory(plugin, "DB Invalid Schema Name List Specification")); - ERROR_CODE_TO_ERROR_CATEGORY.put("0F", new ErrorCategory(plugin, "DB Locator Exception")); - ERROR_CODE_TO_ERROR_CATEGORY.put("0K", new ErrorCategory(plugin, "DB Resignal When Handler Not Active")); - ERROR_CODE_TO_ERROR_CATEGORY.put("0L", new ErrorCategory(plugin, "DB Invalid Grantor")); - ERROR_CODE_TO_ERROR_CATEGORY.put("0M", new ErrorCategory(plugin, "DB Invalid SQL-Invoked Procedure Reference")); - ERROR_CODE_TO_ERROR_CATEGORY.put("0N", new ErrorCategory(plugin, "DB SQL/XML Mapping Error")); - ERROR_CODE_TO_ERROR_CATEGORY.put("0P", new ErrorCategory(plugin, "DB Invalid Role Specification")); - ERROR_CODE_TO_ERROR_CATEGORY.put("0S", - new ErrorCategory(plugin, "DB Invalid Transform Group Name Specification")); - ERROR_CODE_TO_ERROR_CATEGORY.put("0T", - new ErrorCategory(plugin, "DB Target Table Disagrees With Cursor Specification")); - ERROR_CODE_TO_ERROR_CATEGORY.put("0U", - new ErrorCategory(plugin, "DB Attempt To Assign To Non-Updatable Column")); - ERROR_CODE_TO_ERROR_CATEGORY.put("0V", new ErrorCategory(plugin, "DB Attempt To Assign To Ordering Column")); - ERROR_CODE_TO_ERROR_CATEGORY.put("0W", new ErrorCategory(plugin, "DB Prohibited Statement Encountered")); - ERROR_CODE_TO_ERROR_CATEGORY.put("0X", new ErrorCategory(plugin, "DB Invalid Foreign Server Specification")); - ERROR_CODE_TO_ERROR_CATEGORY.put("0Y", new ErrorCategory(plugin, "DB Pass-Through Specific Condition")); - ERROR_CODE_TO_ERROR_CATEGORY.put("0Z", new ErrorCategory(plugin, "DB Diagnostics Exception")); - ERROR_CODE_TO_ERROR_CATEGORY.put("10", new ErrorCategory(plugin, "DB XQuery Error")); - ERROR_CODE_TO_ERROR_CATEGORY.put("20", new ErrorCategory(plugin, "DB Case Not Found For Case Statement")); - ERROR_CODE_TO_ERROR_CATEGORY.put("21", new ErrorCategory(plugin, "DB Cardinality Violation")); - ERROR_CODE_TO_ERROR_CATEGORY.put("22", new ErrorCategory(plugin, "DB Data Exception")); - ERROR_CODE_TO_ERROR_CATEGORY.put("23", new ErrorCategory(plugin, "DB Integrity Constraint Violation")); - ERROR_CODE_TO_ERROR_CATEGORY.put("24", new ErrorCategory(plugin, "DB Invalid Cursor State")); - ERROR_CODE_TO_ERROR_CATEGORY.put("25", new ErrorCategory(plugin, "DB Invalid Transaction State")); - ERROR_CODE_TO_ERROR_CATEGORY.put("26", new ErrorCategory(plugin, "DB Invalid SQL Statement Name")); - ERROR_CODE_TO_ERROR_CATEGORY.put("27", new ErrorCategory(plugin, "DB Triggered Data Change Violation")); - ERROR_CODE_TO_ERROR_CATEGORY.put("28", new ErrorCategory(plugin, "DB Invalid Authorization Specification")); - ERROR_CODE_TO_ERROR_CATEGORY.put("2B", - new ErrorCategory(plugin, "DB Dependent Privilege Descriptors Still Exist")); - ERROR_CODE_TO_ERROR_CATEGORY.put("2C", new ErrorCategory(plugin, "DB Invalid Character Set Name")); - ERROR_CODE_TO_ERROR_CATEGORY.put("2D", new ErrorCategory(plugin, "DB Invalid Transaction Termination")); - ERROR_CODE_TO_ERROR_CATEGORY.put("2E", new ErrorCategory(plugin, "DB Invalid Connection Name")); - ERROR_CODE_TO_ERROR_CATEGORY.put("2F", new ErrorCategory(plugin, "DB SQL Routine Exception")); - ERROR_CODE_TO_ERROR_CATEGORY.put("2H", new ErrorCategory(plugin, "DB Invalid Collation Name")); - ERROR_CODE_TO_ERROR_CATEGORY.put("30", new ErrorCategory(plugin, "DB Invalid SQL Statement Identifier")); - ERROR_CODE_TO_ERROR_CATEGORY.put("33", new ErrorCategory(plugin, "DB Invalid SQL Descriptor Name")); - ERROR_CODE_TO_ERROR_CATEGORY.put("34", new ErrorCategory(plugin, "DB Invalid Cursor Name")); - ERROR_CODE_TO_ERROR_CATEGORY.put("35", new ErrorCategory(plugin, "DB Invalid Condition Number")); - ERROR_CODE_TO_ERROR_CATEGORY.put("36", new ErrorCategory(plugin, "DB Cursor Sensitivity Exception")); - ERROR_CODE_TO_ERROR_CATEGORY.put("38", new ErrorCategory(plugin, "DB External Routine Exception")); - ERROR_CODE_TO_ERROR_CATEGORY.put("39", new ErrorCategory(plugin, "DB External Routine Invocation Exception")); - ERROR_CODE_TO_ERROR_CATEGORY.put("3B", new ErrorCategory(plugin, "DB Savepoint Exception")); - ERROR_CODE_TO_ERROR_CATEGORY.put("3C", new ErrorCategory(plugin, "DB Ambiguous Cursor Name")); - ERROR_CODE_TO_ERROR_CATEGORY.put("3D", new ErrorCategory(plugin, "DB Invalid Catalog Name")); - ERROR_CODE_TO_ERROR_CATEGORY.put("3F", new ErrorCategory(plugin, "DB Invalid Schema Name")); - ERROR_CODE_TO_ERROR_CATEGORY.put("40", new ErrorCategory(plugin, "DB Transaction Rollback")); - ERROR_CODE_TO_ERROR_CATEGORY.put("42", new ErrorCategory(plugin, "DB Syntax Error or Access Rule Violation")); - ERROR_CODE_TO_ERROR_CATEGORY.put("44", new ErrorCategory(plugin, "DB With Check Option Violation")); - ERROR_CODE_TO_ERROR_CATEGORY.put("45", new ErrorCategory(plugin, "DB Unhandled User-Defined Exception")); - ERROR_CODE_TO_ERROR_CATEGORY.put("46", new ErrorCategory(plugin, "DB JAVA DDL")); - ERROR_CODE_TO_ERROR_CATEGORY.put("HW", new ErrorCategory(plugin, "DB Datalink Exception")); - } - public static Connection createConnectionWithRetry(RetryPolicy retryPolicy, String connectionString, - Properties connectionProperties, String externalDocumentationLink) { + Properties connectionProperties, DBErrorDetailsProvider dbErrorDetailsProvider) { try { return Failsafe.with(retryPolicy).get(() -> DriverManager .getConnection(connectionString, connectionProperties) ); } catch (Exception e) { - throw unwrapFailsafeException(e, externalDocumentationLink); + throw unwrapFailsafeException(e, dbErrorDetailsProvider); } } public static Statement createStatementWithRetry(RetryPolicy retryPolicy, Connection connection, - String externalDocumentationLink) { + DBErrorDetailsProvider dbErrorDetailsProvider) { try { return Failsafe.with(retryPolicy).get(connection::createStatement); } catch (Exception e) { - throw unwrapFailsafeException(e, externalDocumentationLink); + throw unwrapFailsafeException(e, dbErrorDetailsProvider); } } public static PreparedStatement prepareStatementWithRetry(RetryPolicy retryPolicy, - Connection connection, String sqlQuery, String externalDocumentationLink) { + Connection connection, String sqlQuery, DBErrorDetailsProvider dbErrorDetailsProvider) { try { return Failsafe.with(retryPolicy).get(() -> connection.prepareStatement(sqlQuery)); } catch (Exception e) { - throw unwrapFailsafeException(e, externalDocumentationLink); + throw unwrapFailsafeException(e, dbErrorDetailsProvider); } } public static ResultSet executeQueryWithRetry(RetryPolicy retryPolicy, - PreparedStatement preparedStatement, String externalDocumentationLink) { + PreparedStatement preparedStatement, DBErrorDetailsProvider dbErrorDetailsProvider) { try { return Failsafe.with(retryPolicy).get(() -> preparedStatement.executeQuery()); } catch (Exception e) { - throw unwrapFailsafeException(e, externalDocumentationLink); + throw unwrapFailsafeException(e, dbErrorDetailsProvider); } } public static ResultSet executeQueryWithRetry(RetryPolicy retryPolicy, Statement statement, - String query, String externalDocumentationLink) { + String query, DBErrorDetailsProvider dbErrorDetailsProvider) { try { return Failsafe.with(retryPolicy).get(() -> statement.executeQuery(query)); } catch (Exception e) { - throw unwrapFailsafeException(e, externalDocumentationLink); + throw unwrapFailsafeException(e, dbErrorDetailsProvider); } } public static void executeInitQueryWithRetry(RetryPolicy retryPolicy, Statement statement, String query, - String externalDocumentationLink) { + DBErrorDetailsProvider dbErrorDetailsProvider) { try { Failsafe.with(retryPolicy).run(() -> statement.execute(query)); } catch (Exception e) { - throw unwrapFailsafeException(e, externalDocumentationLink); + throw unwrapFailsafeException(e, dbErrorDetailsProvider); } } - private static RuntimeException unwrapFailsafeException(Exception e, String externalDocumentationLink) { + private static RuntimeException unwrapFailsafeException(Exception e, + DBErrorDetailsProvider dbErrorDetailsProvider) { if (e instanceof FailsafeException) { Throwable cause = e.getCause(); if (cause instanceof SQLException) { - return programFailureException((SQLException) cause, externalDocumentationLink); + return dbErrorDetailsProvider.getProgramFailureException((SQLException) cause, null); } else if (cause instanceof RuntimeException) { return (RuntimeException) cause; } else if (cause instanceof Error) { @@ -242,68 +112,11 @@ private static RuntimeException unwrapFailsafeException(Exception e, String exte } } if (e instanceof SQLException) { - return programFailureException((SQLException) e, externalDocumentationLink); + return dbErrorDetailsProvider.getProgramFailureException((SQLException) e, null); } if (e instanceof RuntimeException) { return (RuntimeException) e; } return new RuntimeException("Unexpected checked exception", e); } - - private static ProgramFailureException programFailureException(SQLException e, String externalDocumentationLink) { - // wrap exception to ensure SQLException-child instances not exposed to contexts without jdbc - // driver in classpath - String errorMessage = - String.format("SQL Exception occurred: [Message='%s', SQLState='%s', ErrorCode='%s'].", e.getMessage(), - e.getSQLState(), e.getErrorCode()); - String sqlState = e.getSQLState(); - int errorCode = e.getErrorCode(); - String errorMessageWithDetails = String.format("Error occurred while trying to" + - " get schema from database." + "Error message: '%s'. Error code: '%s'. SQLState: '%s'", e.getMessage(), - errorCode, sqlState); - - if (!Strings.isNullOrEmpty(externalDocumentationLink)) { - if (!errorMessage.endsWith(".")) { - errorMessage = errorMessage + "."; - } - errorMessage = String.format("%s For more details, see %s", errorMessage, externalDocumentationLink); - } - return ErrorUtils.getProgramFailureException(Strings.isNullOrEmpty(sqlState) ? - new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN) : getErrorCategoryFromSqlState(sqlState), - errorMessage, errorMessageWithDetails, getErrorTypeFromErrorCodeAndSqlState(errorCode, sqlState), true, - ErrorCodeType.SQLSTATE, e.getSQLState(), externalDocumentationLink, e); - } - - /** - * Get the {@link ErrorCategory} for the given SQL state. - * Implements generic error categories based on the SQL state. - * See SQLSTATE for more information. - * Override this method to provide custom error categories based on the SQL state. - * - * @param sqlState The SQL state. - * @return The {@link ErrorCategory} for the given SQL state. - */ - private static ErrorCategory getErrorCategoryFromSqlState(String sqlState) { - if (!Strings.isNullOrEmpty(sqlState) && sqlState.length() >= 2 && - ERROR_CODE_TO_ERROR_CATEGORY.containsKey(sqlState.substring(0, 2))) { - return ERROR_CODE_TO_ERROR_CATEGORY.get(sqlState.substring(0, 2)); - } - return new ErrorCategory(ErrorCategory.ErrorCategoryEnum.PLUGIN); - } - - /** - * Get the {@link ErrorType} for the given error code and SQL state. - * Override this method to provide custom error types based on the error code and SQL state. - * - * @param errorCode The error code. - * @param sqlState The SQL state. - * @return The {@link ErrorType} for the given error code and SQL state. - */ - private static ErrorType getErrorTypeFromErrorCodeAndSqlState(int errorCode, String sqlState) { - if (!Strings.isNullOrEmpty(sqlState) && sqlState.length() >= 2 && - ERROR_CODE_TO_ERROR_TYPE.containsKey(sqlState.substring(0, 2))) { - return ERROR_CODE_TO_ERROR_TYPE.get(sqlState.substring(0, 2)); - } - return ErrorType.UNKNOWN; - } } diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java index 52a73344a..de7e33482 100644 --- a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java @@ -67,6 +67,11 @@ protected String getErrorDetailsProviderClassName() { return MariadbErrorDetailsProvider.class.getName(); } + @Override + protected MariadbErrorDetailsProvider getErrorDetailsProvider() { + return new MariadbErrorDetailsProvider(); + } + @Override protected String getExternalDocumentationLink() { return DBUtils.MARIADB_SUPPORTED_DOC_URL; diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java index 28204100c..d3014cfbc 100644 --- a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java @@ -88,6 +88,11 @@ protected String getErrorDetailsProviderClassName() { return MariadbErrorDetailsProvider.class.getName(); } + @Override + protected MariadbErrorDetailsProvider getErrorDetailsProvider() { + return new MariadbErrorDetailsProvider(); + } + @Override protected String getExternalDocumentationLink() { return DBUtils.MARIADB_SUPPORTED_DOC_URL; diff --git a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerConnector.java b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerConnector.java index f2274917f..0201d48b4 100644 --- a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerConnector.java +++ b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerConnector.java @@ -149,4 +149,9 @@ protected String getStratifiedQuery(String tableName, int limit, String strata, limit, sessionID, limit, strata); } + @Override + protected SqlServerErrorDetailsProvider getErrorDetailsProvider() { + return new SqlServerErrorDetailsProvider(); + } + } diff --git a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java index dc442d200..eb1199e49 100644 --- a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java +++ b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java @@ -93,6 +93,11 @@ protected String getErrorDetailsProviderClassName() { return SqlServerErrorDetailsProvider.class.getName(); } + @Override + protected SqlServerErrorDetailsProvider getErrorDetailsProvider() { + return new SqlServerErrorDetailsProvider(); + } + @Override protected String getExternalDocumentationLink() { return DBUtils.MSSQL_SUPPORTED_DOC_URL; diff --git a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java index 004532064..dc20ff8dc 100644 --- a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java +++ b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java @@ -80,6 +80,11 @@ protected String getErrorDetailsProviderClassName() { return SqlServerErrorDetailsProvider.class.getName(); } + @Override + protected SqlServerErrorDetailsProvider getErrorDetailsProvider() { + return new SqlServerErrorDetailsProvider(); + } + @Override protected LineageRecorder getLineageRecorder(BatchSourceContext context) { String fqn = DBUtils.constructFQN("mssql", diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnector.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnector.java index e7e935135..0006c9be1 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnector.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnector.java @@ -129,4 +129,9 @@ protected String getStratifiedQuery(String tableName, int limit, String strata, public StructuredRecord transform(LongWritable longWritable, MysqlDBRecord mysqlDBRecord) { return mysqlDBRecord.getRecord(); } + + @Override + protected MysqlErrorDetailsProvider getErrorDetailsProvider() { + return new MysqlErrorDetailsProvider(); + } } diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java index 0a9257a0a..7e2044eed 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java @@ -114,6 +114,11 @@ protected String getErrorDetailsProviderClassName() { return MysqlErrorDetailsProvider.class.getName(); } + @Override + protected MysqlErrorDetailsProvider getErrorDetailsProvider() { + return new MysqlErrorDetailsProvider(); + } + @Override protected String getExternalDocumentationLink() { return DBUtils.MYSQL_SUPPORTED_DOC_URL; diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java index 38642468c..4eb1a4afe 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java @@ -94,6 +94,11 @@ protected String getErrorDetailsProviderClassName() { return MysqlErrorDetailsProvider.class.getName(); } + @Override + protected MysqlErrorDetailsProvider getErrorDetailsProvider() { + return new MysqlErrorDetailsProvider(); + } + /** * MySQL source config. */ diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java index 3d2f7399a..63920260a 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java @@ -188,4 +188,9 @@ protected String generateSessionID() { .replaceAll("-", "") .substring(0, 28); } + + @Override + protected OracleErrorDetailsProvider getErrorDetailsProvider() { + return new OracleErrorDetailsProvider(); + } } diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSink.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSink.java index 511281e9d..d71366168 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSink.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSink.java @@ -87,6 +87,11 @@ protected String getErrorDetailsProviderClassName() { return OracleErrorDetailsProvider.class.getName(); } + @Override + protected OracleErrorDetailsProvider getErrorDetailsProvider() { + return new OracleErrorDetailsProvider(); + } + @Override protected String getExternalDocumentationLink() { return DBUtils.ORACLE_SUPPORTED_DOC_URL; diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java index 53f75613b..b79b6f38d 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java @@ -81,6 +81,11 @@ protected String getErrorDetailsProviderClassName() { return OracleErrorDetailsProvider.class.getName(); } + @Override + protected OracleErrorDetailsProvider getErrorDetailsProvider() { + return new OracleErrorDetailsProvider(); + } + @Override protected LineageRecorder getLineageRecorder(BatchSourceContext context) { String fqn = DBUtils.constructFQN("oracle", diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConnector.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConnector.java index deb56ed79..0fe1a74cd 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConnector.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConnector.java @@ -142,4 +142,9 @@ protected String getStratifiedQuery(String tableName, int limit, String strata, sessionID, strata, sessionID, sessionID, tableName, sessionID, sessionID, sessionID, limit, strata, limit); } + + @Override + protected PostgresErrorDetailsProvider getErrorDetailsProvider() { + return new PostgresErrorDetailsProvider(); + } } diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java index 73430c1e2..a417b665f 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java @@ -121,6 +121,11 @@ protected String getErrorDetailsProviderClassName() { return PostgresErrorDetailsProvider.class.getName(); } + @Override + protected PostgresErrorDetailsProvider getErrorDetailsProvider() { + return new PostgresErrorDetailsProvider(); + } + @Override protected String getExternalDocumentationLink() { return DBUtils.POSTGRES_SUPPORTED_DOC_URL; diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java index b230f3d1e..f90eedd76 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java @@ -72,6 +72,11 @@ protected String getErrorDetailsProviderClassName() { return PostgresErrorDetailsProvider.class.getName(); } + @Override + protected PostgresErrorDetailsProvider getErrorDetailsProvider() { + return new PostgresErrorDetailsProvider(); + } + @Override protected Class getDBRecordType() { return PostgresDBRecord.class; From 5ef4f1523af0ae530ae4a0405e52dbaff58df62f Mon Sep 17 00:00:00 2001 From: Sanchit Garg Date: Wed, 25 Jun 2025 22:36:18 +0530 Subject: [PATCH 18/28] PLUGIN-1823: Create object only once. --- .../cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java | 6 +++++- .../io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java | 6 +++++- .../io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java | 6 +++++- .../cloudsql/postgres/CloudSQLPostgreSQLConnector.java | 6 +++++- .../plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java | 6 +++++- .../plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java | 6 +++++- .../io/cdap/plugin/db/action/AbstractDBArgumentSetter.java | 6 +++++- .../src/main/java/io/cdap/plugin/db/action/DBRun.java | 6 +++++- .../main/java/io/cdap/plugin/db/sink/AbstractDBSink.java | 6 +++++- .../java/io/cdap/plugin/db/source/AbstractDBSource.java | 6 +++++- .../src/main/java/io/cdap/plugin/mariadb/MariadbSink.java | 6 +++++- .../src/main/java/io/cdap/plugin/mariadb/MariadbSource.java | 6 +++++- .../main/java/io/cdap/plugin/mssql/SqlServerConnector.java | 6 +++++- .../src/main/java/io/cdap/plugin/mssql/SqlServerSink.java | 6 +++++- .../src/main/java/io/cdap/plugin/mssql/SqlServerSource.java | 6 +++++- .../src/main/java/io/cdap/plugin/mysql/MysqlConnector.java | 6 +++++- .../src/main/java/io/cdap/plugin/mysql/MysqlSink.java | 6 +++++- .../src/main/java/io/cdap/plugin/mysql/MysqlSource.java | 6 +++++- .../main/java/io/cdap/plugin/oracle/OracleConnector.java | 6 +++++- .../src/main/java/io/cdap/plugin/oracle/OracleSink.java | 6 +++++- .../src/main/java/io/cdap/plugin/oracle/OracleSource.java | 6 +++++- .../java/io/cdap/plugin/postgres/PostgresConnector.java | 6 +++++- .../src/main/java/io/cdap/plugin/postgres/PostgresSink.java | 6 +++++- .../main/java/io/cdap/plugin/postgres/PostgresSource.java | 6 +++++- 24 files changed, 120 insertions(+), 24 deletions(-) diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java index b8045b5cf..e3ba89d65 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java @@ -53,6 +53,7 @@ public class CloudSQLMySQLConnector extends AbstractDBSpecificConnector { private final CloudSQLMySQLSourceConfig cloudsqlMysqlSourceConfig; + private CloudSQLMySQLErrorDetailsProvider cloudSQLMySQLErrorDetailsProvider; public CloudSQLMySQLSource(CloudSQLMySQLSourceConfig cloudsqlMysqlSourceConfig) { super(cloudsqlMysqlSourceConfig); @@ -140,7 +141,10 @@ protected String getErrorDetailsProviderClassName() { @Override protected CloudSQLMySQLErrorDetailsProvider getErrorDetailsProvider() { - return new CloudSQLMySQLErrorDetailsProvider(); + if (cloudSQLMySQLErrorDetailsProvider == null) { + cloudSQLMySQLErrorDetailsProvider = new CloudSQLMySQLErrorDetailsProvider(); + } + return cloudSQLMySQLErrorDetailsProvider; } /** CloudSQL MySQL source config. */ diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnector.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnector.java index 131f09b7b..d89f53346 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnector.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnector.java @@ -52,6 +52,7 @@ public class CloudSQLPostgreSQLConnector extends AbstractDBSpecificConnector { public static final String NAME = CloudSQLPostgreSQLConstants.PLUGIN_NAME; private final CloudSQLPostgreSQLConnectorConfig config; + private CloudSQLPostgreSQLErrorDetailsProvider cloudSQLPostgreSQLErrorDetailsProvider; public CloudSQLPostgreSQLConnector(CloudSQLPostgreSQLConnectorConfig config) { super(config); @@ -121,6 +122,9 @@ protected void setConnectorSpec(ConnectorSpecRequest request, DBConnectorPath pa @Override protected CloudSQLPostgreSQLErrorDetailsProvider getErrorDetailsProvider() { - return new CloudSQLPostgreSQLErrorDetailsProvider(); + if (cloudSQLPostgreSQLErrorDetailsProvider == null) { + cloudSQLPostgreSQLErrorDetailsProvider = new CloudSQLPostgreSQLErrorDetailsProvider(); + } + return cloudSQLPostgreSQLErrorDetailsProvider; } } diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java index 302903d87..d5ff5694d 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java @@ -66,6 +66,7 @@ public class CloudSQLPostgreSQLSink extends AbstractDBSink { private final CloudSQLPostgreSQLSourceConfig cloudsqlPostgresqlSourceConfig; + private CloudSQLPostgreSQLErrorDetailsProvider cloudSQLPostgreSQLErrorDetailsProvider; public CloudSQLPostgreSQLSource(CloudSQLPostgreSQLSourceConfig cloudsqlPostgresqlSourceConfig) { super(cloudsqlPostgresqlSourceConfig); @@ -99,7 +100,10 @@ protected String getErrorDetailsProviderClassName() { @Override protected CloudSQLPostgreSQLErrorDetailsProvider getErrorDetailsProvider() { - return new CloudSQLPostgreSQLErrorDetailsProvider(); + if (cloudSQLPostgreSQLErrorDetailsProvider == null) { + cloudSQLPostgreSQLErrorDetailsProvider = new CloudSQLPostgreSQLErrorDetailsProvider(); + } + return cloudSQLPostgreSQLErrorDetailsProvider; } @Override diff --git a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java index e5788ea94..3d10bfc9c 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java @@ -45,6 +45,7 @@ public class AbstractDBArgumentSetter extends Action { private static final String JDBC_PLUGIN_ID = "driver"; private final ArgumentSetterConfig config; private final RetryPolicy retryPolicy; + private DBErrorDetailsProvider dbErrorDetailsProvider; public AbstractDBArgumentSetter(ArgumentSetterConfig config) { this.config = config; @@ -172,6 +173,9 @@ protected String getExternalDocumentationLink() { * @return DBErrorDetailsProvider instance */ protected DBErrorDetailsProvider getErrorDetailsProvider() { - return new DBErrorDetailsProvider(); + if (dbErrorDetailsProvider == null) { + dbErrorDetailsProvider = new DBErrorDetailsProvider(); + } + return dbErrorDetailsProvider; } } diff --git a/database-commons/src/main/java/io/cdap/plugin/db/action/DBRun.java b/database-commons/src/main/java/io/cdap/plugin/db/action/DBRun.java index 9cf303d5e..90485c723 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/action/DBRun.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/action/DBRun.java @@ -38,6 +38,7 @@ public class DBRun { private final Class driverClass; private boolean enableAutoCommit; private final RetryPolicy retryPolicy; + private DBErrorDetailsProvider dbErrorDetailsProvider; public DBRun(QueryConfig config, Class driverClass, Boolean enableAutocommit) { this.config = config; @@ -55,7 +56,10 @@ public DBRun(QueryConfig config, Class driverClass, Boolean en * @return DBErrorDetailsProvider instance */ protected DBErrorDetailsProvider getErrorDetailsProvider() { - return new DBErrorDetailsProvider(); + if (dbErrorDetailsProvider == null) { + dbErrorDetailsProvider = new DBErrorDetailsProvider(); + } + return dbErrorDetailsProvider; } /** diff --git a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java index d87ba3b89..8db9992fa 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java @@ -98,6 +98,7 @@ public abstract class AbstractDBSink retryPolicy; + private DBErrorDetailsProvider dbErrorDetailsProvider; public AbstractDBSink(T dbSinkConfig) { super(new ReferencePluginConfig(dbSinkConfig.getReferenceName())); @@ -188,7 +189,10 @@ protected String getErrorDetailsProviderClassName() { * @return DBErrorDetailsProvider instance */ protected DBErrorDetailsProvider getErrorDetailsProvider() { - return new DBErrorDetailsProvider(); + if (dbErrorDetailsProvider == null) { + dbErrorDetailsProvider = new DBErrorDetailsProvider(); + } + return dbErrorDetailsProvider; } /** diff --git a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java index 99369b72a..8984b0aed 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java @@ -88,6 +88,7 @@ public abstract class AbstractDBSource retryPolicy; + private DBErrorDetailsProvider dbErrorDetailsProvider; protected final T sourceConfig; protected Class driverClass; @@ -235,7 +236,10 @@ protected String getErrorDetailsProviderClassName() { * @return DBErrorDetailsProvider instance */ protected DBErrorDetailsProvider getErrorDetailsProvider() { - return new DBErrorDetailsProvider(); + if (dbErrorDetailsProvider == null) { + dbErrorDetailsProvider = new DBErrorDetailsProvider(); + } + return dbErrorDetailsProvider; } private DriverCleanup loadPluginClassAndGetDriver(Class driverClass) diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java index de7e33482..418b46c9c 100644 --- a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java @@ -41,6 +41,7 @@ public class MariadbSink extends AbstractDBSink { private final MariadbSinkConfig mariadbSinkConfig; + private MariadbErrorDetailsProvider mariadbErrorDetailsProvider; /** * This is the constructor for MariadbSink. @@ -69,7 +70,10 @@ protected String getErrorDetailsProviderClassName() { @Override protected MariadbErrorDetailsProvider getErrorDetailsProvider() { - return new MariadbErrorDetailsProvider(); + if (mariadbErrorDetailsProvider == null) { + mariadbErrorDetailsProvider = new MariadbErrorDetailsProvider(); + } + return mariadbErrorDetailsProvider; } @Override diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java index d3014cfbc..d0b2cac1b 100644 --- a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java @@ -46,6 +46,7 @@ public class MariadbSource extends AbstractDBSource { private final MariadbSourceConfig mariadbSourceConfig; + private MariadbErrorDetailsProvider mariadbErrorDetailsProvider; /** * This is the constructor for MariadbSource. @@ -90,7 +91,10 @@ protected String getErrorDetailsProviderClassName() { @Override protected MariadbErrorDetailsProvider getErrorDetailsProvider() { - return new MariadbErrorDetailsProvider(); + if (mariadbErrorDetailsProvider == null) { + mariadbErrorDetailsProvider = new MariadbErrorDetailsProvider(); + } + return mariadbErrorDetailsProvider; } @Override diff --git a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerConnector.java b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerConnector.java index 0201d48b4..64d262b88 100644 --- a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerConnector.java +++ b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerConnector.java @@ -53,6 +53,7 @@ public class SqlServerConnector extends AbstractDBSpecificConnector { public static final String NAME = "SQL Server"; private final SqlServerConnectorConfig config; + private SqlServerErrorDetailsProvider sqlServerErrorDetailsProvider; public SqlServerConnector(SqlServerConnectorConfig config) { super(config); @@ -151,7 +152,10 @@ protected String getStratifiedQuery(String tableName, int limit, String strata, @Override protected SqlServerErrorDetailsProvider getErrorDetailsProvider() { - return new SqlServerErrorDetailsProvider(); + if (sqlServerErrorDetailsProvider == null) { + sqlServerErrorDetailsProvider = new SqlServerErrorDetailsProvider(); + } + return sqlServerErrorDetailsProvider; } } diff --git a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java index eb1199e49..bb654f558 100644 --- a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java +++ b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java @@ -57,6 +57,7 @@ public class SqlServerSink extends AbstractDBSink { private final SqlServerSourceConfig sqlServerSourceConfig; + private SqlServerErrorDetailsProvider sqlServerErrorDetailsProvider; public SqlServerSource(SqlServerSourceConfig sqlServerSourceConfig) { super(sqlServerSourceConfig); @@ -82,7 +83,10 @@ protected String getErrorDetailsProviderClassName() { @Override protected SqlServerErrorDetailsProvider getErrorDetailsProvider() { - return new SqlServerErrorDetailsProvider(); + if (sqlServerErrorDetailsProvider == null) { + sqlServerErrorDetailsProvider = new SqlServerErrorDetailsProvider(); + } + return sqlServerErrorDetailsProvider; } @Override diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnector.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnector.java index 0006c9be1..2365c760b 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnector.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnector.java @@ -50,6 +50,7 @@ public class MysqlConnector extends AbstractDBSpecificConnector { public static final String NAME = "MySQL"; private final MysqlConnectorConfig config; + private MysqlErrorDetailsProvider mysqlErrorDetailsProvider; public MysqlConnector(MysqlConnectorConfig config) { super(config); @@ -132,6 +133,9 @@ public StructuredRecord transform(LongWritable longWritable, MysqlDBRecord mysql @Override protected MysqlErrorDetailsProvider getErrorDetailsProvider() { - return new MysqlErrorDetailsProvider(); + if (mysqlErrorDetailsProvider == null) { + mysqlErrorDetailsProvider = new MysqlErrorDetailsProvider(); + } + return mysqlErrorDetailsProvider; } } diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java index 7e2044eed..793e25389 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java @@ -60,6 +60,7 @@ public class MysqlSink extends AbstractDBSink { private final MysqlSinkConfig mysqlSinkConfig; private static final Character ESCAPE_CHAR = '`'; + private MysqlErrorDetailsProvider mysqlErrorDetailsProvider; public MysqlSink(MysqlSinkConfig mysqlSinkConfig) { super(mysqlSinkConfig); @@ -116,7 +117,10 @@ protected String getErrorDetailsProviderClassName() { @Override protected MysqlErrorDetailsProvider getErrorDetailsProvider() { - return new MysqlErrorDetailsProvider(); + if (mysqlErrorDetailsProvider == null) { + mysqlErrorDetailsProvider = new MysqlErrorDetailsProvider(); + } + return mysqlErrorDetailsProvider; } @Override diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java index 4eb1a4afe..d61aabbd1 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java @@ -53,6 +53,7 @@ public class MysqlSource extends AbstractDBSource { private final MysqlSourceConfig mysqlSourceConfig; + private MysqlErrorDetailsProvider mysqlErrorDetailsProvider; public MysqlSource(MysqlSourceConfig mysqlSourceConfig) { super(mysqlSourceConfig); @@ -96,7 +97,10 @@ protected String getErrorDetailsProviderClassName() { @Override protected MysqlErrorDetailsProvider getErrorDetailsProvider() { - return new MysqlErrorDetailsProvider(); + if (mysqlErrorDetailsProvider == null) { + mysqlErrorDetailsProvider = new MysqlErrorDetailsProvider(); + } + return mysqlErrorDetailsProvider; } /** diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java index 63920260a..27e6d97e2 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java @@ -55,6 +55,7 @@ public class OracleConnector extends AbstractDBSpecificConnector { private final OracleSinkConfig oracleSinkConfig; + private OracleErrorDetailsProvider oracleErrorDetailsProvider; public OracleSink(OracleSinkConfig oracleSinkConfig) { super(oracleSinkConfig); @@ -89,7 +90,10 @@ protected String getErrorDetailsProviderClassName() { @Override protected OracleErrorDetailsProvider getErrorDetailsProvider() { - return new OracleErrorDetailsProvider(); + if (oracleErrorDetailsProvider == null) { + oracleErrorDetailsProvider = new OracleErrorDetailsProvider(); + } + return oracleErrorDetailsProvider; } @Override diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java index b79b6f38d..804058611 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java @@ -50,6 +50,7 @@ public class OracleSource extends AbstractDBSource { private final OracleSourceConfig oracleSourceConfig; + private OracleErrorDetailsProvider oracleErrorDetailsProvider; public OracleSource(OracleSourceConfig oracleSourceConfig) { super(oracleSourceConfig); @@ -83,7 +84,10 @@ protected String getErrorDetailsProviderClassName() { @Override protected OracleErrorDetailsProvider getErrorDetailsProvider() { - return new OracleErrorDetailsProvider(); + if (oracleErrorDetailsProvider == null) { + oracleErrorDetailsProvider = new OracleErrorDetailsProvider(); + } + return oracleErrorDetailsProvider; } @Override diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConnector.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConnector.java index 0fe1a74cd..817a1676e 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConnector.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConnector.java @@ -51,6 +51,7 @@ public class PostgresConnector extends AbstractDBSpecificConnector { public static final String NAME = "PostgreSQL"; private final PostgresConnectorConfig config; + private PostgresErrorDetailsProvider postgresErrorDetailsProvider; public PostgresConnector(PostgresConnectorConfig config) { super(config); @@ -145,6 +146,9 @@ protected String getStratifiedQuery(String tableName, int limit, String strata, @Override protected PostgresErrorDetailsProvider getErrorDetailsProvider() { - return new PostgresErrorDetailsProvider(); + if (postgresErrorDetailsProvider == null) { + postgresErrorDetailsProvider = new PostgresErrorDetailsProvider(); + } + return postgresErrorDetailsProvider; } } diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java index a417b665f..fd9363908 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java @@ -65,6 +65,7 @@ public class PostgresSink extends AbstractDBSink { private final PostgresSourceConfig postgresSourceConfig; + private PostgresErrorDetailsProvider postgresErrorDetailsProvider; public PostgresSource(PostgresSourceConfig postgresSourceConfig) { super(postgresSourceConfig); @@ -74,7 +75,10 @@ protected String getErrorDetailsProviderClassName() { @Override protected PostgresErrorDetailsProvider getErrorDetailsProvider() { - return new PostgresErrorDetailsProvider(); + if (postgresErrorDetailsProvider == null) { + postgresErrorDetailsProvider = new PostgresErrorDetailsProvider(); + } + return postgresErrorDetailsProvider; } @Override From 4970405b8295435c449b52466af4f42c2bbb7e7a Mon Sep 17 00:00:00 2001 From: Sanchit Garg Date: Thu, 26 Jun 2025 13:12:02 +0530 Subject: [PATCH 19/28] PLUGIN-1823: Remove instance variables from sub-class and change return types for overridden methods to base class --- .../plugin/cloudsql/mysql/CloudSQLMySQLConnector.java | 10 +++++----- .../cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java | 10 +++++----- .../plugin/cloudsql/mysql/CloudSQLMySQLSource.java | 10 +++++----- .../cloudsql/postgres/CloudSQLPostgreSQLConnector.java | 10 +++++----- .../cloudsql/postgres/CloudSQLPostgreSQLSink.java | 10 +++++----- .../cloudsql/postgres/CloudSQLPostgreSQLSource.java | 10 +++++----- .../plugin/db/action/AbstractDBArgumentSetter.java | 2 +- .../src/main/java/io/cdap/plugin/db/action/DBRun.java | 2 +- .../db/connector/AbstractDBSpecificConnector.java | 6 +++++- .../java/io/cdap/plugin/db/sink/AbstractDBSink.java | 2 +- .../io/cdap/plugin/db/source/AbstractDBSource.java | 2 +- .../main/java/io/cdap/plugin/mariadb/MariadbSink.java | 10 +++++----- .../java/io/cdap/plugin/mariadb/MariadbSource.java | 9 +++++---- .../java/io/cdap/plugin/mssql/SqlServerConnector.java | 10 +++++----- .../main/java/io/cdap/plugin/mssql/SqlServerSink.java | 10 +++++----- .../java/io/cdap/plugin/mssql/SqlServerSource.java | 10 +++++----- .../main/java/io/cdap/plugin/mysql/MysqlConnector.java | 10 +++++----- .../src/main/java/io/cdap/plugin/mysql/MysqlSink.java | 10 +++++----- .../main/java/io/cdap/plugin/mysql/MysqlSource.java | 10 +++++----- .../java/io/cdap/plugin/oracle/OracleConnector.java | 10 +++++----- .../main/java/io/cdap/plugin/oracle/OracleSink.java | 10 +++++----- .../main/java/io/cdap/plugin/oracle/OracleSource.java | 10 +++++----- .../io/cdap/plugin/postgres/PostgresConnector.java | 10 +++++----- .../java/io/cdap/plugin/postgres/PostgresSink.java | 10 +++++----- .../java/io/cdap/plugin/postgres/PostgresSource.java | 10 +++++----- 25 files changed, 109 insertions(+), 104 deletions(-) diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java index e3ba89d65..0751bd160 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLConnector.java @@ -31,6 +31,7 @@ import io.cdap.plugin.common.Constants; import io.cdap.plugin.common.ReferenceNames; import io.cdap.plugin.common.db.DBConnectorPath; +import io.cdap.plugin.common.db.DBErrorDetailsProvider; import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.connector.AbstractDBSpecificConnector; @@ -53,7 +54,6 @@ public class CloudSQLMySQLConnector extends AbstractDBSpecificConnector { private final CloudSQLMySQLSourceConfig cloudsqlMysqlSourceConfig; - private CloudSQLMySQLErrorDetailsProvider cloudSQLMySQLErrorDetailsProvider; public CloudSQLMySQLSource(CloudSQLMySQLSourceConfig cloudsqlMysqlSourceConfig) { super(cloudsqlMysqlSourceConfig); @@ -140,11 +140,11 @@ protected String getErrorDetailsProviderClassName() { } @Override - protected CloudSQLMySQLErrorDetailsProvider getErrorDetailsProvider() { - if (cloudSQLMySQLErrorDetailsProvider == null) { - cloudSQLMySQLErrorDetailsProvider = new CloudSQLMySQLErrorDetailsProvider(); + protected DBErrorDetailsProvider getErrorDetailsProvider() { + if (dbErrorDetailsProvider == null) { + dbErrorDetailsProvider = new CloudSQLMySQLErrorDetailsProvider(); } - return cloudSQLMySQLErrorDetailsProvider; + return dbErrorDetailsProvider; } /** CloudSQL MySQL source config. */ diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnector.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnector.java index d89f53346..348a1b94c 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnector.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLConnector.java @@ -30,6 +30,7 @@ import io.cdap.plugin.common.Constants; import io.cdap.plugin.common.ReferenceNames; import io.cdap.plugin.common.db.DBConnectorPath; +import io.cdap.plugin.common.db.DBErrorDetailsProvider; import io.cdap.plugin.common.db.DBPath; import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.connector.AbstractDBSpecificConnector; @@ -52,7 +53,6 @@ public class CloudSQLPostgreSQLConnector extends AbstractDBSpecificConnector { public static final String NAME = CloudSQLPostgreSQLConstants.PLUGIN_NAME; private final CloudSQLPostgreSQLConnectorConfig config; - private CloudSQLPostgreSQLErrorDetailsProvider cloudSQLPostgreSQLErrorDetailsProvider; public CloudSQLPostgreSQLConnector(CloudSQLPostgreSQLConnectorConfig config) { super(config); @@ -121,10 +121,10 @@ protected void setConnectorSpec(ConnectorSpecRequest request, DBConnectorPath pa } @Override - protected CloudSQLPostgreSQLErrorDetailsProvider getErrorDetailsProvider() { - if (cloudSQLPostgreSQLErrorDetailsProvider == null) { - cloudSQLPostgreSQLErrorDetailsProvider = new CloudSQLPostgreSQLErrorDetailsProvider(); + protected DBErrorDetailsProvider getErrorDetailsProvider() { + if (dbErrorDetailsProvider == null) { + dbErrorDetailsProvider = new CloudSQLPostgreSQLErrorDetailsProvider(); } - return cloudSQLPostgreSQLErrorDetailsProvider; + return dbErrorDetailsProvider; } } diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java index d5ff5694d..496238965 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java @@ -36,6 +36,7 @@ import io.cdap.plugin.common.ConfigUtil; import io.cdap.plugin.common.LineageRecorder; import io.cdap.plugin.common.batch.sink.SinkOutputFormatProvider; +import io.cdap.plugin.common.db.DBErrorDetailsProvider; import io.cdap.plugin.db.DBRecord; import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.config.AbstractDBSpecificSinkConfig; @@ -66,7 +67,6 @@ public class CloudSQLPostgreSQLSink extends AbstractDBSink { private final CloudSQLPostgreSQLSourceConfig cloudsqlPostgresqlSourceConfig; - private CloudSQLPostgreSQLErrorDetailsProvider cloudSQLPostgreSQLErrorDetailsProvider; public CloudSQLPostgreSQLSource(CloudSQLPostgreSQLSourceConfig cloudsqlPostgresqlSourceConfig) { super(cloudsqlPostgresqlSourceConfig); @@ -99,11 +99,11 @@ protected String getErrorDetailsProviderClassName() { } @Override - protected CloudSQLPostgreSQLErrorDetailsProvider getErrorDetailsProvider() { - if (cloudSQLPostgreSQLErrorDetailsProvider == null) { - cloudSQLPostgreSQLErrorDetailsProvider = new CloudSQLPostgreSQLErrorDetailsProvider(); + protected DBErrorDetailsProvider getErrorDetailsProvider() { + if (dbErrorDetailsProvider == null) { + dbErrorDetailsProvider = new CloudSQLPostgreSQLErrorDetailsProvider(); } - return cloudSQLPostgreSQLErrorDetailsProvider; + return dbErrorDetailsProvider; } @Override diff --git a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java index 3d10bfc9c..e59fff513 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java @@ -45,7 +45,7 @@ public class AbstractDBArgumentSetter extends Action { private static final String JDBC_PLUGIN_ID = "driver"; private final ArgumentSetterConfig config; private final RetryPolicy retryPolicy; - private DBErrorDetailsProvider dbErrorDetailsProvider; + protected DBErrorDetailsProvider dbErrorDetailsProvider; public AbstractDBArgumentSetter(ArgumentSetterConfig config) { this.config = config; diff --git a/database-commons/src/main/java/io/cdap/plugin/db/action/DBRun.java b/database-commons/src/main/java/io/cdap/plugin/db/action/DBRun.java index 90485c723..1aaf000bd 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/action/DBRun.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/action/DBRun.java @@ -38,7 +38,7 @@ public class DBRun { private final Class driverClass; private boolean enableAutoCommit; private final RetryPolicy retryPolicy; - private DBErrorDetailsProvider dbErrorDetailsProvider; + protected DBErrorDetailsProvider dbErrorDetailsProvider; public DBRun(QueryConfig config, Class driverClass, Boolean enableAutocommit) { this.config = config; diff --git a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java index 368d04581..0308cf7a4 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/connector/AbstractDBSpecificConnector.java @@ -61,6 +61,7 @@ public abstract class AbstractDBSpecificConnector extends private final AbstractDBConnectorConfig config; private final RetryPolicy retryPolicy; + protected DBErrorDetailsProvider dbErrorDetailsProvider; protected AbstractDBSpecificConnector(AbstractDBConnectorConfig config) { super(config); @@ -130,7 +131,10 @@ public InputFormatProvider getInputFormatProvider(ConnectorContext context, Samp * @return DBErrorDetailsProvider instance */ protected DBErrorDetailsProvider getErrorDetailsProvider() { - return new DBErrorDetailsProvider(); + if (dbErrorDetailsProvider == null) { + dbErrorDetailsProvider = new DBErrorDetailsProvider(); + } + return dbErrorDetailsProvider; } protected Connection getConnection(DBConnectorPath path) { diff --git a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java index 8db9992fa..61f3fda6f 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java @@ -98,7 +98,7 @@ public abstract class AbstractDBSink retryPolicy; - private DBErrorDetailsProvider dbErrorDetailsProvider; + protected DBErrorDetailsProvider dbErrorDetailsProvider; public AbstractDBSink(T dbSinkConfig) { super(new ReferencePluginConfig(dbSinkConfig.getReferenceName())); diff --git a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java index 8984b0aed..a278d0613 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java @@ -88,7 +88,7 @@ public abstract class AbstractDBSource retryPolicy; - private DBErrorDetailsProvider dbErrorDetailsProvider; + protected DBErrorDetailsProvider dbErrorDetailsProvider; protected final T sourceConfig; protected Class driverClass; diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java index 418b46c9c..19335b1fa 100644 --- a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java @@ -21,6 +21,7 @@ import io.cdap.cdap.api.annotation.Plugin; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.etl.api.batch.BatchSink; +import io.cdap.plugin.common.db.DBErrorDetailsProvider; import io.cdap.plugin.db.DBRecord; import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.config.DBSpecificSinkConfig; @@ -41,7 +42,6 @@ public class MariadbSink extends AbstractDBSink { private final MariadbSinkConfig mariadbSinkConfig; - private MariadbErrorDetailsProvider mariadbErrorDetailsProvider; /** * This is the constructor for MariadbSink. @@ -69,11 +69,11 @@ protected String getErrorDetailsProviderClassName() { } @Override - protected MariadbErrorDetailsProvider getErrorDetailsProvider() { - if (mariadbErrorDetailsProvider == null) { - mariadbErrorDetailsProvider = new MariadbErrorDetailsProvider(); + protected DBErrorDetailsProvider getErrorDetailsProvider() { + if (dbErrorDetailsProvider == null) { + dbErrorDetailsProvider = new MariadbErrorDetailsProvider(); } - return mariadbErrorDetailsProvider; + return dbErrorDetailsProvider; } @Override diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java index d0b2cac1b..9f51c482e 100644 --- a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java @@ -25,6 +25,7 @@ import io.cdap.cdap.etl.api.batch.BatchSourceContext; import io.cdap.plugin.common.Asset; import io.cdap.plugin.common.LineageRecorder; +import io.cdap.plugin.common.db.DBErrorDetailsProvider; import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.config.DBSpecificSourceConfig; import io.cdap.plugin.db.source.AbstractDBSource; @@ -90,11 +91,11 @@ protected String getErrorDetailsProviderClassName() { } @Override - protected MariadbErrorDetailsProvider getErrorDetailsProvider() { - if (mariadbErrorDetailsProvider == null) { - mariadbErrorDetailsProvider = new MariadbErrorDetailsProvider(); + protected DBErrorDetailsProvider getErrorDetailsProvider() { + if (dbErrorDetailsProvider == null) { + dbErrorDetailsProvider = new MariadbErrorDetailsProvider(); } - return mariadbErrorDetailsProvider; + return dbErrorDetailsProvider; } @Override diff --git a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerConnector.java b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerConnector.java index 64d262b88..7e86d97ce 100644 --- a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerConnector.java +++ b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerConnector.java @@ -32,6 +32,7 @@ import io.cdap.plugin.common.Constants; import io.cdap.plugin.common.ReferenceNames; import io.cdap.plugin.common.db.DBConnectorPath; +import io.cdap.plugin.common.db.DBErrorDetailsProvider; import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.connector.AbstractDBSpecificConnector; @@ -53,7 +54,6 @@ public class SqlServerConnector extends AbstractDBSpecificConnector { public static final String NAME = "SQL Server"; private final SqlServerConnectorConfig config; - private SqlServerErrorDetailsProvider sqlServerErrorDetailsProvider; public SqlServerConnector(SqlServerConnectorConfig config) { super(config); @@ -151,11 +151,11 @@ protected String getStratifiedQuery(String tableName, int limit, String strata, } @Override - protected SqlServerErrorDetailsProvider getErrorDetailsProvider() { - if (sqlServerErrorDetailsProvider == null) { - sqlServerErrorDetailsProvider = new SqlServerErrorDetailsProvider(); + protected DBErrorDetailsProvider getErrorDetailsProvider() { + if (dbErrorDetailsProvider == null) { + dbErrorDetailsProvider = new SqlServerErrorDetailsProvider(); } - return sqlServerErrorDetailsProvider; + return dbErrorDetailsProvider; } } diff --git a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java index bb654f558..7c3f55068 100644 --- a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java +++ b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java @@ -31,6 +31,7 @@ import io.cdap.plugin.common.Asset; import io.cdap.plugin.common.ConfigUtil; import io.cdap.plugin.common.LineageRecorder; +import io.cdap.plugin.common.db.DBErrorDetailsProvider; import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.db.DBRecord; import io.cdap.plugin.db.SchemaReader; @@ -57,7 +58,6 @@ public class SqlServerSink extends AbstractDBSink { private final SqlServerSourceConfig sqlServerSourceConfig; - private SqlServerErrorDetailsProvider sqlServerErrorDetailsProvider; public SqlServerSource(SqlServerSourceConfig sqlServerSourceConfig) { super(sqlServerSourceConfig); @@ -82,11 +82,11 @@ protected String getErrorDetailsProviderClassName() { } @Override - protected SqlServerErrorDetailsProvider getErrorDetailsProvider() { - if (sqlServerErrorDetailsProvider == null) { - sqlServerErrorDetailsProvider = new SqlServerErrorDetailsProvider(); + protected DBErrorDetailsProvider getErrorDetailsProvider() { + if (dbErrorDetailsProvider == null) { + dbErrorDetailsProvider = new SqlServerErrorDetailsProvider(); } - return sqlServerErrorDetailsProvider; + return dbErrorDetailsProvider; } @Override diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnector.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnector.java index 2365c760b..214d8650e 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnector.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlConnector.java @@ -32,6 +32,7 @@ import io.cdap.plugin.common.Constants; import io.cdap.plugin.common.ReferenceNames; import io.cdap.plugin.common.db.DBConnectorPath; +import io.cdap.plugin.common.db.DBErrorDetailsProvider; import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.connector.AbstractDBSpecificConnector; import org.apache.hadoop.io.LongWritable; @@ -50,7 +51,6 @@ public class MysqlConnector extends AbstractDBSpecificConnector { public static final String NAME = "MySQL"; private final MysqlConnectorConfig config; - private MysqlErrorDetailsProvider mysqlErrorDetailsProvider; public MysqlConnector(MysqlConnectorConfig config) { super(config); @@ -132,10 +132,10 @@ public StructuredRecord transform(LongWritable longWritable, MysqlDBRecord mysql } @Override - protected MysqlErrorDetailsProvider getErrorDetailsProvider() { - if (mysqlErrorDetailsProvider == null) { - mysqlErrorDetailsProvider = new MysqlErrorDetailsProvider(); + protected DBErrorDetailsProvider getErrorDetailsProvider() { + if (dbErrorDetailsProvider == null) { + dbErrorDetailsProvider = new MysqlErrorDetailsProvider(); } - return mysqlErrorDetailsProvider; + return dbErrorDetailsProvider; } } diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java index 793e25389..898b32c17 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java @@ -33,6 +33,7 @@ import io.cdap.plugin.common.Asset; import io.cdap.plugin.common.ConfigUtil; import io.cdap.plugin.common.LineageRecorder; +import io.cdap.plugin.common.db.DBErrorDetailsProvider; import io.cdap.plugin.db.ConnectionConfig; import io.cdap.plugin.db.DBRecord; import io.cdap.plugin.db.SchemaReader; @@ -60,7 +61,6 @@ public class MysqlSink extends AbstractDBSink { private final MysqlSinkConfig mysqlSinkConfig; private static final Character ESCAPE_CHAR = '`'; - private MysqlErrorDetailsProvider mysqlErrorDetailsProvider; public MysqlSink(MysqlSinkConfig mysqlSinkConfig) { super(mysqlSinkConfig); @@ -116,11 +116,11 @@ protected String getErrorDetailsProviderClassName() { } @Override - protected MysqlErrorDetailsProvider getErrorDetailsProvider() { - if (mysqlErrorDetailsProvider == null) { - mysqlErrorDetailsProvider = new MysqlErrorDetailsProvider(); + protected DBErrorDetailsProvider getErrorDetailsProvider() { + if (dbErrorDetailsProvider == null) { + dbErrorDetailsProvider = new MysqlErrorDetailsProvider(); } - return mysqlErrorDetailsProvider; + return dbErrorDetailsProvider; } @Override diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java index d61aabbd1..f9bcccfd6 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java @@ -31,6 +31,7 @@ import io.cdap.plugin.common.Asset; import io.cdap.plugin.common.ConfigUtil; import io.cdap.plugin.common.LineageRecorder; +import io.cdap.plugin.common.db.DBErrorDetailsProvider; import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.config.AbstractDBSpecificSourceConfig; import io.cdap.plugin.db.source.AbstractDBSource; @@ -53,7 +54,6 @@ public class MysqlSource extends AbstractDBSource { private final MysqlSourceConfig mysqlSourceConfig; - private MysqlErrorDetailsProvider mysqlErrorDetailsProvider; public MysqlSource(MysqlSourceConfig mysqlSourceConfig) { super(mysqlSourceConfig); @@ -96,11 +96,11 @@ protected String getErrorDetailsProviderClassName() { } @Override - protected MysqlErrorDetailsProvider getErrorDetailsProvider() { - if (mysqlErrorDetailsProvider == null) { - mysqlErrorDetailsProvider = new MysqlErrorDetailsProvider(); + protected DBErrorDetailsProvider getErrorDetailsProvider() { + if (dbErrorDetailsProvider == null) { + dbErrorDetailsProvider = new MysqlErrorDetailsProvider(); } - return mysqlErrorDetailsProvider; + return dbErrorDetailsProvider; } /** diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java index 27e6d97e2..b2ee1470e 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleConnector.java @@ -32,6 +32,7 @@ import io.cdap.plugin.common.Constants; import io.cdap.plugin.common.ReferenceNames; import io.cdap.plugin.common.db.DBConnectorPath; +import io.cdap.plugin.common.db.DBErrorDetailsProvider; import io.cdap.plugin.common.db.DBPath; import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.connector.AbstractDBSpecificConnector; @@ -55,7 +56,6 @@ public class OracleConnector extends AbstractDBSpecificConnector { private final OracleSinkConfig oracleSinkConfig; - private OracleErrorDetailsProvider oracleErrorDetailsProvider; public OracleSink(OracleSinkConfig oracleSinkConfig) { super(oracleSinkConfig); @@ -89,11 +89,11 @@ protected String getErrorDetailsProviderClassName() { } @Override - protected OracleErrorDetailsProvider getErrorDetailsProvider() { - if (oracleErrorDetailsProvider == null) { - oracleErrorDetailsProvider = new OracleErrorDetailsProvider(); + protected DBErrorDetailsProvider getErrorDetailsProvider() { + if (dbErrorDetailsProvider == null) { + dbErrorDetailsProvider = new OracleErrorDetailsProvider(); } - return oracleErrorDetailsProvider; + return dbErrorDetailsProvider; } @Override diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java index 804058611..5f7d1b928 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java @@ -30,6 +30,7 @@ import io.cdap.plugin.common.Asset; import io.cdap.plugin.common.ConfigUtil; import io.cdap.plugin.common.LineageRecorder; +import io.cdap.plugin.common.db.DBErrorDetailsProvider; import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.config.AbstractDBSpecificSourceConfig; import io.cdap.plugin.db.source.AbstractDBSource; @@ -50,7 +51,6 @@ public class OracleSource extends AbstractDBSource { private final OracleSourceConfig oracleSourceConfig; - private OracleErrorDetailsProvider oracleErrorDetailsProvider; public OracleSource(OracleSourceConfig oracleSourceConfig) { super(oracleSourceConfig); @@ -83,11 +83,11 @@ protected String getErrorDetailsProviderClassName() { } @Override - protected OracleErrorDetailsProvider getErrorDetailsProvider() { - if (oracleErrorDetailsProvider == null) { - oracleErrorDetailsProvider = new OracleErrorDetailsProvider(); + protected DBErrorDetailsProvider getErrorDetailsProvider() { + if (dbErrorDetailsProvider == null) { + dbErrorDetailsProvider = new OracleErrorDetailsProvider(); } - return oracleErrorDetailsProvider; + return dbErrorDetailsProvider; } @Override diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConnector.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConnector.java index 817a1676e..5b23ce6cd 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConnector.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresConnector.java @@ -31,6 +31,7 @@ import io.cdap.plugin.common.Constants; import io.cdap.plugin.common.ReferenceNames; import io.cdap.plugin.common.db.DBConnectorPath; +import io.cdap.plugin.common.db.DBErrorDetailsProvider; import io.cdap.plugin.common.db.DBPath; import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.connector.AbstractDBSpecificConnector; @@ -51,7 +52,6 @@ public class PostgresConnector extends AbstractDBSpecificConnector { public static final String NAME = "PostgreSQL"; private final PostgresConnectorConfig config; - private PostgresErrorDetailsProvider postgresErrorDetailsProvider; public PostgresConnector(PostgresConnectorConfig config) { super(config); @@ -145,10 +145,10 @@ protected String getStratifiedQuery(String tableName, int limit, String strata, } @Override - protected PostgresErrorDetailsProvider getErrorDetailsProvider() { - if (postgresErrorDetailsProvider == null) { - postgresErrorDetailsProvider = new PostgresErrorDetailsProvider(); + protected DBErrorDetailsProvider getErrorDetailsProvider() { + if (dbErrorDetailsProvider == null) { + dbErrorDetailsProvider = new PostgresErrorDetailsProvider(); } - return postgresErrorDetailsProvider; + return dbErrorDetailsProvider; } } diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java index fd9363908..ff7c0d12e 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java @@ -35,6 +35,7 @@ import io.cdap.plugin.common.ConfigUtil; import io.cdap.plugin.common.LineageRecorder; import io.cdap.plugin.common.batch.sink.SinkOutputFormatProvider; +import io.cdap.plugin.common.db.DBErrorDetailsProvider; import io.cdap.plugin.db.DBRecord; import io.cdap.plugin.db.SchemaReader; import io.cdap.plugin.db.config.AbstractDBSpecificSinkConfig; @@ -65,7 +66,6 @@ public class PostgresSink extends AbstractDBSink { private final PostgresSourceConfig postgresSourceConfig; - private PostgresErrorDetailsProvider postgresErrorDetailsProvider; public PostgresSource(PostgresSourceConfig postgresSourceConfig) { super(postgresSourceConfig); @@ -74,11 +74,11 @@ protected String getErrorDetailsProviderClassName() { } @Override - protected PostgresErrorDetailsProvider getErrorDetailsProvider() { - if (postgresErrorDetailsProvider == null) { - postgresErrorDetailsProvider = new PostgresErrorDetailsProvider(); + protected DBErrorDetailsProvider getErrorDetailsProvider() { + if (dbErrorDetailsProvider == null) { + dbErrorDetailsProvider = new PostgresErrorDetailsProvider(); } - return postgresErrorDetailsProvider; + return dbErrorDetailsProvider; } @Override From 7c8815dc3849a51d85d2b8624f75aa866a66a56c Mon Sep 17 00:00:00 2001 From: Sanchit Garg Date: Thu, 26 Jun 2025 16:39:31 +0530 Subject: [PATCH 20/28] PLUGIN-1823: Remove getExternalDocumentationLink() method as it is not longer used now. --- .../cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java | 5 ----- .../plugin/cloudsql/mysql/CloudSQLMySQLSource.java | 5 ----- .../cloudsql/postgres/CloudSQLPostgreSQLSink.java | 5 ----- .../cloudsql/postgres/CloudSQLPostgreSQLSource.java | 5 ----- .../plugin/db/action/AbstractDBArgumentSetter.java | 10 ---------- .../java/io/cdap/plugin/db/sink/AbstractDBSink.java | 10 ---------- .../io/cdap/plugin/db/source/AbstractDBSource.java | 10 ---------- .../main/java/io/cdap/plugin/mariadb/MariadbSink.java | 5 ----- .../java/io/cdap/plugin/mariadb/MariadbSource.java | 5 ----- .../main/java/io/cdap/plugin/mssql/SqlServerSink.java | 5 ----- .../java/io/cdap/plugin/mssql/SqlServerSource.java | 5 ----- .../src/main/java/io/cdap/plugin/mysql/MysqlSink.java | 5 ----- .../main/java/io/cdap/plugin/mysql/MysqlSource.java | 5 ----- .../main/java/io/cdap/plugin/oracle/OracleSink.java | 5 ----- .../main/java/io/cdap/plugin/oracle/OracleSource.java | 5 ----- .../java/io/cdap/plugin/postgres/PostgresSink.java | 5 ----- .../java/io/cdap/plugin/postgres/PostgresSource.java | 5 ----- 17 files changed, 100 deletions(-) diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java index be1555b57..30fc229fe 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSink.java @@ -118,11 +118,6 @@ protected DBErrorDetailsProvider getErrorDetailsProvider() { return dbErrorDetailsProvider; } - @Override - protected String getExternalDocumentationLink() { - return DBUtils.CLOUDSQLMYSQL_SUPPORTED_DOC_URL; - } - @Override protected LineageRecorder getLineageRecorder(BatchSinkContext context) { String host; diff --git a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java index 66f778f23..80358ce4b 100644 --- a/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java +++ b/cloudsql-mysql-plugin/src/main/java/io/cdap/plugin/cloudsql/mysql/CloudSQLMySQLSource.java @@ -83,11 +83,6 @@ protected Class getDBRecordType() { return MysqlDBRecord.class; } - @Override - protected String getExternalDocumentationLink() { - return DBUtils.CLOUDSQLMYSQL_SUPPORTED_DOC_URL; - } - @Override protected String createConnectionString() { if (CloudSQLUtil.PRIVATE_INSTANCE.equalsIgnoreCase( diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java index 496238965..dcfdd3579 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSink.java @@ -162,11 +162,6 @@ protected DBErrorDetailsProvider getErrorDetailsProvider() { return dbErrorDetailsProvider; } - @Override - protected String getExternalDocumentationLink() { - return DBUtils.CLOUDSQLPOSTGRES_SUPPORTED_DOC_URL; - } - /** CloudSQL PostgreSQL sink config. */ public static class CloudSQLPostgreSQLSinkConfig extends AbstractDBSpecificSinkConfig { diff --git a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java index 78e0cac69..7ebc7d809 100644 --- a/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java +++ b/cloudsql-postgresql-plugin/src/main/java/io/cdap/plugin/cloudsql/postgres/CloudSQLPostgreSQLSource.java @@ -88,11 +88,6 @@ protected Class getDBRecordType() { return PostgresDBRecord.class; } - @Override - protected String getExternalDocumentationLink() { - return DBUtils.CLOUDSQLPOSTGRES_SUPPORTED_DOC_URL; - } - @Override protected String getErrorDetailsProviderClassName() { return CloudSQLPostgreSQLErrorDetailsProvider.class.getName(); diff --git a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java index e59fff513..d7aeb51c3 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/action/AbstractDBArgumentSetter.java @@ -156,16 +156,6 @@ private void setArguments(ResultSet resultSet, FailureCollector failureCollector } } - /** - * Returns the external documentation link. - * Override this method to provide a custom external documentation link. - * - * @return external documentation link - */ - protected String getExternalDocumentationLink() { - return "https://en.wikipedia.org/wiki/SQLSTATE"; - } - /** * Returns the DBErrorDetailsProvider instance. * Override this method to provide a custom DBErrorDetailsProvider instance. diff --git a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java index 61f3fda6f..90fda0928 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/sink/AbstractDBSink.java @@ -195,16 +195,6 @@ protected DBErrorDetailsProvider getErrorDetailsProvider() { return dbErrorDetailsProvider; } - /** - * Returns the external documentation link. - * Override this method to provide a custom external documentation link. - * - * @return external documentation link - */ - protected String getExternalDocumentationLink() { - return "https://en.wikipedia.org/wiki/SQLSTATE"; - } - @Override public void prepareRun(BatchSinkContext context) { String connectionString = dbSinkConfig.getConnectionString(); diff --git a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java index a278d0613..b74edac9c 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java @@ -376,16 +376,6 @@ protected Class getDBRecordType() { return DBRecord.class; } - /** - * Returns the external documentation link. - * Override this method to provide a custom external documentation link. - * - * @return external documentation link - */ - protected String getExternalDocumentationLink() { - return "https://en.wikipedia.org/wiki/SQLSTATE"; - } - @Override public void initialize(BatchRuntimeContext context) throws Exception { super.initialize(context); diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java index 19335b1fa..1575374c8 100644 --- a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSink.java @@ -76,11 +76,6 @@ protected DBErrorDetailsProvider getErrorDetailsProvider() { return dbErrorDetailsProvider; } - @Override - protected String getExternalDocumentationLink() { - return DBUtils.MARIADB_SUPPORTED_DOC_URL; - } - @Override protected FieldsValidator getFieldsValidator() { return new MariadbFieldsValidator(); diff --git a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java index 9f51c482e..6361ba86c 100644 --- a/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java +++ b/mariadb-plugin/src/main/java/io/cdap/plugin/mariadb/MariadbSource.java @@ -98,11 +98,6 @@ protected DBErrorDetailsProvider getErrorDetailsProvider() { return dbErrorDetailsProvider; } - @Override - protected String getExternalDocumentationLink() { - return DBUtils.MARIADB_SUPPORTED_DOC_URL; - } - /** * MaraiDB source mariadbSourceConfig. */ diff --git a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java index 7c3f55068..00b627139 100644 --- a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java +++ b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSink.java @@ -102,11 +102,6 @@ protected DBErrorDetailsProvider getErrorDetailsProvider() { return dbErrorDetailsProvider; } - @Override - protected String getExternalDocumentationLink() { - return DBUtils.MSSQL_SUPPORTED_DOC_URL; - } - /** * MSSQL action configuration. */ diff --git a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java index 63d4b8997..d8d4cf39e 100644 --- a/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java +++ b/mssql-plugin/src/main/java/io/cdap/plugin/mssql/SqlServerSource.java @@ -99,11 +99,6 @@ protected LineageRecorder getLineageRecorder(BatchSourceContext context) { return new LineageRecorder(context, asset); } - @Override - protected String getExternalDocumentationLink() { - return DBUtils.MSSQL_SUPPORTED_DOC_URL; - } - /** * MSSQL source config. */ diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java index 898b32c17..6487ffae6 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSink.java @@ -123,11 +123,6 @@ protected DBErrorDetailsProvider getErrorDetailsProvider() { return dbErrorDetailsProvider; } - @Override - protected String getExternalDocumentationLink() { - return DBUtils.MYSQL_SUPPORTED_DOC_URL; - } - /** * MySQL action configuration. */ diff --git a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java index f9bcccfd6..d7d7c400b 100644 --- a/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java +++ b/mysql-plugin/src/main/java/io/cdap/plugin/mysql/MysqlSource.java @@ -70,11 +70,6 @@ protected Class getDBRecordType() { return MysqlDBRecord.class; } - @Override - protected String getExternalDocumentationLink() { - return DBUtils.MYSQL_SUPPORTED_DOC_URL; - } - @Override protected LineageRecorder getLineageRecorder(BatchSourceContext context) { String fqn = DBUtils.constructFQN("mysql", diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSink.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSink.java index 9a2f0be7f..40ecfbe9e 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSink.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSink.java @@ -96,11 +96,6 @@ protected DBErrorDetailsProvider getErrorDetailsProvider() { return dbErrorDetailsProvider; } - @Override - protected String getExternalDocumentationLink() { - return DBUtils.ORACLE_SUPPORTED_DOC_URL; - } - /** * Oracle action configuration. */ diff --git a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java index 5f7d1b928..7f3c2061e 100644 --- a/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java +++ b/oracle-plugin/src/main/java/io/cdap/plugin/oracle/OracleSource.java @@ -72,11 +72,6 @@ protected Class getDBRecordType() { return OracleSourceDBRecord.class; } - @Override - protected String getExternalDocumentationLink() { - return DBUtils.ORACLE_SUPPORTED_DOC_URL; - } - @Override protected String getErrorDetailsProviderClassName() { return OracleErrorDetailsProvider.class.getName(); diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java index ff7c0d12e..6d63b64bf 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSink.java @@ -130,11 +130,6 @@ protected DBErrorDetailsProvider getErrorDetailsProvider() { return dbErrorDetailsProvider; } - @Override - protected String getExternalDocumentationLink() { - return DBUtils.POSTGRES_SUPPORTED_DOC_URL; - } - /** * PostgreSQL action configuration. */ diff --git a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java index c87c2453b..661c22a71 100644 --- a/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java +++ b/postgresql-plugin/src/main/java/io/cdap/plugin/postgres/PostgresSource.java @@ -86,11 +86,6 @@ protected Class getDBRecordType() { return PostgresDBRecord.class; } - @Override - protected String getExternalDocumentationLink() { - return DBUtils.POSTGRES_SUPPORTED_DOC_URL; - } - @Override protected LineageRecorder getLineageRecorder(BatchSourceContext context) { String fqn = DBUtils.constructFQN("postgres", From 0b57ce6062cb3c8ef0b8b7521239cc52b37f5966 Mon Sep 17 00:00:00 2001 From: Sanchit Garg Date: Thu, 26 Jun 2025 17:09:39 +0530 Subject: [PATCH 21/28] PLUGIN-1823: Removed empty lines --- .../src/main/java/io/cdap/plugin/db/RetryExceptions.java | 4 +--- .../main/java/io/cdap/plugin/db/source/AbstractDBSource.java | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/database-commons/src/main/java/io/cdap/plugin/db/RetryExceptions.java b/database-commons/src/main/java/io/cdap/plugin/db/RetryExceptions.java index e16531e33..13720ab9a 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/RetryExceptions.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/RetryExceptions.java @@ -14,16 +14,14 @@ * the License. */ - package io.cdap.plugin.db; import java.sql.SQLTransientException; import java.util.HashSet; import java.util.Set; - /** - * Checks whether the given exception or one of its causes is a known retryable SQLException. + * Check if an exception or any of its causes is a retryable {@link java.sql.SQLTransientException}. */ public class RetryExceptions { public static boolean isRetryable(Throwable t) { diff --git a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java index b74edac9c..908d27eb9 100644 --- a/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java +++ b/database-commons/src/main/java/io/cdap/plugin/db/source/AbstractDBSource.java @@ -87,7 +87,6 @@ public abstract class AbstractDBSource retryPolicy; - protected DBErrorDetailsProvider dbErrorDetailsProvider; protected final T sourceConfig; protected Class driverClass; From 26efc64e077dc4e8586fed4cbf7ca17b6f22c399 Mon Sep 17 00:00:00 2001 From: suryakumari Date: Thu, 3 Jul 2025 15:04:06 +0530 Subject: [PATCH 22/28] new error messages fix --- .../e2e-test/resources/errorMessage.properties | 10 +++++----- .../e2e-test/resources/errorMessage.properties | 10 +++++----- .../e2e-test/resources/errorMessage.properties | 16 ++++++++-------- .../e2e-test/resources/errorMessage.properties | 16 ++++++++-------- .../e2e-test/resources/errorMessage.properties | 12 ++++++------ 5 files changed, 32 insertions(+), 32 deletions(-) diff --git a/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties b/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties index 4556e268c..e492d1400 100644 --- a/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties @@ -1,9 +1,9 @@ -errorMessageInvalidSourceDatabase=Error occurred while trying to get schema from database. +errorMessageInvalidSourceDatabase=SQL Error occurred, sqlState: '42000', errorCode: '1044', errorMessage: SQL Exception occurred: [Message='Access denied errorMessageInvalidImportQuery=Import Query select must contain the string '$CONDITIONS'. if Number of Splits is not set\ \ to 1. Include '$CONDITIONS' in the Import Query errorMessageCloudMySqlInvalidReferenceName=Invalid reference name errorMessageBlankUsername=Username is required when password is given. -errorMessageBlankPassword=Error occurred while trying to get schema from database.Error message: 'Access denied for user +errorMessageBlankPassword=SQL Error occurred, sqlState: '28000', errorCode: '1045', errorMessage: SQL Exception occurred: [Message='Access denied for user ' errorMessageInvalidFetchSize=Invalid fetch size. Fetch size must be a positive integer. errorMessageBlankSplitBy=Split-By Field Name must be specified if Number of Splits is not set to 1. Specify the Split-by Field Name. errorMessageInvalidNumberOfSplits=Invalid value for Number of Splits '0'. Must be at least 1. Specify a Number of Splits no less than 1. @@ -17,9 +17,9 @@ validationSuccessMessage=No errors found. validationErrorMessage=COUNT ERROR found errorLogsMessageInvalidTableName=Spark program 'phase-1' failed with error: Stage 'CloudSQL MySQL' encountered : io.cdap.cdap.etl.api.validation.ValidationException: Errors were encountered during validation. \ Table 'Table123' does not exist.. Please check the system logs for more details. -errorLogsMessageInvalidCredentials =Spark program 'phase-1' failed with error: Stage 'CloudSQL MySQL' encountered : io.cdap.cdap.api.exception.ProgramFailureException: Error occurred while trying to get schema from database.Error message: 'Access denied for user 'testUser' +errorLogsMessageInvalidCredentials=Spark program 'phase-1' failed with error: Stage 'CloudSQL MySQL' encountered : io.cdap.cdap.api.exception.ProgramFailureException: SQL Error occurred, sqlState: '28000', errorCode: '1045', errorMessage: SQL Exception occurred: [Message='Access denied for user 'testUser' errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: Stage 'CloudSQL MySQL' encountered : java.io.IOException: You have an error in your SQL syntax; \ check the manual that corresponds to your MySQL server version for the right syntax to use near 'table' at line 1. Please check the system logs for more details. -errorMessageInvalidPassword=Error occurred while trying to get schema from database.Error message: 'Access denied for user +errorMessageInvalidPassword=SQL Error occurred, sqlState: '28000', errorCode: '1045', errorMessage: SQL Exception occurred: [Message='Access denied for user ' errorMessagePrivateConnectionName=Enter the internal IP address of the Compute Engine VM cloudsql proxy is running on, to connect to a private -errorMessageWithBlankPassword=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'Access denied for user +errorMessageWithBlankPassword=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '28000', errorCode: '1045', errorMessage: SQL Exception occurred: [Message='Access denied for user ' diff --git a/cloudsql-postgresql-plugin/src/e2e-test/resources/errorMessage.properties b/cloudsql-postgresql-plugin/src/e2e-test/resources/errorMessage.properties index 1d7e23e01..a344e472f 100644 --- a/cloudsql-postgresql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/cloudsql-postgresql-plugin/src/e2e-test/resources/errorMessage.properties @@ -7,19 +7,19 @@ errorMessageInvalidNumberOfSplits=Invalid value for Number of Splits '0'. Must b errorMessageNumberOfSplitNotNumber=Unable to create config for batchsource CloudSQLPostgreSQL 'numSplits' is invalid: Value of \ field class io.cdap.plugin.db.config.AbstractDBSpecificSourceConfig.numSplits is expected to be a number. errorMessageInvalidFetchSize=Invalid fetch size. Fetch size must be a positive integer. -errorMessageInvalidSourceDatabase=Error occurred while trying to get schema from database.Error message: 'FATAL: database "invalidDatabase" does not exist'. +errorMessageInvalidSourceDatabase=SQL Error occurred, sqlState: '3D000', errorCode: '0', errorMessage: SQL Exception occurred: [Message='FATAL: database "invalidDatabase" does not exist', SQLState='3D000', ErrorCode='0']. errorMessageInvalidImportQuery=Import Query select must contain the string '$CONDITIONS'. if Number of Splits is not set\ \ to 1. Include '$CONDITIONS' in the Import Query errorMessageBlankUsername=Username is required when password is given. -errorMessageBlankPassword=Error occurred while trying to get schema from database.Error message: 'Something unusual has occurred to cause the driver to fail. -errorMessageInvalidPassword=Error occurred while trying to get schema from database.Error message: 'FATAL: password authentication failed for user +errorMessageBlankPassword=SQL Error occurred, sqlState: '99999', errorCode: '0', errorMessage: SQL Exception occurred: [Message='Something unusual has occurred to cause the driver to fail. Please report this exception.', SQLState='99999', ErrorCode='0']. +errorMessageInvalidPassword=SQL Error occurred, sqlState: '28P01', errorCode: '0', errorMessage: SQL Exception occurred: [Message='FATAL: password authentication failed for user errorMessageInvalidSourceHost=SQL error while getting query schema: The connection attempt failed. errorMessageInvalidTableName=Table 'table' does not exist. Ensure table '"table"' is set correctly and that the -errorMessageInvalidSinkDatabase=Error occurred while trying to get schema from database.Error message: 'FATAL: database "invalidDB" does not exist'. Error code: '0'. SQLState: '3D000'' +errorMessageInvalidSinkDatabase=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '3D000', errorCode: '0', errorMessage: SQL Exception occurred: [Message='FATAL: database "invalidDB" does not exist', SQLState='3D000', ErrorCode='0'].' errorLogsMessageInvalidBoundingQuery=The column index is out of range: 1, number of columns: 0.. errorMessageConnectionName=Connection Name must be in the format :: to connect to \ a public CloudSQL PostgreSQL instance. errorMessagePrivateConnectionName=Enter the internal IP address of the Compute Engine VM cloudsql proxy is running on, \ to connect to a private CloudSQL PostgreSQL instance. -errorMessageWithBlankPassword=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'Something unusual has occurred to cause the driver to fail. Please report this exception.'. Error code: '0'. SQLState: '99999'' +errorMessageWithBlankPassword=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '99999', errorCode: '0', errorMessage: SQL Exception occurred: [Message='Something unusual has occurred to cause the driver to fail. Please report this exception.', SQLState='99999', ErrorCode='0'].' errorMessageUpdateUpsertOperationName=Table key must be set if the operation is 'Update' or 'Upsert'. diff --git a/mssql-plugin/src/e2e-test/resources/errorMessage.properties b/mssql-plugin/src/e2e-test/resources/errorMessage.properties index f33ae97db..1f328f4d6 100644 --- a/mssql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/mssql-plugin/src/e2e-test/resources/errorMessage.properties @@ -1,27 +1,27 @@ validationSuccessMessage=No errors found. -errorMessageInvalidSourceDatabase=Error occurred while trying to get schema from database.Error message: 'Cannot open database "test123" requested by the login. The login failed. +errorMessageInvalidSourceDatabase=SQL Error occurred, sqlState: 'S0001', errorCode: '4060', errorMessage: SQL Exception occurred: [Message='Cannot open database "test123" requested by the login. The login failed. errorMessageInvalidImportQuery=Import Query select must contain the string '$CONDITIONS'. errorMessageMssqlInvalidReferenceName=Invalid reference name 'invalidRef&^*&&*'. errorMessageBlankUsername=Username is required when password is given. -errorMessageBlankPassword=Error occurred while trying to get schema from database.Error message: 'Login failed for user +errorMessageBlankPassword=SQL Error occurred, sqlState: 'S0001', errorCode: '18456', errorMessage: SQL Exception occurred: [Message='Login failed for user ' errorMessageInvalidFetchSize=Invalid fetch size. Fetch size must be a positive integer. errorMessageBlankSplitBy=Split-By Field Name must be specified if Number of Splits is not set to 1. errorMessageInvalidNumberOfSplits=Invalid value for Number of Splits '0'. Must be at least 1. errorMessageNumberOfSplitNotNumber=Unable to create config for batchsource errorMessageBoundingQuery=Bounding Query must be specified if Number of Splits is not set to 1. errorMessagenumofSplit=Split-By Field Name must be specified if Number of Splits is not set to 1. -errorMessageInvalidSinkDatabase=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'Cannot open database "test123" requested by the login. +errorMessageInvalidSinkDatabase=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: 'S0001', errorCode: '4060', errorMessage: SQL Exception occurred: [Message='Cannot open database "test123" requested by the login. The login failed. errorMessageInvalidSinkTableName=Table 'Table123@' does not exist. -errormessageBlankHost=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'The TCP/IP connection to the host localhost, port 1433 has failed. Error: "Connection refused (Connection refused). \ - Verify the connection properties. Make sure that an instance of SQL Server is running on the host and accepting TCP/IP connections at the port. Make sure that TCP connections to the port are not blocked by a firewall.".'. Error code: '0'. SQLState: '08S01'' +errormessageBlankHost=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '08S01', errorCode: '0', errorMessage: SQL Exception occurred: [Message='The TCP/IP connection to the host localhost, port 1433 has failed. \ + Error: "Connection refused (Connection refused). Verify the connection properties. Make sure that an instance of SQL Server is running on the host and accepting TCP/IP connections at the port. Make sure that TCP connections to the port are not blocked by a firewall.".', SQLState='08S01', ErrorCode='0'].' errorMessageInvalidTableName=Spark program 'phase-1' failed with error: Stage 'SQL Server2' encountered : io.cdap.cdap.etl.api.validation.ValidationException: \ Errors were encountered during validation. Table 'Table123@' does not exist.. Please check the system logs for more details. errorMessageInvalidCredentials=Spark program 'phase-1' failed with error: Unable to create config for batchsink SqlServer \ 'connection' is invalid: Failed to assign value -errorMessageInvalidsourcetable=Spark program 'phase-1' failed with error: Stage 'SQL Server' encountered : io.cdap.cdap.api.exception.ProgramFailureException: \ - Error occurred while trying to get schema from database.Error message: 'Incorrect syntax near the keyword 'table'.'. Error code: '156'. SQLState: 'S0001'. Please check the system logs for more details. +errorMessageInvalidsourcetable=Spark program 'phase-1' failed with error: Stage 'SQL Server' encountered : io.cdap.cdap.api.exception.ProgramFailureException: SQL Error occurred, sqlState: 'S0001', errorCode: '156', \ + errorMessage: SQL Exception occurred: [Message='Incorrect syntax near the keyword 'table'.', SQLState='S0001', ErrorCode='156'].. Please check the system logs for more details. errorMessageInvalidCredentialSource=Spark program 'phase-1' failed with error: Stage 'SQL Server' encountered : java.lang.IllegalArgumentException: \ Plugin with id SQL Server:source.jdbc.sqlserver does not exist in program phase-1 of application errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: Stage 'SQL Server' encountered : java.io.IOException: Could not find stored procedure blank.jdbcPluginName.message=Required property 'jdbcPluginName' has no value. -blank.connection.message=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'Login failed for user ' +blank.connection.message=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: 'S0001', errorCode: '18456', errorMessage: SQL Exception occurred: [Message='Login failed for user ' diff --git a/mysql-plugin/src/e2e-test/resources/errorMessage.properties b/mysql-plugin/src/e2e-test/resources/errorMessage.properties index 72f7a5501..b92a4b7c5 100644 --- a/mysql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/mysql-plugin/src/e2e-test/resources/errorMessage.properties @@ -1,19 +1,19 @@ validationSuccessMessage=No errors found. -invalid.username.message=Error occurred while trying to get schema from database.Error message: 'Access denied for user ' +invalid.username.message=SQL Error occurred, sqlState: '28000', errorCode: '1045', errorMessage: SQL Exception occurred: [Message='Access denied for user ' invalidtableName.error.message=Table '123#' does not exist. Ensure table '123#' is set correctly invalidreferenceName.error.message=Invalid reference name -invalid.host.message=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'Communications link failure The last packet sent successfully to the server was 0 milliseconds ago. The driver has not received any packets from the server.'. Error code: '0'. SQLState: '08S01'' -invalid.password.message=Error occurred while trying to get schema from database.Error message: 'Access denied for user ' -invalid.databasename.message=Error occurred while trying to get schema from database. -invalid.query.message=Error occurred while trying to get schema from database.Error message: 'No tables used'. +invalid.host.message=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '08S01', errorCode: '0', errorMessage: SQL Exception occurred: [Message='Communications link failure The last packet sent successfully to the server was 0 milliseconds ago. The driver has not received any packets from the server.', SQLState='08S01', ErrorCode='0'].' +invalid.password.message=SQL Error occurred, sqlState: '28000', errorCode: '1045', errorMessage: SQL Exception occurred: [Message='Access denied for user ' +invalid.databasename.message=SQL Error occurred, sqlState: '42000', errorCode: '1049', errorMessage: SQL Exception occurred: [Message='Unknown database 'test123'', SQLState='42000', ErrorCode='1049']. +invalid.query.message=SQL Error occurred, sqlState: 'HY000', errorCode: '1096', errorMessage: SQL Exception occurred: [Message='No tables used', SQLState='HY000', ErrorCode='1096']. numberofsplits.error.message=Unable to create config for batchsource Mysql 'numSplits' is invalid boundingQuery.error.message=Bounding Query must be specified if Number of Splits is not set to 1. splitfield.error.message=Split-By Field Name must be specified if Number of Splits is not set to 1 -invalid.sink.database.message=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'Unknown database 'test123''. Error code: '1049'. SQLState: '42000'' +invalid.sink.database.message=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '42000', errorCode: '1049', errorMessage: SQL Exception occurred: [Message='Unknown database 'test123'', SQLState='42000', ErrorCode='1049'].' blank.username.message=Username is required when password is given. errorMessageInvalidFetchSize=Invalid fetch size. Fetch size must be a positive integer. errorMessageInvalidNumberOfSplits=Invalid value for Number of Splits '0'. Must be at least 1. errorMessageInvalidImportQuery=Import Query select must contain the string '$CONDITIONS'. -blank.connection.message=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'Access denied for user -blank.HostConnection.message=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'Communications link failure The last packet sent successfully to the server was 0 milliseconds ago. The driver has not received any packets from the server.'. Error code: '0'. SQLState: '08S01'' +blank.connection.message=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '28000', errorCode: '1045', errorMessage: SQL Exception occurred: [Message='Access denied for user ' +blank.HostConnection.message=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '08S01', errorCode: '0', errorMessage: SQL Exception occurred: [Message='Communications link failure The last packet sent successfully to the server was 0 milliseconds ago. The driver has not received any packets from the server.', SQLState='08S01', ErrorCode='0'].' blank.jdbcPluginName.message=Required property 'jdbcPluginName' has no value. diff --git a/oracle-plugin/src/e2e-test/resources/errorMessage.properties b/oracle-plugin/src/e2e-test/resources/errorMessage.properties index ba94fd8ed..28566fa32 100644 --- a/oracle-plugin/src/e2e-test/resources/errorMessage.properties +++ b/oracle-plugin/src/e2e-test/resources/errorMessage.properties @@ -7,15 +7,15 @@ errorMessageInvalidNumberOfSplits=Invalid value for Number of Splits '0'. Must b errorMessageNumberOfSplitNotNumber=Unable to create config for batchsource Oracle 'numSplits' is invalid: Value of field\ \ class io.cdap.plugin.db.config.AbstractDBSpecificSourceConfig.numSplits is expected to be a number. errorMessageInvalidFetchSize=Invalid fetch size. Fetch size must be a positive integer. -errorMessageInvalidSourceDatabase=Error occurred while trying to get schema from database.Error message: 'Listener refused the connection with the following error: ORA-12514, TNS:listener does not currently know of service requested in connect descriptor '. Error code: '12514'. SQLState: '08006' +errorMessageInvalidSourceDatabase=SQL Error occurred, sqlState: '08006', errorCode: '12514', errorMessage: SQL Exception occurred: [Message='Listener refused the connection with the following error: ORA-12514, TNS:listener does not currently know of service requested in connect descriptor ', SQLState='08006', ErrorCode='12514']. errorMessageInvalidImportQuery=Import Query select must contain the string '$CONDITIONS'. if Number of Splits is not set\ \ to 1. Include '$CONDITIONS' in the Import Query errorMessageBlankUsername=Username is required when password is given. -errorMessageInvalidTableName=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'IO Error: Unknown host specified '. Error code: '17002'. SQLState: '08006'' -errorMessageInvalidSinkDatabase=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database. -errorMessageInvalidHost=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'IO Error: Unknown host specified '. Error code: '17002'. SQLState: '08006'' +errorMessageInvalidTableName=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '08006', errorCode: '17002', errorMessage: SQL Exception occurred: [Message='IO Error: Unknown host specified ', SQLState='08006', ErrorCode='17002'].' +errorMessageInvalidSinkDatabase=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '08006', errorCode: '17002', errorMessage: SQL Exception occurred: [Message='IO Error: Unknown host specified ', SQLState='08006', ErrorCode='17002'].' +errorMessageInvalidHost=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '08006', errorCode: '17002', errorMessage: SQL Exception occurred: [Message='IO Error: Unknown host specified ', SQLState='08006', ErrorCode='17002'].' errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: Stage 'Oracle' encountered : \ java.io.IOException: ORA-00936: missing expression . Please check the system logs for more details. blank.database.message=Required property 'database' has no value. -blank.connection.message=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'ORA-01005: null password given; logon denied '. Error code: '1005'. SQLState: '72000'' -blank.HostBlank.message=Error encountered while configuring the stage: 'Error occurred while trying to get schema from database.Error message: 'IO Error: The Network Adapter could not establish the connection'. Error code: '17002'. SQLState: '08006'' +blank.connection.message=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '72000', errorCode: '1005', errorMessage: SQL Exception occurred: [Message='ORA-01005: null password given; logon denied ', SQLState='72000', ErrorCode='1005'].' +blank.HostBlank.message=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '08006', errorCode: '17002', errorMessage: SQL Exception occurred: [Message='IO Error: The Network Adapter could not establish the connection', SQLState='08006', ErrorCode='17002'].' From 74ce49b8602d99a43b534b054511cb1f7372c007 Mon Sep 17 00:00:00 2001 From: suryakumari Date: Mon, 7 Jul 2025 21:25:14 +0530 Subject: [PATCH 23/28] oracle msg update --- oracle-plugin/src/e2e-test/resources/errorMessage.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oracle-plugin/src/e2e-test/resources/errorMessage.properties b/oracle-plugin/src/e2e-test/resources/errorMessage.properties index 28566fa32..b21efc55b 100644 --- a/oracle-plugin/src/e2e-test/resources/errorMessage.properties +++ b/oracle-plugin/src/e2e-test/resources/errorMessage.properties @@ -12,7 +12,7 @@ errorMessageInvalidImportQuery=Import Query select must contain the string '$CON \ to 1. Include '$CONDITIONS' in the Import Query errorMessageBlankUsername=Username is required when password is given. errorMessageInvalidTableName=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '08006', errorCode: '17002', errorMessage: SQL Exception occurred: [Message='IO Error: Unknown host specified ', SQLState='08006', ErrorCode='17002'].' -errorMessageInvalidSinkDatabase=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '08006', errorCode: '17002', errorMessage: SQL Exception occurred: [Message='IO Error: Unknown host specified ', SQLState='08006', ErrorCode='17002'].' +errorMessageInvalidSinkDatabase=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '22000', errorCode: '17868', errorMessage: SQL Exception occurred: [Message='ORA-17868: Unknown host specified.: dsf: Name or service not known https://docs.oracle.com/error-help/db/ora-17868/', SQLState='22000', ErrorCode='17868'].' errorMessageInvalidHost=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '08006', errorCode: '17002', errorMessage: SQL Exception occurred: [Message='IO Error: Unknown host specified ', SQLState='08006', ErrorCode='17002'].' errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: Stage 'Oracle' encountered : \ java.io.IOException: ORA-00936: missing expression . Please check the system logs for more details. From 172e9bd93a18942b30fb978e1e76d561d8459915 Mon Sep 17 00:00:00 2001 From: suryakumari Date: Tue, 8 Jul 2025 10:32:27 +0530 Subject: [PATCH 24/28] oracle msg new update --- oracle-plugin/src/e2e-test/resources/errorMessage.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/oracle-plugin/src/e2e-test/resources/errorMessage.properties b/oracle-plugin/src/e2e-test/resources/errorMessage.properties index b21efc55b..ecc15e8b2 100644 --- a/oracle-plugin/src/e2e-test/resources/errorMessage.properties +++ b/oracle-plugin/src/e2e-test/resources/errorMessage.properties @@ -12,7 +12,7 @@ errorMessageInvalidImportQuery=Import Query select must contain the string '$CON \ to 1. Include '$CONDITIONS' in the Import Query errorMessageBlankUsername=Username is required when password is given. errorMessageInvalidTableName=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '08006', errorCode: '17002', errorMessage: SQL Exception occurred: [Message='IO Error: Unknown host specified ', SQLState='08006', ErrorCode='17002'].' -errorMessageInvalidSinkDatabase=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '22000', errorCode: '17868', errorMessage: SQL Exception occurred: [Message='ORA-17868: Unknown host specified.: dsf: Name or service not known https://docs.oracle.com/error-help/db/ora-17868/', SQLState='22000', ErrorCode='17868'].' +errorMessageInvalidSinkDatabase=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '22000', errorCode: '17868', errorMessage: SQL Exception occurred: [Message='ORA-17868: Unknown host specified.: invalidDB%$^%*: Name or service not known https://docs.oracle.com/error-help/db/ora-17868/', SQLState='22000', ErrorCode='17868'].' errorMessageInvalidHost=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '08006', errorCode: '17002', errorMessage: SQL Exception occurred: [Message='IO Error: Unknown host specified ', SQLState='08006', ErrorCode='17002'].' errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: Stage 'Oracle' encountered : \ java.io.IOException: ORA-00936: missing expression . Please check the system logs for more details. From 2284f2d9be3c42b4fcc4d6de8bd268d223765ff1 Mon Sep 17 00:00:00 2001 From: suryakumari Date: Wed, 9 Jul 2025 19:24:10 +0530 Subject: [PATCH 25/28] test error msg --- .../src/e2e-test/resources/errorMessage.properties | 2 +- oracle-plugin/src/e2e-test/resources/errorMessage.properties | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties b/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties index e492d1400..6bbc5dee4 100644 --- a/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties @@ -1,4 +1,4 @@ -errorMessageInvalidSourceDatabase=SQL Error occurred, sqlState: '42000', errorCode: '1044', errorMessage: SQL Exception occurred: [Message='Access denied +errorMessageInvalidSourceDatabase=errorMessage: SQL Exception occurred errorMessageInvalidImportQuery=Import Query select must contain the string '$CONDITIONS'. if Number of Splits is not set\ \ to 1. Include '$CONDITIONS' in the Import Query errorMessageCloudMySqlInvalidReferenceName=Invalid reference name diff --git a/oracle-plugin/src/e2e-test/resources/errorMessage.properties b/oracle-plugin/src/e2e-test/resources/errorMessage.properties index ecc15e8b2..0d40301fb 100644 --- a/oracle-plugin/src/e2e-test/resources/errorMessage.properties +++ b/oracle-plugin/src/e2e-test/resources/errorMessage.properties @@ -12,7 +12,7 @@ errorMessageInvalidImportQuery=Import Query select must contain the string '$CON \ to 1. Include '$CONDITIONS' in the Import Query errorMessageBlankUsername=Username is required when password is given. errorMessageInvalidTableName=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '08006', errorCode: '17002', errorMessage: SQL Exception occurred: [Message='IO Error: Unknown host specified ', SQLState='08006', ErrorCode='17002'].' -errorMessageInvalidSinkDatabase=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '22000', errorCode: '17868', errorMessage: SQL Exception occurred: [Message='ORA-17868: Unknown host specified.: invalidDB%$^%*: Name or service not known https://docs.oracle.com/error-help/db/ora-17868/', SQLState='22000', ErrorCode='17868'].' +errorMessageInvalidSinkDatabase=errorMessage: SQL Exception occurred errorMessageInvalidHost=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '08006', errorCode: '17002', errorMessage: SQL Exception occurred: [Message='IO Error: Unknown host specified ', SQLState='08006', ErrorCode='17002'].' errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: Stage 'Oracle' encountered : \ java.io.IOException: ORA-00936: missing expression . Please check the system logs for more details. From 1c0bfe579fdc047b7d45fe9bbfe2d71eb99796b3 Mon Sep 17 00:00:00 2001 From: suryakumari Date: Fri, 11 Jul 2025 13:31:50 +0530 Subject: [PATCH 26/28] address review comments --- .../src/e2e-test/resources/errorMessage.properties | 2 +- .../src/e2e-test/features/source/RunTime.feature | 3 +++ .../features/mssql/mssql source/RunTimeWithMacros.feature | 4 ++++ oracle-plugin/src/e2e-test/resources/errorMessage.properties | 2 +- 4 files changed, 9 insertions(+), 2 deletions(-) diff --git a/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties b/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties index 6bbc5dee4..ec101ed02 100644 --- a/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties +++ b/cloudsql-mysql-plugin/src/e2e-test/resources/errorMessage.properties @@ -1,4 +1,4 @@ -errorMessageInvalidSourceDatabase=errorMessage: SQL Exception occurred +errorMessageInvalidSourceDatabase=SQL Error occurred, sqlState: '42000', errorCode: '1049' errorMessageInvalidImportQuery=Import Query select must contain the string '$CONDITIONS'. if Number of Splits is not set\ \ to 1. Include '$CONDITIONS' in the Import Query errorMessageCloudMySqlInvalidReferenceName=Invalid reference name diff --git a/cloudsql-postgresql-plugin/src/e2e-test/features/source/RunTime.feature b/cloudsql-postgresql-plugin/src/e2e-test/features/source/RunTime.feature index a9e3ff26b..c4c2eeae6 100644 --- a/cloudsql-postgresql-plugin/src/e2e-test/features/source/RunTime.feature +++ b/cloudsql-postgresql-plugin/src/e2e-test/features/source/RunTime.feature @@ -147,7 +147,9 @@ Feature: CloudSQL-PostGreSQL Source - Run Time scenarios And Save and Deploy Pipeline And Run the Pipeline in Runtime And Wait till pipeline is in running state + And Open and capture logs And Verify the pipeline status is "Failed" + And Close the pipeline logs Then Open Pipeline logs and verify Log entries having below listed Level and Message: | Level | Message | | ERROR | errorLogsMessageInvalidBoundingQuery | @@ -189,6 +191,7 @@ Feature: CloudSQL-PostGreSQL Source - Run Time scenarios Then Save the pipeline Then Preview and run the pipeline Then Wait till pipeline preview is in running state and check if any error occurs + Then Open and capture pipeline preview logs Then Verify the preview run status of pipeline in the logs is "failed" @CLOUDSQLPOSTGRESQL_SOURCE_TEST @CLOUDSQLPOSTGRESQL_TARGET_TEST diff --git a/mssql-plugin/src/e2e-test/features/mssql/mssql source/RunTimeWithMacros.feature b/mssql-plugin/src/e2e-test/features/mssql/mssql source/RunTimeWithMacros.feature index 036d0e7bc..451bde5a3 100644 --- a/mssql-plugin/src/e2e-test/features/mssql/mssql source/RunTimeWithMacros.feature +++ b/mssql-plugin/src/e2e-test/features/mssql/mssql source/RunTimeWithMacros.feature @@ -249,7 +249,9 @@ Feature: Mssql Source - Run time scenarios (macro) And Enter runtime argument value "invalidSelectQuery" for key "importQuery" And Run the Pipeline in Runtime with runtime arguments Then Wait till pipeline is in running state + And Open and capture logs And Verify the pipeline status is "Failed" + And Close the pipeline logs Then Open Pipeline logs and verify Log entries having below listed Level and Message: | Level | Message | | ERROR | errorMessageInvalidsourcetable | @@ -294,7 +296,9 @@ Feature: Mssql Source - Run time scenarios (macro) And Enter runtime argument value "invalid.password" for key "Password" And Run the Pipeline in Runtime with runtime arguments Then Wait till pipeline is in running state + And Open and capture logs And Verify the pipeline status is "Failed" + And Close the pipeline logs Then Open Pipeline logs and verify Log entries having below listed Level and Message: | Level | Message | | ERROR | errorMessageInvalidCredentialSource | diff --git a/oracle-plugin/src/e2e-test/resources/errorMessage.properties b/oracle-plugin/src/e2e-test/resources/errorMessage.properties index 0d40301fb..895444408 100644 --- a/oracle-plugin/src/e2e-test/resources/errorMessage.properties +++ b/oracle-plugin/src/e2e-test/resources/errorMessage.properties @@ -12,7 +12,7 @@ errorMessageInvalidImportQuery=Import Query select must contain the string '$CON \ to 1. Include '$CONDITIONS' in the Import Query errorMessageBlankUsername=Username is required when password is given. errorMessageInvalidTableName=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '08006', errorCode: '17002', errorMessage: SQL Exception occurred: [Message='IO Error: Unknown host specified ', SQLState='08006', ErrorCode='17002'].' -errorMessageInvalidSinkDatabase=errorMessage: SQL Exception occurred +errorMessageInvalidSinkDatabase=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '08006', errorCode: '17002' errorMessageInvalidHost=Error encountered while configuring the stage: 'SQL Error occurred, sqlState: '08006', errorCode: '17002', errorMessage: SQL Exception occurred: [Message='IO Error: Unknown host specified ', SQLState='08006', ErrorCode='17002'].' errorLogsMessageInvalidBoundingQuery=Spark program 'phase-1' failed with error: Stage 'Oracle' encountered : \ java.io.IOException: ORA-00936: missing expression . Please check the system logs for more details. From 40ff60ae8c0651cda2b9443293beae8f0c7cb8c9 Mon Sep 17 00:00:00 2001 From: Sanchit Garg Date: Mon, 14 Jul 2025 11:28:24 +0530 Subject: [PATCH 27/28] PLUGIN-1823: Updated error message as per the suggestion --- .../src/main/java/io/cdap/plugin/util/RetryUtils.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/database-commons/src/main/java/io/cdap/plugin/util/RetryUtils.java b/database-commons/src/main/java/io/cdap/plugin/util/RetryUtils.java index be11daaa9..765845fa6 100644 --- a/database-commons/src/main/java/io/cdap/plugin/util/RetryUtils.java +++ b/database-commons/src/main/java/io/cdap/plugin/util/RetryUtils.java @@ -104,11 +104,11 @@ private static RuntimeException unwrapFailsafeException(Exception e, if (cause instanceof SQLException) { return dbErrorDetailsProvider.getProgramFailureException((SQLException) cause, null); } else if (cause instanceof RuntimeException) { - return (RuntimeException) cause; + return new RuntimeException("Operation failed with error", cause); } else if (cause instanceof Error) { - return new RuntimeException("Failsafe wrapped an Error", cause); + return new RuntimeException("Operation failed with error", cause); } else { - return new RuntimeException("Failsafe wrapped a non-runtime exception", cause); + return new RuntimeException("Operation failed", cause); } } if (e instanceof SQLException) { From 3f332d1f40be245ac8d37ab8e430f4ee5f2e97aa Mon Sep 17 00:00:00 2001 From: Sanhit Garg Date: Thu, 17 Jul 2025 10:45:14 +0530 Subject: [PATCH 28/28] PLUGIN-1823: Revert exception message update for causes that are instances of RuntimeException --- .../src/main/java/io/cdap/plugin/util/RetryUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/database-commons/src/main/java/io/cdap/plugin/util/RetryUtils.java b/database-commons/src/main/java/io/cdap/plugin/util/RetryUtils.java index 765845fa6..433978358 100644 --- a/database-commons/src/main/java/io/cdap/plugin/util/RetryUtils.java +++ b/database-commons/src/main/java/io/cdap/plugin/util/RetryUtils.java @@ -104,7 +104,7 @@ private static RuntimeException unwrapFailsafeException(Exception e, if (cause instanceof SQLException) { return dbErrorDetailsProvider.getProgramFailureException((SQLException) cause, null); } else if (cause instanceof RuntimeException) { - return new RuntimeException("Operation failed with error", cause); + return (RuntimeException) cause; } else if (cause instanceof Error) { return new RuntimeException("Operation failed with error", cause); } else {