From 0b1c2b1c28167696caa1b2ca0609c9a743d07628 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 24 Jan 2025 22:42:38 +0000 Subject: [PATCH 1/7] fix: handle DECIMAL(38,9) type conversion in BigQuery processor Co-Authored-By: Aaron Steers --- airbyte/_processors/sql/bigquery.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/airbyte/_processors/sql/bigquery.py b/airbyte/_processors/sql/bigquery.py index ffbd5aac..c2031a2f 100644 --- a/airbyte/_processors/sql/bigquery.py +++ b/airbyte/_processors/sql/bigquery.py @@ -117,6 +117,10 @@ def to_sql_type( return self.get_string_type() if isinstance(sql_type, sqlalchemy.types.BIGINT): return sqlalchemy_types.Integer() # All integers are 64-bit in BigQuery + if isinstance(sql_type, sqlalchemy.types.DECIMAL): + # Convert SQLAlchemy DECIMAL to BigQuery NUMERIC type + # BigQuery NUMERIC type has precision of 38 and scale of 9 by default + return sqlalchemy_types.Numeric(precision=38, scale=9) return sql_type From 9caefb43b4a4e54875e7faf3a31b21b4c08b67b1 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 24 Jan 2025 22:57:03 +0000 Subject: [PATCH 2/7] fix: handle DECIMAL(38,9) type conversion in BigQuery processor Co-Authored-By: Aaron Steers --- airbyte/_processors/sql/bigquery.py | 3 +- .../integration_tests/test_bigquery_cache.py | 67 +++++++++++++++++++ 2 files changed, 68 insertions(+), 2 deletions(-) diff --git a/airbyte/_processors/sql/bigquery.py b/airbyte/_processors/sql/bigquery.py index c2031a2f..ff4edd94 100644 --- a/airbyte/_processors/sql/bigquery.py +++ b/airbyte/_processors/sql/bigquery.py @@ -291,8 +291,7 @@ def _swap_temp_table_with_final_table( deletion_name = f"{final_table_name}_deleteme" commands = "\n".join( [ - f"ALTER TABLE {self._fully_qualified(final_table_name)} " - f"RENAME TO {deletion_name};", + f"ALTER TABLE {self._fully_qualified(final_table_name)} RENAME TO {deletion_name};", f"ALTER TABLE {self._fully_qualified(temp_table_name)} " f"RENAME TO {final_table_name};", f"DROP TABLE {self._fully_qualified(deletion_name)};", diff --git a/tests/integration_tests/test_bigquery_cache.py b/tests/integration_tests/test_bigquery_cache.py index 2039399e..dcb05f41 100644 --- a/tests/integration_tests/test_bigquery_cache.py +++ b/tests/integration_tests/test_bigquery_cache.py @@ -4,8 +4,10 @@ from __future__ import annotations import pytest +from sqlalchemy import types as sqlalchemy_types import airbyte as ab +from airbyte._processors.sql.bigquery import BigQueryTypeConverter @pytest.mark.requires_creds @@ -24,3 +26,68 @@ def test_bigquery_props( assert new_bigquery_cache.get_database_name() == new_bigquery_cache.project_name, ( "Database name should be the same as project name." ) + + +@pytest.mark.requires_creds +def test_decimal_type_conversion( + new_bigquery_cache: ab.BigQueryCache, +) -> None: + """Test that DECIMAL(38,9) types are correctly converted to BigQuery NUMERIC types.""" + # Create a test table with a DECIMAL column + table_name = "test_decimal_types" + sql = f""" + CREATE TABLE {new_bigquery_cache.schema_name}.{table_name} ( + id INT64, + amount NUMERIC(38, 9) + ) + """ + new_bigquery_cache._execute_sql(sql) + + # Insert test data + sql = f""" + INSERT INTO {new_bigquery_cache.schema_name}.{table_name} (id, amount) + VALUES (1, 123.456789) + """ + new_bigquery_cache._execute_sql(sql) + + try: + # Verify type conversion + converter = BigQueryTypeConverter() + decimal_type = sqlalchemy_types.DECIMAL(precision=38, scale=9) + converted_type = converter.to_sql_type({"type": "number", "format": "decimal"}) + + # Check that the converted type is a NUMERIC type with correct precision and scale + assert isinstance(converted_type, sqlalchemy_types.Numeric), ( + "DECIMAL type should be converted to NUMERIC" + ) + assert converted_type.precision == 38, "Precision should be 38" + assert converted_type.scale == 9, "Scale should be 9" + + # Verify we can read the data back + sql = f"SELECT amount FROM {new_bigquery_cache.schema_name}.{table_name} WHERE id = 1" + result = new_bigquery_cache._execute_sql(sql).fetchone() + assert result is not None, "Should be able to read NUMERIC data" + assert isinstance(result[0], (float, int, str)), ( + "NUMERIC data should be readable" + ) + + # Test error case with invalid DECIMAL type + with pytest.raises(ValueError, match="Invalid value for type"): + invalid_decimal = sqlalchemy_types.DECIMAL( + precision=100, scale=50 + ) # Too large for BigQuery + new_bigquery_cache._execute_sql( + f""" + CREATE TABLE {new_bigquery_cache.schema_name}.invalid_decimal_test ( + id INT64, + amount NUMERIC(100, 50) + ) + """ + ) + + finally: + # Clean up + sql = f"DROP TABLE IF EXISTS {new_bigquery_cache.schema_name}.{table_name}" + new_bigquery_cache._execute_sql(sql) + sql = f"DROP TABLE IF EXISTS {new_bigquery_cache.schema_name}.invalid_decimal_test" + new_bigquery_cache._execute_sql(sql) From ca326274975dfcc4f51855c391a72ec20374f3b4 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 24 Jan 2025 23:12:37 +0000 Subject: [PATCH 3/7] test: improve BigQuery decimal type test cleanup Co-Authored-By: Aaron Steers --- .../integration_tests/test_bigquery_cache.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/integration_tests/test_bigquery_cache.py b/tests/integration_tests/test_bigquery_cache.py index dcb05f41..b7898975 100644 --- a/tests/integration_tests/test_bigquery_cache.py +++ b/tests/integration_tests/test_bigquery_cache.py @@ -50,6 +50,13 @@ def test_decimal_type_conversion( """ new_bigquery_cache._execute_sql(sql) + # Clean up any existing tables first + cleanup_sql = f""" + DROP TABLE IF EXISTS {new_bigquery_cache.schema_name}.{table_name}; + DROP TABLE IF EXISTS {new_bigquery_cache.schema_name}.invalid_decimal_test; + """ + new_bigquery_cache._execute_sql(cleanup_sql) + try: # Verify type conversion converter = BigQueryTypeConverter() @@ -73,9 +80,6 @@ def test_decimal_type_conversion( # Test error case with invalid DECIMAL type with pytest.raises(ValueError, match="Invalid value for type"): - invalid_decimal = sqlalchemy_types.DECIMAL( - precision=100, scale=50 - ) # Too large for BigQuery new_bigquery_cache._execute_sql( f""" CREATE TABLE {new_bigquery_cache.schema_name}.invalid_decimal_test ( @@ -87,7 +91,8 @@ def test_decimal_type_conversion( finally: # Clean up - sql = f"DROP TABLE IF EXISTS {new_bigquery_cache.schema_name}.{table_name}" - new_bigquery_cache._execute_sql(sql) - sql = f"DROP TABLE IF EXISTS {new_bigquery_cache.schema_name}.invalid_decimal_test" - new_bigquery_cache._execute_sql(sql) + cleanup_sql = f""" + DROP TABLE IF EXISTS {new_bigquery_cache.schema_name}.{table_name}; + DROP TABLE IF EXISTS {new_bigquery_cache.schema_name}.invalid_decimal_test; + """ + new_bigquery_cache._execute_sql(cleanup_sql) From b13538c684080a43ba6428f605ec48d7e9d3ee12 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 24 Jan 2025 23:26:49 +0000 Subject: [PATCH 4/7] test: improve BigQuery decimal type test Co-Authored-By: Aaron Steers --- .../integration_tests/test_bigquery_cache.py | 60 +++++++------------ 1 file changed, 20 insertions(+), 40 deletions(-) diff --git a/tests/integration_tests/test_bigquery_cache.py b/tests/integration_tests/test_bigquery_cache.py index b7898975..d3ac8902 100644 --- a/tests/integration_tests/test_bigquery_cache.py +++ b/tests/integration_tests/test_bigquery_cache.py @@ -8,6 +8,7 @@ import airbyte as ab from airbyte._processors.sql.bigquery import BigQueryTypeConverter +from airbyte._util import text_util @pytest.mark.requires_creds @@ -33,34 +34,11 @@ def test_decimal_type_conversion( new_bigquery_cache: ab.BigQueryCache, ) -> None: """Test that DECIMAL(38,9) types are correctly converted to BigQuery NUMERIC types.""" - # Create a test table with a DECIMAL column - table_name = "test_decimal_types" - sql = f""" - CREATE TABLE {new_bigquery_cache.schema_name}.{table_name} ( - id INT64, - amount NUMERIC(38, 9) - ) - """ - new_bigquery_cache._execute_sql(sql) - - # Insert test data - sql = f""" - INSERT INTO {new_bigquery_cache.schema_name}.{table_name} (id, amount) - VALUES (1, 123.456789) - """ - new_bigquery_cache._execute_sql(sql) - - # Clean up any existing tables first - cleanup_sql = f""" - DROP TABLE IF EXISTS {new_bigquery_cache.schema_name}.{table_name}; - DROP TABLE IF EXISTS {new_bigquery_cache.schema_name}.invalid_decimal_test; - """ - new_bigquery_cache._execute_sql(cleanup_sql) - + table_name = f"test_decimal_{text_util.generate_random_suffix()}" + try: # Verify type conversion converter = BigQueryTypeConverter() - decimal_type = sqlalchemy_types.DECIMAL(precision=38, scale=9) converted_type = converter.to_sql_type({"type": "number", "format": "decimal"}) # Check that the converted type is a NUMERIC type with correct precision and scale @@ -70,6 +48,22 @@ def test_decimal_type_conversion( assert converted_type.precision == 38, "Precision should be 38" assert converted_type.scale == 9, "Scale should be 9" + # Create a test table with a DECIMAL column + sql = f""" + CREATE TABLE {new_bigquery_cache.schema_name}.{table_name} ( + id INT64, + amount NUMERIC(38, 9) + ) + """ + new_bigquery_cache._execute_sql(sql) + + # Insert test data + sql = f""" + INSERT INTO {new_bigquery_cache.schema_name}.{table_name} (id, amount) + VALUES (1, 123.456789) + """ + new_bigquery_cache._execute_sql(sql) + # Verify we can read the data back sql = f"SELECT amount FROM {new_bigquery_cache.schema_name}.{table_name} WHERE id = 1" result = new_bigquery_cache._execute_sql(sql).fetchone() @@ -78,21 +72,7 @@ def test_decimal_type_conversion( "NUMERIC data should be readable" ) - # Test error case with invalid DECIMAL type - with pytest.raises(ValueError, match="Invalid value for type"): - new_bigquery_cache._execute_sql( - f""" - CREATE TABLE {new_bigquery_cache.schema_name}.invalid_decimal_test ( - id INT64, - amount NUMERIC(100, 50) - ) - """ - ) - finally: # Clean up - cleanup_sql = f""" - DROP TABLE IF EXISTS {new_bigquery_cache.schema_name}.{table_name}; - DROP TABLE IF EXISTS {new_bigquery_cache.schema_name}.invalid_decimal_test; - """ + cleanup_sql = f"DROP TABLE IF EXISTS {new_bigquery_cache.schema_name}.{table_name}" new_bigquery_cache._execute_sql(cleanup_sql) From 05d5e53ce7a4ff7528a7332af7272eb80647a40e Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 24 Jan 2025 23:29:12 +0000 Subject: [PATCH 5/7] style: format test_bigquery_cache.py with ruff Co-Authored-By: Aaron Steers --- tests/integration_tests/test_bigquery_cache.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration_tests/test_bigquery_cache.py b/tests/integration_tests/test_bigquery_cache.py index d3ac8902..ea52ffc1 100644 --- a/tests/integration_tests/test_bigquery_cache.py +++ b/tests/integration_tests/test_bigquery_cache.py @@ -35,7 +35,7 @@ def test_decimal_type_conversion( ) -> None: """Test that DECIMAL(38,9) types are correctly converted to BigQuery NUMERIC types.""" table_name = f"test_decimal_{text_util.generate_random_suffix()}" - + try: # Verify type conversion converter = BigQueryTypeConverter() @@ -74,5 +74,7 @@ def test_decimal_type_conversion( finally: # Clean up - cleanup_sql = f"DROP TABLE IF EXISTS {new_bigquery_cache.schema_name}.{table_name}" + cleanup_sql = ( + f"DROP TABLE IF EXISTS {new_bigquery_cache.schema_name}.{table_name}" + ) new_bigquery_cache._execute_sql(cleanup_sql) From 3042f56edf187c4884950d600fb6d5a952111ab2 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Fri, 24 Jan 2025 23:48:55 +0000 Subject: [PATCH 6/7] fix: use execute_sql instead of _execute_sql in BigQuery test Co-Authored-By: Aaron Steers --- tests/integration_tests/test_bigquery_cache.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration_tests/test_bigquery_cache.py b/tests/integration_tests/test_bigquery_cache.py index ea52ffc1..1f9041b5 100644 --- a/tests/integration_tests/test_bigquery_cache.py +++ b/tests/integration_tests/test_bigquery_cache.py @@ -55,18 +55,18 @@ def test_decimal_type_conversion( amount NUMERIC(38, 9) ) """ - new_bigquery_cache._execute_sql(sql) + new_bigquery_cache.execute_sql(sql) # Insert test data sql = f""" INSERT INTO {new_bigquery_cache.schema_name}.{table_name} (id, amount) VALUES (1, 123.456789) """ - new_bigquery_cache._execute_sql(sql) + new_bigquery_cache.execute_sql(sql) # Verify we can read the data back sql = f"SELECT amount FROM {new_bigquery_cache.schema_name}.{table_name} WHERE id = 1" - result = new_bigquery_cache._execute_sql(sql).fetchone() + result = new_bigquery_cache.execute_sql(sql).fetchone() assert result is not None, "Should be able to read NUMERIC data" assert isinstance(result[0], (float, int, str)), ( "NUMERIC data should be readable" @@ -77,4 +77,4 @@ def test_decimal_type_conversion( cleanup_sql = ( f"DROP TABLE IF EXISTS {new_bigquery_cache.schema_name}.{table_name}" ) - new_bigquery_cache._execute_sql(cleanup_sql) + new_bigquery_cache.execute_sql(cleanup_sql) From f35332e8ac26562c780908c91d93da6043f19e6c Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 25 Jan 2025 00:04:19 +0000 Subject: [PATCH 7/7] test: ensure schema exists before creating BigQuery test table Co-Authored-By: Aaron Steers --- tests/integration_tests/test_bigquery_cache.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/integration_tests/test_bigquery_cache.py b/tests/integration_tests/test_bigquery_cache.py index 1f9041b5..9779d2f8 100644 --- a/tests/integration_tests/test_bigquery_cache.py +++ b/tests/integration_tests/test_bigquery_cache.py @@ -48,6 +48,9 @@ def test_decimal_type_conversion( assert converted_type.precision == 38, "Precision should be 38" assert converted_type.scale == 9, "Scale should be 9" + # Ensure schema exists before creating table + new_bigquery_cache._ensure_schema_exists() + # Create a test table with a DECIMAL column sql = f""" CREATE TABLE {new_bigquery_cache.schema_name}.{table_name} (