Merge branch 'main' into saumya/upgrade-odbc

Author: gargsaumya

eng/pipelines/pr-validation-pipeline.yml

@@ -38,6 +38,7 @@
     InternalError,
     ProgrammingError,
     NotSupportedError,
+    sqlstate_to_exception,
 )
 from mssql_python.auth import extract_auth_type, process_connection_string
 from mssql_python.constants import ConstantsDDBC, GetInfoConstants
@@ -57,6 +58,42 @@
 # Note: "utf-16" with BOM is NOT included as it's problematic for SQL_WCHAR
 UTF16_ENCODINGS: frozenset[str] = frozenset(["utf-16le", "utf-16be"])
 
+_SQLSTATE_RE = re.compile(r"^SQLSTATE:([A-Z0-9]{0,5}):(.*)", re.DOTALL)
+
+
+def _raise_connection_error(e: RuntimeError) -> None:
+    """Map a RuntimeError from the C++ pybind layer to the correct DB-API 2.0 exception.
+
+    Connection::checkError() throws "SQLSTATE:XXXXX:<odbc_message>" so the SQLSTATE
+    can be mapped via sqlstate_to_exception(), consistent with cursor-level error handling.
+    """
+    error_msg = str(e)
+    match = _SQLSTATE_RE.match(error_msg)
+    if match:
+        sqlstate, ddbc_error = match.group(1), match.group(2)
+        # Handle malformed SQLSTATE prefix (empty or invalid code)
+        if not sqlstate or len(sqlstate) != 5:
+            logger.error("Connection error (malformed SQLSTATE): %s", ddbc_error)
+            raise OperationalError(
+                driver_error="Connection operation failed",
+                ddbc_error=ddbc_error,
+            ) from None
+        exc = sqlstate_to_exception(sqlstate, ddbc_error)
+        if exc is None:
+            logger.error("Unknown SQLSTATE %s, raising DatabaseError", sqlstate)
+            raise DatabaseError(
+                driver_error=f"An error occurred with SQLSTATE code: {sqlstate}",
+                ddbc_error=ddbc_error,
+            ) from None
+        logger.error("Connection error (SQLSTATE %s): %s", sqlstate, ddbc_error)
+        raise exc from None
+    # Fallback: no SQLSTATE prefix — e.g. "Connection handle not allocated"
+    logger.error("Connection error: %s", error_msg)
+    raise OperationalError(
+        driver_error="Connection operation failed",
+        ddbc_error=error_msg,
+    ) from None
+
 
 def _validate_utf16_wchar_compatibility(
     encoding: str, wchar_type: int, context: str = "SQL_WCHAR"
@@ -261,10 +298,14 @@ def __init__(
         }
 
         # Initialize decoding settings with Python 3 defaults
+        # SQL_CHAR default uses SQL_WCHAR ctype so the ODBC driver returns
+        # UTF-16 data for VARCHAR columns. This avoids encoding mismatches on
+        # Windows where the driver returns raw bytes in the server's native
+        # code page (e.g. CP-1252) that may fail to decode as UTF-8.
         self._decoding_settings = {
             ConstantsDDBC.SQL_CHAR.value: {
-                "encoding": "utf-8",
-                "ctype": ConstantsDDBC.SQL_CHAR.value,
+                "encoding": "utf-16le",
+                "ctype": ConstantsDDBC.SQL_WCHAR.value,
             },
             ConstantsDDBC.SQL_WCHAR.value: {
                 "encoding": "utf-16le",
@@ -329,9 +370,12 @@ def __init__(
         if not PoolingManager.is_initialized():
             PoolingManager.enable()
         self._pooling = PoolingManager.is_enabled()
-        self._conn = ddbc_bindings.Connection(
-            self.connection_str, self._pooling, self._attrs_before
-        )
+        try:
+            self._conn = ddbc_bindings.Connection(
+                self.connection_str, self._pooling, self._attrs_before
+            )
+        except RuntimeError as e:
+            _raise_connection_error(e)
         self.setautocommit(autocommit)
 
         # Register this connection for cleanup before Python shutdown
@@ -452,7 +496,10 @@ def autocommit(self) -> bool:
         Returns:
             bool: True if autocommit is enabled, False otherwise.
         """
-        return self._conn.get_autocommit()
+        try:
+            return self._conn.get_autocommit()
+        except RuntimeError as e:
+            _raise_connection_error(e)
 
     @autocommit.setter
     def autocommit(self, value: bool) -> None:
@@ -492,7 +539,10 @@ def setautocommit(self, value: bool = False) -> None:
         Raises:
             DatabaseError: If there is an error while setting the autocommit mode.
         """
-        self._conn.set_autocommit(value)
+        try:
+            self._conn.set_autocommit(value)
+        except RuntimeError as e:
+            _raise_connection_error(e)
 
     def setencoding(self, encoding: Optional[str] = None, ctype: Optional[int] = None) -> None:
         """
@@ -643,9 +693,13 @@ def setdecoding(
             sqltype (int): The SQL type being configured: SQL_CHAR, SQL_WCHAR, or SQL_WMETADATA.
                 SQL_WMETADATA is a special flag for configuring column name decoding.
             encoding (str, optional): The Python encoding to use when decoding the data.
-                If None, uses default encoding based on sqltype.
+                If None, defaults to ``'utf-16le'`` for all sqltypes (SQL_CHAR,
+                SQL_WCHAR, and SQL_WMETADATA), matching the connection-level
+                defaults set in ``Connection.__init__``. Passing ``encoding=None``
+                therefore resets the sqltype to its initial default.
             ctype (int, optional): The C data type to request from SQLGetData:
-                SQL_CHAR or SQL_WCHAR. If None, uses default based on encoding.
+                SQL_CHAR or SQL_WCHAR. If None, uses default based on encoding
+                (SQL_WCHAR for UTF-16 variants, SQL_CHAR otherwise).
 
         Returns:
             None
@@ -655,7 +709,10 @@ def setdecoding(
             InterfaceError: If the connection is closed.
 
         Example:
-            # Configure SQL_CHAR to use UTF-8 decoding
+            # Reset SQL_CHAR to the connection default (utf-16le + SQL_WCHAR ctype)
+            cnxn.setdecoding(mssql_python.SQL_CHAR)
+
+            # Configure SQL_CHAR to use UTF-8 decoding (opt-in, non-default)
             cnxn.setdecoding(mssql_python.SQL_CHAR, encoding='utf-8')
 
             # Configure column metadata decoding
@@ -691,12 +748,15 @@ def setdecoding(
                 ),
             )
 
-        # Set default encoding based on sqltype if not provided
+        # Set default encoding based on sqltype if not provided.
+        # All sqltypes default to UTF-16LE to match Connection.__init__ defaults.
+        # SQL_CHAR uses utf-16le + SQL_WCHAR ctype so the ODBC driver returns
+        # UTF-16 data for VARCHAR columns, avoiding encoding mismatches on
+        # Windows where the driver may otherwise return raw bytes in the
+        # server's native code page (e.g. CP-1252). This makes
+        # ``setdecoding(SQL_CHAR)`` with no arguments a true reset-to-defaults.
         if encoding is None:
-            if sqltype == ConstantsDDBC.SQL_CHAR.value:
-                encoding = "utf-8"  # Default for SQL_CHAR in Python 3
-            else:  # SQL_WCHAR or SQL_WMETADATA
-                encoding = "utf-16le"  # Default for SQL_WCHAR in Python 3
+            encoding = "utf-16le"
 
         # Validate encoding using cached validation for better performance
         if not _validate_encoding(encoding):
@@ -1477,7 +1537,10 @@ def commit(self) -> None:
             )
 
         # Commit the current transaction
-        self._conn.commit()
+        try:
+            self._conn.commit()
+        except RuntimeError as e:
+            _raise_connection_error(e)
         logger.info("Transaction committed successfully.")
 
     def rollback(self) -> None:
@@ -1500,7 +1563,10 @@ def rollback(self) -> None:
             )
 
         # Roll back the current transaction
-        self._conn.rollback()
+        try:
+            self._conn.rollback()
+        except RuntimeError as e:
+            _raise_connection_error(e)
         logger.info("Transaction rolled back successfully.")
 
     def close(self) -> None:
@@ -1556,7 +1622,11 @@ def close(self) -> None:
                     # For autocommit True, this is not necessary as each statement is
                     # committed immediately
                     logger.debug("Rolling back uncommitted changes before closing connection.")
-                    self._conn.rollback()
+                    try:
+                        self._conn.rollback()
+                    except RuntimeError as e:
+                        # Handle C++ layer RuntimeError with proper DB-API exception mapping
+                        _raise_connection_error(e)
                 # TODO: Check potential race conditions in case of multithreaded scenarios
                 # Close the connection
                 self._conn.close()

mssql_python/connection.py

@@ -2054,19 +2054,27 @@ def _compute_column_type(self, column):
             sample_value: Representative value for type inference and modified_row.
             min_val: Minimum for integers (None otherwise).
             max_val: Maximum for integers (None otherwise).
+            max_decimal_formatted_len: Maximum len(format(d, 'f')) across all
+                Decimal values in the column (0 when no Decimals are present).
+                Used by executemany to correct the SQL_VARCHAR column size when
+                the sample value's formatted string is shorter than another
+                value's (e.g. positive sample vs negative row value) (GH-557).
         """
         non_nulls = [v for v in column if v is not None]
         if not non_nulls:
-            return None, None, None
+            return None, None, None, 0
 
         int_values = [v for v in non_nulls if isinstance(v, int)]
         if int_values:
             min_val, max_val = min(int_values), max(int_values)
             sample_value = max(int_values, key=abs)
-            return sample_value, min_val, max_val
+            return sample_value, min_val, max_val, 0
 
         sample_value = None
+        max_decimal_formatted_len = 0
         for v in non_nulls:
+            if isinstance(v, decimal.Decimal):
+                max_decimal_formatted_len = max(max_decimal_formatted_len, len(format(v, "f")))
             if not sample_value:
                 sample_value = v
             elif isinstance(v, (str, bytes, bytearray)) and isinstance(
@@ -2120,7 +2128,7 @@ def _compute_column_type(self, column):
                 # If comparing Decimal to non-Decimal, prefer Decimal for better type inference
                 sample_value = v
 
-        return sample_value, None, None
+        return sample_value, None, None, max_decimal_formatted_len
 
     def executemany(  # pylint: disable=too-many-locals,too-many-branches,too-many-statements
         self, operation: str, seq_of_parameters: Union[List[Sequence[Any]], List[Mapping[str, Any]]]
@@ -2225,7 +2233,7 @@ def executemany(  # pylint: disable=too-many-locals,too-many-branches,too-many-s
                 if hasattr(seq_of_parameters, "__getitem__")
                 else []
             )
-            sample_value, min_val, max_val = self._compute_column_type(column)
+            sample_value, min_val, max_val, _ = self._compute_column_type(column)
 
             if self._inputsizes and col_index < len(self._inputsizes):
                 # Use explicitly set input sizes
@@ -2301,7 +2309,7 @@ def executemany(  # pylint: disable=too-many-locals,too-many-branches,too-many-s
                     if hasattr(seq_of_parameters, "__getitem__")
                     else []
                 )
-                sample_value, min_val, max_val = self._compute_column_type(column)
+                sample_value, min_val, max_val, max_decimal_len = self._compute_column_type(column)
 
                 dummy_row = list(sample_row)
                 paraminfo = self._create_parameter_types_list(
@@ -2322,6 +2330,17 @@ def executemany(  # pylint: disable=too-many-locals,too-many-branches,too-many-s
                     paraminfo.paramSQLType = ddbc_sql_const.SQL_VARCHAR.value
                     paraminfo.columnSize = 1
 
+                # Correct column size for Decimal columns sent as SQL_VARCHAR (GH-557).
+                # The sample value's formatted string may be shorter than another
+                # row's (e.g. positive sample "1.0" = 3 chars vs negative "-0.1" = 4).
+                # max_decimal_len was already computed during _compute_column_type
+                # so no extra iteration is needed.
+                if (
+                    paraminfo.paramSQLType == ddbc_sql_const.SQL_VARCHAR.value
+                    and max_decimal_len > paraminfo.columnSize
+                ):
+                    paraminfo.columnSize = max_decimal_len
+
                 # Special handling for binary data in auto-detected types
                 if paraminfo.paramSQLType in (
                     ddbc_sql_const.SQL_BINARY.value,
@@ -2462,8 +2481,9 @@ def fetchone(self) -> Union[None, Row]:
             ret = ddbc_bindings.DDBCSQLFetchOne(
                 self.hstmt,
                 row_data,
-                char_decoding.get("encoding", "utf-8"),
+                char_decoding.get("encoding", "utf-16le"),
                 wchar_decoding.get("encoding", "utf-16le"),
+                char_decoding.get("ctype", ddbc_sql_const.SQL_WCHAR.value),
             )
 
             if self.hstmt:
@@ -2528,8 +2548,9 @@ def fetchmany(self, size: Optional[int] = None) -> List[Row]:
                 self.hstmt,
                 rows_data,
                 size,
-                char_decoding.get("encoding", "utf-8"),
+                char_decoding.get("encoding", "utf-16le"),
                 wchar_decoding.get("encoding", "utf-16le"),
+                char_decoding.get("ctype", ddbc_sql_const.SQL_WCHAR.value),
             )
 
             if self.hstmt:
@@ -2586,8 +2607,9 @@ def fetchall(self) -> List[Row]:
             ret = ddbc_bindings.DDBCSQLFetchAll(
                 self.hstmt,
                 rows_data,
-                char_decoding.get("encoding", "utf-8"),
+                char_decoding.get("encoding", "utf-16le"),
                 wchar_decoding.get("encoding", "utf-16le"),
+                char_decoding.get("ctype", ddbc_sql_const.SQL_WCHAR.value),
             )
 
             # Check for errors

mssql_python/cursor.py

@@ -215,6 +215,25 @@ endif()
 
 message(STATUS "Final Python library directory: ${PYTHON_LIB_DIR}")
 
+find_package(simdutf CONFIG QUIET)
+
+if(NOT simdutf_FOUND)
+    include(FetchContent)
+    message(STATUS "simdutf not found via find_package; downloading v8.2.0 source archive with FetchContent")
+    set(simdutf_fetchcontent_args
+        URL https://github.com/simdutf/simdutf/archive/refs/tags/v8.2.0.tar.gz
+        URL_HASH SHA256=033a91b1d7d1cb818c1eff49e61faaa1b64a3a530d59ef9efef0195e56bda8b1
+    )
+    if(CMAKE_VERSION VERSION_GREATER_EQUAL "3.24")
+        list(APPEND simdutf_fetchcontent_args DOWNLOAD_EXTRACT_TIMESTAMP FALSE)
+    endif()
+    FetchContent_Declare(simdutf ${simdutf_fetchcontent_args})
+    set(SIMDUTF_TESTS OFF CACHE BOOL "Disable simdutf tests" FORCE)
+    set(SIMDUTF_TOOLS OFF CACHE BOOL "Disable simdutf tools" FORCE)
+    set(SIMDUTF_BENCHMARKS OFF CACHE BOOL "Disable simdutf benchmarks" FORCE)
+    FetchContent_MakeAvailable(simdutf)
+endif()
+
 set(DDBC_SOURCE "ddbc_bindings.cpp")
 message(STATUS "Using standard source file: ${DDBC_SOURCE}")
 # Include connection module and logger bridge
@@ -293,6 +312,8 @@ else()
     endif()
 endif()
 
+target_link_libraries(ddbc_bindings PRIVATE simdutf::simdutf)
+
 # Compiler definitions
 target_compile_definitions(ddbc_bindings PRIVATE 
     HAVE_SNPRINTF

mssql_python/pybind/CMakeLists.txt

@@ -184,3 +184,6 @@ Examples:
 - Linux x86_64: `ddbc_bindings.cp311-x86_64.so`
 - Linux ARM64: `ddbc_bindings.cp311-arm64.so`
 
+# String Handling for Unicode Data
+
+Use std::u16string or the Python C-API when converting between SQLWCHAR data and Python strings. Use simdutf for any pure c++ UTF transcoding. Do not introduce std::wstring in the C++ bindings.