Fixed bug with SQL containing multibyte characters with certain database

character sets (#133).

Author: anthony-tuininga

doc/src/release_notes.rst

@@ -30,6 +30,9 @@ Thin Mode Changes
 #)  Fixed bug with incorrect values of :data:`Cursor.rowcount` when fetching
     data
     (`issue 147 <https://github.com/oracle/python-oracledb/issues/147>`__).
+#)  Fixed bug with SQL containing multibyte characters with certain database
+    character sets
+    (`issue 133 <https://github.com/oracle/python-oracledb/issues/133>`__).
 
 Thick Mode Changes
 ++++++++++++++++++

src/oracledb/impl/thin/capabilities.pyx

@@ -1,5 +1,5 @@
 #------------------------------------------------------------------------------
-# Copyright (c) 2021, 2022, Oracle and/or its affiliates.
+# Copyright (c) 2021, 2023, Oracle and/or its affiliates.
 #
 # This software is dual-licensed to you under the Universal Permissive License
 # (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl and Apache License
@@ -38,7 +38,6 @@ cdef class Capabilities:
         uint16_t ncharset_id
         bytearray compile_caps
         bytearray runtime_caps
-        bint char_conversion
         bint supports_oob
 
     def __init__(self):

src/oracledb/impl/thin/connection.pyx

@@ -1,5 +1,5 @@
 #------------------------------------------------------------------------------
-# Copyright (c) 2020, 2022, Oracle and/or its affiliates.
+# Copyright (c) 2020, 2023, Oracle and/or its affiliates.
 #
 # This software is dual-licensed to you under the Universal Permissive License
 # (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl and Apache License
@@ -240,7 +240,7 @@ cdef class ThinConnImpl(BaseConnImpl):
             statement = self._statement_cache.get(sql)
             if statement is None:
                 statement = Statement()
-                statement._prepare(sql, self._protocol._caps.char_conversion)
+                statement._prepare(sql)
                 if len(self._statement_cache) < self._statement_cache_size \
                         and cache_statement \
                         and not self._drcp_establish_session:

src/oracledb/impl/thin/constants.pxi

@@ -578,6 +578,8 @@ DEF TNS_CHARSET_UTF8 = 873
 DEF TNS_CHARSET_UTF16 = 2000
 DEF TNS_ENCODING_UTF8 = "UTF-8"
 DEF TNS_ENCODING_UTF16 = "UTF-16BE"
+DEF TNS_ENCODING_MULTI_BYTE = 0x01
+DEF TNS_ENCODING_CONV_LENGTH = 0x02
 
 # compile time capability indices
 DEF TNS_CCAP_SQL_VERSION = 0

src/oracledb/impl/thin/messages.pyx

@@ -1090,7 +1090,7 @@ cdef class MessageWithData(Message):
         buf.write_uint8(1)                  # pointer
         schema_bytes = self.conn_impl._current_schema.encode()
         buf.write_ub4(len(schema_bytes))
-        buf.write_bytes(schema_bytes)
+        buf.write_bytes_with_length(schema_bytes)
 
     cdef int _write_close_temp_lobs_piggyback(self,
                                               WriteBuffer buf) except -1:
@@ -1235,15 +1235,15 @@ cdef class MessageWithData(Message):
         # write strings
         if conn._client_identifier_modified \
                 and conn._client_identifier is not None:
-            buf.write_bytes(client_identifier_bytes)
+            buf.write_bytes_with_length(client_identifier_bytes)
         if conn._module_modified and conn._module is not None:
-            buf.write_bytes(module_bytes)
+            buf.write_bytes_with_length(module_bytes)
         if conn._action_modified and conn._action is not None:
-            buf.write_bytes(action_bytes)
+            buf.write_bytes_with_length(action_bytes)
         if conn._client_info_modified and conn._client_info is not None:
-            buf.write_bytes(client_info_bytes)
+            buf.write_bytes_with_length(client_info_bytes)
         if conn._dbop_modified and conn._dbop is not None:
-            buf.write_bytes(dbop_bytes)
+            buf.write_bytes_with_length(dbop_bytes)
 
         # reset flags and values
         conn._action_modified = False
@@ -1548,7 +1548,7 @@ cdef class AuthMessage(Message):
         buf.write_uint8(1)                  # pointer (authovl)
         buf.write_uint8(1)                  # pointer (authovln)
         if has_user:
-            buf.write_bytes(self.user_bytes)
+            buf.write_bytes_with_length(self.user_bytes)
 
         # write key/value pairs
         if self.function_code == TNS_FUNC_AUTH_PHASE_ONE:
@@ -1712,7 +1712,8 @@ cdef class DataTypesMessage(Message):
         buf.write_uint8(TNS_MSG_TYPE_DATA_TYPES)
         buf.write_uint16(TNS_CHARSET_UTF8, BYTE_ORDER_LSB)
         buf.write_uint16(TNS_CHARSET_UTF8, BYTE_ORDER_LSB)
-        buf.write_ub4(len(buf._caps.compile_caps))
+        buf.write_uint8(TNS_ENCODING_MULTI_BYTE | TNS_ENCODING_CONV_LENGTH)
+        buf.write_uint8(len(buf._caps.compile_caps))
         buf.write_bytes(bytes(buf._caps.compile_caps))
         buf.write_uint8(len(buf._caps.runtime_caps))
         buf.write_bytes(bytes(buf._caps.runtime_caps))
@@ -1867,7 +1868,7 @@ cdef class ExecuteMessage(MessageWithData):
         if stmt._cursor_id == 0 or stmt._is_ddl:
             if stmt._sql_bytes is None:
                 errors._raise_err(errors.ERR_INVALID_REF_CURSOR)
-            buf.write_bytes(stmt._sql_bytes)
+            buf.write_bytes_with_length(stmt._sql_bytes)
             buf.write_ub4(1)                # al8i4[0] parse
         else:
             buf.write_ub4(0)                # al8i4[0] parse
@@ -2133,7 +2134,6 @@ cdef class ProtocolMessage(Message):
                 if c == 0:
                     break
             buf.read_uint16(&caps.charset_id, BYTE_ORDER_LSB)
-            buf._caps.char_conversion = caps.charset_id != TNS_CHARSET_UTF8
             buf.skip_ub1()                  # skip server flags
             buf.read_uint16(&num_elem, BYTE_ORDER_LSB)
             if num_elem > 0:                # skip elements

src/oracledb/impl/thin/statement.pyx

@@ -1,5 +1,5 @@
 #------------------------------------------------------------------------------
-# Copyright (c) 2020, 2022, Oracle and/or its affiliates.
+# Copyright (c) 2020, 2023, Oracle and/or its affiliates.
 #
 # This software is dual-licensed to you under the Universal Permissive License
 # (UPL) 1.0 as shown at https://oss.oracle.com/licenses/upl and Apache License
@@ -156,7 +156,7 @@ cdef class Statement:
             elif sql_keyword in ("CREATE", "ALTER", "DROP", "TRUNCATE"):
                 self._is_ddl = True
 
-    cdef int _prepare(self, str sql, bint char_conversion) except -1:
+    cdef int _prepare(self, str sql) except -1:
         """
         Prepare the SQL for execution by determining the list of bind names
         that are found within it. The length of the SQL text is also calculated
@@ -171,10 +171,7 @@ cdef class Statement:
         # retain normalized SQL (as string and bytes) as well as the length
         self._sql = sql
         self._sql_bytes = self._sql.encode()
-        if char_conversion:
-            self._sql_length = <uint32_t> len(self._sql)
-        else:
-            self._sql_length = <uint32_t> len(self._sql_bytes)
+        self._sql_length = <uint32_t> len(self._sql_bytes)
 
         # create empty list (bind by position) and dict (bind by name)
         self._bind_info_dict = collections.OrderedDict()