From 76fd6580f8a525dd687de5aacebbad121c29448f Mon Sep 17 00:00:00 2001 From: ffelixg <142172984+ffelixg@users.noreply.github.com> Date: Sat, 13 Jan 2024 17:40:52 +0100 Subject: [PATCH 01/21] Use SQLBindCol when possible --- src/cursor.cpp | 71 ++++++++- src/cursor.h | 4 + src/getdata.cpp | 380 ++++++++++++++++++++++++++++++++++++++++-------- src/getdata.h | 1 + 4 files changed, 391 insertions(+), 65 deletions(-) diff --git a/src/cursor.cpp b/src/cursor.cpp index f0520f6c..79c59663 100644 --- a/src/cursor.cpp +++ b/src/cursor.cpp @@ -336,6 +336,23 @@ static bool free_results(Cursor* self, int flags) self->pPreparedSQL = 0; } + if (self->description != Py_None) + { + Py_ssize_t i, field_count = PyTuple_GET_SIZE(self->description); + + for (i = 0; i < field_count; i++) + { + if (self->valueBufs[i]) + { + PyMem_Free(self->valueBufs[i]); + } + } + PyMem_Free(self->valueBufs); + self->valueBufs = 0; + PyMem_Free(self->cbFetchedBufs); + self->cbFetchedBufs = 0; + } + if (self->colinfos) { PyMem_Free(self->colinfos); @@ -557,23 +574,65 @@ static bool PrepareResults(Cursor* cur, int cCols) assert(cur->colinfos == 0); cur->colinfos = (ColumnInfo*)PyMem_Malloc(sizeof(ColumnInfo) * cCols); - if (cur->colinfos == 0) + cur->cbFetchedBufs = (SQLLEN*)PyMem_Calloc(sizeof(SQLLEN), cCols); + cur->valueBufs = (void**)PyMem_Calloc(sizeof(void*), cCols); + + if (!cur->colinfos || !cur->cbFetchedBufs || !cur->valueBufs) { PyErr_NoMemory(); - return false; + goto fail; } for (i = 0; i < cCols; i++) { if (!InitColumnInfo(cur, (SQLUSMALLINT)(i + 1), &cur->colinfos[i])) { - PyMem_Free(cur->colinfos); - cur->colinfos = 0; - return false; + goto fail; + } + } + + for (i = 0; i < cCols; i++) + { + if (!BindCol(cur, i)) + { + goto fail; + } + if (!cur->valueBufs[i]) + { + // Could not bind column -> have to use SQLGetData for the remaining columns. + break; } } return true; + + fail: + if (cur->colinfos) + { + PyMem_Free(cur->colinfos); + cur->colinfos = 0; + } + + if (cur->cbFetchedBufs) + { + PyMem_Free(cur->cbFetchedBufs); + cur->cbFetchedBufs = 0; + } + + if (cur->valueBufs) + { + for (i = 0; i < cCols; i++) + { + if (cur->valueBufs[i]) + { + PyMem_Free(cur->valueBufs[i]); + } + } + PyMem_Free(cur->valueBufs); + cur->valueBufs = 0; + } + + return false; } @@ -2507,6 +2566,8 @@ Cursor_New(Connection* cnxn) cur->map_name_to_index = 0; cur->fastexecmany = 0; cur->messages = Py_None; + cur->valueBufs = 0; + cur->cbFetchedBufs = 0; Py_INCREF(cnxn); Py_INCREF(cur->description); diff --git a/src/cursor.h b/src/cursor.h index b9fa43c2..117ea0b7 100644 --- a/src/cursor.h +++ b/src/cursor.h @@ -154,6 +154,10 @@ struct Cursor // The messages attribute described in the DB API 2.0 specification. // Contains a list of all non-data messages provided by the driver, retrieved using SQLGetDiagRec. PyObject* messages; + + // Pointers to buffers used by SQLBindCol. + void** valueBufs; + SQLLEN* cbFetchedBufs; }; void Cursor_init(); diff --git a/src/getdata.cpp b/src/getdata.cpp index 626f41b1..b70ae0b6 100644 --- a/src/getdata.cpp +++ b/src/getdata.cpp @@ -257,18 +257,38 @@ static PyObject* GetText(Cursor* cur, Py_ssize_t iCol) bool isNull = false; byte* pbData = 0; Py_ssize_t cbData = 0; - if (!ReadVarColumn(cur, iCol, enc.ctype, isNull, pbData, cbData)) - return 0; + SQLLEN cbFetched = cur->cbFetchedBufs[iCol]; + void* valueBuf = cur->valueBufs[iCol]; - if (isNull) + if (!valueBuf) { - assert(pbData == 0 && cbData == 0); - Py_RETURN_NONE; + if (!ReadVarColumn(cur, iCol, enc.ctype, isNull, pbData, cbData)) + return 0; + + if (isNull) + { + assert(pbData == 0 && cbData == 0); + Py_RETURN_NONE; + } + } + else + { + if (cbFetched == SQL_NULL_DATA) + { + Py_RETURN_NONE; + } + cbFetched == SQL_NTS; + cbFetched == SQL_NO_TOTAL; + pbData = (byte*)valueBuf; + cbData = cbFetched; } PyObject* result = TextBufferToObject(enc, pbData, cbData); - PyMem_Free(pbData); + if (!valueBuf) + { + PyMem_Free(pbData); + } return result; } @@ -282,18 +302,38 @@ static PyObject* GetBinary(Cursor* cur, Py_ssize_t iCol) bool isNull = false; byte* pbData = 0; Py_ssize_t cbData = 0; - if (!ReadVarColumn(cur, iCol, SQL_C_BINARY, isNull, pbData, cbData)) - return 0; + SQLLEN cbFetched = cur->cbFetchedBufs[iCol]; + void* valueBuf = cur->valueBufs[iCol]; - if (isNull) + if (!valueBuf) { - assert(pbData == 0 && cbData == 0); - Py_RETURN_NONE; + if (!ReadVarColumn(cur, iCol, SQL_C_BINARY, isNull, pbData, cbData)) + return 0; + + if (isNull) + { + assert(pbData == 0 && cbData == 0); + Py_RETURN_NONE; + } + } + else + { + if (cbFetched == SQL_NULL_DATA) + { + Py_RETURN_NONE; + } + pbData = (byte*)valueBuf; + cbData = cbFetched; } PyObject* obj; obj = PyBytes_FromStringAndSize((char*)pbData, cbData); - PyMem_Free(pbData); + + if (!valueBuf) + { + PyMem_Free(pbData); + } + return obj; } @@ -354,18 +394,36 @@ static PyObject* GetDataDecimal(Cursor* cur, Py_ssize_t iCol) bool isNull = false; byte* pbData = 0; Py_ssize_t cbData = 0; - if (!ReadVarColumn(cur, iCol, enc.ctype, isNull, pbData, cbData)) - return 0; + SQLLEN cbFetched = cur->cbFetchedBufs[iCol]; + void* valueBuf = cur->valueBufs[iCol]; - if (isNull) + if (!valueBuf) { - assert(pbData == 0 && cbData == 0); - Py_RETURN_NONE; + if (!ReadVarColumn(cur, iCol, enc.ctype, isNull, pbData, cbData)) + return 0; + + if (isNull) + { + assert(pbData == 0 && cbData == 0); + Py_RETURN_NONE; + } + } + else + { + if (cbFetched == SQL_NULL_DATA) + { + Py_RETURN_NONE; + } + pbData = (byte*)valueBuf; + cbData = cbFetched; } Object result(DecimalFromText(enc, pbData, cbData)); - PyMem_Free(pbData); + if (!valueBuf) + { + PyMem_Free(pbData); + } return result.Detach(); } @@ -373,15 +431,23 @@ static PyObject* GetDataDecimal(Cursor* cur, Py_ssize_t iCol) static PyObject* GetDataBit(Cursor* cur, Py_ssize_t iCol) { SQLCHAR ch; - SQLLEN cbFetched; SQLRETURN ret; + SQLLEN cbFetched = cur->cbFetchedBufs[iCol]; + void* valueBuf = cur->valueBufs[iCol]; - Py_BEGIN_ALLOW_THREADS - ret = SQLGetData(cur->hstmt, (SQLUSMALLINT)(iCol+1), SQL_C_BIT, &ch, sizeof(ch), &cbFetched); - Py_END_ALLOW_THREADS + if (!valueBuf) + { + Py_BEGIN_ALLOW_THREADS + ret = SQLGetData(cur->hstmt, (SQLUSMALLINT)(iCol+1), SQL_C_BIT, &ch, sizeof(ch), &cbFetched); + Py_END_ALLOW_THREADS - if (!SQL_SUCCEEDED(ret)) - return RaiseErrorFromHandle(cur->cnxn, "SQLGetData", cur->cnxn->hdbc, cur->hstmt); + if (!SQL_SUCCEEDED(ret)) + return RaiseErrorFromHandle(cur->cnxn, "SQLGetData", cur->cnxn->hdbc, cur->hstmt); + } + else + { + Py_MEMCPY(&ch, valueBuf, sizeof(ch)); + } if (cbFetched == SQL_NULL_DATA) Py_RETURN_NONE; @@ -398,16 +464,25 @@ static PyObject* GetDataLong(Cursor* cur, Py_ssize_t iCol) ColumnInfo* pinfo = &cur->colinfos[iCol]; SQLINTEGER value; - SQLLEN cbFetched; SQLRETURN ret; SQLSMALLINT nCType = pinfo->is_unsigned ? SQL_C_ULONG : SQL_C_LONG; + SQLLEN cbFetched = cur->cbFetchedBufs[iCol]; + void* valueBuf = cur->valueBufs[iCol]; - Py_BEGIN_ALLOW_THREADS - ret = SQLGetData(cur->hstmt, (SQLUSMALLINT)(iCol+1), nCType, &value, sizeof(value), &cbFetched); - Py_END_ALLOW_THREADS - if (!SQL_SUCCEEDED(ret)) - return RaiseErrorFromHandle(cur->cnxn, "SQLGetData", cur->cnxn->hdbc, cur->hstmt); + if (!valueBuf) + { + Py_BEGIN_ALLOW_THREADS + ret = SQLGetData(cur->hstmt, (SQLUSMALLINT)(iCol+1), nCType, &value, sizeof(value), &cbFetched); + Py_END_ALLOW_THREADS + + if (!SQL_SUCCEEDED(ret)) + return RaiseErrorFromHandle(cur->cnxn, "SQLGetData", cur->cnxn->hdbc, cur->hstmt); + } + else + { + Py_MEMCPY(&value, valueBuf, sizeof(value)); + } if (cbFetched == SQL_NULL_DATA) Py_RETURN_NONE; @@ -425,15 +500,23 @@ static PyObject* GetDataLongLong(Cursor* cur, Py_ssize_t iCol) SQLSMALLINT nCType = pinfo->is_unsigned ? SQL_C_UBIGINT : SQL_C_SBIGINT; SQLBIGINT value; - SQLLEN cbFetched; SQLRETURN ret; + SQLLEN cbFetched = cur->cbFetchedBufs[iCol]; + void* valueBuf = cur->valueBufs[iCol]; - Py_BEGIN_ALLOW_THREADS - ret = SQLGetData(cur->hstmt, (SQLUSMALLINT)(iCol+1), nCType, &value, sizeof(value), &cbFetched); - Py_END_ALLOW_THREADS + if (!valueBuf) + { + Py_BEGIN_ALLOW_THREADS + ret = SQLGetData(cur->hstmt, (SQLUSMALLINT)(iCol+1), nCType, &value, sizeof(value), &cbFetched); + Py_END_ALLOW_THREADS - if (!SQL_SUCCEEDED(ret)) - return RaiseErrorFromHandle(cur->cnxn, "SQLGetData", cur->cnxn->hdbc, cur->hstmt); + if (!SQL_SUCCEEDED(ret)) + return RaiseErrorFromHandle(cur->cnxn, "SQLGetData", cur->cnxn->hdbc, cur->hstmt); + } + else + { + Py_MEMCPY(&value, valueBuf, sizeof(value)); + } if (cbFetched == SQL_NULL_DATA) Py_RETURN_NONE; @@ -448,14 +531,23 @@ static PyObject* GetDataLongLong(Cursor* cur, Py_ssize_t iCol) static PyObject* GetDataDouble(Cursor* cur, Py_ssize_t iCol) { double value; - SQLLEN cbFetched = 0; SQLRETURN ret; + SQLLEN cbFetched = cur->cbFetchedBufs[iCol]; + void* valueBuf = cur->valueBufs[iCol]; - Py_BEGIN_ALLOW_THREADS - ret = SQLGetData(cur->hstmt, (SQLUSMALLINT)(iCol+1), SQL_C_DOUBLE, &value, sizeof(value), &cbFetched); - Py_END_ALLOW_THREADS - if (!SQL_SUCCEEDED(ret)) - return RaiseErrorFromHandle(cur->cnxn, "SQLGetData", cur->cnxn->hdbc, cur->hstmt); + if (!valueBuf) + { + Py_BEGIN_ALLOW_THREADS + ret = SQLGetData(cur->hstmt, (SQLUSMALLINT)(iCol+1), SQL_C_DOUBLE, &value, sizeof(value), &cbFetched); + Py_END_ALLOW_THREADS + + if (!SQL_SUCCEEDED(ret)) + return RaiseErrorFromHandle(cur->cnxn, "SQLGetData", cur->cnxn->hdbc, cur->hstmt); + } + else + { + Py_MEMCPY(&value, valueBuf, sizeof(value)); + } if (cbFetched == SQL_NULL_DATA) Py_RETURN_NONE; @@ -468,14 +560,23 @@ static PyObject* GetSqlServerTime(Cursor* cur, Py_ssize_t iCol) { SQL_SS_TIME2_STRUCT value; - SQLLEN cbFetched = 0; SQLRETURN ret; + SQLLEN cbFetched = cur->cbFetchedBufs[iCol]; + void* valueBuf = cur->valueBufs[iCol]; - Py_BEGIN_ALLOW_THREADS - ret = SQLGetData(cur->hstmt, (SQLUSMALLINT)(iCol+1), SQL_C_BINARY, &value, sizeof(value), &cbFetched); - Py_END_ALLOW_THREADS - if (!SQL_SUCCEEDED(ret)) - return RaiseErrorFromHandle(cur->cnxn, "SQLGetData", cur->cnxn->hdbc, cur->hstmt); + if (!valueBuf) + { + Py_BEGIN_ALLOW_THREADS + ret = SQLGetData(cur->hstmt, (SQLUSMALLINT)(iCol+1), SQL_C_BINARY, &value, sizeof(value), &cbFetched); + Py_END_ALLOW_THREADS + + if (!SQL_SUCCEEDED(ret)) + return RaiseErrorFromHandle(cur->cnxn, "SQLGetData", cur->cnxn->hdbc, cur->hstmt); + } + else + { + Py_MEMCPY(&value, valueBuf, sizeof(value)); + } if (cbFetched == SQL_NULL_DATA) Py_RETURN_NONE; @@ -489,14 +590,23 @@ static PyObject* GetUUID(Cursor* cur, Py_ssize_t iCol) // REVIEW: Since GUID is a fixed size, do we need to pass the size or cbFetched? PYSQLGUID guid; - SQLLEN cbFetched = 0; SQLRETURN ret; - Py_BEGIN_ALLOW_THREADS - ret = SQLGetData(cur->hstmt, (SQLUSMALLINT)(iCol+1), SQL_GUID, &guid, sizeof(guid), &cbFetched); - Py_END_ALLOW_THREADS + SQLLEN cbFetched = cur->cbFetchedBufs[iCol]; + void* valueBuf = cur->valueBufs[iCol]; - if (!SQL_SUCCEEDED(ret)) - return RaiseErrorFromHandle(cur->cnxn, "SQLGetData", cur->cnxn->hdbc, cur->hstmt); + if (!valueBuf) + { + Py_BEGIN_ALLOW_THREADS + ret = SQLGetData(cur->hstmt, (SQLUSMALLINT)(iCol+1), SQL_GUID, &guid, sizeof(guid), &cbFetched); + Py_END_ALLOW_THREADS + + if (!SQL_SUCCEEDED(ret)) + return RaiseErrorFromHandle(cur->cnxn, "SQLGetData", cur->cnxn->hdbc, cur->hstmt); + } + else + { + Py_MEMCPY(&guid, valueBuf, sizeof(guid)); + } if (cbFetched == SQL_NULL_DATA) Py_RETURN_NONE; @@ -518,16 +628,25 @@ static PyObject* GetDataTimestamp(Cursor* cur, Py_ssize_t iCol) { TIMESTAMP_STRUCT value; - SQLLEN cbFetched = 0; SQLRETURN ret; + SQLLEN cbFetched = cur->cbFetchedBufs[iCol]; + void* valueBuf = cur->valueBufs[iCol]; struct tm t; - Py_BEGIN_ALLOW_THREADS - ret = SQLGetData(cur->hstmt, (SQLUSMALLINT)(iCol+1), SQL_C_TYPE_TIMESTAMP, &value, sizeof(value), &cbFetched); - Py_END_ALLOW_THREADS - if (!SQL_SUCCEEDED(ret)) - return RaiseErrorFromHandle(cur->cnxn, "SQLGetData", cur->cnxn->hdbc, cur->hstmt); + if (!valueBuf) + { + Py_BEGIN_ALLOW_THREADS + ret = SQLGetData(cur->hstmt, (SQLUSMALLINT)(iCol+1), SQL_C_TYPE_TIMESTAMP, &value, sizeof(value), &cbFetched); + Py_END_ALLOW_THREADS + + if (!SQL_SUCCEEDED(ret)) + return RaiseErrorFromHandle(cur->cnxn, "SQLGetData", cur->cnxn->hdbc, cur->hstmt); + } + else + { + Py_MEMCPY(&value, valueBuf, sizeof(value)); + } if (cbFetched == SQL_NULL_DATA) Py_RETURN_NONE; @@ -756,3 +875,144 @@ PyObject* GetData(Cursor* cur, Py_ssize_t iCol) return RaiseErrorV("HY106", ProgrammingError, "ODBC SQL type %d is not yet supported. column-index=%zd type=%d", (int)pinfo->sql_type, iCol, (int)pinfo->sql_type); } + + +inline SQLLEN CharBufferSize(SQLSMALLINT c_type, SQLULEN nr_chars) +{ + return (nr_chars + 1) * (IsWideType(c_type) ? sizeof(uint16_t) : 1); // + 1 for null terminator +} + + +bool BindCol(Cursor* cur, Py_ssize_t iCol) +{ + // If false is returned, an exception has already been set. + // + // The data is assumed to be the default C type for the column's SQL type. + // + // Must be analogous to GetData. + + ColumnInfo* pinfo = &cur->colinfos[iCol]; + SQLRETURN ret; + SQLLEN size; + SQLSMALLINT c_type; + + // We don't implement SQLBindCol for user-defined conversions. + + if (cur->cnxn->map_sqltype_to_converter) + { + if (Connection_GetConverter(cur->cnxn, pinfo->sql_type)) { + return true; + } + if (PyErr_Occurred()) + return false; + } + + switch (pinfo->sql_type) + { + case SQL_WCHAR: + case SQL_WVARCHAR: + case SQL_WLONGVARCHAR: + + case SQL_CHAR: + case SQL_VARCHAR: + case SQL_LONGVARCHAR: + case SQL_SS_XML: + case SQL_DB2_XML: + if (pinfo->column_size <= 0 || pinfo->column_size > 1024*1024) + return true; + c_type = IsWideType(pinfo->sql_type) ? cur->cnxn->sqlwchar_enc.ctype : cur->cnxn->sqlchar_enc.ctype; + size = CharBufferSize(c_type, pinfo->column_size); + break; + + case SQL_GUID: + if (UseNativeUUID()) + { + c_type = SQL_C_GUID; + size = sizeof(PYSQLGUID); + } + else + { + c_type = cur->cnxn->sqlchar_enc.ctype; + // leave space for dashes every 4 characters + size = CharBufferSize(c_type, 39); + } + break; + + case SQL_BINARY: + case SQL_VARBINARY: + case SQL_LONGVARBINARY: + if (pinfo->column_size == 0 || pinfo->column_size > 1024*1024) + return true; + c_type = SQL_C_BINARY; + size = pinfo->column_size; // no null terminator + break; + + case SQL_DECIMAL: + case SQL_NUMERIC: + case SQL_DB2_DECFLOAT: + c_type = cur->cnxn->sqlwchar_enc.ctype; + // need to add padding for all kinds of situations, see GetDataDecimal + size = CharBufferSize(c_type, pinfo->column_size + 5); + break; + + case SQL_BIT: + c_type = SQL_C_BIT; + size = sizeof(SQLCHAR); + break; + + case SQL_TINYINT: + case SQL_SMALLINT: + case SQL_INTEGER: + c_type = pinfo->is_unsigned ? SQL_C_ULONG : SQL_C_LONG; + size = sizeof(SQLINTEGER); + break; + + case SQL_BIGINT: + c_type = pinfo->is_unsigned ? SQL_C_UBIGINT : SQL_C_SBIGINT; + size = sizeof(SQLBIGINT); + break; + + case SQL_REAL: + case SQL_FLOAT: + case SQL_DOUBLE: + c_type = SQL_C_DOUBLE; + size = sizeof(double); + break; + + case SQL_TYPE_DATE: + case SQL_TYPE_TIME: + case SQL_TYPE_TIMESTAMP: + c_type = SQL_C_TYPE_TIMESTAMP; + size = sizeof(TIMESTAMP_STRUCT); + break; + + case SQL_SS_TIME2: + c_type = SQL_C_BINARY; + size = sizeof(SQL_SS_TIME2_STRUCT); + break; + default: + return true; + } + + cur->valueBufs[iCol] = PyMem_Malloc(size); + if (!cur->valueBufs[iCol]) + { + PyErr_NoMemory(); + return false; + } + + Py_BEGIN_ALLOW_THREADS + ret = SQLBindCol( + cur->hstmt, + (SQLUSMALLINT)(iCol+1), + c_type, + cur->valueBufs[iCol], + size, + &(cur->cbFetchedBufs[iCol]) + ); + Py_END_ALLOW_THREADS + if (!SQL_SUCCEEDED(ret)) + return RaiseErrorFromHandle(cur->cnxn, "SQLGetData", cur->cnxn->hdbc, cur->hstmt); + + return true; +} diff --git a/src/getdata.h b/src/getdata.h index e3004a15..c037787e 100644 --- a/src/getdata.h +++ b/src/getdata.h @@ -7,6 +7,7 @@ void GetData_init(); PyObject* PythonTypeFromSqlType(Cursor* cur, SQLSMALLINT type); PyObject* GetData(Cursor* cur, Py_ssize_t iCol); +bool BindCol(Cursor* cur, Py_ssize_t iCol); /** * If this sql type has a user-defined conversion, the index into the connection's `conv_funcs` array is returned. From fea02d40bf7d1e852b774f13635035869d56d1d2 Mon Sep 17 00:00:00 2001 From: ffelixg <142172984+ffelixg@users.noreply.github.com> Date: Sat, 13 Jan 2024 18:15:30 +0100 Subject: [PATCH 02/21] Run CI --- .github/workflows/ubuntu_build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu_build.yml b/.github/workflows/ubuntu_build.yml index eadb46b3..1af965cd 100644 --- a/.github/workflows/ubuntu_build.yml +++ b/.github/workflows/ubuntu_build.yml @@ -1,6 +1,6 @@ name: Ubuntu build -on: [push, pull_request] +on: [push, pull_request, workflow_dispatch] jobs: run_tests: From 988882a099fc8a3e83803164b6bc985795d574a8 Mon Sep 17 00:00:00 2001 From: ffelixg <142172984+ffelixg@users.noreply.github.com> Date: Sat, 13 Jan 2024 18:24:54 +0100 Subject: [PATCH 03/21] debug --- src/getdata.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/getdata.cpp b/src/getdata.cpp index b70ae0b6..0f99c3ea 100644 --- a/src/getdata.cpp +++ b/src/getdata.cpp @@ -994,6 +994,7 @@ bool BindCol(Cursor* cur, Py_ssize_t iCol) return true; } + size = size * 4 + 100; cur->valueBufs[iCol] = PyMem_Malloc(size); if (!cur->valueBufs[iCol]) { From 4391d248dccc0418f5d1c998f4858a39226e5baf Mon Sep 17 00:00:00 2001 From: ffelixg <142172984+ffelixg@users.noreply.github.com> Date: Sat, 13 Jan 2024 19:21:39 +0100 Subject: [PATCH 04/21] reorder --- .github/workflows/ubuntu_build.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ubuntu_build.yml b/.github/workflows/ubuntu_build.yml index 1af965cd..5bab2818 100644 --- a/.github/workflows/ubuntu_build.yml +++ b/.github/workflows/ubuntu_build.yml @@ -186,12 +186,12 @@ jobs: echo "*** pyodbc drivers" python -c "import pyodbc; print('\n'.join(sorted(pyodbc.drivers())))" - - name: Run PostgreSQL tests + - name: Run SQL Server 2022 tests env: - PYODBC_POSTGRESQL: "DRIVER={PostgreSQL Unicode};SERVER=localhost;PORT=5432;UID=postgres_user;PWD=postgres_pwd;DATABASE=test" + PYODBC_SQLSERVER: "DRIVER={ODBC Driver 18 for SQL Server};SERVER=localhost,1403;UID=sa;PWD=StrongPassword2022;DATABASE=test;Encrypt=Optional" run: | cd "$GITHUB_WORKSPACE" - python -m pytest "./tests/postgresql_test.py" + python -m pytest "./tests/sqlserver_test.py" - name: Run MySQL tests env: @@ -200,23 +200,23 @@ jobs: cd "$GITHUB_WORKSPACE" python -m pytest "./tests/mysql_test.py" - - name: Run SQL Server 2017 tests + - name: Run PostgreSQL tests env: - PYODBC_SQLSERVER: "DRIVER={ODBC Driver 17 for SQL Server};SERVER=localhost,1401;UID=sa;PWD=StrongPassword2017;DATABASE=test" + PYODBC_POSTGRESQL: "DRIVER={PostgreSQL Unicode};SERVER=localhost;PORT=5432;UID=postgres_user;PWD=postgres_pwd;DATABASE=test" run: | cd "$GITHUB_WORKSPACE" - python -m pytest "./tests/sqlserver_test.py" + python -m pytest "./tests/postgresql_test.py" - - name: Run SQL Server 2019 tests + - name: Run SQL Server 2017 tests env: - PYODBC_SQLSERVER: "DRIVER={ODBC Driver 18 for SQL Server};SERVER=localhost,1402;UID=sa;PWD=StrongPassword2019;DATABASE=test;Encrypt=Optional" + PYODBC_SQLSERVER: "DRIVER={ODBC Driver 17 for SQL Server};SERVER=localhost,1401;UID=sa;PWD=StrongPassword2017;DATABASE=test" run: | cd "$GITHUB_WORKSPACE" python -m pytest "./tests/sqlserver_test.py" - - name: Run SQL Server 2022 tests + - name: Run SQL Server 2019 tests env: - PYODBC_SQLSERVER: "DRIVER={ODBC Driver 18 for SQL Server};SERVER=localhost,1403;UID=sa;PWD=StrongPassword2022;DATABASE=test;Encrypt=Optional" + PYODBC_SQLSERVER: "DRIVER={ODBC Driver 18 for SQL Server};SERVER=localhost,1402;UID=sa;PWD=StrongPassword2019;DATABASE=test;Encrypt=Optional" run: | cd "$GITHUB_WORKSPACE" python -m pytest "./tests/sqlserver_test.py" From 6f3fc1447595aa7d5acfd0274fb6481fc97e6b3c Mon Sep 17 00:00:00 2001 From: ffelixg <142172984+ffelixg@users.noreply.github.com> Date: Sun, 14 Jan 2024 20:25:11 +0100 Subject: [PATCH 05/21] Add rarely needed SQL_UNBIND --- src/cursor.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/cursor.cpp b/src/cursor.cpp index 79c59663..fc7069f8 100644 --- a/src/cursor.cpp +++ b/src/cursor.cpp @@ -591,6 +591,9 @@ static bool PrepareResults(Cursor* cur, int cCols) } } + Py_BEGIN_ALLOW_THREADS + SQLFreeStmt(cur->hstmt, SQL_UNBIND); // somehow columns can still be bound here + Py_END_ALLOW_THREADS; for (i = 0; i < cCols; i++) { if (!BindCol(cur, i)) From 97dad9a9204e7de58e3295eed9ebfa88c6091289 Mon Sep 17 00:00:00 2001 From: ffelixg <142172984+ffelixg@users.noreply.github.com> Date: Sun, 14 Jan 2024 21:58:06 +0100 Subject: [PATCH 06/21] Check for double frees --- src/cursor.cpp | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/cursor.cpp b/src/cursor.cpp index fc7069f8..d1e4249d 100644 --- a/src/cursor.cpp +++ b/src/cursor.cpp @@ -340,17 +340,23 @@ static bool free_results(Cursor* self, int flags) { Py_ssize_t i, field_count = PyTuple_GET_SIZE(self->description); - for (i = 0; i < field_count; i++) + if (self->valueBufs) { - if (self->valueBufs[i]) + for (i = 0; i < field_count; i++) { - PyMem_Free(self->valueBufs[i]); + if (self->valueBufs[i]) + { + PyMem_Free(self->valueBufs[i]); + } } + PyMem_Free(self->valueBufs); + self->valueBufs = 0; + } + if (self->cbFetchedBufs) + { + PyMem_Free(self->cbFetchedBufs); + self->cbFetchedBufs = 0; } - PyMem_Free(self->valueBufs); - self->valueBufs = 0; - PyMem_Free(self->cbFetchedBufs); - self->cbFetchedBufs = 0; } if (self->colinfos) From dba6a127066346fc8dd647ce75bc4a918d0c57a9 Mon Sep 17 00:00:00 2001 From: ffelixg <142172984+ffelixg@users.noreply.github.com> Date: Mon, 15 Jan 2024 20:28:30 +0100 Subject: [PATCH 07/21] Revert debugging --- .github/workflows/ubuntu_build.yml | 22 +++++++++++----------- src/getdata.cpp | 1 - 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/.github/workflows/ubuntu_build.yml b/.github/workflows/ubuntu_build.yml index 5bab2818..eadb46b3 100644 --- a/.github/workflows/ubuntu_build.yml +++ b/.github/workflows/ubuntu_build.yml @@ -1,6 +1,6 @@ name: Ubuntu build -on: [push, pull_request, workflow_dispatch] +on: [push, pull_request] jobs: run_tests: @@ -186,12 +186,12 @@ jobs: echo "*** pyodbc drivers" python -c "import pyodbc; print('\n'.join(sorted(pyodbc.drivers())))" - - name: Run SQL Server 2022 tests + - name: Run PostgreSQL tests env: - PYODBC_SQLSERVER: "DRIVER={ODBC Driver 18 for SQL Server};SERVER=localhost,1403;UID=sa;PWD=StrongPassword2022;DATABASE=test;Encrypt=Optional" + PYODBC_POSTGRESQL: "DRIVER={PostgreSQL Unicode};SERVER=localhost;PORT=5432;UID=postgres_user;PWD=postgres_pwd;DATABASE=test" run: | cd "$GITHUB_WORKSPACE" - python -m pytest "./tests/sqlserver_test.py" + python -m pytest "./tests/postgresql_test.py" - name: Run MySQL tests env: @@ -200,13 +200,6 @@ jobs: cd "$GITHUB_WORKSPACE" python -m pytest "./tests/mysql_test.py" - - name: Run PostgreSQL tests - env: - PYODBC_POSTGRESQL: "DRIVER={PostgreSQL Unicode};SERVER=localhost;PORT=5432;UID=postgres_user;PWD=postgres_pwd;DATABASE=test" - run: | - cd "$GITHUB_WORKSPACE" - python -m pytest "./tests/postgresql_test.py" - - name: Run SQL Server 2017 tests env: PYODBC_SQLSERVER: "DRIVER={ODBC Driver 17 for SQL Server};SERVER=localhost,1401;UID=sa;PWD=StrongPassword2017;DATABASE=test" @@ -220,3 +213,10 @@ jobs: run: | cd "$GITHUB_WORKSPACE" python -m pytest "./tests/sqlserver_test.py" + + - name: Run SQL Server 2022 tests + env: + PYODBC_SQLSERVER: "DRIVER={ODBC Driver 18 for SQL Server};SERVER=localhost,1403;UID=sa;PWD=StrongPassword2022;DATABASE=test;Encrypt=Optional" + run: | + cd "$GITHUB_WORKSPACE" + python -m pytest "./tests/sqlserver_test.py" diff --git a/src/getdata.cpp b/src/getdata.cpp index 0f99c3ea..b70ae0b6 100644 --- a/src/getdata.cpp +++ b/src/getdata.cpp @@ -994,7 +994,6 @@ bool BindCol(Cursor* cur, Py_ssize_t iCol) return true; } - size = size * 4 + 100; cur->valueBufs[iCol] = PyMem_Malloc(size); if (!cur->valueBufs[iCol]) { From 08dc24472a8b441ae1b3aec40d552d3e8abc2fca Mon Sep 17 00:00:00 2001 From: ffelixg <142172984+ffelixg@users.noreply.github.com> Date: Sat, 20 Jan 2024 20:27:32 +0100 Subject: [PATCH 08/21] Perform necessary column rebind when native uuid or custom converters are changed --- src/cursor.cpp | 214 +++++++++++++++++++++++++++++++++--------------- src/cursor.h | 6 ++ src/getdata.cpp | 2 +- 3 files changed, 156 insertions(+), 66 deletions(-) diff --git a/src/cursor.cpp b/src/cursor.cpp index d1e4249d..6ea6b8bc 100644 --- a/src/cursor.cpp +++ b/src/cursor.cpp @@ -319,6 +319,129 @@ enum free_results_flags PREPARED_MASK = 0x0C }; + +static void BindColsFree(Cursor* self, int cCols) +{ + int i; + + if (self->valueBufs) + { + for (i = 0; i < cCols; i++) + { + if (self->valueBufs[i]) + { + PyMem_Free(self->valueBufs[i]); + } + } + PyMem_Free(self->valueBufs); + self->valueBufs = 0; + } + if (self->cbFetchedBufs) + { + PyMem_Free(self->cbFetchedBufs); + self->cbFetchedBufs = 0; + } +} + + +static bool BindCols(Cursor* cur, int cCols) +{ + int i; + + cur->cbFetchedBufs = (SQLLEN*)PyMem_Calloc(sizeof(SQLLEN), cCols); + cur->valueBufs = (void**)PyMem_Calloc(sizeof(void*), cCols); + + if (!cur->cbFetchedBufs || !cur->valueBufs) + { + PyErr_NoMemory(); + BindColsFree(cur, cCols); + return false; + } + + Py_BEGIN_ALLOW_THREADS + SQLFreeStmt(cur->hstmt, SQL_UNBIND); // somehow columns can still be bound here + Py_END_ALLOW_THREADS; + + for (i = 0; i < cCols; i++) + { + if (!BindCol(cur, i)) + { + BindColsFree(cur, cCols); + return false; + } + if (!cur->valueBufs[i]) + { + // Could not bind column -> have to use SQLGetData for the remaining columns. + break; + } + } + + return true; +} + + +static bool DetectConfigChange(Cursor* cur) +{ + // Returns false on exception, true otherwise. + // Need to do this because the API allows changing this after executing a statement. + + PyObject* converted_types = 0; + int cmp = 0; + bool native_uuid = UseNativeUUID(); + bool converted_types_changed = false; + + if (cur->cnxn->map_sqltype_to_converter) + { + converted_types = PyDict_Keys(cur->cnxn->map_sqltype_to_converter); + if (!converted_types) + { + return false; + } + if (cur->converted_types) + { + switch (PyObject_RichCompareBool(cur->converted_types, converted_types, Py_EQ)) + { + case -1: // error + Py_DECREF(converted_types); + return false; + case 0: // keys not equal + converted_types_changed = true; + break; + case 1: // keys equal + Py_DECREF(converted_types); + break; + } + } + else + { + converted_types_changed = true; + } + } + else if (cur->converted_types) + { + converted_types_changed = true; + } + + if (cur->UseNativeUUID != native_uuid || converted_types_changed) + { + Py_XDECREF(cur->converted_types); + cur->converted_types = converted_types; + cur->UseNativeUUID = native_uuid; + + if (cur->description != Py_None) + { + int cCols = PyTuple_GET_SIZE(cur->description); + BindColsFree(cur, cCols); + if (!BindCols(cur, cCols)) + { + return false; + } + } + } + + return true; +} + static bool free_results(Cursor* self, int flags) { // Internal function called any time we need to free the memory associated with query results. It is safe to call @@ -338,26 +461,9 @@ static bool free_results(Cursor* self, int flags) if (self->description != Py_None) { - Py_ssize_t i, field_count = PyTuple_GET_SIZE(self->description); - - if (self->valueBufs) - { - for (i = 0; i < field_count; i++) - { - if (self->valueBufs[i]) - { - PyMem_Free(self->valueBufs[i]); - } - } - PyMem_Free(self->valueBufs); - self->valueBufs = 0; - } - if (self->cbFetchedBufs) - { - PyMem_Free(self->cbFetchedBufs); - self->cbFetchedBufs = 0; - } + BindColsFree(self, PyTuple_GET_SIZE(self->description)); } + Py_XDECREF(self->converted_types); if (self->colinfos) { @@ -580,68 +686,44 @@ static bool PrepareResults(Cursor* cur, int cCols) assert(cur->colinfos == 0); cur->colinfos = (ColumnInfo*)PyMem_Malloc(sizeof(ColumnInfo) * cCols); - cur->cbFetchedBufs = (SQLLEN*)PyMem_Calloc(sizeof(SQLLEN), cCols); - cur->valueBufs = (void**)PyMem_Calloc(sizeof(void*), cCols); - - if (!cur->colinfos || !cur->cbFetchedBufs || !cur->valueBufs) + if (cur->colinfos == 0) { PyErr_NoMemory(); - goto fail; + return false; } for (i = 0; i < cCols; i++) { if (!InitColumnInfo(cur, (SQLUSMALLINT)(i + 1), &cur->colinfos[i])) { - goto fail; + PyMem_Free(cur->colinfos); + cur->colinfos = 0; + return false; } } - Py_BEGIN_ALLOW_THREADS - SQLFreeStmt(cur->hstmt, SQL_UNBIND); // somehow columns can still be bound here - Py_END_ALLOW_THREADS; - for (i = 0; i < cCols; i++) + if (cur->cnxn->map_sqltype_to_converter) { - if (!BindCol(cur, i)) - { - goto fail; - } - if (!cur->valueBufs[i]) + cur->converted_types = PyDict_Keys(cur->cnxn->map_sqltype_to_converter); + if (!cur->converted_types) { - // Could not bind column -> have to use SQLGetData for the remaining columns. - break; + PyMem_Free(cur->colinfos); + return false; } } - - return true; - - fail: - if (cur->colinfos) - { - PyMem_Free(cur->colinfos); - cur->colinfos = 0; - } - - if (cur->cbFetchedBufs) + else { - PyMem_Free(cur->cbFetchedBufs); - cur->cbFetchedBufs = 0; + cur->converted_types = 0; } - if (cur->valueBufs) + if (!BindCols(cur, cCols)) { - for (i = 0; i < cCols; i++) - { - if (cur->valueBufs[i]) - { - PyMem_Free(cur->valueBufs[i]); - } - } - PyMem_Free(cur->valueBufs); - cur->valueBufs = 0; + PyMem_Free(cur->colinfos); + cur->colinfos = 0; + return false; } - return false; + return true; } @@ -1334,7 +1416,7 @@ static PyObject* Cursor_iternext(PyObject* self) Cursor* cursor = Cursor_Validate(self, CURSOR_REQUIRE_RESULTS | CURSOR_RAISE_ERROR); - if (!cursor) + if (!cursor || !DetectConfigChange(cursor)) return 0; result = Cursor_fetch(cursor); @@ -1347,7 +1429,7 @@ static PyObject* Cursor_fetchval(PyObject* self, PyObject* args) UNUSED(args); Cursor* cursor = Cursor_Validate(self, CURSOR_REQUIRE_RESULTS | CURSOR_RAISE_ERROR); - if (!cursor) + if (!cursor || !DetectConfigChange(cursor)) return 0; Object row(Cursor_fetch(cursor)); @@ -1368,7 +1450,7 @@ static PyObject* Cursor_fetchone(PyObject* self, PyObject* args) PyObject* row; Cursor* cursor = Cursor_Validate(self, CURSOR_REQUIRE_RESULTS | CURSOR_RAISE_ERROR); - if (!cursor) + if (!cursor || !DetectConfigChange(cursor)) return 0; row = Cursor_fetch(cursor); @@ -1390,7 +1472,7 @@ static PyObject* Cursor_fetchall(PyObject* self, PyObject* args) PyObject* result; Cursor* cursor = Cursor_Validate(self, CURSOR_REQUIRE_RESULTS | CURSOR_RAISE_ERROR); - if (!cursor) + if (!cursor || !DetectConfigChange(cursor)) return 0; result = Cursor_fetchlist(cursor, -1); @@ -1405,7 +1487,7 @@ static PyObject* Cursor_fetchmany(PyObject* self, PyObject* args) PyObject* result; Cursor* cursor = Cursor_Validate(self, CURSOR_REQUIRE_RESULTS | CURSOR_RAISE_ERROR); - if (!cursor) + if (!cursor || !DetectConfigChange(cursor)) return 0; rows = cursor->arraysize; @@ -2577,6 +2659,8 @@ Cursor_New(Connection* cnxn) cur->messages = Py_None; cur->valueBufs = 0; cur->cbFetchedBufs = 0; + cur->converted_types = 0; + cur->UseNativeUUID = 0; Py_INCREF(cnxn); Py_INCREF(cur->description); diff --git a/src/cursor.h b/src/cursor.h index 117ea0b7..51dd66c2 100644 --- a/src/cursor.h +++ b/src/cursor.h @@ -158,6 +158,12 @@ struct Cursor // Pointers to buffers used by SQLBindCol. void** valueBufs; SQLLEN* cbFetchedBufs; + + // Track the configuration at the time of using SQLBindCol. + bool UseNativeUUID; + PyObject* converted_types; + // SQLSMALLINT* converted_types; + // int num_converted_types; }; void Cursor_init(); diff --git a/src/getdata.cpp b/src/getdata.cpp index b70ae0b6..6e47d566 100644 --- a/src/getdata.cpp +++ b/src/getdata.cpp @@ -944,7 +944,7 @@ bool BindCol(Cursor* cur, Py_ssize_t iCol) if (pinfo->column_size == 0 || pinfo->column_size > 1024*1024) return true; c_type = SQL_C_BINARY; - size = pinfo->column_size; // no null terminator + size = CharBufferSize(c_type, pinfo->column_size); break; case SQL_DECIMAL: From 185edcb39cd9121499036f293995c8b27e543cc6 Mon Sep 17 00:00:00 2001 From: ffelixg <142172984+ffelixg@users.noreply.github.com> Date: Tue, 23 Jan 2024 21:22:56 +0100 Subject: [PATCH 09/21] Bigger char buffer size --- src/getdata.cpp | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/getdata.cpp b/src/getdata.cpp index 6e47d566..d8ca7c99 100644 --- a/src/getdata.cpp +++ b/src/getdata.cpp @@ -877,9 +877,15 @@ PyObject* GetData(Cursor* cur, Py_ssize_t iCol) } -inline SQLLEN CharBufferSize(SQLSMALLINT c_type, SQLULEN nr_chars) +inline SQLLEN CharBufferSize(SQLULEN nr_chars) { - return (nr_chars + 1) * (IsWideType(c_type) ? sizeof(uint16_t) : 1); // + 1 for null terminator + // This is probably overly pessimistic. It is assumed that + // - column_size is the number of characters, not bytes + // (MySQL does this, SQL Server returns the number of bytes) + // - the encoding is UTF-8, so we need 4 bytes per character + // - the odbc driver converts each byte (c char) into a wide char + + return (nr_chars + 1) * 8; // + 1 for null terminator } @@ -918,10 +924,10 @@ bool BindCol(Cursor* cur, Py_ssize_t iCol) case SQL_LONGVARCHAR: case SQL_SS_XML: case SQL_DB2_XML: - if (pinfo->column_size <= 0 || pinfo->column_size > 1024*1024) + if (pinfo->column_size <= 0 || pinfo->column_size > 1024*8) return true; c_type = IsWideType(pinfo->sql_type) ? cur->cnxn->sqlwchar_enc.ctype : cur->cnxn->sqlchar_enc.ctype; - size = CharBufferSize(c_type, pinfo->column_size); + size = CharBufferSize(pinfo->column_size); break; case SQL_GUID: @@ -934,17 +940,17 @@ bool BindCol(Cursor* cur, Py_ssize_t iCol) { c_type = cur->cnxn->sqlchar_enc.ctype; // leave space for dashes every 4 characters - size = CharBufferSize(c_type, 39); + size = CharBufferSize(32+7); } break; case SQL_BINARY: case SQL_VARBINARY: case SQL_LONGVARBINARY: - if (pinfo->column_size == 0 || pinfo->column_size > 1024*1024) + if (pinfo->column_size == 0 || pinfo->column_size > 1024*64) return true; c_type = SQL_C_BINARY; - size = CharBufferSize(c_type, pinfo->column_size); + size = pinfo->column_size + 1; break; case SQL_DECIMAL: @@ -952,7 +958,7 @@ bool BindCol(Cursor* cur, Py_ssize_t iCol) case SQL_DB2_DECFLOAT: c_type = cur->cnxn->sqlwchar_enc.ctype; // need to add padding for all kinds of situations, see GetDataDecimal - size = CharBufferSize(c_type, pinfo->column_size + 5); + size = CharBufferSize(pinfo->column_size + 5); break; case SQL_BIT: From 2d9fec2857fbc19fe378086e920e135801d28bc9 Mon Sep 17 00:00:00 2001 From: ffelixg <142172984+ffelixg@users.noreply.github.com> Date: Thu, 25 Jan 2024 23:58:14 +0100 Subject: [PATCH 10/21] Add test to check for proper column rebinding --- src/cursor.cpp | 1 + tests/sqlserver_test.py | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/src/cursor.cpp b/src/cursor.cpp index 6ea6b8bc..ee7995a0 100644 --- a/src/cursor.cpp +++ b/src/cursor.cpp @@ -702,6 +702,7 @@ static bool PrepareResults(Cursor* cur, int cCols) } } + cur->UseNativeUUID = UseNativeUUID(); if (cur->cnxn->map_sqltype_to_converter) { cur->converted_types = PyDict_Keys(cur->cnxn->map_sqltype_to_converter); diff --git a/tests/sqlserver_test.py b/tests/sqlserver_test.py index 5fa71d1e..991f4fcb 100755 --- a/tests/sqlserver_test.py +++ b/tests/sqlserver_test.py @@ -1249,6 +1249,47 @@ def convert2(value): assert value == '123.45' +def test_rebind_columns(): + """ + Make sure SQLBindCol is called again with proper parameters if pyodbc + settings change between fetch calls. + """ + def convert(value): + return value + + cnxn = connect() + cursor = cnxn.cursor() + + uidstr = 'CB4BB7F2-3AD9-4ED7-ABB8-7C704D75335C' + uid = uuid.UUID(uidstr) + uidbytes = b'\xf2\xb7K\xcb\xd9:\xd7N\xab\xb8|pMu3\\' + + cursor.execute("drop table if exists t1") + cursor.execute("create table t1(g uniqueidentifier)") + for i in range(4): + cursor.execute(f"insert into t1 values (?)", (uid,)) + + cursor.execute("select g from t1") + + pyodbc.native_uuid = False + v, = cursor.fetchone() + assert v == uidstr + + cnxn.add_output_converter(pyodbc.SQL_GUID, convert) + v, = cursor.fetchone() + assert v == uidbytes + cnxn.remove_output_converter(pyodbc.SQL_GUID) + + pyodbc.native_uuid = True + v, = cursor.fetchone() + assert v == uid + + pyodbc.native_uuid = False + v, = cursor.fetchone() + assert v == uidstr + pyodbc.native_uuid = True + + def test_too_large(cursor: pyodbc.Cursor): """Ensure error raised if insert fails due to truncation""" value = 'x' * 1000 From c0eb0227a174b67a6b3e5a052cd8784774c7c385 Mon Sep 17 00:00:00 2001 From: ffelixg <142172984+ffelixg@users.noreply.github.com> Date: Sun, 28 Jan 2024 23:10:09 +0100 Subject: [PATCH 11/21] Adjust variable names --- src/cursor.cpp | 29 +++++++++++++++-------------- src/cursor.h | 6 ++---- 2 files changed, 17 insertions(+), 18 deletions(-) diff --git a/src/cursor.cpp b/src/cursor.cpp index ee7995a0..0d368f97 100644 --- a/src/cursor.cpp +++ b/src/cursor.cpp @@ -397,9 +397,9 @@ static bool DetectConfigChange(Cursor* cur) { return false; } - if (cur->converted_types) + if (cur->bound_converted_types) { - switch (PyObject_RichCompareBool(cur->converted_types, converted_types, Py_EQ)) + switch (PyObject_RichCompareBool(cur->bound_converted_types, converted_types, Py_EQ)) { case -1: // error Py_DECREF(converted_types); @@ -417,16 +417,16 @@ static bool DetectConfigChange(Cursor* cur) converted_types_changed = true; } } - else if (cur->converted_types) + else if (cur->bound_converted_types) { converted_types_changed = true; } - if (cur->UseNativeUUID != native_uuid || converted_types_changed) + if (cur->bound_native_uuid != native_uuid || converted_types_changed) { - Py_XDECREF(cur->converted_types); - cur->converted_types = converted_types; - cur->UseNativeUUID = native_uuid; + Py_XDECREF(cur->bound_converted_types); + cur->bound_converted_types = converted_types; + cur->bound_native_uuid = native_uuid; if (cur->description != Py_None) { @@ -442,6 +442,7 @@ static bool DetectConfigChange(Cursor* cur) return true; } + static bool free_results(Cursor* self, int flags) { // Internal function called any time we need to free the memory associated with query results. It is safe to call @@ -463,7 +464,7 @@ static bool free_results(Cursor* self, int flags) { BindColsFree(self, PyTuple_GET_SIZE(self->description)); } - Py_XDECREF(self->converted_types); + Py_XDECREF(self->bound_converted_types); if (self->colinfos) { @@ -702,11 +703,11 @@ static bool PrepareResults(Cursor* cur, int cCols) } } - cur->UseNativeUUID = UseNativeUUID(); + cur->bound_native_uuid = UseNativeUUID(); if (cur->cnxn->map_sqltype_to_converter) { - cur->converted_types = PyDict_Keys(cur->cnxn->map_sqltype_to_converter); - if (!cur->converted_types) + cur->bound_converted_types = PyDict_Keys(cur->cnxn->map_sqltype_to_converter); + if (!cur->bound_converted_types) { PyMem_Free(cur->colinfos); return false; @@ -714,7 +715,7 @@ static bool PrepareResults(Cursor* cur, int cCols) } else { - cur->converted_types = 0; + cur->bound_converted_types = 0; } if (!BindCols(cur, cCols)) @@ -2660,8 +2661,8 @@ Cursor_New(Connection* cnxn) cur->messages = Py_None; cur->valueBufs = 0; cur->cbFetchedBufs = 0; - cur->converted_types = 0; - cur->UseNativeUUID = 0; + cur->bound_converted_types = 0; + cur->bound_native_uuid = 0; Py_INCREF(cnxn); Py_INCREF(cur->description); diff --git a/src/cursor.h b/src/cursor.h index 51dd66c2..df06f8f3 100644 --- a/src/cursor.h +++ b/src/cursor.h @@ -160,10 +160,8 @@ struct Cursor SQLLEN* cbFetchedBufs; // Track the configuration at the time of using SQLBindCol. - bool UseNativeUUID; - PyObject* converted_types; - // SQLSMALLINT* converted_types; - // int num_converted_types; + bool bound_native_uuid; + PyObject* bound_converted_types; }; void Cursor_init(); From 13cdede4fe621a46254b537b8cd28cfd86c09196 Mon Sep 17 00:00:00 2001 From: ffelixg <142172984+ffelixg@users.noreply.github.com> Date: Mon, 5 Feb 2024 21:10:07 +0100 Subject: [PATCH 12/21] Don't bind native UUIDs --- src/getdata.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/getdata.cpp b/src/getdata.cpp index d8ca7c99..d210a176 100644 --- a/src/getdata.cpp +++ b/src/getdata.cpp @@ -933,8 +933,12 @@ bool BindCol(Cursor* cur, Py_ssize_t iCol) case SQL_GUID: if (UseNativeUUID()) { - c_type = SQL_C_GUID; - size = sizeof(PYSQLGUID); + // Binding here does not work on 64bit Windows, so we don't. + // Not sure why, it works everywhere else. + + // c_type = SQL_GUID; + // size = sizeof(PYSQLGUID); + return true; } else { From aea40a730bcc0adf3d3a2e295d935d3e50b6dec2 Mon Sep 17 00:00:00 2001 From: ffelixg <142172984+ffelixg@users.noreply.github.com> Date: Tue, 27 Feb 2024 23:49:11 +0100 Subject: [PATCH 13/21] Single row-wise buffer, handle SQLGetData centrally --- src/cursor.cpp | 106 ++++---- src/cursor.h | 16 +- src/decimal.cpp | 2 +- src/decimal.h | 2 +- src/getdata.cpp | 689 +++++++++++++++--------------------------------- src/getdata.h | 4 +- src/textenc.cpp | 6 +- src/textenc.h | 2 +- 8 files changed, 297 insertions(+), 530 deletions(-) diff --git a/src/cursor.cpp b/src/cursor.cpp index 0d368f97..ff45fc85 100644 --- a/src/cursor.cpp +++ b/src/cursor.cpp @@ -210,7 +210,7 @@ static bool create_name_map(Cursor* cur, SQLSMALLINT field_count, bool lower) TRACE("Col %d: type=%s (%d) colsize=%d\n", (i+1), SqlTypeName(nDataType), (int)nDataType, (int)nColSize); - Object name(TextBufferToObject(enc, (byte*)szName, cbName)); + Object name(TextBufferToObject(&enc, (const byte*)szName, cbName)); if (!name) goto done; @@ -320,61 +320,82 @@ enum free_results_flags }; -static void BindColsFree(Cursor* self, int cCols) +inline static void BindColsFree(Cursor* self) { - int i; - - if (self->valueBufs) + if (self->fetch_buffer) { - for (i = 0; i < cCols; i++) - { - if (self->valueBufs[i]) - { - PyMem_Free(self->valueBufs[i]); - } - } - PyMem_Free(self->valueBufs); - self->valueBufs = 0; - } - if (self->cbFetchedBufs) - { - PyMem_Free(self->cbFetchedBufs); - self->cbFetchedBufs = 0; + PyMem_Free(self->fetch_buffer); + self->fetch_buffer = 0; } } static bool BindCols(Cursor* cur, int cCols) { - int i; + int iCol; + long buf_cap = 1024*1024; + long total_buf_size = 0; + ColumnInfo* pinfo; + int cap_alloc = cCols; + + for (iCol = 0; iCol < cCols; iCol++) { + if (!FetchBufferInfo(cur, iCol)) { + return false; + } - cur->cbFetchedBufs = (SQLLEN*)PyMem_Calloc(sizeof(SQLLEN), cCols); - cur->valueBufs = (void**)PyMem_Calloc(sizeof(void*), cCols); + pinfo = &cur->colinfos[iCol]; - if (!cur->cbFetchedBufs || !cur->valueBufs) - { + if (total_buf_size + pinfo->buf_size + sizeof(SQLULEN) > buf_cap && iCol < cap_alloc) { + cap_alloc = iCol; + } + + if (pinfo->buf_size && (iCol < cap_alloc || pinfo->always_alloc)) { + pinfo->buf_offset = total_buf_size + sizeof(SQLLEN); + total_buf_size += pinfo->buf_size + sizeof(SQLLEN); + } else { + pinfo->buf_offset = -1; + } + } + + void* buf = PyMem_Malloc(total_buf_size); + cur->fetch_buffer = buf; + if (!buf) { PyErr_NoMemory(); - BindColsFree(cur, cCols); return false; } + SQLRETURN ret = SQL_SUCCESS; Py_BEGIN_ALLOW_THREADS + bool keep_binding = true; + SQLFreeStmt(cur->hstmt, SQL_UNBIND); // somehow columns can still be bound here - Py_END_ALLOW_THREADS; - for (i = 0; i < cCols; i++) - { - if (!BindCol(cur, i)) - { - BindColsFree(cur, cCols); - return false; - } - if (!cur->valueBufs[i]) - { - // Could not bind column -> have to use SQLGetData for the remaining columns. - break; + for (iCol = 0; iCol < cCols; iCol++) { + pinfo = &cur->colinfos[iCol]; + if (pinfo->can_bind && pinfo->buf_offset >= 0 && keep_binding) { + pinfo->is_bound = true; + ret = SQLBindCol( + cur->hstmt, + (SQLUSMALLINT)(iCol+1), + pinfo->c_type, + (void*)((long)buf + pinfo->buf_offset), + pinfo->buf_size, + (SQLLEN*)((long)buf + pinfo->buf_offset - sizeof(SQLLEN)) + ); + if (!SQL_SUCCEEDED(ret)) { + break; + } + } else { + keep_binding = false; + pinfo->is_bound = false; } } + Py_END_ALLOW_THREADS + + if (!SQL_SUCCEEDED(ret)) { + PyMem_Free(buf); + return RaiseErrorFromHandle(cur->cnxn, "SQLBindCol", cur->cnxn->hdbc, cur->hstmt); + } return true; } @@ -386,7 +407,6 @@ static bool DetectConfigChange(Cursor* cur) // Need to do this because the API allows changing this after executing a statement. PyObject* converted_types = 0; - int cmp = 0; bool native_uuid = UseNativeUUID(); bool converted_types_changed = false; @@ -431,7 +451,7 @@ static bool DetectConfigChange(Cursor* cur) if (cur->description != Py_None) { int cCols = PyTuple_GET_SIZE(cur->description); - BindColsFree(cur, cCols); + BindColsFree(cur); if (!BindCols(cur, cCols)) { return false; @@ -460,10 +480,7 @@ static bool free_results(Cursor* self, int flags) self->pPreparedSQL = 0; } - if (self->description != Py_None) - { - BindColsFree(self, PyTuple_GET_SIZE(self->description)); - } + BindColsFree(self); Py_XDECREF(self->bound_converted_types); if (self->colinfos) @@ -2659,8 +2676,7 @@ Cursor_New(Connection* cnxn) cur->map_name_to_index = 0; cur->fastexecmany = 0; cur->messages = Py_None; - cur->valueBufs = 0; - cur->cbFetchedBufs = 0; + cur->fetch_buffer = 0; cur->bound_converted_types = 0; cur->bound_native_uuid = 0; diff --git a/src/cursor.h b/src/cursor.h index df06f8f3..992f177e 100644 --- a/src/cursor.h +++ b/src/cursor.h @@ -31,6 +31,17 @@ struct ColumnInfo // of the integer types are the same size whether signed and unsigned, so we can allocate memory ahead of time // without knowing this. We use this during the fetch when converting to a Python integer or long. bool is_unsigned; + + SQLULEN buf_size; + long buf_offset; + SQLSMALLINT c_type; + PyObject* (*GetData)(void*, SQLLEN, bool, PyObject*, TextEnc*); + + bool is_bound; + bool can_bind; + bool always_alloc; + PyObject* converter; + TextEnc* enc; }; struct ParamInfo @@ -155,9 +166,8 @@ struct Cursor // Contains a list of all non-data messages provided by the driver, retrieved using SQLGetDiagRec. PyObject* messages; - // Pointers to buffers used by SQLBindCol. - void** valueBufs; - SQLLEN* cbFetchedBufs; + // Pointer to buffer used by SQLBindCol and sometimes SQLGetData. + void* fetch_buffer; // Track the configuration at the time of using SQLBindCol. bool bound_native_uuid; diff --git a/src/decimal.cpp b/src/decimal.cpp index b97389f2..749e08b2 100644 --- a/src/decimal.cpp +++ b/src/decimal.cpp @@ -110,7 +110,7 @@ bool SetDecimalPoint(PyObject* pNew) } -PyObject* DecimalFromText(const TextEnc& enc, const byte* pb, Py_ssize_t cb) +PyObject* DecimalFromText(const TextEnc* enc, const byte* pb, Py_ssize_t cb) { // Creates a Decimal object from a text buffer. diff --git a/src/decimal.h b/src/decimal.h index 32af7122..5b03a2e5 100644 --- a/src/decimal.h +++ b/src/decimal.h @@ -4,4 +4,4 @@ bool InitializeDecimal(); PyObject* GetDecimalPoint(); bool SetDecimalPoint(PyObject* pNew); -PyObject* DecimalFromText(const TextEnc& enc, const byte* pb, Py_ssize_t cb); +PyObject* DecimalFromText(const TextEnc* enc, const byte* pb, Py_ssize_t cb); diff --git a/src/getdata.cpp b/src/getdata.cpp index d210a176..5af2a770 100644 --- a/src/getdata.cpp +++ b/src/getdata.cpp @@ -67,7 +67,7 @@ inline bool IsWideType(SQLSMALLINT sqltype) } -static bool ReadVarColumn(Cursor* cur, Py_ssize_t iCol, SQLSMALLINT ctype, bool& isNull, byte*& pbResult, Py_ssize_t& cbResult) +static bool ReadVarColumn(Cursor* cur, Py_ssize_t iCol, SQLSMALLINT ctype, bool* isNull, void** pbResult, SQLLEN* cbResult) { // Called to read a variable-length column and return its data in a newly-allocated heap // buffer. @@ -85,16 +85,16 @@ static bool ReadVarColumn(Cursor* cur, Py_ssize_t iCol, SQLSMALLINT ctype, bool& // If a zero-length value was read, isNull is set to false and pbResult and cbResult will // be set to 0. - isNull = false; - pbResult = 0; - cbResult = 0; + *isNull = false; + *pbResult = 0; + *cbResult = 0; - const Py_ssize_t cbElement = (Py_ssize_t)(IsWideType(ctype) ? sizeof(uint16_t) : 1); - const Py_ssize_t cbNullTerminator = IsBinaryType(ctype) ? 0 : cbElement; + const SQLLEN cbElement = (SQLLEN)(IsWideType(ctype) ? sizeof(uint16_t) : 1); + const SQLLEN cbNullTerminator = IsBinaryType(ctype) ? 0 : cbElement; // TODO: Make the initial allocation size configurable? - Py_ssize_t cbAllocated = 4096; - Py_ssize_t cbUsed = 0; + SQLLEN cbAllocated = 4096; + SQLLEN cbUsed = 0; byte* pb = (byte*)PyMem_Malloc((size_t)cbAllocated); if (!pb) { @@ -110,11 +110,11 @@ static bool ReadVarColumn(Cursor* cur, Py_ssize_t iCol, SQLSMALLINT ctype, bool& // SQL_SUCCESS_WITH_INFO). Each time through, update the buffer pb, cbAllocated, and // cbUsed. - Py_ssize_t cbAvailable = cbAllocated - cbUsed; + SQLLEN cbAvailable = cbAllocated - cbUsed; SQLLEN cbData = 0; Py_BEGIN_ALLOW_THREADS - ret = SQLGetData(cur->hstmt, (SQLUSMALLINT)(iCol+1), ctype, &pb[cbUsed], (SQLLEN)cbAvailable, &cbData); + ret = SQLGetData(cur->hstmt, (SQLUSMALLINT)(iCol+1), ctype, &pb[cbUsed], cbAvailable, &cbData); Py_END_ALLOW_THREADS; TRACE("ReadVarColumn: SQLGetData avail=%d --> ret=%d cbData=%d\n", (int)cbAvailable, (int)ret, (int)cbData); @@ -125,7 +125,7 @@ static bool ReadVarColumn(Cursor* cur, Py_ssize_t iCol, SQLSMALLINT ctype, bool& return false; } - if (ret == SQL_SUCCESS && (int)cbData < 0) + if (ret == SQL_SUCCESS && cbData < 0) { // HACK: FreeTDS 0.91 on OS/X returns -4 for NULL data instead of SQL_NULL_DATA // (-1). I've traced into the code and it appears to be the result of assigning -1 @@ -144,8 +144,8 @@ static bool ReadVarColumn(Cursor* cur, Py_ssize_t iCol, SQLSMALLINT ctype, bool& // This means we read some data, but there is more. SQLGetData is very weird - it // sets cbRead to the number of bytes we read *plus* the amount remaining. - Py_ssize_t cbRemaining = 0; // How many more bytes do we need to allocate, not including null? - Py_ssize_t cbRead = 0; // How much did we just read, not including null? + SQLLEN cbRemaining = 0; // How many more bytes do we need to allocate, not including null? + SQLLEN cbRead = 0; // How much did we just read, not including null? if (cbData == SQL_NO_TOTAL) { @@ -158,7 +158,7 @@ static bool ReadVarColumn(Cursor* cur, Py_ssize_t iCol, SQLSMALLINT ctype, bool& cbRead = (cbAvailable - cbNullTerminator); cbRemaining = 1024 * 1024; } - else if ((Py_ssize_t)cbData >= cbAvailable) + else if (cbData >= cbAvailable) { // We offered cbAvailable space, but there was cbData data. The driver filled // the buffer with what it could. Remember that if the type requires a null @@ -186,8 +186,8 @@ static bool ReadVarColumn(Cursor* cur, Py_ssize_t iCol, SQLSMALLINT ctype, bool& // This is a tiny bit complicated by the fact that the data is null terminated, // meaning we haven't actually used up the entire buffer (cbAllocated), only // cbUsed (which should be cbAllocated - cbNullTerminator). - Py_ssize_t cbNeed = cbUsed + cbRemaining + cbNullTerminator; - pb = ReallocOrFreeBuffer(pb, cbNeed); + SQLLEN cbNeed = cbUsed + cbRemaining + cbNullTerminator; + pb = ReallocOrFreeBuffer(pb, (Py_ssize_t)cbNeed); if (!pb) return false; cbAllocated = cbNeed; @@ -206,12 +206,12 @@ static bool ReadVarColumn(Cursor* cur, Py_ssize_t iCol, SQLSMALLINT ctype, bool& } while (ret == SQL_SUCCESS_WITH_INFO); - isNull = (ret == SQL_NULL_DATA); + *isNull = (ret == SQL_NULL_DATA); - if (!isNull && cbUsed > 0) + if (!*isNull && cbUsed > 0) { - pbResult = pb; - cbResult = cbUsed; + *pbResult = pb; + *cbResult = cbUsed; } else { @@ -238,7 +238,7 @@ static byte* ReallocOrFreeBuffer(byte* pb, Py_ssize_t cbNeed) } -static PyObject* GetText(Cursor* cur, Py_ssize_t iCol) +static PyObject* GetText(void* buffer, SQLLEN cbFetched, bool bound, PyObject* converter, TextEnc* enc) { // We are reading one of the SQL_WCHAR, SQL_WVARCHAR, etc., and will return // a string. @@ -251,113 +251,25 @@ static PyObject* GetText(Cursor* cur, Py_ssize_t iCol) // (Otherwise it is just UTF-8 with each character stored as 2 bytes.) That's why we allow // the user to configure. - ColumnInfo* pinfo = &cur->colinfos[iCol]; - const TextEnc& enc = IsWideType(pinfo->sql_type) ? cur->cnxn->sqlwchar_enc : cur->cnxn->sqlchar_enc; - - bool isNull = false; - byte* pbData = 0; - Py_ssize_t cbData = 0; - SQLLEN cbFetched = cur->cbFetchedBufs[iCol]; - void* valueBuf = cur->valueBufs[iCol]; - - if (!valueBuf) - { - if (!ReadVarColumn(cur, iCol, enc.ctype, isNull, pbData, cbData)) - return 0; - - if (isNull) - { - assert(pbData == 0 && cbData == 0); - Py_RETURN_NONE; - } - } - else - { - if (cbFetched == SQL_NULL_DATA) - { - Py_RETURN_NONE; - } - cbFetched == SQL_NTS; - cbFetched == SQL_NO_TOTAL; - pbData = (byte*)valueBuf; - cbData = cbFetched; - } - - PyObject* result = TextBufferToObject(enc, pbData, cbData); - - if (!valueBuf) - { - PyMem_Free(pbData); - } - - return result; + return TextBufferToObject(enc, (byte*)buffer, (Py_ssize_t)cbFetched); } - -static PyObject* GetBinary(Cursor* cur, Py_ssize_t iCol) +static PyObject* GetBinary(void* buffer, SQLLEN cbFetched, bool bound, PyObject* converter, TextEnc* enc) { // Reads SQL_BINARY. - bool isNull = false; - byte* pbData = 0; - Py_ssize_t cbData = 0; - SQLLEN cbFetched = cur->cbFetchedBufs[iCol]; - void* valueBuf = cur->valueBufs[iCol]; - - if (!valueBuf) - { - if (!ReadVarColumn(cur, iCol, SQL_C_BINARY, isNull, pbData, cbData)) - return 0; - - if (isNull) - { - assert(pbData == 0 && cbData == 0); - Py_RETURN_NONE; - } - } - else - { - if (cbFetched == SQL_NULL_DATA) - { - Py_RETURN_NONE; - } - pbData = (byte*)valueBuf; - cbData = cbFetched; - } - - PyObject* obj; - obj = PyBytes_FromStringAndSize((char*)pbData, cbData); - - if (!valueBuf) - { - PyMem_Free(pbData); - } - - return obj; + return PyBytes_FromStringAndSize((char*)buffer, (Py_ssize_t)cbFetched); } -static PyObject* GetDataUser(Cursor* cur, Py_ssize_t iCol, PyObject* func) +static PyObject* GetDataUser(void* buffer, SQLLEN cbFetched, bool bound, PyObject* converter, TextEnc* enc) { - bool isNull = false; - byte* pbData = 0; - Py_ssize_t cbData = 0; - if (!ReadVarColumn(cur, iCol, SQL_C_BINARY, isNull, pbData, cbData)) - return 0; - - if (isNull) - { - assert(pbData == 0 && cbData == 0); - Py_RETURN_NONE; - } - - PyObject* value = PyBytes_FromStringAndSize((char*)pbData, cbData); - PyMem_Free(pbData); + PyObject* value = PyBytes_FromStringAndSize((char*)buffer, (Py_ssize_t)cbFetched); if (!value) return 0; - PyObject* result = PyObject_CallFunction(func, "(O)", value); + PyObject* result = PyObject_CallFunction(converter, "(O)", value); Py_DECREF(value); if (!result) return 0; @@ -366,7 +278,7 @@ static PyObject* GetDataUser(Cursor* cur, Py_ssize_t iCol, PyObject* func) } -static PyObject* GetDataDecimal(Cursor* cur, Py_ssize_t iCol) +static PyObject* GetDataDecimal(void* buffer, SQLLEN cbFetched, bool bound, PyObject* converter, TextEnc* enc) { // The SQL_NUMERIC_STRUCT support is hopeless (SQL Server ignores scale on input parameters // and output columns, Oracle does something else weird, and many drivers don't support it @@ -387,229 +299,69 @@ static PyObject* GetDataDecimal(Cursor* cur, Py_ssize_t iCol) // to ignore that for right now. Therefore if we ask for the data in SQLCHAR, it should be // ASCII even if the encoding is UTF-8. - const TextEnc& enc = cur->cnxn->sqlwchar_enc; // I'm going to request the data as Unicode in case there is a weird currency symbol. If // this is a performance problems we may want a flag on this. - bool isNull = false; - byte* pbData = 0; - Py_ssize_t cbData = 0; - SQLLEN cbFetched = cur->cbFetchedBufs[iCol]; - void* valueBuf = cur->valueBufs[iCol]; - - if (!valueBuf) - { - if (!ReadVarColumn(cur, iCol, enc.ctype, isNull, pbData, cbData)) - return 0; - - if (isNull) - { - assert(pbData == 0 && cbData == 0); - Py_RETURN_NONE; - } - } - else - { - if (cbFetched == SQL_NULL_DATA) - { - Py_RETURN_NONE; - } - pbData = (byte*)valueBuf; - cbData = cbFetched; - } - - Object result(DecimalFromText(enc, pbData, cbData)); - - if (!valueBuf) - { - PyMem_Free(pbData); - } - + Object result(DecimalFromText(enc, (byte*)buffer, (Py_ssize_t)cbFetched)); return result.Detach(); } -static PyObject* GetDataBit(Cursor* cur, Py_ssize_t iCol) +static PyObject* GetDataBit(void* buffer, SQLLEN cbFetched, bool bound, PyObject* converter, TextEnc* enc) { - SQLCHAR ch; - SQLRETURN ret; - SQLLEN cbFetched = cur->cbFetchedBufs[iCol]; - void* valueBuf = cur->valueBufs[iCol]; - - if (!valueBuf) - { - Py_BEGIN_ALLOW_THREADS - ret = SQLGetData(cur->hstmt, (SQLUSMALLINT)(iCol+1), SQL_C_BIT, &ch, sizeof(ch), &cbFetched); - Py_END_ALLOW_THREADS - - if (!SQL_SUCCEEDED(ret)) - return RaiseErrorFromHandle(cur->cnxn, "SQLGetData", cur->cnxn->hdbc, cur->hstmt); - } - else - { - Py_MEMCPY(&ch, valueBuf, sizeof(ch)); - } - - if (cbFetched == SQL_NULL_DATA) - Py_RETURN_NONE; - - if (ch == SQL_TRUE) + if (*(SQLCHAR*)buffer == SQL_TRUE) Py_RETURN_TRUE; - Py_RETURN_FALSE; } -static PyObject* GetDataLong(Cursor* cur, Py_ssize_t iCol) +static PyObject* GetDataULong(void* buffer, SQLLEN cbFetched, bool bound, PyObject* converter, TextEnc* enc) { - ColumnInfo* pinfo = &cur->colinfos[iCol]; - - SQLINTEGER value; - SQLRETURN ret; - - SQLSMALLINT nCType = pinfo->is_unsigned ? SQL_C_ULONG : SQL_C_LONG; - SQLLEN cbFetched = cur->cbFetchedBufs[iCol]; - void* valueBuf = cur->valueBufs[iCol]; - - if (!valueBuf) - { - Py_BEGIN_ALLOW_THREADS - ret = SQLGetData(cur->hstmt, (SQLUSMALLINT)(iCol+1), nCType, &value, sizeof(value), &cbFetched); - Py_END_ALLOW_THREADS - - if (!SQL_SUCCEEDED(ret)) - return RaiseErrorFromHandle(cur->cnxn, "SQLGetData", cur->cnxn->hdbc, cur->hstmt); - } - else - { - Py_MEMCPY(&value, valueBuf, sizeof(value)); - } - - if (cbFetched == SQL_NULL_DATA) - Py_RETURN_NONE; + SQLINTEGER value = *(SQLINTEGER*)buffer; + return PyLong_FromLong(*(SQLINTEGER*)&value); +} - if (pinfo->is_unsigned) - return PyLong_FromLong(*(SQLINTEGER*)&value); +static PyObject* GetDataLong(void* buffer, SQLLEN cbFetched, bool bound, PyObject* converter, TextEnc* enc) +{ + SQLINTEGER value = *(SQLINTEGER*)buffer; return PyLong_FromLong(value); } -static PyObject* GetDataLongLong(Cursor* cur, Py_ssize_t iCol) +static PyObject* GetDataULongLong(void* buffer, SQLLEN cbFetched, bool bound, PyObject* converter, TextEnc* enc) { - ColumnInfo* pinfo = &cur->colinfos[iCol]; - - SQLSMALLINT nCType = pinfo->is_unsigned ? SQL_C_UBIGINT : SQL_C_SBIGINT; - SQLBIGINT value; - SQLRETURN ret; - SQLLEN cbFetched = cur->cbFetchedBufs[iCol]; - void* valueBuf = cur->valueBufs[iCol]; - - if (!valueBuf) - { - Py_BEGIN_ALLOW_THREADS - ret = SQLGetData(cur->hstmt, (SQLUSMALLINT)(iCol+1), nCType, &value, sizeof(value), &cbFetched); - Py_END_ALLOW_THREADS - - if (!SQL_SUCCEEDED(ret)) - return RaiseErrorFromHandle(cur->cnxn, "SQLGetData", cur->cnxn->hdbc, cur->hstmt); - } - else - { - Py_MEMCPY(&value, valueBuf, sizeof(value)); - } - - if (cbFetched == SQL_NULL_DATA) - Py_RETURN_NONE; + SQLBIGINT value = *(SQLBIGINT*)buffer; + return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG)(SQLUBIGINT)value); +} - if (pinfo->is_unsigned) - return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG)(SQLUBIGINT)value); +static PyObject* GetDataLongLong(void* buffer, SQLLEN cbFetched, bool bound, PyObject* converter, TextEnc* enc) +{ + SQLBIGINT value = *(SQLBIGINT*)buffer; return PyLong_FromLongLong((PY_LONG_LONG)value); } -static PyObject* GetDataDouble(Cursor* cur, Py_ssize_t iCol) +static PyObject* GetDataDouble(void* buffer, SQLLEN cbFetched, bool bound, PyObject* converter, TextEnc* enc) { - double value; - SQLRETURN ret; - SQLLEN cbFetched = cur->cbFetchedBufs[iCol]; - void* valueBuf = cur->valueBufs[iCol]; - - if (!valueBuf) - { - Py_BEGIN_ALLOW_THREADS - ret = SQLGetData(cur->hstmt, (SQLUSMALLINT)(iCol+1), SQL_C_DOUBLE, &value, sizeof(value), &cbFetched); - Py_END_ALLOW_THREADS - - if (!SQL_SUCCEEDED(ret)) - return RaiseErrorFromHandle(cur->cnxn, "SQLGetData", cur->cnxn->hdbc, cur->hstmt); - } - else - { - Py_MEMCPY(&value, valueBuf, sizeof(value)); - } - - if (cbFetched == SQL_NULL_DATA) - Py_RETURN_NONE; - + double value = *(double*)buffer; return PyFloat_FromDouble(value); } -static PyObject* GetSqlServerTime(Cursor* cur, Py_ssize_t iCol) +static PyObject* GetSqlServerTime(void* buffer, SQLLEN cbFetched, bool bound, PyObject* converter, TextEnc* enc) { - SQL_SS_TIME2_STRUCT value; - - SQLRETURN ret; - SQLLEN cbFetched = cur->cbFetchedBufs[iCol]; - void* valueBuf = cur->valueBufs[iCol]; - - if (!valueBuf) - { - Py_BEGIN_ALLOW_THREADS - ret = SQLGetData(cur->hstmt, (SQLUSMALLINT)(iCol+1), SQL_C_BINARY, &value, sizeof(value), &cbFetched); - Py_END_ALLOW_THREADS - - if (!SQL_SUCCEEDED(ret)) - return RaiseErrorFromHandle(cur->cnxn, "SQLGetData", cur->cnxn->hdbc, cur->hstmt); - } - else - { - Py_MEMCPY(&value, valueBuf, sizeof(value)); - } - - if (cbFetched == SQL_NULL_DATA) - Py_RETURN_NONE; - + SQL_SS_TIME2_STRUCT value = *(SQL_SS_TIME2_STRUCT*)buffer; int micros = (int)(value.fraction / 1000); // nanos --> micros return PyTime_FromTime(value.hour, value.minute, value.second, micros); } -static PyObject* GetUUID(Cursor* cur, Py_ssize_t iCol) + +static PyObject* GetUUID(void* buffer, SQLLEN cbFetched, bool bound, PyObject* converter, TextEnc* enc) { // REVIEW: Since GUID is a fixed size, do we need to pass the size or cbFetched? - PYSQLGUID guid; - SQLRETURN ret; - SQLLEN cbFetched = cur->cbFetchedBufs[iCol]; - void* valueBuf = cur->valueBufs[iCol]; - - if (!valueBuf) - { - Py_BEGIN_ALLOW_THREADS - ret = SQLGetData(cur->hstmt, (SQLUSMALLINT)(iCol+1), SQL_GUID, &guid, sizeof(guid), &cbFetched); - Py_END_ALLOW_THREADS - - if (!SQL_SUCCEEDED(ret)) - return RaiseErrorFromHandle(cur->cnxn, "SQLGetData", cur->cnxn->hdbc, cur->hstmt); - } - else - { - Py_MEMCPY(&guid, valueBuf, sizeof(guid)); - } - - if (cbFetched == SQL_NULL_DATA) - Py_RETURN_NONE; + PYSQLGUID guid = *(PYSQLGUID*)buffer; const char* szFmt = "(yyy#)"; Object args(Py_BuildValue(szFmt, NULL, NULL, &guid, (int)sizeof(guid))); @@ -624,57 +376,35 @@ static PyObject* GetUUID(Cursor* cur, Py_ssize_t iCol) return uuid; } -static PyObject* GetDataTimestamp(Cursor* cur, Py_ssize_t iCol) + +static PyObject* GetDataDate(void* buffer, SQLLEN cbFetched, bool bound, PyObject* converter, TextEnc* enc) { - TIMESTAMP_STRUCT value; + TIMESTAMP_STRUCT value = *(TIMESTAMP_STRUCT*)buffer; + return PyDate_FromDate(value.year, value.month, value.day); +} - SQLRETURN ret; - SQLLEN cbFetched = cur->cbFetchedBufs[iCol]; - void* valueBuf = cur->valueBufs[iCol]; - struct tm t; +static PyObject* GetDataTime(void* buffer, SQLLEN cbFetched, bool bound, PyObject* converter, TextEnc* enc) +{ + TIMESTAMP_STRUCT value = *(TIMESTAMP_STRUCT*)buffer; + int micros = (int)(value.fraction / 1000); // nanos --> micros + return PyTime_FromTime(value.hour, value.minute, value.second, micros); +} - if (!valueBuf) - { - Py_BEGIN_ALLOW_THREADS - ret = SQLGetData(cur->hstmt, (SQLUSMALLINT)(iCol+1), SQL_C_TYPE_TIMESTAMP, &value, sizeof(value), &cbFetched); - Py_END_ALLOW_THREADS - if (!SQL_SUCCEEDED(ret)) - return RaiseErrorFromHandle(cur->cnxn, "SQLGetData", cur->cnxn->hdbc, cur->hstmt); - } - else +static PyObject* GetDataTimestamp(void* buffer, SQLLEN cbFetched, bool bound, PyObject* converter, TextEnc* enc) +{ + struct tm t; + TIMESTAMP_STRUCT value = *(TIMESTAMP_STRUCT*)buffer; + + if (value.year < 1) { - Py_MEMCPY(&value, valueBuf, sizeof(value)); + value.year = 1; } - - if (cbFetched == SQL_NULL_DATA) - Py_RETURN_NONE; - - switch (cur->colinfos[iCol].sql_type) + else if (value.year > 9999) { - case SQL_TYPE_TIME: - { - int micros = (int)(value.fraction / 1000); // nanos --> micros - return PyTime_FromTime(value.hour, value.minute, value.second, micros); - } - - case SQL_TYPE_DATE: - return PyDate_FromDate(value.year, value.month, value.day); - - case SQL_TYPE_TIMESTAMP: - { - if (value.year < 1) - { - value.year = 1; - } - else if (value.year > 9999) - { - value.year = 9999; - } - } + value.year = 9999; } - int micros = (int)(value.fraction / 1000); // nanos --> micros @@ -799,85 +529,67 @@ PyObject* PythonTypeFromSqlType(Cursor* cur, SQLSMALLINT type) PyObject* GetData(Cursor* cur, Py_ssize_t iCol) { - // Returns an object representing the value in the row/field. If 0 is returned, an exception has already been set. - // - // The data is assumed to be the default C type for the column's SQL type. - ColumnInfo* pinfo = &cur->colinfos[iCol]; - // First see if there is a user-defined conversion. + void* ptr_value; + SQLLEN len; + SQLLEN* ptr_len; + bool isNull = false; + if (pinfo->is_bound || pinfo->always_alloc) { + assert(pinfo->buf_offset > 0); + ptr_value = (void*)((long)cur->fetch_buffer + pinfo->buf_offset); + ptr_len = (SQLLEN*)((long)cur->fetch_buffer + pinfo->buf_offset - sizeof(SQLLEN)); + } else { + ptr_value = 0; + ptr_len = &len; + } - if (cur->cnxn->map_sqltype_to_converter) { - PyObject* func = Connection_GetConverter(cur->cnxn, pinfo->sql_type); - if (func) { - return GetDataUser(cur, iCol, func); + if (!pinfo->is_bound && pinfo->always_alloc) { + SQLRETURN ret; + Py_BEGIN_ALLOW_THREADS + ret = SQLGetData( + cur->hstmt, + (SQLUSMALLINT)(iCol+1), + pinfo->c_type, + ptr_value, + pinfo->buf_size, + ptr_len + ); + Py_END_ALLOW_THREADS + if (!SQL_SUCCEEDED(ret)) { + return RaiseErrorFromHandle(cur->cnxn, "SQLGetData", cur->cnxn->hdbc, cur->hstmt); } - if (PyErr_Occurred()) - return 0; } - - switch (pinfo->sql_type) - { - case SQL_WCHAR: - case SQL_WVARCHAR: - case SQL_WLONGVARCHAR: - return GetText(cur, iCol); - - case SQL_CHAR: - case SQL_VARCHAR: - case SQL_LONGVARCHAR: - case SQL_SS_XML: - case SQL_DB2_XML: - return GetText(cur, iCol); - - case SQL_GUID: - if (UseNativeUUID()) - return GetUUID(cur, iCol); - return GetText(cur, iCol); - break; - - case SQL_BINARY: - case SQL_VARBINARY: - case SQL_LONGVARBINARY: - return GetBinary(cur, iCol); - - case SQL_DECIMAL: - case SQL_NUMERIC: - case SQL_DB2_DECFLOAT: - return GetDataDecimal(cur, iCol); - - case SQL_BIT: - return GetDataBit(cur, iCol); - - case SQL_TINYINT: - case SQL_SMALLINT: - case SQL_INTEGER: - return GetDataLong(cur, iCol); - - case SQL_BIGINT: - return GetDataLongLong(cur, iCol); - - case SQL_REAL: - case SQL_FLOAT: - case SQL_DOUBLE: - return GetDataDouble(cur, iCol); - - - case SQL_TYPE_DATE: - case SQL_TYPE_TIME: - case SQL_TYPE_TIMESTAMP: - return GetDataTimestamp(cur, iCol); - - case SQL_SS_TIME2: - return GetSqlServerTime(cur, iCol); + if (!pinfo->is_bound && !pinfo->always_alloc) { + if (!ReadVarColumn(cur, iCol, pinfo->c_type, &isNull, &ptr_value, ptr_len)) { + return 0; + } + assert(!ptr_value == isNull); + } else { + isNull = *ptr_len == SQL_NULL_DATA; + } + + PyObject* value; + if (isNull) { + value = Py_None; + } else { + value = (*pinfo->GetData)( + ptr_value, + *ptr_len, + pinfo->is_bound, + pinfo->converter, + pinfo->enc + ); + if (!pinfo->is_bound && !pinfo->always_alloc) { + PyMem_Free(ptr_value); + } } - return RaiseErrorV("HY106", ProgrammingError, "ODBC SQL type %d is not yet supported. column-index=%zd type=%d", - (int)pinfo->sql_type, iCol, (int)pinfo->sql_type); + return value; } -inline SQLLEN CharBufferSize(SQLULEN nr_chars) +inline SQLULEN CharBufferSize(SQLULEN nr_chars) { // This is probably overly pessimistic. It is assumed that // - column_size is the number of characters, not bytes @@ -889,8 +601,9 @@ inline SQLLEN CharBufferSize(SQLULEN nr_chars) } -bool BindCol(Cursor* cur, Py_ssize_t iCol) +bool FetchBufferInfo(Cursor* cur, Py_ssize_t iCol) { + // 0 means error, 1 means can be bound, -1 means cannot be bound // If false is returned, an exception has already been set. // // The data is assumed to be the default C type for the column's SQL type. @@ -898,21 +611,27 @@ bool BindCol(Cursor* cur, Py_ssize_t iCol) // Must be analogous to GetData. ColumnInfo* pinfo = &cur->colinfos[iCol]; - SQLRETURN ret; - SQLLEN size; - SQLSMALLINT c_type; // We don't implement SQLBindCol for user-defined conversions. - if (cur->cnxn->map_sqltype_to_converter) - { - if (Connection_GetConverter(cur->cnxn, pinfo->sql_type)) { - return true; + // First see if there is a user-defined conversion. + + if (cur->cnxn->map_sqltype_to_converter) { + PyObject* converter = Connection_GetConverter(cur->cnxn, pinfo->sql_type); + if (converter) { + pinfo->converter = converter; + pinfo->GetData = GetDataUser; + pinfo->can_bind = false; + pinfo->always_alloc = false; + pinfo->c_type = SQL_C_BINARY; + pinfo->buf_size = 0; + return PyErr_Occurred() ? false : true; } - if (PyErr_Occurred()) - return false; } + pinfo->converter = 0; + pinfo->can_bind = true; + pinfo->always_alloc = true; switch (pinfo->sql_type) { case SQL_WCHAR: @@ -924,105 +643,127 @@ bool BindCol(Cursor* cur, Py_ssize_t iCol) case SQL_LONGVARCHAR: case SQL_SS_XML: case SQL_DB2_XML: - if (pinfo->column_size <= 0 || pinfo->column_size > 1024*8) - return true; - c_type = IsWideType(pinfo->sql_type) ? cur->cnxn->sqlwchar_enc.ctype : cur->cnxn->sqlchar_enc.ctype; - size = CharBufferSize(pinfo->column_size); + pinfo->enc = &(IsWideType(pinfo->sql_type) ? cur->cnxn->sqlwchar_enc : cur->cnxn->sqlchar_enc); + pinfo->c_type = pinfo->enc->ctype; + pinfo->GetData = GetText; + if (pinfo->column_size <= 0) { + pinfo->buf_size = 0; + pinfo->can_bind = false; + } else { + pinfo->buf_size = CharBufferSize(pinfo->column_size); + } + pinfo->always_alloc = false; break; case SQL_GUID: - if (UseNativeUUID()) - { + if (UseNativeUUID()) { // Binding here does not work on 64bit Windows, so we don't. // Not sure why, it works everywhere else. - // c_type = SQL_GUID; - // size = sizeof(PYSQLGUID); - return true; - } - else - { - c_type = cur->cnxn->sqlchar_enc.ctype; + pinfo->c_type = SQL_GUID; + pinfo->buf_size = sizeof(PYSQLGUID); + pinfo->GetData = GetUUID; + pinfo->can_bind = false; + } else { + pinfo->enc = &cur->cnxn->sqlchar_enc; + pinfo->c_type = pinfo->enc->ctype; // leave space for dashes every 4 characters - size = CharBufferSize(32+7); + pinfo->buf_size = CharBufferSize(32+7); + pinfo->GetData = GetText; } break; case SQL_BINARY: case SQL_VARBINARY: case SQL_LONGVARBINARY: - if (pinfo->column_size == 0 || pinfo->column_size > 1024*64) - return true; - c_type = SQL_C_BINARY; - size = pinfo->column_size + 1; + pinfo->c_type = SQL_C_BINARY; + if (pinfo->column_size == 0) { + pinfo->buf_size = 0; + pinfo->can_bind = false; + } else { + pinfo->buf_size = pinfo->column_size + 1; + } + pinfo->GetData = GetBinary; + pinfo->always_alloc = false; break; case SQL_DECIMAL: case SQL_NUMERIC: case SQL_DB2_DECFLOAT: - c_type = cur->cnxn->sqlwchar_enc.ctype; + pinfo->enc = &cur->cnxn->sqlwchar_enc; + pinfo->c_type = pinfo->enc->ctype; // need to add padding for all kinds of situations, see GetDataDecimal - size = CharBufferSize(pinfo->column_size + 5); + pinfo->buf_size = CharBufferSize(pinfo->column_size + 5); + pinfo->GetData = GetDataDecimal; break; case SQL_BIT: - c_type = SQL_C_BIT; - size = sizeof(SQLCHAR); + pinfo->c_type = SQL_C_BIT; + pinfo->buf_size = sizeof(SQLCHAR); + pinfo->GetData = GetDataBit; break; case SQL_TINYINT: case SQL_SMALLINT: case SQL_INTEGER: - c_type = pinfo->is_unsigned ? SQL_C_ULONG : SQL_C_LONG; - size = sizeof(SQLINTEGER); + if (pinfo->is_unsigned) { + pinfo->c_type = SQL_C_ULONG; + pinfo->buf_size = sizeof(SQLINTEGER); + pinfo->GetData = GetDataULong; + } else { + pinfo->c_type = SQL_C_LONG; + pinfo->GetData = GetDataLong; + pinfo->buf_size = sizeof(SQLINTEGER); + } break; case SQL_BIGINT: - c_type = pinfo->is_unsigned ? SQL_C_UBIGINT : SQL_C_SBIGINT; - size = sizeof(SQLBIGINT); + if (pinfo->is_unsigned) { + pinfo->c_type = SQL_C_UBIGINT; + pinfo->buf_size = sizeof(SQLBIGINT); + pinfo->GetData = GetDataULongLong; + } else { + pinfo->c_type = SQL_C_SBIGINT; + pinfo->buf_size = sizeof(SQLBIGINT); + pinfo->GetData = GetDataLongLong; + } break; case SQL_REAL: case SQL_FLOAT: case SQL_DOUBLE: - c_type = SQL_C_DOUBLE; - size = sizeof(double); + pinfo->c_type = SQL_C_DOUBLE; + pinfo->buf_size = sizeof(double); + pinfo->GetData = GetDataDouble; break; case SQL_TYPE_DATE: + pinfo->c_type = SQL_C_TYPE_TIMESTAMP; + pinfo->buf_size = sizeof(TIMESTAMP_STRUCT); + pinfo->GetData = GetDataDate; + break; + case SQL_TYPE_TIME: + pinfo->c_type = SQL_C_TYPE_TIMESTAMP; + pinfo->buf_size = sizeof(TIMESTAMP_STRUCT); + pinfo->GetData = GetDataTime; + break; + case SQL_TYPE_TIMESTAMP: - c_type = SQL_C_TYPE_TIMESTAMP; - size = sizeof(TIMESTAMP_STRUCT); + pinfo->c_type = SQL_C_TYPE_TIMESTAMP; + pinfo->buf_size = sizeof(TIMESTAMP_STRUCT); + pinfo->GetData = GetDataTimestamp; break; case SQL_SS_TIME2: - c_type = SQL_C_BINARY; - size = sizeof(SQL_SS_TIME2_STRUCT); + pinfo->c_type = SQL_C_BINARY; + pinfo->buf_size = sizeof(SQL_SS_TIME2_STRUCT); + pinfo->GetData = GetSqlServerTime; break; default: - return true; - } - - cur->valueBufs[iCol] = PyMem_Malloc(size); - if (!cur->valueBufs[iCol]) - { - PyErr_NoMemory(); + RaiseErrorV("HY106", ProgrammingError, "ODBC SQL type %d is not yet supported. column-index=%zd type=%d", + (int)pinfo->sql_type, iCol, (int)pinfo->sql_type); return false; } - - Py_BEGIN_ALLOW_THREADS - ret = SQLBindCol( - cur->hstmt, - (SQLUSMALLINT)(iCol+1), - c_type, - cur->valueBufs[iCol], - size, - &(cur->cbFetchedBufs[iCol]) - ); - Py_END_ALLOW_THREADS - if (!SQL_SUCCEEDED(ret)) - return RaiseErrorFromHandle(cur->cnxn, "SQLGetData", cur->cnxn->hdbc, cur->hstmt); - return true; } diff --git a/src/getdata.h b/src/getdata.h index c037787e..b40fb6a4 100644 --- a/src/getdata.h +++ b/src/getdata.h @@ -7,12 +7,12 @@ void GetData_init(); PyObject* PythonTypeFromSqlType(Cursor* cur, SQLSMALLINT type); PyObject* GetData(Cursor* cur, Py_ssize_t iCol); -bool BindCol(Cursor* cur, Py_ssize_t iCol); +bool FetchBufferInfo(Cursor* cur, Py_ssize_t iCol); /** * If this sql type has a user-defined conversion, the index into the connection's `conv_funcs` array is returned. * Otherwise -1 is returned. */ -int GetUserConvIndex(Cursor* cur, SQLSMALLINT sql_type); +// int GetUserConvIndex(Cursor* cur, SQLSMALLINT sql_type); #endif // _GETDATA_H_ diff --git a/src/textenc.cpp b/src/textenc.cpp index 725265f0..fed2eed7 100644 --- a/src/textenc.cpp +++ b/src/textenc.cpp @@ -89,7 +89,7 @@ PyObject* TextEnc::Encode(PyObject* obj) const -PyObject* TextBufferToObject(const TextEnc& enc, const byte* pbData, Py_ssize_t cbData) +PyObject* TextBufferToObject(const TextEnc* enc, const byte* pbData, Py_ssize_t cbData) { // cbData // The length of data in bytes (cb == 'count of bytes'). @@ -104,7 +104,7 @@ PyObject* TextBufferToObject(const TextEnc& enc, const byte* pbData, Py_ssize_t if (cbData == 0) return PyUnicode_FromStringAndSize("", 0); - switch (enc.optenc) + switch (enc->optenc) { case OPTENC_UTF8: return PyUnicode_DecodeUTF8((char*)pbData, cbData, "strict"); @@ -129,5 +129,5 @@ PyObject* TextBufferToObject(const TextEnc& enc, const byte* pbData, Py_ssize_t } // The user set an encoding by name. - return PyUnicode_Decode((char*)pbData, cbData, enc.name, "strict"); + return PyUnicode_Decode((char*)pbData, cbData, enc->name, "strict"); } diff --git a/src/textenc.h b/src/textenc.h index 19214023..4b0f7add 100644 --- a/src/textenc.h +++ b/src/textenc.h @@ -135,7 +135,7 @@ class SQLWChar }; -PyObject* TextBufferToObject(const TextEnc& enc, const byte* p, Py_ssize_t len); +PyObject* TextBufferToObject(const TextEnc* enc, const byte* p, Py_ssize_t len); // Convert a text buffer to a Python object using the given encoding. // // - pbData :: The buffer, which is an array of SQLCHAR or SQLWCHAR. We treat it as bytes here From a1540a0f17559d76ed477b0d299a27fbe2da3e67 Mon Sep 17 00:00:00 2001 From: ffelixg <142172984+ffelixg@users.noreply.github.com> Date: Fri, 1 Mar 2024 19:34:22 +0100 Subject: [PATCH 14/21] Add support for fetching multiple rows --- src/cursor.cpp | 72 ++++++++++++++++++++++++++++++++++++------------- src/cursor.h | 6 +++++ src/getdata.cpp | 10 ++++--- src/getdata.h | 2 +- 4 files changed, 67 insertions(+), 23 deletions(-) diff --git a/src/cursor.cpp b/src/cursor.cpp index ff45fc85..d7db6c7d 100644 --- a/src/cursor.cpp +++ b/src/cursor.cpp @@ -337,6 +337,7 @@ static bool BindCols(Cursor* cur, int cCols) long total_buf_size = 0; ColumnInfo* pinfo; int cap_alloc = cCols; + bool bind_all = true; for (iCol = 0; iCol < cCols; iCol++) { if (!FetchBufferInfo(cur, iCol)) { @@ -354,22 +355,44 @@ static bool BindCols(Cursor* cur, int cCols) total_buf_size += pinfo->buf_size + sizeof(SQLLEN); } else { pinfo->buf_offset = -1; + bind_all = false; } } - void* buf = PyMem_Malloc(total_buf_size); - cur->fetch_buffer = buf; + cur->fetch_buffer_width = total_buf_size; + if (bind_all) { + cur->fetch_buffer_length = buf_cap / cur->fetch_buffer_width; + if (cur->fetch_buffer_length > cur->arraysize) { // cur->arraysize can be negative + cur->fetch_buffer_length = cur->arraysize; + } + if (cur->fetch_buffer_length < 1) { + cur->fetch_buffer_length = 1; + } + } else { + cur->fetch_buffer_length = 1; + } + + void* buf = PyMem_Malloc((cur->fetch_buffer_width + sizeof(SQLUSMALLINT)) * cur->fetch_buffer_length); if (!buf) { PyErr_NoMemory(); return false; } + cur->fetch_buffer = buf; + cur->row_status_array = (SQLUSMALLINT*)((uintptr_t)buf + cur->fetch_buffer_width * cur->fetch_buffer_length); + cur->current_row = 0; - SQLRETURN ret = SQL_SUCCESS; + SQLRETURN ret = SQL_SUCCESS, ret1, ret2, ret3, ret4; Py_BEGIN_ALLOW_THREADS bool keep_binding = true; SQLFreeStmt(cur->hstmt, SQL_UNBIND); // somehow columns can still be bound here + // First two will be read as SQLULEN by driver + ret1 = SQLSetStmtAttr(cur->hstmt, SQL_ATTR_ROW_BIND_TYPE, (SQLPOINTER)cur->fetch_buffer_width, 0); + ret2 = SQLSetStmtAttr(cur->hstmt, SQL_ATTR_ROW_ARRAY_SIZE, (SQLPOINTER)cur->fetch_buffer_length, 0); + ret3 = SQLSetStmtAttr(cur->hstmt, SQL_ATTR_ROW_STATUS_PTR, cur->row_status_array, 0); + ret4 = SQLSetStmtAttr(cur->hstmt, SQL_ATTR_ROWS_FETCHED_PTR, &cur->rows_fetched, 0); + for (iCol = 0; iCol < cCols; iCol++) { pinfo = &cur->colinfos[iCol]; if (pinfo->can_bind && pinfo->buf_offset >= 0 && keep_binding) { @@ -378,9 +401,9 @@ static bool BindCols(Cursor* cur, int cCols) cur->hstmt, (SQLUSMALLINT)(iCol+1), pinfo->c_type, - (void*)((long)buf + pinfo->buf_offset), + (void*)((uintptr_t)buf + pinfo->buf_offset), pinfo->buf_size, - (SQLLEN*)((long)buf + pinfo->buf_offset - sizeof(SQLLEN)) + (SQLLEN*)((uintptr_t)buf + pinfo->buf_offset - sizeof(SQLLEN)) ); if (!SQL_SUCCEEDED(ret)) { break; @@ -397,6 +420,11 @@ static bool BindCols(Cursor* cur, int cCols) return RaiseErrorFromHandle(cur->cnxn, "SQLBindCol", cur->cnxn->hdbc, cur->hstmt); } + if (!SQL_SUCCEEDED(ret1) || !SQL_SUCCEEDED(ret2) || !SQL_SUCCEEDED(ret3) || !SQL_SUCCEEDED(ret4)) { + PyMem_Free(buf); + return RaiseErrorFromHandle(cur->cnxn, "SQLSetStmtAttr", cur->cnxn->hdbc, cur->hstmt); + } + return true; } @@ -1340,21 +1368,28 @@ static PyObject* Cursor_fetch(Cursor* cur) Py_ssize_t field_count, i; PyObject** apValues; - Py_BEGIN_ALLOW_THREADS - ret = SQLFetch(cur->hstmt); - Py_END_ALLOW_THREADS + // One fetch per cycle. + if (cur->current_row == 0) { + Py_BEGIN_ALLOW_THREADS + ret = SQLFetch(cur->hstmt); + Py_END_ALLOW_THREADS - if (cur->cnxn->hdbc == SQL_NULL_HANDLE) - { - // The connection was closed by another thread in the ALLOW_THREADS block above. - return RaiseErrorV(0, ProgrammingError, "The cursor's connection was closed."); - } + if (cur->cnxn->hdbc == SQL_NULL_HANDLE) + { + // The connection was closed by another thread in the ALLOW_THREADS block above. + return RaiseErrorV(0, ProgrammingError, "The cursor's connection was closed."); + } - if (ret == SQL_NO_DATA) - return 0; + if (ret == SQL_NO_DATA) + return 0; - if (!SQL_SUCCEEDED(ret)) - return RaiseErrorFromHandle(cur->cnxn, "SQLFetch", cur->cnxn->hdbc, cur->hstmt); + if (!SQL_SUCCEEDED(ret)) + return RaiseErrorFromHandle(cur->cnxn, "SQLFetch", cur->cnxn->hdbc, cur->hstmt); + } else { + if (cur->current_row >= cur->rows_fetched) { + return 0; + } + } field_count = PyTuple_GET_SIZE(cur->description); @@ -1365,7 +1400,7 @@ static PyObject* Cursor_fetch(Cursor* cur) for (i = 0; i < field_count; i++) { - PyObject* value = GetData(cur, i); + PyObject* value = GetData(cur, i, cur->current_row); if (!value) { @@ -1375,6 +1410,7 @@ static PyObject* Cursor_fetch(Cursor* cur) apValues[i] = value; } + cur->current_row = (cur->current_row + 1) % cur->fetch_buffer_length; return (PyObject*)Row_InternalNew(cur->description, cur->map_name_to_index, field_count, apValues); } diff --git a/src/cursor.h b/src/cursor.h index 992f177e..2d56a484 100644 --- a/src/cursor.h +++ b/src/cursor.h @@ -146,6 +146,7 @@ struct Cursor // The description tuple described in the DB API 2.0 specification. Set to None when there are no results. PyObject* description; + // Also serves as a cap on the number of rows allocated for the buffer. int arraysize; // The Cursor.rowcount attribute from the DB API specification. @@ -168,6 +169,11 @@ struct Cursor // Pointer to buffer used by SQLBindCol and sometimes SQLGetData. void* fetch_buffer; + SQLULEN fetch_buffer_width; + SQLULEN fetch_buffer_length; + SQLULEN rows_fetched; + SQLUSMALLINT* row_status_array; + SQLULEN current_row; // Track the configuration at the time of using SQLBindCol. bool bound_native_uuid; diff --git a/src/getdata.cpp b/src/getdata.cpp index 5af2a770..4702a3eb 100644 --- a/src/getdata.cpp +++ b/src/getdata.cpp @@ -527,24 +527,25 @@ PyObject* PythonTypeFromSqlType(Cursor* cur, SQLSMALLINT type) return pytype; } -PyObject* GetData(Cursor* cur, Py_ssize_t iCol) +PyObject* GetData(Cursor* cur, Py_ssize_t iCol, Py_ssize_t iRow) { ColumnInfo* pinfo = &cur->colinfos[iCol]; - void* ptr_value; SQLLEN len; SQLLEN* ptr_len; bool isNull = false; + if (pinfo->is_bound || pinfo->always_alloc) { assert(pinfo->buf_offset > 0); - ptr_value = (void*)((long)cur->fetch_buffer + pinfo->buf_offset); - ptr_len = (SQLLEN*)((long)cur->fetch_buffer + pinfo->buf_offset - sizeof(SQLLEN)); + ptr_value = (void*)((uintptr_t)cur->fetch_buffer + pinfo->buf_offset + iRow * cur->fetch_buffer_width); + ptr_len = (SQLLEN*)((uintptr_t)cur->fetch_buffer + pinfo->buf_offset + iRow * cur->fetch_buffer_width - sizeof(SQLLEN)); } else { ptr_value = 0; ptr_len = &len; } if (!pinfo->is_bound && pinfo->always_alloc) { + assert(iRow == 1); SQLRETURN ret; Py_BEGIN_ALLOW_THREADS ret = SQLGetData( @@ -561,6 +562,7 @@ PyObject* GetData(Cursor* cur, Py_ssize_t iCol) } } if (!pinfo->is_bound && !pinfo->always_alloc) { + assert(iRow == 1); if (!ReadVarColumn(cur, iCol, pinfo->c_type, &isNull, &ptr_value, ptr_len)) { return 0; } diff --git a/src/getdata.h b/src/getdata.h index b40fb6a4..d587e342 100644 --- a/src/getdata.h +++ b/src/getdata.h @@ -6,7 +6,7 @@ void GetData_init(); PyObject* PythonTypeFromSqlType(Cursor* cur, SQLSMALLINT type); -PyObject* GetData(Cursor* cur, Py_ssize_t iCol); +PyObject* GetData(Cursor* cur, Py_ssize_t iCol, Py_ssize_t iRow); bool FetchBufferInfo(Cursor* cur, Py_ssize_t iCol); /** From 0a5bb9722261e8dd96404794bb5c24806f0972f7 Mon Sep 17 00:00:00 2001 From: ffelixg <142172984+ffelixg@users.noreply.github.com> Date: Sat, 2 Mar 2024 19:27:12 +0100 Subject: [PATCH 15/21] Rewrite GetUUID & bind --- src/cursor.cpp | 3 +++ src/getdata.cpp | 28 +++++++++++++++------------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/cursor.cpp b/src/cursor.cpp index d7db6c7d..b56475a8 100644 --- a/src/cursor.cpp +++ b/src/cursor.cpp @@ -353,6 +353,9 @@ static bool BindCols(Cursor* cur, int cCols) if (pinfo->buf_size && (iCol < cap_alloc || pinfo->always_alloc)) { pinfo->buf_offset = total_buf_size + sizeof(SQLLEN); total_buf_size += pinfo->buf_size + sizeof(SQLLEN); + if (!pinfo->can_bind) { + bind_all = false; + } } else { pinfo->buf_offset = -1; bind_all = false; diff --git a/src/getdata.cpp b/src/getdata.cpp index 4702a3eb..aec4a396 100644 --- a/src/getdata.cpp +++ b/src/getdata.cpp @@ -361,17 +361,23 @@ static PyObject* GetUUID(void* buffer, SQLLEN cbFetched, bool bound, PyObject* c { // REVIEW: Since GUID is a fixed size, do we need to pass the size or cbFetched? - PYSQLGUID guid = *(PYSQLGUID*)buffer; + PyObject* guid_bytes = PyBytes_FromStringAndSize((char*)buffer, (Py_ssize_t)sizeof(PYSQLGUID)); + PyObject* uuid_args = PyTuple_New(3); + PyObject* uuid_type = GetClassForThread("uuid", "UUID"); - const char* szFmt = "(yyy#)"; - Object args(Py_BuildValue(szFmt, NULL, NULL, &guid, (int)sizeof(guid))); - if (!args) + if(!guid_bytes || !uuid_args || !uuid_type) { + Py_XDECREF(guid_bytes); + Py_XDECREF(uuid_args); + Py_XDECREF(uuid_type); return 0; + } - PyObject* uuid_type = GetClassForThread("uuid", "UUID"); - if (!uuid_type) - return 0; - PyObject* uuid = PyObject_CallObject(uuid_type, args.Get()); + PyTuple_SET_ITEM(uuid_args, 0, Py_None); + PyTuple_SET_ITEM(uuid_args, 1, Py_None); + PyTuple_SET_ITEM(uuid_args, 2, guid_bytes); + + PyObject* uuid = PyObject_CallObject(uuid_type, uuid_args); + Py_DECREF(uuid_args); Py_DECREF(uuid_type); return uuid; } @@ -659,13 +665,9 @@ bool FetchBufferInfo(Cursor* cur, Py_ssize_t iCol) case SQL_GUID: if (UseNativeUUID()) { - // Binding here does not work on 64bit Windows, so we don't. - // Not sure why, it works everywhere else. - - pinfo->c_type = SQL_GUID; + pinfo->c_type = SQL_C_GUID; pinfo->buf_size = sizeof(PYSQLGUID); pinfo->GetData = GetUUID; - pinfo->can_bind = false; } else { pinfo->enc = &cur->cnxn->sqlchar_enc; pinfo->c_type = pinfo->enc->ctype; From 78235507d2389c61d29071b768db6c0f4c74a84b Mon Sep 17 00:00:00 2001 From: ffelixg <142172984+ffelixg@users.noreply.github.com> Date: Sun, 3 Mar 2024 23:21:47 +0100 Subject: [PATCH 16/21] Only (re)bind before fetch, cleanup --- src/cursor.cpp | 112 +++++++++---------- src/cursor.h | 2 +- src/getdata.cpp | 280 ++++++++++++++++++++++-------------------------- 3 files changed, 185 insertions(+), 209 deletions(-) diff --git a/src/cursor.cpp b/src/cursor.cpp index b56475a8..88c7915b 100644 --- a/src/cursor.cpp +++ b/src/cursor.cpp @@ -335,7 +335,7 @@ static bool BindCols(Cursor* cur, int cCols) int iCol; long buf_cap = 1024*1024; long total_buf_size = 0; - ColumnInfo* pinfo; + ColumnInfo* cInfo; int cap_alloc = cCols; bool bind_all = true; @@ -344,20 +344,20 @@ static bool BindCols(Cursor* cur, int cCols) return false; } - pinfo = &cur->colinfos[iCol]; + cInfo = &cur->colinfos[iCol]; - if (total_buf_size + pinfo->buf_size + sizeof(SQLULEN) > buf_cap && iCol < cap_alloc) { + if (total_buf_size + cInfo->buf_size + sizeof(SQLULEN) > buf_cap && iCol < cap_alloc) { cap_alloc = iCol; } - if (pinfo->buf_size && (iCol < cap_alloc || pinfo->always_alloc)) { - pinfo->buf_offset = total_buf_size + sizeof(SQLLEN); - total_buf_size += pinfo->buf_size + sizeof(SQLLEN); - if (!pinfo->can_bind) { + if (cInfo->buf_size && (iCol < cap_alloc || cInfo->always_alloc)) { + cInfo->buf_offset = total_buf_size + sizeof(SQLLEN); + total_buf_size += cInfo->buf_size + sizeof(SQLLEN); + if (!cInfo->can_bind) { bind_all = false; } } else { - pinfo->buf_offset = -1; + cInfo->buf_offset = -1; bind_all = false; } } @@ -380,59 +380,74 @@ static bool BindCols(Cursor* cur, int cCols) PyErr_NoMemory(); return false; } + assert(cur->fetch_buffer == 0); cur->fetch_buffer = buf; cur->row_status_array = (SQLUSMALLINT*)((uintptr_t)buf + cur->fetch_buffer_width * cur->fetch_buffer_length); cur->current_row = 0; - SQLRETURN ret = SQL_SUCCESS, ret1, ret2, ret3, ret4; + SQLRETURN ret_bind = SQL_SUCCESS, ret_attr; Py_BEGIN_ALLOW_THREADS bool keep_binding = true; SQLFreeStmt(cur->hstmt, SQL_UNBIND); // somehow columns can still be bound here // First two will be read as SQLULEN by driver - ret1 = SQLSetStmtAttr(cur->hstmt, SQL_ATTR_ROW_BIND_TYPE, (SQLPOINTER)cur->fetch_buffer_width, 0); - ret2 = SQLSetStmtAttr(cur->hstmt, SQL_ATTR_ROW_ARRAY_SIZE, (SQLPOINTER)cur->fetch_buffer_length, 0); - ret3 = SQLSetStmtAttr(cur->hstmt, SQL_ATTR_ROW_STATUS_PTR, cur->row_status_array, 0); - ret4 = SQLSetStmtAttr(cur->hstmt, SQL_ATTR_ROWS_FETCHED_PTR, &cur->rows_fetched, 0); + ret_attr = SQLSetStmtAttr(cur->hstmt, SQL_ATTR_ROW_BIND_TYPE, (SQLPOINTER)cur->fetch_buffer_width, 0); + if (!SQL_SUCCEEDED(ret_attr)) { + goto skip; + } + ret_attr = SQLSetStmtAttr(cur->hstmt, SQL_ATTR_ROW_ARRAY_SIZE, (SQLPOINTER)cur->fetch_buffer_length, 0); + if (!SQL_SUCCEEDED(ret_attr)) { + goto skip; + } + // TODO use for error checking + ret_attr = SQLSetStmtAttr(cur->hstmt, SQL_ATTR_ROW_STATUS_PTR, cur->row_status_array, 0); + if (!SQL_SUCCEEDED(ret_attr)) { + goto skip; + } + ret_attr = SQLSetStmtAttr(cur->hstmt, SQL_ATTR_ROWS_FETCHED_PTR, &cur->rows_fetched, 0); + if (!SQL_SUCCEEDED(ret_attr)) { + goto skip; + } for (iCol = 0; iCol < cCols; iCol++) { - pinfo = &cur->colinfos[iCol]; - if (pinfo->can_bind && pinfo->buf_offset >= 0 && keep_binding) { - pinfo->is_bound = true; - ret = SQLBindCol( + cInfo = &cur->colinfos[iCol]; + if (cInfo->can_bind && cInfo->buf_offset >= 0 && keep_binding) { + cInfo->is_bound = true; + ret_bind = SQLBindCol( cur->hstmt, (SQLUSMALLINT)(iCol+1), - pinfo->c_type, - (void*)((uintptr_t)buf + pinfo->buf_offset), - pinfo->buf_size, - (SQLLEN*)((uintptr_t)buf + pinfo->buf_offset - sizeof(SQLLEN)) + cInfo->c_type, + (void*)((uintptr_t)buf + cInfo->buf_offset), + cInfo->buf_size, + (SQLLEN*)((uintptr_t)buf + cInfo->buf_offset - sizeof(SQLLEN)) ); - if (!SQL_SUCCEEDED(ret)) { + if (!SQL_SUCCEEDED(ret_bind)) { break; } } else { keep_binding = false; - pinfo->is_bound = false; + cInfo->is_bound = false; } } + skip: Py_END_ALLOW_THREADS - if (!SQL_SUCCEEDED(ret)) { + if (!SQL_SUCCEEDED(ret_attr)) { PyMem_Free(buf); - return RaiseErrorFromHandle(cur->cnxn, "SQLBindCol", cur->cnxn->hdbc, cur->hstmt); + return RaiseErrorFromHandle(cur->cnxn, "SQLSetStmtAttr", cur->cnxn->hdbc, cur->hstmt); } - if (!SQL_SUCCEEDED(ret1) || !SQL_SUCCEEDED(ret2) || !SQL_SUCCEEDED(ret3) || !SQL_SUCCEEDED(ret4)) { + if (!SQL_SUCCEEDED(ret_bind)) { PyMem_Free(buf); - return RaiseErrorFromHandle(cur->cnxn, "SQLSetStmtAttr", cur->cnxn->hdbc, cur->hstmt); + return RaiseErrorFromHandle(cur->cnxn, "SQLBindCol", cur->cnxn->hdbc, cur->hstmt); } return true; } -static bool DetectConfigChange(Cursor* cur) +static bool PrepareFetch(Cursor* cur) { // Returns false on exception, true otherwise. // Need to do this because the API allows changing this after executing a statement. @@ -443,7 +458,7 @@ static bool DetectConfigChange(Cursor* cur) if (cur->cnxn->map_sqltype_to_converter) { - converted_types = PyDict_Keys(cur->cnxn->map_sqltype_to_converter); + converted_types = PyDict_Copy(cur->cnxn->map_sqltype_to_converter); if (!converted_types) { return false; @@ -455,10 +470,10 @@ static bool DetectConfigChange(Cursor* cur) case -1: // error Py_DECREF(converted_types); return false; - case 0: // keys not equal + case 0: // not equal converted_types_changed = true; break; - case 1: // keys equal + case 1: // equal Py_DECREF(converted_types); break; } @@ -473,7 +488,7 @@ static bool DetectConfigChange(Cursor* cur) converted_types_changed = true; } - if (cur->bound_native_uuid != native_uuid || converted_types_changed) + if (cur->bound_native_uuid != native_uuid || converted_types_changed || !cur->fetch_buffer) { Py_XDECREF(cur->bound_converted_types); cur->bound_converted_types = converted_types; @@ -733,6 +748,7 @@ static bool PrepareResults(Cursor* cur, int cCols) int i; assert(cur->colinfos == 0); + assert(cur->fetch_buffer == 0); cur->colinfos = (ColumnInfo*)PyMem_Malloc(sizeof(ColumnInfo) * cCols); if (cur->colinfos == 0) @@ -751,28 +767,6 @@ static bool PrepareResults(Cursor* cur, int cCols) } } - cur->bound_native_uuid = UseNativeUUID(); - if (cur->cnxn->map_sqltype_to_converter) - { - cur->bound_converted_types = PyDict_Keys(cur->cnxn->map_sqltype_to_converter); - if (!cur->bound_converted_types) - { - PyMem_Free(cur->colinfos); - return false; - } - } - else - { - cur->bound_converted_types = 0; - } - - if (!BindCols(cur, cCols)) - { - PyMem_Free(cur->colinfos); - cur->colinfos = 0; - return false; - } - return true; } @@ -1474,7 +1468,7 @@ static PyObject* Cursor_iternext(PyObject* self) Cursor* cursor = Cursor_Validate(self, CURSOR_REQUIRE_RESULTS | CURSOR_RAISE_ERROR); - if (!cursor || !DetectConfigChange(cursor)) + if (!cursor || !PrepareFetch(cursor)) return 0; result = Cursor_fetch(cursor); @@ -1487,7 +1481,7 @@ static PyObject* Cursor_fetchval(PyObject* self, PyObject* args) UNUSED(args); Cursor* cursor = Cursor_Validate(self, CURSOR_REQUIRE_RESULTS | CURSOR_RAISE_ERROR); - if (!cursor || !DetectConfigChange(cursor)) + if (!cursor || !PrepareFetch(cursor)) return 0; Object row(Cursor_fetch(cursor)); @@ -1508,7 +1502,7 @@ static PyObject* Cursor_fetchone(PyObject* self, PyObject* args) PyObject* row; Cursor* cursor = Cursor_Validate(self, CURSOR_REQUIRE_RESULTS | CURSOR_RAISE_ERROR); - if (!cursor || !DetectConfigChange(cursor)) + if (!cursor || !PrepareFetch(cursor)) return 0; row = Cursor_fetch(cursor); @@ -1530,7 +1524,7 @@ static PyObject* Cursor_fetchall(PyObject* self, PyObject* args) PyObject* result; Cursor* cursor = Cursor_Validate(self, CURSOR_REQUIRE_RESULTS | CURSOR_RAISE_ERROR); - if (!cursor || !DetectConfigChange(cursor)) + if (!cursor || !PrepareFetch(cursor)) return 0; result = Cursor_fetchlist(cursor, -1); @@ -1545,7 +1539,7 @@ static PyObject* Cursor_fetchmany(PyObject* self, PyObject* args) PyObject* result; Cursor* cursor = Cursor_Validate(self, CURSOR_REQUIRE_RESULTS | CURSOR_RAISE_ERROR); - if (!cursor || !DetectConfigChange(cursor)) + if (!cursor || !PrepareFetch(cursor)) return 0; rows = cursor->arraysize; diff --git a/src/cursor.h b/src/cursor.h index 2d56a484..062f7e86 100644 --- a/src/cursor.h +++ b/src/cursor.h @@ -35,7 +35,7 @@ struct ColumnInfo SQLULEN buf_size; long buf_offset; SQLSMALLINT c_type; - PyObject* (*GetData)(void*, SQLLEN, bool, PyObject*, TextEnc*); + PyObject* (*GetData)(void*, SQLLEN, PyObject*, TextEnc*); bool is_bound; bool can_bind; diff --git a/src/getdata.cpp b/src/getdata.cpp index aec4a396..bbfa71fc 100644 --- a/src/getdata.cpp +++ b/src/getdata.cpp @@ -238,32 +238,19 @@ static byte* ReallocOrFreeBuffer(byte* pb, Py_ssize_t cbNeed) } -static PyObject* GetText(void* buffer, SQLLEN cbFetched, bool bound, PyObject* converter, TextEnc* enc) +static PyObject* GetText(void* buffer, SQLLEN cbFetched, PyObject* converter, TextEnc* enc) { - // We are reading one of the SQL_WCHAR, SQL_WVARCHAR, etc., and will return - // a string. - // - // If there is no configuration we would expect this to be UTF-16 encoded data. (If no - // byte-order-mark, we would expect it to be big-endian.) - // - // Now, just because the driver is telling us it is wide data doesn't mean it is true. - // psqlodbc with UTF-8 will tell us it is wide data but you must ask for single-byte. - // (Otherwise it is just UTF-8 with each character stored as 2 bytes.) That's why we allow - // the user to configure. - return TextBufferToObject(enc, (byte*)buffer, (Py_ssize_t)cbFetched); } -static PyObject* GetBinary(void* buffer, SQLLEN cbFetched, bool bound, PyObject* converter, TextEnc* enc) +static PyObject* GetBinary(void* buffer, SQLLEN cbFetched, PyObject* converter, TextEnc* enc) { - // Reads SQL_BINARY. - return PyBytes_FromStringAndSize((char*)buffer, (Py_ssize_t)cbFetched); } -static PyObject* GetDataUser(void* buffer, SQLLEN cbFetched, bool bound, PyObject* converter, TextEnc* enc) +static PyObject* GetDataUser(void* buffer, SQLLEN cbFetched, PyObject* converter, TextEnc* enc) { PyObject* value = PyBytes_FromStringAndSize((char*)buffer, (Py_ssize_t)cbFetched); if (!value) @@ -278,35 +265,13 @@ static PyObject* GetDataUser(void* buffer, SQLLEN cbFetched, bool bound, PyObjec } -static PyObject* GetDataDecimal(void* buffer, SQLLEN cbFetched, bool bound, PyObject* converter, TextEnc* enc) +static PyObject* GetDataDecimal(void* buffer, SQLLEN cbFetched, PyObject* converter, TextEnc* enc) { - // The SQL_NUMERIC_STRUCT support is hopeless (SQL Server ignores scale on input parameters - // and output columns, Oracle does something else weird, and many drivers don't support it - // at all), so we'll rely on the Decimal's string parsing. Unfortunately, the Decimal - // author does not pay attention to the locale, so we have to modify the string ourselves. - // - // Oracle inserts group separators (commas in US, periods in some countries), so leave room - // for that too. - // - // Some databases support a 'money' type which also inserts currency symbols. Since we - // don't want to keep track of all these, we'll ignore all characters we don't recognize. - // We will look for digits, negative sign (which I hope is universal), and a decimal point - // ('.' or ',' usually). We'll do everything as Unicode in case currencies, etc. are too - // far out. - // - // This seems very inefficient. We know the characters we are interested in are ASCII - // since they are -, ., and 0-9. There /could/ be a Unicode currency symbol, but I'm going - // to ignore that for right now. Therefore if we ask for the data in SQLCHAR, it should be - // ASCII even if the encoding is UTF-8. - - // I'm going to request the data as Unicode in case there is a weird currency symbol. If - // this is a performance problems we may want a flag on this. - Object result(DecimalFromText(enc, (byte*)buffer, (Py_ssize_t)cbFetched)); return result.Detach(); } -static PyObject* GetDataBit(void* buffer, SQLLEN cbFetched, bool bound, PyObject* converter, TextEnc* enc) +static PyObject* GetDataBit(void* buffer, SQLLEN cbFetched, PyObject* converter, TextEnc* enc) { if (*(SQLCHAR*)buffer == SQL_TRUE) Py_RETURN_TRUE; @@ -314,42 +279,35 @@ static PyObject* GetDataBit(void* buffer, SQLLEN cbFetched, bool bound, PyObject } -static PyObject* GetDataULong(void* buffer, SQLLEN cbFetched, bool bound, PyObject* converter, TextEnc* enc) -{ - SQLINTEGER value = *(SQLINTEGER*)buffer; - return PyLong_FromLong(*(SQLINTEGER*)&value); -} - - -static PyObject* GetDataLong(void* buffer, SQLLEN cbFetched, bool bound, PyObject* converter, TextEnc* enc) +static PyObject* GetDataLong(void* buffer, SQLLEN cbFetched, PyObject* converter, TextEnc* enc) { SQLINTEGER value = *(SQLINTEGER*)buffer; return PyLong_FromLong(value); } -static PyObject* GetDataULongLong(void* buffer, SQLLEN cbFetched, bool bound, PyObject* converter, TextEnc* enc) +static PyObject* GetDataULongLong(void* buffer, SQLLEN cbFetched, PyObject* converter, TextEnc* enc) { SQLBIGINT value = *(SQLBIGINT*)buffer; return PyLong_FromUnsignedLongLong((unsigned PY_LONG_LONG)(SQLUBIGINT)value); } -static PyObject* GetDataLongLong(void* buffer, SQLLEN cbFetched, bool bound, PyObject* converter, TextEnc* enc) +static PyObject* GetDataLongLong(void* buffer, SQLLEN cbFetched, PyObject* converter, TextEnc* enc) { SQLBIGINT value = *(SQLBIGINT*)buffer; return PyLong_FromLongLong((PY_LONG_LONG)value); } -static PyObject* GetDataDouble(void* buffer, SQLLEN cbFetched, bool bound, PyObject* converter, TextEnc* enc) +static PyObject* GetDataDouble(void* buffer, SQLLEN cbFetched, PyObject* converter, TextEnc* enc) { double value = *(double*)buffer; return PyFloat_FromDouble(value); } -static PyObject* GetSqlServerTime(void* buffer, SQLLEN cbFetched, bool bound, PyObject* converter, TextEnc* enc) +static PyObject* GetSqlServerTime(void* buffer, SQLLEN cbFetched, PyObject* converter, TextEnc* enc) { SQL_SS_TIME2_STRUCT value = *(SQL_SS_TIME2_STRUCT*)buffer; int micros = (int)(value.fraction / 1000); // nanos --> micros @@ -357,10 +315,8 @@ static PyObject* GetSqlServerTime(void* buffer, SQLLEN cbFetched, bool bound, Py } -static PyObject* GetUUID(void* buffer, SQLLEN cbFetched, bool bound, PyObject* converter, TextEnc* enc) +static PyObject* GetUUID(void* buffer, SQLLEN cbFetched, PyObject* converter, TextEnc* enc) { - // REVIEW: Since GUID is a fixed size, do we need to pass the size or cbFetched? - PyObject* guid_bytes = PyBytes_FromStringAndSize((char*)buffer, (Py_ssize_t)sizeof(PYSQLGUID)); PyObject* uuid_args = PyTuple_New(3); PyObject* uuid_type = GetClassForThread("uuid", "UUID"); @@ -372,8 +328,8 @@ static PyObject* GetUUID(void* buffer, SQLLEN cbFetched, bool bound, PyObject* c return 0; } - PyTuple_SET_ITEM(uuid_args, 0, Py_None); - PyTuple_SET_ITEM(uuid_args, 1, Py_None); + PyTuple_SET_ITEM(uuid_args, 0, Py_NewRef(Py_None)); + PyTuple_SET_ITEM(uuid_args, 1, Py_NewRef(Py_None)); PyTuple_SET_ITEM(uuid_args, 2, guid_bytes); PyObject* uuid = PyObject_CallObject(uuid_type, uuid_args); @@ -383,14 +339,14 @@ static PyObject* GetUUID(void* buffer, SQLLEN cbFetched, bool bound, PyObject* c } -static PyObject* GetDataDate(void* buffer, SQLLEN cbFetched, bool bound, PyObject* converter, TextEnc* enc) +static PyObject* GetDataDate(void* buffer, SQLLEN cbFetched, PyObject* converter, TextEnc* enc) { TIMESTAMP_STRUCT value = *(TIMESTAMP_STRUCT*)buffer; return PyDate_FromDate(value.year, value.month, value.day); } -static PyObject* GetDataTime(void* buffer, SQLLEN cbFetched, bool bound, PyObject* converter, TextEnc* enc) +static PyObject* GetDataTime(void* buffer, SQLLEN cbFetched, PyObject* converter, TextEnc* enc) { TIMESTAMP_STRUCT value = *(TIMESTAMP_STRUCT*)buffer; int micros = (int)(value.fraction / 1000); // nanos --> micros @@ -398,7 +354,7 @@ static PyObject* GetDataTime(void* buffer, SQLLEN cbFetched, bool bound, PyObjec } -static PyObject* GetDataTimestamp(void* buffer, SQLLEN cbFetched, bool bound, PyObject* converter, TextEnc* enc) +static PyObject* GetDataTimestamp(void* buffer, SQLLEN cbFetched, PyObject* converter, TextEnc* enc) { struct tm t; TIMESTAMP_STRUCT value = *(TIMESTAMP_STRUCT*)buffer; @@ -535,31 +491,31 @@ PyObject* PythonTypeFromSqlType(Cursor* cur, SQLSMALLINT type) PyObject* GetData(Cursor* cur, Py_ssize_t iCol, Py_ssize_t iRow) { - ColumnInfo* pinfo = &cur->colinfos[iCol]; + ColumnInfo* cInfo = &cur->colinfos[iCol]; void* ptr_value; SQLLEN len; SQLLEN* ptr_len; bool isNull = false; - if (pinfo->is_bound || pinfo->always_alloc) { - assert(pinfo->buf_offset > 0); - ptr_value = (void*)((uintptr_t)cur->fetch_buffer + pinfo->buf_offset + iRow * cur->fetch_buffer_width); - ptr_len = (SQLLEN*)((uintptr_t)cur->fetch_buffer + pinfo->buf_offset + iRow * cur->fetch_buffer_width - sizeof(SQLLEN)); + if (cInfo->is_bound || cInfo->always_alloc) { + assert(cInfo->buf_offset > 0); + ptr_value = (void*)((uintptr_t)cur->fetch_buffer + cInfo->buf_offset + iRow * cur->fetch_buffer_width); + ptr_len = (SQLLEN*)((uintptr_t)cur->fetch_buffer + cInfo->buf_offset + iRow * cur->fetch_buffer_width - sizeof(SQLLEN)); } else { ptr_value = 0; ptr_len = &len; } - if (!pinfo->is_bound && pinfo->always_alloc) { + if (!cInfo->is_bound && cInfo->always_alloc) { assert(iRow == 1); SQLRETURN ret; Py_BEGIN_ALLOW_THREADS ret = SQLGetData( cur->hstmt, (SQLUSMALLINT)(iCol+1), - pinfo->c_type, + cInfo->c_type, ptr_value, - pinfo->buf_size, + cInfo->buf_size, ptr_len ); Py_END_ALLOW_THREADS @@ -567,9 +523,9 @@ PyObject* GetData(Cursor* cur, Py_ssize_t iCol, Py_ssize_t iRow) return RaiseErrorFromHandle(cur->cnxn, "SQLGetData", cur->cnxn->hdbc, cur->hstmt); } } - if (!pinfo->is_bound && !pinfo->always_alloc) { + if (!cInfo->is_bound && !cInfo->always_alloc) { assert(iRow == 1); - if (!ReadVarColumn(cur, iCol, pinfo->c_type, &isNull, &ptr_value, ptr_len)) { + if (!ReadVarColumn(cur, iCol, cInfo->c_type, &isNull, &ptr_value, ptr_len)) { return 0; } assert(!ptr_value == isNull); @@ -579,16 +535,15 @@ PyObject* GetData(Cursor* cur, Py_ssize_t iCol, Py_ssize_t iRow) PyObject* value; if (isNull) { - value = Py_None; + value = Py_NewRef(Py_None); } else { - value = (*pinfo->GetData)( + value = (*cInfo->GetData)( ptr_value, *ptr_len, - pinfo->is_bound, - pinfo->converter, - pinfo->enc + cInfo->converter, + cInfo->enc ); - if (!pinfo->is_bound && !pinfo->always_alloc) { + if (!cInfo->is_bound && !cInfo->always_alloc) { PyMem_Free(ptr_value); } } @@ -618,29 +573,29 @@ bool FetchBufferInfo(Cursor* cur, Py_ssize_t iCol) // // Must be analogous to GetData. - ColumnInfo* pinfo = &cur->colinfos[iCol]; + ColumnInfo* cInfo = &cur->colinfos[iCol]; // We don't implement SQLBindCol for user-defined conversions. // First see if there is a user-defined conversion. if (cur->cnxn->map_sqltype_to_converter) { - PyObject* converter = Connection_GetConverter(cur->cnxn, pinfo->sql_type); + PyObject* converter = Connection_GetConverter(cur->cnxn, cInfo->sql_type); if (converter) { - pinfo->converter = converter; - pinfo->GetData = GetDataUser; - pinfo->can_bind = false; - pinfo->always_alloc = false; - pinfo->c_type = SQL_C_BINARY; - pinfo->buf_size = 0; + cInfo->converter = converter; + cInfo->GetData = GetDataUser; + cInfo->can_bind = false; + cInfo->always_alloc = false; + cInfo->c_type = SQL_C_BINARY; + cInfo->buf_size = 0; return PyErr_Occurred() ? false : true; } } - pinfo->converter = 0; - pinfo->can_bind = true; - pinfo->always_alloc = true; - switch (pinfo->sql_type) + cInfo->converter = 0; + cInfo->can_bind = true; + cInfo->always_alloc = true; + switch (cInfo->sql_type) { case SQL_WCHAR: case SQL_WVARCHAR: @@ -651,122 +606,149 @@ bool FetchBufferInfo(Cursor* cur, Py_ssize_t iCol) case SQL_LONGVARCHAR: case SQL_SS_XML: case SQL_DB2_XML: - pinfo->enc = &(IsWideType(pinfo->sql_type) ? cur->cnxn->sqlwchar_enc : cur->cnxn->sqlchar_enc); - pinfo->c_type = pinfo->enc->ctype; - pinfo->GetData = GetText; - if (pinfo->column_size <= 0) { - pinfo->buf_size = 0; - pinfo->can_bind = false; + // We are reading one of the SQL_WCHAR, SQL_WVARCHAR, etc., and will return + // a string. + // + // If there is no configuration we would expect this to be UTF-16 encoded data. (If no + // byte-order-mark, we would expect it to be big-endian.) + // + // Now, just because the driver is telling us it is wide data doesn't mean it is true. + // psqlodbc with UTF-8 will tell us it is wide data but you must ask for single-byte. + // (Otherwise it is just UTF-8 with each character stored as 2 bytes.) That's why we allow + // the user to configure. + + cInfo->enc = &(IsWideType(cInfo->sql_type) ? cur->cnxn->sqlwchar_enc : cur->cnxn->sqlchar_enc); + cInfo->c_type = cInfo->enc->ctype; + cInfo->GetData = GetText; + if (cInfo->column_size <= 0) { + cInfo->buf_size = 0; + cInfo->can_bind = false; } else { - pinfo->buf_size = CharBufferSize(pinfo->column_size); + cInfo->buf_size = CharBufferSize(cInfo->column_size); } - pinfo->always_alloc = false; + cInfo->always_alloc = false; break; case SQL_GUID: if (UseNativeUUID()) { - pinfo->c_type = SQL_C_GUID; - pinfo->buf_size = sizeof(PYSQLGUID); - pinfo->GetData = GetUUID; + cInfo->c_type = SQL_C_GUID; + cInfo->buf_size = sizeof(PYSQLGUID); + cInfo->GetData = GetUUID; } else { - pinfo->enc = &cur->cnxn->sqlchar_enc; - pinfo->c_type = pinfo->enc->ctype; + cInfo->enc = &cur->cnxn->sqlchar_enc; + cInfo->c_type = cInfo->enc->ctype; // leave space for dashes every 4 characters - pinfo->buf_size = CharBufferSize(32+7); - pinfo->GetData = GetText; + cInfo->buf_size = CharBufferSize(32+7); + cInfo->GetData = GetText; } break; case SQL_BINARY: case SQL_VARBINARY: case SQL_LONGVARBINARY: - pinfo->c_type = SQL_C_BINARY; - if (pinfo->column_size == 0) { - pinfo->buf_size = 0; - pinfo->can_bind = false; + cInfo->c_type = SQL_C_BINARY; + if (cInfo->column_size == 0) { + cInfo->buf_size = 0; + cInfo->can_bind = false; } else { - pinfo->buf_size = pinfo->column_size + 1; + cInfo->buf_size = cInfo->column_size + 1; } - pinfo->GetData = GetBinary; - pinfo->always_alloc = false; + cInfo->GetData = GetBinary; + cInfo->always_alloc = false; break; case SQL_DECIMAL: case SQL_NUMERIC: case SQL_DB2_DECFLOAT: - pinfo->enc = &cur->cnxn->sqlwchar_enc; - pinfo->c_type = pinfo->enc->ctype; + // The SQL_NUMERIC_STRUCT support is hopeless (SQL Server ignores scale on input parameters + // and output columns, Oracle does something else weird, and many drivers don't support it + // at all), so we'll rely on the Decimal's string parsing. Unfortunately, the Decimal + // author does not pay attention to the locale, so we have to modify the string ourselves. + // + // Oracle inserts group separators (commas in US, periods in some countries), so leave room + // for that too. + // + // Some databases support a 'money' type which also inserts currency symbols. Since we + // don't want to keep track of all these, we'll ignore all characters we don't recognize. + // We will look for digits, negative sign (which I hope is universal), and a decimal point + // ('.' or ',' usually). We'll do everything as Unicode in case currencies, etc. are too + // far out. + // + // This seems very inefficient. We know the characters we are interested in are ASCII + // since they are -, ., and 0-9. There /could/ be a Unicode currency symbol, but I'm going + // to ignore that for right now. Therefore if we ask for the data in SQLCHAR, it should be + // ASCII even if the encoding is UTF-8. + + // I'm going to request the data as Unicode in case there is a weird currency symbol. If + // this is a performance problems we may want a flag on this. + + cInfo->enc = &cur->cnxn->sqlwchar_enc; + cInfo->c_type = cInfo->enc->ctype; // need to add padding for all kinds of situations, see GetDataDecimal - pinfo->buf_size = CharBufferSize(pinfo->column_size + 5); - pinfo->GetData = GetDataDecimal; + cInfo->buf_size = CharBufferSize(cInfo->column_size + 5); + cInfo->GetData = GetDataDecimal; break; case SQL_BIT: - pinfo->c_type = SQL_C_BIT; - pinfo->buf_size = sizeof(SQLCHAR); - pinfo->GetData = GetDataBit; + cInfo->c_type = SQL_C_BIT; + cInfo->buf_size = sizeof(SQLCHAR); + cInfo->GetData = GetDataBit; break; case SQL_TINYINT: case SQL_SMALLINT: case SQL_INTEGER: - if (pinfo->is_unsigned) { - pinfo->c_type = SQL_C_ULONG; - pinfo->buf_size = sizeof(SQLINTEGER); - pinfo->GetData = GetDataULong; - } else { - pinfo->c_type = SQL_C_LONG; - pinfo->GetData = GetDataLong; - pinfo->buf_size = sizeof(SQLINTEGER); - } + cInfo->c_type = cInfo->is_unsigned ? SQL_C_ULONG : SQL_C_LONG; + cInfo->buf_size = sizeof(SQLINTEGER); + cInfo->GetData = GetDataLong; break; case SQL_BIGINT: - if (pinfo->is_unsigned) { - pinfo->c_type = SQL_C_UBIGINT; - pinfo->buf_size = sizeof(SQLBIGINT); - pinfo->GetData = GetDataULongLong; + if (cInfo->is_unsigned) { + cInfo->c_type = SQL_C_UBIGINT; + cInfo->buf_size = sizeof(SQLBIGINT); + cInfo->GetData = GetDataULongLong; } else { - pinfo->c_type = SQL_C_SBIGINT; - pinfo->buf_size = sizeof(SQLBIGINT); - pinfo->GetData = GetDataLongLong; + cInfo->c_type = SQL_C_SBIGINT; + cInfo->buf_size = sizeof(SQLBIGINT); + cInfo->GetData = GetDataLongLong; } break; case SQL_REAL: case SQL_FLOAT: case SQL_DOUBLE: - pinfo->c_type = SQL_C_DOUBLE; - pinfo->buf_size = sizeof(double); - pinfo->GetData = GetDataDouble; + cInfo->c_type = SQL_C_DOUBLE; + cInfo->buf_size = sizeof(double); + cInfo->GetData = GetDataDouble; break; case SQL_TYPE_DATE: - pinfo->c_type = SQL_C_TYPE_TIMESTAMP; - pinfo->buf_size = sizeof(TIMESTAMP_STRUCT); - pinfo->GetData = GetDataDate; + cInfo->c_type = SQL_C_TYPE_TIMESTAMP; + cInfo->buf_size = sizeof(TIMESTAMP_STRUCT); + cInfo->GetData = GetDataDate; break; case SQL_TYPE_TIME: - pinfo->c_type = SQL_C_TYPE_TIMESTAMP; - pinfo->buf_size = sizeof(TIMESTAMP_STRUCT); - pinfo->GetData = GetDataTime; + cInfo->c_type = SQL_C_TYPE_TIMESTAMP; + cInfo->buf_size = sizeof(TIMESTAMP_STRUCT); + cInfo->GetData = GetDataTime; break; case SQL_TYPE_TIMESTAMP: - pinfo->c_type = SQL_C_TYPE_TIMESTAMP; - pinfo->buf_size = sizeof(TIMESTAMP_STRUCT); - pinfo->GetData = GetDataTimestamp; + cInfo->c_type = SQL_C_TYPE_TIMESTAMP; + cInfo->buf_size = sizeof(TIMESTAMP_STRUCT); + cInfo->GetData = GetDataTimestamp; break; case SQL_SS_TIME2: - pinfo->c_type = SQL_C_BINARY; - pinfo->buf_size = sizeof(SQL_SS_TIME2_STRUCT); - pinfo->GetData = GetSqlServerTime; + cInfo->c_type = SQL_C_BINARY; + cInfo->buf_size = sizeof(SQL_SS_TIME2_STRUCT); + cInfo->GetData = GetSqlServerTime; break; default: RaiseErrorV("HY106", ProgrammingError, "ODBC SQL type %d is not yet supported. column-index=%zd type=%d", - (int)pinfo->sql_type, iCol, (int)pinfo->sql_type); + (int)cInfo->sql_type, iCol, (int)cInfo->sql_type); return false; } return true; From 64fc661d79c3cd787cbff5d043db23c4b4a85f23 Mon Sep 17 00:00:00 2001 From: ffelixg <142172984+ffelixg@users.noreply.github.com> Date: Sun, 3 Mar 2024 23:42:48 +0100 Subject: [PATCH 17/21] Fix py3.9 compatibility --- src/getdata.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/getdata.cpp b/src/getdata.cpp index bbfa71fc..08de3fde 100644 --- a/src/getdata.cpp +++ b/src/getdata.cpp @@ -328,8 +328,10 @@ static PyObject* GetUUID(void* buffer, SQLLEN cbFetched, PyObject* converter, Te return 0; } - PyTuple_SET_ITEM(uuid_args, 0, Py_NewRef(Py_None)); - PyTuple_SET_ITEM(uuid_args, 1, Py_NewRef(Py_None)); + Py_IncRef(Py_None); + Py_IncRef(Py_None); + PyTuple_SET_ITEM(uuid_args, 0, Py_None); + PyTuple_SET_ITEM(uuid_args, 1, Py_None); PyTuple_SET_ITEM(uuid_args, 2, guid_bytes); PyObject* uuid = PyObject_CallObject(uuid_type, uuid_args); @@ -535,7 +537,8 @@ PyObject* GetData(Cursor* cur, Py_ssize_t iCol, Py_ssize_t iRow) PyObject* value; if (isNull) { - value = Py_NewRef(Py_None); + Py_IncRef(Py_None); + value = Py_None; } else { value = (*cInfo->GetData)( ptr_value, From 057cafedbd61f0b770573dade7bf6495e6772b9f Mon Sep 17 00:00:00 2001 From: ffelixg <142172984+ffelixg@users.noreply.github.com> Date: Mon, 4 Mar 2024 20:05:03 +0100 Subject: [PATCH 18/21] Rewrite GetUUID and bind native uuids --- src/getdata.cpp | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/getdata.cpp b/src/getdata.cpp index d210a176..1650f316 100644 --- a/src/getdata.cpp +++ b/src/getdata.cpp @@ -277,8 +277,6 @@ static PyObject* GetText(Cursor* cur, Py_ssize_t iCol) { Py_RETURN_NONE; } - cbFetched == SQL_NTS; - cbFetched == SQL_NO_TOTAL; pbData = (byte*)valueBuf; cbData = cbFetched; } @@ -611,15 +609,25 @@ static PyObject* GetUUID(Cursor* cur, Py_ssize_t iCol) if (cbFetched == SQL_NULL_DATA) Py_RETURN_NONE; - const char* szFmt = "(yyy#)"; - Object args(Py_BuildValue(szFmt, NULL, NULL, &guid, (int)sizeof(guid))); - if (!args) - return 0; - + PyObject* guid_bytes = PyBytes_FromStringAndSize((char*)&guid, (Py_ssize_t)sizeof(PYSQLGUID)); + PyObject* uuid_args = PyTuple_New(3); PyObject* uuid_type = GetClassForThread("uuid", "UUID"); - if (!uuid_type) + + if(!guid_bytes || !uuid_args || !uuid_type) { + Py_XDECREF(guid_bytes); + Py_XDECREF(uuid_args); + Py_XDECREF(uuid_type); return 0; - PyObject* uuid = PyObject_CallObject(uuid_type, args.Get()); + } + + Py_IncRef(Py_None); + Py_IncRef(Py_None); + PyTuple_SET_ITEM(uuid_args, 0, Py_None); + PyTuple_SET_ITEM(uuid_args, 1, Py_None); + PyTuple_SET_ITEM(uuid_args, 2, guid_bytes); + + PyObject* uuid = PyObject_CallObject(uuid_type, uuid_args); + Py_DECREF(uuid_args); Py_DECREF(uuid_type); return uuid; } @@ -933,12 +941,8 @@ bool BindCol(Cursor* cur, Py_ssize_t iCol) case SQL_GUID: if (UseNativeUUID()) { - // Binding here does not work on 64bit Windows, so we don't. - // Not sure why, it works everywhere else. - - // c_type = SQL_GUID; - // size = sizeof(PYSQLGUID); - return true; + c_type = SQL_GUID; + size = sizeof(PYSQLGUID); } else { From 2df52d41516a1f84e0e794b5c93293ab487e0379 Mon Sep 17 00:00:00 2001 From: ffelixg <142172984+ffelixg@users.noreply.github.com> Date: Tue, 5 Mar 2024 00:15:26 +0100 Subject: [PATCH 19/21] Add configuration variables for the number of rows to allocate for fetching --- src/cursor.cpp | 80 +++++++++++++++++++++++++++++++++++++++----------- src/cursor.h | 4 +++ 2 files changed, 67 insertions(+), 17 deletions(-) diff --git a/src/cursor.cpp b/src/cursor.cpp index 88c7915b..d07cab3e 100644 --- a/src/cursor.cpp +++ b/src/cursor.cpp @@ -330,15 +330,15 @@ inline static void BindColsFree(Cursor* self) } -static bool BindCols(Cursor* cur, int cCols) +static bool BindCols(Cursor* cur, int cCols, int fetch_rows_cap) { int iCol; - long buf_cap = 1024*1024; long total_buf_size = 0; ColumnInfo* cInfo; int cap_alloc = cCols; bool bind_all = true; + // init buffer info, compute sizes & offsets for (iCol = 0; iCol < cCols; iCol++) { if (!FetchBufferInfo(cur, iCol)) { return false; @@ -346,7 +346,7 @@ static bool BindCols(Cursor* cur, int cCols) cInfo = &cur->colinfos[iCol]; - if (total_buf_size + cInfo->buf_size + sizeof(SQLULEN) > buf_cap && iCol < cap_alloc) { + if (total_buf_size + cInfo->buf_size + sizeof(SQLULEN) > cur->bind_byte_cap && iCol < cap_alloc) { cap_alloc = iCol; } @@ -362,11 +362,24 @@ static bool BindCols(Cursor* cur, int cCols) } } + if (cur->bind_byte_cap < 0) { + PyErr_SetString(ProgrammingError, "Cursor attribute bind_byte_cap must be non negative."); + return 0; + } + if (cur->bind_cell_cap < 0) { + PyErr_SetString(ProgrammingError, "Cursor attribute bind_cell_cap must be non negative."); + return 0; + } + + // number of rows to be fetched at a time cur->fetch_buffer_width = total_buf_size; if (bind_all) { - cur->fetch_buffer_length = buf_cap / cur->fetch_buffer_width; - if (cur->fetch_buffer_length > cur->arraysize) { // cur->arraysize can be negative - cur->fetch_buffer_length = cur->arraysize; + cur->fetch_buffer_length = cur->bind_byte_cap / cur->fetch_buffer_width; + if (cur->fetch_buffer_length > fetch_rows_cap) { + cur->fetch_buffer_length = fetch_rows_cap; + } + if (cur->fetch_buffer_length > cur->bind_cell_cap / cCols) { + cur->fetch_buffer_length = cur->bind_cell_cap / cCols; } if (cur->fetch_buffer_length < 1) { cur->fetch_buffer_length = 1; @@ -375,13 +388,12 @@ static bool BindCols(Cursor* cur, int cCols) cur->fetch_buffer_length = 1; } + // single large buffer using row-wise layout with row status array at the end void* buf = PyMem_Malloc((cur->fetch_buffer_width + sizeof(SQLUSMALLINT)) * cur->fetch_buffer_length); if (!buf) { PyErr_NoMemory(); return false; } - assert(cur->fetch_buffer == 0); - cur->fetch_buffer = buf; cur->row_status_array = (SQLUSMALLINT*)((uintptr_t)buf + cur->fetch_buffer_width * cur->fetch_buffer_length); cur->current_row = 0; @@ -410,6 +422,7 @@ static bool BindCols(Cursor* cur, int cCols) goto skip; } + cur->bound_columns_count = 0; for (iCol = 0; iCol < cCols; iCol++) { cInfo = &cur->colinfos[iCol]; if (cInfo->can_bind && cInfo->buf_offset >= 0 && keep_binding) { @@ -425,6 +438,7 @@ static bool BindCols(Cursor* cur, int cCols) if (!SQL_SUCCEEDED(ret_bind)) { break; } + cur->bound_columns_count++; } else { keep_binding = false; cInfo->is_bound = false; @@ -434,7 +448,6 @@ static bool BindCols(Cursor* cur, int cCols) Py_END_ALLOW_THREADS if (!SQL_SUCCEEDED(ret_attr)) { - PyMem_Free(buf); return RaiseErrorFromHandle(cur->cnxn, "SQLSetStmtAttr", cur->cnxn->hdbc, cur->hstmt); } @@ -443,11 +456,14 @@ static bool BindCols(Cursor* cur, int cCols) return RaiseErrorFromHandle(cur->cnxn, "SQLBindCol", cur->cnxn->hdbc, cur->hstmt); } + assert(cur->fetch_buffer == 0); + cur->fetch_buffer = buf; + return true; } -static bool PrepareFetch(Cursor* cur) +static bool PrepareFetch(Cursor* cur, int n_rows) { // Returns false on exception, true otherwise. // Need to do this because the API allows changing this after executing a statement. @@ -498,7 +514,7 @@ static bool PrepareFetch(Cursor* cur) { int cCols = PyTuple_GET_SIZE(cur->description); BindColsFree(cur); - if (!BindCols(cur, cCols)) + if (!BindCols(cur, cCols, n_rows)) { return false; } @@ -1468,7 +1484,7 @@ static PyObject* Cursor_iternext(PyObject* self) Cursor* cursor = Cursor_Validate(self, CURSOR_REQUIRE_RESULTS | CURSOR_RAISE_ERROR); - if (!cursor || !PrepareFetch(cursor)) + if (!cursor || !PrepareFetch(cursor, 1)) return 0; result = Cursor_fetch(cursor); @@ -1481,7 +1497,7 @@ static PyObject* Cursor_fetchval(PyObject* self, PyObject* args) UNUSED(args); Cursor* cursor = Cursor_Validate(self, CURSOR_REQUIRE_RESULTS | CURSOR_RAISE_ERROR); - if (!cursor || !PrepareFetch(cursor)) + if (!cursor || !PrepareFetch(cursor, 1)) return 0; Object row(Cursor_fetch(cursor)); @@ -1502,7 +1518,7 @@ static PyObject* Cursor_fetchone(PyObject* self, PyObject* args) PyObject* row; Cursor* cursor = Cursor_Validate(self, CURSOR_REQUIRE_RESULTS | CURSOR_RAISE_ERROR); - if (!cursor || !PrepareFetch(cursor)) + if (!cursor || !PrepareFetch(cursor, 1)) return 0; row = Cursor_fetch(cursor); @@ -1524,7 +1540,7 @@ static PyObject* Cursor_fetchall(PyObject* self, PyObject* args) PyObject* result; Cursor* cursor = Cursor_Validate(self, CURSOR_REQUIRE_RESULTS | CURSOR_RAISE_ERROR); - if (!cursor || !PrepareFetch(cursor)) + if (!cursor || !PrepareFetch(cursor, -1)) return 0; result = Cursor_fetchlist(cursor, -1); @@ -1539,11 +1555,11 @@ static PyObject* Cursor_fetchmany(PyObject* self, PyObject* args) PyObject* result; Cursor* cursor = Cursor_Validate(self, CURSOR_REQUIRE_RESULTS | CURSOR_RAISE_ERROR); - if (!cursor || !PrepareFetch(cursor)) + if (!cursor) return 0; rows = cursor->arraysize; - if (!PyArg_ParseTuple(args, "|l", &rows)) + if (!PyArg_ParseTuple(args, "|l", &rows) || !PrepareFetch(cursor, rows)) return 0; result = Cursor_fetchlist(cursor, rows); @@ -2416,6 +2432,29 @@ static char messages_doc[] = "This read-only attribute is a list of all the diagnostic messages in the\n" \ "current result set."; +static char bound_columns_count_doc[] = + "This read-only attribute provides the number of columns bound for fetching.\n" \ + "Binding columns requires a small enough cap on the size of a column, i.e.\n" \ + "varchar(200) instead of varchar(max). We bind as many columns as allowed by\n" \ + "cur.bind_byte_cap."; + +static char bound_buffer_rows_doc[] = + "This read-only attribute provides the number of rows allocated for fetching.\n" \ + "It is determined by bind_byte_cap and the size of each row, bind_cell_cap and the\n" \ + "number of columns as well as the fetch function called.\n" \ + "Can only be > 1 if all columns are bound."; + +static char bind_cell_cap_doc[] = + "This read/write attribute specifies a cap on the number of rows fetched at a time\n" \ + "when binding columns. For example when the cap is 10 and there are 5 columns,\n" \ + "2 rows will be fetched at a time. If there are 3 columns, 3 rows will be fetched.\n" \ + "Only takes effect when all columns can be bound."; + +static char bind_byte_cap_doc[] = + "This read/write attribute specifies a cap on the size of the buffer used for\n" \ + "fetching. When the cap is reached, further variable length columns will be obtained\n" \ + "using SQLGetData."; + static PyMemberDef Cursor_members[] = { {"rowcount", T_INT, offsetof(Cursor, rowcount), READONLY, rowcount_doc }, @@ -2424,6 +2463,10 @@ static PyMemberDef Cursor_members[] = {"connection", T_OBJECT_EX, offsetof(Cursor, cnxn), READONLY, connection_doc }, {"fast_executemany",T_BOOL, offsetof(Cursor, fastexecmany), 0, fastexecmany_doc }, {"messages", T_OBJECT_EX, offsetof(Cursor, messages), READONLY, messages_doc }, + {"bound_columns_count", T_UINT, offsetof(Cursor, bound_columns_count), READONLY, bound_columns_count_doc }, + {"bound_buffer_rows", T_ULONG, offsetof(Cursor, fetch_buffer_length), READONLY, bound_buffer_rows_doc }, + {"bind_cell_cap", T_LONG, offsetof(Cursor, bind_cell_cap), 0, bind_cell_cap_doc }, + {"bind_byte_cap", T_LONG, offsetof(Cursor, bind_byte_cap), 0, bind_byte_cap_doc }, { 0 } }; @@ -2712,6 +2755,9 @@ Cursor_New(Connection* cnxn) cur->fetch_buffer = 0; cur->bound_converted_types = 0; cur->bound_native_uuid = 0; + cur->bind_cell_cap = 10; + cur->bind_byte_cap = 20 * 1024 * 1024; + cur->bound_columns_count = 0; Py_INCREF(cnxn); Py_INCREF(cur->description); diff --git a/src/cursor.h b/src/cursor.h index 062f7e86..4986adfd 100644 --- a/src/cursor.h +++ b/src/cursor.h @@ -178,6 +178,10 @@ struct Cursor // Track the configuration at the time of using SQLBindCol. bool bound_native_uuid; PyObject* bound_converted_types; + + unsigned int bound_columns_count; + long bind_cell_cap; + long bind_byte_cap; }; void Cursor_init(); From ff0423f860390180e2447ad4e5fa932994e73825 Mon Sep 17 00:00:00 2001 From: ffelixg <142172984+ffelixg@users.noreply.github.com> Date: Wed, 15 May 2024 00:34:47 +0200 Subject: [PATCH 20/21] Undo some unnecessary changes --- src/cursor.cpp | 15 +++- src/cursor.h | 8 +- src/decimal.cpp | 2 +- src/decimal.h | 2 +- src/getdata.cpp | 223 ++++++++++++++++++------------------------------ src/getdata.h | 2 +- src/textenc.cpp | 6 +- src/textenc.h | 2 +- 8 files changed, 104 insertions(+), 156 deletions(-) diff --git a/src/cursor.cpp b/src/cursor.cpp index d07cab3e..9a8f9916 100644 --- a/src/cursor.cpp +++ b/src/cursor.cpp @@ -210,7 +210,7 @@ static bool create_name_map(Cursor* cur, SQLSMALLINT field_count, bool lower) TRACE("Col %d: type=%s (%d) colsize=%d\n", (i+1), SqlTypeName(nDataType), (int)nDataType, (int)nColSize); - Object name(TextBufferToObject(&enc, (const byte*)szName, cbName)); + Object name(TextBufferToObject(enc, (const byte*)szName, cbName)); if (!name) goto done; @@ -333,11 +333,20 @@ inline static void BindColsFree(Cursor* self) static bool BindCols(Cursor* cur, int cCols, int fetch_rows_cap) { int iCol; - long total_buf_size = 0; + unsigned long total_buf_size = 0; ColumnInfo* cInfo; int cap_alloc = cCols; bool bind_all = true; + if (cur->bind_byte_cap < 0) { + PyErr_SetString(ProgrammingError, "Cursor attribute bind_byte_cap must be non negative."); + return 0; + } + if (cur->bind_cell_cap < 0) { + PyErr_SetString(ProgrammingError, "Cursor attribute bind_cell_cap must be non negative."); + return 0; + } + // init buffer info, compute sizes & offsets for (iCol = 0; iCol < cCols; iCol++) { if (!FetchBufferInfo(cur, iCol)) { @@ -346,7 +355,7 @@ static bool BindCols(Cursor* cur, int cCols, int fetch_rows_cap) cInfo = &cur->colinfos[iCol]; - if (total_buf_size + cInfo->buf_size + sizeof(SQLULEN) > cur->bind_byte_cap && iCol < cap_alloc) { + if (total_buf_size + cInfo->buf_size + sizeof(SQLULEN) > (unsigned long)cur->bind_byte_cap && iCol < cap_alloc) { cap_alloc = iCol; } diff --git a/src/cursor.h b/src/cursor.h index 4986adfd..3762cb61 100644 --- a/src/cursor.h +++ b/src/cursor.h @@ -169,11 +169,11 @@ struct Cursor // Pointer to buffer used by SQLBindCol and sometimes SQLGetData. void* fetch_buffer; - SQLULEN fetch_buffer_width; - SQLULEN fetch_buffer_length; - SQLULEN rows_fetched; + long fetch_buffer_width; + long fetch_buffer_length; + long rows_fetched; SQLUSMALLINT* row_status_array; - SQLULEN current_row; + long current_row; // Track the configuration at the time of using SQLBindCol. bool bound_native_uuid; diff --git a/src/decimal.cpp b/src/decimal.cpp index 749e08b2..b97389f2 100644 --- a/src/decimal.cpp +++ b/src/decimal.cpp @@ -110,7 +110,7 @@ bool SetDecimalPoint(PyObject* pNew) } -PyObject* DecimalFromText(const TextEnc* enc, const byte* pb, Py_ssize_t cb) +PyObject* DecimalFromText(const TextEnc& enc, const byte* pb, Py_ssize_t cb) { // Creates a Decimal object from a text buffer. diff --git a/src/decimal.h b/src/decimal.h index 5b03a2e5..32af7122 100644 --- a/src/decimal.h +++ b/src/decimal.h @@ -4,4 +4,4 @@ bool InitializeDecimal(); PyObject* GetDecimalPoint(); bool SetDecimalPoint(PyObject* pNew); -PyObject* DecimalFromText(const TextEnc* enc, const byte* pb, Py_ssize_t cb); +PyObject* DecimalFromText(const TextEnc& enc, const byte* pb, Py_ssize_t cb); diff --git a/src/getdata.cpp b/src/getdata.cpp index 08de3fde..46dc1f84 100644 --- a/src/getdata.cpp +++ b/src/getdata.cpp @@ -67,7 +67,7 @@ inline bool IsWideType(SQLSMALLINT sqltype) } -static bool ReadVarColumn(Cursor* cur, Py_ssize_t iCol, SQLSMALLINT ctype, bool* isNull, void** pbResult, SQLLEN* cbResult) +static bool ReadVarColumn(Cursor* cur, Py_ssize_t iCol, SQLSMALLINT ctype, bool& isNull, byte*& pbResult, Py_ssize_t& cbResult) { // Called to read a variable-length column and return its data in a newly-allocated heap // buffer. @@ -85,16 +85,16 @@ static bool ReadVarColumn(Cursor* cur, Py_ssize_t iCol, SQLSMALLINT ctype, bool* // If a zero-length value was read, isNull is set to false and pbResult and cbResult will // be set to 0. - *isNull = false; - *pbResult = 0; - *cbResult = 0; + isNull = false; + pbResult = 0; + cbResult = 0; - const SQLLEN cbElement = (SQLLEN)(IsWideType(ctype) ? sizeof(uint16_t) : 1); - const SQLLEN cbNullTerminator = IsBinaryType(ctype) ? 0 : cbElement; + const Py_ssize_t cbElement = (Py_ssize_t)(IsWideType(ctype) ? sizeof(uint16_t) : 1); + const Py_ssize_t cbNullTerminator = IsBinaryType(ctype) ? 0 : cbElement; // TODO: Make the initial allocation size configurable? - SQLLEN cbAllocated = 4096; - SQLLEN cbUsed = 0; + Py_ssize_t cbAllocated = 4096; + Py_ssize_t cbUsed = 0; byte* pb = (byte*)PyMem_Malloc((size_t)cbAllocated); if (!pb) { @@ -110,11 +110,11 @@ static bool ReadVarColumn(Cursor* cur, Py_ssize_t iCol, SQLSMALLINT ctype, bool* // SQL_SUCCESS_WITH_INFO). Each time through, update the buffer pb, cbAllocated, and // cbUsed. - SQLLEN cbAvailable = cbAllocated - cbUsed; + Py_ssize_t cbAvailable = cbAllocated - cbUsed; SQLLEN cbData = 0; Py_BEGIN_ALLOW_THREADS - ret = SQLGetData(cur->hstmt, (SQLUSMALLINT)(iCol+1), ctype, &pb[cbUsed], cbAvailable, &cbData); + ret = SQLGetData(cur->hstmt, (SQLUSMALLINT)(iCol+1), ctype, &pb[cbUsed], (SQLLEN)cbAvailable, &cbData); Py_END_ALLOW_THREADS; TRACE("ReadVarColumn: SQLGetData avail=%d --> ret=%d cbData=%d\n", (int)cbAvailable, (int)ret, (int)cbData); @@ -125,7 +125,7 @@ static bool ReadVarColumn(Cursor* cur, Py_ssize_t iCol, SQLSMALLINT ctype, bool* return false; } - if (ret == SQL_SUCCESS && cbData < 0) + if (ret == SQL_SUCCESS && (int)cbData < 0) { // HACK: FreeTDS 0.91 on OS/X returns -4 for NULL data instead of SQL_NULL_DATA // (-1). I've traced into the code and it appears to be the result of assigning -1 @@ -144,8 +144,8 @@ static bool ReadVarColumn(Cursor* cur, Py_ssize_t iCol, SQLSMALLINT ctype, bool* // This means we read some data, but there is more. SQLGetData is very weird - it // sets cbRead to the number of bytes we read *plus* the amount remaining. - SQLLEN cbRemaining = 0; // How many more bytes do we need to allocate, not including null? - SQLLEN cbRead = 0; // How much did we just read, not including null? + Py_ssize_t cbRemaining = 0; // How many more bytes do we need to allocate, not including null? + Py_ssize_t cbRead = 0; // How much did we just read, not including null? if (cbData == SQL_NO_TOTAL) { @@ -158,7 +158,7 @@ static bool ReadVarColumn(Cursor* cur, Py_ssize_t iCol, SQLSMALLINT ctype, bool* cbRead = (cbAvailable - cbNullTerminator); cbRemaining = 1024 * 1024; } - else if (cbData >= cbAvailable) + else if ((Py_ssize_t)cbData >= cbAvailable) { // We offered cbAvailable space, but there was cbData data. The driver filled // the buffer with what it could. Remember that if the type requires a null @@ -186,8 +186,8 @@ static bool ReadVarColumn(Cursor* cur, Py_ssize_t iCol, SQLSMALLINT ctype, bool* // This is a tiny bit complicated by the fact that the data is null terminated, // meaning we haven't actually used up the entire buffer (cbAllocated), only // cbUsed (which should be cbAllocated - cbNullTerminator). - SQLLEN cbNeed = cbUsed + cbRemaining + cbNullTerminator; - pb = ReallocOrFreeBuffer(pb, (Py_ssize_t)cbNeed); + Py_ssize_t cbNeed = cbUsed + cbRemaining + cbNullTerminator; + pb = ReallocOrFreeBuffer(pb, cbNeed); if (!pb) return false; cbAllocated = cbNeed; @@ -206,12 +206,12 @@ static bool ReadVarColumn(Cursor* cur, Py_ssize_t iCol, SQLSMALLINT ctype, bool* } while (ret == SQL_SUCCESS_WITH_INFO); - *isNull = (ret == SQL_NULL_DATA); + isNull = (ret == SQL_NULL_DATA); - if (!*isNull && cbUsed > 0) + if (!isNull && cbUsed > 0) { - *pbResult = pb; - *cbResult = cbUsed; + pbResult = pb; + cbResult = cbUsed; } else { @@ -240,7 +240,7 @@ static byte* ReallocOrFreeBuffer(byte* pb, Py_ssize_t cbNeed) static PyObject* GetText(void* buffer, SQLLEN cbFetched, PyObject* converter, TextEnc* enc) { - return TextBufferToObject(enc, (byte*)buffer, (Py_ssize_t)cbFetched); + return TextBufferToObject(*enc, (byte*)buffer, (Py_ssize_t)cbFetched); } @@ -252,23 +252,13 @@ static PyObject* GetBinary(void* buffer, SQLLEN cbFetched, PyObject* converter, static PyObject* GetDataUser(void* buffer, SQLLEN cbFetched, PyObject* converter, TextEnc* enc) { - PyObject* value = PyBytes_FromStringAndSize((char*)buffer, (Py_ssize_t)cbFetched); - if (!value) - return 0; - - PyObject* result = PyObject_CallFunction(converter, "(O)", value); - Py_DECREF(value); - if (!result) - return 0; - - return result; + return PyObject_CallFunction(converter, "y#", (char*)buffer, (Py_ssize_t)cbFetched); } static PyObject* GetDataDecimal(void* buffer, SQLLEN cbFetched, PyObject* converter, TextEnc* enc) { - Object result(DecimalFromText(enc, (byte*)buffer, (Py_ssize_t)cbFetched)); - return result.Detach(); + return DecimalFromText(*enc, (byte*)buffer, (Py_ssize_t)cbFetched); } static PyObject* GetDataBit(void* buffer, SQLLEN cbFetched, PyObject* converter, TextEnc* enc) @@ -315,29 +305,15 @@ static PyObject* GetSqlServerTime(void* buffer, SQLLEN cbFetched, PyObject* conv } +PyObject* uuid_type = 0; static PyObject* GetUUID(void* buffer, SQLLEN cbFetched, PyObject* converter, TextEnc* enc) { - PyObject* guid_bytes = PyBytes_FromStringAndSize((char*)buffer, (Py_ssize_t)sizeof(PYSQLGUID)); - PyObject* uuid_args = PyTuple_New(3); - PyObject* uuid_type = GetClassForThread("uuid", "UUID"); - - if(!guid_bytes || !uuid_args || !uuid_type) { - Py_XDECREF(guid_bytes); - Py_XDECREF(uuid_args); - Py_XDECREF(uuid_type); - return 0; + if (!uuid_type) { + uuid_type = GetClassForThread("uuid", "UUID"); + if (!uuid_type) + return 0; } - - Py_IncRef(Py_None); - Py_IncRef(Py_None); - PyTuple_SET_ITEM(uuid_args, 0, Py_None); - PyTuple_SET_ITEM(uuid_args, 1, Py_None); - PyTuple_SET_ITEM(uuid_args, 2, guid_bytes); - - PyObject* uuid = PyObject_CallObject(uuid_type, uuid_args); - Py_DECREF(uuid_args); - Py_DECREF(uuid_type); - return uuid; + return PyObject_CallFunction(uuid_type, "yyy#", 0, 0, (char*)buffer, (Py_ssize_t)16); } @@ -509,7 +485,6 @@ PyObject* GetData(Cursor* cur, Py_ssize_t iCol, Py_ssize_t iRow) } if (!cInfo->is_bound && cInfo->always_alloc) { - assert(iRow == 1); SQLRETURN ret; Py_BEGIN_ALLOW_THREADS ret = SQLGetData( @@ -526,10 +501,13 @@ PyObject* GetData(Cursor* cur, Py_ssize_t iCol, Py_ssize_t iRow) } } if (!cInfo->is_bound && !cInfo->always_alloc) { - assert(iRow == 1); - if (!ReadVarColumn(cur, iCol, cInfo->c_type, &isNull, &ptr_value, ptr_len)) { + byte* readVarPtr = 0; + Py_ssize_t readVarLen = 0; + if (!ReadVarColumn(cur, iCol, cInfo->c_type, isNull, readVarPtr, readVarLen)) { return 0; } + ptr_value = (void*)readVarPtr; + *ptr_len = (SQLLEN)readVarLen; assert(!ptr_value == isNull); } else { isNull = *ptr_len == SQL_NULL_DATA; @@ -566,39 +544,40 @@ inline SQLULEN CharBufferSize(SQLULEN nr_chars) return (nr_chars + 1) * 8; // + 1 for null terminator } - +#define SetInfoAndBreak(ctp, bufsz, get) \ + info->c_type = ctp; \ + info->buf_size = bufsz; \ + info->GetData = get; \ + break; bool FetchBufferInfo(Cursor* cur, Py_ssize_t iCol) { - // 0 means error, 1 means can be bound, -1 means cannot be bound // If false is returned, an exception has already been set. // // The data is assumed to be the default C type for the column's SQL type. - // - // Must be analogous to GetData. - ColumnInfo* cInfo = &cur->colinfos[iCol]; - - // We don't implement SQLBindCol for user-defined conversions. + ColumnInfo* info = &cur->colinfos[iCol]; // First see if there is a user-defined conversion. + // We don't implement SQLBindCol for user-defined conversions. if (cur->cnxn->map_sqltype_to_converter) { - PyObject* converter = Connection_GetConverter(cur->cnxn, cInfo->sql_type); + PyObject* converter = Connection_GetConverter(cur->cnxn, info->sql_type); if (converter) { - cInfo->converter = converter; - cInfo->GetData = GetDataUser; - cInfo->can_bind = false; - cInfo->always_alloc = false; - cInfo->c_type = SQL_C_BINARY; - cInfo->buf_size = 0; + info->converter = converter; + info->GetData = GetDataUser; + info->can_bind = false; + info->always_alloc = false; + info->c_type = SQL_C_BINARY; + info->buf_size = 0; return PyErr_Occurred() ? false : true; } } - cInfo->converter = 0; - cInfo->can_bind = true; - cInfo->always_alloc = true; - switch (cInfo->sql_type) + info->converter = 0; + info->can_bind = true; + info->always_alloc = true; + info->enc = 0; + switch (info->sql_type) { case SQL_WCHAR: case SQL_WVARCHAR: @@ -620,45 +599,34 @@ bool FetchBufferInfo(Cursor* cur, Py_ssize_t iCol) // (Otherwise it is just UTF-8 with each character stored as 2 bytes.) That's why we allow // the user to configure. - cInfo->enc = &(IsWideType(cInfo->sql_type) ? cur->cnxn->sqlwchar_enc : cur->cnxn->sqlchar_enc); - cInfo->c_type = cInfo->enc->ctype; - cInfo->GetData = GetText; - if (cInfo->column_size <= 0) { - cInfo->buf_size = 0; - cInfo->can_bind = false; + info->enc = &(IsWideType(info->sql_type) ? cur->cnxn->sqlwchar_enc : cur->cnxn->sqlchar_enc); + info->always_alloc = false; + if (info->column_size <= 0) { + info->can_bind = false; + SetInfoAndBreak(info->enc->ctype, 0, GetText); } else { - cInfo->buf_size = CharBufferSize(cInfo->column_size); + SetInfoAndBreak(info->enc->ctype, CharBufferSize(info->column_size), GetText); } - cInfo->always_alloc = false; - break; case SQL_GUID: if (UseNativeUUID()) { - cInfo->c_type = SQL_C_GUID; - cInfo->buf_size = sizeof(PYSQLGUID); - cInfo->GetData = GetUUID; + SetInfoAndBreak(SQL_C_GUID, sizeof(PYSQLGUID), GetUUID); } else { - cInfo->enc = &cur->cnxn->sqlchar_enc; - cInfo->c_type = cInfo->enc->ctype; + info->enc = &cur->cnxn->sqlchar_enc; // leave space for dashes every 4 characters - cInfo->buf_size = CharBufferSize(32+7); - cInfo->GetData = GetText; + SetInfoAndBreak(info->enc->ctype, CharBufferSize(32+7), GetText); } - break; case SQL_BINARY: case SQL_VARBINARY: case SQL_LONGVARBINARY: - cInfo->c_type = SQL_C_BINARY; - if (cInfo->column_size == 0) { - cInfo->buf_size = 0; - cInfo->can_bind = false; + info->always_alloc = false; + if (info->column_size == 0) { + info->can_bind = false; + SetInfoAndBreak(SQL_C_BINARY, 0, GetBinary); } else { - cInfo->buf_size = cInfo->column_size + 1; + SetInfoAndBreak(SQL_C_BINARY, info->column_size + 1, GetBinary); } - cInfo->GetData = GetBinary; - cInfo->always_alloc = false; - break; case SQL_DECIMAL: case SQL_NUMERIC: @@ -685,73 +653,44 @@ bool FetchBufferInfo(Cursor* cur, Py_ssize_t iCol) // I'm going to request the data as Unicode in case there is a weird currency symbol. If // this is a performance problems we may want a flag on this. - cInfo->enc = &cur->cnxn->sqlwchar_enc; - cInfo->c_type = cInfo->enc->ctype; - // need to add padding for all kinds of situations, see GetDataDecimal - cInfo->buf_size = CharBufferSize(cInfo->column_size + 5); - cInfo->GetData = GetDataDecimal; - break; + info->enc = &cur->cnxn->sqlwchar_enc; + SetInfoAndBreak(info->enc->ctype, CharBufferSize(info->column_size + 5), GetDataDecimal); case SQL_BIT: - cInfo->c_type = SQL_C_BIT; - cInfo->buf_size = sizeof(SQLCHAR); - cInfo->GetData = GetDataBit; - break; + SetInfoAndBreak(SQL_C_BIT, sizeof(SQLCHAR), GetDataBit); case SQL_TINYINT: case SQL_SMALLINT: case SQL_INTEGER: - cInfo->c_type = cInfo->is_unsigned ? SQL_C_ULONG : SQL_C_LONG; - cInfo->buf_size = sizeof(SQLINTEGER); - cInfo->GetData = GetDataLong; - break; + SetInfoAndBreak(info->is_unsigned ? SQL_C_ULONG : SQL_C_LONG, sizeof(SQLINTEGER), GetDataLong); case SQL_BIGINT: - if (cInfo->is_unsigned) { - cInfo->c_type = SQL_C_UBIGINT; - cInfo->buf_size = sizeof(SQLBIGINT); - cInfo->GetData = GetDataULongLong; + if (info->is_unsigned) { + SetInfoAndBreak(SQL_C_UBIGINT, sizeof(SQLBIGINT), GetDataULongLong); } else { - cInfo->c_type = SQL_C_SBIGINT; - cInfo->buf_size = sizeof(SQLBIGINT); - cInfo->GetData = GetDataLongLong; + SetInfoAndBreak(SQL_C_SBIGINT, sizeof(SQLBIGINT), GetDataLongLong); } - break; case SQL_REAL: case SQL_FLOAT: case SQL_DOUBLE: - cInfo->c_type = SQL_C_DOUBLE; - cInfo->buf_size = sizeof(double); - cInfo->GetData = GetDataDouble; - break; + SetInfoAndBreak(SQL_C_DOUBLE, sizeof(double), GetDataDouble); case SQL_TYPE_DATE: - cInfo->c_type = SQL_C_TYPE_TIMESTAMP; - cInfo->buf_size = sizeof(TIMESTAMP_STRUCT); - cInfo->GetData = GetDataDate; - break; + SetInfoAndBreak(SQL_C_TYPE_TIMESTAMP, sizeof(TIMESTAMP_STRUCT), GetDataDate); case SQL_TYPE_TIME: - cInfo->c_type = SQL_C_TYPE_TIMESTAMP; - cInfo->buf_size = sizeof(TIMESTAMP_STRUCT); - cInfo->GetData = GetDataTime; - break; + SetInfoAndBreak(SQL_C_TYPE_TIMESTAMP, sizeof(TIMESTAMP_STRUCT), GetDataTime); case SQL_TYPE_TIMESTAMP: - cInfo->c_type = SQL_C_TYPE_TIMESTAMP; - cInfo->buf_size = sizeof(TIMESTAMP_STRUCT); - cInfo->GetData = GetDataTimestamp; - break; + SetInfoAndBreak(SQL_C_TYPE_TIMESTAMP, sizeof(TIMESTAMP_STRUCT), GetDataTimestamp); case SQL_SS_TIME2: - cInfo->c_type = SQL_C_BINARY; - cInfo->buf_size = sizeof(SQL_SS_TIME2_STRUCT); - cInfo->GetData = GetSqlServerTime; - break; + SetInfoAndBreak(SQL_C_BINARY, sizeof(SQL_SS_TIME2_STRUCT), GetSqlServerTime); + default: RaiseErrorV("HY106", ProgrammingError, "ODBC SQL type %d is not yet supported. column-index=%zd type=%d", - (int)cInfo->sql_type, iCol, (int)cInfo->sql_type); + (int)info->sql_type, iCol, (int)info->sql_type); return false; } return true; diff --git a/src/getdata.h b/src/getdata.h index d587e342..9e8dfa5a 100644 --- a/src/getdata.h +++ b/src/getdata.h @@ -13,6 +13,6 @@ bool FetchBufferInfo(Cursor* cur, Py_ssize_t iCol); * If this sql type has a user-defined conversion, the index into the connection's `conv_funcs` array is returned. * Otherwise -1 is returned. */ -// int GetUserConvIndex(Cursor* cur, SQLSMALLINT sql_type); +int GetUserConvIndex(Cursor* cur, SQLSMALLINT sql_type); #endif // _GETDATA_H_ diff --git a/src/textenc.cpp b/src/textenc.cpp index fed2eed7..725265f0 100644 --- a/src/textenc.cpp +++ b/src/textenc.cpp @@ -89,7 +89,7 @@ PyObject* TextEnc::Encode(PyObject* obj) const -PyObject* TextBufferToObject(const TextEnc* enc, const byte* pbData, Py_ssize_t cbData) +PyObject* TextBufferToObject(const TextEnc& enc, const byte* pbData, Py_ssize_t cbData) { // cbData // The length of data in bytes (cb == 'count of bytes'). @@ -104,7 +104,7 @@ PyObject* TextBufferToObject(const TextEnc* enc, const byte* pbData, Py_ssize_t if (cbData == 0) return PyUnicode_FromStringAndSize("", 0); - switch (enc->optenc) + switch (enc.optenc) { case OPTENC_UTF8: return PyUnicode_DecodeUTF8((char*)pbData, cbData, "strict"); @@ -129,5 +129,5 @@ PyObject* TextBufferToObject(const TextEnc* enc, const byte* pbData, Py_ssize_t } // The user set an encoding by name. - return PyUnicode_Decode((char*)pbData, cbData, enc->name, "strict"); + return PyUnicode_Decode((char*)pbData, cbData, enc.name, "strict"); } diff --git a/src/textenc.h b/src/textenc.h index 4b0f7add..19214023 100644 --- a/src/textenc.h +++ b/src/textenc.h @@ -135,7 +135,7 @@ class SQLWChar }; -PyObject* TextBufferToObject(const TextEnc* enc, const byte* p, Py_ssize_t len); +PyObject* TextBufferToObject(const TextEnc& enc, const byte* p, Py_ssize_t len); // Convert a text buffer to a Python object using the given encoding. // // - pbData :: The buffer, which is an array of SQLCHAR or SQLWCHAR. We treat it as bytes here From b2be6a9cf7c4ce20636b5afde4742b6c8ffdad43 Mon Sep 17 00:00:00 2001 From: ffelixg <142172984+ffelixg@users.noreply.github.com> Date: Sun, 19 May 2024 01:03:47 +0200 Subject: [PATCH 21/21] Finalize column rebinding - Track encoding ctypes and trigger rebinding on changes - Prevent losing rows when switching from fetchmany to fetchone and then triggering a rebind - Add tests for the above - Avoid unnecessary dict copy when checking if rebind is necessary - Minor improvements to diagnostic variables and error handling --- src/cursor.cpp | 99 ++++++++++++++++++++++------------------- src/cursor.h | 10 ++++- tests/sqlserver_test.py | 66 +++++++++++++++++---------- 3 files changed, 105 insertions(+), 70 deletions(-) diff --git a/src/cursor.cpp b/src/cursor.cpp index 9a8f9916..e123f289 100644 --- a/src/cursor.cpp +++ b/src/cursor.cpp @@ -371,15 +371,6 @@ static bool BindCols(Cursor* cur, int cCols, int fetch_rows_cap) } } - if (cur->bind_byte_cap < 0) { - PyErr_SetString(ProgrammingError, "Cursor attribute bind_byte_cap must be non negative."); - return 0; - } - if (cur->bind_cell_cap < 0) { - PyErr_SetString(ProgrammingError, "Cursor attribute bind_cell_cap must be non negative."); - return 0; - } - // number of rows to be fetched at a time cur->fetch_buffer_width = total_buf_size; if (bind_all) { @@ -396,14 +387,15 @@ static bool BindCols(Cursor* cur, int cCols, int fetch_rows_cap) } else { cur->fetch_buffer_length = 1; } + cur->fetch_buffer_length_used = 0; // single large buffer using row-wise layout with row status array at the end - void* buf = PyMem_Malloc((cur->fetch_buffer_width + sizeof(SQLUSMALLINT)) * cur->fetch_buffer_length); - if (!buf) { + cur->fetch_buffer = PyMem_Malloc((cur->fetch_buffer_width + sizeof(SQLUSMALLINT)) * cur->fetch_buffer_length); + if (!cur->fetch_buffer) { PyErr_NoMemory(); return false; } - cur->row_status_array = (SQLUSMALLINT*)((uintptr_t)buf + cur->fetch_buffer_width * cur->fetch_buffer_length); + cur->row_status_array = (SQLUSMALLINT*)((uintptr_t)cur->fetch_buffer + cur->fetch_buffer_width * cur->fetch_buffer_length); cur->current_row = 0; SQLRETURN ret_bind = SQL_SUCCESS, ret_attr; @@ -417,10 +409,6 @@ static bool BindCols(Cursor* cur, int cCols, int fetch_rows_cap) if (!SQL_SUCCEEDED(ret_attr)) { goto skip; } - ret_attr = SQLSetStmtAttr(cur->hstmt, SQL_ATTR_ROW_ARRAY_SIZE, (SQLPOINTER)cur->fetch_buffer_length, 0); - if (!SQL_SUCCEEDED(ret_attr)) { - goto skip; - } // TODO use for error checking ret_attr = SQLSetStmtAttr(cur->hstmt, SQL_ATTR_ROW_STATUS_PTR, cur->row_status_array, 0); if (!SQL_SUCCEEDED(ret_attr)) { @@ -440,9 +428,9 @@ static bool BindCols(Cursor* cur, int cCols, int fetch_rows_cap) cur->hstmt, (SQLUSMALLINT)(iCol+1), cInfo->c_type, - (void*)((uintptr_t)buf + cInfo->buf_offset), + (void*)((uintptr_t)cur->fetch_buffer + cInfo->buf_offset), cInfo->buf_size, - (SQLLEN*)((uintptr_t)buf + cInfo->buf_offset - sizeof(SQLLEN)) + (SQLLEN*)((uintptr_t)cur->fetch_buffer + cInfo->buf_offset - sizeof(SQLLEN)) ); if (!SQL_SUCCEEDED(ret_bind)) { break; @@ -461,13 +449,9 @@ static bool BindCols(Cursor* cur, int cCols, int fetch_rows_cap) } if (!SQL_SUCCEEDED(ret_bind)) { - PyMem_Free(buf); return RaiseErrorFromHandle(cur->cnxn, "SQLBindCol", cur->cnxn->hdbc, cur->hstmt); } - assert(cur->fetch_buffer == 0); - cur->fetch_buffer = buf; - return true; } @@ -477,29 +461,21 @@ static bool PrepareFetch(Cursor* cur, int n_rows) // Returns false on exception, true otherwise. // Need to do this because the API allows changing this after executing a statement. - PyObject* converted_types = 0; bool native_uuid = UseNativeUUID(); bool converted_types_changed = false; if (cur->cnxn->map_sqltype_to_converter) { - converted_types = PyDict_Copy(cur->cnxn->map_sqltype_to_converter); - if (!converted_types) - { - return false; - } if (cur->bound_converted_types) { - switch (PyObject_RichCompareBool(cur->bound_converted_types, converted_types, Py_EQ)) + switch (PyObject_RichCompareBool(cur->bound_converted_types, cur->cnxn->map_sqltype_to_converter, Py_EQ)) { case -1: // error - Py_DECREF(converted_types); return false; case 0: // not equal converted_types_changed = true; break; case 1: // equal - Py_DECREF(converted_types); break; } } @@ -513,11 +489,22 @@ static bool PrepareFetch(Cursor* cur, int n_rows) converted_types_changed = true; } - if (cur->bound_native_uuid != native_uuid || converted_types_changed || !cur->fetch_buffer) + if ( + !cur->fetch_buffer || cur->bound_native_uuid != native_uuid || converted_types_changed || + cur->ctype_of_char_enc != cur->cnxn->sqlchar_enc.ctype || cur->ctype_of_wchar_enc != cur->cnxn->sqlwchar_enc.ctype + ) { - Py_XDECREF(cur->bound_converted_types); + Py_CLEAR(cur->bound_converted_types); + PyObject* converted_types = 0; + if (cur->cnxn->map_sqltype_to_converter) { + converted_types = PyDict_Copy(cur->cnxn->map_sqltype_to_converter); + if (!converted_types) + return false; + } cur->bound_converted_types = converted_types; cur->bound_native_uuid = native_uuid; + cur->ctype_of_char_enc = cur->cnxn->sqlchar_enc.ctype; + cur->ctype_of_wchar_enc = cur->cnxn->sqlwchar_enc.ctype; if (cur->description != Py_None) { @@ -525,6 +512,7 @@ static bool PrepareFetch(Cursor* cur, int n_rows) BindColsFree(cur); if (!BindCols(cur, cCols, n_rows)) { + BindColsFree(cur); return false; } } @@ -552,7 +540,7 @@ static bool free_results(Cursor* self, int flags) } BindColsFree(self); - Py_XDECREF(self->bound_converted_types); + Py_CLEAR(self->bound_converted_types); if (self->colinfos) { @@ -1378,7 +1366,7 @@ static PyObject* Cursor_setinputsizes(PyObject* self, PyObject* sizes) Py_RETURN_NONE; } -static PyObject* Cursor_fetch(Cursor* cur) +static PyObject* Cursor_fetch(Cursor* cur, Py_ssize_t max) { // Internal function to fetch a single row and construct a Row object from it. Used by all of the fetching // functions. @@ -1387,15 +1375,33 @@ static PyObject* Cursor_fetch(Cursor* cur) // exception is set and zero is returned. (To differentiate between the last two, use PyErr_Occurred.) SQLRETURN ret = 0; + SQLRETURN ret_attr = 0; Py_ssize_t field_count, i; PyObject** apValues; // One fetch per cycle. if (cur->current_row == 0) { Py_BEGIN_ALLOW_THREADS - ret = SQLFetch(cur->hstmt); + // Make sure no more rows are fetched than are requested by fetchone/fetchmany. + // Otherwise rows might get lost if buffers need to be rebound between fetches. + long fetch_buffer_length_used; + if (max >= 0 && (long)max < cur->fetch_buffer_length) { + fetch_buffer_length_used = (long)max; + } else { + fetch_buffer_length_used = cur->fetch_buffer_length; + } + if (cur->fetch_buffer_length_used != fetch_buffer_length_used) { + cur->fetch_buffer_length_used = fetch_buffer_length_used; + ret_attr = SQLSetStmtAttr(cur->hstmt, SQL_ATTR_ROW_ARRAY_SIZE, (SQLPOINTER)cur->fetch_buffer_length_used, 0); + } + if (SQL_SUCCEEDED(ret_attr)) + ret = SQLFetch(cur->hstmt); Py_END_ALLOW_THREADS + if (!SQL_SUCCEEDED(ret_attr)) { + return RaiseErrorFromHandle(cur->cnxn, "SQLSetStmtAttr", cur->cnxn->hdbc, cur->hstmt); + } + if (cur->cnxn->hdbc == SQL_NULL_HANDLE) { // The connection was closed by another thread in the ALLOW_THREADS block above. @@ -1408,7 +1414,7 @@ static PyObject* Cursor_fetch(Cursor* cur) if (!SQL_SUCCEEDED(ret)) return RaiseErrorFromHandle(cur->cnxn, "SQLFetch", cur->cnxn->hdbc, cur->hstmt); } else { - if (cur->current_row >= cur->rows_fetched) { + if (cur->current_row >= (long)cur->rows_fetched) { return 0; } } @@ -1432,7 +1438,7 @@ static PyObject* Cursor_fetch(Cursor* cur) apValues[i] = value; } - cur->current_row = (cur->current_row + 1) % cur->fetch_buffer_length; + cur->current_row = (cur->current_row + 1) % cur->fetch_buffer_length_used; return (PyObject*)Row_InternalNew(cur->description, cur->map_name_to_index, field_count, apValues); } @@ -1454,7 +1460,7 @@ static PyObject* Cursor_fetchlist(Cursor* cur, Py_ssize_t max) while (max == -1 || max > 0) { - row = Cursor_fetch(cur); + row = Cursor_fetch(cur, max); if (!row) { @@ -1496,7 +1502,7 @@ static PyObject* Cursor_iternext(PyObject* self) if (!cursor || !PrepareFetch(cursor, 1)) return 0; - result = Cursor_fetch(cursor); + result = Cursor_fetch(cursor, 1); return result; } @@ -1509,7 +1515,7 @@ static PyObject* Cursor_fetchval(PyObject* self, PyObject* args) if (!cursor || !PrepareFetch(cursor, 1)) return 0; - Object row(Cursor_fetch(cursor)); + Object row(Cursor_fetch(cursor, 1)); if (!row) { @@ -1530,7 +1536,7 @@ static PyObject* Cursor_fetchone(PyObject* self, PyObject* args) if (!cursor || !PrepareFetch(cursor, 1)) return 0; - row = Cursor_fetch(cursor); + row = Cursor_fetch(cursor, 1); if (!row) { @@ -2472,8 +2478,8 @@ static PyMemberDef Cursor_members[] = {"connection", T_OBJECT_EX, offsetof(Cursor, cnxn), READONLY, connection_doc }, {"fast_executemany",T_BOOL, offsetof(Cursor, fastexecmany), 0, fastexecmany_doc }, {"messages", T_OBJECT_EX, offsetof(Cursor, messages), READONLY, messages_doc }, - {"bound_columns_count", T_UINT, offsetof(Cursor, bound_columns_count), READONLY, bound_columns_count_doc }, - {"bound_buffer_rows", T_ULONG, offsetof(Cursor, fetch_buffer_length), READONLY, bound_buffer_rows_doc }, + {"bound_columns_count", T_INT, offsetof(Cursor, bound_columns_count), READONLY, bound_columns_count_doc }, + {"bound_buffer_rows", T_LONG, offsetof(Cursor, fetch_buffer_length), READONLY, bound_buffer_rows_doc }, {"bind_cell_cap", T_LONG, offsetof(Cursor, bind_cell_cap), 0, bind_cell_cap_doc }, {"bind_byte_cap", T_LONG, offsetof(Cursor, bind_byte_cap), 0, bind_byte_cap_doc }, { 0 } @@ -2764,9 +2770,12 @@ Cursor_New(Connection* cnxn) cur->fetch_buffer = 0; cur->bound_converted_types = 0; cur->bound_native_uuid = 0; + cur->ctype_of_char_enc = SQL_C_CHAR; + cur->ctype_of_wchar_enc = SQL_C_WCHAR; cur->bind_cell_cap = 10; cur->bind_byte_cap = 20 * 1024 * 1024; - cur->bound_columns_count = 0; + cur->bound_columns_count = -1; // should indicate that it's not initialized yet + cur->fetch_buffer_length = -1; Py_INCREF(cnxn); Py_INCREF(cur->description); diff --git a/src/cursor.h b/src/cursor.h index 3762cb61..9050225c 100644 --- a/src/cursor.h +++ b/src/cursor.h @@ -40,6 +40,7 @@ struct ColumnInfo bool is_bound; bool can_bind; bool always_alloc; + // No need to do refcounting no the converter, since at least cur->bound_converted_types will have one. PyObject* converter; TextEnc* enc; }; @@ -171,15 +172,20 @@ struct Cursor void* fetch_buffer; long fetch_buffer_width; long fetch_buffer_length; - long rows_fetched; + long fetch_buffer_length_used; + SQLULEN rows_fetched; SQLUSMALLINT* row_status_array; long current_row; // Track the configuration at the time of using SQLBindCol. bool bound_native_uuid; PyObject* bound_converted_types; + // Only track the ctype of cur->cnxn->sql(w)char_enc. Changing any other attribute of the encoding + // would not change the binding process. + SQLSMALLINT ctype_of_wchar_enc; + SQLSMALLINT ctype_of_char_enc; - unsigned int bound_columns_count; + int bound_columns_count; long bind_cell_cap; long bind_byte_cap; }; diff --git a/tests/sqlserver_test.py b/tests/sqlserver_test.py index 991f4fcb..e75cf683 100755 --- a/tests/sqlserver_test.py +++ b/tests/sqlserver_test.py @@ -1258,36 +1258,56 @@ def convert(value): return value cnxn = connect() - cursor = cnxn.cursor() uidstr = 'CB4BB7F2-3AD9-4ED7-ABB8-7C704D75335C' uid = uuid.UUID(uidstr) uidbytes = b'\xf2\xb7K\xcb\xd9:\xd7N\xab\xb8|pMu3\\' - cursor.execute("drop table if exists t1") - cursor.execute("create table t1(g uniqueidentifier)") - for i in range(4): - cursor.execute(f"insert into t1 values (?)", (uid,)) - - cursor.execute("select g from t1") - - pyodbc.native_uuid = False - v, = cursor.fetchone() - assert v == uidstr - - cnxn.add_output_converter(pyodbc.SQL_GUID, convert) - v, = cursor.fetchone() - assert v == uidbytes - cnxn.remove_output_converter(pyodbc.SQL_GUID) + with cnxn: + cursor = cnxn.cursor() + cursor.execute("drop table if exists t1") + cursor.execute("create table t1(g uniqueidentifier)") + for i in range(6): + cursor.execute("insert into t1 values (?)", (uid,)) + + cursor.execute("select g from t1") + + pyodbc.native_uuid = False + v, = cursor.fetchone() + assert v == uidstr + + cnxn.add_output_converter(pyodbc.SQL_GUID, convert) + v, = cursor.fetchone() + assert v == uidbytes + cnxn.remove_output_converter(pyodbc.SQL_GUID) + + pyodbc.native_uuid = True + v, = cursor.fetchone() + assert v == uid + + pyodbc.native_uuid = False + v, = cursor.fetchone() + assert v == uidstr + + cnxn.setdecoding(pyodbc.SQL_CHAR, encoding='utf-16-le') + v, = cursor.fetchone() # fetches into SQL_C_WCHAR buffer + assert v == uidstr + cnxn.setdecoding(pyodbc.SQL_CHAR, encoding='cp1252') + v, = cursor.fetchone() # fetches into SQL_C_CHAR buffer + assert v == uidstr + + cursor.close() + cursor = cnxn.cursor() + cursor.execute("select 1 union select 2 union select 3 union select 4") - pyodbc.native_uuid = True - v, = cursor.fetchone() - assert v == uid + cursor.fetchmany(2) # should fetch 2 rows per SQLFetch call with rows left over + v, = cursor.fetchone() # even though 2 rows are allocated, only one should be used so we can rebind witout discarding data + assert v == 3 + pyodbc.native_uuid = not pyodbc.native_uuid # force rebind + v, = cursor.fetchone() + assert v == 4 - pyodbc.native_uuid = False - v, = cursor.fetchone() - assert v == uidstr - pyodbc.native_uuid = True + pyodbc.native_uuid = True def test_too_large(cursor: pyodbc.Cursor):