From 5673faccea931883944fcd8f134348d6bc04d1dd Mon Sep 17 00:00:00 2001 From: Oleg Broytman Date: Sun, 4 Dec 2022 11:25:29 +0300 Subject: [PATCH 1/8] Fix: Always close cursor, even in case of errors --- sqlobject/dbconnection.py | 30 +++++--- sqlobject/firebird/firebirdconnection.py | 22 +++--- sqlobject/inheritance/iteration.py | 57 ++++++++------- sqlobject/manager/command.py | 7 +- sqlobject/maxdb/maxdbconnection.py | 26 +++---- sqlobject/mssql/mssqlconnection.py | 89 +++++++++++++----------- sqlobject/mysql/mysqlconnection.py | 36 +++++----- sqlobject/postgres/pgconnection.py | 74 ++++++++++---------- sqlobject/sqlite/sqliteconnection.py | 24 ++++--- sqlobject/sybase/sybaseconnection.py | 58 ++++++++------- 10 files changed, 233 insertions(+), 190 deletions(-) diff --git a/sqlobject/dbconnection.py b/sqlobject/dbconnection.py index 6a383e79..dd130f2a 100644 --- a/sqlobject/dbconnection.py +++ b/sqlobject/dbconnection.py @@ -428,8 +428,10 @@ def _query(self, conn, s): if self.debug: self.printDebug(conn, s, 'Query') c = conn.cursor() - self._executeRetry(conn, c, s) - c.close() + try: + self._executeRetry(conn, c, s) + finally: + c.close() def query(self, s): return self._runWithConnection(self._query, s) @@ -438,9 +440,11 @@ def _queryAll(self, conn, s): if self.debug: self.printDebug(conn, s, 'QueryAll') c = conn.cursor() - self._executeRetry(conn, c, s) - value = c.fetchall() - c.close() + try: + self._executeRetry(conn, c, s) + value = c.fetchall() + finally: + c.close() if self.debugOutput: self.printDebug(conn, value, 'QueryAll', 'result') return value @@ -456,9 +460,11 @@ def _queryAllDescription(self, conn, s): if self.debug: self.printDebug(conn, s, 'QueryAllDesc') c = conn.cursor() - self._executeRetry(conn, c, s) - value = c.fetchall() - c.close() + try: + self._executeRetry(conn, c, s) + value = c.fetchall() + finally: + c.close() if self.debugOutput: self.printDebug(conn, value, 'QueryAll', 'result') return c.description, value @@ -470,9 +476,11 @@ def _queryOne(self, conn, s): if self.debug: self.printDebug(conn, s, 'QueryOne') c = conn.cursor() - self._executeRetry(conn, c, s) - value = c.fetchone() - c.close() + try: + self._executeRetry(conn, c, s) + value = c.fetchone() + finally: + c.close() if self.debugOutput: self.printDebug(conn, value, 'QueryOne', 'result') return value diff --git a/sqlobject/firebird/firebirdconnection.py b/sqlobject/firebird/firebirdconnection.py index 4624b487..051b9648 100644 --- a/sqlobject/firebird/firebirdconnection.py +++ b/sqlobject/firebird/firebirdconnection.py @@ -126,16 +126,18 @@ def _queryInsertID(self, conn, soInstance, id, names, values): idName = soInstance.sqlmeta.idName sequenceName = soInstance.sqlmeta.idSequence or 'GEN_%s' % table c = conn.cursor() - if id is None: - c.execute('SELECT gen_id(%s,1) FROM rdb$database' % sequenceName) - id = c.fetchone()[0] - names = [idName] + names - values = [id] + values - q = self._insertSQL(table, names, values) - if self.debug: - self.printDebug(conn, q, 'QueryIns') - c.execute(q) - c.close() + try: + if id is None: + c.execute('SELECT gen_id(%s,1) FROM rdb$database' % sequenceName) + id = c.fetchone()[0] + names = [idName] + names + values = [id] + values + q = self._insertSQL(table, names, values) + if self.debug: + self.printDebug(conn, q, 'QueryIns') + c.execute(q) + finally: + c.close() if self.debugOutput: self.printDebug(conn, id, 'QueryIns', 'result') return id diff --git a/sqlobject/inheritance/iteration.py b/sqlobject/inheritance/iteration.py index 558700ea..25edbce8 100644 --- a/sqlobject/inheritance/iteration.py +++ b/sqlobject/inheritance/iteration.py @@ -77,30 +77,33 @@ def fetchChildren(self): dbconn = self.dbconn rawconn = self.rawconn cursor = rawconn.cursor() - registry = self.select.sourceClass.sqlmeta.registry - for childName, ids in childIdsNames.items(): - klass = findClass(childName, registry) - if len(ids) == 1: - select = klass.select(klass.q.id == ids[0], - childUpdate=True, connection=dbconn) - else: - select = klass.select(sqlbuilder.IN(klass.q.id, ids), - childUpdate=True, connection=dbconn) - query = dbconn.queryForSelect(select) - if dbconn.debug: - dbconn.printDebug(rawconn, query, - 'Select children of the class %s' % - childName) - self.dbconn._executeRetry(rawconn, cursor, query) - for result in cursor.fetchall(): - # Inheritance child classes may have no own columns - # (that makes sense when child class has a join - # that does not apply to parent class objects). - # In such cases result[1:] gives an empty tuple - # which is interpreted as "no results fetched" in .get(). - # So .get() issues another query which is absolutely - # meaningless (like "SELECT NULL FROM child WHERE id=1"). - # In order to avoid this, we replace empty results - # with non-empty tuple. Extra values in selectResults - # are Ok - they will be ignored by ._SO_selectInit(). - self._childrenResults[result[0]] = result[1:] or (None,) + try: + registry = self.select.sourceClass.sqlmeta.registry + for childName, ids in childIdsNames.items(): + klass = findClass(childName, registry) + if len(ids) == 1: + select = klass.select(klass.q.id == ids[0], + childUpdate=True, connection=dbconn) + else: + select = klass.select(sqlbuilder.IN(klass.q.id, ids), + childUpdate=True, connection=dbconn) + query = dbconn.queryForSelect(select) + if dbconn.debug: + dbconn.printDebug(rawconn, query, + 'Select children of the class %s' % + childName) + self.dbconn._executeRetry(rawconn, cursor, query) + for result in cursor.fetchall(): + # Inheritance child classes may have no own columns + # (that makes sense when child class has a join + # that does not apply to parent class objects). + # In such cases result[1:] gives an empty tuple + # which is interpreted as "no results fetched" in .get(). + # So .get() issues another query which is absolutely + # meaningless (like "SELECT NULL FROM child WHERE id=1"). + # In order to avoid this, we replace empty results + # with non-empty tuple. Extra values in selectResults + # are Ok - they will be ignored by ._SO_selectInit(). + self._childrenResults[result[0]] = result[1:] or (None,) + finally: + cursor.close() diff --git a/sqlobject/manager/command.py b/sqlobject/manager/command.py index fc8db834..97923591 100755 --- a/sqlobject/manager/command.py +++ b/sqlobject/manager/command.py @@ -869,8 +869,11 @@ def command(self): args.append(sys.stdin.read()) self.conn = self.connection().getConnection() self.cursor = self.conn.cursor() - for sql in args: - self.execute_sql(sql) + try: + for sql in args: + self.execute_sql(sql) + finally: + self.cursor.close() def execute_sql(self, sql): if self.options.verbose: diff --git a/sqlobject/maxdb/maxdbconnection.py b/sqlobject/maxdb/maxdbconnection.py index 4fd53b1b..4720e440 100644 --- a/sqlobject/maxdb/maxdbconnection.py +++ b/sqlobject/maxdb/maxdbconnection.py @@ -127,18 +127,20 @@ def _queryInsertID(self, conn, soInstance, id, names, values): table = soInstance.sqlmeta.table idName = soInstance.sqlmeta.idName c = conn.cursor() - if id is None: - c.execute( - 'SELECT %s.NEXTVAL FROM DUAL' % ( - self.createSequenceName(table))) - id = c.fetchone()[0] - names = [idName] + names - values = [id] + values - q = self._insertSQL(table, names, values) - if self.debug: - self.printDebug(conn, q, 'QueryIns') - c.execute(q) - c.close() + try: + if id is None: + c.execute( + 'SELECT %s.NEXTVAL FROM DUAL' % ( + self.createSequenceName(table))) + id = c.fetchone()[0] + names = [idName] + names + values = [id] + values + q = self._insertSQL(table, names, values) + if self.debug: + self.printDebug(conn, q, 'QueryIns') + c.execute(q) + finally: + c.close() if self.debugOutput: self.printDebug(conn, id, 'QueryIns', 'result') return id diff --git a/sqlobject/mssql/mssqlconnection.py b/sqlobject/mssql/mssqlconnection.py index b135fe44..d439543e 100644 --- a/sqlobject/mssql/mssqlconnection.py +++ b/sqlobject/mssql/mssqlconnection.py @@ -134,11 +134,13 @@ def insert_id(self, conn): insert_id method. """ c = conn.cursor() - # converting the identity to an int is ugly, but it gets returned - # as a decimal otherwise :S - c.execute('SELECT CONVERT(INT, @@IDENTITY)') - result = c.fetchone()[0] - c.close() + try: + # converting the identity to an int is ugly, but it gets returned + # as a decimal otherwise :S + c.execute('SELECT CONVERT(INT, @@IDENTITY)') + result = c.fetchone()[0] + finally: + c.close() return result def makeConnection(self): @@ -160,10 +162,12 @@ def makeConnection(self): else: conn = self.dbconnection(conn_descr) cur = conn.cursor() - cur.execute('SET ANSI_NULLS ON') - cur.execute("SELECT CAST('12345.21' AS DECIMAL(10, 2))") - self.decimalSeparator = str(cur.fetchone()[0])[-3] - cur.close() + try: + cur.execute('SET ANSI_NULLS ON') + cur.execute("SELECT CAST('12345.21' AS DECIMAL(10, 2))") + self.decimalSeparator = str(cur.fetchone()[0])[-3] + finally: + cur.close() return conn def _setAutoCommit(self, conn, auto): @@ -174,7 +178,10 @@ def _setAutoCommit(self, conn, auto): else: option = "OFF" c = conn.cursor() - c.execute("SET AUTOCOMMIT " + option) + try: + c.execute("SET AUTOCOMMIT " + option) + finally: + c.close() elif self.driver == 'pymssql': conn.autocommit(auto) elif self.driver in ('odbc', 'pyodbc', 'pypyodbc'): @@ -190,9 +197,11 @@ def _setAutoCommit(self, conn, auto): def _hasIdentity(self, conn, table): query = self.HAS_IDENTITY % table c = conn.cursor() - c.execute(query) - r = c.fetchone() - c.close() + try: + c.execute(query) + r = c.fetchone() + finally: + c.close() return r is not None def _queryInsertID(self, conn, soInstance, id, names, values): @@ -202,35 +211,37 @@ def _queryInsertID(self, conn, soInstance, id, names, values): table = soInstance.sqlmeta.table idName = soInstance.sqlmeta.idName c = conn.cursor() - has_identity = self._hasIdentity(conn, table) - if id is not None: - names = [idName] + names - values = [id] + values - elif has_identity and idName in names: - try: - i = names.index(idName) - if i: - del names[i] - del values[i] - except ValueError: - pass - - if has_identity: + try: + has_identity = self._hasIdentity(conn, table) if id is not None: - c.execute('SET IDENTITY_INSERT %s ON' % table) + names = [idName] + names + values = [id] + values + elif has_identity and idName in names: + try: + i = names.index(idName) + if i: + del names[i] + del values[i] + except ValueError: + pass + + if has_identity: + if id is not None: + c.execute('SET IDENTITY_INSERT %s ON' % table) + else: + c.execute('SET IDENTITY_INSERT %s OFF' % table) + + if names and values: + q = self._insertSQL(table, names, values) else: + q = "INSERT INTO %s DEFAULT VALUES" % table + if self.debug: + self.printDebug(conn, q, 'QueryIns') + c.execute(q) + if has_identity: c.execute('SET IDENTITY_INSERT %s OFF' % table) - - if names and values: - q = self._insertSQL(table, names, values) - else: - q = "INSERT INTO %s DEFAULT VALUES" % table - if self.debug: - self.printDebug(conn, q, 'QueryIns') - c.execute(q) - if has_identity: - c.execute('SET IDENTITY_INSERT %s OFF' % table) - c.close() + finally: + c.close() if id is None: id = self.insert_id(conn) diff --git a/sqlobject/mysql/mysqlconnection.py b/sqlobject/mysql/mysqlconnection.py index c4a72a6e..2dabe999 100644 --- a/sqlobject/mysql/mysqlconnection.py +++ b/sqlobject/mysql/mysqlconnection.py @@ -313,25 +313,27 @@ def _queryInsertID(self, conn, soInstance, id, names, values): table = soInstance.sqlmeta.table idName = soInstance.sqlmeta.idName c = conn.cursor() - if id is not None: - names = [idName] + names - values = [id] + values - q = self._insertSQL(table, names, values) - if self.debug: - self.printDebug(conn, q, 'QueryIns') - self._executeRetry(conn, c, q) - if id is None: - try: - id = c.lastrowid - except AttributeError: + try: + if id is not None: + names = [idName] + names + values = [id] + values + q = self._insertSQL(table, names, values) + if self.debug: + self.printDebug(conn, q, 'QueryIns') + self._executeRetry(conn, c, q) + if id is None: try: - id = c.insert_id + id = c.lastrowid except AttributeError: - self._executeRetry(conn, c, "SELECT LAST_INSERT_ID();") - id = c.fetchone()[0] - else: - id = c.insert_id() - c.close() + try: + id = c.insert_id + except AttributeError: + self._executeRetry(conn, c, "SELECT LAST_INSERT_ID();") + id = c.fetchone()[0] + else: + id = c.insert_id() + finally: + c.close() if self.debugOutput: self.printDebug(conn, id, 'QueryIns', 'result') return id diff --git a/sqlobject/postgres/pgconnection.py b/sqlobject/postgres/pgconnection.py index 722f3d4e..0f745de2 100644 --- a/sqlobject/postgres/pgconnection.py +++ b/sqlobject/postgres/pgconnection.py @@ -229,21 +229,23 @@ def makeConnection(self): if self.autoCommit: self._setAutoCommit(conn, 1) c = conn.cursor() - if self.schema: - self._executeRetry(conn, c, "SET search_path TO " + self.schema) - dbEncoding = self.dbEncoding - if dbEncoding: - if self.driver in ('odbc', 'pyodbc'): - conn.setdecoding(self.module.SQL_CHAR, encoding=dbEncoding) - conn.setdecoding(self.module.SQL_WCHAR, encoding=dbEncoding) - if PY2: - conn.setencoding(str, encoding=dbEncoding) - conn.setencoding(unicode, encoding=dbEncoding) # noqa - else: - conn.setencoding(encoding=dbEncoding) - self._executeRetry(conn, c, - "SET client_encoding TO '%s'" % dbEncoding) - c.close() + try: + if self.schema: + self._executeRetry(conn, c, "SET search_path TO " + self.schema) + dbEncoding = self.dbEncoding + if dbEncoding: + if self.driver in ('odbc', 'pyodbc'): + conn.setdecoding(self.module.SQL_CHAR, encoding=dbEncoding) + conn.setdecoding(self.module.SQL_WCHAR, encoding=dbEncoding) + if PY2: + conn.setencoding(str, encoding=dbEncoding) + conn.setencoding(unicode, encoding=dbEncoding) # noqa + else: + conn.setencoding(encoding=dbEncoding) + self._executeRetry(conn, c, + "SET client_encoding TO '%s'" % dbEncoding) + finally: + c.close() return conn def _executeRetry(self, conn, cursor, query): @@ -299,26 +301,28 @@ def _queryInsertID(self, conn, soInstance, id, names, values): table = soInstance.sqlmeta.table idName = soInstance.sqlmeta.idName c = conn.cursor() - if id is None and self.driver in ('py-postgresql', 'pypostgresql'): - sequenceName = soInstance.sqlmeta.idSequence or \ - '%s_%s_seq' % (table, idName) - self._executeRetry(conn, c, "SELECT NEXTVAL('%s')" % sequenceName) - id = c.fetchone()[0] - if id is not None: - names = [idName] + names - values = [id] + values - if names and values: - q = self._insertSQL(table, names, values) - else: - q = "INSERT INTO %s DEFAULT VALUES" % table - if id is None: - q += " RETURNING " + idName - if self.debug: - self.printDebug(conn, q, 'QueryIns') - self._executeRetry(conn, c, q) - if id is None: - id = c.fetchone()[0] - c.close() + try: + if id is None and self.driver in ('py-postgresql', 'pypostgresql'): + sequenceName = soInstance.sqlmeta.idSequence or \ + '%s_%s_seq' % (table, idName) + self._executeRetry(conn, c, "SELECT NEXTVAL('%s')" % sequenceName) + id = c.fetchone()[0] + if id is not None: + names = [idName] + names + values = [id] + values + if names and values: + q = self._insertSQL(table, names, values) + else: + q = "INSERT INTO %s DEFAULT VALUES" % table + if id is None: + q += " RETURNING " + idName + if self.debug: + self.printDebug(conn, q, 'QueryIns') + self._executeRetry(conn, c, q) + if id is None: + id = c.fetchone()[0] + finally: + c.close() if self.debugOutput: self.printDebug(conn, id, 'QueryIns', 'result') return id diff --git a/sqlobject/sqlite/sqliteconnection.py b/sqlobject/sqlite/sqliteconnection.py index 06cf3c43..97877e0b 100644 --- a/sqlobject/sqlite/sqliteconnection.py +++ b/sqlobject/sqlite/sqliteconnection.py @@ -253,17 +253,19 @@ def _queryInsertID(self, conn, soInstance, id, names, values): table = soInstance.sqlmeta.table idName = soInstance.sqlmeta.idName c = conn.cursor() - if id is not None: - names = [idName] + names - values = [id] + values - q = self._insertSQL(table, names, values) - if self.debug: - self.printDebug(conn, q, 'QueryIns') - self._executeRetry(conn, c, q) - # lastrowid is a DB-API extension from "PEP 0249": - if id is None: - id = int(c.lastrowid) - c.close() + try: + if id is not None: + names = [idName] + names + values = [id] + values + q = self._insertSQL(table, names, values) + if self.debug: + self.printDebug(conn, q, 'QueryIns') + self._executeRetry(conn, c, q) + # lastrowid is a DB-API extension from "PEP 0249": + if id is None: + id = int(c.lastrowid) + finally: + c.close() if self.debugOutput: self.printDebug(conn, id, 'QueryIns', 'result') return id diff --git a/sqlobject/sybase/sybaseconnection.py b/sqlobject/sybase/sybaseconnection.py index 11f44344..6731f1cb 100644 --- a/sqlobject/sybase/sybaseconnection.py +++ b/sqlobject/sybase/sybaseconnection.py @@ -45,9 +45,11 @@ def insert_id(self, conn): insert_id method. """ c = conn.cursor() - c.execute('SELECT @@IDENTITY') - result = c.fetchone()[0] - c.close() + try: + c.execute('SELECT @@IDENTITY') + result = c.fetchone()[0] + finally: + c.close() return result def makeConnection(self): @@ -68,35 +70,39 @@ def makeConnection(self): def _hasIdentity(self, conn, table): query = self.HAS_IDENTITY % table c = conn.cursor() - c.execute(query) - r = c.fetchone() - c.close() + try: + c.execute(query) + r = c.fetchone() + finally: + c.close() return r is not None def _queryInsertID(self, conn, soInstance, id, names, values): table = soInstance.sqlmeta.table idName = soInstance.sqlmeta.idName c = conn.cursor() - if id is not None: - names = [idName] + names - values = [id] + values - - has_identity = self._hasIdentity(conn, table) - identity_insert_on = False - if has_identity and (id is not None): - identity_insert_on = True - c.execute('SET IDENTITY_INSERT %s ON' % table) - - if names and values: - q = self._insertSQL(table, names, values) - else: - q = "INSERT INTO %s DEFAULT VALUES" % table - if self.debug: - self.printDebug(conn, q, 'QueryIns') - c.execute(q) - if has_identity and identity_insert_on: - c.execute('SET IDENTITY_INSERT %s OFF' % table) - c.close() + try: + if id is not None: + names = [idName] + names + values = [id] + values + + has_identity = self._hasIdentity(conn, table) + identity_insert_on = False + if has_identity and (id is not None): + identity_insert_on = True + c.execute('SET IDENTITY_INSERT %s ON' % table) + + if names and values: + q = self._insertSQL(table, names, values) + else: + q = "INSERT INTO %s DEFAULT VALUES" % table + if self.debug: + self.printDebug(conn, q, 'QueryIns') + c.execute(q) + if has_identity and identity_insert_on: + c.execute('SET IDENTITY_INSERT %s OFF' % table) + finally: + c.close() if id is None: id = self.insert_id(conn) if self.debugOutput: From 55f85966c333f3488e7ca7a912ef09d2bae8b2d0 Mon Sep 17 00:00:00 2001 From: Oleg Broytman Date: Thu, 24 Nov 2022 18:31:03 +0300 Subject: [PATCH 2/8] Refactor(mssqlconnection): Use `self.module` Move ODBC-related code. --- sqlobject/mssql/mssqlconnection.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/sqlobject/mssql/mssqlconnection.py b/sqlobject/mssql/mssqlconnection.py index d439543e..13cb89a1 100644 --- a/sqlobject/mssql/mssqlconnection.py +++ b/sqlobject/mssql/mssqlconnection.py @@ -23,9 +23,11 @@ def __init__(self, db, user, password='', host='localhost', port=None, continue try: if driver in ('adodb', 'adodbapi'): - import adodbapi as sqlmodule + import adodbapi + self.module = adodbapi elif driver == 'pymssql': - import pymssql as sqlmodule + import pymssql + self.module = pymssql elif driver == 'pyodbc': import pyodbc self.module = pyodbc @@ -56,14 +58,7 @@ def __init__(self, db, user, password='', host='localhost', port=None, timeout = int(timeout) self.timeout = timeout - if driver in ('odbc', 'pyodbc', 'pypyodbc'): - self.make_odbc_conn_str(kw.pop('odbcdrv', 'SQL Server'), - db, host, port, user, password - ) - - elif driver in ('adodb', 'adodbapi'): - self.module = sqlmodule - self.dbconnection = sqlmodule.connect + if driver in ('adodb', 'adodbapi'): # ADO uses unicode only (AFAIK) self.usingUnicodeStrings = True @@ -90,9 +85,7 @@ def __init__(self, db, user, password='', host='localhost', port=None, kw.pop("sspi", None) elif driver == 'pymssql': - self.module = sqlmodule - self.dbconnection = sqlmodule.connect - sqlmodule.Binary = lambda st: str(st) + self.module.Binary = lambda st: str(st) # don't know whether pymssql uses unicode self.usingUnicodeStrings = False @@ -110,6 +103,12 @@ def _make_conn_str(keys): keys_dict[attr] = value return keys_dict self.make_conn_str = _make_conn_str + + elif driver in ('odbc', 'pyodbc', 'pypyodbc'): + self.make_odbc_conn_str(kw.pop('odbcdrv', 'SQL Server'), + db, host, port, user, password + ) + self.driver = driver self.autoCommit = int(autoCommit) @@ -158,9 +157,9 @@ def makeConnection(self): else: conn_descr = self.make_conn_str(self) if isinstance(conn_descr, dict): - conn = self.dbconnection(**conn_descr) + conn = self.module.connect(**conn_descr) else: - conn = self.dbconnection(conn_descr) + conn = self.module.connect(conn_descr) cur = conn.cursor() try: cur.execute('SET ANSI_NULLS ON') From 57213a91846acad97526d3b8ab5796b98ca45499 Mon Sep 17 00:00:00 2001 From: Oleg Broytman Date: Thu, 24 Nov 2022 18:37:38 +0300 Subject: [PATCH 3/8] Feat(MSSQL): Use driver `pytds` --- README.rst | 6 +++--- docs/News.rst | 2 ++ docs/SQLObject.rst | 5 +++-- setup.py | 7 ++++--- sqlobject/mssql/mssqlconnection.py | 18 ++++++++++++------ 5 files changed, 24 insertions(+), 14 deletions(-) diff --git a/README.rst b/README.rst index 8c310399..3591a363 100644 --- a/README.rst +++ b/README.rst @@ -10,9 +10,9 @@ SQLObject supports a number of backends: MySQL/MariaDB (with a number of DB API drivers: ``MySQLdb``, ``mysqlclient``, ``mysql-connector``, ``PyMySQL``, ``mariadb``), PostgreSQL (``psycopg2``, ``PyGreSQL``, partially ``pg8000`` and ``py-postgresql``), SQLite (builtin ``sqlite``, -``pysqlite``, partially ``supersqlite``); connections to other backends -- Firebird, Sybase, MSSQL and MaxDB (also known as SAPDB) - are less -debugged). +``pysqlite``, partially ``supersqlite``); MSSQL Server (``pymssql`` or +``pytds``); connections to other backends - Firebird, Sybase and MaxDB +(also known as SAPDB) - are less debugged). Python 2.7 or 3.4+ is required. diff --git a/docs/News.rst b/docs/News.rst index 0eaaa97d..6bb8c303 100644 --- a/docs/News.rst +++ b/docs/News.rst @@ -14,6 +14,8 @@ Minor features * Use ``module_loader.exec_module(module_loader.create_module())`` instead of ``module_loader.load_module()`` when available. +* Use driver ``pytds``. + Tests, CI --------- diff --git a/docs/SQLObject.rst b/docs/SQLObject.rst index 56f25415..772cc352 100644 --- a/docs/SQLObject.rst +++ b/docs/SQLObject.rst @@ -53,8 +53,8 @@ PostgreSQL_ psycopg2_ is recommended; PyGreSQL_, py-postgresql_ and pg8000_ are supported; SQLite_ has a built-in driver, PySQLite_ or supersqlite_. Firebird_ is supported via fdb_ or kinterbasdb_; pyfirebirdsql_ is supported but has problems. `MAX DB`_ (also known as SAP DB) is supported -via sapdb_. Sybase via Sybase_. `MSSQL Server`_ via pymssql_ (+ FreeTDS_) -or adodbapi_ (Win32). PyODBC_ and PyPyODBC_ are supported for MySQL, +via sapdb_. Sybase via Sybase_. `MSSQL Server`_ via pymssql_ (+ FreeTDS_), +`pytds`_ or adodbapi_ (Win32). PyODBC_ and PyPyODBC_ are supported for MySQL, PostgreSQL and MSSQL but have problems (not all tests passed). .. _MySQL: https://www.mysql.com/ @@ -83,6 +83,7 @@ PostgreSQL and MSSQL but have problems (not all tests passed). .. _`MSSQL Server`: http://www.microsoft.com/sql/ .. _pymssql: http://www.pymssql.org/en/latest/index.html .. _FreeTDS: http://www.freetds.org/ +.. _pytds: https://pypi.org/project/python-tds/ .. _adodbapi: http://adodbapi.sourceforge.net/ .. _PyODBC: https://pypi.org/project/pyodbc/ .. _PyPyODBC: https://pypi.org/project/pypyodbc/ diff --git a/setup.py b/setup.py index b67aa317..32c7e42a 100755 --- a/setup.py +++ b/setup.py @@ -110,9 +110,6 @@ 'fdb': ['fdb'], 'firebirdsql': ['firebirdsql'], 'kinterbasdb': ['kinterbasdb'], - # MS SQL - 'adodbapi': ['adodbapi'], - 'pymssql': ['pymssql'], # MySQL 'mysql:python_version=="2.7"': ['MySQL-python'], 'mysql:python_version>="3.4"': ['mysqlclient'], @@ -123,6 +120,10 @@ 'oursql3 @ git+https://github.com/sqlobject/oursql.git@py3k'], 'pymysql': ['pymysql'], 'mariadb': ['mariadb'], + # MS SQL + 'adodbapi': ['adodbapi'], + 'pymssql': ['pymssql'], + 'pytds': ['python-tds'], # ODBC 'odbc': ['pyodbc'], 'pyodbc': ['pyodbc'], diff --git a/sqlobject/mssql/mssqlconnection.py b/sqlobject/mssql/mssqlconnection.py index 13cb89a1..4d2ac35f 100644 --- a/sqlobject/mssql/mssqlconnection.py +++ b/sqlobject/mssql/mssqlconnection.py @@ -16,7 +16,7 @@ class MSSQLConnection(DBAPI): def __init__(self, db, user, password='', host='localhost', port=None, autoCommit=0, **kw): - drivers = kw.pop('driver', None) or 'adodb,pymssql' + drivers = kw.pop('driver', None) or 'adodb,pymssql,pytds' for driver in drivers.split(','): driver = driver.strip() if not driver: @@ -28,6 +28,9 @@ def __init__(self, db, user, password='', host='localhost', port=None, elif driver == 'pymssql': import pymssql self.module = pymssql + elif driver == 'pytds': + import pytds + self.module = pytds elif driver == 'pyodbc': import pyodbc self.module = pyodbc @@ -43,7 +46,7 @@ def __init__(self, db, user, password='', host='localhost', port=None, else: raise ValueError( 'Unknown MSSQL driver "%s", ' - 'expected adodb, pymssql, ' + 'expected adodb, pymssql, pytds, ' 'odbc, pyodbc or pypyodbc' % driver) except ImportError: pass @@ -84,9 +87,10 @@ def __init__(self, db, user, password='', host='localhost', port=None, kw.pop("ncli", None) kw.pop("sspi", None) - elif driver == 'pymssql': + elif driver in ('pymssql', 'pytds'): + self.dbconnection = self.module.connect self.module.Binary = lambda st: str(st) - # don't know whether pymssql uses unicode + # don't know whether pymssql/pytds use unicode self.usingUnicodeStrings = False def _make_conn_str(keys): @@ -95,7 +99,9 @@ def _make_conn_str(keys): ('database', keys.db), ('user', keys.user), ('password', keys.password), - ('host', keys.host), + ('host', keys.host) + if driver == 'pymssql' else # pytds + ('dsn', keys.host), ('port', keys.port), ('timeout', keys.timeout), ): @@ -183,7 +189,7 @@ def _setAutoCommit(self, conn, auto): c.close() elif self.driver == 'pymssql': conn.autocommit(auto) - elif self.driver in ('odbc', 'pyodbc', 'pypyodbc'): + elif self.driver in ('pytds', 'odbc', 'pyodbc', 'pypyodbc'): conn.autocommit = auto HAS_IDENTITY = """ From 4988fdf775add1d96a57a4cf06a45cb809efc2e8 Mon Sep 17 00:00:00 2001 From: Oleg Broytman Date: Thu, 24 Nov 2022 18:48:01 +0300 Subject: [PATCH 4/8] CI: Run tests with MS SQL at GH Actions --- .github/workflows/run-tests.yaml | 9 +++ .../requirements/requirements_pymssql.txt | 2 + docs/News.rst | 2 + tox.ini | 71 ++++++++++++++----- 4 files changed, 65 insertions(+), 19 deletions(-) create mode 100644 devscripts/requirements/requirements_pymssql.txt diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 7b5c8b45..642afeee 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -36,6 +36,11 @@ jobs: psql --command="CREATE USER runner CREATEDB ENCRYPTED PASSWORD 'test'" if: ${{ runner.os == 'Windows' }} + # Setup MS SQL + - uses: ankane/setup-sqlserver@v1 + with: + accept-eula: true + # Setup Python/pip - uses: actions/checkout@v2 - uses: actions/setup-python@v4 @@ -71,6 +76,8 @@ jobs: devscripts/tox-select-envs $PYVER-mysql devscripts/tox-select-envs $PYVER-postgres devscripts/tox-select-envs $PYVER-sqlite + devscripts/tox-select-envs $PYVER-mssql + devscripts/tox-select-envs $PYVER-pytds devscripts/tox-select-envs $PYVER-flake8 if: ${{ runner.os == 'Linux' }} - name: Run tox @ w32 @@ -78,4 +85,6 @@ jobs: devscripts\tox-select-envs.cmd %PYVER%-mysql devscripts\tox-select-envs.cmd %PYVER%-postgres devscripts\tox-select-envs.cmd %PYVER%-sqlite + devscripts\tox-select-envs.cmd %PYVER%-mssql + devscripts\tox-select-envs.cmd %PYVER%-pytds if: ${{ runner.os == 'Windows' }} diff --git a/devscripts/requirements/requirements_pymssql.txt b/devscripts/requirements/requirements_pymssql.txt new file mode 100644 index 00000000..da5918ee --- /dev/null +++ b/devscripts/requirements/requirements_pymssql.txt @@ -0,0 +1,2 @@ +pymssql < 2.2; python_version <= '3.5' +pymssql; python_version >= '3.6' diff --git a/docs/News.rst b/docs/News.rst index 6bb8c303..10acacbb 100644 --- a/docs/News.rst +++ b/docs/News.rst @@ -21,6 +21,8 @@ Tests, CI * Run tests with Python 3.11. +* Run tests with MS SQL at GH Actions. + SQLObject 3.10.0 ================ diff --git a/tox.ini b/tox.ini index 81df7f69..fcf70dd5 100644 --- a/tox.ini +++ b/tox.ini @@ -31,6 +31,8 @@ deps = pyodbc: pyodbc pypyodbc: pypyodbc supersqlite: supersqlite + mssql-pymssql: -rdevscripts/requirements/requirements_pymssql.txt + mssql-pytds: python-tds firebird-fdb: fdb firebirdsql: firebirdsql passenv = CI @@ -369,25 +371,6 @@ commands = # Windows testing -[mssql-pyodbc-w32] -platform = win32 -commands = - {envpython} -c "import pyodbc; print(pyodbc.drivers())" - -sqlcmd -U sa -P "Password12!" -S .\SQL2014 -Q "DROP DATABASE sqlobject_test" - sqlcmd -U sa -P "Password12!" -S .\SQL2014 -Q "CREATE DATABASE sqlobject_test" - pytest -D "mssql://sa:Password12!@localhost\SQL2014/sqlobject_test?driver=pyodbc&odbcdrv=SQL%20Server&timeout=30&debug=1" - sqlcmd -U sa -P "Password12!" -S .\SQL2014 -Q "DROP DATABASE sqlobject_test" - -[testenv:py27-mssql-pyodbc-noauto-w32] -platform = win32 -commands = - easy_install -i https://downloads.egenix.com/python/index/ucs2/ egenix-mx-base - {[mssql-pyodbc-w32]commands} - -[testenv:py3{4,5,6,7,8,9,10,11}-mssql-pyodbc-noauto-w32] -platform = win32 -commands = {[mssql-pyodbc-w32]commands} - [mysql-connector-w32] platform = win32 commands = @@ -631,3 +614,53 @@ commands = [testenv:py3{4,5,6,7,8,9,10,11}-sqlite-memory-w32] platform = win32 commands = {[sqlite-memory-w32]commands} + +[mssql-pymssql-w32] +platform = win32 +commands = + -sqlcmd -U SA -P "YourStrong!Passw0rd" -Q "DROP DATABASE sqlobject_test" + sqlcmd -U SA -P "YourStrong!Passw0rd" -Q "CREATE DATABASE sqlobject_test" + pytest -D "mssql://SA:YourStrong!Passw0rd@localhost:1433/sqlobject_test?driver=pymssql&debug=1" + sqlcmd -U SA -P "YourStrong!Passw0rd" -Q "DROP DATABASE sqlobject_test" + +[testenv:py27-mssql-pymssql-w32] +platform = win32 +commands = + easy_install -i https://downloads.egenix.com/python/index/ucs2/ egenix-mx-base + {[mssql-pymssql-w32]commands} + +[testenv:py3{4,5,6,7,8,9,10,11}-mssql-pymssql-w32] +platform = win32 +commands = {[mssql-pymssql-w32]commands} + +[mssql-pytds-w32] +platform = win32 +commands = + -sqlcmd -U SA -P "YourStrong!Passw0rd" -Q "DROP DATABASE sqlobject_test" + sqlcmd -U SA -P "YourStrong!Passw0rd" -Q "CREATE DATABASE sqlobject_test" + pytest -D "mssql://SA:YourStrong!Passw0rd@localhost:1433/sqlobject_test?driver=pytds&debug=1" + sqlcmd -U SA -P "YourStrong!Passw0rd" -Q "DROP DATABASE sqlobject_test" + +[testenv:py27-mssql-pytds-w32] +platform = win32 +commands = + easy_install -i https://downloads.egenix.com/python/index/ucs2/ egenix-mx-base + {[mssql-pytds-w32]commands} + +[testenv:py3{4,5,6,7,8,9,10,11}-mssql-pytds-w32] +platform = win32 +commands = {[mssql-pytds-w32]commands} + +[mssql-pyodbc-w32] +platform = win32 +commands = + {envpython} -c "import pyodbc; print(pyodbc.drivers())" + -sqlcmd -U SA -P "YourStrong!Passw0rd" -Q "DROP DATABASE sqlobject_test" + sqlcmd -U SA -P "YourStrong!Passw0rd" -Q "CREATE DATABASE sqlobject_test" + pytest -D "mssql://SA:YourStrong!Passw0rd@localhost/sqlobject_test?driver=pyodbc&odbcdrv=SQL%20Server&debug=1" + sqlcmd -U SA -P "YourStrong!Passw0rd" -Q "DROP DATABASE sqlobject_test" + +[testenv:py27-mssql-pyodbc-noauto-w32] +platform = win32 +commands = + easy_install -i https://downloads.egenix.com/python/index/ucs2/ egenix-mx-base From 945f696258e623ebf27b6e98aed44b777c9221c2 Mon Sep 17 00:00:00 2001 From: Oleg Broytman Date: Fri, 2 Dec 2022 19:19:01 +0300 Subject: [PATCH 5/8] CI(MSSQL): Install and run `tsql` --- .github/workflows/run-tests.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 642afeee..75da13ef 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -40,6 +40,12 @@ jobs: - uses: ankane/setup-sqlserver@v1 with: accept-eula: true + - name: tsql + run: | + sudo apt-get update --yes + sudo apt-get install --yes freetds-bin + echo "SELECT @@VERSION" | tsql -H localhost -p 1433 -U sa -P 'YourStrong!Passw0rd' + if: ${{ runner.os == 'Linux' }} # Setup Python/pip - uses: actions/checkout@v2 From d25545bf6d5def67ed4eb656787f80bd71c7bfee Mon Sep 17 00:00:00 2001 From: Oleg Broytman Date: Fri, 2 Dec 2022 19:20:06 +0300 Subject: [PATCH 6/8] CI(MSSQL): Enable TCP for MSSQL --- .github/workflows/run-tests.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 75da13ef..3a37c58e 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -46,6 +46,16 @@ jobs: sudo apt-get install --yes freetds-bin echo "SELECT @@VERSION" | tsql -H localhost -p 1433 -U sa -P 'YourStrong!Passw0rd' if: ${{ runner.os == 'Linux' }} + - name: Enable TCP for MSSQL + run: | + [System.Reflection.Assembly]::LoadWithPartialName('Microsoft.SqlServer.SqlWmiManagement') + $wmi = New-Object 'Microsoft.SqlServer.Management.Smo.Wmi.ManagedComputer' localhost + $tcp = $wmi.ServerInstances['MSSQLSERVER'].ServerProtocols['Tcp'] + $tcp.IsEnabled = $true + $tcp.Alter() + Restart-Service -Name MSSQLSERVER -Force + shell: powershell + if: ${{ runner.os == 'Windows' }} # Setup Python/pip - uses: actions/checkout@v2 From c3ca3de7003527f009821bc0149bcd55c2f6a552 Mon Sep 17 00:00:00 2001 From: Oleg Broytman Date: Thu, 24 Nov 2022 18:07:35 +0300 Subject: [PATCH 7/8] WIP: Limit OS, Python versions, backends --- .github/workflows/run-tests.yaml | 44 +++++++++++++++++--------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index 3a37c58e..c6a2ffe2 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -7,12 +7,14 @@ jobs: strategy: matrix: - os: [ubuntu-latest, windows-latest] - python-version: ["2.7", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] + #os: [ubuntu-latest, windows-latest] + os: [windows-latest] + #python-version: ["2.7", "3.5", "3.6", "3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.7"] include: - - os: ubuntu-latest - os-name: Linux - pip-cache-path: ~/.cache/pip + #- os: ubuntu-latest + # os-name: Linux + # pip-cache-path: ~/.cache/pip - os: windows-latest os-name: w32 pip-cache-path: ~\AppData\Local\pip\Cache @@ -23,18 +25,18 @@ jobs: steps: # Setup MySQL - - uses: ankane/setup-mysql@v1 + #- uses: ankane/setup-mysql@v1 # Setup PostgreSQL - - uses: ankane/setup-postgres@v1 - - name: Setup Postgres user - run: | - sudo -u postgres psql --command="ALTER USER runner CREATEDB ENCRYPTED PASSWORD 'test'" - if: ${{ runner.os == 'Linux' }} - - name: Setup Postgres user - run: | - psql --command="CREATE USER runner CREATEDB ENCRYPTED PASSWORD 'test'" - if: ${{ runner.os == 'Windows' }} + #- uses: ankane/setup-postgres@v1 + #- name: Setup Postgres user + # run: | + # sudo -u postgres psql --command="ALTER USER runner CREATEDB ENCRYPTED PASSWORD 'test'" + # if: ${{ runner.os == 'Linux' }} + #- name: Setup Postgres user + # run: | + # psql --command="CREATE USER runner CREATEDB ENCRYPTED PASSWORD 'test'" + # if: ${{ runner.os == 'Windows' }} # Setup MS SQL - uses: ankane/setup-sqlserver@v1 @@ -89,18 +91,18 @@ jobs: tox --version - name: Run tox @ Linux run: | - devscripts/tox-select-envs $PYVER-mysql - devscripts/tox-select-envs $PYVER-postgres - devscripts/tox-select-envs $PYVER-sqlite + #devscripts/tox-select-envs $PYVER-mysql + #devscripts/tox-select-envs $PYVER-postgres + #devscripts/tox-select-envs $PYVER-sqlite devscripts/tox-select-envs $PYVER-mssql devscripts/tox-select-envs $PYVER-pytds devscripts/tox-select-envs $PYVER-flake8 if: ${{ runner.os == 'Linux' }} - name: Run tox @ w32 run: | - devscripts\tox-select-envs.cmd %PYVER%-mysql - devscripts\tox-select-envs.cmd %PYVER%-postgres - devscripts\tox-select-envs.cmd %PYVER%-sqlite + #devscripts\tox-select-envs.cmd %PYVER%-mysql + #devscripts\tox-select-envs.cmd %PYVER%-postgres + #devscripts\tox-select-envs.cmd %PYVER%-sqlite devscripts\tox-select-envs.cmd %PYVER%-mssql devscripts\tox-select-envs.cmd %PYVER%-pytds if: ${{ runner.os == 'Windows' }} From 0ab2d1d6ae3f50a15f234386b6185ed968d3bd72 Mon Sep 17 00:00:00 2001 From: Oleg Broytman Date: Fri, 2 Dec 2022 19:21:45 +0300 Subject: [PATCH 8/8] WIP: Tests(MSSQL): Run only two test files; set timeout --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index fcf70dd5..d53fbc36 100644 --- a/tox.ini +++ b/tox.ini @@ -620,7 +620,7 @@ platform = win32 commands = -sqlcmd -U SA -P "YourStrong!Passw0rd" -Q "DROP DATABASE sqlobject_test" sqlcmd -U SA -P "YourStrong!Passw0rd" -Q "CREATE DATABASE sqlobject_test" - pytest -D "mssql://SA:YourStrong!Passw0rd@localhost:1433/sqlobject_test?driver=pymssql&debug=1" + pytest -D "mssql://SA:YourStrong!Passw0rd@localhost:1433/sqlobject_test?driver=pymssql&debug=1&timeout=30" tests/test_basic.py tests/test_transactions.py sqlcmd -U SA -P "YourStrong!Passw0rd" -Q "DROP DATABASE sqlobject_test" [testenv:py27-mssql-pymssql-w32] @@ -638,7 +638,7 @@ platform = win32 commands = -sqlcmd -U SA -P "YourStrong!Passw0rd" -Q "DROP DATABASE sqlobject_test" sqlcmd -U SA -P "YourStrong!Passw0rd" -Q "CREATE DATABASE sqlobject_test" - pytest -D "mssql://SA:YourStrong!Passw0rd@localhost:1433/sqlobject_test?driver=pytds&debug=1" + pytest -D "mssql://SA:YourStrong!Passw0rd@localhost:1433/sqlobject_test?driver=pytds&debug=1&timeout=30" tests/test_basic.py tests/test_transactions.py sqlcmd -U SA -P "YourStrong!Passw0rd" -Q "DROP DATABASE sqlobject_test" [testenv:py27-mssql-pytds-w32]