From cc105911c7956b85a79479e821da21232f470b05 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Mon, 5 Apr 2021 17:19:33 -0500 Subject: [PATCH 01/12] activate warnings --- .github/workflows/database.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/database.yml b/.github/workflows/database.yml index ba5a0a1fd0909..42077b04cc355 100644 --- a/.github/workflows/database.yml +++ b/.github/workflows/database.yml @@ -13,6 +13,7 @@ env: PANDAS_CI: 1 PATTERN: ((not slow and not network and not clipboard) or (single and db)) COVERAGE: true + SQLALCHEMY_WARN_20: true jobs: Linux_py37_IO: From 4c57caa9bcd3fcc2320927e4c1474794f2f2e261 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Sat, 10 Jul 2021 21:55:23 -0500 Subject: [PATCH 02/12] fix warnings --- pandas/tests/io/test_sql.py | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 290e063a59be7..afdc45032f51a 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -25,6 +25,7 @@ ) from io import StringIO import sqlite3 +from sqlite3.dbapi2 import Connection import warnings import numpy as np @@ -66,6 +67,7 @@ from sqlalchemy.orm import session as sa_session import sqlalchemy.schema import sqlalchemy.sql.sqltypes as sqltypes + from sqlalchemy import text SQLALCHEMY_INSTALLED = True except ImportError: @@ -288,10 +290,11 @@ class PandasSQLTest: """ def _get_exec(self): - if hasattr(self.conn, "execute"): - return self.conn - else: - return self.conn.cursor() + return ( + self.conn.cursor() + if isinstance(self.conn, sqlite3.Connection) + else self.conn + ) @pytest.fixture(params=[("io", "data", "csv", "iris.csv")]) def load_iris_data(self, datapath, request): @@ -302,15 +305,27 @@ def load_iris_data(self, datapath, request): self.setup_connect() self.drop_table("iris") - self._get_exec().execute(SQL_STRINGS["create_iris"][self.flavor]) + + if isinstance(self._get_exec(), sqlite3.Connection): + self._get_exec().execute(SQL_STRINGS["create_iris"][self.flavor]) + else: + with self._get_exec().connect() as conn: + with conn.begin(): + conn.execute(text(SQL_STRINGS["create_iris"][self.flavor])) with open(iris_csv_file, newline=None) as iris_csv: - r = csv.reader(iris_csv) - next(r) # skip header row + reader = csv.reader(iris_csv) + next(reader) # skip header row ins = SQL_STRINGS["insert_iris"][self.flavor] - for row in r: - self._get_exec().execute(ins, row) + if isinstance(self._get_exec(), sqlite3.Connection): + for row in reader: + self._get_exec().execute(ins, row) + else: + with self._get_exec().connect() as conn: + with conn.begin(): + for row in reader: + conn.execute(text(ins), row) def _load_iris_view(self): self.drop_table("iris_view") From 07d6ee489eb6d8764b5ff4606e86020e9d1a9f52 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Sat, 10 Jul 2021 22:19:47 -0500 Subject: [PATCH 03/12] fix warnings --- pandas/tests/io/test_sql.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index afdc45032f51a..721162e26d43a 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -306,7 +306,7 @@ def load_iris_data(self, datapath, request): self.drop_table("iris") - if isinstance(self._get_exec(), sqlite3.Connection): + if isinstance(self._get_exec(), sqlite3.Cursor): self._get_exec().execute(SQL_STRINGS["create_iris"][self.flavor]) else: with self._get_exec().connect() as conn: @@ -318,7 +318,7 @@ def load_iris_data(self, datapath, request): next(reader) # skip header row ins = SQL_STRINGS["insert_iris"][self.flavor] - if isinstance(self._get_exec(), sqlite3.Connection): + if isinstance(self._get_exec(), sqlite3.Cursor): for row in reader: self._get_exec().execute(ins, row) else: From c88ec69b071c007fa45dce81359f86b4377c8712 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Sat, 10 Jul 2021 23:24:29 -0500 Subject: [PATCH 04/12] remove text --- pandas/tests/io/test_sql.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 721162e26d43a..6fc3bb13f9976 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -67,7 +67,6 @@ from sqlalchemy.orm import session as sa_session import sqlalchemy.schema import sqlalchemy.sql.sqltypes as sqltypes - from sqlalchemy import text SQLALCHEMY_INSTALLED = True except ImportError: @@ -311,7 +310,7 @@ def load_iris_data(self, datapath, request): else: with self._get_exec().connect() as conn: with conn.begin(): - conn.execute(text(SQL_STRINGS["create_iris"][self.flavor])) + conn.execute(SQL_STRINGS["create_iris"][self.flavor]) with open(iris_csv_file, newline=None) as iris_csv: reader = csv.reader(iris_csv) @@ -325,7 +324,7 @@ def load_iris_data(self, datapath, request): with self._get_exec().connect() as conn: with conn.begin(): for row in reader: - conn.execute(text(ins), row) + conn.execute(ins, row) def _load_iris_view(self): self.drop_table("iris_view") From 6ad9aa69286935a0af9cd430bde2948367c45c99 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Sun, 11 Jul 2021 10:47:08 -0500 Subject: [PATCH 05/12] fix more warnings in tests --- pandas/tests/io/test_sql.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 6fc3bb13f9976..4f0685ea26338 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -328,7 +328,12 @@ def load_iris_data(self, datapath, request): def _load_iris_view(self): self.drop_table("iris_view") - self._get_exec().execute(SQL_STRINGS["create_view"][self.flavor]) + if isinstance(self._get_exec(), sqlite3.Cursor): + self._get_exec().execute(SQL_STRINGS["create_view"][self.flavor]) + else: + with self._get_exec().connect() as conn: + with conn.begin(): + conn.execute(SQL_STRINGS["create_view"][self.flavor]) def _check_iris_loaded_frame(self, iris_frame): pytype = iris_frame.dtypes[0].type @@ -447,7 +452,12 @@ def _filter_to_flavor(flavor, df): def _load_raw_sql(self): self.drop_table("types_test_data") - self._get_exec().execute(SQL_STRINGS["create_test_types"][self.flavor]) + if isinstance(self._get_exec(), sqlite3.Cursor): + self._get_exec().execute(SQL_STRINGS["create_test_types"][self.flavor]) + else: + with self._get_exec().connect() as conn: + with conn.begin(): + conn.execute(SQL_STRINGS["create_test_types"][self.flavor]) ins = SQL_STRINGS["insert_test_types"][self.flavor] data = [ { @@ -475,11 +485,18 @@ def _load_raw_sql(self): "BoolColWithNull": None, }, ] - - for d in data: - self._get_exec().execute( - ins["query"], [d[field] for field in ins["fields"]] - ) + if isinstance(self._get_exec(), sqlite3.Cursor): + for d in data: + self._get_exec().execute( + ins["query"], [d[field] for field in ins["fields"]] + ) + else: + with self._get_exec().connect() as conn: + with conn.begin(): + for d in data: + conn.execute( + ins["query"], [d[field] for field in ins["fields"]] + ) self._load_types_test_data(data) From 9aa8fb05ed1771ccfccaf03b7e37134d811d89ce Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Mon, 12 Jul 2021 23:19:32 -0500 Subject: [PATCH 06/12] fix more errors --- pandas/io/sql.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 1a8012df5cb46..ee56f2191db83 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -89,8 +89,12 @@ def _gt14() -> bool: return Version(sqlalchemy.__version__) >= Version("1.4.0") -def _convert_params(sql, params): +def _convert_params(sql, params, using_sqlite=False): """Convert SQL and params args to DBAPI2.0 compliant format.""" + if isinstance(sql, str) and not using_sqlite: + from sqlalchemy import text + + sql = text(sql) args = [sql] if params is not None: if hasattr(params, "keys"): # test if params is a mapping @@ -202,11 +206,15 @@ def execute(sql, con, cur=None, params=None): ------- Results Iterable """ + import sqlite3 + if cur is None: pandas_sql = pandasSQL_builder(con) else: pandas_sql = pandasSQL_builder(cur, is_cursor=True) - args = _convert_params(sql, params) + args = _convert_params( + sql, params, using_sqlite=isinstance(con, sqlite3.Connection) + ) return pandas_sql.execute(*args) @@ -1421,7 +1429,7 @@ def run_transaction(self): def execute(self, *args, **kwargs): """Simple passthrough to SQLAlchemy connectable""" - return self.connectable.execution_options().execute(*args, **kwargs) + return self.connectable.connect().execute(*args, **kwargs) def read_table( self, @@ -2112,7 +2120,7 @@ def read_query( dtype: DtypeArg | None = None, ): - args = _convert_params(sql, params) + args = _convert_params(sql, params, using_sqlite=True) cursor = self.execute(*args) columns = [col_desc[0] for col_desc in cursor.description] From b9fb23d26b2812f17d9f31743b6c9bca4de01122 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Tue, 13 Jul 2021 13:39:40 -0500 Subject: [PATCH 07/12] revert changes in tests --- pandas/io/sql.py | 2 ++ pandas/tests/io/test_sql.py | 52 ++++++++----------------------------- 2 files changed, 13 insertions(+), 41 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index f5c349837b6fa..c204d995c4cb3 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -205,6 +205,8 @@ def execute(sql, con, params=None): ------- Results Iterable """ + import sqlite3 + pandas_sql = pandasSQL_builder(con) args = _convert_params( sql, params, using_sqlite=isinstance(con, sqlite3.Connection) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 37d3a2395315a..a54f51068d289 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -25,7 +25,6 @@ ) from io import StringIO import sqlite3 -from sqlite3.dbapi2 import Connection import warnings import numpy as np @@ -305,35 +304,19 @@ def load_iris_data(self, datapath, request): self.drop_table("iris") - if isinstance(self._get_exec(), sqlite3.Cursor): - self._get_exec().execute(SQL_STRINGS["create_iris"][self.flavor]) - else: - with self._get_exec().connect() as conn: - with conn.begin(): - conn.execute(SQL_STRINGS["create_iris"][self.flavor]) + self._get_exec().execute(SQL_STRINGS["create_iris"][self.flavor]) with open(iris_csv_file, newline=None) as iris_csv: - reader = csv.reader(iris_csv) - next(reader) # skip header row + r = csv.reader(iris_csv) + next(r) # skip header row ins = SQL_STRINGS["insert_iris"][self.flavor] - if isinstance(self._get_exec(), sqlite3.Cursor): - for row in reader: - self._get_exec().execute(ins, row) - else: - with self._get_exec().connect() as conn: - with conn.begin(): - for row in reader: - conn.execute(ins, row) + for row in r: + self._get_exec().execute(ins, row) def _load_iris_view(self): self.drop_table("iris_view") - if isinstance(self._get_exec(), sqlite3.Cursor): - self._get_exec().execute(SQL_STRINGS["create_view"][self.flavor]) - else: - with self._get_exec().connect() as conn: - with conn.begin(): - conn.execute(SQL_STRINGS["create_view"][self.flavor]) + self._get_exec().execute(SQL_STRINGS["create_view"][self.flavor]) def _check_iris_loaded_frame(self, iris_frame): pytype = iris_frame.dtypes[0].type @@ -452,12 +435,7 @@ def _filter_to_flavor(flavor, df): def _load_raw_sql(self): self.drop_table("types_test_data") - if isinstance(self._get_exec(), sqlite3.Cursor): - self._get_exec().execute(SQL_STRINGS["create_test_types"][self.flavor]) - else: - with self._get_exec().connect() as conn: - with conn.begin(): - conn.execute(SQL_STRINGS["create_test_types"][self.flavor]) + self._get_exec().execute(SQL_STRINGS["create_test_types"][self.flavor]) ins = SQL_STRINGS["insert_test_types"][self.flavor] data = [ { @@ -485,18 +463,10 @@ def _load_raw_sql(self): "BoolColWithNull": None, }, ] - if isinstance(self._get_exec(), sqlite3.Cursor): - for d in data: - self._get_exec().execute( - ins["query"], [d[field] for field in ins["fields"]] - ) - else: - with self._get_exec().connect() as conn: - with conn.begin(): - for d in data: - conn.execute( - ins["query"], [d[field] for field in ins["fields"]] - ) + for d in data: + self._get_exec().execute( + ins["query"], [d[field] for field in ins["fields"]] + ) self._load_types_test_data(data) From d768cdc1e0a8c1e7c4c0b08bbcca7af5eb4edd67 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Wed, 14 Jul 2021 14:05:13 -0500 Subject: [PATCH 08/12] update insert --- pandas/io/sql.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index c204d995c4cb3..063f41e0e9cf7 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -889,8 +889,11 @@ def _execute_insert_multi(self, conn, keys: list[str], data_iter): and tables containing a few columns but performance degrades quickly with increase of columns. """ + from sqlalchemy import insert + data = [dict(zip(keys, row)) for row in data_iter] - conn.execute(self.table.insert(data)) + insert_stmt = insert(self.table).values(data) + conn.execute(insert_stmt) def insert_data(self): if self.index is not None: @@ -1401,8 +1404,9 @@ class SQLDatabase(PandasSQL): arguments in the MetaData object. """ + from sqlalchemy import Engine - def __init__(self, engine, schema: str | None = None, meta=None): + def __init__(self, engine: Engine, schema: str | None = None, meta=None): self.connectable = engine if not meta: from sqlalchemy.schema import MetaData From 9d4f03be95ef51e17f1603a283c7cdbe18bde1ff Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Tue, 17 Aug 2021 16:46:00 -0500 Subject: [PATCH 09/12] revert changes --- pandas/io/sql.py | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 2161f122b1a03..afd045bd8bb2b 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -65,12 +65,8 @@ def _gt14() -> bool: return Version(sqlalchemy.__version__) >= Version("1.4.0") -def _convert_params(sql, params, using_sqlite=False): +def _convert_params(sql, params): """Convert SQL and params args to DBAPI2.0 compliant format.""" - if isinstance(sql, str) and not using_sqlite: - from sqlalchemy import text - - sql = text(sql) args = [sql] if params is not None: if hasattr(params, "keys"): # test if params is a mapping @@ -181,12 +177,8 @@ def execute(sql, con, params=None): ------- Results Iterable """ - import sqlite3 - pandas_sql = pandasSQL_builder(con) - args = _convert_params( - sql, params, using_sqlite=isinstance(con, sqlite3.Connection) - ) + args = _convert_params(sql, params) return pandas_sql.execute(*args) @@ -850,11 +842,8 @@ def _execute_insert_multi(self, conn, keys: list[str], data_iter): and tables containing a few columns but performance degrades quickly with increase of columns. """ - from sqlalchemy import insert - data = [dict(zip(keys, row)) for row in data_iter] - insert_stmt = insert(self.table).values(data) - conn.execute(insert_stmt) + conn.execute(self.table.insert(data)) def insert_data(self): if self.index is not None: @@ -1361,7 +1350,6 @@ class SQLDatabase(PandasSQL): supports this). If None, use default schema (default). """ - from sqlalchemy import Engine def __init__(self, engine, schema: str | None = None): from sqlalchemy.schema import MetaData @@ -1379,7 +1367,7 @@ def run_transaction(self): def execute(self, *args, **kwargs): """Simple passthrough to SQLAlchemy connectable""" - return self.connectable.connect().execute(*args, **kwargs) + return self.connectable.execution_options().execute(*args, **kwargs) def read_table( self, @@ -2066,7 +2054,7 @@ def read_query( dtype: DtypeArg | None = None, ): - args = _convert_params(sql, params, using_sqlite=True) + args = _convert_params(sql, params) cursor = self.execute(*args) columns = [col_desc[0] for col_desc in cursor.description] From c123658d2bee75f0055ab52114edacfa24ecf062 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Tue, 17 Aug 2021 16:47:18 -0500 Subject: [PATCH 10/12] revert changes --- pandas/tests/io/test_sql.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py index 009a9ad5e985a..6136bd0e1e057 100644 --- a/pandas/tests/io/test_sql.py +++ b/pandas/tests/io/test_sql.py @@ -416,11 +416,10 @@ class PandasSQLTest: """ def _get_exec(self): - return ( - self.conn.cursor() - if isinstance(self.conn, sqlite3.Connection) - else self.conn - ) + if hasattr(self.conn, "execute"): + return self.conn + else: + return self.conn.cursor() @pytest.fixture def load_iris_data(self, iris_path): From c2868f9e348b8194da6418aa75cf5b339fee5c3f Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Fri, 20 Aug 2021 21:07:29 -0500 Subject: [PATCH 11/12] first try --- pandas/io/sql.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 45444852c99a6..6cb529c59b934 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -178,6 +178,10 @@ def execute(sql, con, params=None): Results Iterable """ pandas_sql = pandasSQL_builder(con) + if isinstance(pandas_sql, SQLDatabase): + from sqlalchemy import text + + sql = text(sql) args = _convert_params(sql, params) return pandas_sql.execute(*args) @@ -1367,8 +1371,19 @@ def run_transaction(self): yield self.connectable def execute(self, *args, **kwargs): - """Simple passthrough to SQLAlchemy connectable""" - return self.connectable.execution_options().execute(*args, **kwargs) + from sqlalchemy.engine import Engine + + if isinstance(self.connectable, Engine): + if not self.connectable.dialect.name == "sqlite": + with self.connectable.connect() as conn: + with conn.begin(): + result = conn.execute(*args, **kwargs) + else: + conn = self.connectable.connect() + result = conn.execute(*args, **kwargs) + else: + result = self.connectable.execute(*args, **kwargs) + return result def read_table( self, @@ -1521,6 +1536,11 @@ def read_query( read_sql """ + from sqlalchemy import text + + if isinstance(sql, str) and hasattr(params, "keys"): + sql = text(sql) + args = _convert_params(sql, params) result = self.execute(*args) From 4f9dca139fe3406be8462e81361bc732419e40a8 Mon Sep 17 00:00:00 2001 From: Fangchen Li Date: Sun, 22 Aug 2021 19:23:16 -0500 Subject: [PATCH 12/12] rollback --- pandas/io/sql.py | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/pandas/io/sql.py b/pandas/io/sql.py index 6cb529c59b934..45444852c99a6 100644 --- a/pandas/io/sql.py +++ b/pandas/io/sql.py @@ -178,10 +178,6 @@ def execute(sql, con, params=None): Results Iterable """ pandas_sql = pandasSQL_builder(con) - if isinstance(pandas_sql, SQLDatabase): - from sqlalchemy import text - - sql = text(sql) args = _convert_params(sql, params) return pandas_sql.execute(*args) @@ -1371,19 +1367,8 @@ def run_transaction(self): yield self.connectable def execute(self, *args, **kwargs): - from sqlalchemy.engine import Engine - - if isinstance(self.connectable, Engine): - if not self.connectable.dialect.name == "sqlite": - with self.connectable.connect() as conn: - with conn.begin(): - result = conn.execute(*args, **kwargs) - else: - conn = self.connectable.connect() - result = conn.execute(*args, **kwargs) - else: - result = self.connectable.execute(*args, **kwargs) - return result + """Simple passthrough to SQLAlchemy connectable""" + return self.connectable.execution_options().execute(*args, **kwargs) def read_table( self, @@ -1536,11 +1521,6 @@ def read_query( read_sql """ - from sqlalchemy import text - - if isinstance(sql, str) and hasattr(params, "keys"): - sql = text(sql) - args = _convert_params(sql, params) result = self.execute(*args)