From 7d070c46cb3f23c9ff093a5b56105b53647e64f5 Mon Sep 17 00:00:00 2001 From: mjs Date: Sun, 17 Sep 2023 22:28:41 +0900 Subject: [PATCH 1/8] test add percona8 --- docker-compose-test.yml | 14 +++++++++++++- docker-compose.yml | 6 ++++++ pymysqlreplication/tests/base.py | 24 ++++++++++++++++++++++++ test.Dockerfile | 6 ++++++ 4 files changed, 49 insertions(+), 1 deletion(-) diff --git a/docker-compose-test.yml b/docker-compose-test.yml index 3634e304..e79173fd 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -37,6 +37,15 @@ services: networks: - default + percona-8.0: + <<: *mysql + image: percona:8.0 + platform: linux/amd64 + ports: + - "3309:3306" + networks: + - default + mariadb-10.6: <<: *mariadb image: mariadb:10.6 @@ -61,6 +70,8 @@ services: MYSQL_5_7: percona-5.7 MYSQL_5_7_CTL: percona-5.7-ctl MYSQL_5_7_CTL_PORT: 3306 + MYSQL_8_0: percona-8.0 + MYSQL_8_0_PORT: 3306 MARIADB_10_6: mariadb-10.6 MARIADB_10_6_PORT: 3306 @@ -72,7 +83,7 @@ services: while : do - if mysql -h percona-5.7 --user=root --execute "SELECT version();" 2>&1 >/dev/null && mysql -h percona-5.7-ctl --user=root --execute "SELECT version();" 2>&1 >/dev/null; then + if mysql -h percona-5.7 --user=root --execute "SELECT version();" 2>&1 >/dev/null && mysql -h percona-5.7-ctl --user=root --execute "SELECT version();" 2>&1 >/dev/null && mysql -h percona-8.0 --user=root --execute "SELECT version();" 2>&1 >/dev/null; then break fi sleep 1 @@ -87,6 +98,7 @@ services: depends_on: - percona-5.7 - percona-5.7-ctl + - percona-8.0 networks: default: diff --git a/docker-compose.yml b/docker-compose.yml index 9e68758c..102ff631 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,6 +33,12 @@ services: ports: - "3307:3306" + percona-8.0: + <<: *mysql + image: percona:8.0 + ports: + - "3309:3306" + mariadb-10.6: <<: *mariadb image: mariadb:10.6 diff --git a/pymysqlreplication/tests/base.py b/pymysqlreplication/tests/base.py index f2ddbcef..281e6273 100644 --- a/pymysqlreplication/tests/base.py +++ b/pymysqlreplication/tests/base.py @@ -160,3 +160,27 @@ def bin_log_basename(self): bin_log_basename = cursor.fetchone()[0] bin_log_basename = bin_log_basename.split("/")[-1] return bin_log_basename + +class PyMySQLReplicationPercona8TestCase(PyMySQLReplicationTestCase): + def setUp(self): + # default + self.database = { + "host": os.environ.get("MYSQL_8_0") or "localhost", + "user": "root", + "passwd": "", + "port": int(os.environ.get("MYSQL_8_0_PORT") or 3309), + "use_unicode": True, + "charset": "utf8", + "db": "pymysqlreplication_test", + } + + self.conn_control = None + db = copy.copy(self.database) + db["db"] = None + self.connect_conn_control(db) + self.execute("DROP DATABASE IF EXISTS pymysqlreplication_test") + self.execute("CREATE DATABASE pymysqlreplication_test") + db = copy.copy(self.database) + self.connect_conn_control(db) + self.stream = None + self.resetBinLog() diff --git a/test.Dockerfile b/test.Dockerfile index 0272e00f..ca6df276 100644 --- a/test.Dockerfile +++ b/test.Dockerfile @@ -19,6 +19,12 @@ ENV MYSQL_5_7_CTL ${MYSQL_5_7_CTL} ARG MYSQL_5_7_CTL_PORT ENV MYSQL_5_7_CTL_PORT ${MYSQL_5_7_CTL_PORT} +ARG MYSQL_8_0 +ENV MYSQL_8_0 ${MYSQL_8_0} + +ARG MYSQL_8_0_PORT +ENV MYSQL_8_0_PORT ${MYSQL_8_0_PORT} + ARG MARIADB_10_6 ENV MARIADB_10_6 ${MARIADB_10_6} From ff58278350451902fdec3b3d78ca1983a4df9b55 Mon Sep 17 00:00:00 2001 From: Heeseon Cheon Date: Mon, 18 Sep 2023 23:43:12 +0900 Subject: [PATCH 2/8] feat: change TestOptionalMetaData with percona8 --- pymysqlreplication/tests/base.py | 3 +++ pymysqlreplication/tests/test_basic.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pymysqlreplication/tests/base.py b/pymysqlreplication/tests/base.py index 281e6273..eec03f65 100644 --- a/pymysqlreplication/tests/base.py +++ b/pymysqlreplication/tests/base.py @@ -163,6 +163,7 @@ def bin_log_basename(self): class PyMySQLReplicationPercona8TestCase(PyMySQLReplicationTestCase): def setUp(self): + super().setUp() # default self.database = { "host": os.environ.get("MYSQL_8_0") or "localhost", @@ -184,3 +185,5 @@ def setUp(self): self.connect_conn_control(db) self.stream = None self.resetBinLog() + self.isMySQL80AndMore() + self.__is_mariaDB = None diff --git a/pymysqlreplication/tests/test_basic.py b/pymysqlreplication/tests/test_basic.py index ac3df684..aac9a8b6 100644 --- a/pymysqlreplication/tests/test_basic.py +++ b/pymysqlreplication/tests/test_basic.py @@ -1537,7 +1537,7 @@ def test_query_event_latin1(self): assert event.query == r"CREATE TABLE test_latin1_\xd6\xc6\xdb (a INT)" -class TestOptionalMetaData(base.PyMySQLReplicationTestCase): +class TestOptionalMetaData(base.PyMySQLReplicationPercona8TestCase): def setUp(self): super(TestOptionalMetaData, self).setUp() self.stream.close() From eff5596bdd14f4c9cd83b8b8a8c6e2906118110c Mon Sep 17 00:00:00 2001 From: heehehe Date: Tue, 19 Sep 2023 22:45:41 +0900 Subject: [PATCH 3/8] feat: add version8 test in test_data_type.py related PR: https://github.com/julien-duponchelle/python-mysql-replication/pull/355 --- pymysqlreplication/tests/base.py | 3 ++- pymysqlreplication/tests/test_data_type.py | 20 +++++++++++++++++++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/pymysqlreplication/tests/base.py b/pymysqlreplication/tests/base.py index eec03f65..2cd7e2cf 100644 --- a/pymysqlreplication/tests/base.py +++ b/pymysqlreplication/tests/base.py @@ -161,9 +161,10 @@ def bin_log_basename(self): bin_log_basename = bin_log_basename.split("/")[-1] return bin_log_basename + class PyMySQLReplicationPercona8TestCase(PyMySQLReplicationTestCase): def setUp(self): - super().setUp() + super().setUp() # default self.database = { "host": os.environ.get("MYSQL_8_0") or "localhost", diff --git a/pymysqlreplication/tests/test_data_type.py b/pymysqlreplication/tests/test_data_type.py index ed30cf9c..c4f74bb3 100644 --- a/pymysqlreplication/tests/test_data_type.py +++ b/pymysqlreplication/tests/test_data_type.py @@ -19,7 +19,7 @@ from pymysqlreplication._compat import text_type -__all__ = ["TestDataType"] +__all__ = ["TestDataType", "TestDataTypeVersion8"] def to_binary_dict(d): @@ -894,5 +894,23 @@ def test_mariadb_only_status_vars(self): self.assertEqual(event.query, create_query) +class TestDataTypeVersion8(base.PyMySQLReplicationPercona8TestCase): + def test_partition_id(self): + if not self.isMySQL80AndMore(): + self.skipTest("Not supported in this version of MySQL") + create_query = "CREATE TABLE test (id INTEGER) \ + PARTITION BY RANGE (id) ( \ + PARTITION p0 VALUES LESS THAN (1), \ + PARTITION p1 VALUES LESS THAN (2), \ + PARTITION p2 VALUES LESS THAN (3), \ + PARTITION p3 VALUES LESS THAN (4), \ + PARTITION p4 VALUES LESS THAN (5) \ + )" + insert_query = "INSERT INTO test (id) VALUES(3)" + event = self.create_and_insert_value(create_query, insert_query) + self.assertEqual(event.extra_data_type, 1) + self.assertEqual(event.partition_id, 3) + + if __name__ == "__main__": unittest.main() From 6cf18b8e04117cd5ff2e6c5e3fa204cc36001c6e Mon Sep 17 00:00:00 2001 From: mjs Date: Tue, 19 Sep 2023 22:55:58 +0900 Subject: [PATCH 4/8] refactor: upgrade github actions --- .github/workflows/pytest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 47ea5f40..b0a5286f 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -17,10 +17,10 @@ jobs: steps: - name: Check out code - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} From 5e35074281d5305b7cbdbfd6cd8be82b1eafead9 Mon Sep 17 00:00:00 2001 From: heehehe Date: Tue, 19 Sep 2023 22:56:23 +0900 Subject: [PATCH 5/8] fix: add ignoredEvents and create_and_insert_value in TestDataTypeVersion8 --- pymysqlreplication/tests/test_data_type.py | 26 ++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/pymysqlreplication/tests/test_data_type.py b/pymysqlreplication/tests/test_data_type.py index c4f74bb3..9af7d899 100644 --- a/pymysqlreplication/tests/test_data_type.py +++ b/pymysqlreplication/tests/test_data_type.py @@ -895,6 +895,32 @@ def test_mariadb_only_status_vars(self): class TestDataTypeVersion8(base.PyMySQLReplicationPercona8TestCase): + def ignoredEvents(self): + return [GtidEvent, PreviousGtidsEvent] + + def create_and_insert_value(self, create_query, insert_query): + self.execute(create_query) + self.execute(insert_query) + self.execute("COMMIT") + + self.assertIsInstance(self.stream.fetchone(), RotateEvent) + self.assertIsInstance(self.stream.fetchone(), FormatDescriptionEvent) + # QueryEvent for the Create Table + self.assertIsInstance(self.stream.fetchone(), QueryEvent) + + # QueryEvent for the BEGIN + self.assertIsInstance(self.stream.fetchone(), QueryEvent) + + self.assertIsInstance(self.stream.fetchone(), TableMapEvent) + + event = self.stream.fetchone() + if self.isMySQL56AndMore(): + self.assertEqual(event.event_type, WRITE_ROWS_EVENT_V2) + else: + self.assertEqual(event.event_type, WRITE_ROWS_EVENT_V1) + self.assertIsInstance(event, WriteRowsEvent) + return event + def test_partition_id(self): if not self.isMySQL80AndMore(): self.skipTest("Not supported in this version of MySQL") From a6686c0f5da7b449b6219cfef3e1b99a4901be83 Mon Sep 17 00:00:00 2001 From: heehehe Date: Tue, 19 Sep 2023 23:30:24 +0900 Subject: [PATCH 6/8] refactor: Percona8 -> Version8 --- pymysqlreplication/tests/base.py | 2 +- pymysqlreplication/tests/test_basic.py | 2 +- pymysqlreplication/tests/test_data_type.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pymysqlreplication/tests/base.py b/pymysqlreplication/tests/base.py index 2cd7e2cf..5061d96a 100644 --- a/pymysqlreplication/tests/base.py +++ b/pymysqlreplication/tests/base.py @@ -162,7 +162,7 @@ def bin_log_basename(self): return bin_log_basename -class PyMySQLReplicationPercona8TestCase(PyMySQLReplicationTestCase): +class PyMySQLReplicationVersion8TestCase(PyMySQLReplicationTestCase): def setUp(self): super().setUp() # default diff --git a/pymysqlreplication/tests/test_basic.py b/pymysqlreplication/tests/test_basic.py index aac9a8b6..8c884c1e 100644 --- a/pymysqlreplication/tests/test_basic.py +++ b/pymysqlreplication/tests/test_basic.py @@ -1537,7 +1537,7 @@ def test_query_event_latin1(self): assert event.query == r"CREATE TABLE test_latin1_\xd6\xc6\xdb (a INT)" -class TestOptionalMetaData(base.PyMySQLReplicationPercona8TestCase): +class TestOptionalMetaData(base.PyMySQLReplicationVersion8TestCase): def setUp(self): super(TestOptionalMetaData, self).setUp() self.stream.close() diff --git a/pymysqlreplication/tests/test_data_type.py b/pymysqlreplication/tests/test_data_type.py index 9af7d899..900e47de 100644 --- a/pymysqlreplication/tests/test_data_type.py +++ b/pymysqlreplication/tests/test_data_type.py @@ -894,7 +894,7 @@ def test_mariadb_only_status_vars(self): self.assertEqual(event.query, create_query) -class TestDataTypeVersion8(base.PyMySQLReplicationPercona8TestCase): +class TestDataTypeVersion8(base.PyMySQLReplicationVersion8TestCase): def ignoredEvents(self): return [GtidEvent, PreviousGtidsEvent] From 85c10b6270d9d964ed42e85799856a8454f6f486 Mon Sep 17 00:00:00 2001 From: mjs Date: Thu, 21 Sep 2023 19:31:05 +0900 Subject: [PATCH 7/8] resolve conflict test case --- pymysqlreplication/tests/test_basic.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pymysqlreplication/tests/test_basic.py b/pymysqlreplication/tests/test_basic.py index 6553ba3a..711df5dc 100644 --- a/pymysqlreplication/tests/test_basic.py +++ b/pymysqlreplication/tests/test_basic.py @@ -1703,7 +1703,7 @@ def test_sync_drop_table_map_event_table_schema(self): event = self.stream.fetchone() self.assertIsInstance(event, TableMapEvent) - self.assertEqual(event.table_obj.data["columns"][0].name, "name") + self.assertEqual(event.table_obj.data["columns"][0].name, None) self.assertEqual(len(column_schemas), 0) def test_sync_column_drop_event_table_schema(self): @@ -1734,9 +1734,9 @@ def test_sync_column_drop_event_table_schema(self): self.assertEqual(len(event.table_obj.data["columns"]), 3) self.assertEqual(column_schemas[0][0], "drop_column1") self.assertEqual(column_schemas[1][0], "drop_column3") - self.assertEqual(event.table_obj.data["columns"][0].name, "drop_column1") - self.assertEqual(event.table_obj.data["columns"][1].name, "drop_column2") - self.assertEqual(event.table_obj.data["columns"][2].name, "drop_column3") + self.assertEqual(event.table_obj.data["columns"][0].name, None) + self.assertEqual(event.table_obj.data["columns"][1].name, None) + self.assertEqual(event.table_obj.data["columns"][2].name, None) def tearDown(self): self.execute("SET GLOBAL binlog_row_metadata='MINIMAL';") From a58f1dffac538267d05b78f2a0489d2136f36e2a Mon Sep 17 00:00:00 2001 From: mjs Date: Thu, 21 Sep 2023 19:47:37 +0900 Subject: [PATCH 8/8] refactor: update time limit --- .github/workflows/pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index b0a5286f..33b8f63c 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -13,7 +13,7 @@ jobs: - {name: 'Pypy 3.9', python: 'pypy-3.9'} name: ${{ matrix.name }} runs-on: ubuntu-latest - timeout-minutes: 2 + timeout-minutes: 3 steps: - name: Check out code