From a84f2af259515d633ea292e39aabe35bc9daa569 Mon Sep 17 00:00:00 2001 From: "J.C. Jones" Date: Fri, 22 Oct 2021 13:33:50 -0700 Subject: [PATCH 1/6] Don't require PyMySQL, as it's an optional runtime dependency now --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a63253f..8d1fc75 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ zip_safe=False, include_package_data=True, python_requires=">=3.6", - install_requires=["PyMySQL >= 1.0.2", "pyyaml"], + install_requires=["pyyaml"], packages=["partitionmanager"], entry_points={"console_scripts": ["partition-manager=partitionmanager.cli:main"]}, ) From e9b1b392816e9a2d1ea096e71a5672ba2a903359 Mon Sep 17 00:00:00 2001 From: "J.C. Jones" Date: Fri, 22 Oct 2021 13:35:25 -0700 Subject: [PATCH 2/6] Make logging more specific if we turn down the level from INFO to WARNING --- partitionmanager/cli.py | 7 ++- partitionmanager/table_append_partition.py | 14 +++-- .../table_append_partition_test.py | 56 +++++++++++++------ 3 files changed, 54 insertions(+), 23 deletions(-) diff --git a/partitionmanager/cli.py b/partitionmanager/cli.py index 09a0484..6e9d498 100644 --- a/partitionmanager/cli.py +++ b/partitionmanager/cli.py @@ -156,12 +156,14 @@ def all_configured_tables_are_compatible(conf): Returns True only if all are compatible, otherwise logs errors and returns False. """ + log = logging.getLogger("all_configured_tables_are_compatible") + problems = dict() for table in conf.tables: table_problems = pm_tap.get_table_compatibility_problems(conf.dbcmd, table) if table_problems: problems[table.name] = table_problems - logging.error(f"Cannot proceed: {table} {table_problems}") + log.error(f"Cannot proceed: {table} {table_problems}") return len(problems) == 0 @@ -433,7 +435,8 @@ def do_stats(conf, metrics=partitionmanager.stats.PrometheusMetrics()): def main(): """Start here.""" args = PARSER.parse_args() - logging.basicConfig(level=args.log_level) + log_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + logging.basicConfig(level=args.log_level, format=log_format) if "func" not in args: PARSER.print_help() return diff --git a/partitionmanager/table_append_partition.py b/partitionmanager/table_append_partition.py index 8cde6d0..759d21e 100644 --- a/partitionmanager/table_append_partition.py +++ b/partitionmanager/table_append_partition.py @@ -361,6 +361,7 @@ def _calculate_start_time(last_changed_time, evaluation_time, allowed_lifespan): def _plan_partition_changes( + table, partition_list, current_position, evaluation_time, @@ -373,7 +374,7 @@ def _plan_partition_changes( requirements, using an estimate as to the rate of fill from the supplied partition_list, current_position, and evaluation_time. """ - log = logging.getLogger("plan_partition_changes") + log = logging.getLogger(f"plan_partition_changes:{table.name}") filled_partitions, active_partition, empty_partitions = _split_partitions_around_position( partition_list, current_position @@ -526,14 +527,14 @@ def _plan_partition_changes( return results -def _should_run_changes(altered_partitions): +def _should_run_changes(table, altered_partitions): """Returns True if the changeset should run, otherwise returns False. Evaluate the list from plan_partition_changes and determine if the set of changes should be performed - if all the changes are minor, they shouldn't be run. """ - log = logging.getLogger("should_run_changes") + log = logging.getLogger(f"should_run_changes:{table.name}") for p in altered_partitions: if isinstance(p, partitionmanager.types.NewPlannedPartition): @@ -650,9 +651,12 @@ def get_pending_sql_reorganize_partition_commands( algorithm is running. """ - log = logging.getLogger("get_pending_sql_reorganize_partition_commands") + log = logging.getLogger( + f"get_pending_sql_reorganize_partition_commands:{table.name}" + ) partition_changes = _plan_partition_changes( + table, partition_list, current_position, evaluation_time, @@ -660,7 +664,7 @@ def get_pending_sql_reorganize_partition_commands( num_empty_partitions, ) - if not _should_run_changes(partition_changes): + if not _should_run_changes(table, partition_changes): log.info(f"{table} does not need to be modified currently.") return list() diff --git a/partitionmanager/table_append_partition_test.py b/partitionmanager/table_append_partition_test.py index 8a29607..ad33d6f 100644 --- a/partitionmanager/table_append_partition_test.py +++ b/partitionmanager/table_append_partition_test.py @@ -497,6 +497,7 @@ def test_predict_forward_time(self): def test_plan_partition_changes_no_empty_partitions(self): with self.assertRaises(NoEmptyPartitionsAvailableException): _plan_partition_changes( + Table("table"), [mkPPart("p_20201231", 0), mkPPart("p_20210102", 200)], mkPos(50), datetime(2021, 1, 1, tzinfo=timezone.utc), @@ -505,8 +506,9 @@ def test_plan_partition_changes_no_empty_partitions(self): ) def test_plan_partition_changes_imminent(self): - with self.assertLogs("plan_partition_changes", level="INFO") as logctx: + with self.assertLogs("plan_partition_changes:table", level="INFO") as logctx: planned = _plan_partition_changes( + Table("table"), [ mkPPart("p_20201231", 100), mkPPart("p_20210102", 200), @@ -521,7 +523,7 @@ def test_plan_partition_changes_imminent(self): self.assertEqual( logctx.output, [ - "INFO:plan_partition_changes:Start-of-fill predicted at " + "INFO:plan_partition_changes:table:Start-of-fill predicted at " "2021-01-03 which is not 2021-01-02. This change will be marked " "as important to ensure that p_20210102: (200) is moved to " "2021-01-03" @@ -545,8 +547,9 @@ def test_plan_partition_changes_imminent(self): ) def test_plan_partition_changes_wildly_off_dates(self): - with self.assertLogs("plan_partition_changes", level="INFO") as logctx: + with self.assertLogs("plan_partition_changes:table", level="INFO") as logctx: planned = _plan_partition_changes( + Table("table"), [ mkPPart("p_20201231", 100), mkPPart("p_20210104", 200), @@ -561,7 +564,7 @@ def test_plan_partition_changes_wildly_off_dates(self): self.assertEqual( logctx.output, [ - "INFO:plan_partition_changes:Start-of-fill predicted at " + "INFO:plan_partition_changes:table:Start-of-fill predicted at " "2021-01-02 which is not 2021-01-04. This change will be marked " "as important to ensure that p_20210104: (200) is moved to " "2021-01-02" @@ -583,6 +586,7 @@ def test_plan_partition_changes_wildly_off_dates(self): def test_plan_partition_changes_long_delay(self): planned = _plan_partition_changes( + Table("table"), [ mkPPart("p_20210101", 100), mkPPart("p_20210415", 200), @@ -610,6 +614,7 @@ def test_plan_partition_changes_long_delay(self): def test_plan_partition_changes_short_names(self): self.maxDiff = None planned = _plan_partition_changes( + Table("table"), [ mkPPart("p_2019", 1912499867), mkPPart("p_2020", 8890030931), @@ -651,6 +656,7 @@ def test_plan_partition_changes_short_names(self): def test_plan_partition_changes_bespoke_names(self): planned = _plan_partition_changes( + Table("table"), [mkPPart("p_start", 100), mkTailPart("p_future")], mkPos(50), datetime(2021, 1, 6, tzinfo=timezone.utc), @@ -686,6 +692,7 @@ def test_plan_partition_changes_bespoke_names(self): def test_plan_partition_changes(self): self.maxDiff = None planned = _plan_partition_changes( + Table("table"), [ mkPPart("p_20201231", 100), mkPPart("p_20210102", 200), @@ -710,6 +717,7 @@ def test_plan_partition_changes(self): self.assertEqual( _plan_partition_changes( + Table("table"), [ mkPPart("p_20201231", 100), mkPPart("p_20210102", 200), @@ -739,6 +747,7 @@ def test_plan_partition_changes_misprediction(self): match reality. """ self.maxDiff = None planned = _plan_partition_changes( + Table("table"), [ mkPPart("p_20210505", 9505010028), mkPPart("p_20210604", 10152257517), @@ -770,12 +779,18 @@ def test_plan_partition_changes_misprediction(self): def test_should_run_changes(self): self.assertFalse( _should_run_changes( - [ChangePlannedPartition(mkPPart("p_20210102", 200)).set_position([300])] + Table("table"), + [ + ChangePlannedPartition(mkPPart("p_20210102", 200)).set_position( + [300] + ) + ], ) ) self.assertFalse( _should_run_changes( + Table("table"), [ ChangePlannedPartition(mkPPart("p_20210102", 200)).set_position( [300] @@ -783,12 +798,13 @@ def test_should_run_changes(self): ChangePlannedPartition(mkPPart("p_20210109", 1000)).set_position( [1300] ), - ] + ], ) ) - with self.assertLogs("should_run_changes", level="DEBUG") as logctx: + with self.assertLogs("should_run_changes:table", level="DEBUG") as logctx: self.assertTrue( _should_run_changes( + Table("table"), [ ChangePlannedPartition(mkPPart("p_20210102", 200)).set_position( [302] @@ -802,17 +818,21 @@ def test_should_run_changes(self): NewPlannedPartition() .set_position([662]) .set_timestamp(datetime(2021, 1, 23, tzinfo=timezone.utc)), - ] + ], ) ) self.assertEqual( logctx.output, - ["DEBUG:should_run_changes:Add: [542] 2021-01-16 " "00:00:00+00:00 is new"], + [ + "DEBUG:should_run_changes:table:Add: [542] 2021-01-16 " + "00:00:00+00:00 is new" + ], ) - with self.assertLogs("should_run_changes", level="DEBUG") as logctx: + with self.assertLogs("should_run_changes:table", level="DEBUG") as logctx: self.assertTrue( _should_run_changes( + Table("table"), [ ChangePlannedPartition(mkPPart("p_20210102", 200)), NewPlannedPartition() @@ -821,12 +841,15 @@ def test_should_run_changes(self): NewPlannedPartition() .set_position([662]) .set_timestamp(datetime(2021, 1, 23, tzinfo=timezone.utc)), - ] + ], ) ) self.assertEqual( logctx.output, - ["DEBUG:should_run_changes:Add: [542] 2021-01-16 " "00:00:00+00:00 is new"], + [ + "DEBUG:should_run_changes:table:Add: [542] 2021-01-16 " + "00:00:00+00:00 is new" + ], ) def testgenerate_sql_reorganize_partition_commands_no_change(self): @@ -976,6 +999,7 @@ def test_plan_andgenerate_sql_reorganize_partition_commands_with_future_partitio self ): planned = _plan_partition_changes( + Table("table"), [ mkPPart("p_20201231", 100), mkPPart("p_20210104", 200), @@ -999,7 +1023,7 @@ def test_plan_andgenerate_sql_reorganize_partition_commands_with_future_partitio def test_get_pending_sql_reorganize_partition_commands_no_changes(self): with self.assertLogs( - "get_pending_sql_reorganize_partition_commands", level="INFO" + "get_pending_sql_reorganize_partition_commands:plushies", level="INFO" ) as logctx: cmds = get_pending_sql_reorganize_partition_commands( table=Table("plushies"), @@ -1017,7 +1041,7 @@ def test_get_pending_sql_reorganize_partition_commands_no_changes(self): self.assertEqual( logctx.output, [ - "INFO:get_pending_sql_reorganize_partition_commands:" + "INFO:get_pending_sql_reorganize_partition_commands:plushies:" "Table plushies does not need to be modified currently." ], ) @@ -1026,7 +1050,7 @@ def test_get_pending_sql_reorganize_partition_commands_no_changes(self): def test_get_pending_sql_reorganize_partition_commands_with_changes(self): with self.assertLogs( - "get_pending_sql_reorganize_partition_commands", level="DEBUG" + "get_pending_sql_reorganize_partition_commands:plushies", level="DEBUG" ) as logctx: cmds = get_pending_sql_reorganize_partition_commands( table=Table("plushies"), @@ -1044,7 +1068,7 @@ def test_get_pending_sql_reorganize_partition_commands_with_changes(self): self.assertEqual( logctx.output, [ - "DEBUG:get_pending_sql_reorganize_partition_commands:" + "DEBUG:get_pending_sql_reorganize_partition_commands:plushies:" "Table plushies has changes waiting." ], ) From ba6a4097ed4d0b03c7609048b3bd4b34b7b20ff5 Mon Sep 17 00:00:00 2001 From: "J.C. Jones" Date: Fri, 22 Oct 2021 13:38:57 -0700 Subject: [PATCH 3/6] Don't use --noop if read-only, let's explicitly just do statistics --- partitionmanager/cli.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/partitionmanager/cli.py b/partitionmanager/cli.py index 6e9d498..f1a2f8c 100644 --- a/partitionmanager/cli.py +++ b/partitionmanager/cli.py @@ -284,8 +284,10 @@ def do_partition(conf): # Preflight if is_read_only(conf): - log.info("Database is read-only, forcing noop mode") - conf.noop = True + log.info("Database is read-only, only emitting statistics") + if conf.prometheus_stats_path: + do_stats(conf) + return dict() if not all_configured_tables_are_compatible(conf): return dict() @@ -357,7 +359,7 @@ def do_partition(conf): ) if conf.prometheus_stats_path: - do_stats(conf, metrics) + do_stats(conf, metrics=metrics) return all_results From 407603b963cdb3d4e487019f0332d8c8912e3c0d Mon Sep 17 00:00:00 2001 From: "J.C. Jones" Date: Fri, 22 Oct 2021 14:25:28 -0700 Subject: [PATCH 4/6] Make pyyaml an optional dependency for CircleCI / tests --- .circleci/config.yml | 2 +- setup.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3848970..b955c87 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -8,7 +8,7 @@ jobs: executor: python/default steps: - checkout - - run: pip install --editable . + - run: pip install --editable ".[pymysql]" - run: name: make test-reesults dir command: | diff --git a/setup.py b/setup.py index 8d1fc75..cb4b86e 100644 --- a/setup.py +++ b/setup.py @@ -20,6 +20,7 @@ include_package_data=True, python_requires=">=3.6", install_requires=["pyyaml"], + extras_require={"pymysql": ["PyMySQL >= 1.0.2"]}, packages=["partitionmanager"], entry_points={"console_scripts": ["partition-manager=partitionmanager.cli:main"]}, ) From 23dff617f2e998fa3067ef0d2eb19029e8a038b3 Mon Sep 17 00:00:00 2001 From: "J.C. Jones" Date: Fri, 22 Oct 2021 14:33:21 -0700 Subject: [PATCH 5/6] Update README to note the extra dependency --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a926d1a..0c7fe47 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Similar tools: → cd mariadb-sequential-partition-manager-py → python3 -m venv .venv → . .venv/bin/activate - → python3 -m pip install . + → python3 -m pip install ".[pymysql]" → tee /tmp/partman.conf.yml < Date: Fri, 22 Oct 2021 14:34:35 -0700 Subject: [PATCH 6/6] Permit Stats to work even if there are misbehaving tables --- partitionmanager/cli.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/partitionmanager/cli.py b/partitionmanager/cli.py index f1a2f8c..2f7356e 100644 --- a/partitionmanager/cli.py +++ b/partitionmanager/cli.py @@ -365,11 +365,16 @@ def do_partition(conf): def do_stats(conf, metrics=partitionmanager.stats.PrometheusMetrics()): """Populates a metrics object from the tables in the configuration.""" - if not all_configured_tables_are_compatible(conf): - return dict() + + log = logging.getLogger("do_stats") all_results = dict() for table in conf.tables: + table_problems = pm_tap.get_table_compatibility_problems(conf.dbcmd, table) + if table_problems: + log.error(f"Cannot proceed: {table} {table_problems}") + continue + map_data = pm_tap.get_partition_map(conf.dbcmd, table) statistics = partitionmanager.stats.get_statistics( map_data["partitions"], conf.curtime, table