diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 5a725c9..0000000 --- a/.flake8 +++ /dev/null @@ -1,4 +0,0 @@ -[flake8] -# See http://pep8.readthedocs.io/en/latest/intro.html#configuration -ignore = E121, E123, E126, E129, E133, E203, E226, E241, E242, E704, W503, E402, E741 -max-line-length = 99 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c7d07fa..634c642 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,19 +8,16 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python 3.9 - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.9 - name: Install Linting Tools run: | python -m pip install --upgrade pip - pip install --user pylint==3.1.0 - pip install --user black~=23.9.1 - pip install --user flake8~=6.1.0 - pip install --user pytest + pip install --user pylint==3.1.0 pytest ruff - name: Install Partition Manager run: | @@ -30,21 +27,21 @@ jobs: run: | python -m pylint -E partitionmanager - - name: Checking format with Black + - name: Checking format with Ruff run: | - python -m black --check . + python -m ruff format . - - name: Checking format with Flake8 + - name: Lint Python code with Ruff run: | - python -m flake8 + python -m ruff --output-format=github test: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python 3.9 - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.9 - name: Install Partition Manager diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 021a047..1d2cacc 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v4.5.0 hooks: - id: check-ast - id: check-merge-conflict @@ -8,14 +8,13 @@ repos: - id: end-of-file-fixer - id: requirements-txt-fixer - id: trailing-whitespace -- repo: https://github.com/psf/black - rev: "23.9.1" + +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.2.2 hooks: - - id: black -- repo: https://github.com/pycqa/flake8 - rev: "6.1.0" - hooks: - - id: flake8 + - id: ruff + # - id: ruff-format + - repo: https://github.com/PyCQA/pylint rev: v3.1.0 hooks: @@ -26,6 +25,8 @@ repos: - PyMySQL - pyyaml - pytest + - setuptools + - repo: local hooks: - id: pytest diff --git a/partitionmanager/database_helpers.py b/partitionmanager/database_helpers.py index 98400d1..6f80408 100644 --- a/partitionmanager/database_helpers.py +++ b/partitionmanager/database_helpers.py @@ -58,7 +58,7 @@ def calculate_exact_timestamp_via_query(database, table, position_partition): raise partitionmanager.types.NoExactTimeException( "Unexpected column count for the timestamp result" ) - for key, value in exact_time_result[0].items(): + for value in exact_time_result[0].values(): exact_time = datetime.fromtimestamp(value, tz=timezone.utc) break diff --git a/partitionmanager/migrate.py b/partitionmanager/migrate.py index b893b06..8ea899f 100644 --- a/partitionmanager/migrate.py +++ b/partitionmanager/migrate.py @@ -175,8 +175,7 @@ def _generate_sql_copy_commands( yield f"\tPARTITION {max_val_part.name} VALUES LESS THAN {max_val_string}" yield ");" - for command in alter_commands_iter: - yield command + yield from alter_commands_iter cols = set(columns) @@ -214,9 +213,11 @@ def _generate_sql_copy_commands( ): yield line - yield "\t\tWHERE " + " AND ".join( - _trigger_column_copies(map_data["range_cols"]) - ) + ";" + yield ( + "\t\tWHERE " + + " AND ".join(_trigger_column_copies(map_data["range_cols"])) + + ";" + ) return diff --git a/partitionmanager/stats_test.py b/partitionmanager/stats_test.py index 6d85023..9bfad2c 100644 --- a/partitionmanager/stats_test.py +++ b/partitionmanager/stats_test.py @@ -46,7 +46,7 @@ def test_statistics_two_partitions(self): def test_statistics_weekly_partitions_year(self): parts = list() base = datetime(2020, 5, 20, tzinfo=timezone.utc) - for w in range(0, 52): + for w in range(52): partName = f"p_{base + timedelta(weeks=w):%Y%m%d}" parts.append(mkPPart(partName, w * 1024)) parts.append(MaxValuePartition(f"p_{base + timedelta(weeks=52):%Y%m%d}", 1)) diff --git a/partitionmanager/table_append_partition.py b/partitionmanager/table_append_partition.py index 7ac0a81..266bdd0 100644 --- a/partitionmanager/table_append_partition.py +++ b/partitionmanager/table_append_partition.py @@ -107,13 +107,13 @@ def _parse_partition_map(rows): options = rows[0] - for l in options["Create Table"].split("\n"): - range_match = partition_range.match(l) + for line in options["Create Table"].split("\n"): + range_match = partition_range.match(line) if range_match: range_cols = [x.strip("` ") for x in range_match.group("cols").split(",")] log.debug(f"Partition range columns: {range_cols}") - member_match = partition_member.match(l) + member_match = partition_member.match(line) if member_match: part_name = member_match.group("name") part_vals_str = member_match.group("cols") @@ -139,7 +139,7 @@ def _parse_partition_map(rows): ) partitions.append(pos_part) - member_tail = partition_tail.match(l) + member_tail = partition_tail.match(line) if member_tail: if range_cols is None: raise partitionmanager.types.TableInformationException( @@ -200,7 +200,7 @@ def _split_partitions_around_position(partition_list, current_position): if not partitionmanager.types.is_partition_type(p): raise partitionmanager.types.UnexpectedPartitionException(p) if not isinstance(current_position, partitionmanager.types.Position): - raise ValueError() + raise ValueError less_than_partitions = list() greater_or_equal_partitions = list() @@ -481,7 +481,7 @@ def _plan_partition_changes( "as without an empty partition to manipulate, you'll need to " "perform an expensive copy operation. See the bootstrap mode." ) - raise partitionmanager.types.NoEmptyPartitionsAvailableException() + raise partitionmanager.types.NoEmptyPartitionsAvailableException if not active_partition: raise Exception("Active Partition can't be None") @@ -626,8 +626,7 @@ def _should_run_changes(table, altered_partitions): log.debug(f"{p} is new") return True - if isinstance(p, partitionmanager.types.ChangePlannedPartition): - if p.important(): + if isinstance(p, partitionmanager.types.ChangePlannedPartition) and p.important(): log.debug(f"{p} is marked important") return True return False diff --git a/partitionmanager/table_append_partition_test.py b/partitionmanager/table_append_partition_test.py index c89b8ed..7858b84 100644 --- a/partitionmanager/table_append_partition_test.py +++ b/partitionmanager/table_append_partition_test.py @@ -1,4 +1,4 @@ -# flake8: noqa: E501 +# ruff: noqa: E501 import unittest import argparse @@ -7,11 +7,8 @@ ChangePlannedPartition, DatabaseCommand, DuplicatePartitionException, - MaxValuePartition, - MismatchedIdException, NewPlannedPartition, NoEmptyPartitionsAvailableException, - PositionPartition, InstantPartition, SqlInput, SqlQuery, @@ -49,7 +46,7 @@ def __init__(self): self._response = [] self._num_queries = 0 - def run(self, cmd): + def run(self, cmd): # noqa: ARG002 self._num_queries += 1 return self._response.pop() diff --git a/partitionmanager/types.py b/partitionmanager/types.py index afb5fd4..cf1e3ca 100644 --- a/partitionmanager/types.py +++ b/partitionmanager/types.py @@ -18,9 +18,8 @@ def timedelta_from_dict(r): for k, v in r.items(): if k == "days": return timedelta(days=v) - raise argparse.ArgumentTypeError( - f"Unknown retention period definition: {k}={v}" - ) + raise argparse.ArgumentTypeError(f"Unknown retention period definition: {k}={v}") + return None class Table: @@ -96,32 +95,22 @@ def __new__(cls, *args): raise argparse.ArgumentTypeError(f"{args} is not a single argument") query_string = args[0].strip() if not query_string.endswith(";"): - raise argparse.ArgumentTypeError( - f"[{query_string}] does not end with a ';'" - ) + raise argparse.ArgumentTypeError(f"[{query_string}] does not end with a ';'") if query_string.count(";") > 1: - raise argparse.ArgumentTypeError( - f"[{query_string}] has more than one statement" - ) + raise argparse.ArgumentTypeError(f"[{query_string}] has more than one statement") if "?" not in query_string: - raise argparse.ArgumentTypeError( - f"[{query_string}] has no substitution variable '?'" - ) + raise argparse.ArgumentTypeError(f"[{query_string}] has no substitution variable '?'") if query_string.count("?") > 1: raise argparse.ArgumentTypeError( f"[{query_string}] has more than one substitution variable '?'" ) if not query_string.upper().startswith("SELECT "): - raise argparse.ArgumentTypeError( - f"[{query_string}] is not a SELECT statement" - ) + raise argparse.ArgumentTypeError(f"[{query_string}] is not a SELECT statement") for term in SqlQuery.forbidden_terms: if term in query_string.upper(): - raise argparse.ArgumentTypeError( - f"[{query_string}] has a forbidden term [{term}]" - ) + raise argparse.ArgumentTypeError(f"[{query_string}] has a forbidden term [{term}]") return super().__new__(cls, query_string) @@ -142,7 +131,7 @@ def to_sql_url(urlstring): urltuple = urlparse(urlstring) if urltuple.scheme.lower() != "sql": raise argparse.ArgumentTypeError(f"{urlstring} is not a valid sql://") - if urltuple.path == "/" or urltuple.path == "": + if urltuple.path in {"/", ""}: raise argparse.ArgumentTypeError(f"{urlstring} should include a db path") return urltuple except ValueError as ve: @@ -259,7 +248,7 @@ def set_position(self, position_in): """Set the list of identifiers for this position.""" if isinstance(position_in, Position): self._position = position_in.as_list() - elif isinstance(position_in, list) or isinstance(position_in, tuple): + elif isinstance(position_in, (list, tuple)): self._position = [int(p) for p in position_in] else: raise ValueError(f"Unexpected position input: {position_in}") @@ -345,10 +334,10 @@ def __lt__(self, other): # If ALL of v_mine >= v_other, then self is greater than other # If ANY of v_mine < v_other, then self is less than other - for v_mine, v_other in zip(self._position.as_list(), other_position_list): - if v_mine < v_other: - return True - return False + return any( + v_mine < v_other + for v_mine, v_other in zip(self._position.as_list(), other_position_list) + ) def __ge__(self, other): return not self < other @@ -388,7 +377,7 @@ def values(self): def __lt__(self, other): """MaxValuePartitions are always greater than every other partition.""" - if isinstance(other, list) or isinstance(other, Position): + if isinstance(other, (Position, list)): if self._count != len(other): raise UnexpectedPartitionException( f"Expected {self._count} columns but list has {len(other)}." @@ -506,11 +495,9 @@ def set_as_max_value(self): def as_partition(self): """Return a concrete Partition that can be rendered into a SQL ALTER.""" if not self._timestamp: - raise ValueError() + raise ValueError if self._position: - return PositionPartition(f"p_{self._timestamp:%Y%m%d}").set_position( - self._position - ) + return PositionPartition(f"p_{self._timestamp:%Y%m%d}").set_position(self._position) return MaxValuePartition(f"p_{self._timestamp:%Y%m%d}", count=self._num_columns) def __repr__(self): @@ -535,7 +522,7 @@ class ChangePlannedPartition(_PlannedPartition): def __init__(self, old_part): if not is_partition_type(old_part): - raise ValueError() + raise ValueError super().__init__() self._old = old_part self._num_columns = self._old.num_columns diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..779563e --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,83 @@ +[tool.ruff] +line-length = 99 # default is 88 +target-version = "py38" + +[tool.ruff.lint] +select = [ + "A", # flake8-builtins + "AIR", # Airflow + "ARG", # flake8-unused-arguments + "ASYNC", # flake8-async + "B", # flake8-bugbear + "BLE", # flake8-blind-except + "C90", # McCabe cyclomatic complexity + "DJ", # flake8-django + "E", # pycodestyle + "EXE", # flake8-executable + "F", # Pyflakes + "FA", # flake8-future-annotations + "FBT", # flake8-boolean-trap + "FIX", # flake8-fixme + "FLY", # flynt + "ICN", # flake8-import-conventions + "INP", # flake8-no-pep420 + "INT", # flake8-gettext + "LOG", # flake8-logging + "NPY", # NumPy-specific rules + "PD", # pandas-vet + "PERF", # Perflint + "PGH", # pygrep-hooks + "PIE", # flake8-pie + "PL", # Pylint + "PTH", # flake8-use-pathlib + "PYI", # flake8-pyi + "RET", # flake8-return + "RSE", # flake8-raise + "S", # flake8-bandit + "SIM", # flake8-simplify + "SLOT", # flake8-slots + "T10", # flake8-debugger + "TCH", # flake8-type-checking + "TD", # flake8-todos + "TID", # flake8-tidy-imports + "TRIO", # flake8-trio + "UP", # pyupgrade + "W", # pycodestyle + "YTT", # flake8-2020 + # "ANN", # flake8-annotations + # "C4", # flake8-comprehensions + # "COM", # flake8-commas + # "CPY", # flake8-copyright + # "D", # pydocstyle + # "DTZ", # flake8-datetimez + # "EM", # flake8-errmsg + # "ERA", # eradicate + # "FURB", # refurb + # "G", # flake8-logging-format + # "I", # isort + # "ISC", # flake8-implicit-str-concat + # "N", # pep8-naming + # "PT", # flake8-pytest-style + # "Q", # flake8-quotes + # "RUF", # Ruff-specific rules + # "SLF", # flake8-self + # "T20", # flake8-print + # "TRY", # tryceratops +] +ignore = ["S101"] # Allow assert statements + +[tool.ruff.lint.mccabe] +max-complexity = 16 # default is 10 + +[tool.ruff.lint.per-file-ignores] +"partitionmanager/cli.py" = ["B008"] # TODO: Fix me +"partitionmanager/cli_test.py" = ["S608", "SIM115", "SIM117"] # TODO: Fix SIMs +"partitionmanager/sql.py" = ["B904", "S603"] # TODO: Fix S603 +"partitionmanager/table_append_partition.py" = ["S608", "SIM102"] # TODO: Fix S608 +"partitionmanager/types.py" = ["B904", "RET505", "SLOT000"] # TODO: Fix B904 and SLOT000 +"partitionmanager/types_test.py" = ["B015"] # TODO: Fix me + +[tool.ruff.lint.pylint] +max-args = 7 # default is 5 +max-branches = 15 # default is 12 +max-statements = 52 # default is 50