diff --git a/.gitignore b/.gitignore index 97ebaa67..80fbf68f 100644 --- a/.gitignore +++ b/.gitignore @@ -28,3 +28,4 @@ dist/ # Ignore PyCharm / IntelliJ files .idea/ +*.iml \ No newline at end of file diff --git a/docs/source/table_partitioning.rst b/docs/source/table_partitioning.rst index 1bb5ba6f..eba33c10 100644 --- a/docs/source/table_partitioning.rst +++ b/docs/source/table_partitioning.rst @@ -177,6 +177,16 @@ Time-based partitioning ]) +Running management operations in a non atomic way +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Partitions creation and deletion can be done in a non-atomic way. +This can be useful to reduce lock contention when performing partition operations on a table while it is under heavy load. +Note that obviously this can lead to partially created/deleted partitions if something goes wrong during the operations. +By default all operations are done in an atomic way. + +You can disable atomic operations by setting the `atomic` parameter to `False` in the `PostgresPartitioningConfig` constructor. + Changing a time partitioning strategy ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/psqlextra/backend/schema.py b/psqlextra/backend/schema.py index 28e9211a..9c0a6ef6 100644 --- a/psqlextra/backend/schema.py +++ b/psqlextra/backend/schema.py @@ -75,6 +75,7 @@ class PostgresSchemaEditor(SchemaEditor): sql_add_range_partition = ( "CREATE TABLE %s PARTITION OF %s FOR VALUES FROM (%s) TO (%s)" ) + sql_detach_partition = "ALTER TABLE %s DETACH PARTITION %s" sql_add_list_partition = ( "CREATE TABLE %s PARTITION OF %s FOR VALUES IN (%s)" ) @@ -807,11 +808,16 @@ def add_default_partition( def delete_partition(self, model: Type[Model], name: str) -> None: """Deletes the partition with the specified name.""" - - sql = self.sql_delete_partition % self.quote_name( - self.create_partition_table_name(model, name) + partition_table_name = self.create_partition_table_name(model, name) + detach_sql = self.sql_detach_partition % ( + self.quote_name(model._meta.db_table), + self.quote_name(partition_table_name), ) - self.execute(sql) + delete_sql = self.sql_delete_partition % self.quote_name( + partition_table_name + ) + self.execute(detach_sql) + self.execute(delete_sql) def alter_db_table( self, model: Type[Model], old_db_table: str, new_db_table: str diff --git a/psqlextra/partitioning/config.py b/psqlextra/partitioning/config.py index 976bf1ae..07cc729b 100644 --- a/psqlextra/partitioning/config.py +++ b/psqlextra/partitioning/config.py @@ -13,9 +13,11 @@ def __init__( self, model: Type[PostgresPartitionedModel], strategy: PostgresPartitioningStrategy, + atomic: bool = True, ) -> None: self.model = model self.strategy = strategy + self.atomic = atomic __all__ = ["PostgresPartitioningConfig"] diff --git a/psqlextra/partitioning/plan.py b/psqlextra/partitioning/plan.py index 3fcac44d..a5882d0a 100644 --- a/psqlextra/partitioning/plan.py +++ b/psqlextra/partitioning/plan.py @@ -1,5 +1,8 @@ +import contextlib +import sys + from dataclasses import dataclass, field -from typing import TYPE_CHECKING, List, Optional, cast +from typing import TYPE_CHECKING, ContextManager, List, Optional, Union, cast from django.db import connections, transaction @@ -36,8 +39,10 @@ def apply(self, using: Optional[str]) -> None: connection = connections[using or "default"] - with transaction.atomic(): - with connection.schema_editor() as schema_editor: + with self._migration_context_manager(): + with connection.schema_editor( + atomic=self.config.atomic + ) as schema_editor: for partition in self.creations: partition.create( self.config.model, @@ -51,6 +56,22 @@ def apply(self, using: Optional[str]) -> None: cast("PostgresSchemaEditor", schema_editor), ) + def _migration_context_manager( + self, + ) -> Union[transaction.Atomic, ContextManager[None]]: + if sys.version_info >= (3, 7): + return ( + transaction.atomic() + if self.config.atomic + else contextlib.nullcontext() + ) + else: + return ( + transaction.atomic() + if self.config.atomic + else contextlib.suppress() + ) + def print(self) -> None: """Prints this model plan to the terminal in a readable format.""" diff --git a/psqlextra/partitioning/shorthands.py b/psqlextra/partitioning/shorthands.py index 30175273..e3f0dcdc 100644 --- a/psqlextra/partitioning/shorthands.py +++ b/psqlextra/partitioning/shorthands.py @@ -18,6 +18,7 @@ def partition_by_current_time( days: Optional[int] = None, max_age: Optional[relativedelta] = None, name_format: Optional[str] = None, + atomic: bool = True, ) -> PostgresPartitioningConfig: """Short-hand for generating a partitioning config that partitions the specified model by time. @@ -53,6 +54,9 @@ def partition_by_current_time( name_format: The datetime format which is being passed to datetime.strftime to generate the partition name. + + atomic: + If set to True, the partitioning operations will be run inside a transaction. """ size = PostgresTimePartitionSize( @@ -61,6 +65,7 @@ def partition_by_current_time( return PostgresPartitioningConfig( model=model, + atomic=atomic, strategy=PostgresCurrentTimePartitioningStrategy( size=size, count=count, diff --git a/tests/test_partitioning_time.py b/tests/test_partitioning_time.py index 9f6b5bf1..f1952f99 100644 --- a/tests/test_partitioning_time.py +++ b/tests/test_partitioning_time.py @@ -458,6 +458,37 @@ def test_partitioning_time_delete(kwargs, timepoints): assert len(table.partitions) == partition_count +@pytest.mark.postgres_version(lt=110000) +def test_partitioning_time_when_non_atomic(): + model = define_fake_partitioned_model( + {"timestamp": models.DateTimeField()}, {"key": ["timestamp"]} + ) + + schema_editor = connection.schema_editor() + schema_editor.create_partitioned_model(model) + + manager = PostgresPartitioningManager( + [ + partition_by_current_time( + model=model, + count=6, + days=7, + max_age=relativedelta(weeks=1), + atomic=False, + ) + ] + ) + + with freezegun.freeze_time("2019-1-1"): + manager.plan().apply() + + with freezegun.freeze_time("2019-1-15"): + manager.plan(skip_create=True).apply() + + table = _get_partitioned_table(model) + assert len(table.partitions) == 4 + + @pytest.mark.postgres_version(lt=110000) def test_partitioning_time_delete_ignore_manual(): """Tests whether partitions that were created manually are ignored.