From ba9bc71f60b34d5ba9ef005089ad4fbdd07ff659 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B0=D1=80=D1=82=D1=8B=D0=BD=D0=BE=D0=B2=20=D0=9C?= =?UTF-8?q?=D0=B0=D0=BA=D1=81=D0=B8=D0=BC=20=D0=A1=D0=B5=D1=80=D0=B3=D0=B5?= =?UTF-8?q?=D0=B5=D0=B2=D0=B8=D1=87?= Date: Wed, 13 May 2026 15:04:31 +0300 Subject: [PATCH] [DOP-36877] Fix cleanup_partitions script --- Makefile | 2 +- data_rentgen/db/scripts/cleanup_partitions.py | 61 ++++++++++--------- docker-compose.yml | 2 +- docs/changelog/next_release/449.bugfix.rst | 1 + .../database/cleanup_partitions_cli.rst | 4 +- docs/reference/database/index.rst | 2 +- .../database/cleanup_partitions_cli.md | 4 +- mddocs/reference/database/index.md | 2 +- 8 files changed, 41 insertions(+), 37 deletions(-) create mode 100644 docs/changelog/next_release/449.bugfix.rst diff --git a/Makefile b/Makefile index 6d937e6d..0d2f94ca 100644 --- a/Makefile +++ b/Makefile @@ -64,7 +64,7 @@ db-cleanup-partitions: ##@DB Clean partitions ${PYTHON} -m data_rentgen.db.scripts.cleanup_partitions $(ARGS) db-cleanup-partitions-ci: ##@DB Clean partitions in CI - ${PYTHON} -m data_rentgen.db.scripts.cleanup_partitions $(ARGS) + ${COVERAGE} run -m data_rentgen.db.scripts.cleanup_partitions $(ARGS) db-views: ##@DB Create views ${UV} run coverage run python -m data_rentgen.db.scripts.refresh_analytic_views $(ARGS) diff --git a/data_rentgen/db/scripts/cleanup_partitions.py b/data_rentgen/db/scripts/cleanup_partitions.py index 6bcb1f53..fe1d8a3f 100755 --- a/data_rentgen/db/scripts/cleanup_partitions.py +++ b/data_rentgen/db/scripts/cleanup_partitions.py @@ -40,10 +40,9 @@ ] POSTGRES_GET_PARTITIONS_QUERY = ( - "SELECT relname as table_name " + "SELECT relname as table_name, relispartition as is_attached " "FROM pg_class " "WHERE relname like ANY(:table_prefixes) " - "AND relispartition = True " "AND relkind = 'r' " "ORDER BY relname" ) @@ -90,6 +89,7 @@ class TablePartition: name: str begin_date: date granularity: Literal["year", "month", "day"] = field(default="year") + is_attached: bool = True @property def qualified_name(self): @@ -105,7 +105,7 @@ async def get_partitioned_tables(session: AsyncSession) -> dict[str, list[TableP logger.info("There is no partitioned tables") return {} # parse tabel_names - for (tabel_name,) in table_names: + for tabel_name, is_attached in table_names: granularity: Literal["year", "month", "day"] = "year" match = re.search(PARTITION_GRANULARITY_PATERN, tabel_name) if not match: @@ -124,11 +124,18 @@ async def get_partitioned_tables(session: AsyncSession) -> dict[str, list[TableP else: day = 1 - tables[name].append(TablePartition(name=name, begin_date=date(year, month, day), granularity=granularity)) + tables[name].append( + TablePartition( + name=name, + begin_date=date(year, month, day), + granularity=granularity, + is_attached=is_attached, + ), + ) return tables -def get_tables_partitions(tables: dict[str, list[TablePartition]], keep_after: date) -> dict[str, list[TablePartition]]: +def filter_partitions(tables: dict[str, list[TablePartition]], keep_after: date) -> dict[str, list[TablePartition]]: return { table_name: [partition for partition in partitions if partition.begin_date < keep_after] for table_name, partitions in tables.items() @@ -136,24 +143,20 @@ def get_tables_partitions(tables: dict[str, list[TablePartition]], keep_after: d def get_queries(tables: dict[str, list[TablePartition]], command: Command) -> list[str]: - match command: - case Command.DETACH: - query_template = "ALTER TABLE {table_name} DETACH PARTITION {qualified_name};" - case Command.DROP: - query_template = "DROP TABLE {qualified_name};" - case Command.TRUNCATE: - query_template = "TRUNCATE TABLE {qualified_name};" - case _: - return [] - - return [ - query_template.format( - table_name=table_name, - qualified_name=partition.qualified_name, - ) - for table_name, partitions in tables.items() - for partition in partitions - ] + result: list[str] = [] + for table_name, partitions in tables.items(): + for partition in partitions: + match command: + case Command.DETACH if partition.is_attached: + query_template = f"ALTER TABLE {table_name} DETACH PARTITION {partition.qualified_name};" + case Command.TRUNCATE if partition.is_attached: + query_template = f"TRUNCATE TABLE {partition.qualified_name};" + case Command.DROP: + query_template = f"DROP TABLE {partition.qualified_name};" + case _: + continue + result.append(query_template) + return result def print_partitions(tables: dict[str, list[TablePartition]]): @@ -214,20 +217,20 @@ async def main(args: list[str]) -> None: session_factory = create_session_factory(db_settings) async with session_factory() as session: tables = await get_partitioned_tables(session) - tables_to_remove = get_tables_partitions(tables, keep_after) # type: ignore[arg-type] + partitions = filter_partitions(tables, keep_after) # type: ignore[arg-type] logger.info("Partitions matching --keep-after %s:", keep_after) - print_partitions(tables_to_remove) + print_partitions(partitions) match params.command: case Command.DRY_RUN: pass case Command.DETACH: - await detach_partitions(tables_to_remove, session) + await detach_partitions(partitions, session) case Command.DROP: - await detach_partitions(tables_to_remove, session) - await drop_partitions(tables_to_remove, session) + await detach_partitions(partitions, session) + await drop_partitions(partitions, session) case Command.TRUNCATE: - await truncate_partitions(tables_to_remove, session) + await truncate_partitions(partitions, session) case _: logger.error("No such command: %s", params.command) diff --git a/docker-compose.yml b/docker-compose.yml index 07b2b4ca..78423177 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,7 +47,7 @@ services: db-cleanup-partitions: image: mtsrus/data-rentgen:${VERSION:-latest} command: | - python -m data_rentgen.db.scripts.cleanup_partitions truncate --keep-after $(date --date='-1year' '+%Y-%m-%d') + python -m data_rentgen.db.scripts.cleanup_partitions truncate --keep-after $(date --date='-1 year' '+%Y-%m-%d') env_file: .env.docker depends_on: db-migration: diff --git a/docs/changelog/next_release/449.bugfix.rst b/docs/changelog/next_release/449.bugfix.rst new file mode 100644 index 00000000..dd69d217 --- /dev/null +++ b/docs/changelog/next_release/449.bugfix.rst @@ -0,0 +1 @@ +Fix ``data_rentgen.db.scripts.cleanup_partitions`` script ignored detached partitions while dropping old partitions. diff --git a/docs/reference/database/cleanup_partitions_cli.rst b/docs/reference/database/cleanup_partitions_cli.rst index 1014a136..54fefd1a 100644 --- a/docs/reference/database/cleanup_partitions_cli.rst +++ b/docs/reference/database/cleanup_partitions_cli.rst @@ -50,7 +50,7 @@ This command will log which partitions would be affected if you were to clean up .. code:: shell - python3 -m data_rentgen.db.scripts.cleanup_partitions detach_partitions --keep-after 2024-01-01 + python3 -m data_rentgen.db.scripts.cleanup_partitions detach --keep-after 2024-01-01 This will detach all partitions created before January 1, 2024, from their parent tables. The detached tables will still exist with their data. @@ -58,7 +58,7 @@ This will detach all partitions created before January 1, 2024, from their paren .. code:: shell - python3 -m data_rentgen.db.scripts.cleanup_partitions remove_data --keep-after 2024-01-01 + python3 -m data_rentgen.db.scripts.cleanup_partitions drop --keep-after 2024-01-01 This will detach and then **drop all partitions** created before January 1, 2024, permanently deleting their data. diff --git a/docs/reference/database/index.rst b/docs/reference/database/index.rst index ab87cf96..4976b158 100644 --- a/docs/reference/database/index.rst +++ b/docs/reference/database/index.rst @@ -166,7 +166,7 @@ Without Docker # read settings from .env file, and run script using a specific venv with all required dependencies 0 0 * * * /bin/bash -c "source /some/.env && /some/.venv/bin/python -m data_rentgen.db.scripts.create_partitions" - 0 0 * * * /bin/bash -c "source /some/.env && /some/.venv/bin/python -m data_rentgen.db.scripts.cleanup_partitions truncate --keep-after $(date --date='-1year' '+%Y-%m-%d')" + 0 0 * * * /bin/bash -c "source /some/.env && /some/.venv/bin/python -m data_rentgen.db.scripts.cleanup_partitions truncate --keep-after $(date --date='-1 year' '+%Y-%m-%d')" 0 0 * * * /bin/bash -c "source /some/.env && /some/.venv/bin/python -m data_rentgen.db.scripts.refresh_analytic_views" See also diff --git a/mddocs/reference/database/cleanup_partitions_cli.md b/mddocs/reference/database/cleanup_partitions_cli.md index e5ee34df..f6a05bd0 100644 --- a/mddocs/reference/database/cleanup_partitions_cli.md +++ b/mddocs/reference/database/cleanup_partitions_cli.md @@ -37,7 +37,7 @@ It's automatically inditifies partitioned tables and their granularity. 2. Detach Partitions Older Than a Specific Date: ```shell - python3 -m data_rentgen.db.scripts.cleanup_partitions detach_partitions --keep-after 2024-01-01 + python3 -m data_rentgen.db.scripts.cleanup_partitions detach --keep-after 2024-01-01 ``` This will detach all partitions created before January 1, 2024, from their parent tables. The detached tables will still exist with their data. @@ -45,7 +45,7 @@ It's automatically inditifies partitioned tables and their granularity. 3. Remove Data and Drop Partitions Older Than a Specific Date: ```shell - python3 -m data_rentgen.db.scripts.cleanup_partitions remove_data --keep-after 2024-01-01 + python3 -m data_rentgen.db.scripts.cleanup_partitions drop --keep-after 2024-01-01 ``` This will detach and then **drop all partitions** created before January 1, 2024, permanently deleting their data. diff --git a/mddocs/reference/database/index.md b/mddocs/reference/database/index.md index 77fd040f..6000f07c 100644 --- a/mddocs/reference/database/index.md +++ b/mddocs/reference/database/index.md @@ -158,7 +158,7 @@ By default, database is created with no data. To seed database with some example ```text # read settings from .env file, and run script using a specific venv with all required dependencies 0 0 * * * /bin/bash -c "source /some/.env && /some/.venv/bin/python -m data_rentgen.db.scripts.create_partitions" - 0 0 * * * /bin/bash -c "source /some/.env && /some/.venv/bin/python -m data_rentgen.db.scripts.cleanup_partitions truncate --keep-after $(date --date='-1year' '+%Y-%m-%d')" + 0 0 * * * /bin/bash -c "source /some/.env && /some/.venv/bin/python -m data_rentgen.db.scripts.cleanup_partitions truncate --keep-after $(date --date='-1 year' '+%Y-%m-%d')" 0 0 * * * /bin/bash -c "source /some/.env && /some/.venv/bin/python -m data_rentgen.db.scripts.refresh_analytic_views" ```