From 9993eb7b53f7c1af4466477531ff18ee406aed7b Mon Sep 17 00:00:00 2001 From: Cornelius Roemer Date: Tue, 30 Jun 2026 16:57:03 +0200 Subject: [PATCH] feat(backend): keep only latest preprocessing pipeline version Previously the pipeline-version garbage collector kept the two most recent preprocessing pipeline versions (passing `latestVersion - 1` as the earliest version to keep). Change it to keep only the latest version (`latestVersion`), so outdated preprocessed data is no longer retained unnecessarily. Also updates the GC test assertions to expect only the latest version surviving, and clarifies the backend AGENTS.md test instructions to prefer the Docker route when Docker is present. Co-Authored-By: Claude Opus 4.8 --- backend/AGENTS.md | 4 ++-- .../submission/UseNewerProcessingPipelineVersionTask.kt | 2 +- .../UseNewerProcessingPipelineVersionTaskTest.kt | 9 +++++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/backend/AGENTS.md b/backend/AGENTS.md index d1a85fe88d..c1dc755b36 100644 --- a/backend/AGENTS.md +++ b/backend/AGENTS.md @@ -1,10 +1,10 @@ Kotlin dependent packages have already been installed for you. -Run tests like this (if you have Docker set up properly): +First check whether Docker is present (e.g. `docker info`). If it is, prefer running tests through Docker: ./gradlew test --console=plain -If that doesn't work due to Docker issues because you're running inside a cloud environment, try this: +Only fall back to the non-Docker route if Docker is not present or the Docker-based run fails (e.g. when running inside a cloud environment): USE_NONDOCKER_INFRA=true ./gradlew test --console=plain diff --git a/backend/src/main/kotlin/org/loculus/backend/service/submission/UseNewerProcessingPipelineVersionTask.kt b/backend/src/main/kotlin/org/loculus/backend/service/submission/UseNewerProcessingPipelineVersionTask.kt index d3953ba565..5e8d9c3bcd 100644 --- a/backend/src/main/kotlin/org/loculus/backend/service/submission/UseNewerProcessingPipelineVersionTask.kt +++ b/backend/src/main/kotlin/org/loculus/backend/service/submission/UseNewerProcessingPipelineVersionTask.kt @@ -31,7 +31,7 @@ class UseNewerProcessingPipelineVersionTask(private val submissionDatabaseServic newVersions.forEach { (organism, latestVersion) -> if (latestVersion != null) { - submissionDatabaseService.cleanUpOutdatedPreprocessingData(organism, latestVersion - 1) + submissionDatabaseService.cleanUpOutdatedPreprocessingData(organism, latestVersion) } } diff --git a/backend/src/test/kotlin/org/loculus/backend/service/submission/UseNewerProcessingPipelineVersionTaskTest.kt b/backend/src/test/kotlin/org/loculus/backend/service/submission/UseNewerProcessingPipelineVersionTaskTest.kt index a164a95c61..5084755e06 100644 --- a/backend/src/test/kotlin/org/loculus/backend/service/submission/UseNewerProcessingPipelineVersionTaskTest.kt +++ b/backend/src/test/kotlin/org/loculus/backend/service/submission/UseNewerProcessingPipelineVersionTaskTest.kt @@ -153,8 +153,9 @@ class UseNewerProcessingPipelineVersionTaskTest( useNewerProcessingPipelineVersionTask.task() transaction { - // check that nothing got deleted yet - assertThat(getExistingPipelineVersions(DEFAULT_ORGANISM), `is`(listOf(1L, 2L))) + // check that v1 for DEFAULT_ORGANISM is deleted (only the latest version is kept), + // but OTHER_ORGANISM is untouched + assertThat(getExistingPipelineVersions(DEFAULT_ORGANISM), `is`(listOf(2L))) assertThat(getExistingPipelineVersions(OTHER_ORGANISM), `is`(listOf(1L))) } @@ -166,8 +167,8 @@ class UseNewerProcessingPipelineVersionTaskTest( assertThat(submissionDatabaseService.getCurrentProcessingPipelineVersion(Organism(DEFAULT_ORGANISM)), `is`(3L)) transaction { - // check that v1 for DEFAULT_ORGANISM is deleted, but not for OTHER_ORGANISM - assertThat(getExistingPipelineVersions(DEFAULT_ORGANISM), `is`(listOf(2L, 3L))) + // check that only the latest version (v3) for DEFAULT_ORGANISM remains, but not for OTHER_ORGANISM + assertThat(getExistingPipelineVersions(DEFAULT_ORGANISM), `is`(listOf(3L))) assertThat(getExistingPipelineVersions(OTHER_ORGANISM), `is`(listOf(1L))) } }