From b85fddc62b6bc52d4be3b1ee9fe6ed576d2bd181 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 24 Jun 2026 15:03:38 +0200 Subject: [PATCH 1/5] Add collection tagging system (issue #1265) Implements tags on collections: database schema, CRUD support, tag filtering on GET /collections, and a new GET /collections/tags endpoint for frontend autocomplete. Co-Authored-By: Claude Sonnet 4.6 --- .../dashboardsbackend/api/Collection.kt | 8 + .../controller/CollectionsController.kt | 13 +- .../model/collection/CollectionModel.kt | 65 ++++- .../model/collection/CollectionTable.kt | 8 + .../model/collection/CollectionTagsTable.kt | 10 + .../migration/V1.5__add_collection_tags.sql | 9 + .../controller/CollectionsClient.kt | 12 +- .../controller/CollectionsTagsTest.kt | 230 ++++++++++++++++++ 8 files changed, 352 insertions(+), 3 deletions(-) create mode 100644 backend/src/main/kotlin/org/genspectrum/dashboardsbackend/model/collection/CollectionTagsTable.kt create mode 100644 backend/src/main/resources/db/migration/V1.5__add_collection_tags.sql create mode 100644 backend/src/test/kotlin/org/genspectrum/dashboardsbackend/controller/CollectionsTagsTest.kt diff --git a/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/api/Collection.kt b/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/api/Collection.kt index 4e9784173..3a38cb397 100644 --- a/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/api/Collection.kt +++ b/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/api/Collection.kt @@ -14,6 +14,7 @@ import kotlin.time.Instant "organism": "covid", "description": "A collection of interesting variants", "variantCount": 1, + "tags": ["europe", "flu"], "createdAt": "2026-01-01T00:00:00Z", "updatedAt": "2026-01-02T00:00:00Z" } @@ -28,6 +29,7 @@ data class Collection( val variantCount: Int, @JsonInclude(JsonInclude.Include.NON_NULL) val variants: List?, + val tags: List, val createdAt: Instant, val updatedAt: Instant, ) @@ -39,6 +41,7 @@ data class Collection( "name": "My Collection", "organism": "covid", "description": "A collection of interesting variants", + "tags": ["europe", "flu"], "variants": [ { "type": "query", @@ -55,6 +58,7 @@ data class CollectionRequest( val name: String, val organism: String, val description: String? = null, + val tags: List = emptyList(), val variants: List, ) @@ -85,5 +89,9 @@ data class CollectionRequest( data class CollectionUpdate( val name: String? = null, val description: String? = null, + val tags: List? = null, val variants: List? = null, ) + +@Schema(description = "Response containing all distinct tags in use") +data class CollectionTagsResponse(val tags: List) diff --git a/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/controller/CollectionsController.kt b/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/controller/CollectionsController.kt index d596c76a1..cb23d82dd 100644 --- a/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/controller/CollectionsController.kt +++ b/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/controller/CollectionsController.kt @@ -4,6 +4,7 @@ import io.swagger.v3.oas.annotations.Operation import io.swagger.v3.oas.annotations.Parameter import org.genspectrum.dashboardsbackend.api.Collection import org.genspectrum.dashboardsbackend.api.CollectionRequest +import org.genspectrum.dashboardsbackend.api.CollectionTagsResponse import org.genspectrum.dashboardsbackend.api.CollectionUpdate import org.genspectrum.dashboardsbackend.model.collection.CollectionModel import org.springframework.http.HttpStatus @@ -25,20 +26,30 @@ class CollectionsController(private val collectionModel: CollectionModel) { summary = "Get collections", description = "Returns collections filtered by optional userId and/or organism parameters. " + "Set includeVariants=true to include the full variant list; by default only variantCount is returned. " + - "Set excludeSystemCollections=true to exclude collections owned by the system user.", + "Set excludeSystemCollections=true to exclude collections owned by the system user. " + + "Repeatable tags parameter filters to collections that have ALL specified tags (AND semantics).", ) fun getCollections( @RequestParam(required = false) userId: Long?, @RequestParam(required = false) organism: String?, @RequestParam(required = false, defaultValue = "false") includeVariants: Boolean, @RequestParam(required = false, defaultValue = "false") excludeSystemCollections: Boolean, + @RequestParam(required = false) tags: List?, ): List = collectionModel.getCollections( userId = userId, organism = organism, includeVariants = includeVariants, excludeSystemCollections = excludeSystemCollections, + tags = tags, ) + @GetMapping("/collections/tags", produces = [MediaType.APPLICATION_JSON_VALUE]) + @Operation( + summary = "Get all distinct collection tags", + description = "Returns all distinct tags in use across all collections, alphabetically sorted.", + ) + fun getCollectionTags(): CollectionTagsResponse = collectionModel.getAllTags() + @GetMapping("/collections/{id}", produces = [MediaType.APPLICATION_JSON_VALUE]) @Operation( summary = "Get a collection by ID", diff --git a/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/model/collection/CollectionModel.kt b/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/model/collection/CollectionModel.kt index 7b645d8d7..8dd537ffd 100644 --- a/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/model/collection/CollectionModel.kt +++ b/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/model/collection/CollectionModel.kt @@ -2,6 +2,7 @@ package org.genspectrum.dashboardsbackend.model.collection import org.genspectrum.dashboardsbackend.api.Collection import org.genspectrum.dashboardsbackend.api.CollectionRequest +import org.genspectrum.dashboardsbackend.api.CollectionTagsResponse import org.genspectrum.dashboardsbackend.api.CollectionUpdate import org.genspectrum.dashboardsbackend.api.FilterObject import org.genspectrum.dashboardsbackend.api.VariantRequest @@ -18,9 +19,11 @@ import org.jetbrains.exposed.v1.core.Op import org.jetbrains.exposed.v1.core.and import org.jetbrains.exposed.v1.core.count import org.jetbrains.exposed.v1.core.eq +import org.jetbrains.exposed.v1.core.inList import org.jetbrains.exposed.v1.core.neq import org.jetbrains.exposed.v1.core.notInList import org.jetbrains.exposed.v1.jdbc.deleteWhere +import org.jetbrains.exposed.v1.jdbc.insert import org.jetbrains.exposed.v1.jdbc.select import org.jetbrains.exposed.v1.jdbc.selectAll import org.springframework.stereotype.Service @@ -35,6 +38,7 @@ class CollectionModel(private val dashboardsConfig: DashboardsConfig, private va organism: String?, includeVariants: Boolean = false, excludeSystemCollections: Boolean = false, + tags: List? = null, ): List { if (userId != null) { UserEntity.findById(userId) ?: throw NotFoundException("User $userId not found") @@ -57,10 +61,23 @@ class CollectionModel(private val dashboardsConfig: DashboardsConfig, private va collectionConditions = collectionConditions and (CollectionTable.ownedBy neq systemUserId) } } + if (!tags.isNullOrEmpty()) { + val distinctTags = tags.map { it.lowercase() }.distinct() + var matchingIds: Set? = null + for (tag in distinctTags) { + val idsWithTag = CollectionTagsTable + .selectAll() + .where { CollectionTagsTable.tag eq tag } + .mapTo(mutableSetOf()) { it[CollectionTagsTable.collectionId].value } + matchingIds = if (matchingIds == null) idsWithTag else (matchingIds intersect idsWithTag) + } + collectionConditions = collectionConditions and + (CollectionTable.id inList (matchingIds?.toList() ?: emptyList())) + } val join = CollectionTable.join(VariantTable, JoinType.LEFT) - return if (includeVariants) { + val initialCollections = if (includeVariants) { join.selectAll() .where { collectionConditions } .groupBy { it[CollectionTable.id] } @@ -77,6 +94,7 @@ class CollectionModel(private val dashboardsConfig: DashboardsConfig, private va description = first[CollectionTable.description], variantCount = variants.size, variants = variants, + tags = emptyList(), createdAt = first[CollectionTable.createdAt], updatedAt = first[CollectionTable.updatedAt], ) @@ -112,11 +130,36 @@ class CollectionModel(private val dashboardsConfig: DashboardsConfig, private va description = row[CollectionTable.description], variantCount = row[countExpr].toInt(), variants = null, + tags = emptyList(), createdAt = row[CollectionTable.createdAt], updatedAt = row[CollectionTable.updatedAt], ) } } + + return withTagsAttached(initialCollections) + } + + private fun withTagsAttached(collections: List): List { + if (collections.isEmpty()) return collections + val collectionIds = collections.map { it.id } + val tagsByCollectionId = CollectionTagsTable + .selectAll() + .where { CollectionTagsTable.collectionId inList collectionIds } + .groupBy { it[CollectionTagsTable.collectionId].value } + .mapValues { (_, rows) -> rows.map { it[CollectionTagsTable.tag] }.sorted() } + return collections.map { collection -> + collection.copy(tags = tagsByCollectionId[collection.id] ?: emptyList()) + } + } + + fun getAllTags(): CollectionTagsResponse { + val tags = CollectionTagsTable + .selectAll() + .map { it[CollectionTagsTable.tag] } + .distinct() + .sorted() + return CollectionTagsResponse(tags = tags) } fun getCollection(id: Long): Collection { @@ -146,6 +189,14 @@ class CollectionModel(private val dashboardsConfig: DashboardsConfig, private va variantEntity } + val insertedTags = request.tags.map { it.lowercase() }.distinct().sorted() + insertedTags.forEach { tag -> + CollectionTagsTable.insert { + it[collectionId] = collectionEntity.id + it[CollectionTagsTable.tag] = tag + } + } + val variants = variantEntities.map { it.toVariant() } return Collection( id = collectionEntity.id.value, @@ -155,6 +206,7 @@ class CollectionModel(private val dashboardsConfig: DashboardsConfig, private va description = collectionEntity.description, variantCount = variants.size, variants = variants, + tags = insertedTags, createdAt = collectionEntity.createdAt, updatedAt = collectionEntity.updatedAt, ) @@ -185,6 +237,17 @@ class CollectionModel(private val dashboardsConfig: DashboardsConfig, private va collectionEntity.description = update.description } + if (update.tags != null) { + CollectionTagsTable.deleteWhere { CollectionTagsTable.collectionId eq id } + val newTags = update.tags.map { it.lowercase() }.distinct() + newTags.forEach { tag -> + CollectionTagsTable.insert { + it[collectionId] = collectionEntity.id + it[CollectionTagsTable.tag] = tag + } + } + } + if (update.variants != null) { // Track which variant IDs should be kept val variantIdsToKeep = mutableSetOf() diff --git a/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/model/collection/CollectionTable.kt b/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/model/collection/CollectionTable.kt index 3b04f8282..e6bf907a3 100644 --- a/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/model/collection/CollectionTable.kt +++ b/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/model/collection/CollectionTable.kt @@ -3,9 +3,11 @@ package org.genspectrum.dashboardsbackend.model.collection import org.genspectrum.dashboardsbackend.api.Collection import org.jetbrains.exposed.v1.core.dao.id.EntityID import org.jetbrains.exposed.v1.core.dao.id.LongIdTable +import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.dao.LongEntity import org.jetbrains.exposed.v1.dao.LongEntityClass import org.jetbrains.exposed.v1.datetime.timestamp +import org.jetbrains.exposed.v1.jdbc.selectAll const val COLLECTION_TABLE = "collections_table" @@ -36,6 +38,11 @@ class CollectionEntity(id: EntityID) : LongEntity(id) { fun toCollection(): Collection { val variantList = variants.map { it.toVariant() } + val tagList = CollectionTagsTable + .selectAll() + .where { CollectionTagsTable.collectionId eq id.value } + .map { it[CollectionTagsTable.tag] } + .sorted() return Collection( id = id.value, name = name, @@ -44,6 +51,7 @@ class CollectionEntity(id: EntityID) : LongEntity(id) { description = description, variantCount = variantList.size, variants = variantList, + tags = tagList, createdAt = createdAt, updatedAt = updatedAt, ) diff --git a/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/model/collection/CollectionTagsTable.kt b/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/model/collection/CollectionTagsTable.kt new file mode 100644 index 000000000..53967bceb --- /dev/null +++ b/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/model/collection/CollectionTagsTable.kt @@ -0,0 +1,10 @@ +package org.genspectrum.dashboardsbackend.model.collection + +import org.jetbrains.exposed.v1.core.ReferenceOption +import org.jetbrains.exposed.v1.core.Table + +object CollectionTagsTable : Table("collection_tags") { + val collectionId = reference("collection_id", CollectionTable, onDelete = ReferenceOption.CASCADE) + val tag = text("tag") + override val primaryKey = PrimaryKey(collectionId, tag) +} diff --git a/backend/src/main/resources/db/migration/V1.5__add_collection_tags.sql b/backend/src/main/resources/db/migration/V1.5__add_collection_tags.sql new file mode 100644 index 000000000..a640399d1 --- /dev/null +++ b/backend/src/main/resources/db/migration/V1.5__add_collection_tags.sql @@ -0,0 +1,9 @@ +create table collection_tags ( + collection_id bigint not null, + tag text not null, + primary key (collection_id, tag), + constraint fk_collection_tags_collection foreign key (collection_id) references collections_table(id) on delete cascade, + constraint chk_tag_lowercase check (tag = lower(tag)) +); + +create index idx_collection_tags_tag on collection_tags(tag); diff --git a/backend/src/test/kotlin/org/genspectrum/dashboardsbackend/controller/CollectionsClient.kt b/backend/src/test/kotlin/org/genspectrum/dashboardsbackend/controller/CollectionsClient.kt index 7e0115666..6c26ea098 100644 --- a/backend/src/test/kotlin/org/genspectrum/dashboardsbackend/controller/CollectionsClient.kt +++ b/backend/src/test/kotlin/org/genspectrum/dashboardsbackend/controller/CollectionsClient.kt @@ -4,6 +4,7 @@ import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.module.kotlin.readValue import org.genspectrum.dashboardsbackend.api.Collection import org.genspectrum.dashboardsbackend.api.CollectionRequest +import org.genspectrum.dashboardsbackend.api.CollectionTagsResponse import org.genspectrum.dashboardsbackend.api.CollectionUpdate import org.springframework.http.MediaType import org.springframework.test.web.servlet.MockMvc @@ -32,6 +33,7 @@ class CollectionsClient(private val mockMvc: MockMvc, private val objectMapper: organism: String? = null, includeVariants: Boolean = false, excludeSystemCollections: Boolean = false, + tags: List? = null, ): ResultActions { val params = buildString { val queryParams = mutableListOf() @@ -39,6 +41,7 @@ class CollectionsClient(private val mockMvc: MockMvc, private val objectMapper: if (organism != null) queryParams.add("organism=$organism") if (includeVariants) queryParams.add("includeVariants=true") if (excludeSystemCollections) queryParams.add("excludeSystemCollections=true") + tags?.forEach { queryParams.add("tags=$it") } if (queryParams.isNotEmpty()) { append("?") append(queryParams.joinToString("&")) @@ -52,11 +55,18 @@ class CollectionsClient(private val mockMvc: MockMvc, private val objectMapper: organism: String? = null, includeVariants: Boolean = false, excludeSystemCollections: Boolean = false, + tags: List? = null, ): List = deserializeJsonResponse( - getCollectionsRaw(userId, organism, includeVariants, excludeSystemCollections) + getCollectionsRaw(userId, organism, includeVariants, excludeSystemCollections, tags) .andExpect(status().isOk), ) + fun getCollectionTagsRaw(): ResultActions = mockMvc.perform(get("/collections/tags")) + + fun getCollectionTags(): CollectionTagsResponse = deserializeJsonResponse( + getCollectionTagsRaw().andExpect(status().isOk), + ) + fun getCollectionRaw(id: Long): ResultActions = mockMvc.perform(get("/collections/$id")) fun getCollection(id: Long): Collection = deserializeJsonResponse( diff --git a/backend/src/test/kotlin/org/genspectrum/dashboardsbackend/controller/CollectionsTagsTest.kt b/backend/src/test/kotlin/org/genspectrum/dashboardsbackend/controller/CollectionsTagsTest.kt new file mode 100644 index 000000000..b9bf2958e --- /dev/null +++ b/backend/src/test/kotlin/org/genspectrum/dashboardsbackend/controller/CollectionsTagsTest.kt @@ -0,0 +1,230 @@ +package org.genspectrum.dashboardsbackend.controller + +import org.genspectrum.dashboardsbackend.api.CollectionUpdate +import org.genspectrum.dashboardsbackend.dummyCollectionRequest +import org.hamcrest.MatcherAssert.assertThat +import org.hamcrest.Matchers.contains +import org.hamcrest.Matchers.containsInAnyOrder +import org.hamcrest.Matchers.empty +import org.hamcrest.Matchers.equalTo +import org.hamcrest.Matchers.hasItem +import org.hamcrest.Matchers.not +import org.junit.jupiter.api.Test +import org.springframework.beans.factory.annotation.Autowired +import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc +import org.springframework.boot.test.context.SpringBootTest +import org.springframework.context.annotation.Import +import org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath +import org.springframework.test.web.servlet.result.MockMvcResultMatchers.status + +@SpringBootTest +@AutoConfigureMockMvc +@Import(CollectionsClient::class, UsersClient::class) +class CollectionsTagsTest( + @param:Autowired private val collectionsClient: CollectionsClient, + @param:Autowired private val usersClient: UsersClient, +) { + + @Test + fun `WHEN creating collection with tags THEN tags are returned`() { + val userId = usersClient.createUser() + val collection = collectionsClient.postCollection( + dummyCollectionRequest.copy(tags = listOf("europe", "flu")), + userId, + ) + + assertThat(collection.tags, containsInAnyOrder("europe", "flu")) + } + + @Test + fun `WHEN creating collection without tags THEN empty tags list is returned`() { + val userId = usersClient.createUser() + val collection = collectionsClient.postCollection(dummyCollectionRequest, userId) + + assertThat(collection.tags, empty()) + } + + @Test + fun `WHEN creating collection with tags THEN tags are lowercased`() { + val userId = usersClient.createUser() + val collection = collectionsClient.postCollection( + dummyCollectionRequest.copy(tags = listOf("Europe", "FLU")), + userId, + ) + + assertThat(collection.tags, containsInAnyOrder("europe", "flu")) + } + + @Test + fun `WHEN getting collection by ID THEN tags are included`() { + val userId = usersClient.createUser() + val created = collectionsClient.postCollection( + dummyCollectionRequest.copy(tags = listOf("flu")), + userId, + ) + + val retrieved = collectionsClient.getCollection(created.id) + + assertThat(retrieved.tags, contains("flu")) + } + + @Test + fun `WHEN getting collections THEN tags are included`() { + val userId = usersClient.createUser() + collectionsClient.postCollection( + dummyCollectionRequest.copy(tags = listOf("europe", "flu")), + userId, + ) + + val collections = collectionsClient.getCollections(userId = userId) + + assertThat(collections.first().tags, containsInAnyOrder("europe", "flu")) + } + + @Test + fun `GIVEN collections with different tags WHEN filtering by one tag THEN returns matching collections only`() { + val userId = usersClient.createUser() + val fluCollection = collectionsClient.postCollection( + dummyCollectionRequest.copy(name = "Flu", tags = listOf("flu")), + userId, + ) + val covidCollection = collectionsClient.postCollection( + dummyCollectionRequest.copy(name = "Covid", tags = listOf("covid")), + userId, + ) + + val results = collectionsClient.getCollections(userId = userId, tags = listOf("flu"), includeVariants = true) + + assertThat(results, hasItem(fluCollection)) + assertThat(results, not(hasItem(covidCollection))) + } + + @Test + fun `GIVEN collections WHEN filtering by two tags THEN only collections with both tags are returned`() { + val userId = usersClient.createUser() + val bothTags = collectionsClient.postCollection( + dummyCollectionRequest.copy(name = "Both", tags = listOf("flu", "europe")), + userId, + ) + val onlyFlu = collectionsClient.postCollection( + dummyCollectionRequest.copy(name = "Only flu", tags = listOf("flu")), + userId, + ) + val noTags = collectionsClient.postCollection( + dummyCollectionRequest.copy(name = "No tags"), + userId, + ) + + val results = collectionsClient.getCollections( + userId = userId, + tags = listOf("flu", "europe"), + includeVariants = true, + ) + + assertThat(results, hasItem(bothTags)) + assertThat(results, not(hasItem(onlyFlu))) + assertThat(results, not(hasItem(noTags))) + } + + @Test + fun `WHEN filtering by tag that no collection has THEN returns empty list`() { + val userId = usersClient.createUser() + collectionsClient.postCollection(dummyCollectionRequest.copy(tags = listOf("flu")), userId) + + val results = collectionsClient.getCollections(userId = userId, tags = listOf("nonexistent")) + + assertThat(results, empty()) + } + + @Test + fun `WHEN updating collection with new tags THEN tags are replaced`() { + val userId = usersClient.createUser() + val created = collectionsClient.postCollection( + dummyCollectionRequest.copy(tags = listOf("flu", "europe")), + userId, + ) + + val updated = collectionsClient.putCollection( + CollectionUpdate(tags = listOf("covid")), + created.id, + userId, + ) + + assertThat(updated.tags, contains("covid")) + } + + @Test + fun `WHEN updating collection with empty tags list THEN all tags are removed`() { + val userId = usersClient.createUser() + val created = collectionsClient.postCollection( + dummyCollectionRequest.copy(tags = listOf("flu")), + userId, + ) + + val updated = collectionsClient.putCollection( + CollectionUpdate(tags = emptyList()), + created.id, + userId, + ) + + assertThat(updated.tags, empty()) + } + + @Test + fun `WHEN updating collection without tags field THEN existing tags are preserved`() { + val userId = usersClient.createUser() + val created = collectionsClient.postCollection( + dummyCollectionRequest.copy(tags = listOf("flu")), + userId, + ) + + val updated = collectionsClient.putCollection( + CollectionUpdate(name = "New Name"), + created.id, + userId, + ) + + assertThat(updated.tags, contains("flu")) + } + + @Test + fun `GIVEN collections with tags WHEN getting all tags THEN all created tags are present and sorted`() { + val userId = usersClient.createUser() + val uniqueTag1 = "xtest-unique-alpha-tag" + val uniqueTag2 = "xtest-unique-beta-tag" + val uniqueTag3 = "xtest-unique-gamma-tag" + collectionsClient.postCollection(dummyCollectionRequest.copy(tags = listOf(uniqueTag1, uniqueTag2)), userId) + collectionsClient.postCollection(dummyCollectionRequest.copy(tags = listOf(uniqueTag2, uniqueTag3)), userId) + + val response = collectionsClient.getCollectionTags() + + assertThat(response.tags, hasItem(uniqueTag1)) + assertThat(response.tags, hasItem(uniqueTag2)) + assertThat(response.tags, hasItem(uniqueTag3)) + val idx1 = response.tags.indexOf(uniqueTag1) + val idx2 = response.tags.indexOf(uniqueTag2) + val idx3 = response.tags.indexOf(uniqueTag3) + assertThat(idx1 < idx2, equalTo(true)) + assertThat(idx2 < idx3, equalTo(true)) + } + + @Test + fun `WHEN getting collection tags THEN tags response contains tags field`() { + collectionsClient.getCollectionTagsRaw() + .andExpect(status().isOk) + .andExpect(jsonPath("$.tags").isArray) + } + + @Test + fun `WHEN filtering by tag with uppercase THEN matches lowercased stored tags`() { + val userId = usersClient.createUser() + val collection = collectionsClient.postCollection( + dummyCollectionRequest.copy(tags = listOf("flu")), + userId, + ) + + val results = collectionsClient.getCollections(userId = userId, tags = listOf("FLU"), includeVariants = true) + + assertThat(results, hasItem(collection)) + } +} From 87ffbd8d3d49bc9dec1c2113012324de180485e8 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Wed, 24 Jun 2026 17:44:18 +0200 Subject: [PATCH 2/5] refactor(collections): load tags via three-table JOIN with string_agg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the separate batch query for tags with a PostgreSQL string_agg aggregate over a three-table JOIN (collections × variants × tags), eliminating the extra round-trip to the database. Uses GROUP BY to avoid the cartesian product: - includeVariants=true: GROUP BY (c.id, v.id) + string_agg(tag ORDER BY tag) - includeVariants=false: GROUP BY c.id + COUNT(DISTINCT v.id) + string_agg(DISTINCT tag) Co-Authored-By: Claude Sonnet 4.6 --- .../model/collection/CollectionModel.kt | 49 +++++++------------ .../model/collection/CollectionTagsTable.kt | 22 +++++++++ 2 files changed, 40 insertions(+), 31 deletions(-) diff --git a/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/model/collection/CollectionModel.kt b/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/model/collection/CollectionModel.kt index 8dd537ffd..d1bbe8511 100644 --- a/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/model/collection/CollectionModel.kt +++ b/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/model/collection/CollectionModel.kt @@ -14,10 +14,10 @@ import org.genspectrum.dashboardsbackend.controller.NotFoundException import org.genspectrum.dashboardsbackend.model.user.UserEntity import org.genspectrum.dashboardsbackend.model.user.UserModel import org.genspectrum.dashboardsbackend.util.now +import org.jetbrains.exposed.v1.core.Count import org.jetbrains.exposed.v1.core.JoinType import org.jetbrains.exposed.v1.core.Op import org.jetbrains.exposed.v1.core.and -import org.jetbrains.exposed.v1.core.count import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.core.inList import org.jetbrains.exposed.v1.core.neq @@ -75,11 +75,18 @@ class CollectionModel(private val dashboardsConfig: DashboardsConfig, private va (CollectionTable.id inList (matchingIds?.toList() ?: emptyList())) } - val join = CollectionTable.join(VariantTable, JoinType.LEFT) + val join = CollectionTable + .join(VariantTable, JoinType.LEFT) + .join(CollectionTagsTable, JoinType.LEFT, CollectionTable.id, CollectionTagsTable.collectionId) - val initialCollections = if (includeVariants) { - join.selectAll() + return if (includeVariants) { + val tagsExpr = StringAgg(CollectionTagsTable.tag, orderBy = true) + val columns = CollectionTable.columns + VariantTable.columns + listOf(tagsExpr) + + join.select(columns) .where { collectionConditions } + .groupBy(CollectionTable.id, VariantTable.id) + .toList() .groupBy { it[CollectionTable.id] } .map { (_, rows) -> val first = rows.first() @@ -94,13 +101,15 @@ class CollectionModel(private val dashboardsConfig: DashboardsConfig, private va description = first[CollectionTable.description], variantCount = variants.size, variants = variants, - tags = emptyList(), + tags = first[tagsExpr]?.split(",") ?: emptyList(), createdAt = first[CollectionTable.createdAt], updatedAt = first[CollectionTable.updatedAt], ) } } else { - val countExpr = VariantTable.id.count() + val countExpr = Count(VariantTable.id, distinct = true) + val tagsExpr = StringAgg(CollectionTagsTable.tag, distinct = true) + join.select( CollectionTable.id, CollectionTable.name, @@ -110,17 +119,10 @@ class CollectionModel(private val dashboardsConfig: DashboardsConfig, private va CollectionTable.createdAt, CollectionTable.updatedAt, countExpr, + tagsExpr, ) .where { collectionConditions } - .groupBy( - CollectionTable.id, - CollectionTable.name, - CollectionTable.ownedBy, - CollectionTable.organism, - CollectionTable.description, - CollectionTable.createdAt, - CollectionTable.updatedAt, - ) + .groupBy(CollectionTable.id) .map { row -> Collection( id = row[CollectionTable.id].value, @@ -130,27 +132,12 @@ class CollectionModel(private val dashboardsConfig: DashboardsConfig, private va description = row[CollectionTable.description], variantCount = row[countExpr].toInt(), variants = null, - tags = emptyList(), + tags = row[tagsExpr]?.split(",")?.sorted() ?: emptyList(), createdAt = row[CollectionTable.createdAt], updatedAt = row[CollectionTable.updatedAt], ) } } - - return withTagsAttached(initialCollections) - } - - private fun withTagsAttached(collections: List): List { - if (collections.isEmpty()) return collections - val collectionIds = collections.map { it.id } - val tagsByCollectionId = CollectionTagsTable - .selectAll() - .where { CollectionTagsTable.collectionId inList collectionIds } - .groupBy { it[CollectionTagsTable.collectionId].value } - .mapValues { (_, rows) -> rows.map { it[CollectionTagsTable.tag] }.sorted() } - return collections.map { collection -> - collection.copy(tags = tagsByCollectionId[collection.id] ?: emptyList()) - } } fun getAllTags(): CollectionTagsResponse { diff --git a/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/model/collection/CollectionTagsTable.kt b/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/model/collection/CollectionTagsTable.kt index 53967bceb..7affce6bb 100644 --- a/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/model/collection/CollectionTagsTable.kt +++ b/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/model/collection/CollectionTagsTable.kt @@ -1,10 +1,32 @@ package org.genspectrum.dashboardsbackend.model.collection +import org.jetbrains.exposed.v1.core.Column +import org.jetbrains.exposed.v1.core.Function +import org.jetbrains.exposed.v1.core.QueryBuilder import org.jetbrains.exposed.v1.core.ReferenceOption import org.jetbrains.exposed.v1.core.Table +import org.jetbrains.exposed.v1.core.TextColumnType object CollectionTagsTable : Table("collection_tags") { val collectionId = reference("collection_id", CollectionTable, onDelete = ReferenceOption.CASCADE) val tag = text("tag") override val primaryKey = PrimaryKey(collectionId, tag) } + +class StringAgg( + private val column: Column, + private val distinct: Boolean = false, + private val orderBy: Boolean = false, +) : Function(TextColumnType()) { + override fun toQueryBuilder(queryBuilder: QueryBuilder) = queryBuilder { + append("string_agg(") + if (distinct) append("DISTINCT ") + append(column) + append(", ','") + if (orderBy && !distinct) { + append(" ORDER BY ") + append(column) + } + append(")") + } +} From 0eb0dbd1fd006a018dacc4550919d61476812873 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Thu, 25 Jun 2026 11:23:28 +0200 Subject: [PATCH 3/5] some changes --- .../model/collection/CollectionModel.kt | 74 ++++++++++++------- 1 file changed, 48 insertions(+), 26 deletions(-) diff --git a/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/model/collection/CollectionModel.kt b/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/model/collection/CollectionModel.kt index d1bbe8511..b7890c830 100644 --- a/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/model/collection/CollectionModel.kt +++ b/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/model/collection/CollectionModel.kt @@ -33,6 +33,14 @@ import kotlin.time.Instant @Service @Transactional class CollectionModel(private val dashboardsConfig: DashboardsConfig, private val userModel: UserModel) { + /** + * The core function to fetch collections. Also fetches associated variants and tags (if desired). + * + * @param includeVariants whether to fetch variants as well. Quicker to omit if not needed. + * @param excludeSystemCollections whether to exclude collections belonging to the system user. + * Useful when only community collections are desired, and filtering makes responses smaller. + * @param tags if provided, only collections containing all specified tags are returned. + */ fun getCollections( userId: Long?, organism: String?, @@ -48,32 +56,7 @@ class CollectionModel(private val dashboardsConfig: DashboardsConfig, private va dashboardsConfig.validateCollectionsEnabled(organism) } - var collectionConditions: Op = Op.TRUE - if (userId != null) { - collectionConditions = collectionConditions and (CollectionTable.ownedBy eq userId) - } - if (organism != null) { - collectionConditions = collectionConditions and (CollectionTable.organism eq organism) - } - if (excludeSystemCollections) { - val systemUserId = userModel.getSystemUserId() - if (systemUserId != null) { - collectionConditions = collectionConditions and (CollectionTable.ownedBy neq systemUserId) - } - } - if (!tags.isNullOrEmpty()) { - val distinctTags = tags.map { it.lowercase() }.distinct() - var matchingIds: Set? = null - for (tag in distinctTags) { - val idsWithTag = CollectionTagsTable - .selectAll() - .where { CollectionTagsTable.tag eq tag } - .mapTo(mutableSetOf()) { it[CollectionTagsTable.collectionId].value } - matchingIds = if (matchingIds == null) idsWithTag else (matchingIds intersect idsWithTag) - } - collectionConditions = collectionConditions and - (CollectionTable.id inList (matchingIds?.toList() ?: emptyList())) - } + val collectionConditions = buildCollectionConditions(userId, organism, excludeSystemCollections, tags) val join = CollectionTable .join(VariantTable, JoinType.LEFT) @@ -289,6 +272,45 @@ class CollectionModel(private val dashboardsConfig: DashboardsConfig, private va return collectionEntity.toCollection() } + private fun buildCollectionConditions( + userId: Long?, + organism: String?, + excludeSystemCollections: Boolean, + tags: List?, + ): Op { + var conditions: Op = Op.TRUE + if (userId != null) { + conditions = conditions and (CollectionTable.ownedBy eq userId) + } + if (organism != null) { + conditions = conditions and (CollectionTable.organism eq organism) + } + if (excludeSystemCollections) { + val systemUserId = userModel.getSystemUserId() + if (systemUserId != null) { + conditions = conditions and (CollectionTable.ownedBy neq systemUserId) + } + } + if (!tags.isNullOrEmpty()) { + val distinctTags = tags.map { it.lowercase() }.distinct() + val matchingIds = getIdsMatchingAllTags(distinctTags) + conditions = conditions and (CollectionTable.id inList matchingIds.toList()) + } + return conditions + } + + private fun getIdsMatchingAllTags(tags: List): Set { + var matchingIds: Set? = null + for (tag in tags) { + val idsWithTag = CollectionTagsTable + .selectAll() + .where { CollectionTagsTable.tag eq tag } + .mapTo(mutableSetOf()) { it[CollectionTagsTable.collectionId].value } + matchingIds = if (matchingIds == null) idsWithTag else (matchingIds intersect idsWithTag) + } + return matchingIds ?: emptySet() + } + private fun createVariantEntity( collectionEntity: CollectionEntity, variantRequest: VariantRequest, From 8df6c3e5d1c661bb676ab369100169eb437bc246 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Thu, 25 Jun 2026 11:28:45 +0200 Subject: [PATCH 4/5] refactor(collections): replace Kotlin-side tag intersection with SQL subquery Instead of iterating over tags and intersecting ID sets in Kotlin (one query per tag), use a single subquery: SELECT collection_id FROM collection_tags WHERE tag IN (...) GROUP BY collection_id HAVING COUNT(DISTINCT tag) = N This keeps the filter as one round-trip and avoids materializing ID sets in application memory. Co-Authored-By: Claude Sonnet 4.6 --- .../model/collection/CollectionModel.kt | 21 +++++++------------ 1 file changed, 7 insertions(+), 14 deletions(-) diff --git a/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/model/collection/CollectionModel.kt b/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/model/collection/CollectionModel.kt index b7890c830..27b78fb2a 100644 --- a/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/model/collection/CollectionModel.kt +++ b/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/model/collection/CollectionModel.kt @@ -20,6 +20,7 @@ import org.jetbrains.exposed.v1.core.Op import org.jetbrains.exposed.v1.core.and import org.jetbrains.exposed.v1.core.eq import org.jetbrains.exposed.v1.core.inList +import org.jetbrains.exposed.v1.core.inSubQuery import org.jetbrains.exposed.v1.core.neq import org.jetbrains.exposed.v1.core.notInList import org.jetbrains.exposed.v1.jdbc.deleteWhere @@ -293,24 +294,16 @@ class CollectionModel(private val dashboardsConfig: DashboardsConfig, private va } if (!tags.isNullOrEmpty()) { val distinctTags = tags.map { it.lowercase() }.distinct() - val matchingIds = getIdsMatchingAllTags(distinctTags) - conditions = conditions and (CollectionTable.id inList matchingIds.toList()) + val tagSubquery = CollectionTagsTable + .select(CollectionTagsTable.collectionId) + .where { CollectionTagsTable.tag inList distinctTags } + .groupBy(CollectionTagsTable.collectionId) + .having { Count(CollectionTagsTable.tag, distinct = true) eq distinctTags.size.toLong() } + conditions = conditions and (CollectionTable.id inSubQuery tagSubquery) } return conditions } - private fun getIdsMatchingAllTags(tags: List): Set { - var matchingIds: Set? = null - for (tag in tags) { - val idsWithTag = CollectionTagsTable - .selectAll() - .where { CollectionTagsTable.tag eq tag } - .mapTo(mutableSetOf()) { it[CollectionTagsTable.collectionId].value } - matchingIds = if (matchingIds == null) idsWithTag else (matchingIds intersect idsWithTag) - } - return matchingIds ?: emptySet() - } - private fun createVariantEntity( collectionEntity: CollectionEntity, variantRequest: VariantRequest, From 6e856c0ca3fee4190bcfefbfc2e4bf685d5c5c02 Mon Sep 17 00:00:00 2001 From: Felix Hennig Date: Thu, 25 Jun 2026 13:58:59 +0200 Subject: [PATCH 5/5] refactor(collections): use SQL DISTINCT+ORDER BY in getAllTags instead of in-memory sort Co-Authored-By: Claude Sonnet 4.6 --- .../dashboardsbackend/model/collection/CollectionModel.kt | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/model/collection/CollectionModel.kt b/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/model/collection/CollectionModel.kt index 27b78fb2a..3e84eea79 100644 --- a/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/model/collection/CollectionModel.kt +++ b/backend/src/main/kotlin/org/genspectrum/dashboardsbackend/model/collection/CollectionModel.kt @@ -26,7 +26,6 @@ import org.jetbrains.exposed.v1.core.notInList import org.jetbrains.exposed.v1.jdbc.deleteWhere import org.jetbrains.exposed.v1.jdbc.insert import org.jetbrains.exposed.v1.jdbc.select -import org.jetbrains.exposed.v1.jdbc.selectAll import org.springframework.stereotype.Service import org.springframework.transaction.annotation.Transactional import kotlin.time.Instant @@ -126,10 +125,10 @@ class CollectionModel(private val dashboardsConfig: DashboardsConfig, private va fun getAllTags(): CollectionTagsResponse { val tags = CollectionTagsTable - .selectAll() + .select(CollectionTagsTable.tag) + .withDistinct(true) + .orderBy(CollectionTagsTable.tag) .map { it[CollectionTagsTable.tag] } - .distinct() - .sorted() return CollectionTagsResponse(tags = tags) }