From 5d5da0d021d55fa82681402a482eaee7a2d01bc0 Mon Sep 17 00:00:00 2001 From: Kiryl_Kurnosenka Date: Mon, 10 Aug 2026 18:25:39 +0300 Subject: [PATCH 01/15] feat: short-name resolution via derivation + schema $id index #1783 Makes every materialized platform-bucket config entity short-name addressed, inbound and outbound, via deterministic derivation (no per-entity flag, no stored alias index) for models/interceptors/ roles/applications/toolsets, and adds a $id -> canonical-id index so blob-stored app-type/catalog schemas resolve by their JSON-Schema $id. --- .../com/epam/aidial/core/config/Config.java | 72 ++++++- .../epam/aidial/core/config/ConfigTest.java | 76 ++++++++ .../server/config/BlobEntityValidator.java | 4 +- .../server/config/ConfigPostProcessor.java | 59 ++++-- .../core/server/config/MergedConfigStore.java | 147 +++++++++++--- .../controller/BaseInterceptorController.java | 2 +- .../controller/DeploymentController.java | 2 +- .../server/controller/ModelController.java | 2 +- .../server/controller/ResourceController.java | 2 +- .../CollectResponseAttachmentsFn.java | 2 +- .../core/server/limiter/RateLimiter.java | 23 ++- .../core/server/service/ShareService.java | 5 +- .../core/server/CanonicalIdListingTest.java | 34 ++-- .../core/server/MergedConfigStoreApiTest.java | 180 +++++++++++++++++- .../config/ConfigPostProcessorTest.java | 3 +- 15 files changed, 518 insertions(+), 95 deletions(-) diff --git a/config/src/main/java/com/epam/aidial/core/config/Config.java b/config/src/main/java/com/epam/aidial/core/config/Config.java index 2842af901..3917e0a5a 100644 --- a/config/src/main/java/com/epam/aidial/core/config/Config.java +++ b/config/src/main/java/com/epam/aidial/core/config/Config.java @@ -56,43 +56,97 @@ public class Config { private List globalInterceptors = List.of(); + /** + * $id → canonical-id index for {@code platform}-bucket schema entities, built at rebuild + * time from blob bodies. Bridges $id-keyed file entries and canonical-id-keyed blob entries + * in {@link #applicationTypeSchemas}, since a schema's $id is not derivable from its path. + */ + @JsonIgnore + private Map schemaAliasesById = Map.of(); + + @JsonIgnore + private Map catalogSchemaAliasesById = Map.of(); + @JsonIgnore public Deployment selectDeployment(String deploymentId) { - Application application = applications.get(deploymentId); + Application application = resolve(applications, "applications", deploymentId); if (application != null) { return application; } - Model model = models.get(deploymentId); + Model model = resolve(models, "models", deploymentId); if (model != null) { return model; } - ToolSet toolSet = toolsets.get(deploymentId); + ToolSet toolSet = resolve(toolsets, "toolsets", deploymentId); if (toolSet != null) { return toolSet; } - return interceptors.get(deploymentId); + return resolve(interceptors, "interceptors", deploymentId); } public boolean isDeploymentExists(String deploymentId) { return selectDeployment(deploymentId) != null; } + @JsonIgnore + public Model getModel(String id) { + return resolve(models, "models", id); + } + + @JsonIgnore + public Role getRole(String id) { + return resolve(roles, "roles", id); + } + + @JsonIgnore + public Interceptor getInterceptor(String id) { + return resolve(interceptors, "interceptors", id); + } + @JsonIgnore public String getCustomApplicationSchema(URI schemaId) { - if (schemaId == null) { - return null; - } - return applicationTypeSchemas.get(schemaId.toString()); + return resolveSchema(applicationTypeSchemas, schemaAliasesById, schemaId); } @JsonIgnore public String getCatalogSchema(URI schemaId) { + return resolveSchema(catalogSchemas, catalogSchemaAliasesById, schemaId); + } + + /** + * Resolves a schema by its $id: verbatim lookup first (canonical-id callers, and file entries + * already keyed by $id), then falls back through the $id → canonical-id alias index for a + * migrated blob entry. A schema's $id is not derivable from its path, so unlike {@link + * #resolve}, the alias index must be maintained explicitly (see {@code MergedConfigStore}). + */ + private static String resolveSchema(Map schemas, Map aliasesById, URI schemaId) { if (schemaId == null) { return null; } - return catalogSchemas.get(schemaId.toString()); + String id = schemaId.toString(); + String body = schemas.get(id); + if (body != null) { + return body; + } + String canonicalId = aliasesById.get(id); + return canonicalId == null ? null : schemas.get(canonicalId); + } + + /** + * Resolves a deployment-map lookup by id. Tries {@code id} verbatim first (canonical-id + * callers, and not-yet-migrated file entries keyed by short name), then falls back to the + * derived canonical id {@code typeSegment/platform/id} for a short-name lookup against a + * migrated blob entry. {@code typeSegment} is a string literal because this module has no + * dependency on storage/ResourceTypes. + */ + private static V resolve(Map entities, String typeSegment, String id) { + V direct = entities.get(id); + if (direct != null) { + return direct; + } + return entities.get(typeSegment + "/platform/" + id); } } diff --git a/config/src/test/java/com/epam/aidial/core/config/ConfigTest.java b/config/src/test/java/com/epam/aidial/core/config/ConfigTest.java index 464585405..a607d846f 100644 --- a/config/src/test/java/com/epam/aidial/core/config/ConfigTest.java +++ b/config/src/test/java/com/epam/aidial/core/config/ConfigTest.java @@ -2,10 +2,12 @@ import org.junit.jupiter.api.Test; +import java.net.URI; import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertSame; public class ConfigTest { @@ -27,4 +29,78 @@ public void testSelectDeployment() { assertEquals(interceptor, config.selectDeployment("interceptor")); assertNull(config.selectDeployment("unknown")); } + + @Test + public void testSelectDeploymentResolvesShortNameAgainstCanonicalId() { + Config config = new Config(); + Model model = new Model(); + config.setModels(Map.of("models/platform/gpt-4", model)); + + assertSame(model, config.selectDeployment("models/platform/gpt-4"), "verbatim (canonical) hit"); + assertSame(model, config.selectDeployment("gpt-4"), "derived (short-name) hit"); + assertNull(config.selectDeployment("unknown")); + } + + @Test + public void testGetModelResolvesVerbatimAndDerived() { + Config config = new Config(); + Model model = new Model(); + config.setModels(Map.of("models/platform/gpt-4", model)); + + assertSame(model, config.getModel("models/platform/gpt-4")); + assertSame(model, config.getModel("gpt-4")); + assertNull(config.getModel("unknown")); + } + + @Test + public void testGetRoleResolvesVerbatimAndDerived() { + Config config = new Config(); + Role role = new Role(); + config.setRoles(Map.of("roles/platform/admin", role)); + + assertSame(role, config.getRole("roles/platform/admin")); + assertSame(role, config.getRole("admin")); + assertNull(config.getRole("unknown")); + } + + @Test + public void testGetInterceptorResolvesVerbatimAndDerived() { + Config config = new Config(); + Interceptor interceptor = new Interceptor(); + config.setInterceptors(Map.of("interceptors/platform/my-interceptor", interceptor)); + + assertSame(interceptor, config.getInterceptor("interceptors/platform/my-interceptor")); + assertSame(interceptor, config.getInterceptor("my-interceptor")); + assertNull(config.getInterceptor("unknown")); + } + + @Test + public void testGetCustomApplicationSchemaFallsBackThroughAliasIndex() { + Config config = new Config(); + String canonicalId = "application_type_schemas/platform/my-schema"; + String schemaId = "https://mydial.epam.com/custom_application_schemas/specific_application_type"; + String body = "{\"$id\":\"" + schemaId + "\"}"; + config.setApplicationTypeSchemas(Map.of(canonicalId, body)); + config.setSchemaAliasesById(Map.of(schemaId, canonicalId)); + + assertEquals(body, config.getCustomApplicationSchema(URI.create(schemaId)), "$id lookup via alias index"); + assertEquals(body, config.getCustomApplicationSchema(URI.create(canonicalId)), "verbatim canonical-id lookup"); + assertNull(config.getCustomApplicationSchema(URI.create("https://mydial.epam.com/custom_application_schemas/unknown"))); + assertNull(config.getCustomApplicationSchema(null)); + } + + @Test + public void testGetCatalogSchemaFallsBackThroughAliasIndex() { + Config config = new Config(); + String canonicalId = "catalog_schemas/platform/my-schema"; + String schemaId = "https://dial.epam.com/catalog-schemas/model"; + String body = "{\"$id\":\"" + schemaId + "\"}"; + config.setCatalogSchemas(Map.of(canonicalId, body)); + config.setCatalogSchemaAliasesById(Map.of(schemaId, canonicalId)); + + assertEquals(body, config.getCatalogSchema(URI.create(schemaId)), "$id lookup via alias index"); + assertEquals(body, config.getCatalogSchema(URI.create(canonicalId)), "verbatim canonical-id lookup"); + assertNull(config.getCatalogSchema(URI.create("https://dial.epam.com/catalog-schemas/unknown"))); + assertNull(config.getCatalogSchema(null)); + } } diff --git a/server/src/main/java/com/epam/aidial/core/server/config/BlobEntityValidator.java b/server/src/main/java/com/epam/aidial/core/server/config/BlobEntityValidator.java index 80193d9c0..43f431ffc 100644 --- a/server/src/main/java/com/epam/aidial/core/server/config/BlobEntityValidator.java +++ b/server/src/main/java/com/epam/aidial/core/server/config/BlobEntityValidator.java @@ -37,7 +37,7 @@ private static void appendInterceptorWarnings(List refs, Config config, } for (int i = 0; i < refs.size(); i++) { String ref = refs.get(i); - if (ref == null || !config.getInterceptors().containsKey(ref)) { + if (ref == null || config.getInterceptor(ref) == null) { warnings.add(new ValidationWarning("interceptors[" + i + "]", "Interceptor '" + ref + "' not found")); } @@ -48,7 +48,7 @@ private static void appendSchemaWarning(URI schemaId, Config config, List warnings = new ArrayList<>(); validatePricing(model, warnings); if (onSkip != null) { @@ -149,7 +149,7 @@ static void validateSingleModel(Config config, String canonicalId, static void validateSingleInterceptor(Config config, String canonicalId) { Interceptor interceptor = config.getInterceptors().get(canonicalId); if (interceptor != null) { - interceptor.setName(canonicalId); + interceptor.setName(lastSegment(canonicalId)); } } @@ -160,7 +160,30 @@ static void validateSingleInterceptor(Config config, String canonicalId) { static void validateSingleRole(Config config, String canonicalId) { Role role = config.getRoles().get(canonicalId); if (role != null) { - role.setName(canonicalId); + role.setName(lastSegment(canonicalId)); + } + } + + /** + * Targeted per-type helper for {@link MergedConfigStore} partial-update path. Sets + * {@code application.name} from the map key. No cross-ref validation — applications have + * no outbound refs checked here. + */ + static void validateSingleApplication(Config config, String canonicalId) { + Application application = config.getApplications().get(canonicalId); + if (application != null) { + application.setName(lastSegment(canonicalId)); + } + } + + /** + * Targeted per-type helper for {@link MergedConfigStore} partial-update path. Sets + * {@code toolSet.name} from the map key. + */ + static void validateSingleToolSet(Config config, String canonicalId) { + ToolSet toolSet = config.getToolsets().get(canonicalId); + if (toolSet != null) { + toolSet.setName(lastSegment(canonicalId)); } } @@ -225,7 +248,7 @@ private static void processModels(Config config, Set deploymentIds, continue; } Model model = entry.getValue(); - model.setName(name); + model.setName(lastSegment(name)); log.debug("Loading {}", model); List warnings = new ArrayList<>(); validatePricing(model, warnings); @@ -248,20 +271,19 @@ private static void processModels(Config config, Set deploymentIds, /** * Validates that every interceptor reference on the supplied model resolves - * within the merged {@code config.interceptors} map. {@link MergedConfigStore} - * keys file entries by simple name and API entries by canonical ID; either - * shape is accepted via {@code containsKey}. Returns {@code true} when every - * reference resolves (no warnings appended). + * within {@code config}. Resolve-aware ({@code config.getInterceptor}) rather than a raw + * {@code containsKey}, so a short-name reference to a migrated (canonical-id-keyed, with the + * file entry shadowed) interceptor is not wrongly treated as dangling. Returns {@code true} + * when every reference resolves (no warnings appended). */ public static boolean validateCrossReferences(Model model, Config config, List warnings) { List refs = model.getInterceptors(); if (refs == null || refs.isEmpty()) { return true; } - Map interceptors = config.getInterceptors(); for (int i = 0; i < refs.size(); i++) { String ref = refs.get(i); - if (ref == null || !interceptors.containsKey(ref)) { + if (ref == null || config.getInterceptor(ref) == null) { warnings.add(new ValidationWarning("interceptors[" + i + "]", "Interceptor '" + ref + "' not found in config")); } @@ -297,7 +319,7 @@ private static void processApplications(Config config, Set deploymentIds continue; } Application application = entry.getValue(); - application.setName(name); + application.setName(lastSegment(name)); validateExternalServices(application); log.debug("Loading {}", application); } @@ -351,7 +373,7 @@ private static void processRoles(Config config) { for (Map.Entry entry : config.getRoles().entrySet()) { String name = entry.getKey(); Role role = entry.getValue(); - role.setName(name); + role.setName(lastSegment(name)); log.debug("Start loading role `{}`", role.getName()); for (Map.Entry limitEntry : role.getLimits().entrySet()) { log.debug("Loading {} for deployment `{}`", limitEntry.getValue(), limitEntry.getKey()); @@ -370,7 +392,7 @@ private static void processInterceptors(Config config, Set deploymentIds continue; } Interceptor interceptor = entry.getValue(); - interceptor.setName(name); + interceptor.setName(lastSegment(name)); log.debug("Loading {}", interceptor); } } @@ -386,7 +408,7 @@ private static void processToolSets(Config config, Set deploymentIds, } if (isValidToolSetKey(name)) { ToolSet toolSet = entry.getValue(); - toolSet.setName(name); + toolSet.setName(lastSegment(name)); log.debug("Loading {}", entry.getValue()); } else { log.warn("Invalid ToolSet name: {}", name); @@ -416,6 +438,15 @@ private static boolean skipOnDuplicate(String name, ResourceTypes type, Set ConfigPostProcessor.validateSingleRole(next, canonicalId); - case PROJECT_KEY, APP_TYPE_SCHEMA, CATALOG_SCHEMA, APPLICATION, TOOL_SET -> { /* no post-processing */ } + case APPLICATION -> ConfigPostProcessor.validateSingleApplication(next, canonicalId); + case TOOL_SET -> ConfigPostProcessor.validateSingleToolSet(next, canonicalId); + case PROJECT_KEY, APP_TYPE_SCHEMA, CATALOG_SCHEMA -> { /* no post-processing */ } case ROUTE -> ConfigPostProcessor.sortRoutesInPlace(next); default -> throw new IllegalArgumentException("Unsupported type for partial update: " + type); } @@ -882,6 +884,8 @@ private static Config shallowClone(Config base) { next.setRoutes(base.getRoutes()); next.setApplicationTypeSchemas(base.getApplicationTypeSchemas()); next.setCatalogSchemas(base.getCatalogSchemas()); + next.setSchemaAliasesById(base.getSchemaAliasesById()); + next.setCatalogSchemaAliasesById(base.getCatalogSchemaAliasesById()); next.setApplications(base.getApplications()); next.setToolsets(base.getToolsets()); next.setRetriableErrorCodes(base.getRetriableErrorCodes()); @@ -924,8 +928,14 @@ private static void cloneTypeMap(Config config, ResourceTypes type) { case ROLE -> config.setRoles(new HashMap<>(config.getRoles())); case PROJECT_KEY -> config.setKeys(new HashMap<>(config.getKeys())); case ROUTE -> config.setRoutes(new LinkedHashMap<>(config.getRoutes())); - case APP_TYPE_SCHEMA -> config.setApplicationTypeSchemas(new LinkedHashMap<>(config.getApplicationTypeSchemas())); - case CATALOG_SCHEMA -> config.setCatalogSchemas(new LinkedHashMap<>(config.getCatalogSchemas())); + case APP_TYPE_SCHEMA -> { + config.setApplicationTypeSchemas(new LinkedHashMap<>(config.getApplicationTypeSchemas())); + config.setSchemaAliasesById(new HashMap<>(config.getSchemaAliasesById())); + } + case CATALOG_SCHEMA -> { + config.setCatalogSchemas(new LinkedHashMap<>(config.getCatalogSchemas())); + config.setCatalogSchemaAliasesById(new HashMap<>(config.getCatalogSchemaAliasesById())); + } case APPLICATION -> config.setApplications(new LinkedHashMap<>(config.getApplications())); case TOOL_SET -> config.setToolsets(new LinkedHashMap<>(config.getToolsets())); default -> throw new IllegalArgumentException("Unsupported type for partial update: " + type); @@ -949,19 +959,62 @@ private static Object peekEntity(Config config, ResourceTypes type, String canon private static void putEntityInPlace(Config config, ResourceTypes type, String canonicalId, Object entity) { switch (type) { - case MODEL -> config.getModels().put(canonicalId, (Model) entity); - case INTERCEPTOR -> config.getInterceptors().put(canonicalId, (Interceptor) entity); - case ROLE -> config.getRoles().put(canonicalId, (Role) entity); + case MODEL -> putNameAddressed(config.getModels(), canonicalId, (Model) entity); + case INTERCEPTOR -> putNameAddressed(config.getInterceptors(), canonicalId, (Interceptor) entity); + case ROLE -> putNameAddressed(config.getRoles(), canonicalId, (Role) entity); case PROJECT_KEY -> config.getKeys().put(canonicalId, (Key) entity); case ROUTE -> config.getRoutes().put(canonicalId, (Route) entity); - case APP_TYPE_SCHEMA -> config.getApplicationTypeSchemas().put(canonicalId, schemaBody(entity)); - case CATALOG_SCHEMA -> config.getCatalogSchemas().put(canonicalId, schemaBody(entity)); - case APPLICATION -> config.getApplications().put(canonicalId, (Application) entity); - case TOOL_SET -> config.getToolsets().put(canonicalId, (ToolSet) entity); + case APP_TYPE_SCHEMA -> + putSchemaInPlace(config.getApplicationTypeSchemas(), config.getSchemaAliasesById(), canonicalId, entity); + case CATALOG_SCHEMA -> + putSchemaInPlace(config.getCatalogSchemas(), config.getCatalogSchemaAliasesById(), canonicalId, entity); + case APPLICATION -> putNameAddressed(config.getApplications(), canonicalId, (Application) entity); + case TOOL_SET -> putNameAddressed(config.getToolsets(), canonicalId, (ToolSet) entity); default -> throw new IllegalArgumentException("Unsupported type for partial update: " + type); } } + /** + * Puts a name-addressed entity under its canonical id, then removes the file-defined entry + * sharing its short name (blob shadows file), so a subsequent short-name {@code resolve} + * hits the freshly-written blob entity rather than a stale file entry. + */ + private static void putNameAddressed(Map map, String canonicalId, V entity) { + map.put(canonicalId, entity); + map.remove(lastSegment(canonicalId)); + } + + private static void putSchemaInPlace(Map schemas, Map aliasesById, + String canonicalId, Object entity) { + String body = schemaBody(entity); + schemas.put(canonicalId, body); + try { + recordSchemaAlias(schemas, aliasesById, canonicalId, ProxyUtil.BLOB_MAPPER.readTree(body)); + } catch (JsonProcessingException e) { + log.warn("Failed to parse schema body for $id alias index: {} ({})", canonicalId, e.getMessage()); + } + } + + /** + * Records the {@code $id → canonicalId} alias read from a schema's body, and removes the + * file-defined entry keyed by that same $id — the schema-specific counterpart of the + * blob-shadows-file behavior used for name-addressed types: file entries are keyed by $id, + * blob entries by canonical id, so a migrated schema would otherwise appear twice in + * $id-keyed listings ({@code ApplicationTypeSchemaController}/{@code CatalogSchemaController} + * both iterate the whole map and read each entry's own body $id, not the map key). App-type + * and catalog schemas are referenced by the arbitrary $id embedded in the body, not by a + * derivable last-path-segment, so this index bridges $id-keyed file entries and + * canonical-id-keyed blob entries. + */ + private static void recordSchemaAlias(Map schemas, Map aliasesById, + String canonicalId, JsonNode node) { + JsonNode idNode = node.get("$id"); + if (idNode != null && idNode.isTextual()) { + schemas.remove(idNode.asText()); + aliasesById.put(idNode.asText(), canonicalId); + } + } + private static void removeEntityInPlace(Config config, ResourceTypes type, String canonicalId) { switch (type) { case MODEL -> config.getModels().remove(canonicalId); @@ -969,8 +1022,14 @@ private static void removeEntityInPlace(Config config, ResourceTypes type, Strin case ROLE -> config.getRoles().remove(canonicalId); case PROJECT_KEY -> config.getKeys().remove(canonicalId); case ROUTE -> config.getRoutes().remove(canonicalId); - case APP_TYPE_SCHEMA -> config.getApplicationTypeSchemas().remove(canonicalId); - case CATALOG_SCHEMA -> config.getCatalogSchemas().remove(canonicalId); + case APP_TYPE_SCHEMA -> { + config.getApplicationTypeSchemas().remove(canonicalId); + config.getSchemaAliasesById().values().removeIf(canonicalId::equals); + } + case CATALOG_SCHEMA -> { + config.getCatalogSchemas().remove(canonicalId); + config.getCatalogSchemaAliasesById().values().removeIf(canonicalId::equals); + } case APPLICATION -> config.getApplications().remove(canonicalId); case TOOL_SET -> config.getToolsets().remove(canonicalId); default -> throw new IllegalArgumentException("Unsupported type for partial update: " + type); @@ -1016,6 +1075,10 @@ private Config rebuild() { Map catalogSchemas = new LinkedHashMap<>(base.getCatalogSchemas()); Map applications = new LinkedHashMap<>(base.getApplications()); Map toolsets = new LinkedHashMap<>(base.getToolsets()); + // $id -> canonicalId index, built fresh each rebuild from the blob scan below; file + // entries need no alias since they're already keyed by $id. + Map schemaAliasesById = new HashMap<>(); + Map catalogSchemaAliasesById = new HashMap<>(); merged.setRetriableErrorCodes(base.getRetriableErrorCodes()); merged.setGlobalInterceptors(base.getGlobalInterceptors()); @@ -1060,11 +1123,11 @@ private Config rebuild() { ResourceDescriptor descriptor = ResourceDescriptorFactory.fromDecoded( type, bucket, bucketLocation, name); - AddedEntity added; + Object added; try { added = addBlobEntity(type, canonicalId, node, models, interceptors, roles, keys, routes, schemas, catalogSchemas, - applications, toolsets); + applications, toolsets, schemaAliasesById, catalogSchemaAliasesById); } catch (Exception parseError) { recordInvalid(pendingInvalid, type, canonicalId, name, "JSON parse failure: " + parseError.getMessage(), @@ -1073,9 +1136,9 @@ private Config rebuild() { continue; } - if (added != null && added.entity() != null) { + if (added != null) { try { - decryptManagedEntity(type, added.entity(), descriptor); + decryptManagedEntity(type, added, descriptor); } catch (Exception decryptError) { // Roll back the partial insertion so decryption-failure entities never // reach addProjectKeys (locked 2S.9 invariant). @@ -1088,8 +1151,12 @@ private Config rebuild() { continue; } if (type == ResourceTypes.PROJECT_KEY) { - apiKeysByCanonicalId.put(canonicalId, (Key) added.entity()); + apiKeysByCanonicalId.put(canonicalId, (Key) added); } + // Shadow the file-defined entry sharing this canonical id's short name, + // gated on successful decryption above — if decryption failed, removeAddedEntity + // rolled the blob entity back, so the file entry must stay in this rebuild's map. + shadowFileEntry(type, canonicalId, models, interceptors, roles, applications, toolsets); } blobBodies.put(canonicalId, node); } @@ -1105,6 +1172,8 @@ private Config rebuild() { merged.setCatalogSchemas(catalogSchemas); merged.setApplications(applications); merged.setToolsets(toolsets); + merged.setSchemaAliasesById(schemaAliasesById); + merged.setCatalogSchemaAliasesById(catalogSchemaAliasesById); // Semantic pass — under MODE_SKIP, route per-entity violations to invalidEntities and // continue; under MODE_ABORT, the post-processor throws and the rebuild aborts (this.config @@ -1237,56 +1306,60 @@ public static String canonicalId(ResourceDescriptor descriptor) { descriptor.getBucketName(), descriptor.getName()); } - private static AddedEntity addBlobEntity(ResourceTypes type, String canonicalId, JsonNode node, + private static Object addBlobEntity(ResourceTypes type, String canonicalId, JsonNode node, Map models, Map interceptors, Map roles, Map keys, LinkedHashMap routes, Map schemas, Map catalogSchemas, - Map applications, Map toolsets) + Map applications, Map toolsets, + Map schemaAliasesById, + Map catalogSchemaAliasesById) throws JsonProcessingException { switch (type) { case MODEL -> { Model entity = ProxyUtil.BLOB_MAPPER.treeToValue(node, Model.class); warnIfReplaced(type, canonicalId, models.put(canonicalId, entity)); - return new AddedEntity(entity); + return entity; } case INTERCEPTOR -> { Interceptor entity = ProxyUtil.BLOB_MAPPER.treeToValue(node, Interceptor.class); warnIfReplaced(type, canonicalId, interceptors.put(canonicalId, entity)); - return new AddedEntity(entity); + return entity; } case ROLE -> { Role entity = ProxyUtil.BLOB_MAPPER.treeToValue(node, Role.class); warnIfReplaced(type, canonicalId, roles.put(canonicalId, entity)); - return new AddedEntity(entity); + return entity; } case PROJECT_KEY -> { Key entity = ProxyUtil.BLOB_MAPPER.treeToValue(node, Key.class); warnIfReplaced(type, canonicalId, keys.put(canonicalId, entity)); - return new AddedEntity(entity); + return entity; } case ROUTE -> { Route entity = ProxyUtil.BLOB_MAPPER.treeToValue(node, Route.class); warnIfReplaced(type, canonicalId, routes.put(canonicalId, entity)); - return new AddedEntity(entity); + return entity; } case APP_TYPE_SCHEMA -> { warnIfReplaced(type, canonicalId, schemas.put(canonicalId, node.toString())); + recordSchemaAlias(schemas, schemaAliasesById, canonicalId, node); return null; } case CATALOG_SCHEMA -> { warnIfReplaced(type, canonicalId, catalogSchemas.put(canonicalId, node.toString())); + recordSchemaAlias(catalogSchemas, catalogSchemaAliasesById, canonicalId, node); return null; } case APPLICATION -> { Application entity = ProxyUtil.BLOB_MAPPER.treeToValue(node, Application.class); warnIfReplaced(type, canonicalId, applications.put(canonicalId, entity)); - return new AddedEntity(entity); + return entity; } case TOOL_SET -> { ToolSet entity = ProxyUtil.BLOB_MAPPER.treeToValue(node, ToolSet.class); warnIfReplaced(type, canonicalId, toolsets.put(canonicalId, entity)); - return new AddedEntity(entity); + return entity; } default -> { /* GLOBAL_SETTINGS is a singleton — design 02 §4 leaves union-by-key out of scope. */ @@ -1295,6 +1368,26 @@ private static AddedEntity addBlobEntity(ResourceTypes type, String canonicalId, } } + /** + * Removes the file-defined entry sharing {@code canonicalId}'s short name from the + * name-addressed type maps (models/interceptors/roles/applications/toolsets), so blob wins + * over the file entry it just shadowed. No-op for types that aren't name-addressed (keys, + * routes, schemas — schemas use the $id index instead, see {@link #recordSchemaAlias}). + */ + private static void shadowFileEntry(ResourceTypes type, String canonicalId, + Map models, Map interceptors, + Map roles, Map applications, + Map toolsets) { + switch (type) { + case MODEL -> models.remove(lastSegment(canonicalId)); + case INTERCEPTOR -> interceptors.remove(lastSegment(canonicalId)); + case ROLE -> roles.remove(lastSegment(canonicalId)); + case APPLICATION -> applications.remove(lastSegment(canonicalId)); + case TOOL_SET -> toolsets.remove(lastSegment(canonicalId)); + default -> { /* not name-addressed */ } + } + } + private static void warnIfReplaced(ResourceTypes type, String canonicalId, Object previous) { if (previous != null) { log.warn("Duplicate canonical ID during merged Config rebuild: {} '{}' overwrote a prior entry", @@ -1321,6 +1414,4 @@ private static void removeAddedEntity(ResourceTypes type, String canonicalId, default -> { /* no-op */ } } } - - private record AddedEntity(Object entity) { } } diff --git a/server/src/main/java/com/epam/aidial/core/server/controller/BaseInterceptorController.java b/server/src/main/java/com/epam/aidial/core/server/controller/BaseInterceptorController.java index ac149d74a..26880635a 100644 --- a/server/src/main/java/com/epam/aidial/core/server/controller/BaseInterceptorController.java +++ b/server/src/main/java/com/epam/aidial/core/server/controller/BaseInterceptorController.java @@ -48,7 +48,7 @@ protected BaseInterceptorController(Proxy proxy, ProxyContext context, int inter public Future handle() { List interceptors = context.getInterceptors(); String interceptorName = interceptors.get(interceptorIndex); - Interceptor interceptor = context.getConfig().getInterceptors().get(interceptorName); + Interceptor interceptor = context.getConfig().getInterceptor(interceptorName); if (interceptor == null) { log.warn("Interceptor is not found: {}", interceptorName); return respond(HttpStatus.NOT_FOUND, "Interceptor is not found"); diff --git a/server/src/main/java/com/epam/aidial/core/server/controller/DeploymentController.java b/server/src/main/java/com/epam/aidial/core/server/controller/DeploymentController.java index 5836c031f..761ab9e50 100644 --- a/server/src/main/java/com/epam/aidial/core/server/controller/DeploymentController.java +++ b/server/src/main/java/com/epam/aidial/core/server/controller/DeploymentController.java @@ -84,7 +84,7 @@ public DeploymentController(Proxy proxy, ProxyContext context) { ) public Future getDeployment(String deploymentId) { Config config = context.getConfig(); - Model model = config.getModels().get(deploymentId); + Model model = config.getModel(deploymentId); if (model == null) { return context.respond(HttpStatus.NOT_FOUND); diff --git a/server/src/main/java/com/epam/aidial/core/server/controller/ModelController.java b/server/src/main/java/com/epam/aidial/core/server/controller/ModelController.java index 8226fe34f..4317558d6 100644 --- a/server/src/main/java/com/epam/aidial/core/server/controller/ModelController.java +++ b/server/src/main/java/com/epam/aidial/core/server/controller/ModelController.java @@ -47,7 +47,7 @@ public class ModelController { ) public Future getModel(String modelId) { Config config = context.getConfig(); - Model model = config.getModels().get(modelId); + Model model = config.getModel(modelId); if (model == null) { return context.respond(HttpStatus.NOT_FOUND); diff --git a/server/src/main/java/com/epam/aidial/core/server/controller/ResourceController.java b/server/src/main/java/com/epam/aidial/core/server/controller/ResourceController.java index ebf423017..52fdde33e 100644 --- a/server/src/main/java/com/epam/aidial/core/server/controller/ResourceController.java +++ b/server/src/main/java/com/epam/aidial/core/server/controller/ResourceController.java @@ -678,7 +678,7 @@ private void validateCustomApplication(Application application) { } Config config = context.getConfig(); for (String interceptor : application.getInterceptors()) { - if (!config.getInterceptors().containsKey(interceptor)) { + if (config.getInterceptor(interceptor) == null) { throw new HttpException(BAD_REQUEST, "Unknown interceptor: " + interceptor); } } diff --git a/server/src/main/java/com/epam/aidial/core/server/function/CollectResponseAttachmentsFn.java b/server/src/main/java/com/epam/aidial/core/server/function/CollectResponseAttachmentsFn.java index 2ec5afc39..44f9e5378 100644 --- a/server/src/main/java/com/epam/aidial/core/server/function/CollectResponseAttachmentsFn.java +++ b/server/src/main/java/com/epam/aidial/core/server/function/CollectResponseAttachmentsFn.java @@ -77,7 +77,7 @@ private void processAttachedFile(String url, Map return; } String sourceDeployment = context.getApiKeyData().getSourceDeployment(); - if (context.getConfig().getInterceptors().containsKey(sourceDeployment)) { + if (context.getConfig().getInterceptor(sourceDeployment) != null) { // Note. permission check: make sure that the target deployment has access to the resource only // we don't check other permissions like admin, share or publishing access since we give full permissions to the source deployment Map> result = AccessService.getAppResourceAccess(Set.of(resource), diff --git a/server/src/main/java/com/epam/aidial/core/server/limiter/RateLimiter.java b/server/src/main/java/com/epam/aidial/core/server/limiter/RateLimiter.java index 011126d4f..d8d9cb4c2 100644 --- a/server/src/main/java/com/epam/aidial/core/server/limiter/RateLimiter.java +++ b/server/src/main/java/com/epam/aidial/core/server/limiter/RateLimiter.java @@ -1,7 +1,7 @@ package com.epam.aidial.core.server.limiter; +import com.epam.aidial.core.config.Config; import com.epam.aidial.core.config.CostLimit; -import com.epam.aidial.core.config.Deployment; import com.epam.aidial.core.config.Limit; import com.epam.aidial.core.config.Role; import com.epam.aidial.core.config.RoleBasedEntity; @@ -26,7 +26,6 @@ import java.math.BigDecimal; import java.util.List; -import java.util.Map; import java.util.Optional; @Slf4j @@ -325,14 +324,14 @@ private Limit getLimitByUser(ProxyContext context, RoleBasedEntity roleBasedEnti // find limits for user roles which match to required roles userRoles = context.getUserRoles().stream().filter(role -> roleBasedEntity.getUserRoles().contains(role)).toList(); } - Map roles = context.getConfig().getRoles(); - Limit defaultUserLimit = getLimit(roles, DEFAULT_USER_ROLE, name, DEFAULT_LIMIT); + Config config = context.getConfig(); + Limit defaultUserLimit = getLimit(config, DEFAULT_USER_ROLE, name, DEFAULT_LIMIT); if (userRoles.isEmpty()) { return defaultUserLimit; } Limit limit = null; for (String userRole : userRoles) { - Limit candidate = getLimit(roles, userRole, name, null); + Limit candidate = getLimit(config, userRole, name, null); if (candidate != null) { if (limit == null) { limit = new Limit(); @@ -357,14 +356,14 @@ private Limit getLimitByUser(ProxyContext context, RoleBasedEntity roleBasedEnti private CostLimit getCostLimitByUser(ProxyContext context) { List userRoles = context.getUserRoles(); - Map roles = context.getConfig().getRoles(); - CostLimit defaultUserCostLimit = getCostLimit(roles, DEFAULT_USER_ROLE, DEFAULT_COST_LIMIT); + Config config = context.getConfig(); + CostLimit defaultUserCostLimit = getCostLimit(config, DEFAULT_USER_ROLE, DEFAULT_COST_LIMIT); if (userRoles.isEmpty()) { return defaultUserCostLimit; } CostLimit costLimit = null; for (String userRole : userRoles) { - CostLimit candidate = getCostLimit(roles, userRole, null); + CostLimit candidate = getCostLimit(config, userRole, null); if (candidate != null) { if (costLimit == null) { costLimit = new CostLimit(); @@ -396,14 +395,14 @@ private static String getPathToCosts() { return "costs"; } - private static Limit getLimit(Map roles, String userRole, String name, Limit defaultLimit) { - return Optional.ofNullable(roles.get(userRole)) + private static Limit getLimit(Config config, String userRole, String name, Limit defaultLimit) { + return Optional.ofNullable(config.getRole(userRole)) .map(role -> role.getLimits().get(name)) .orElse(defaultLimit); } - private static CostLimit getCostLimit(Map roles, String userRole, CostLimit defaultCostLimit) { - return Optional.ofNullable(roles.get(userRole)) + private static CostLimit getCostLimit(Config config, String userRole, CostLimit defaultCostLimit) { + return Optional.ofNullable(config.getRole(userRole)) .map(Role::getCostLimit) .orElse(defaultCostLimit); } diff --git a/server/src/main/java/com/epam/aidial/core/server/service/ShareService.java b/server/src/main/java/com/epam/aidial/core/server/service/ShareService.java index 599889af0..e31ae91f7 100644 --- a/server/src/main/java/com/epam/aidial/core/server/service/ShareService.java +++ b/server/src/main/java/com/epam/aidial/core/server/service/ShareService.java @@ -1,6 +1,7 @@ package com.epam.aidial.core.server.service; import com.epam.aidial.core.config.Application; +import com.epam.aidial.core.config.Config; import com.epam.aidial.core.config.CredentialsLevel; import com.epam.aidial.core.config.ResourceAccessType; import com.epam.aidial.core.config.Role; @@ -292,11 +293,11 @@ private void updateLimits(ProxyContext context, ResourceType resourceType, Share private ShareResourceLimit getLimit(ProxyContext context, ResourceType resourceType) { List userRoles = context.getUserRoles(); - Map roles = context.getConfig().getRoles(); + Config config = context.getConfig(); ShareResourceLimit defaultLimit = DEFAULT_LIMITS.get(resourceType); ShareResourceLimit limit = null; for (String userRole : userRoles) { - ShareResourceLimit candidate = Optional.ofNullable(roles.get(userRole)).map(Role::getShare).map(limits -> limits.get(resourceType.name())).orElse(null); + ShareResourceLimit candidate = Optional.ofNullable(config.getRole(userRole)).map(Role::getShare).map(limits -> limits.get(resourceType.name())).orElse(null); if (candidate != null) { if (limit == null) { limit = new ShareResourceLimit(candidate.getMaxAcceptedUsers(), candidate.getInvitationTtl()); diff --git a/server/src/test/java/com/epam/aidial/core/server/CanonicalIdListingTest.java b/server/src/test/java/com/epam/aidial/core/server/CanonicalIdListingTest.java index 4a13e3c61..51219dcaa 100644 --- a/server/src/test/java/com/epam/aidial/core/server/CanonicalIdListingTest.java +++ b/server/src/test/java/com/epam/aidial/core/server/CanonicalIdListingTest.java @@ -3,15 +3,17 @@ import io.vertx.core.http.HttpMethod; import org.junit.jupiter.api.Test; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; /** - * HTTP integration tests for slice 2S.15 + Polish.1 (2026-05-08): canonical IDs surface as the - * {@code id}/{@code model} fields on the legacy {@code /openai/models} and {@code /openai/deployments} - * listings for API-managed entries, and on the admin Configuration API - * ({@code /v1/{type}/{bucket}/...}) GET + listing projection. File-sourced entries continue to - * surface their simple names. Locks the OQ-23 + Polish.1 contract that clients can copy a - * listing's identifier verbatim into per-entity URLs. + * HTTP integration tests locking the short-name-addressing contract: the {@code id}/{@code model} + * fields on the legacy {@code /openai/models} and {@code /openai/deployments} listings surface + * the short name (last path segment) for API-managed entries, matching file-sourced entries — + * never the canonical id. The admin Configuration API ({@code /v1/{type}/{bucket}/...}) GET + + * listing projection is unaffected: it independently projects the canonical ID (map key) + * regardless of entity name, so operators can still copy-paste the identifier verbatim into + * per-entity URLs. */ public class CanonicalIdListingTest extends ResourceBaseTest { @@ -23,27 +25,31 @@ public class CanonicalIdListingTest extends ResourceBaseTest { """; @Test - void testApiManagedModelSurfacedAsCanonicalIdInOpenAiModels() { + void testApiManagedModelSurfacedAsShortNameInOpenAiModels() { verify(send(HttpMethod.PUT, "/v1/models/platform/canonical-test", null, API_MODEL_BODY, "authorization", "admin", "If-None-Match", "*"), 200); Response list = send(HttpMethod.GET, "/openai/models", null, ""); verify(list, 200); - assertTrue(list.body().contains("\"id\":\"models/platform/canonical-test\""), - () -> "Expected canonical id for API-managed model: " + list.body()); - assertTrue(list.body().contains("\"model\":\"models/platform/canonical-test\""), - () -> "Expected canonical model field for API-managed model: " + list.body()); + assertTrue(list.body().contains("\"id\":\"canonical-test\""), + () -> "Expected short name for API-managed model: " + list.body()); + assertTrue(list.body().contains("\"model\":\"canonical-test\""), + () -> "Expected short name model field for API-managed model: " + list.body()); + assertFalse(list.body().contains("models/platform/canonical-test"), + () -> "Canonical id must not leak into the outbound listing: " + list.body()); } @Test - void testApiManagedModelSurfacedAsCanonicalIdInOpenAiDeployments() { + void testApiManagedModelSurfacedAsShortNameInOpenAiDeployments() { verify(send(HttpMethod.PUT, "/v1/models/platform/canonical-deployments", null, API_MODEL_BODY, "authorization", "admin", "If-None-Match", "*"), 200); Response list = send(HttpMethod.GET, "/openai/deployments", null, ""); verify(list, 200); - assertTrue(list.body().contains("\"id\":\"models/platform/canonical-deployments\""), - () -> "Expected canonical id for API-managed deployment: " + list.body()); + assertTrue(list.body().contains("\"id\":\"canonical-deployments\""), + () -> "Expected short name for API-managed deployment: " + list.body()); + assertFalse(list.body().contains("models/platform/canonical-deployments"), + () -> "Canonical id must not leak into the outbound listing: " + list.body()); } @Test diff --git a/server/src/test/java/com/epam/aidial/core/server/MergedConfigStoreApiTest.java b/server/src/test/java/com/epam/aidial/core/server/MergedConfigStoreApiTest.java index b088e1891..316bc08eb 100644 --- a/server/src/test/java/com/epam/aidial/core/server/MergedConfigStoreApiTest.java +++ b/server/src/test/java/com/epam/aidial/core/server/MergedConfigStoreApiTest.java @@ -11,6 +11,8 @@ import io.vertx.core.http.HttpMethod; import org.junit.jupiter.api.Test; +import java.net.URI; + import static com.epam.aidial.core.server.util.ResourceDescriptorFactory.fromDecoded; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -72,12 +74,12 @@ void testBlobModelSurfacesAfterReload() { Config merged = dial.getProxy().getConfigStore().get(); Model blobModel = merged.getModels().get("models/platform/" + blobName); assertNotNull(blobModel, () -> "Expected canonical-ID key in merged Config: " + merged.getModels().keySet()); - // Slice 2S.15 / OQ-23: Model.name carries the canonical ID for API-managed entries so - // legacy /openai/models, /openai/deployments, and rate-limit role-limit lookups see the - // canonical form. Polish.1 (2026-05-08) extends this to the admin Configuration API GET - // / listing projection — canonical ID for API entries, simple name for file entries. - assertEquals("models/platform/" + blobName, blobModel.getName(), - "Entity.name carries the canonical ID for API-managed entries"); + // Model.name carries the short name for API-managed entries too, so legacy /openai/models, + // /openai/deployments, and rate-limit role-limit lookups see the same short form file + // entries already use. The admin Configuration API GET / listing projection is unaffected + // — it independently projects the canonical ID (map key). + assertEquals(blobName, blobModel.getName(), + "Entity.name carries the short name for API-managed entries"); assertNotNull(merged.getModels().get("test-model-v1"), "File model must still coexist by simple name"); Response get = send(HttpMethod.GET, "/v1/models/platform/" + blobName, null, "", @@ -106,11 +108,173 @@ void testBlobInterceptorSurfacesAfterReload() { Config merged = dial.getProxy().getConfigStore().get(); Interceptor blob = merged.getInterceptors().get("interceptors/platform/" + blobName); assertNotNull(blob, () -> "Expected canonical-ID key in merged Config: " + merged.getInterceptors().keySet()); - // Slice 2S.15: API-managed entries carry the canonical ID as their name (per OQ-23). - assertEquals("interceptors/platform/" + blobName, blob.getName()); + // API-managed entries carry the short name (last path segment), not the canonical ID. + assertEquals(blobName, blob.getName()); assertNotNull(merged.getInterceptors().get("interceptor1"), "File interceptor must still coexist"); } + @Test + void testBlobModelShadowsFileEntryByShortNameAfterReload() { + // A blob model written under the SAME short name as an existing file-sourced model must + // replace it in the merged Config, not coexist alongside it: the file entry keyed by the + // bare short name is removed, and resolving that short name (verbatim or via getModel) + // hits the blob entity, which is authoritative once migrated. + String shortName = "test-model-v1"; + String canonicalId = "models/platform/" + shortName; + String body = """ + { + "type": "chat", + "displayName": "Migrated Test Model", + "endpoint": "http://localhost:7001/openai/deployments/migrated/chat/completions" + } + """; + putBlob(ResourceTypes.MODEL, ResourceDescriptor.PLATFORM_BUCKET, ResourceDescriptor.PLATFORM_LOCATION, + shortName, body); + + Response reload = operationRequest("/v1/ops/config/reload", null, "Authorization", "admin"); + assertEquals(200, reload.status()); + + Config merged = dial.getProxy().getConfigStore().get(); + Model blobModel = merged.getModels().get(canonicalId); + assertNotNull(blobModel, () -> "Expected canonical-ID key in merged Config: " + merged.getModels().keySet()); + assertNull(merged.getModels().get(shortName), + () -> "File entry must be shadowed by the migrated blob entity: " + merged.getModels().keySet()); + assertEquals(blobModel, merged.getModel(shortName), "getModel must resolve the short name to the blob entity"); + } + + @Test + void testBlobInterceptorShadowsFileEntryByShortNameAfterReload() { + String shortName = "interceptor1"; + String canonicalId = "interceptors/platform/" + shortName; + String body = """ + { + "endpoint": "http://localhost:9000/migrated-intercept" + } + """; + putBlob(ResourceTypes.INTERCEPTOR, ResourceDescriptor.PLATFORM_BUCKET, ResourceDescriptor.PLATFORM_LOCATION, + shortName, body); + + Response reload = operationRequest("/v1/ops/config/reload", null, "Authorization", "admin"); + assertEquals(200, reload.status()); + + Config merged = dial.getProxy().getConfigStore().get(); + Interceptor blob = merged.getInterceptors().get(canonicalId); + assertNotNull(blob, () -> "Expected canonical-ID key in merged Config: " + merged.getInterceptors().keySet()); + assertNull(merged.getInterceptors().get(shortName), + () -> "File entry must be shadowed by the migrated blob entity: " + merged.getInterceptors().keySet()); + assertEquals(blob, merged.getInterceptor(shortName), + "getInterceptor must resolve the short name to the blob entity"); + } + + @Test + void testBlobRoleShadowsFileEntryByShortNameAfterReload() { + String shortName = "default"; + String canonicalId = "roles/platform/" + shortName; + String body = """ + { + "limits": {} + } + """; + putBlob(ResourceTypes.ROLE, ResourceDescriptor.PLATFORM_BUCKET, ResourceDescriptor.PLATFORM_LOCATION, + shortName, body); + + Response reload = operationRequest("/v1/ops/config/reload", null, "Authorization", "admin"); + assertEquals(200, reload.status()); + + Config merged = dial.getProxy().getConfigStore().get(); + assertNotNull(merged.getRoles().get(canonicalId), + () -> "Expected canonical-ID key in merged Config: " + merged.getRoles().keySet()); + assertNull(merged.getRoles().get(shortName), + () -> "File entry must be shadowed by the migrated blob entity: " + merged.getRoles().keySet()); + assertEquals(merged.getRoles().get(canonicalId), merged.getRole(shortName), + "getRole must resolve the short name to the blob entity"); + } + + @Test + void testBlobApplicationShadowsFileEntryByShortNameAfterReload() { + String shortName = "app"; + String canonicalId = "applications/platform/" + shortName; + String body = """ + { + "endpoint": "http://application1/v1/completions", + "display_name": "Migrated Platform App" + } + """; + putBlob(ResourceTypes.APPLICATION, ResourceDescriptor.PLATFORM_BUCKET, ResourceDescriptor.PLATFORM_LOCATION, + shortName, body); + + Response reload = operationRequest("/v1/ops/config/reload", null, "Authorization", "admin"); + assertEquals(200, reload.status()); + + Config merged = dial.getProxy().getConfigStore().get(); + assertNotNull(merged.getApplications().get(canonicalId), + () -> "Expected canonical-ID key in merged Config: " + merged.getApplications().keySet()); + assertNull(merged.getApplications().get(shortName), + () -> "File entry must be shadowed by the migrated blob entity: " + merged.getApplications().keySet()); + assertEquals(merged.getApplications().get(canonicalId), merged.selectDeployment(shortName), + "selectDeployment must resolve the short name to the blob entity"); + } + + @Test + void testBlobToolSetShadowsFileEntryByShortNameAfterReload() { + String shortName = "git"; + String canonicalId = "toolsets/platform/" + shortName; + String body = """ + { + "endpoint": "http://localhost:9876", + "transport": "HTTP", + "display_name": "Migrated Git Toolset" + } + """; + putBlob(ResourceTypes.TOOL_SET, ResourceDescriptor.PLATFORM_BUCKET, ResourceDescriptor.PLATFORM_LOCATION, + shortName, body); + + Response reload = operationRequest("/v1/ops/config/reload", null, "Authorization", "admin"); + assertEquals(200, reload.status()); + + Config merged = dial.getProxy().getConfigStore().get(); + assertNotNull(merged.getToolsets().get(canonicalId), + () -> "Expected canonical-ID key in merged Config: " + merged.getToolsets().keySet()); + assertNull(merged.getToolsets().get(shortName), + () -> "File entry must be shadowed by the migrated blob entity: " + merged.getToolsets().keySet()); + assertEquals(merged.getToolsets().get(canonicalId), merged.selectDeployment(shortName), + "selectDeployment must resolve the short name to the blob entity"); + } + + @Test + void testBlobAppTypeSchemaShadowsFileEntryByIdAfterReload() { + // App-type/catalog schemas are keyed by $id (file) vs. canonical id (blob), so the + // blob-shadows-file removal used for name-addressed types (by short name) doesn't apply + // directly — instead, the migrated blob entity's own $id is used to remove the file entry + // sharing it, so $id-keyed listings (ApplicationTypeSchemaController) don't surface the + // schema twice. + String fileSchemaId = "https://mydial.somewhere.com/custom_application_schemas/specific_application_type"; + String blobName = "blob-schema-1"; + String body = """ + { + "$schema": "https://dial.epam.com/application_type_schemas/schema#", + "$id": "%s", + "display_name": "Blob-migrated schema" + } + """.formatted(fileSchemaId); + putBlob(ResourceTypes.APP_TYPE_SCHEMA, ResourceDescriptor.PLATFORM_BUCKET, ResourceDescriptor.PLATFORM_LOCATION, + blobName, body); + + Response reload = operationRequest("/v1/ops/config/reload", null, "Authorization", "admin"); + assertEquals(200, reload.status()); + + Config merged = dial.getProxy().getConfigStore().get(); + String canonicalId = "schemas/platform/" + blobName; + assertTrue(merged.getApplicationTypeSchemas().containsKey(canonicalId), + () -> "Expected canonical-ID key in merged Config: " + merged.getApplicationTypeSchemas().keySet()); + assertFalse(merged.getApplicationTypeSchemas().containsKey(fileSchemaId), + () -> "File entry keyed by $id must be shadowed by the migrated blob entity: " + + merged.getApplicationTypeSchemas().keySet()); + assertEquals(merged.getApplicationTypeSchemas().get(canonicalId), + merged.getCustomApplicationSchema(URI.create(fileSchemaId)), + "getCustomApplicationSchema must still resolve the $id via the alias index"); + } + @Test void testReloadConfigSucceedsUnderMergedStore() { Response resp = operationRequest("/v1/ops/config/reload", null, "Authorization", "admin"); diff --git a/server/src/test/java/com/epam/aidial/core/server/config/ConfigPostProcessorTest.java b/server/src/test/java/com/epam/aidial/core/server/config/ConfigPostProcessorTest.java index 255b42403..9d0bc127d 100644 --- a/server/src/test/java/com/epam/aidial/core/server/config/ConfigPostProcessorTest.java +++ b/server/src/test/java/com/epam/aidial/core/server/config/ConfigPostProcessorTest.java @@ -104,7 +104,8 @@ void testSemanticKeepsCanonicalIdKeyedToolSet() { ConfigPostProcessor.processSemantic(config, null, Map.of(), Map.of(), null); assertTrue(config.getToolsets().containsKey("toolsets/platform/my-toolset")); - assertEquals("toolsets/platform/my-toolset", config.getToolsets().get("toolsets/platform/my-toolset").getName()); + // Name is the short name (last path segment), not the canonical id. + assertEquals("my-toolset", config.getToolsets().get("toolsets/platform/my-toolset").getName()); } @Test From 6b97d1e2426020b1bf6404681c98c02d1125b3ee Mon Sep 17 00:00:00 2001 From: Kiryl_Kurnosenka Date: Mon, 10 Aug 2026 20:10:51 +0300 Subject: [PATCH 02/15] fix: use correct app-type-schema canonical id in ConfigTest #1783 APP_TYPE_SCHEMA's urlSegment is "schemas", not "application_type_schemas" (its group and urlSegment differ, unlike CATALOG_SCHEMA), so the test's "canonical id" example didn't match what MergedConfigStore actually generates. --- .../src/test/java/com/epam/aidial/core/config/ConfigTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/src/test/java/com/epam/aidial/core/config/ConfigTest.java b/config/src/test/java/com/epam/aidial/core/config/ConfigTest.java index a607d846f..a1fc3d2cf 100644 --- a/config/src/test/java/com/epam/aidial/core/config/ConfigTest.java +++ b/config/src/test/java/com/epam/aidial/core/config/ConfigTest.java @@ -77,7 +77,7 @@ public void testGetInterceptorResolvesVerbatimAndDerived() { @Test public void testGetCustomApplicationSchemaFallsBackThroughAliasIndex() { Config config = new Config(); - String canonicalId = "application_type_schemas/platform/my-schema"; + String canonicalId = "schemas/platform/my-schema"; String schemaId = "https://mydial.epam.com/custom_application_schemas/specific_application_type"; String body = "{\"$id\":\"" + schemaId + "\"}"; config.setApplicationTypeSchemas(Map.of(canonicalId, body)); From b8f84a7c00acf8ec3526928ea0dea516af285c1f Mon Sep 17 00:00:00 2001 From: Kiryl_Kurnosenka Date: Tue, 11 Aug 2026 19:46:26 +0300 Subject: [PATCH 03/15] fix: close deployment-id uniqueness and short-name compat gaps #1783 Code-review follow-up on the short-name resolution feature: - Deployment-id uniqueness now dedupes on the derived short name (not the raw map key), and is enforced on the partial-update and /v1/admin/apply paths too, not just full rebuilds. - RateLimiter/ConsentService fall back to a canonical-id match so pre-existing role limits and consent records keep resolving now that deployment names revert to short form. - AnalyticsLogContext normalizes canonical-id interceptor refs before comparing against the (always short-name) execution path. - Added a bucket-scope guard to MergedConfigStore's rebuild shadow step, mirroring the existing replica-event guard. - Deduplicated the several lastSegment implementations into PlatformCanonicalIdUtil. Co-Authored-By: Claude Sonnet 5 --- .../server/config/ConfigPostProcessor.java | 39 ++++- .../core/server/config/MergedConfigStore.java | 136 +++++++++--------- .../controller/AdminApplyController.java | 107 +++++++++++--- .../controller/ConfigResourceController.java | 26 ++++ .../core/server/limiter/RateLimiter.java | 22 ++- .../core/server/log/AnalyticsLogContext.java | 8 +- .../core/server/service/ConsentService.java | 27 +++- .../server/util/PlatformCanonicalIdUtil.java | 21 +++ 8 files changed, 291 insertions(+), 95 deletions(-) create mode 100644 server/src/main/java/com/epam/aidial/core/server/util/PlatformCanonicalIdUtil.java diff --git a/server/src/main/java/com/epam/aidial/core/server/config/ConfigPostProcessor.java b/server/src/main/java/com/epam/aidial/core/server/config/ConfigPostProcessor.java index 4e7e6e54d..6d07c9c8b 100644 --- a/server/src/main/java/com/epam/aidial/core/server/config/ConfigPostProcessor.java +++ b/server/src/main/java/com/epam/aidial/core/server/config/ConfigPostProcessor.java @@ -31,6 +31,8 @@ import java.util.regex.Pattern; import javax.annotation.Nullable; +import static com.epam.aidial.core.server.util.PlatformCanonicalIdUtil.lastSegment; + /** * Post-processes a freshly-loaded {@link Config} in two passes (slice 2S.9): * @@ -421,11 +423,18 @@ private static void processToolSets(Config config, Set deploymentIds, * Returns true and removes the offending entry when the name was already seen. * Abort mode ({@code onSkip == null}) preserves {@link FileConfigStore}'s today-behavior: * throw {@link IllegalStateException} and roll back the load. + * + *

Dedupes on the derived short name + * ({@link com.epam.aidial.core.server.util.PlatformCanonicalIdUtil#lastSegment}), not the raw map key — + * deployment-id uniqueness is a client-facing (short-name) concept, and a file entry + * (bare key {@code gpt-4}) and a blob entry of a different type (canonical key + * {@code applications/platform/gpt-4}) resolve to the same short name for + * {@link Config#selectDeployment}. Comparing raw keys would miss that collision entirely. */ private static boolean skipOnDuplicate(String name, ResourceTypes type, Set deploymentIds, @Nullable BiConsumer onSkip, Iterator iterator) { - if (deploymentIds.add(name)) { + if (deploymentIds.add(lastSegment(name))) { return false; } if (onSkip == null) { @@ -439,12 +448,30 @@ private static boolean skipOnDuplicate(String name, ResourceTypes type, Setdifferent model/application/interceptor/toolset entry in {@code config}. + * {@link #skipOnDuplicate} enforces the same invariant during a full rebuild via a shared + * {@code deploymentIds} set spanning all four types; this is the equivalent check for a write + * that only touches one entity at a time and never runs skipOnDuplicate. Excludes + * {@code canonicalId} itself so updating an existing entity in place isn't flagged as a + * duplicate of its own prior version. */ - static String lastSegment(String key) { - int slash = key.lastIndexOf('/'); - return slash < 0 ? key : key.substring(slash + 1); + public static boolean isDeploymentIdTaken(Config config, String canonicalId) { + String shortName = lastSegment(canonicalId); + return shortNameTakenIn(config.getModels(), canonicalId, shortName) + || shortNameTakenIn(config.getApplications(), canonicalId, shortName) + || shortNameTakenIn(config.getInterceptors(), canonicalId, shortName) + || shortNameTakenIn(config.getToolsets(), canonicalId, shortName); + } + + private static boolean shortNameTakenIn(Map map, String canonicalId, String shortName) { + for (String key : map.keySet()) { + if (!key.equals(canonicalId) && lastSegment(key).equals(shortName)) { + return true; + } + } + return false; } private static boolean isValidResourceKey(String resourceKey) { diff --git a/server/src/main/java/com/epam/aidial/core/server/config/MergedConfigStore.java b/server/src/main/java/com/epam/aidial/core/server/config/MergedConfigStore.java index 7731e5ef6..18ab4d08d 100644 --- a/server/src/main/java/com/epam/aidial/core/server/config/MergedConfigStore.java +++ b/server/src/main/java/com/epam/aidial/core/server/config/MergedConfigStore.java @@ -48,6 +48,8 @@ import java.util.function.BiConsumer; import java.util.function.Function; +import static com.epam.aidial.core.server.util.PlatformCanonicalIdUtil.lastSegment; + /** * {@link ConfigStore} implementation that builds the runtime {@link Config} as the * union of {@link FileConfigStore} and API-managed entities loaded from @@ -1015,6 +1017,16 @@ private static void recordSchemaAlias(Map schemas, Mapnot restore the file-defined + * entry that {@link #putNameAddressed}/{@link #shadowFileEntry} shadowed when the blob entity + * was created — that resurrection happens naturally on the next full {@link #rebuild()}, which + * always starts from a fresh copy of the file-derived {@link Config} and only shadows short + * names that still have a live blob. This is an accepted, transient gap: a short name deleted + * via this partial-update path stays unreachable until the next periodic rebuild, bounded by + * {@link FileConfigStore}'s poll interval. Do not add restoration logic here — that would make + * delete-then-rebuild diverge instead of converge. + */ private static void removeEntityInPlace(Config config, ResourceTypes type, String canonicalId) { switch (type) { case MODEL -> config.getModels().remove(canonicalId); @@ -1081,6 +1093,21 @@ private Config rebuild() { Map catalogSchemaAliasesById = new HashMap<>(); merged.setRetriableErrorCodes(base.getRetriableErrorCodes()); merged.setGlobalInterceptors(base.getGlobalInterceptors()); + // Wire the (still-being-populated) local maps onto merged now rather than after the blob + // scan below — same references either way, but it lets addBlobEntity/removeAddedEntity/ + // shadowFileEntry dispatch off a single Config parameter instead of a positional map per + // type (they used to take 9-11 map parameters each). + merged.setModels(models); + merged.setInterceptors(interceptors); + merged.setRoles(roles); + merged.setKeys(keys); + merged.setRoutes(routes); + merged.setApplicationTypeSchemas(schemas); + merged.setCatalogSchemas(catalogSchemas); + merged.setApplications(applications); + merged.setToolsets(toolsets); + merged.setSchemaAliasesById(schemaAliasesById); + merged.setCatalogSchemaAliasesById(catalogSchemaAliasesById); Map blobBodies = new HashMap<>(); Map> pendingInvalid = new EnumMap<>(ResourceTypes.class); @@ -1096,6 +1123,12 @@ private Config rebuild() { if (bucket == null) { continue; } + // File-shadowing only makes sense for the platform-scoped copy of a type — the same + // guard onResourceEvent applies for the identical reason (APPLICATION/TOOL_SET also + // resolve for the public bucket). Currently a no-op since listScopes only ever + // returns the platform scope, but keeps this loop safe if that changes. + boolean isPlatformBucket = bucket.equals( + locationStrategy.resolveBucket(type, EntityLocationStrategy.PLATFORM_SCOPE)); String bucketLocation = bucket + ResourceDescriptor.PATH_SEPARATOR; ResourceDescriptor folder = ResourceDescriptorFactory.fromDecoded(type, bucket, bucketLocation, ""); List> items = @@ -1125,9 +1158,7 @@ private Config rebuild() { type, bucket, bucketLocation, name); Object added; try { - added = addBlobEntity(type, canonicalId, node, - models, interceptors, roles, keys, routes, schemas, catalogSchemas, - applications, toolsets, schemaAliasesById, catalogSchemaAliasesById); + added = addBlobEntity(merged, type, canonicalId, node); } catch (Exception parseError) { recordInvalid(pendingInvalid, type, canonicalId, name, "JSON parse failure: " + parseError.getMessage(), @@ -1142,8 +1173,7 @@ private Config rebuild() { } catch (Exception decryptError) { // Roll back the partial insertion so decryption-failure entities never // reach addProjectKeys (locked 2S.9 invariant). - removeAddedEntity(type, canonicalId, models, interceptors, roles, keys, routes, schemas, catalogSchemas, - applications, toolsets); + removeAddedEntity(merged, type, canonicalId); recordInvalid(pendingInvalid, type, canonicalId, name, "Decryption failure: " + decryptError.getMessage(), List.of(new ValidationWarning("body", decryptError.getMessage())), @@ -1156,25 +1186,16 @@ private Config rebuild() { // Shadow the file-defined entry sharing this canonical id's short name, // gated on successful decryption above — if decryption failed, removeAddedEntity // rolled the blob entity back, so the file entry must stay in this rebuild's map. - shadowFileEntry(type, canonicalId, models, interceptors, roles, applications, toolsets); + // Also gated on isPlatformBucket — see the comment where it's computed above. + if (isPlatformBucket) { + shadowFileEntry(merged, type, canonicalId); + } } blobBodies.put(canonicalId, node); } } } - merged.setModels(models); - merged.setInterceptors(interceptors); - merged.setRoles(roles); - merged.setKeys(keys); - merged.setRoutes(routes); - merged.setApplicationTypeSchemas(schemas); - merged.setCatalogSchemas(catalogSchemas); - merged.setApplications(applications); - merged.setToolsets(toolsets); - merged.setSchemaAliasesById(schemaAliasesById); - merged.setCatalogSchemaAliasesById(catalogSchemaAliasesById); - // Semantic pass — under MODE_SKIP, route per-entity violations to invalidEntities and // continue; under MODE_ABORT, the post-processor throws and the rebuild aborts (this.config // stays at the previous value because we only swap below). @@ -1292,11 +1313,6 @@ private void recordInvalid(Map> log.warn("Skipped {} '{}' from merged Config: {}", type.urlSegment(), canonicalId, reason); } - private static String lastSegment(String key) { - int slash = key.lastIndexOf('/'); - return slash < 0 ? key : key.substring(slash + 1); - } - public static String canonicalId(ResourceTypes type, String bucket, String name) { return type.urlSegment() + ResourceDescriptor.PATH_SEPARATOR + bucket + ResourceDescriptor.PATH_SEPARATOR + name; } @@ -1306,59 +1322,59 @@ public static String canonicalId(ResourceDescriptor descriptor) { descriptor.getBucketName(), descriptor.getName()); } - private static Object addBlobEntity(ResourceTypes type, String canonicalId, JsonNode node, - Map models, Map interceptors, - Map roles, Map keys, - LinkedHashMap routes, Map schemas, - Map catalogSchemas, - Map applications, Map toolsets, - Map schemaAliasesById, - Map catalogSchemaAliasesById) + /** + * Reads the type-map to mutate off {@code config} rather than taking one map parameter per + * managed type — {@code config}'s maps are wired up by {@link #rebuild} before the blob scan + * that calls this runs, so they're already the right (still being populated) instances. + */ + private static Object addBlobEntity(Config config, ResourceTypes type, String canonicalId, JsonNode node) throws JsonProcessingException { switch (type) { case MODEL -> { Model entity = ProxyUtil.BLOB_MAPPER.treeToValue(node, Model.class); - warnIfReplaced(type, canonicalId, models.put(canonicalId, entity)); + warnIfReplaced(type, canonicalId, config.getModels().put(canonicalId, entity)); return entity; } case INTERCEPTOR -> { Interceptor entity = ProxyUtil.BLOB_MAPPER.treeToValue(node, Interceptor.class); - warnIfReplaced(type, canonicalId, interceptors.put(canonicalId, entity)); + warnIfReplaced(type, canonicalId, config.getInterceptors().put(canonicalId, entity)); return entity; } case ROLE -> { Role entity = ProxyUtil.BLOB_MAPPER.treeToValue(node, Role.class); - warnIfReplaced(type, canonicalId, roles.put(canonicalId, entity)); + warnIfReplaced(type, canonicalId, config.getRoles().put(canonicalId, entity)); return entity; } case PROJECT_KEY -> { Key entity = ProxyUtil.BLOB_MAPPER.treeToValue(node, Key.class); - warnIfReplaced(type, canonicalId, keys.put(canonicalId, entity)); + warnIfReplaced(type, canonicalId, config.getKeys().put(canonicalId, entity)); return entity; } case ROUTE -> { Route entity = ProxyUtil.BLOB_MAPPER.treeToValue(node, Route.class); - warnIfReplaced(type, canonicalId, routes.put(canonicalId, entity)); + warnIfReplaced(type, canonicalId, config.getRoutes().put(canonicalId, entity)); return entity; } case APP_TYPE_SCHEMA -> { + Map schemas = config.getApplicationTypeSchemas(); warnIfReplaced(type, canonicalId, schemas.put(canonicalId, node.toString())); - recordSchemaAlias(schemas, schemaAliasesById, canonicalId, node); + recordSchemaAlias(schemas, config.getSchemaAliasesById(), canonicalId, node); return null; } case CATALOG_SCHEMA -> { + Map catalogSchemas = config.getCatalogSchemas(); warnIfReplaced(type, canonicalId, catalogSchemas.put(canonicalId, node.toString())); - recordSchemaAlias(catalogSchemas, catalogSchemaAliasesById, canonicalId, node); + recordSchemaAlias(catalogSchemas, config.getCatalogSchemaAliasesById(), canonicalId, node); return null; } case APPLICATION -> { Application entity = ProxyUtil.BLOB_MAPPER.treeToValue(node, Application.class); - warnIfReplaced(type, canonicalId, applications.put(canonicalId, entity)); + warnIfReplaced(type, canonicalId, config.getApplications().put(canonicalId, entity)); return entity; } case TOOL_SET -> { ToolSet entity = ProxyUtil.BLOB_MAPPER.treeToValue(node, ToolSet.class); - warnIfReplaced(type, canonicalId, toolsets.put(canonicalId, entity)); + warnIfReplaced(type, canonicalId, config.getToolsets().put(canonicalId, entity)); return entity; } default -> { @@ -1374,16 +1390,13 @@ private static Object addBlobEntity(ResourceTypes type, String canonicalId, Json * over the file entry it just shadowed. No-op for types that aren't name-addressed (keys, * routes, schemas — schemas use the $id index instead, see {@link #recordSchemaAlias}). */ - private static void shadowFileEntry(ResourceTypes type, String canonicalId, - Map models, Map interceptors, - Map roles, Map applications, - Map toolsets) { + private static void shadowFileEntry(Config config, ResourceTypes type, String canonicalId) { switch (type) { - case MODEL -> models.remove(lastSegment(canonicalId)); - case INTERCEPTOR -> interceptors.remove(lastSegment(canonicalId)); - case ROLE -> roles.remove(lastSegment(canonicalId)); - case APPLICATION -> applications.remove(lastSegment(canonicalId)); - case TOOL_SET -> toolsets.remove(lastSegment(canonicalId)); + case MODEL -> config.getModels().remove(lastSegment(canonicalId)); + case INTERCEPTOR -> config.getInterceptors().remove(lastSegment(canonicalId)); + case ROLE -> config.getRoles().remove(lastSegment(canonicalId)); + case APPLICATION -> config.getApplications().remove(lastSegment(canonicalId)); + case TOOL_SET -> config.getToolsets().remove(lastSegment(canonicalId)); default -> { /* not name-addressed */ } } } @@ -1395,22 +1408,17 @@ private static void warnIfReplaced(ResourceTypes type, String canonicalId, Objec } } - private static void removeAddedEntity(ResourceTypes type, String canonicalId, - Map models, Map interceptors, - Map roles, Map keys, - LinkedHashMap routes, Map schemas, - Map catalogSchemas, - Map applications, Map toolsets) { + private static void removeAddedEntity(Config config, ResourceTypes type, String canonicalId) { switch (type) { - case MODEL -> models.remove(canonicalId); - case INTERCEPTOR -> interceptors.remove(canonicalId); - case ROLE -> roles.remove(canonicalId); - case PROJECT_KEY -> keys.remove(canonicalId); - case ROUTE -> routes.remove(canonicalId); - case APP_TYPE_SCHEMA -> schemas.remove(canonicalId); - case CATALOG_SCHEMA -> catalogSchemas.remove(canonicalId); - case APPLICATION -> applications.remove(canonicalId); - case TOOL_SET -> toolsets.remove(canonicalId); + case MODEL -> config.getModels().remove(canonicalId); + case INTERCEPTOR -> config.getInterceptors().remove(canonicalId); + case ROLE -> config.getRoles().remove(canonicalId); + case PROJECT_KEY -> config.getKeys().remove(canonicalId); + case ROUTE -> config.getRoutes().remove(canonicalId); + case APP_TYPE_SCHEMA -> config.getApplicationTypeSchemas().remove(canonicalId); + case CATALOG_SCHEMA -> config.getCatalogSchemas().remove(canonicalId); + case APPLICATION -> config.getApplications().remove(canonicalId); + case TOOL_SET -> config.getToolsets().remove(canonicalId); default -> { /* no-op */ } } } diff --git a/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java b/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java index c6d710421..d9f11f969 100644 --- a/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java +++ b/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java @@ -308,8 +308,18 @@ static ValidationResult validateOnly(AdminManifest entry, Config scratch, boolea if (!warnings.isEmpty() && !softValidation) { return new ValidationResult(id, ValidationStatus.FAILED, joinWarnings(warnings)); } + String dupError = duplicateDeploymentIdMessage(scratch, ResourceTypes.MODEL, parsed); + if (dupError != null) { + return new ValidationResult(id, ValidationStatus.FAILED, dupError); + } + } + case "Interceptor" -> { + ConfigResourceController.treeToEntity(entry.spec(), Interceptor.class); + String dupError = duplicateDeploymentIdMessage(scratch, ResourceTypes.INTERCEPTOR, parsed); + if (dupError != null) { + return new ValidationResult(id, ValidationStatus.FAILED, dupError); + } } - case "Interceptor" -> ConfigResourceController.treeToEntity(entry.spec(), Interceptor.class); case "Role" -> ConfigResourceController.treeToEntity(entry.spec(), Role.class); case "Route" -> ConfigResourceController.treeToEntity(entry.spec(), Route.class); case "Key" -> { @@ -325,8 +335,24 @@ static ValidationResult validateOnly(AdminManifest entry, Config scratch, boolea "Invalid key: at least one role must be assigned to the key " + key.getProject()); } } - case "Application" -> ConfigResourceController.treeToEntity(entry.spec(), Application.class); - case "ToolSet" -> ConfigResourceController.treeToEntity(entry.spec(), ToolSet.class); + case "Application" -> { + ConfigResourceController.treeToEntity(entry.spec(), Application.class); + if (ResourceDescriptor.PLATFORM_BUCKET.equals(parsed.bucket())) { + String dupError = duplicateDeploymentIdMessage(scratch, ResourceTypes.APPLICATION, parsed); + if (dupError != null) { + return new ValidationResult(id, ValidationStatus.FAILED, dupError); + } + } + } + case "ToolSet" -> { + ConfigResourceController.treeToEntity(entry.spec(), ToolSet.class); + if (ResourceDescriptor.PLATFORM_BUCKET.equals(parsed.bucket())) { + String dupError = duplicateDeploymentIdMessage(scratch, ResourceTypes.TOOL_SET, parsed); + if (dupError != null) { + return new ValidationResult(id, ValidationStatus.FAILED, dupError); + } + } + } case "Schema" -> { if (!entry.spec().isObject()) { return new ValidationResult(id, ValidationStatus.FAILED, "Schema spec must be a JSON object"); @@ -347,6 +373,21 @@ static ValidationResult validateOnly(AdminManifest entry, Config scratch, boolea return new ValidationResult(id, ValidationStatus.VALID, null); } + /** + * Shared by {@link #validateOnly} (precheck) and the real-apply {@code applyX} methods: + * non-null iff {@code type}'s canonical id under {@code parsed} has a short name already + * claimed by a different model/application/interceptor/toolset in {@code scratch}. See + * {@link ConfigPostProcessor#isDeploymentIdTaken}. + */ + private static String duplicateDeploymentIdMessage(Config scratch, ResourceTypes type, ParsedName parsed) { + ResourceDescriptor descriptor = ResourceDescriptorFactory.fromDecoded( + type, parsed.bucket(), parsed.location(), parsed.name()); + if (ConfigPostProcessor.isDeploymentIdTaken(scratch, MergedConfigStore.canonicalId(descriptor))) { + return "Deployment ID '" + parsed.name() + "' is already used by a different entity"; + } + return null; + } + private EntityResult applySingle(AdminManifest entry, Config scratch, List pending) { String id = entry.name(); ParsedName parsed; @@ -359,13 +400,13 @@ private EntityResult applySingle(AdminManifest entry, Config scratch, List applySettings(entry, id, parsed); case "Schema" -> applySchema(entry, id, parsed, pending, ResourceTypes.APP_TYPE_SCHEMA); case "CatalogSchema" -> applySchema(entry, id, parsed, pending, ResourceTypes.CATALOG_SCHEMA); - case "Interceptor" -> applyManagedEntity(entry, id, parsed, ResourceTypes.INTERCEPTOR, Interceptor.class, pending); - case "Role" -> applyManagedEntity(entry, id, parsed, ResourceTypes.ROLE, Role.class, pending); - case "Route" -> applyManagedEntity(entry, id, parsed, ResourceTypes.ROUTE, Route.class, pending); + case "Interceptor" -> applyManagedEntity(entry, id, parsed, ResourceTypes.INTERCEPTOR, Interceptor.class, scratch, pending); + case "Role" -> applyManagedEntity(entry, id, parsed, ResourceTypes.ROLE, Role.class, scratch, pending); + case "Route" -> applyManagedEntity(entry, id, parsed, ResourceTypes.ROUTE, Route.class, scratch, pending); case "Key" -> applyKey(entry, id, parsed, pending); case "Model" -> applyModel(entry, id, parsed, scratch, pending); - case "ToolSet" -> applyToolSet(entry, id, parsed, pending); - case "Application" -> applyApplication(entry, id, parsed, pending); + case "ToolSet" -> applyToolSet(entry, id, parsed, scratch, pending); + case "Application" -> applyApplication(entry, id, parsed, scratch, pending); default -> new EntityResult(id, AdminApplyStatus.FAILED, "Unknown kind: " + entry.kind()); }; } @@ -403,10 +444,19 @@ private EntityResult applySchema(AdminManifest entry, String id, ParsedName pars } private EntityResult applyManagedEntity(AdminManifest entry, String id, ParsedName parsed, - ResourceTypes type, Class entityClass, List pending) { + ResourceTypes type, Class entityClass, Config scratch, + List pending) { T entity = ConfigResourceController.treeToEntity(entry.spec(), entityClass); ResourceDescriptor descriptor = ResourceDescriptorFactory.fromDecoded( type, parsed.bucket(), parsed.location(), parsed.name()); + // Deployment-id uniqueness only applies to INTERCEPTOR here — ROLE/ROUTE aren't deployments + // resolved through Config.selectDeployment, so they don't share the short-name namespace. + if (type == ResourceTypes.INTERCEPTOR) { + String dupError = duplicateDeploymentIdMessage(scratch, type, parsed); + if (dupError != null) { + return new EntityResult(id, AdminApplyStatus.FAILED, dupError); + } + } String blobBody = ConfigResourceController.serializeForBlob(entity); resourceService.putResource(descriptor, blobBody, EtagHeader.ANY); pending.add(new EntityChange(type, MergedConfigStore.canonicalId(descriptor), entity)); @@ -472,6 +522,10 @@ private EntityResult applyModel(AdminManifest entry, String id, ParsedName parse } ResourceDescriptor descriptor = ResourceDescriptorFactory.fromDecoded( ResourceTypes.MODEL, parsed.bucket(), parsed.location(), parsed.name()); + String dupError = duplicateDeploymentIdMessage(scratch, ResourceTypes.MODEL, parsed); + if (dupError != null) { + return new EntityResult(id, AdminApplyStatus.FAILED, dupError); + } secretFieldProcessor.encryptFields(model, descriptor); String blobBody = ConfigResourceController.serializeForBlob(model); resourceService.putResource(descriptor, blobBody, EtagHeader.ANY); @@ -481,32 +535,47 @@ private EntityResult applyModel(AdminManifest entry, String id, ParsedName parse return new EntityResult(id, invalid ? AdminApplyStatus.APPLIED_INVALID : AdminApplyStatus.APPLIED, null); } - private EntityResult applyApplication(AdminManifest entry, String id, ParsedName parsed, List pending) { + private EntityResult applyApplication(AdminManifest entry, String id, ParsedName parsed, Config scratch, List pending) { Application application = ConfigResourceController.treeToEntity(entry.spec(), Application.class); ResourceDescriptor descriptor = ResourceDescriptorFactory.fromDecoded( ResourceTypes.APPLICATION, parsed.bucket(), parsed.location(), parsed.name()); + // Only the platform bucket is materialized into MergedConfigStore (see EntityLocationStrategy) — + // public-bucket apps stay outside it and are served lazily by ApplicationService, so they're + // exempt from deployment-id uniqueness and pushing them into `pending` below would spuriously + // duplicate them in config.getApplications()-backed listings (e.g. ApplicationController/ + // DeploymentController) until the next full rebuild. + boolean platform = ResourceDescriptor.PLATFORM_BUCKET.equals(parsed.bucket()); + if (platform) { + String dupError = duplicateDeploymentIdMessage(scratch, ResourceTypes.APPLICATION, parsed); + if (dupError != null) { + return new EntityResult(id, AdminApplyStatus.FAILED, dupError); + } + } // Bulk admin apply is always admin context — preserve forwardAuthToken if the manifest set it. applicationService.putApplication(descriptor, EtagHeader.ANY, null, application, true, AdminManagedFieldsWriteMode.AUTHORITATIVE); - // Only the platform bucket is materialized into MergedConfigStore (see EntityLocationStrategy) — - // public-bucket apps stay outside it and are served lazily by ApplicationService, so pushing - // them into `pending` here would spuriously duplicate them in config.getApplications()-backed - // listings (e.g. ApplicationController/DeploymentController) until the next full rebuild. - if (ResourceDescriptor.PLATFORM_BUCKET.equals(parsed.bucket())) { + if (platform) { Application decrypted = applicationService.getApplicationWithDecryptedSecrets(descriptor).getValue(); pending.add(new EntityChange(ResourceTypes.APPLICATION, MergedConfigStore.canonicalId(descriptor), decrypted)); } return new EntityResult(id, AdminApplyStatus.APPLIED, null); } - private EntityResult applyToolSet(AdminManifest entry, String id, ParsedName parsed, List pending) { + private EntityResult applyToolSet(AdminManifest entry, String id, ParsedName parsed, Config scratch, List pending) { ToolSet toolSet = ConfigResourceController.treeToEntity(entry.spec(), ToolSet.class); ResourceDescriptor descriptor = ResourceDescriptorFactory.fromDecoded( ResourceTypes.TOOL_SET, parsed.bucket(), parsed.location(), parsed.name()); - toolSetService.putToolSet(descriptor, EtagHeader.ANY, null, toolSet, true); // Same rationale as applyApplication above — only platform-bucket toolsets belong in - // MergedConfigStore. - if (ResourceDescriptor.PLATFORM_BUCKET.equals(parsed.bucket())) { + // MergedConfigStore / are subject to deployment-id uniqueness. + boolean platform = ResourceDescriptor.PLATFORM_BUCKET.equals(parsed.bucket()); + if (platform) { + String dupError = duplicateDeploymentIdMessage(scratch, ResourceTypes.TOOL_SET, parsed); + if (dupError != null) { + return new EntityResult(id, AdminApplyStatus.FAILED, dupError); + } + } + toolSetService.putToolSet(descriptor, EtagHeader.ANY, null, toolSet, true); + if (platform) { ToolSet decrypted = toolSetService.getToolSetWithDecryptedAuthSettings(descriptor).getValue(); pending.add(new EntityChange(ResourceTypes.TOOL_SET, MergedConfigStore.canonicalId(descriptor), decrypted)); } diff --git a/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java b/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java index cd91b406f..f8387626f 100644 --- a/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java +++ b/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java @@ -1305,6 +1305,10 @@ private Future handleAppOrToolSetPut() { throw new HttpException(HttpStatus.BAD_REQUEST, "Request body must be a JSON object"); } return taskExecutor.submit(() -> lockService.underBucketLocks(MergedConfigStore.ADMIN_BUCKET_LOCATIONS, () -> { + // This route is platform-bucket-only (see the dedicated /v1/(applications|toolsets)/ + // platform/... routes), so unlike the generic ResourceController path every write + // here is materialized into Config and subject to deployment-id uniqueness. + rejectDuplicateDeploymentId(descriptor); Object decrypted; // The platform bucket requires explicit admin access for every operation (see // AdminRoleAuthorizationService), not just an admin-AND-public-bucket combination like @@ -1443,6 +1447,10 @@ private Future handlePut() { if (entity instanceof Model m) { checkCrossReferences(m); } + ResourceTypes writeType = resourceType(); + if (writeType == ResourceTypes.MODEL || writeType == ResourceTypes.INTERCEPTOR) { + rejectDuplicateDeploymentId(descriptor); + } if (spec.isKey()) { keyEntity = (Key) entity; validateKeyForApiWrite(keyEntity, "PUT"); @@ -1645,6 +1653,24 @@ private void handleWriteError(Throwable error) { } } + /** + * Rejects a MODEL/INTERCEPTOR/APPLICATION/TOOL_SET write whose derived short name is already + * claimed by a different deployment (of any of those four types) in the live merged Config — + * the partial-update-path counterpart of {@code ConfigPostProcessor.skipOnDuplicate}, which + * only runs during a full rebuild. See {@link ConfigPostProcessor#isDeploymentIdTaken}. + */ + private void rejectDuplicateDeploymentId(ResourceDescriptor descriptor) { + Config snapshot = mergedConfigStore.get(); + if (snapshot == null) { + return; + } + String canonicalId = MergedConfigStore.canonicalId(descriptor); + if (ConfigPostProcessor.isDeploymentIdTaken(snapshot, canonicalId)) { + throw new HttpException(HttpStatus.CONFLICT, + "Deployment ID '" + path + "' is already used by a different entity"); + } + } + /** * Cross-reference check for Model writes. Strict mode aborts with HTTP 422 carrying a * {@code {"validationWarnings":[...]}} JSON body. Soft mode logs and proceeds — the next diff --git a/server/src/main/java/com/epam/aidial/core/server/limiter/RateLimiter.java b/server/src/main/java/com/epam/aidial/core/server/limiter/RateLimiter.java index d8d9cb4c2..7a6cdf7b7 100644 --- a/server/src/main/java/com/epam/aidial/core/server/limiter/RateLimiter.java +++ b/server/src/main/java/com/epam/aidial/core/server/limiter/RateLimiter.java @@ -26,6 +26,7 @@ import java.math.BigDecimal; import java.util.List; +import java.util.Map; import java.util.Optional; @Slf4j @@ -396,9 +397,24 @@ private static String getPathToCosts() { } private static Limit getLimit(Config config, String userRole, String name, Limit defaultLimit) { - return Optional.ofNullable(config.getRole(userRole)) - .map(role -> role.getLimits().get(name)) - .orElse(defaultLimit); + Role role = config.getRole(userRole); + if (role == null) { + return defaultLimit; + } + Limit limit = role.getLimits().get(name); + if (limit != null) { + return limit; + } + // Compat: a Role.limits entry may still be keyed by the canonical id a previously-shipped + // build exposed for API-managed deployments (e.g. "models/platform/gpt-4") before + // deployment.getName() reverted to the short name. Fall back to a last-segment match so + // those pre-existing entries keep resolving after upgrade. + for (Map.Entry entry : role.getLimits().entrySet()) { + if (entry.getKey().endsWith("/" + name)) { + return entry.getValue(); + } + } + return defaultLimit; } private static CostLimit getCostLimit(Config config, String userRole, CostLimit defaultCostLimit) { diff --git a/server/src/main/java/com/epam/aidial/core/server/log/AnalyticsLogContext.java b/server/src/main/java/com/epam/aidial/core/server/log/AnalyticsLogContext.java index b860bfb10..e3703076d 100644 --- a/server/src/main/java/com/epam/aidial/core/server/log/AnalyticsLogContext.java +++ b/server/src/main/java/com/epam/aidial/core/server/log/AnalyticsLogContext.java @@ -26,6 +26,8 @@ import java.util.Scanner; import javax.annotation.Nullable; +import static com.epam.aidial.core.server.util.PlatformCanonicalIdUtil.lastSegment; + @Slf4j @Getter @Builder @@ -148,7 +150,11 @@ public static String getParentDeployment(String sourceDeployment, List i int i = executionPath.size() - 2; for (int j = interceptors.size() - 1; i >= 0 && j >= 0; i--, j--) { String deployment = executionPath.get(i); - String interceptor = interceptors.get(j); + // executionPath entries are always the resolved deployment.getName() (short name); + // interceptors holds the raw config reference, which may be a short name or a + // canonical id (e.g. "interceptors/platform/my-interceptor") - normalize before + // comparing so a canonical-id reference doesn't look like a path mismatch. + String interceptor = lastSegment(interceptors.get(j)); if (!deployment.equals(interceptor)) { log.warn("Can't find parent deployment because interceptor path doesn't match: expected - {}, actual - {}", interceptor, deployment); return null; diff --git a/server/src/main/java/com/epam/aidial/core/server/service/ConsentService.java b/server/src/main/java/com/epam/aidial/core/server/service/ConsentService.java index 0e388c36d..38d705558 100644 --- a/server/src/main/java/com/epam/aidial/core/server/service/ConsentService.java +++ b/server/src/main/java/com/epam/aidial/core/server/service/ConsentService.java @@ -17,9 +17,12 @@ import java.util.ArrayDeque; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Set; +import static com.epam.aidial.core.server.util.PlatformCanonicalIdUtil.lastSegment; + @Slf4j public class ConsentService { @@ -88,17 +91,37 @@ public void verifyUserConsent(ProxyContext context, Deployment deployment) { } if (executionPath != null) { for (String dep : executionPath) { - if (!consent.getDeployments().containsKey(dep)) { + if (findConsentDeployment(consent, dep) == null) { fail(currentDeploymentId); } } } - Consent.Deployment consentDeployment = consent.getDeployments().get(currentDeploymentId); + Consent.Deployment consentDeployment = findConsentDeployment(consent, currentDeploymentId); if (consentDeployment == null || !consentDeployment.isConsentRequired()) { fail(currentDeploymentId); } } + /** + * Looks up a deployment's consent record by id, falling back to a last-segment match when the + * exact key misses. Compat for consent records persisted before deployment.getName() reverted + * from canonical id (e.g. "models/platform/gpt-4") to short name for API-managed deployments — + * without this, every such record would silently fail re-consent after upgrade. + */ + private static Consent.Deployment findConsentDeployment(Consent consent, String id) { + Consent.Deployment exact = consent.getDeployments().get(id); + if (exact != null) { + return exact; + } + String shortId = lastSegment(id); + for (Map.Entry entry : consent.getDeployments().entrySet()) { + if (lastSegment(entry.getKey()).equals(shortId)) { + return entry.getValue(); + } + } + return null; + } + private String getRootDeploymentId(ProxyContext context, Deployment current) { if (context.getApiKeyData().getPerRequestKey() == null) { return current.getName(); diff --git a/server/src/main/java/com/epam/aidial/core/server/util/PlatformCanonicalIdUtil.java b/server/src/main/java/com/epam/aidial/core/server/util/PlatformCanonicalIdUtil.java new file mode 100644 index 000000000..a4f579b36 --- /dev/null +++ b/server/src/main/java/com/epam/aidial/core/server/util/PlatformCanonicalIdUtil.java @@ -0,0 +1,21 @@ +package com.epam.aidial.core.server.util; + +/** + * Shared helper for the canonical-id ↔ short-name relationship used across config loading + * ({@code ConfigPostProcessor}, {@code MergedConfigStore}), consent records ({@code ConsentService}), + * and analytics logging ({@code AnalyticsLogContext}). Applies only to {@code platform}-bucket + * canonical ids, which are always exactly {@code type/platform/name} with no further nesting + * (enforced at the write boundary — see {@code ConfigResourceController.ENTITY_NAME_PATTERN}), so + * the last path segment is unambiguously the short name. A bare (slash-free) name is already its + * own short name. + */ +public final class PlatformCanonicalIdUtil { + + private PlatformCanonicalIdUtil() { + } + + public static String lastSegment(String key) { + int slash = key.lastIndexOf('/'); + return slash < 0 ? key : key.substring(slash + 1); + } +} From 068c896d528137a0f3d751856659a050ab2c16f9 Mon Sep 17 00:00:00 2001 From: Kiryl_Kurnosenka Date: Tue, 11 Aug 2026 22:51:15 +0300 Subject: [PATCH 04/15] fix: correct schema $id alias index eviction and add collision check #1783 recordSchemaAlias never evicted a schema's previous $id alias when its $id changed, leaving a dangling entry pointing at a body that no longer carries it. Fix by evicting any alias currently held by this canonical id before recording the new one (same removeIf-by-value pattern already used on delete), and add a write-time check rejecting a write whose $id is already claimed by a different canonical id (wired into the single- entity PUT, admin-apply real-apply, and admin-apply precheck paths). Co-Authored-By: Claude Sonnet 5 --- .../core/server/config/MergedConfigStore.java | 20 +++++++++- .../controller/AdminApplyController.java | 24 ++++++++++-- .../controller/ConfigResourceController.java | 39 +++++++++++++++++++ 3 files changed, 78 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/com/epam/aidial/core/server/config/MergedConfigStore.java b/server/src/main/java/com/epam/aidial/core/server/config/MergedConfigStore.java index 18ab4d08d..73a08fadf 100644 --- a/server/src/main/java/com/epam/aidial/core/server/config/MergedConfigStore.java +++ b/server/src/main/java/com/epam/aidial/core/server/config/MergedConfigStore.java @@ -1007,9 +1007,23 @@ private static void putSchemaInPlace(Map schemas, MapAn update may change this entity's own $id — evict any alias {@code canonicalId} held + * under its previous $id first, so the old $id stops resolving once the body no longer + * carries it. No-op during a full rebuild, where {@code aliasesById} starts empty each pass. + * + *

Same transient gap as {@link #removeEntityInPlace}'s javadoc describes for a deleted + * canonical id, but triggered by an update rather than a delete: the file entry + * shadowed under the old $id when this canonical id was first migrated is removed from + * {@code schemas} at that point and is never restored here, even though this canonical id no + * longer claims that $id after the change — the old $id stays unreachable via either the file + * entry or this canonical id until the next full {@link #rebuild()}. Changing a schema's $id + * away from a value is, from the shadowed file entry's perspective, indistinguishable from + * deleting the entity that was shadowing it. */ - private static void recordSchemaAlias(Map schemas, Map aliasesById, + public static void recordSchemaAlias(Map schemas, Map aliasesById, String canonicalId, JsonNode node) { + aliasesById.values().removeIf(canonicalId::equals); JsonNode idNode = node.get("$id"); if (idNode != null && idNode.isTextual()) { schemas.remove(idNode.asText()); @@ -1026,6 +1040,10 @@ private static void recordSchemaAlias(Map schemas, MapFor {@code APP_TYPE_SCHEMA}/{@code CATALOG_SCHEMA} the identical gap also arises without + * a delete at all — see {@link #recordSchemaAlias}'s javadoc for why an in-place $id change + * has the same effect on the old $id's shadowed file entry. */ private static void removeEntityInPlace(Config config, ResourceTypes type, String canonicalId) { switch (type) { diff --git a/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java b/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java index d9f11f969..a142a7f6b 100644 --- a/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java +++ b/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java @@ -272,6 +272,8 @@ static Config newScratch(MergedConfigStore mergedConfigStore) { scratch.setInterceptors(new HashMap<>(live.getInterceptors())); scratch.setApplicationTypeSchemas(new HashMap<>(live.getApplicationTypeSchemas())); scratch.setCatalogSchemas(new HashMap<>(live.getCatalogSchemas())); + scratch.setSchemaAliasesById(new HashMap<>(live.getSchemaAliasesById())); + scratch.setCatalogSchemaAliasesById(new HashMap<>(live.getCatalogSchemaAliasesById())); scratch.setApplications(new HashMap<>(live.getApplications())); scratch.setToolsets(new HashMap<>(live.getToolsets())); scratch.setRoles(new HashMap<>(live.getRoles())); @@ -357,11 +359,15 @@ static ValidationResult validateOnly(AdminManifest entry, Config scratch, boolea if (!entry.spec().isObject()) { return new ValidationResult(id, ValidationStatus.FAILED, "Schema spec must be a JSON object"); } + ConfigResourceController.rejectSchemaIdCollision(scratch, ResourceTypes.APP_TYPE_SCHEMA, + entry.name(), entry.spec()); } case "CatalogSchema" -> { if (!entry.spec().isObject()) { return new ValidationResult(id, ValidationStatus.FAILED, "CatalogSchema spec must be a JSON object"); } + ConfigResourceController.rejectSchemaIdCollision(scratch, ResourceTypes.CATALOG_SCHEMA, + entry.name(), entry.spec()); } default -> { return new ValidationResult(id, ValidationStatus.FAILED, "Unknown kind: " + entry.kind()); @@ -398,8 +404,8 @@ private EntityResult applySingle(AdminManifest entry, Config scratch, List applySettings(entry, id, parsed); - case "Schema" -> applySchema(entry, id, parsed, pending, ResourceTypes.APP_TYPE_SCHEMA); - case "CatalogSchema" -> applySchema(entry, id, parsed, pending, ResourceTypes.CATALOG_SCHEMA); + case "Schema" -> applySchema(entry, id, parsed, scratch, pending, ResourceTypes.APP_TYPE_SCHEMA); + case "CatalogSchema" -> applySchema(entry, id, parsed, scratch, pending, ResourceTypes.CATALOG_SCHEMA); case "Interceptor" -> applyManagedEntity(entry, id, parsed, ResourceTypes.INTERCEPTOR, Interceptor.class, scratch, pending); case "Role" -> applyManagedEntity(entry, id, parsed, ResourceTypes.ROLE, Role.class, scratch, pending); case "Route" -> applyManagedEntity(entry, id, parsed, ResourceTypes.ROUTE, Route.class, scratch, pending); @@ -423,13 +429,19 @@ private EntityResult applySettings(AdminManifest entry, String id, ParsedName pa return new EntityResult(id, AdminApplyStatus.APPLIED, null); } - private EntityResult applySchema(AdminManifest entry, String id, ParsedName parsed, List pending, - ResourceTypes type) { + private EntityResult applySchema(AdminManifest entry, String id, ParsedName parsed, Config scratch, + List pending, ResourceTypes type) { if (!entry.spec().isObject()) { return new EntityResult(id, AdminApplyStatus.FAILED, "Schema spec must be a JSON object"); } ResourceDescriptor descriptor = ResourceDescriptorFactory.fromDecoded( type, parsed.bucket(), parsed.location(), parsed.name()); + try { + ConfigResourceController.rejectSchemaIdCollision(scratch, type, + MergedConfigStore.canonicalId(descriptor), entry.spec()); + } catch (HttpException e) { + return new EntityResult(id, AdminApplyStatus.FAILED, e.getMessage()); + } String blobBody; try { blobBody = ProxyUtil.BLOB_MAPPER.writeValueAsString(entry.spec()); @@ -626,6 +638,8 @@ static void mutateScratch(Config scratch, AdminManifest entry) { return; } scratch.getApplicationTypeSchemas().put(entry.name(), json); + MergedConfigStore.recordSchemaAlias(scratch.getApplicationTypeSchemas(), + scratch.getSchemaAliasesById(), entry.name(), entry.spec()); } case "CatalogSchema" -> { String json; @@ -635,6 +649,8 @@ static void mutateScratch(Config scratch, AdminManifest entry) { return; } scratch.getCatalogSchemas().put(entry.name(), json); + MergedConfigStore.recordSchemaAlias(scratch.getCatalogSchemas(), + scratch.getCatalogSchemaAliasesById(), entry.name(), entry.spec()); } default -> { /* unknown kinds never reach this code path */ } } diff --git a/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java b/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java index f8387626f..871671625 100644 --- a/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java +++ b/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java @@ -1402,6 +1402,11 @@ private Future handlePut() { String oldSecret = null; Object entity = null; if (spec.entityClass() == null) { + ResourceTypes schemaType = resourceType(); + if (schemaType == ResourceTypes.APP_TYPE_SCHEMA || schemaType == ResourceTypes.CATALOG_SCHEMA) { + rejectSchemaIdCollision(mergedConfigStore.get(), schemaType, + MergedConfigStore.canonicalId(descriptor), requestNode); + } blobBody = requestNode.toString(); } else { if (!requestNode.isObject()) { @@ -1560,6 +1565,40 @@ private static ResourceTypes typeOf(ResourceDescriptor descriptor) { return (ResourceTypes) descriptor.getType(); } + /** + * App-type/catalog schemas are looked up by their body-embedded {@code $id} + * ({@link Config#getCustomApplicationSchema}/{@link Config#getCatalogSchema}) via + * {@code MergedConfigStore}'s {@code $id -> canonicalId} alias index, which only ever holds + * one canonical id per $id. Reject a write whose $id is already claimed by a *different* + * canonical id, rather than let the index silently pick whichever write landed most recently. + * The alias index only ever contains blob-sourced entries (file-sourced schemas are keyed + * directly by $id and never added to it), so a file-schema sharing this $id — the expected + * pre-migration predecessor a blob write is meant to shadow — is never mistaken for a + * collision here. + * + *

Package-visible: shared with {@link AdminApplyController#applySchema}/ + * {@link AdminApplyController#validateOnly}, the other write/precheck paths for these types. + * Takes a {@link Config} snapshot rather than {@link MergedConfigStore} so callers can pass + * either the live merged config or a batch-apply {@code scratch} clone. + */ + static void rejectSchemaIdCollision(Config snapshot, ResourceTypes type, String thisCanonicalId, JsonNode requestNode) { + if (snapshot == null) { + return; + } + JsonNode idNode = requestNode.get("$id"); + if (idNode == null || !idNode.isTextual()) { + return; + } + String id = idNode.asText(); + Map aliasesById = type == ResourceTypes.APP_TYPE_SCHEMA + ? snapshot.getSchemaAliasesById() : snapshot.getCatalogSchemaAliasesById(); + String owner = aliasesById.get(id); + if (owner != null && !owner.equals(thisCanonicalId)) { + throw new HttpException(HttpStatus.CONFLICT, + "Schema $id '" + id + "' is already used by '" + owner + "'"); + } + } + private static ApiKeyData apiKeyData(Key key) { ApiKeyData data = new ApiKeyData(); data.setOriginalKey(key); From 8d35051291795e38a6daa12ddc86604a8208913e Mon Sep 17 00:00:00 2001 From: Kiryl_Kurnosenka Date: Wed, 12 Aug 2026 12:26:01 +0300 Subject: [PATCH 05/15] fix: fail loudly on unsupported type in schema $id collision check rejectSchemaIdCollision silently treated any non-APP_TYPE_SCHEMA type as CATALOG_SCHEMA via a fallback ternary. Switch to an explicit type check that throws for anything other than the two supported schema types. Co-Authored-By: Claude Sonnet 5 --- .../core/server/controller/ConfigResourceController.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java b/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java index 871671625..038bc0178 100644 --- a/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java +++ b/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java @@ -1590,8 +1590,11 @@ static void rejectSchemaIdCollision(Config snapshot, ResourceTypes type, String return; } String id = idNode.asText(); - Map aliasesById = type == ResourceTypes.APP_TYPE_SCHEMA - ? snapshot.getSchemaAliasesById() : snapshot.getCatalogSchemaAliasesById(); + Map aliasesById = switch (type) { + case APP_TYPE_SCHEMA -> snapshot.getSchemaAliasesById(); + case CATALOG_SCHEMA -> snapshot.getCatalogSchemaAliasesById(); + default -> throw new IllegalArgumentException("Unsupported type for schema $id check: " + type); + }; String owner = aliasesById.get(id); if (owner != null && !owner.equals(thisCanonicalId)) { throw new HttpException(HttpStatus.CONFLICT, From 40cca57efaf437c0ee12dc9ae34239058e1e664e Mon Sep 17 00:00:00 2001 From: Kiryl_Kurnosenka Date: Wed, 12 Aug 2026 15:29:46 +0300 Subject: [PATCH 06/15] fix: evict schema $id alias by key instead of scanning by value recordSchemaAlias evicted a stale alias via aliasesById.values().removeIf, an O(index size) scan on every schema write/delete (and, since it was always a guaranteed no-op mid-rebuild, effectively O(n^2) extra work across a full rebuild for no benefit). Map.put/remove already return the previous value at a key, so callers now pass that previous body through and recordSchemaAlias reads its $id directly and evicts by that key - O(1) instead of a full index scan for every schema mutation. Co-Authored-By: Claude Sonnet 5 --- .../core/server/config/MergedConfigStore.java | 62 +++++++++++++------ .../controller/AdminApplyController.java | 8 +-- 2 files changed, 48 insertions(+), 22 deletions(-) diff --git a/server/src/main/java/com/epam/aidial/core/server/config/MergedConfigStore.java b/server/src/main/java/com/epam/aidial/core/server/config/MergedConfigStore.java index 73a08fadf..c51bc57f6 100644 --- a/server/src/main/java/com/epam/aidial/core/server/config/MergedConfigStore.java +++ b/server/src/main/java/com/epam/aidial/core/server/config/MergedConfigStore.java @@ -47,6 +47,7 @@ import java.util.concurrent.locks.ReentrantLock; import java.util.function.BiConsumer; import java.util.function.Function; +import javax.annotation.Nullable; import static com.epam.aidial.core.server.util.PlatformCanonicalIdUtil.lastSegment; @@ -989,9 +990,9 @@ private static void putNameAddressed(Map map, String canonicalId, private static void putSchemaInPlace(Map schemas, Map aliasesById, String canonicalId, Object entity) { String body = schemaBody(entity); - schemas.put(canonicalId, body); + String previousBody = schemas.put(canonicalId, body); try { - recordSchemaAlias(schemas, aliasesById, canonicalId, ProxyUtil.BLOB_MAPPER.readTree(body)); + recordSchemaAlias(schemas, aliasesById, canonicalId, previousBody, ProxyUtil.BLOB_MAPPER.readTree(body)); } catch (JsonProcessingException e) { log.warn("Failed to parse schema body for $id alias index: {} ({})", canonicalId, e.getMessage()); } @@ -1008,9 +1009,12 @@ private static void putSchemaInPlace(Map schemas, MapAn update may change this entity's own $id — evict any alias {@code canonicalId} held - * under its previous $id first, so the old $id stops resolving once the body no longer - * carries it. No-op during a full rebuild, where {@code aliasesById} starts empty each pass. + *

An update may change this entity's own $id. {@code previousBody} — the body this same + * canonical id held immediately before this call (the return value of the {@code schemas.put}/ + * {@code remove} the caller just performed, or {@code null} on a fresh create) — is read + * directly for its own $id and evicted by that key, an O(1) targeted removal rather than a + * scan over every alias for one matching this canonical id's value. {@code null} on a fresh + * create, so there is nothing to evict there. * *

Same transient gap as {@link #removeEntityInPlace}'s javadoc describes for a deleted * canonical id, but triggered by an update rather than a delete: the file entry @@ -1022,12 +1026,26 @@ private static void putSchemaInPlace(Map schemas, Map schemas, Map aliasesById, - String canonicalId, JsonNode node) { - aliasesById.values().removeIf(canonicalId::equals); + String canonicalId, String previousBody, JsonNode node) { JsonNode idNode = node.get("$id"); - if (idNode != null && idNode.isTextual()) { - schemas.remove(idNode.asText()); - aliasesById.put(idNode.asText(), canonicalId); + String newId = idNode != null && idNode.isTextual() ? idNode.asText() : null; + String oldId = previousBody == null ? null : extractSchemaId(previousBody); + if (oldId != null && !oldId.equals(newId)) { + aliasesById.remove(oldId); + } + if (newId != null) { + schemas.remove(newId); + aliasesById.put(newId, canonicalId); + } + } + + @Nullable + private static String extractSchemaId(String body) { + try { + JsonNode idNode = ProxyUtil.BLOB_MAPPER.readTree(body).get("$id"); + return idNode != null && idNode.isTextual() ? idNode.asText() : null; + } catch (JsonProcessingException e) { + return null; } } @@ -1053,12 +1071,18 @@ private static void removeEntityInPlace(Config config, ResourceTypes type, Strin case PROJECT_KEY -> config.getKeys().remove(canonicalId); case ROUTE -> config.getRoutes().remove(canonicalId); case APP_TYPE_SCHEMA -> { - config.getApplicationTypeSchemas().remove(canonicalId); - config.getSchemaAliasesById().values().removeIf(canonicalId::equals); + String removedBody = config.getApplicationTypeSchemas().remove(canonicalId); + String id = removedBody == null ? null : extractSchemaId(removedBody); + if (id != null) { + config.getSchemaAliasesById().remove(id); + } } case CATALOG_SCHEMA -> { - config.getCatalogSchemas().remove(canonicalId); - config.getCatalogSchemaAliasesById().values().removeIf(canonicalId::equals); + String removedBody = config.getCatalogSchemas().remove(canonicalId); + String id = removedBody == null ? null : extractSchemaId(removedBody); + if (id != null) { + config.getCatalogSchemaAliasesById().remove(id); + } } case APPLICATION -> config.getApplications().remove(canonicalId); case TOOL_SET -> config.getToolsets().remove(canonicalId); @@ -1375,14 +1399,16 @@ private static Object addBlobEntity(Config config, ResourceTypes type, String ca } case APP_TYPE_SCHEMA -> { Map schemas = config.getApplicationTypeSchemas(); - warnIfReplaced(type, canonicalId, schemas.put(canonicalId, node.toString())); - recordSchemaAlias(schemas, config.getSchemaAliasesById(), canonicalId, node); + String previousBody = schemas.put(canonicalId, node.toString()); + warnIfReplaced(type, canonicalId, previousBody); + recordSchemaAlias(schemas, config.getSchemaAliasesById(), canonicalId, previousBody, node); return null; } case CATALOG_SCHEMA -> { Map catalogSchemas = config.getCatalogSchemas(); - warnIfReplaced(type, canonicalId, catalogSchemas.put(canonicalId, node.toString())); - recordSchemaAlias(catalogSchemas, config.getCatalogSchemaAliasesById(), canonicalId, node); + String previousBody = catalogSchemas.put(canonicalId, node.toString()); + warnIfReplaced(type, canonicalId, previousBody); + recordSchemaAlias(catalogSchemas, config.getCatalogSchemaAliasesById(), canonicalId, previousBody, node); return null; } case APPLICATION -> { diff --git a/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java b/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java index a142a7f6b..c0fc1a618 100644 --- a/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java +++ b/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java @@ -637,9 +637,9 @@ static void mutateScratch(Config scratch, AdminManifest entry) { } catch (JsonProcessingException e) { return; } - scratch.getApplicationTypeSchemas().put(entry.name(), json); + String previousJson = scratch.getApplicationTypeSchemas().put(entry.name(), json); MergedConfigStore.recordSchemaAlias(scratch.getApplicationTypeSchemas(), - scratch.getSchemaAliasesById(), entry.name(), entry.spec()); + scratch.getSchemaAliasesById(), entry.name(), previousJson, entry.spec()); } case "CatalogSchema" -> { String json; @@ -648,9 +648,9 @@ static void mutateScratch(Config scratch, AdminManifest entry) { } catch (JsonProcessingException e) { return; } - scratch.getCatalogSchemas().put(entry.name(), json); + String previousJson = scratch.getCatalogSchemas().put(entry.name(), json); MergedConfigStore.recordSchemaAlias(scratch.getCatalogSchemas(), - scratch.getCatalogSchemaAliasesById(), entry.name(), entry.spec()); + scratch.getCatalogSchemaAliasesById(), entry.name(), previousJson, entry.spec()); } default -> { /* unknown kinds never reach this code path */ } } From 6ac1a273eeacad4a82d77d49429805f543eda3bd Mon Sep 17 00:00:00 2001 From: Kiryl_Kurnosenka Date: Wed, 12 Aug 2026 17:26:46 +0300 Subject: [PATCH 07/15] fix: address PR #1813 review feedback on schema alias naming and javadoc Rename schemaAliasesById to applicationSchemaAliasesById to disambiguate from catalogSchemaAliasesById, clarify its key/value semantics with an example, document nullability on schema accessors, and trim the recordSchemaAlias/removeEntityInPlace javadocs to their essential contract. Co-Authored-By: Claude Sonnet 5 --- .../com/epam/aidial/core/config/Config.java | 24 +++++++-- .../epam/aidial/core/config/ConfigTest.java | 2 +- .../core/server/config/MergedConfigStore.java | 54 +++++-------------- .../controller/AdminApplyController.java | 4 +- .../controller/ConfigResourceController.java | 2 +- 5 files changed, 36 insertions(+), 50 deletions(-) diff --git a/config/src/main/java/com/epam/aidial/core/config/Config.java b/config/src/main/java/com/epam/aidial/core/config/Config.java index 3917e0a5a..163c2e366 100644 --- a/config/src/main/java/com/epam/aidial/core/config/Config.java +++ b/config/src/main/java/com/epam/aidial/core/config/Config.java @@ -57,12 +57,18 @@ public class Config { private List globalInterceptors = List.of(); /** - * $id → canonical-id index for {@code platform}-bucket schema entities, built at rebuild - * time from blob bodies. Bridges $id-keyed file entries and canonical-id-keyed blob entries - * in {@link #applicationTypeSchemas}, since a schema's $id is not derivable from its path. + * $id → canonical-id index for {@link #applicationTypeSchemas}, built at rebuild time from + * blob bodies: each key is a schema's own {@code $id} (as declared in its body), each value + * is the canonical id of the blob entry storing that schema. Bridges $id-keyed file entries + * and canonical-id-keyed blob entries, since a schema's $id is not derivable from its path. + * + *

For example, given a blob entry stored under canonical id + * {@code schemas/platform/my-schema} whose body declares + * {@code "$id": "https://example.com/schemas/my-schema.json"}, this map holds + * {@code "https://example.com/schemas/my-schema.json" → "schemas/platform/my-schema"}. */ @JsonIgnore - private Map schemaAliasesById = Map.of(); + private Map applicationSchemaAliasesById = Map.of(); @JsonIgnore private Map catalogSchemaAliasesById = Map.of(); @@ -106,11 +112,17 @@ public Interceptor getInterceptor(String id) { return resolve(interceptors, "interceptors", id); } + /** + * @return the schema body, or {@code null} if {@code schemaId} is null or unresolved + */ @JsonIgnore public String getCustomApplicationSchema(URI schemaId) { - return resolveSchema(applicationTypeSchemas, schemaAliasesById, schemaId); + return resolveSchema(applicationTypeSchemas, applicationSchemaAliasesById, schemaId); } + /** + * @return the schema body, or {@code null} if {@code schemaId} is null or unresolved + */ @JsonIgnore public String getCatalogSchema(URI schemaId) { return resolveSchema(catalogSchemas, catalogSchemaAliasesById, schemaId); @@ -121,6 +133,8 @@ public String getCatalogSchema(URI schemaId) { * already keyed by $id), then falls back through the $id → canonical-id alias index for a * migrated blob entry. A schema's $id is not derivable from its path, so unlike {@link * #resolve}, the alias index must be maintained explicitly (see {@code MergedConfigStore}). + * + * @return the schema body, or {@code null} if {@code schemaId} is null or unresolved */ private static String resolveSchema(Map schemas, Map aliasesById, URI schemaId) { if (schemaId == null) { diff --git a/config/src/test/java/com/epam/aidial/core/config/ConfigTest.java b/config/src/test/java/com/epam/aidial/core/config/ConfigTest.java index a1fc3d2cf..761a13992 100644 --- a/config/src/test/java/com/epam/aidial/core/config/ConfigTest.java +++ b/config/src/test/java/com/epam/aidial/core/config/ConfigTest.java @@ -81,7 +81,7 @@ public void testGetCustomApplicationSchemaFallsBackThroughAliasIndex() { String schemaId = "https://mydial.epam.com/custom_application_schemas/specific_application_type"; String body = "{\"$id\":\"" + schemaId + "\"}"; config.setApplicationTypeSchemas(Map.of(canonicalId, body)); - config.setSchemaAliasesById(Map.of(schemaId, canonicalId)); + config.setApplicationSchemaAliasesById(Map.of(schemaId, canonicalId)); assertEquals(body, config.getCustomApplicationSchema(URI.create(schemaId)), "$id lookup via alias index"); assertEquals(body, config.getCustomApplicationSchema(URI.create(canonicalId)), "verbatim canonical-id lookup"); diff --git a/server/src/main/java/com/epam/aidial/core/server/config/MergedConfigStore.java b/server/src/main/java/com/epam/aidial/core/server/config/MergedConfigStore.java index c51bc57f6..b2dabfae2 100644 --- a/server/src/main/java/com/epam/aidial/core/server/config/MergedConfigStore.java +++ b/server/src/main/java/com/epam/aidial/core/server/config/MergedConfigStore.java @@ -887,7 +887,7 @@ private static Config shallowClone(Config base) { next.setRoutes(base.getRoutes()); next.setApplicationTypeSchemas(base.getApplicationTypeSchemas()); next.setCatalogSchemas(base.getCatalogSchemas()); - next.setSchemaAliasesById(base.getSchemaAliasesById()); + next.setApplicationSchemaAliasesById(base.getApplicationSchemaAliasesById()); next.setCatalogSchemaAliasesById(base.getCatalogSchemaAliasesById()); next.setApplications(base.getApplications()); next.setToolsets(base.getToolsets()); @@ -933,7 +933,7 @@ private static void cloneTypeMap(Config config, ResourceTypes type) { case ROUTE -> config.setRoutes(new LinkedHashMap<>(config.getRoutes())); case APP_TYPE_SCHEMA -> { config.setApplicationTypeSchemas(new LinkedHashMap<>(config.getApplicationTypeSchemas())); - config.setSchemaAliasesById(new HashMap<>(config.getSchemaAliasesById())); + config.setApplicationSchemaAliasesById(new HashMap<>(config.getApplicationSchemaAliasesById())); } case CATALOG_SCHEMA -> { config.setCatalogSchemas(new LinkedHashMap<>(config.getCatalogSchemas())); @@ -968,7 +968,7 @@ private static void putEntityInPlace(Config config, ResourceTypes type, String c case PROJECT_KEY -> config.getKeys().put(canonicalId, (Key) entity); case ROUTE -> config.getRoutes().put(canonicalId, (Route) entity); case APP_TYPE_SCHEMA -> - putSchemaInPlace(config.getApplicationTypeSchemas(), config.getSchemaAliasesById(), canonicalId, entity); + putSchemaInPlace(config.getApplicationTypeSchemas(), config.getApplicationSchemaAliasesById(), canonicalId, entity); case CATALOG_SCHEMA -> putSchemaInPlace(config.getCatalogSchemas(), config.getCatalogSchemaAliasesById(), canonicalId, entity); case APPLICATION -> putNameAddressed(config.getApplications(), canonicalId, (Application) entity); @@ -999,31 +999,11 @@ private static void putSchemaInPlace(Map schemas, MapAn update may change this entity's own $id. {@code previousBody} — the body this same - * canonical id held immediately before this call (the return value of the {@code schemas.put}/ - * {@code remove} the caller just performed, or {@code null} on a fresh create) — is read - * directly for its own $id and evicted by that key, an O(1) targeted removal rather than a - * scan over every alias for one matching this canonical id's value. {@code null} on a fresh - * create, so there is nothing to evict there. - * - *

Same transient gap as {@link #removeEntityInPlace}'s javadoc describes for a deleted - * canonical id, but triggered by an update rather than a delete: the file entry - * shadowed under the old $id when this canonical id was first migrated is removed from - * {@code schemas} at that point and is never restored here, even though this canonical id no - * longer claims that $id after the change — the old $id stays unreachable via either the file - * entry or this canonical id until the next full {@link #rebuild()}. Changing a schema's $id - * away from a value is, from the shadowed file entry's perspective, indistinguishable from - * deleting the entity that was shadowing it. + *

If this canonical id previously held a different $id ({@code previousBody}), that stale + * alias is evicted first — otherwise it would keep pointing here after the $id changed. */ public static void recordSchemaAlias(Map schemas, Map aliasesById, String canonicalId, String previousBody, JsonNode node) { @@ -1052,16 +1032,8 @@ private static String extractSchemaId(String body) { /** * Removes only the canonical-id entry. Deliberately does not restore the file-defined * entry that {@link #putNameAddressed}/{@link #shadowFileEntry} shadowed when the blob entity - * was created — that resurrection happens naturally on the next full {@link #rebuild()}, which - * always starts from a fresh copy of the file-derived {@link Config} and only shadows short - * names that still have a live blob. This is an accepted, transient gap: a short name deleted - * via this partial-update path stays unreachable until the next periodic rebuild, bounded by - * {@link FileConfigStore}'s poll interval. Do not add restoration logic here — that would make - * delete-then-rebuild diverge instead of converge. - * - *

For {@code APP_TYPE_SCHEMA}/{@code CATALOG_SCHEMA} the identical gap also arises without - * a delete at all — see {@link #recordSchemaAlias}'s javadoc for why an in-place $id change - * has the same effect on the old $id's shadowed file entry. + * was created — an accepted, transient gap: the short name stays unreachable until the next + * full {@link #rebuild()} restores it. Do not add restoration logic here. */ private static void removeEntityInPlace(Config config, ResourceTypes type, String canonicalId) { switch (type) { @@ -1074,7 +1046,7 @@ private static void removeEntityInPlace(Config config, ResourceTypes type, Strin String removedBody = config.getApplicationTypeSchemas().remove(canonicalId); String id = removedBody == null ? null : extractSchemaId(removedBody); if (id != null) { - config.getSchemaAliasesById().remove(id); + config.getApplicationSchemaAliasesById().remove(id); } } case CATALOG_SCHEMA -> { @@ -1131,7 +1103,7 @@ private Config rebuild() { Map toolsets = new LinkedHashMap<>(base.getToolsets()); // $id -> canonicalId index, built fresh each rebuild from the blob scan below; file // entries need no alias since they're already keyed by $id. - Map schemaAliasesById = new HashMap<>(); + Map applicationSchemaAliasesById = new HashMap<>(); Map catalogSchemaAliasesById = new HashMap<>(); merged.setRetriableErrorCodes(base.getRetriableErrorCodes()); merged.setGlobalInterceptors(base.getGlobalInterceptors()); @@ -1148,7 +1120,7 @@ private Config rebuild() { merged.setCatalogSchemas(catalogSchemas); merged.setApplications(applications); merged.setToolsets(toolsets); - merged.setSchemaAliasesById(schemaAliasesById); + merged.setApplicationSchemaAliasesById(applicationSchemaAliasesById); merged.setCatalogSchemaAliasesById(catalogSchemaAliasesById); Map blobBodies = new HashMap<>(); @@ -1401,7 +1373,7 @@ private static Object addBlobEntity(Config config, ResourceTypes type, String ca Map schemas = config.getApplicationTypeSchemas(); String previousBody = schemas.put(canonicalId, node.toString()); warnIfReplaced(type, canonicalId, previousBody); - recordSchemaAlias(schemas, config.getSchemaAliasesById(), canonicalId, previousBody, node); + recordSchemaAlias(schemas, config.getApplicationSchemaAliasesById(), canonicalId, previousBody, node); return null; } case CATALOG_SCHEMA -> { diff --git a/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java b/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java index c0fc1a618..fb6fe95bd 100644 --- a/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java +++ b/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java @@ -272,7 +272,7 @@ static Config newScratch(MergedConfigStore mergedConfigStore) { scratch.setInterceptors(new HashMap<>(live.getInterceptors())); scratch.setApplicationTypeSchemas(new HashMap<>(live.getApplicationTypeSchemas())); scratch.setCatalogSchemas(new HashMap<>(live.getCatalogSchemas())); - scratch.setSchemaAliasesById(new HashMap<>(live.getSchemaAliasesById())); + scratch.setApplicationSchemaAliasesById(new HashMap<>(live.getApplicationSchemaAliasesById())); scratch.setCatalogSchemaAliasesById(new HashMap<>(live.getCatalogSchemaAliasesById())); scratch.setApplications(new HashMap<>(live.getApplications())); scratch.setToolsets(new HashMap<>(live.getToolsets())); @@ -639,7 +639,7 @@ static void mutateScratch(Config scratch, AdminManifest entry) { } String previousJson = scratch.getApplicationTypeSchemas().put(entry.name(), json); MergedConfigStore.recordSchemaAlias(scratch.getApplicationTypeSchemas(), - scratch.getSchemaAliasesById(), entry.name(), previousJson, entry.spec()); + scratch.getApplicationSchemaAliasesById(), entry.name(), previousJson, entry.spec()); } case "CatalogSchema" -> { String json; diff --git a/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java b/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java index 038bc0178..496a8f5f2 100644 --- a/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java +++ b/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java @@ -1591,7 +1591,7 @@ static void rejectSchemaIdCollision(Config snapshot, ResourceTypes type, String } String id = idNode.asText(); Map aliasesById = switch (type) { - case APP_TYPE_SCHEMA -> snapshot.getSchemaAliasesById(); + case APP_TYPE_SCHEMA -> snapshot.getApplicationSchemaAliasesById(); case CATALOG_SCHEMA -> snapshot.getCatalogSchemaAliasesById(); default -> throw new IllegalArgumentException("Unsupported type for schema $id check: " + type); }; From 55cd21855d9aa47a679992ec5382ba7c4382055d Mon Sep 17 00:00:00 2001 From: Kiryl_Kurnosenka Date: Thu, 13 Aug 2026 19:52:53 +0300 Subject: [PATCH 08/15] feat: key Config maps by short name for models/apps/toolsets/interceptors/roles Reverts the canonical-id-keyed + derivation/shadow mechanism from #1813 for these five types back to short-name-keyed maps (file and blob entries share one key, no resolve()/lastSegment/shadow step needed), and extends it further: admin GET now reads/decrypts blob storage directly by descriptor instead of the in-memory Config map, since that map can no longer distinguish a blob-managed entity from a file-only one sharing the same short name. - Config.java: selectDeployment/getModel/getRole/getInterceptor are plain Map.get(id); removed resolve() and accessor methods. - ConfigPostProcessor.java: entity.setName(mapKey) directly, no lastSegment. - MergedConfigStore.java: blob entries for these five types are inserted keyed by short name (mapKeyFor helper); removed shadowFileEntry. Schema handling (alias-index-based) is unchanged, deferred to a follow-up. - ConfigResourceController.java: new handleSingleGetFromBlob reads/decrypts storage directly for these five types; PROJECT_KEY/ROUTE keep the existing map-based handleSingleGet (no ambiguity there). - AdminApplyController.java: EntityChange construction uses mapKeyFor instead of always building the canonical id. - Call sites (RateLimiter, ShareService, ConsentService, AnalyticsLogContext, BlobEntityValidator, and related controllers) reverted to raw map access now that Config no longer needs derivation accessors. Co-Authored-By: Claude Sonnet 5 --- .../com/epam/aidial/core/config/Config.java | 42 +-- .../epam/aidial/core/config/ConfigTest.java | 76 ----- docs/design/github-updates-draft.md | 133 +++++++++ docs/design/schema-id-atomic-derivation.md | 274 ++++++++++++++++++ docs/design/short-name-keyed-config-maps.md | 178 ++++++++++++ .../server/config/BlobEntityValidator.java | 4 +- .../server/config/ConfigPostProcessor.java | 69 ++--- .../core/server/config/MergedConfigStore.java | 149 +++++----- .../controller/AdminApplyController.java | 115 ++------ .../controller/BaseInterceptorController.java | 2 +- .../controller/ConfigResourceController.java | 128 +++++--- .../controller/DeploymentController.java | 2 +- .../server/controller/ModelController.java | 2 +- .../server/controller/ResourceController.java | 2 +- .../CollectResponseAttachmentsFn.java | 2 +- .../core/server/limiter/RateLimiter.java | 41 +-- .../core/server/log/AnalyticsLogContext.java | 8 +- .../core/server/service/ConsentService.java | 28 +- .../core/server/service/ShareService.java | 5 +- .../server/util/PlatformCanonicalIdUtil.java | 4 +- .../core/server/CanonicalIdListingTest.java | 19 +- .../core/server/MergedConfigStoreApiTest.java | 98 +++---- .../config/ConfigPostProcessorTest.java | 3 +- .../MergedConfigStorePartialUpdateTest.java | 36 ++- .../MergedConfigStoreReplicaUpdateTest.java | 2 +- .../server/config/MergedConfigStoreTest.java | 4 +- 26 files changed, 890 insertions(+), 536 deletions(-) create mode 100644 docs/design/github-updates-draft.md create mode 100644 docs/design/schema-id-atomic-derivation.md create mode 100644 docs/design/short-name-keyed-config-maps.md diff --git a/config/src/main/java/com/epam/aidial/core/config/Config.java b/config/src/main/java/com/epam/aidial/core/config/Config.java index 163c2e366..cd4110c99 100644 --- a/config/src/main/java/com/epam/aidial/core/config/Config.java +++ b/config/src/main/java/com/epam/aidial/core/config/Config.java @@ -75,43 +75,28 @@ public class Config { @JsonIgnore public Deployment selectDeployment(String deploymentId) { - Application application = resolve(applications, "applications", deploymentId); + Application application = applications.get(deploymentId); if (application != null) { return application; } - Model model = resolve(models, "models", deploymentId); + Model model = models.get(deploymentId); if (model != null) { return model; } - ToolSet toolSet = resolve(toolsets, "toolsets", deploymentId); + ToolSet toolSet = toolsets.get(deploymentId); if (toolSet != null) { return toolSet; } - return resolve(interceptors, "interceptors", deploymentId); + return interceptors.get(deploymentId); } public boolean isDeploymentExists(String deploymentId) { return selectDeployment(deploymentId) != null; } - @JsonIgnore - public Model getModel(String id) { - return resolve(models, "models", id); - } - - @JsonIgnore - public Role getRole(String id) { - return resolve(roles, "roles", id); - } - - @JsonIgnore - public Interceptor getInterceptor(String id) { - return resolve(interceptors, "interceptors", id); - } - /** * @return the schema body, or {@code null} if {@code schemaId} is null or unresolved */ @@ -131,8 +116,8 @@ public String getCatalogSchema(URI schemaId) { /** * Resolves a schema by its $id: verbatim lookup first (canonical-id callers, and file entries * already keyed by $id), then falls back through the $id → canonical-id alias index for a - * migrated blob entry. A schema's $id is not derivable from its path, so unlike {@link - * #resolve}, the alias index must be maintained explicitly (see {@code MergedConfigStore}). + * migrated blob entry. A schema's $id is not derivable from its path, so the alias index must + * be maintained explicitly (see {@code MergedConfigStore}). * * @return the schema body, or {@code null} if {@code schemaId} is null or unresolved */ @@ -148,19 +133,4 @@ private static String resolveSchema(Map schemas, Map V resolve(Map entities, String typeSegment, String id) { - V direct = entities.get(id); - if (direct != null) { - return direct; - } - return entities.get(typeSegment + "/platform/" + id); - } } diff --git a/config/src/test/java/com/epam/aidial/core/config/ConfigTest.java b/config/src/test/java/com/epam/aidial/core/config/ConfigTest.java index 761a13992..464585405 100644 --- a/config/src/test/java/com/epam/aidial/core/config/ConfigTest.java +++ b/config/src/test/java/com/epam/aidial/core/config/ConfigTest.java @@ -2,12 +2,10 @@ import org.junit.jupiter.api.Test; -import java.net.URI; import java.util.Map; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertSame; public class ConfigTest { @@ -29,78 +27,4 @@ public void testSelectDeployment() { assertEquals(interceptor, config.selectDeployment("interceptor")); assertNull(config.selectDeployment("unknown")); } - - @Test - public void testSelectDeploymentResolvesShortNameAgainstCanonicalId() { - Config config = new Config(); - Model model = new Model(); - config.setModels(Map.of("models/platform/gpt-4", model)); - - assertSame(model, config.selectDeployment("models/platform/gpt-4"), "verbatim (canonical) hit"); - assertSame(model, config.selectDeployment("gpt-4"), "derived (short-name) hit"); - assertNull(config.selectDeployment("unknown")); - } - - @Test - public void testGetModelResolvesVerbatimAndDerived() { - Config config = new Config(); - Model model = new Model(); - config.setModels(Map.of("models/platform/gpt-4", model)); - - assertSame(model, config.getModel("models/platform/gpt-4")); - assertSame(model, config.getModel("gpt-4")); - assertNull(config.getModel("unknown")); - } - - @Test - public void testGetRoleResolvesVerbatimAndDerived() { - Config config = new Config(); - Role role = new Role(); - config.setRoles(Map.of("roles/platform/admin", role)); - - assertSame(role, config.getRole("roles/platform/admin")); - assertSame(role, config.getRole("admin")); - assertNull(config.getRole("unknown")); - } - - @Test - public void testGetInterceptorResolvesVerbatimAndDerived() { - Config config = new Config(); - Interceptor interceptor = new Interceptor(); - config.setInterceptors(Map.of("interceptors/platform/my-interceptor", interceptor)); - - assertSame(interceptor, config.getInterceptor("interceptors/platform/my-interceptor")); - assertSame(interceptor, config.getInterceptor("my-interceptor")); - assertNull(config.getInterceptor("unknown")); - } - - @Test - public void testGetCustomApplicationSchemaFallsBackThroughAliasIndex() { - Config config = new Config(); - String canonicalId = "schemas/platform/my-schema"; - String schemaId = "https://mydial.epam.com/custom_application_schemas/specific_application_type"; - String body = "{\"$id\":\"" + schemaId + "\"}"; - config.setApplicationTypeSchemas(Map.of(canonicalId, body)); - config.setApplicationSchemaAliasesById(Map.of(schemaId, canonicalId)); - - assertEquals(body, config.getCustomApplicationSchema(URI.create(schemaId)), "$id lookup via alias index"); - assertEquals(body, config.getCustomApplicationSchema(URI.create(canonicalId)), "verbatim canonical-id lookup"); - assertNull(config.getCustomApplicationSchema(URI.create("https://mydial.epam.com/custom_application_schemas/unknown"))); - assertNull(config.getCustomApplicationSchema(null)); - } - - @Test - public void testGetCatalogSchemaFallsBackThroughAliasIndex() { - Config config = new Config(); - String canonicalId = "catalog_schemas/platform/my-schema"; - String schemaId = "https://dial.epam.com/catalog-schemas/model"; - String body = "{\"$id\":\"" + schemaId + "\"}"; - config.setCatalogSchemas(Map.of(canonicalId, body)); - config.setCatalogSchemaAliasesById(Map.of(schemaId, canonicalId)); - - assertEquals(body, config.getCatalogSchema(URI.create(schemaId)), "$id lookup via alias index"); - assertEquals(body, config.getCatalogSchema(URI.create(canonicalId)), "verbatim canonical-id lookup"); - assertNull(config.getCatalogSchema(URI.create("https://dial.epam.com/catalog-schemas/unknown"))); - assertNull(config.getCatalogSchema(null)); - } } diff --git a/docs/design/github-updates-draft.md b/docs/design/github-updates-draft.md new file mode 100644 index 000000000..8a07087fd --- /dev/null +++ b/docs/design/github-updates-draft.md @@ -0,0 +1,133 @@ +# Draft GitHub Updates — Review Before Pushing + +Drafted text for #1781, #1783, #1784, and PR #1813, reflecting `schema-id-atomic-derivation.md` +and `short-name-keyed-config-maps.md`. **Not pushed to GitHub yet** — for your review. The PR #1813 +draft assumes the code has actually been rewritten to match; don't paste it until that's true, or +it'll describe a PR that doesn't match its own diff. + +--- + +## Issue #1781 (epic) — suggested edits + +**Requirement table** — add a row/footnote under the existing table: + +> **Update:** the "canonical id keeps working as a harmless superset" property in the Inbound row +> no longer holds for models, applications, toolsets, interceptors, roles, or schemas. Once +> `Config`'s maps are keyed by each entity's natural identity (short name, or `$id` for schemas — +> see #1813's follow-up), a caller addressing an entity by its full canonical id +> (`models/platform/gpt-4`) instead of its short name (`gpt-4`) no longer resolves. This is an +> accepted, deliberate regression, not a residual to design around. + +**"Rejected alternatives" section** — replace the "Short-name-keyed `Config` maps" entry: + +> - ~~**Short-name-keyed `Config` maps.** Would avoid derivation but create a canonical-vs-short +> impedance mismatch with the deeply canonical CRUD / pub-sub / partial-update machinery. +> Rejected in favor of canonical-keyed maps + derivation.~~ +> **Reopened.** For models/applications/toolsets/interceptors/roles, the canonical-id-to-map-key +> transform is the same `lastSegment` string operation already computed today for outbound +> naming — not new per-type parsing logic — so the mismatch is narrower than originally assessed. +> Schemas need a bespoke version (body-JSON `$id` extraction) because their natural identity isn't +> a path segment; see `schema-id-atomic-derivation.md` for that case and +> `short-name-keyed-config-maps.md` for the generalization to the other five types. + +--- + +## Issue #1783 (Slice B+C) — suggested rewrite + +Replace **Part B** with: + +> ## Part B — short-name resolution (`Config` maps keyed by short name) +> +> `Config`'s maps for `applications`/`models`/`toolsets`/`interceptors`/`roles` are keyed by short +> name uniformly — file-sourced and blob-sourced entries for the same logical entity share the same +> map key. Canonical id (`{type}/platform/{shortName}`) is derived only where it's actually +> needed: the physical blob address, and the admin CRUD URL. It is never a `Config` map key. +> +> ### `config/.../Config.java` +> - `selectDeployment`, `getModel`, `getRole`, `getInterceptor` are plain `Map.get(shortName)` — no +> derivation step, no verbatim/canonical-id fallback. +> +> ### `server/.../config/ConfigPostProcessor.java` +> - `entity.setName(mapKey)` — the map key already is the short name. +> +> ### `server/.../config/MergedConfigStore.java` +> - Blob-sourced entities are inserted keyed by `lastSegment(canonicalId)`, not the raw canonical +> id — the same key a file entry for that entity already uses. Migrating a file entry to blob is +> an ordinary overwrite of that key; there's no separate "shadow the file entry" removal step. +> +> **Accepted regression:** a caller addressing an entity by canonical id instead of short name no +> longer resolves (see #1781's updated Requirement table). + +Replace **Part C** with: + +> ## Part C — schema `$id` resolution +> +> App-type and catalog schemas are keyed by `$id` in `Config`'s maps — both file- and blob-sourced +> entries, uniformly (file entries already work this way via +> `JsonArrayToSchemaMapDeserializer`). Canonical id for schemas is **derived** from `$id` +> (`schemas/platform/encode($id)` / `catalog_schemas/platform/encode($id)`), used only for the +> physical blob address and the admin CRUD URL — never a map key. No side index +> (`schemaAliasesById`/`catalogSchemaAliasesById`) is needed; `$id` collisions become structurally +> impossible (two schemas can't occupy the same derived blob path). See +> `schema-id-atomic-derivation.md` for the full design, including the non-splitting +> `ResourceDescriptorFactory` addition schemas need (their `$id` is a URI and legitimately contains +> `/`, unlike every other type's short name). + +--- + +## Issue #1784 (Slice D) — suggested edit + +Replace the **"Open item — schema migration naming"** section with: + +> ## Schema migration +> +> Resolved by `schema-id-atomic-derivation.md`: a migrated schema's canonical id is derived +> directly from its `$id` (`schemas/platform/encode($id)`), so there's no separate naming decision +> to make during migration — unlike models/applications/etc., where the file entry's key already +> is the short name to migrate to. + +No other change needed — the migration endpoint's behavior for models/applications/toolsets/ +interceptors/roles/schemas is otherwise unaffected (blob write path and address are unchanged; only +`Config`'s in-memory key changes, which Slice D's migration endpoint doesn't touch directly). + +--- + +## PR #1813 — suggested rewrite (only once the code matches this) + +> Makes every materialized `platform`-bucket config entity short-name (or, for schemas, `$id`) +> addressed by keying `Config`'s in-memory maps directly by that natural identity — not by +> canonical id — and deriving canonical id only where it's actually needed (the blob address, the +> admin CRUD URL). +> +> ### Applicable issues +> - fixes #1783 +> +> ### Description of changes +> - `Config.java`: `selectDeployment`/`getModel`/`getRole`/`getInterceptor` are plain +> `Map.get(shortName)`; `getCustomApplicationSchema`/`getCatalogSchema` are plain +> `Map.get($id)`. No derivation helper, no alias index. +> - `ConfigPostProcessor.java`: `entity.setName(mapKey)` directly. +> - `MergedConfigStore.java`: blob-sourced name-addressed entities are inserted keyed by +> `lastSegment(canonicalId)`; blob-sourced schemas are inserted keyed by their body's `$id` +> (extracted via the existing schema-body parse). No shadow-file-entry step, no schema alias +> index — a migrated entity's blob write is an ordinary overwrite of the same key its file +> predecessor used. +> - `ResourceDescriptorFactory.java`: new atomic (non-splitting) descriptor-factory method for +> schemas, whose `$id` is a URI and legitimately contains `/`. +> - Call-site sweep: unchanged from the original PR — deployment/role/interceptor resolution was +> already funneled through `Config`'s accessors. +> - Tests: [update to match whatever actually lands] +> +> ### Checklist +> - [X] Title of the pull request follows [Conventional Commits specification] + +--- + +## Recommended sequencing for pushing these + +1. Implement `schema-id-atomic-derivation.md` + `short-name-keyed-config-maps.md` on this PR's + branch (or a follow-up branch). +2. Once the diff matches, replace PR #1813's description with the draft above. +3. Update #1783 and #1784 to match (they're still open, no urgency conflict). +4. Update #1781 last, since it's the most "public" (epic) summary and should reflect the settled + state, not an in-flight one. diff --git a/docs/design/schema-id-atomic-derivation.md b/docs/design/schema-id-atomic-derivation.md new file mode 100644 index 000000000..1b321c462 --- /dev/null +++ b/docs/design/schema-id-atomic-derivation.md @@ -0,0 +1,274 @@ +# Schema `$id` Resolution: Key `Config` by `$id`, Derive the Canonical Id + +## Status + +Design proposal. Not implemented. Supersedes the alias-index design in `schema-id-derivation.md` +(itself already implemented — see `Config.applicationSchemaAliasesById`/`catalogSchemaAliasesById`, +`MergedConfigStore.putSchemaInPlace`/`recordSchemaAlias`) for `applicationTypeSchemas` and +`catalogSchemas` only. Nothing else in `expose-as-short-name.md` changes. + +This is the option that came out of re-examining Option A ("encode `$id` into the canonical id", +rejected in `schema-id-resolution-design-options.md`) after tracing the actual request flow. It +combines Option A's derivation (`canonicalId = "{type}/platform/" + encode($id)`) with Option D2's +idea of keying the in-memory map by `$id` — but avoids D2's O(n) canonical-id cost, because +derivation lets admin `GET`/`PUT`/`DELETE` decode the URL segment straight back to `$id` and do one +map lookup, instead of scanning. In the original doc's taxonomy this is a fifth point not +enumerated as its own option: **A's derivation + D2's keying, without D2's scan.** + +## Why revisit this + +The previously-shipped fix (`schema-id-derivation.md`) is real and correct, but it's a second data +structure (`*AliasesById`) with an eviction discipline someone has to remember (`putSchemaInPlace` +using `Map.put`'s return value) and three independent write-time collision checks +(`ConfigResourceController.handlePut`, `AdminApplyController.applySchema`, `.validateOnly`) that all +have to agree. This proposal removes the index and the collision checks entirely by making +collisions structurally impossible — the same free lunch models/apps/toolsets/interceptors get from +D7 (two entities can't share a short name because they can't occupy the same blob path). + +## What changes + +### S1 — `Config`'s schema maps are keyed by `$id`, uniformly + +Today `applicationTypeSchemas`/`catalogSchemas` mix keying: file entries are keyed by `$id` +(`JsonArrayToSchemaMapDeserializer` already does this — confirmed, no change needed there), blob +entries are keyed by **canonical id**, and a separate `*AliasesById` index bridges the two for `$id` +lookups. + +New shape: **both sources key by `$id`, always.** Canonical id stops being a map key for these two +types; it survives only as the admin-facing address (URL / blob path). + +```java +private static String resolveSchema(Map schemas, URI schemaId) { + return schemaId == null ? null : schemas.get(schemaId.toString()); +} +``` + +`schemaAliasesById`/`catalogSchemaAliasesById` fields, `recordSchemaAlias`, `putSchemaInPlace`'s +eviction logic, and `rejectSchemaIdCollision` (`ConfigResourceController.java:1584-1601`) are all +deleted — there's no longer an index to maintain or a collision to detect at write time (see S3). + +### S2 — Canonical id is derived, not admin-chosen + +``` +canonicalId = "schemas/platform/" + encode($id) // app-type schemas +canonicalId = "catalog_schemas/platform/" + encode($id) // catalog schemas +``` + +using the existing `UrlUtil.encodePathSegment` (`storage/.../util/UrlUtil.java:28-33`) — no new +encoder, and it never needs to be reachable from `config` (see S6). This directly resolves the two +open residuals in `expose-as-short-name.md`: schema migration no longer needs a naming decision (the +name **is** `encode($id)`), and `$id` uniqueness becomes storage-structural (D7-style) instead of +"admin error, matches today's file behavior." + +### S3 — Write-time reconciliation mirrors the `name` precedent + +Just as a model's PUT body disagrees with the URL and the stored `name` is silently overridden to +match the URL, a schema PUT whose body's `$id` disagrees with what the URL segment decodes to gets +its stored `$id` silently overwritten to `decode(urlSegment)` before persisting. This is a policy +choice, not a technical requirement — the docs flagged this as "genuinely hard" because `$id` is +`$ref`-able by other tooling, more consequential than a cosmetic `name` — but it's the same choice +already made for `name`, applied consistently, and it's what makes the map key (`$id`) and the +canonical id (`encode($id)`) provably agree after every write, with no separate check needed. + +### S4 — A new, non-splitting `ResourceDescriptorFactory` method for schemas only + +Confirmed from `ControllerSelector.configResourceController` (`:626-641`): the `{path}` URL segment +is decoded (`UrlUtil.decodePath`) **before** it reaches `ConfigResourceController`, matching +`fromDecoded`'s "url decoded relative path" contract. So whatever encoding scheme the canonical id +uses on the wire, `fromDecoded` receives the **decoded** `$id` — literal `/`s and all. `fromDecoded` +(`ResourceDescriptorFactory.java:52-59`) does `path.split("/")`, so a schema `$id` like +`https://dial.epam.com/catalog-schemas/model` would be chopped into five bogus path elements +(including an empty one from `//`) instead of treated as one atomic resource name. This is not +avoidable by choosing a better encoder — the decode happens generically upstream, for every +config-resource route, before schema-specific code ever runs. + +New method, used only by `descriptorFor(APP_TYPE_SCHEMA)`/`descriptorFor(CATALOG_SCHEMA)` +(`ConfigResourceController.java:1146-1148`): + +```java +/** + * Like {@link #fromDecoded}, but treats {@code decodedName} as a single atomic resource name — + * never splits it on '/'. For identifiers (like a JSON-Schema {@code $id}) that are themselves + * URIs and therefore expected to contain '/', not folder-hierarchy separators. + */ +public static ResourceDescriptor fromDecodedAtomicName(ResourceType type, String bucketName, + String bucketLocation, String decodedName) { + verify(bucketLocation.endsWith(PATH_SEPARATOR), "Bucket location must end with /"); + String physicalName = UrlUtil.encodePathSegment(decodedName); // keep the physical blob key flat — see S5 + ResourceDescriptor resource = from(type, bucketName, bucketLocation, List.of(physicalName), false); + verify(resource.getAbsoluteFilePath().getBytes(StandardCharsets.UTF_8).length <= MAX_PATH_SIZE, + "Resource path exceeds max allowed size: " + MAX_PATH_SIZE); + return resource; +} +``` + +(Exact placement/signature TBD during implementation — shown here to fix the shape of the fix: no +`split`, explicit length check reusing `fromEncoded`'s existing `MAX_PATH_SIZE = 900`.) + +### S5 — Double-encoding is accepted, not fixed, for `getUrl()` + +`ResourceDescriptor.name` is used two ways with nothing reconciling them: `getAbsoluteFilePath()` +(`:107+`) embeds it raw for the physical blob key; `getUrl()` (`:51-75`) encodes it once, +unconditionally, for the client-facing address. A schema's `$id` forces a choice between the two +being safe: + +- `name` = raw `$id` → clean, single-encoded `getUrl()`, but the **physical blob key** contains + literal `/`, indistinguishable from nesting to most blob backends (folder listing / GC / prefix + scans could misbehave). +- `name` = `encode($id)` (S4's approach) → flat, atomic physical key, but `getUrl()` encodes it a + second time (`%2F` → `%252F`). + +Take the second option — a flat blob key is the load-bearing property; a cosmetically double-encoded +admin URL is not (nothing round-trips DIAL's own emitted URL by hand; clients just echo it back). +Document this as a known, deliberate quirk on `ResourceDescriptor`/wherever schema descriptors are +built. Revisit only if it becomes a real complaint — e.g. by adding an "already encoded, don't +re-encode" flag to `ResourceDescriptor` — but that's out of scope for this pass. + +### S6 — No encoder needed in `config` + +`Config.resolveSchema` (S1) is a bare `Map.get`. `UrlUtil` stays exactly where it is +(`storage`, which `config` cannot depend on — confirmed via `storage/build.gradle`'s +`implementation project(':config')`, dependency runs one way). All encoding happens in `server` +(S2's write-time derivation, S4's descriptor factory), which already depends on `storage`. This is +the thing that makes this design cheaper than Option A as originally scoped — Option A needed the +encoder reachable from both write and read paths; here the read path (`Config`) needs no encoding at +all. + +### S7 — New pattern for schemas, validated structurally + +Don't reuse `ENTITY_NAME_PATTERN` (`^[A-Za-z0-9._%:-]+$`, `ConfigResourceController.java:86`) as-is — +confirmed it rejects some characters a correctly percent-encoded `$id` can legitimately contain +(`+`, `@`, `!`, `$`, `&`, `'`, `(`, `)`, `*`, `,`, `;`, `=` — all left unescaped by Guava's +`urlPathSegmentEscaper`, none in the current allowlist). Rather than widen the regex and hope it +matches the escaper's actual output space, validate structurally at the schema write path: reject +unless `encode(decode(pathSegment)) == pathSegment` — i.e. the segment round-trips as a validly +encoded string. This is stronger than any fixed character class and stays correct even if the +underlying escaper's exact character set changes. + +## Critical open decision: existing schemas already written under the old scheme + +`/v1/schemas/{bucket}/{path}` and `/v1/catalog_schemas/{bucket}/{path}` (`GET`/`PUT`/`DELETE`) are +**already implemented and shippable** today (confirmed: `ConfigResourceController`'s `saveSchema`/ +`getSchema`/`deleteSchema` operations, `descriptorFor`, `handleSchemaGet` all exist and use +`fromDecoded` with an admin-chosen canonical id, unrelated to `$id`). Any schema an admin has already +written lives at a canonical id that generally does **not** decode to its own `$id`. Under this +design, `Config`'s map is keyed by `$id`, and admin `GET`/`PUT`/`DELETE` by canonical id needs +`decode(urlSegment) == $id` to find it — which breaks for every schema written before this change. + +Three ways to handle it (pick one before implementing): + +1. **One-time migration, admin-triggered** (recommended — matches this codebase's existing pattern + for exactly this kind of transition, see `POST /v1/admin/config/file/migrate` in + `expose-as-short-name.md` §6). Add a step that reads every existing schema blob, computes its + correct new path (`schemas/platform/encode($id)`), writes it there, and deletes the old blob. + Simple, bounded, explicit, no dual-mode code to maintain afterward. +2. **Reject the redesign's premise for already-existing schemas** — keep them resolvable only via + their old canonical id (permanently), and only apply `$id`-keying to schemas created after this + ships. Avoids a migration step but means two schema addressing regimes coexist indefinitely, + which is exactly the kind of permanent special-casing this whole redesign line has been trying to + avoid elsewhere. +3. **Dual-mode transition window** — keep a legacy canonical-id-keyed fallback map alongside the new + `$id`-keyed one, drop the fallback after a deprecation period. More moving parts than (1) for a + supposedly-temporary need. + +This document proceeds assuming **(1)**. If schemas haven't actually been used in production yet +(worth confirming — this is a recently-added surface), this whole section may be moot and can be +dropped. + +## Implementation plan + +### 1. `config/.../Config.java` + +- Delete `applicationSchemaAliasesById`/`catalogSchemaAliasesById` fields and their getters/setters. +- Replace `resolveSchema`'s two-step lookup with the single `Map.get` in S1. +- No change to `getCustomApplicationSchema(URI)`/`getCatalogSchema(URI)` signatures. + +### 2. `config/.../databind/JsonArrayToSchemaMapDeserializer.java` + +No change — file schemas are already keyed by `$id` (confirmed). + +### 3. `server/.../util/ResourceDescriptorFactory.java` + +- Add `fromDecodedAtomicName` (S4): no `split("/")`, explicit `MAX_PATH_SIZE` check, encodes the + decoded name once internally before building the descriptor (S5). + +### 4. `server/.../config/MergedConfigStore.java` + +This is the biggest piece of surgery — `APP_TYPE_SCHEMA`/`CATALOG_SCHEMA` stop fitting the generic +"canonical id is both the map key and the blob address" pattern every other managed type uses +(the same invariant break the original design-options doc flagged for "D3"). + +- **`rebuild()`** (`:1100-1124`, `:1373-1383`): when scanning `platform` blobs, key + `schemas`/`catalogSchemas` insertion by `extractSchemaId(body)` instead of the blob's canonical id. + Delete `applicationSchemaAliasesById`/`catalogSchemaAliasesById` construction entirely (S1). +- **`peekEntity`** (switch around `:955-956`): for schema types, look up by `$id` (extracted from + the incoming body being validated) rather than by canonical id. +- **`putEntityInPlace`** (`:971-973`, `:990-1023`): replace `putSchemaInPlace`/`recordSchemaAlias` + with a schema-specific put that (a) computes `newId = extractSchemaId(newBody)`, (b) if an existing + entry's canonical id/blob address maps to a *different* `$id` currently in the map, removes that + old map entry (this is the one place eviction logic survives, but it's a single `remove` keyed by + the *old* `$id` — no index, no scan, since S3's override guarantees canonical id and `$id` agree + going forward), (c) inserts under `newId`. +- **`removeEntityInPlace`** (`:1046-1056`, `:1434-1435`): resolve the `$id` to remove from the + canonical id being deleted (decode it — S2's derivation makes this direct) rather than doing a + raw-map `.remove(canonicalId)`. +- **`deserializeReplicaEntity`, `cloneTypeMap`, `shallowClone`** (`:888-891`, `:935-940`): drop the + alias-map cloning (S1); schema map cloning itself is unchanged in shape (`Map`). + +### 5. `server/.../controller/ConfigResourceController.java` + +- **`descriptorFor`** (`:1134-1153`): route `APP_TYPE_SCHEMA`/`CATALOG_SCHEMA` through + `fromDecodedAtomicName` (S4) instead of `fromDecoded`. +- **`canonicalId()`** (`:1158`) / schema write path: apply S7's round-trip validation instead of + `ENTITY_NAME_PATTERN` for these two types. +- **`handleSchemaGet`** (`:1162-1207`): change `schemas.get(canonicalId())` to decode the canonical + id back to `$id` (S2's derivation, direct decode — no map involved) and look that up. +- **PUT path**: after parsing the body, apply S3's override (`node.set("$id", decode(pathSegment))` + when they disagree) before computing the physical write. Delete `rejectSchemaIdCollision` + (`:1584-1601`) — collisions are now structurally impossible (S2). +- Delete the `rejectSchemaIdCollision` call sites here and in `AdminApplyController`. + +### 6. `server/.../controller/AdminApplyController.java` + +- **`scratch` setup** (`:273-276`): drop `ApplicationSchemaAliasesById`/`CatalogSchemaAliasesById` + cloning (S1). +- **`mutateScratch`** (`:633-653`): replace `scratch.getApplicationTypeSchemas().put(entry.name(), json)` + + `recordSchemaAlias` with the same schema-specific put logic as `MergedConfigStore.putEntityInPlace` + (§4) — key by `$id`, not `entry.name()`. +- **Precheck/real-apply `"Schema"`/`"CatalogSchema"` cases** (`:358-369`, `:432-451`): drop the + `rejectSchemaIdCollision` calls; apply S3's override before persisting. + +### 7. `storage/.../util/UrlUtil.java` + +No change — `encodePathSegment`/`decodePath` already do exactly what S2/S4 need. + +## Testing + +- **`ConfigTest`**: `resolveSchema` — `$id`-keyed hit (file or blob entry), miss; confirm it no + longer needs a canonical-id-shaped input to work at all. +- **`MergedConfigStoreTest`**: rebuild keys blob schemas by `$id`, not canonical id; updating a + schema's own `$id` in place evicts the old key and inserts the new one (single `remove`, no + scan); a schema's canonical id always decodes back to its own `$id` after any write (S3's + invariant); deleting resolves and removes the right `$id` entry. +- **`ResourceDescriptorFactoryTest`**: `fromDecodedAtomicName` — a `$id` containing `/` produces one + atomic resource (no `parentFolders`), a path exceeding `MAX_PATH_SIZE` is rejected, output is used + consistently for both `getAbsoluteFilePath()` (flat) and `getUrl()` (documented double-encoded). +- **`ConfigResourceControllerTest`/integration**: PUT with a body `$id` disagreeing with the URL + segment succeeds and the stored body's `$id` is silently corrected (S3); PUT/GET/DELETE by the + derived canonical id round-trips; two different schemas can no longer collide on `$id` because + they physically cannot share a blob path (replaces the old 409-based `AdminApplyApiTest` + assertions — collisions now fail as an ordinary "resource already exists at a different identity" + case, not a dedicated check). +- **`AdminApplyApiTest`**: batch apply of two `Schema` entries with the same `$id` — confirm the + outcome (still an error, just surfaced differently now that there's no dedicated collision check). +- **Migration test** (if S6's decision (1) is taken): a schema written under the old scheme is + migrated to `schemas/platform/encode($id)`, old blob removed, resolves correctly afterward by both + its `$id` and its new canonical id. + +## Sequencing + +This is independent of Slices A/B/D in `expose-as-short-name.md` (schemas were always their own +Slice C) but is **not** independently shippable if any schemas already exist in `platform` — the +migration step (see "Critical open decision") must land in the same release, or immediately before, +this change goes live; otherwise existing schemas become unreachable by canonical id the moment this +deploys. diff --git a/docs/design/short-name-keyed-config-maps.md b/docs/design/short-name-keyed-config-maps.md new file mode 100644 index 000000000..262fd0e42 --- /dev/null +++ b/docs/design/short-name-keyed-config-maps.md @@ -0,0 +1,178 @@ +# Generalizing `$id`-Style Keying: `Config` Maps Keyed by Short Name, Canonical Id Purely Derived + +## Status + +Design proposal. Not implemented. Extends `schema-id-atomic-derivation.md`'s core idea (key +`Config`'s maps by the entity's natural identity; derive canonical id, don't store it as the map +key) from schemas to the five other short-name-addressed, `platform`-materialized types: **models, +applications, toolsets, interceptors, roles**. `keys`, `routes`, and `settings` are unaffected — +they were never name-addressed (see the "Coverage by entity type" table in +`expose-as-short-name.md`). + +This **reverses** an alternative explicitly rejected in the epic (issue #1781, "Rejected +alternatives"): *"Short-name-keyed `Config` maps. Would avoid derivation but create a +canonical-vs-short impedance mismatch with the deeply canonical CRUD / pub-sub / partial-update +machinery. Rejected in favor of canonical-keyed maps + derivation."* Reopening it is deliberate — +see "Why the earlier rejection doesn't apply the same way" below — not an oversight. + +Supersedes, for these five types: D2 (`Config.resolve`) and D4 (blob-shadows-file) in +`expose-as-short-name.md`; the corresponding implementation already shipped on this branch/in PR +#1813 (`Config.resolve`/`selectDeployment`/`getModel`/`getRole`/`getInterceptor`, +`Config.java:77-159`; `MergedConfigStore.shadowFileEntry`, `:1409-1415`; the `lastSegment(...)` +outbound-naming calls throughout `ConfigPostProcessor.java`). + +## The idea, generalized + +For models/applications/toolsets/interceptors/roles, canonical id is *already* a pure, derivable +function of short name: `canonicalId = "{type}/platform/" + shortName`. Unlike a schema's `$id` +(an opaque URI that needs an atomic-descriptor workaround because it contains `/`), a short name is +already a valid, slash-free path segment — nothing new needs inventing to make it a map key. + +So: key blob entries in `Config`'s maps by **short name directly** — the same key a file-sourced +entry for the same logical entity already uses — instead of by canonical id. Consequences: + +- **`Config.resolve` (`:159`) disappears.** It becomes a plain `map.get(shortName)` — there's no + "verbatim, then derived" two-step to perform, because there's only ever one key shape now. + `selectDeployment`, `getModel`, `getRole`, `getInterceptor` all collapse to direct `Map.get`. +- **`MergedConfigStore.shadowFileEntry` (`:1409-1415`) disappears.** There's nothing to shadow — a + file entry and its migrated blob counterpart share the same map key, so writing the blob entry + during rebuild is an ordinary overwrite of that key, not an additional removal step elsewhere. +- **The `lastSegment(...)` outbound-naming calls throughout `ConfigPostProcessor.java` (currently + ~10 call sites: `:131,154,165,177,188,253,324,378,397,437,461,470`) revert to `entity.setName(mapKey)`.** + The map key already *is* the short name; there's no canonical form to shorten. +- `PlatformCanonicalIdUtil.lastSegment` itself doesn't disappear — it's still needed wherever code + goes the other direction (derives the *storage* canonical id/blob path from a short name for + writes), just not for keying or outbound naming anymore. + +Net: this is a genuine deletion of code #1813 added, not a lateral move — `resolve`'s two-step +lookup, `shadowFileEntry`, and the ten-plus `lastSegment(...)`-for-naming call sites all go away for +these five types, leaving the map key, the blob address derivation (short name → canonical id, for +writes and for admin CRUD), and the outbound name in permanent agreement by construction, the same +way S1-S3 achieve for schemas in `schema-id-atomic-derivation.md`. + +## Why the earlier rejection (#1781) doesn't apply the same way here + +The epic's stated reason — "impedance mismatch with the deeply canonical CRUD / pub-sub / +partial-update machinery" — is real, and it's exactly what makes the schema version of this change +the larger lift in `schema-id-atomic-derivation.md` (`MergedConfigStore.rebuild`/`peekEntity`/ +`putEntityInPlace`/`removeEntityInPlace` all currently assume "canonical id is both the blob address +and the map key," and schemas need bespoke per-call-site logic to break that assumption cleanly, +since deriving a schema's map key from its canonical id requires parsing the JSON body for `$id`). + +For these five types, the mismatch is narrower, because the transform is not type-specific +content-parsing — it's the same string operation (`lastSegment`) `MergedConfigStore` and +`ConfigPostProcessor` already compute today, just applied one layer earlier (as the map key at +write/rebuild time, not only as the outbound `name` after the fact). There's no new per-type parsing +logic to write; every managed-type call site in `MergedConfigStore` that currently does +`map.put(canonicalId, entity)` for these five types does `map.put(lastSegment(canonicalId), entity)` +instead, uniformly. Worth flagging explicitly to reviewers that this reopens a recorded decision, +with this narrower-mismatch argument as the justification — not silently reversing it. + +## Accepted regression: canonical-id-shaped inbound resolution breaks + +Per direction received: **explicitly accepted, not designed around.** Once these five types are +keyed by short name only, any caller — or any stored reference — that addresses a `platform`-bucket +entity by its full canonical id (`models/platform/gpt-4`) instead of its short name (`gpt-4`) stops +resolving. This affects, at minimum: + +- Entities natively created via the `platform`-bucket CRUD API *before* short-name resolution + (#1783/PR #1813) shipped, if anything still holds a reference to them by canonical id. +- The "canonical id keeps working as a harmless superset" property that #1781/#1783 explicitly + designed in (Requirement table, row 1) — this document removes that property for these five types, + matching the same accepted tradeoff already made for schemas (where nothing ever relied on + canonical-id-shaped resolution in the first place). + +No compatibility shim, dual-mode lookup, or migration bridge is planned for this. If a canonical-id +reference needs to keep working somewhere specific, that needs to be raised as an exception before +implementation — not discovered afterward. + +## Implementation plan + +### 1. `config/.../Config.java` + +- Delete `resolve` (`:159`). Change `selectDeployment` (`:77-96`), `getModel` (`:101-104`), + `getRole` (`:106-109`), `getInterceptor` (`:111-113`) to plain `Map.get(id)` on + `applications`/`models`/`toolsets`/`interceptors`/`roles`. +- No change to map field types (`Map` etc.) — only what's used as the key changes, + at the write/rebuild side (§3), not here. + +### 2. `server/.../config/ConfigPostProcessor.java` + +- Revert every `entity.setName(lastSegment(...))` call (the ~10 sites listed above) to + `entity.setName(mapKey)` (or the already-short key directly, depending on the call site's local + variable naming). +- `validateCrossReferences`'s resolve-aware fix (already landed, docblock at `:278`) stays — + irrelevant to this change; it was about lookup semantics, not the map key. +- `deploymentIds`/de-duplication logic at `:437,461,470` that currently reasons about + `lastSegment(name)` vs. raw map key can simplify back to comparing map keys directly (they're now + always short names). + +### 3. `server/.../config/MergedConfigStore.java` + +- Delete `shadowFileEntry` (`:1409-1415`) and its call site (`:1200-1205`) and the surrounding + comments describing the shadow mechanism (`:982-987`, `:1034`, `:1112`, `:1140`). +- Every place that currently inserts a blob-sourced entity into `Config`'s maps keyed by its + canonical id, for these five types, keys by `lastSegment(canonicalId)` instead — this is the one + place `lastSegment` is still needed, moved from "outbound naming after the fact" to "the map key, + from the start." Audit `rebuild()`, `putEntityInPlace`, `removeEntityInPlace`, + `deserializeReplicaEntity`, `peekEntity`, `cloneTypeMap` for every `MODEL`/`INTERCEPTOR`/`ROLE`/ + `APPLICATION`/`TOOL_SET` case that currently uses the raw canonical id as the key. +- `:839`'s `simpleName = fromApi ? lastSegment(mapKey) : mapKey` ternary becomes unconditional + (`mapKey` is already short in both branches) — audit whatever this feeds to confirm the `fromApi` + distinction isn't load-bearing for something else first. +- Admin `GET`/`PUT`/`DELETE`-by-canonical-id (`ConfigResourceController`, unaffected by this plan + directly) still derives the *storage* address the same way as always + (`{type}/platform/{shortName}`); only the in-memory materialized key changes. Confirm nothing in + the admin CRUD path was implicitly relying on `Config`'s map being canonical-id-keyed (it + shouldn't be — `ConfigResourceController` addresses blob storage directly by descriptor, not + through `Config`'s maps, for writes). + +### 4. Tests + +- **`ConfigTest`**: delete/replace `resolve`-specific test cases (verbatim vs. derived hit) with + plain `Map.get` coverage; `selectDeployment`/`getRole`/`getInterceptor`/`getModel` — short-name hit, + miss; **explicitly assert a canonical-id-shaped input now misses** (locks in the accepted + regression rather than leaving it to silently regress further/inconsistently later). +- **`MergedConfigStoreTest`**: rebuild with a file entry and its migrated blob counterpart — one map + entry, same key, blob's content wins (whichever insertion-order rule is chosen); delete + `shadowFileEntry`-specific test cases. +- **`ConfigPostProcessorTest`**: `name = mapKey` directly, no `lastSegment`; existing + `validateCrossReferences` coverage unaffected. +- **`CanonicalIdListingTest`**: rename/re-scope — it currently covers exactly the + canonical-id-emits-as-short-name transition; confirm what of it still applies once canonical id is + never a map key at all for these types. +- **`MergedConfigStoreApiTest`, `MergedConfigStorePartialUpdateTest`, + `MergedConfigStoreReplicaUpdateTest`**: update key-shape assertions throughout. + +## Updates needed to already-filed issues and the open PR + +- **Issue #1781 (epic)**: update "Rejected alternatives" — either remove the "short-name-keyed + `Config` maps" rejection and replace with a forward reference to this document, or add a note that + it was reopened and why (the narrower-mismatch argument above). Update the Requirement table's row + 1 ("canonical id keeps working as a harmless superset") to state this no longer holds for + models/applications/toolsets/interceptors/roles/schemas, and is an accepted, deliberate change. +- **Issue #1783 (Slice B+C)**: the "Part B" section (derivation via `resolve`, blob-shadows-file, + `lastSegment` outbound naming) describes exactly the mechanism this document removes. Needs a + rewrite describing short-name-keyed maps instead, dropping the `resolve`/shadow-file + language entirely. "Part C" (schema `$id` index) needs the same rewrite pointed at + `schema-id-atomic-derivation.md` instead of the alias-index approach. +- **Issue #1784 (Slice D, migration endpoint)**: the "Open item — schema migration naming" section + is resolved by `schema-id-atomic-derivation.md` (canonical id derives from `$id`, no naming + decision needed) — update or remove that section. The migration behavior for models/apps/etc. is + otherwise unaffected by this document (blob write path/address is unchanged; only the in-memory + key changes), so Slice D's core behavior stands. +- **PR #1813**: still open, not yet merged. Its description documents exactly the `resolve()`/ + shadow-file/`lastSegment` mechanism this document replaces. Two options once the above lands as + actual commits on this PR (or its branch): (a) rewrite the PR description to describe the final + (short-name-keyed) state directly, since GitHub PR descriptions are editable and don't carry + historical baggage the way commit messages do; (b) at merge time, use squash-merge with a fresh + commit message describing the shipped state, rather than the incremental "add index, then replace + index with a bigger rewrite" history — squash-merge naturally collapses this without needing any + destructive history rewrite before merge. + +## Sequencing + +Independent of Slice A (apps/toolsets → `platform`) and Slice D (migration endpoint) in scope, but +touches the same files Slice B (#1783/PR #1813) already modified — this should land as a follow-up +on top of that work (or be squashed into it before merge, per the PR note above), not as a +separate, later PR that has to un-migrate short-name derivation that was just added. diff --git a/server/src/main/java/com/epam/aidial/core/server/config/BlobEntityValidator.java b/server/src/main/java/com/epam/aidial/core/server/config/BlobEntityValidator.java index 43f431ffc..80193d9c0 100644 --- a/server/src/main/java/com/epam/aidial/core/server/config/BlobEntityValidator.java +++ b/server/src/main/java/com/epam/aidial/core/server/config/BlobEntityValidator.java @@ -37,7 +37,7 @@ private static void appendInterceptorWarnings(List refs, Config config, } for (int i = 0; i < refs.size(); i++) { String ref = refs.get(i); - if (ref == null || config.getInterceptor(ref) == null) { + if (ref == null || !config.getInterceptors().containsKey(ref)) { warnings.add(new ValidationWarning("interceptors[" + i + "]", "Interceptor '" + ref + "' not found")); } @@ -48,7 +48,7 @@ private static void appendSchemaWarning(URI schemaId, Config config, List warnings = new ArrayList<>(); validatePricing(model, warnings); if (onSkip != null) { @@ -151,7 +149,7 @@ static void validateSingleModel(Config config, String canonicalId, static void validateSingleInterceptor(Config config, String canonicalId) { Interceptor interceptor = config.getInterceptors().get(canonicalId); if (interceptor != null) { - interceptor.setName(lastSegment(canonicalId)); + interceptor.setName(canonicalId); } } @@ -162,7 +160,7 @@ static void validateSingleInterceptor(Config config, String canonicalId) { static void validateSingleRole(Config config, String canonicalId) { Role role = config.getRoles().get(canonicalId); if (role != null) { - role.setName(lastSegment(canonicalId)); + role.setName(canonicalId); } } @@ -174,7 +172,7 @@ static void validateSingleRole(Config config, String canonicalId) { static void validateSingleApplication(Config config, String canonicalId) { Application application = config.getApplications().get(canonicalId); if (application != null) { - application.setName(lastSegment(canonicalId)); + application.setName(canonicalId); } } @@ -185,7 +183,7 @@ static void validateSingleApplication(Config config, String canonicalId) { static void validateSingleToolSet(Config config, String canonicalId) { ToolSet toolSet = config.getToolsets().get(canonicalId); if (toolSet != null) { - toolSet.setName(lastSegment(canonicalId)); + toolSet.setName(canonicalId); } } @@ -250,7 +248,7 @@ private static void processModels(Config config, Set deploymentIds, continue; } Model model = entry.getValue(); - model.setName(lastSegment(name)); + model.setName(name); log.debug("Loading {}", model); List warnings = new ArrayList<>(); validatePricing(model, warnings); @@ -273,19 +271,20 @@ private static void processModels(Config config, Set deploymentIds, /** * Validates that every interceptor reference on the supplied model resolves - * within {@code config}. Resolve-aware ({@code config.getInterceptor}) rather than a raw - * {@code containsKey}, so a short-name reference to a migrated (canonical-id-keyed, with the - * file entry shadowed) interceptor is not wrongly treated as dangling. Returns {@code true} - * when every reference resolves (no warnings appended). + * within the merged {@code config.interceptors} map. {@link MergedConfigStore} + * keys file entries by simple name and API entries by canonical ID; either + * shape is accepted via {@code containsKey}. Returns {@code true} when every + * reference resolves (no warnings appended). */ public static boolean validateCrossReferences(Model model, Config config, List warnings) { List refs = model.getInterceptors(); if (refs == null || refs.isEmpty()) { return true; } + Map interceptors = config.getInterceptors(); for (int i = 0; i < refs.size(); i++) { String ref = refs.get(i); - if (ref == null || config.getInterceptor(ref) == null) { + if (ref == null || !interceptors.containsKey(ref)) { warnings.add(new ValidationWarning("interceptors[" + i + "]", "Interceptor '" + ref + "' not found in config")); } @@ -321,7 +320,7 @@ private static void processApplications(Config config, Set deploymentIds continue; } Application application = entry.getValue(); - application.setName(lastSegment(name)); + application.setName(name); validateExternalServices(application); log.debug("Loading {}", application); } @@ -375,7 +374,7 @@ private static void processRoles(Config config) { for (Map.Entry entry : config.getRoles().entrySet()) { String name = entry.getKey(); Role role = entry.getValue(); - role.setName(lastSegment(name)); + role.setName(name); log.debug("Start loading role `{}`", role.getName()); for (Map.Entry limitEntry : role.getLimits().entrySet()) { log.debug("Loading {} for deployment `{}`", limitEntry.getValue(), limitEntry.getKey()); @@ -394,7 +393,7 @@ private static void processInterceptors(Config config, Set deploymentIds continue; } Interceptor interceptor = entry.getValue(); - interceptor.setName(lastSegment(name)); + interceptor.setName(name); log.debug("Loading {}", interceptor); } } @@ -410,7 +409,7 @@ private static void processToolSets(Config config, Set deploymentIds, } if (isValidToolSetKey(name)) { ToolSet toolSet = entry.getValue(); - toolSet.setName(lastSegment(name)); + toolSet.setName(name); log.debug("Loading {}", entry.getValue()); } else { log.warn("Invalid ToolSet name: {}", name); @@ -423,18 +422,11 @@ private static void processToolSets(Config config, Set deploymentIds, * Returns true and removes the offending entry when the name was already seen. * Abort mode ({@code onSkip == null}) preserves {@link FileConfigStore}'s today-behavior: * throw {@link IllegalStateException} and roll back the load. - * - *

Dedupes on the derived short name - * ({@link com.epam.aidial.core.server.util.PlatformCanonicalIdUtil#lastSegment}), not the raw map key — - * deployment-id uniqueness is a client-facing (short-name) concept, and a file entry - * (bare key {@code gpt-4}) and a blob entry of a different type (canonical key - * {@code applications/platform/gpt-4}) resolve to the same short name for - * {@link Config#selectDeployment}. Comparing raw keys would miss that collision entirely. */ private static boolean skipOnDuplicate(String name, ResourceTypes type, Set deploymentIds, @Nullable BiConsumer onSkip, Iterator iterator) { - if (deploymentIds.add(lastSegment(name))) { + if (deploymentIds.add(name)) { return false; } if (onSkip == null) { @@ -447,33 +439,6 @@ private static boolean skipOnDuplicate(String name, ResourceTypes type, Setdifferent model/application/interceptor/toolset entry in {@code config}. - * {@link #skipOnDuplicate} enforces the same invariant during a full rebuild via a shared - * {@code deploymentIds} set spanning all four types; this is the equivalent check for a write - * that only touches one entity at a time and never runs skipOnDuplicate. Excludes - * {@code canonicalId} itself so updating an existing entity in place isn't flagged as a - * duplicate of its own prior version. - */ - public static boolean isDeploymentIdTaken(Config config, String canonicalId) { - String shortName = lastSegment(canonicalId); - return shortNameTakenIn(config.getModels(), canonicalId, shortName) - || shortNameTakenIn(config.getApplications(), canonicalId, shortName) - || shortNameTakenIn(config.getInterceptors(), canonicalId, shortName) - || shortNameTakenIn(config.getToolsets(), canonicalId, shortName); - } - - private static boolean shortNameTakenIn(Map map, String canonicalId, String shortName) { - for (String key : map.keySet()) { - if (!key.equals(canonicalId) && lastSegment(key).equals(shortName)) { - return true; - } - } - return false; - } - private static boolean isValidResourceKey(String resourceKey) { return RESOURCE_KEY_PATTERN.matcher(resourceKey).matches(); } diff --git a/server/src/main/java/com/epam/aidial/core/server/config/MergedConfigStore.java b/server/src/main/java/com/epam/aidial/core/server/config/MergedConfigStore.java index b2dabfae2..54b4f9404 100644 --- a/server/src/main/java/com/epam/aidial/core/server/config/MergedConfigStore.java +++ b/server/src/main/java/com/epam/aidial/core/server/config/MergedConfigStore.java @@ -337,14 +337,14 @@ static boolean isManagedEventUrl(String url) { void applyReplicaEvent(ResourceDescriptor descriptor, ResourceEvent.Action action) { try { ResourceTypes type = (ResourceTypes) descriptor.getType(); - String canonicalId = canonicalId(descriptor); + String mapKey = mapKeyFor(type, descriptor); if (action == ResourceEvent.Action.DELETE) { - applyReplicaDelete(type, canonicalId); + applyReplicaDelete(type, mapKey); return; } String body = resourceService.getResource(descriptor); if (body == null) { - applyReplicaDelete(type, canonicalId); + applyReplicaDelete(type, mapKey); return; } JsonNode node = ProxyUtil.BLOB_MAPPER.readTree(body); @@ -372,7 +372,7 @@ void applyReplicaEvent(ResourceDescriptor descriptor, ResourceEvent.Action actio // applyReplicaDelete). On rotation (secret changed) the old auth bearer must be // revoked so the previous secret no longer authenticates (FINDING #2). Config snapshot = this.config; - Key prior = snapshot == null ? null : snapshot.getKeys().get(canonicalId); + Key prior = snapshot == null ? null : snapshot.getKeys().get(mapKey); String oldSecret = prior == null ? null : prior.getKey(); if (secret != null && !secret.isBlank()) { ApiKeyData data = new ApiKeyData(); @@ -383,7 +383,7 @@ void applyReplicaEvent(ResourceDescriptor descriptor, ResourceEvent.Action actio apiKeyStore.removeKey(oldSecret); } } - applyEntityWrite(type, canonicalId, entity); + applyEntityWrite(type, mapKey, entity); } finally { rebuildLock.unlock(); } @@ -394,7 +394,7 @@ void applyReplicaEvent(ResourceDescriptor descriptor, ResourceEvent.Action actio } } - private void applyReplicaDelete(ResourceTypes type, String canonicalId) { + private void applyReplicaDelete(ResourceTypes type, String mapKey) { if (type == ResourceTypes.GLOBAL_SETTINGS) { applySettingsDelete(); return; @@ -407,13 +407,13 @@ private void applyReplicaDelete(ResourceTypes type, String canonicalId) { try { if (type == ResourceTypes.PROJECT_KEY) { Config snapshot = this.config; - Key existing = snapshot == null ? null : snapshot.getKeys().get(canonicalId); + Key existing = snapshot == null ? null : snapshot.getKeys().get(mapKey); String secret = existing == null ? null : existing.getKey(); if (secret != null && !secret.isBlank()) { apiKeyStore.removeKey(secret); } } - applyEntityDelete(type, canonicalId); + applyEntityDelete(type, mapKey); } finally { rebuildLock.unlock(); } @@ -819,24 +819,27 @@ private BiConsumer skipRouter( if (!MODE_SKIP.equals(onInvalidEntity)) { return null; } - return (skippedType, error) -> recordSkippedByMapKey(nextInvalid, skippedType, error, id -> null); + // Partial-update writes are always API-sourced — this path never touches file config. + return (skippedType, error) -> recordSkippedByMapKey(nextInvalid, skippedType, error, id -> null, true); } /** - * Records a {@link InvalidEntityException} routed via {@code onSkip}, classifying the source the - * same way the full {@code rebuild()} path does: a map key carrying a {@code '/'} is an API-sourced - * canonical id, while a bare key is a file-defined simple name that we expand to its canonical id. + * Records a {@link InvalidEntityException} routed via {@code onSkip}. The map key's shape still + * tells us the canonical id and display name: a key carrying a {@code '/'} already is the + * canonical id, while a bare key is a short name we expand via {@code type/platform/name}. That + * derivation is independent of source, but the {@code source} label itself needs an explicit + * {@code fromApi} — for the five short-name-keyed types a file entry and a blob entry now share + * the identical (bare) key shape, so the caller must supply real provenance instead of guessing. * {@code payloadByCanonicalId} supplies the parsed blob body (rebuild) or {@code null} (partial update). */ private void recordSkippedByMapKey(Map> nextInvalid, ResourceTypes type, InvalidEntityException error, - Function payloadByCanonicalId) { + Function payloadByCanonicalId, boolean fromApi) { String mapKey = error.getMapKey(); - boolean fromApi = mapKey.contains("/"); - String canonicalId = fromApi + String canonicalId = mapKey.contains("/") ? mapKey : canonicalId(type, locationStrategy.resolveBucket(type, EntityLocationStrategy.PLATFORM_SCOPE), mapKey); - String simpleName = fromApi ? lastSegment(mapKey) : mapKey; + String simpleName = lastSegment(mapKey); recordInvalid(nextInvalid, type, canonicalId, simpleName, error.getMessage(), error.getWarnings(), payloadByCanonicalId.apply(canonicalId), REASON_VALIDATION, fromApi ? "api" : "file"); } @@ -962,31 +965,21 @@ private static Object peekEntity(Config config, ResourceTypes type, String canon private static void putEntityInPlace(Config config, ResourceTypes type, String canonicalId, Object entity) { switch (type) { - case MODEL -> putNameAddressed(config.getModels(), canonicalId, (Model) entity); - case INTERCEPTOR -> putNameAddressed(config.getInterceptors(), canonicalId, (Interceptor) entity); - case ROLE -> putNameAddressed(config.getRoles(), canonicalId, (Role) entity); + case MODEL -> config.getModels().put(canonicalId, (Model) entity); + case INTERCEPTOR -> config.getInterceptors().put(canonicalId, (Interceptor) entity); + case ROLE -> config.getRoles().put(canonicalId, (Role) entity); case PROJECT_KEY -> config.getKeys().put(canonicalId, (Key) entity); case ROUTE -> config.getRoutes().put(canonicalId, (Route) entity); case APP_TYPE_SCHEMA -> putSchemaInPlace(config.getApplicationTypeSchemas(), config.getApplicationSchemaAliasesById(), canonicalId, entity); case CATALOG_SCHEMA -> putSchemaInPlace(config.getCatalogSchemas(), config.getCatalogSchemaAliasesById(), canonicalId, entity); - case APPLICATION -> putNameAddressed(config.getApplications(), canonicalId, (Application) entity); - case TOOL_SET -> putNameAddressed(config.getToolsets(), canonicalId, (ToolSet) entity); + case APPLICATION -> config.getApplications().put(canonicalId, (Application) entity); + case TOOL_SET -> config.getToolsets().put(canonicalId, (ToolSet) entity); default -> throw new IllegalArgumentException("Unsupported type for partial update: " + type); } } - /** - * Puts a name-addressed entity under its canonical id, then removes the file-defined entry - * sharing its short name (blob shadows file), so a subsequent short-name {@code resolve} - * hits the freshly-written blob entity rather than a stale file entry. - */ - private static void putNameAddressed(Map map, String canonicalId, V entity) { - map.put(canonicalId, entity); - map.remove(lastSegment(canonicalId)); - } - private static void putSchemaInPlace(Map schemas, Map aliasesById, String canonicalId, Object entity) { String body = schemaBody(entity); @@ -1029,12 +1022,6 @@ private static String extractSchemaId(String body) { } } - /** - * Removes only the canonical-id entry. Deliberately does not restore the file-defined - * entry that {@link #putNameAddressed}/{@link #shadowFileEntry} shadowed when the blob entity - * was created — an accepted, transient gap: the short name stays unreachable until the next - * full {@link #rebuild()} restores it. Do not add restoration logic here. - */ private static void removeEntityInPlace(Config config, ResourceTypes type, String canonicalId) { switch (type) { case MODEL -> config.getModels().remove(canonicalId); @@ -1108,9 +1095,9 @@ private Config rebuild() { merged.setRetriableErrorCodes(base.getRetriableErrorCodes()); merged.setGlobalInterceptors(base.getGlobalInterceptors()); // Wire the (still-being-populated) local maps onto merged now rather than after the blob - // scan below — same references either way, but it lets addBlobEntity/removeAddedEntity/ - // shadowFileEntry dispatch off a single Config parameter instead of a positional map per - // type (they used to take 9-11 map parameters each). + // scan below — same references either way, but it lets addBlobEntity/removeAddedEntity + // dispatch off a single Config parameter instead of a positional map per type (they used + // to take 9-11 map parameters each). merged.setModels(models); merged.setInterceptors(interceptors); merged.setRoles(roles); @@ -1130,6 +1117,12 @@ private Config rebuild() { // can classify the source without inferring from map-key shape — file secrets may be Base64 // and contain '/' (OQ-12). The merged keys map itself still folds both for config consumers. Map apiKeysByCanonicalId = new HashMap<>(); + // For models/interceptors/roles/applications/toolsets, a blob entry's map key (its short + // name) is indistinguishable in shape from a file entry's — both are bare, slash-free names. + // Track which (type, short name) pairs actually came from a blob this rebuild, so the + // semantic pass below can classify a skipped entity as "api" vs "file" correctly instead of + // guessing from key shape (which only works for the still-canonical-id-keyed types). + Map> apiSourcedKeys = new EnumMap<>(ResourceTypes.class); for (ResourceTypes type : MANAGED_TYPES) { for (String scope : locationStrategy.listScopes(type)) { @@ -1137,12 +1130,6 @@ private Config rebuild() { if (bucket == null) { continue; } - // File-shadowing only makes sense for the platform-scoped copy of a type — the same - // guard onResourceEvent applies for the identical reason (APPLICATION/TOOL_SET also - // resolve for the public bucket). Currently a no-op since listScopes only ever - // returns the platform scope, but keeps this loop safe if that changes. - boolean isPlatformBucket = bucket.equals( - locationStrategy.resolveBucket(type, EntityLocationStrategy.PLATFORM_SCOPE)); String bucketLocation = bucket + ResourceDescriptor.PATH_SEPARATOR; ResourceDescriptor folder = ResourceDescriptorFactory.fromDecoded(type, bucket, bucketLocation, ""); List> items = @@ -1155,6 +1142,11 @@ private Config rebuild() { continue; } String canonicalId = canonicalId(type, bucket, name); + // Blob entries for these five types key Config's map by short name, the same + // key a file-sourced entry for the same logical entity already uses — no + // derivation, no shadowing. Every other managed type keeps using the canonical + // id as its map key. + String mapKey = isNameAddressed(type) ? name : canonicalId; JsonNode node; try { // Parse once into JsonNode; reused below for typed deserialization and as the @@ -1172,7 +1164,7 @@ private Config rebuild() { type, bucket, bucketLocation, name); Object added; try { - added = addBlobEntity(merged, type, canonicalId, node); + added = addBlobEntity(merged, type, mapKey, node); } catch (Exception parseError) { recordInvalid(pendingInvalid, type, canonicalId, name, "JSON parse failure: " + parseError.getMessage(), @@ -1187,7 +1179,7 @@ private Config rebuild() { } catch (Exception decryptError) { // Roll back the partial insertion so decryption-failure entities never // reach addProjectKeys (locked 2S.9 invariant). - removeAddedEntity(merged, type, canonicalId); + removeAddedEntity(merged, type, mapKey); recordInvalid(pendingInvalid, type, canonicalId, name, "Decryption failure: " + decryptError.getMessage(), List.of(new ValidationWarning("body", decryptError.getMessage())), @@ -1197,12 +1189,8 @@ private Config rebuild() { if (type == ResourceTypes.PROJECT_KEY) { apiKeysByCanonicalId.put(canonicalId, (Key) added); } - // Shadow the file-defined entry sharing this canonical id's short name, - // gated on successful decryption above — if decryption failed, removeAddedEntity - // rolled the blob entity back, so the file entry must stay in this rebuild's map. - // Also gated on isPlatformBucket — see the comment where it's computed above. - if (isPlatformBucket) { - shadowFileEntry(merged, type, canonicalId); + if (isNameAddressed(type)) { + apiSourcedKeys.computeIfAbsent(type, t -> new HashSet<>()).add(mapKey); } } blobBodies.put(canonicalId, node); @@ -1226,7 +1214,11 @@ private Config rebuild() { error.getMapKey(), error.getMessage()); return; } - recordSkippedByMapKey(pendingInvalid, type, error, blobBodies::get); + // For the short-name-keyed types, key shape no longer tells file from blob + // (see apiSourcedKeys above); everything else still resolves via shape alone. + boolean fromApi = error.getMapKey().contains("/") + || apiSourcedKeys.getOrDefault(type, Set.of()).contains(error.getMapKey()); + recordSkippedByMapKey(pendingInvalid, type, error, blobBodies::get, fromApi); } : null; // File partition = base file keys (keyed by secret); API partition = PROJECT_KEY blob entries @@ -1234,11 +1226,11 @@ private Config rebuild() { // consumers, but the ApiKeyStore feed stays explicitly source-classified (FINDING #7). ConfigPostProcessor.processSemantic(merged, apiKeyStore, base.getKeys(), apiKeysByCanonicalId, onSkip); - // ConfigPostProcessor sets entity.name = mapKey: canonical ID for API entries - // ("models/platform/foo"), simple name for file entries ("gpt-4"). This is the OQ-23 contract: - // canonical IDs surface on legacy /openai/models, /openai/deployments, and rate-limit - // role-limit lookups for API-managed deployments. The new admin Configuration API listing - // controller projects simpleName(mapKey) independently per design 03 §4. + // ConfigPostProcessor sets entity.name = mapKey. For models/interceptors/roles/ + // applications/toolsets the map key is now always the short name ("gpt-4"), whether the + // entry is file- or blob-sourced, so /openai/models, /openai/deployments, and rate-limit + // role-limit lookups see the same short form either way. Keys/routes/schemas are unaffected + // — their map key stays the canonical id for API entries, as before. boolean overlayFromApi = applySettingsOverlay(merged, pendingInvalid); Map> finalInvalid = @@ -1336,6 +1328,26 @@ public static String canonicalId(ResourceDescriptor descriptor) { descriptor.getBucketName(), descriptor.getName()); } + /** + * The key {@code Config}'s in-memory maps use for {@code descriptor}'s entity: the short name + * (last path segment) for models/interceptors/roles/applications/toolsets, so blob-sourced and + * file-sourced entries of the same logical entity share one map key — no derivation, no + * shadowing, no separate canonical-id keying to reconcile. Every other managed type (keys, + * routes, schemas) keeps using the canonical id as its map key, unchanged. Callers that already + * have a {@link ResourceDescriptor} should use this instead of building the canonical id and + * then parsing the short name back out of it. + */ + public static String mapKeyFor(ResourceTypes type, ResourceDescriptor descriptor) { + return isNameAddressed(type) ? descriptor.getName() : canonicalId(descriptor); + } + + private static boolean isNameAddressed(ResourceTypes type) { + return switch (type) { + case MODEL, INTERCEPTOR, ROLE, APPLICATION, TOOL_SET -> true; + default -> false; + }; + } + /** * Reads the type-map to mutate off {@code config} rather than taking one map parameter per * managed type — {@code config}'s maps are wired up by {@link #rebuild} before the blob scan @@ -1400,23 +1412,6 @@ private static Object addBlobEntity(Config config, ResourceTypes type, String ca } } - /** - * Removes the file-defined entry sharing {@code canonicalId}'s short name from the - * name-addressed type maps (models/interceptors/roles/applications/toolsets), so blob wins - * over the file entry it just shadowed. No-op for types that aren't name-addressed (keys, - * routes, schemas — schemas use the $id index instead, see {@link #recordSchemaAlias}). - */ - private static void shadowFileEntry(Config config, ResourceTypes type, String canonicalId) { - switch (type) { - case MODEL -> config.getModels().remove(lastSegment(canonicalId)); - case INTERCEPTOR -> config.getInterceptors().remove(lastSegment(canonicalId)); - case ROLE -> config.getRoles().remove(lastSegment(canonicalId)); - case APPLICATION -> config.getApplications().remove(lastSegment(canonicalId)); - case TOOL_SET -> config.getToolsets().remove(lastSegment(canonicalId)); - default -> { /* not name-addressed */ } - } - } - private static void warnIfReplaced(ResourceTypes type, String canonicalId, Object previous) { if (previous != null) { log.warn("Duplicate canonical ID during merged Config rebuild: {} '{}' overwrote a prior entry", diff --git a/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java b/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java index fb6fe95bd..6e9cb6e76 100644 --- a/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java +++ b/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java @@ -310,18 +310,8 @@ static ValidationResult validateOnly(AdminManifest entry, Config scratch, boolea if (!warnings.isEmpty() && !softValidation) { return new ValidationResult(id, ValidationStatus.FAILED, joinWarnings(warnings)); } - String dupError = duplicateDeploymentIdMessage(scratch, ResourceTypes.MODEL, parsed); - if (dupError != null) { - return new ValidationResult(id, ValidationStatus.FAILED, dupError); - } - } - case "Interceptor" -> { - ConfigResourceController.treeToEntity(entry.spec(), Interceptor.class); - String dupError = duplicateDeploymentIdMessage(scratch, ResourceTypes.INTERCEPTOR, parsed); - if (dupError != null) { - return new ValidationResult(id, ValidationStatus.FAILED, dupError); - } } + case "Interceptor" -> ConfigResourceController.treeToEntity(entry.spec(), Interceptor.class); case "Role" -> ConfigResourceController.treeToEntity(entry.spec(), Role.class); case "Route" -> ConfigResourceController.treeToEntity(entry.spec(), Route.class); case "Key" -> { @@ -337,24 +327,8 @@ static ValidationResult validateOnly(AdminManifest entry, Config scratch, boolea "Invalid key: at least one role must be assigned to the key " + key.getProject()); } } - case "Application" -> { - ConfigResourceController.treeToEntity(entry.spec(), Application.class); - if (ResourceDescriptor.PLATFORM_BUCKET.equals(parsed.bucket())) { - String dupError = duplicateDeploymentIdMessage(scratch, ResourceTypes.APPLICATION, parsed); - if (dupError != null) { - return new ValidationResult(id, ValidationStatus.FAILED, dupError); - } - } - } - case "ToolSet" -> { - ConfigResourceController.treeToEntity(entry.spec(), ToolSet.class); - if (ResourceDescriptor.PLATFORM_BUCKET.equals(parsed.bucket())) { - String dupError = duplicateDeploymentIdMessage(scratch, ResourceTypes.TOOL_SET, parsed); - if (dupError != null) { - return new ValidationResult(id, ValidationStatus.FAILED, dupError); - } - } - } + case "Application" -> ConfigResourceController.treeToEntity(entry.spec(), Application.class); + case "ToolSet" -> ConfigResourceController.treeToEntity(entry.spec(), ToolSet.class); case "Schema" -> { if (!entry.spec().isObject()) { return new ValidationResult(id, ValidationStatus.FAILED, "Schema spec must be a JSON object"); @@ -379,21 +353,6 @@ static ValidationResult validateOnly(AdminManifest entry, Config scratch, boolea return new ValidationResult(id, ValidationStatus.VALID, null); } - /** - * Shared by {@link #validateOnly} (precheck) and the real-apply {@code applyX} methods: - * non-null iff {@code type}'s canonical id under {@code parsed} has a short name already - * claimed by a different model/application/interceptor/toolset in {@code scratch}. See - * {@link ConfigPostProcessor#isDeploymentIdTaken}. - */ - private static String duplicateDeploymentIdMessage(Config scratch, ResourceTypes type, ParsedName parsed) { - ResourceDescriptor descriptor = ResourceDescriptorFactory.fromDecoded( - type, parsed.bucket(), parsed.location(), parsed.name()); - if (ConfigPostProcessor.isDeploymentIdTaken(scratch, MergedConfigStore.canonicalId(descriptor))) { - return "Deployment ID '" + parsed.name() + "' is already used by a different entity"; - } - return null; - } - private EntityResult applySingle(AdminManifest entry, Config scratch, List pending) { String id = entry.name(); ParsedName parsed; @@ -406,13 +365,13 @@ private EntityResult applySingle(AdminManifest entry, Config scratch, List applySettings(entry, id, parsed); case "Schema" -> applySchema(entry, id, parsed, scratch, pending, ResourceTypes.APP_TYPE_SCHEMA); case "CatalogSchema" -> applySchema(entry, id, parsed, scratch, pending, ResourceTypes.CATALOG_SCHEMA); - case "Interceptor" -> applyManagedEntity(entry, id, parsed, ResourceTypes.INTERCEPTOR, Interceptor.class, scratch, pending); - case "Role" -> applyManagedEntity(entry, id, parsed, ResourceTypes.ROLE, Role.class, scratch, pending); - case "Route" -> applyManagedEntity(entry, id, parsed, ResourceTypes.ROUTE, Route.class, scratch, pending); + case "Interceptor" -> applyManagedEntity(entry, id, parsed, ResourceTypes.INTERCEPTOR, Interceptor.class, pending); + case "Role" -> applyManagedEntity(entry, id, parsed, ResourceTypes.ROLE, Role.class, pending); + case "Route" -> applyManagedEntity(entry, id, parsed, ResourceTypes.ROUTE, Route.class, pending); case "Key" -> applyKey(entry, id, parsed, pending); case "Model" -> applyModel(entry, id, parsed, scratch, pending); - case "ToolSet" -> applyToolSet(entry, id, parsed, scratch, pending); - case "Application" -> applyApplication(entry, id, parsed, scratch, pending); + case "ToolSet" -> applyToolSet(entry, id, parsed, pending); + case "Application" -> applyApplication(entry, id, parsed, pending); default -> new EntityResult(id, AdminApplyStatus.FAILED, "Unknown kind: " + entry.kind()); }; } @@ -456,22 +415,13 @@ private EntityResult applySchema(AdminManifest entry, String id, ParsedName pars } private EntityResult applyManagedEntity(AdminManifest entry, String id, ParsedName parsed, - ResourceTypes type, Class entityClass, Config scratch, - List pending) { + ResourceTypes type, Class entityClass, List pending) { T entity = ConfigResourceController.treeToEntity(entry.spec(), entityClass); ResourceDescriptor descriptor = ResourceDescriptorFactory.fromDecoded( type, parsed.bucket(), parsed.location(), parsed.name()); - // Deployment-id uniqueness only applies to INTERCEPTOR here — ROLE/ROUTE aren't deployments - // resolved through Config.selectDeployment, so they don't share the short-name namespace. - if (type == ResourceTypes.INTERCEPTOR) { - String dupError = duplicateDeploymentIdMessage(scratch, type, parsed); - if (dupError != null) { - return new EntityResult(id, AdminApplyStatus.FAILED, dupError); - } - } String blobBody = ConfigResourceController.serializeForBlob(entity); resourceService.putResource(descriptor, blobBody, EtagHeader.ANY); - pending.add(new EntityChange(type, MergedConfigStore.canonicalId(descriptor), entity)); + pending.add(new EntityChange(type, MergedConfigStore.mapKeyFor(type, descriptor), entity)); return new EntityResult(id, AdminApplyStatus.APPLIED, null); } @@ -534,62 +484,43 @@ private EntityResult applyModel(AdminManifest entry, String id, ParsedName parse } ResourceDescriptor descriptor = ResourceDescriptorFactory.fromDecoded( ResourceTypes.MODEL, parsed.bucket(), parsed.location(), parsed.name()); - String dupError = duplicateDeploymentIdMessage(scratch, ResourceTypes.MODEL, parsed); - if (dupError != null) { - return new EntityResult(id, AdminApplyStatus.FAILED, dupError); - } secretFieldProcessor.encryptFields(model, descriptor); String blobBody = ConfigResourceController.serializeForBlob(model); resourceService.putResource(descriptor, blobBody, EtagHeader.ANY); // Slice 4S.4: decrypt-in-place so partial-update receives plaintext upstream secrets. secretFieldProcessor.decryptFields(model, descriptor); - pending.add(new EntityChange(ResourceTypes.MODEL, MergedConfigStore.canonicalId(descriptor), model)); + pending.add(new EntityChange(ResourceTypes.MODEL, MergedConfigStore.mapKeyFor(ResourceTypes.MODEL, descriptor), model)); return new EntityResult(id, invalid ? AdminApplyStatus.APPLIED_INVALID : AdminApplyStatus.APPLIED, null); } - private EntityResult applyApplication(AdminManifest entry, String id, ParsedName parsed, Config scratch, List pending) { + private EntityResult applyApplication(AdminManifest entry, String id, ParsedName parsed, List pending) { Application application = ConfigResourceController.treeToEntity(entry.spec(), Application.class); ResourceDescriptor descriptor = ResourceDescriptorFactory.fromDecoded( ResourceTypes.APPLICATION, parsed.bucket(), parsed.location(), parsed.name()); - // Only the platform bucket is materialized into MergedConfigStore (see EntityLocationStrategy) — - // public-bucket apps stay outside it and are served lazily by ApplicationService, so they're - // exempt from deployment-id uniqueness and pushing them into `pending` below would spuriously - // duplicate them in config.getApplications()-backed listings (e.g. ApplicationController/ - // DeploymentController) until the next full rebuild. - boolean platform = ResourceDescriptor.PLATFORM_BUCKET.equals(parsed.bucket()); - if (platform) { - String dupError = duplicateDeploymentIdMessage(scratch, ResourceTypes.APPLICATION, parsed); - if (dupError != null) { - return new EntityResult(id, AdminApplyStatus.FAILED, dupError); - } - } // Bulk admin apply is always admin context — preserve forwardAuthToken if the manifest set it. applicationService.putApplication(descriptor, EtagHeader.ANY, null, application, true, AdminManagedFieldsWriteMode.AUTHORITATIVE); - if (platform) { + // Only the platform bucket is materialized into MergedConfigStore (see EntityLocationStrategy) — + // public-bucket apps stay outside it and are served lazily by ApplicationService, so pushing + // them into `pending` here would spuriously duplicate them in config.getApplications()-backed + // listings (e.g. ApplicationController/DeploymentController) until the next full rebuild. + if (ResourceDescriptor.PLATFORM_BUCKET.equals(parsed.bucket())) { Application decrypted = applicationService.getApplicationWithDecryptedSecrets(descriptor).getValue(); - pending.add(new EntityChange(ResourceTypes.APPLICATION, MergedConfigStore.canonicalId(descriptor), decrypted)); + pending.add(new EntityChange(ResourceTypes.APPLICATION, MergedConfigStore.mapKeyFor(ResourceTypes.APPLICATION, descriptor), decrypted)); } return new EntityResult(id, AdminApplyStatus.APPLIED, null); } - private EntityResult applyToolSet(AdminManifest entry, String id, ParsedName parsed, Config scratch, List pending) { + private EntityResult applyToolSet(AdminManifest entry, String id, ParsedName parsed, List pending) { ToolSet toolSet = ConfigResourceController.treeToEntity(entry.spec(), ToolSet.class); ResourceDescriptor descriptor = ResourceDescriptorFactory.fromDecoded( ResourceTypes.TOOL_SET, parsed.bucket(), parsed.location(), parsed.name()); - // Same rationale as applyApplication above — only platform-bucket toolsets belong in - // MergedConfigStore / are subject to deployment-id uniqueness. - boolean platform = ResourceDescriptor.PLATFORM_BUCKET.equals(parsed.bucket()); - if (platform) { - String dupError = duplicateDeploymentIdMessage(scratch, ResourceTypes.TOOL_SET, parsed); - if (dupError != null) { - return new EntityResult(id, AdminApplyStatus.FAILED, dupError); - } - } toolSetService.putToolSet(descriptor, EtagHeader.ANY, null, toolSet, true); - if (platform) { + // Same rationale as applyApplication above — only platform-bucket toolsets belong in + // MergedConfigStore. + if (ResourceDescriptor.PLATFORM_BUCKET.equals(parsed.bucket())) { ToolSet decrypted = toolSetService.getToolSetWithDecryptedAuthSettings(descriptor).getValue(); - pending.add(new EntityChange(ResourceTypes.TOOL_SET, MergedConfigStore.canonicalId(descriptor), decrypted)); + pending.add(new EntityChange(ResourceTypes.TOOL_SET, MergedConfigStore.mapKeyFor(ResourceTypes.TOOL_SET, descriptor), decrypted)); } return new EntityResult(id, AdminApplyStatus.APPLIED, null); } diff --git a/server/src/main/java/com/epam/aidial/core/server/controller/BaseInterceptorController.java b/server/src/main/java/com/epam/aidial/core/server/controller/BaseInterceptorController.java index 26880635a..ac149d74a 100644 --- a/server/src/main/java/com/epam/aidial/core/server/controller/BaseInterceptorController.java +++ b/server/src/main/java/com/epam/aidial/core/server/controller/BaseInterceptorController.java @@ -48,7 +48,7 @@ protected BaseInterceptorController(Proxy proxy, ProxyContext context, int inter public Future handle() { List interceptors = context.getInterceptors(); String interceptorName = interceptors.get(interceptorIndex); - Interceptor interceptor = context.getConfig().getInterceptor(interceptorName); + Interceptor interceptor = context.getConfig().getInterceptors().get(interceptorName); if (interceptor == null) { log.warn("Interceptor is not found: {}", interceptorName); return respond(HttpStatus.NOT_FOUND, "Interceptor is not found"); diff --git a/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java b/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java index 496a8f5f2..3974240bb 100644 --- a/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java +++ b/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java @@ -1011,14 +1011,11 @@ private Future handleGet() throws JsonProcessingException { // Slice U.4: secret fields drop on response via @JsonProperty(WRITE_ONLY) — there is no // ?reveal_secrets=true reveal flow and no security-admin tier. return switch (resourceType()) { - case MODEL -> handleSingleGet( - config.getModels(), ResourceTypes.MODEL, + case MODEL -> handleSingleGetFromBlob(ResourceTypes.MODEL, (key, model) -> projectItem(model, key)); - case INTERCEPTOR -> handleSingleGet( - config.getInterceptors(), ResourceTypes.INTERCEPTOR, + case INTERCEPTOR -> handleSingleGetFromBlob(ResourceTypes.INTERCEPTOR, (key, interceptor) -> projectItem(interceptor, key)); - case ROLE -> handleSingleGet( - config.getRoles(), ResourceTypes.ROLE, + case ROLE -> handleSingleGetFromBlob(ResourceTypes.ROLE, (key, role) -> projectItem(role, key)); case PROJECT_KEY -> handleSingleGet( config.getKeys(), ResourceTypes.PROJECT_KEY, @@ -1028,17 +1025,21 @@ private Future handleGet() throws JsonProcessingException { (key, route) -> projectItem(route, key)); case APP_TYPE_SCHEMA -> handleSchemaGet(config.getApplicationTypeSchemas(), ResourceTypes.APP_TYPE_SCHEMA, admin); case CATALOG_SCHEMA -> handleSchemaGet(config.getCatalogSchemas(), ResourceTypes.CATALOG_SCHEMA, admin); - case APPLICATION -> handleSingleGet( - config.getApplications(), ResourceTypes.APPLICATION, + case APPLICATION -> handleSingleGetFromBlob(ResourceTypes.APPLICATION, (key, application) -> redactExternalServiceSecrets(projectItem(application, key))); - case TOOL_SET -> handleSingleGet( - config.getToolsets(), ResourceTypes.TOOL_SET, + case TOOL_SET -> handleSingleGetFromBlob(ResourceTypes.TOOL_SET, (key, toolSet) -> redactAuthSettingsSecrets(projectItem(toolSet, key))); case GLOBAL_SETTINGS -> handleSettingsGet(config); default -> respondMethodNotAllowed(); }; } + /** + * Per-entity GET for {@code PROJECT_KEY}/{@code ROUTE} only — these two types still key + * {@code Config}'s in-memory map by canonical id (never migrated to short-name keying, see + * {@code short-name-keyed-config-maps.md}), so file-sourced entries never share a key with a + * blob-sourced one and this lookup can safely stay in-memory. + */ private Future handleSingleGet(Map source, ResourceTypes resourceType, BiFunction projector) { @@ -1078,6 +1079,79 @@ private Future handleSingleGet(Map source, return Future.succeededFuture(); } + /** + * Per-entity GET for {@code MODEL}/{@code INTERCEPTOR}/{@code ROLE}/{@code APPLICATION}/ + * {@code TOOL_SET} — these five types key {@code Config}'s in-memory map by short name, the + * same key a file-sourced entry for the same logical entity already uses (see + * {@code short-name-keyed-config-maps.md}), so the map can no longer tell a genuinely + * blob-managed entity apart from a file-only one sharing that short name. This reads and + * decrypts blob storage directly by descriptor instead — the same pattern PUT/DELETE already + * use for these types — so this endpoint only ever serves entities that actually exist in the + * {@code platform} bucket. {@code Config}'s map stays purely a runtime-resolution structure. + */ + private Future handleSingleGetFromBlob(ResourceTypes resourceType, BiFunction projector) { + if (path == null || path.isEmpty()) { + context.respond(HttpStatus.NOT_FOUND); + return Future.succeededFuture(); + } + ResourceDescriptor descriptor = descriptorFor(resourceType); + boolean admin = authorizationService.isAdmin(context); + EtagHeader etag = ProxyUtil.etag(context.getRequest()); + + taskExecutor.submit(() -> { + // getResourceWithMetadata validates the conditional header itself (throws the + // appropriate HttpException for If-None-Match/If-Match), so a 304/412 short-circuits + // here before any decrypt work runs. + Pair existing = resourceService.getResourceWithMetadata(descriptor, etag); + if (existing == null) { + return null; + } + Object entity = readAndDecrypt(resourceType, existing.getValue(), descriptor); + return Pair.of(existing.getKey().getEtag(), projector.apply(path, entity)); + }).onSuccess(result -> { + if (result == null) { + Map invalid = mergedConfigStore.getInvalidEntities() + .getOrDefault(resourceType, Map.of()); + InvalidEntityRecord invalidRecord = invalid.get(canonicalId()); + if (invalidRecord != null) { + context.respond(HttpStatus.OK, projectInvalidItem(invalidRecord, admin)); + } else { + context.respond(HttpStatus.NOT_FOUND); + } + return; + } + context.putHeader(HttpHeaders.ETAG, result.getKey()) + .respond(HttpStatus.OK, result.getValue()); + }).onFailure(this::handleWriteError); + + return Future.succeededFuture(); + } + + /** + * Deserializes and decrypts a blob body for {@link #handleSingleGetFromBlob}. Applications and + * toolsets encrypt secrets outside the {@code @EncryptedField}/{@link SecretFieldProcessor} + * path (per-resource, via {@link ApplicationService}/{@link ToolSetService}), so they're + * re-fetched through those services' own decrypting reads rather than decoded from + * {@code body} directly — mirrors {@code MergedConfigStore.decryptManagedEntity}. + */ + private Object readAndDecrypt(ResourceTypes type, String body, ResourceDescriptor descriptor) { + return switch (type) { + case APPLICATION -> applicationService.getApplicationWithDecryptedSecrets(descriptor).getValue(); + case TOOL_SET -> toolSetService.getToolSetWithDecryptedAuthSettings(descriptor).getValue(); + default -> { + Object entity; + try { + entity = treeToEntity(ProxyUtil.BLOB_MAPPER.readTree(body), entityClassFor(entityType)); + } catch (JsonProcessingException e) { + throw new HttpException(HttpStatus.INTERNAL_SERVER_ERROR, + "Stored entity is malformed at " + locationOf(e)); + } + secretFieldProcessor.decryptFields(entity, descriptor); + yield entity; + } + }; + } + /** * Emits a {@code 304 Not Modified} if the matched entity's stored blob has an ETag * that matches the client's {@code If-None-Match} header. The blob-metadata fetch @@ -1305,10 +1379,6 @@ private Future handleAppOrToolSetPut() { throw new HttpException(HttpStatus.BAD_REQUEST, "Request body must be a JSON object"); } return taskExecutor.submit(() -> lockService.underBucketLocks(MergedConfigStore.ADMIN_BUCKET_LOCATIONS, () -> { - // This route is platform-bucket-only (see the dedicated /v1/(applications|toolsets)/ - // platform/... routes), so unlike the generic ResourceController path every write - // here is materialized into Config and subject to deployment-id uniqueness. - rejectDuplicateDeploymentId(descriptor); Object decrypted; // The platform bucket requires explicit admin access for every operation (see // AdminRoleAuthorizationService), not just an admin-AND-public-bucket combination like @@ -1323,7 +1393,7 @@ private Future handleAppOrToolSetPut() { toolSetService.putToolSet(descriptor, etag, author, toolSet, true); decrypted = toolSetService.getToolSetWithDecryptedAuthSettings(descriptor).getValue(); } - mergedConfigStore.applyEntityWrite(type, MergedConfigStore.canonicalId(descriptor), decrypted); + mergedConfigStore.applyEntityWrite(type, MergedConfigStore.mapKeyFor(type, descriptor), decrypted); return resourceService.getResourceMetadata(descriptor); })); }).onSuccess(meta -> context.putHeader(HttpHeaders.ETAG, meta.getEtag()) @@ -1353,7 +1423,7 @@ private Future handleAppOrToolSetDelete() { throw new HttpException(HttpStatus.NOT_FOUND, "Resource not found: " + descriptor.getUrl()); } } - mergedConfigStore.applyEntityDelete(type, MergedConfigStore.canonicalId(descriptor)); + mergedConfigStore.applyEntityDelete(type, MergedConfigStore.mapKeyFor(type, descriptor)); return true; })).onSuccess(v -> context.respond(HttpStatus.NO_CONTENT)).onFailure(this::handleWriteError); @@ -1452,10 +1522,6 @@ private Future handlePut() { if (entity instanceof Model m) { checkCrossReferences(m); } - ResourceTypes writeType = resourceType(); - if (writeType == ResourceTypes.MODEL || writeType == ResourceTypes.INTERCEPTOR) { - rejectDuplicateDeploymentId(descriptor); - } if (spec.isKey()) { keyEntity = (Key) entity; validateKeyForApiWrite(keyEntity, "PUT"); @@ -1487,7 +1553,7 @@ private Future handlePut() { secretFieldProcessor.decryptFields(entity, descriptor); } mergedConfigStore.applyEntityWrite(typeOf(descriptor), - MergedConfigStore.canonicalId(descriptor), + MergedConfigStore.mapKeyFor(typeOf(descriptor), descriptor), entity != null ? entity : requestNode); return meta; })); @@ -1552,7 +1618,7 @@ private Future handleDelete() { if (deletedSecret != null) { apiKeyStore.removeKey(deletedSecret); } - mergedConfigStore.applyEntityDelete(typeOf(descriptor), MergedConfigStore.canonicalId(descriptor)); + mergedConfigStore.applyEntityDelete(typeOf(descriptor), MergedConfigStore.mapKeyFor(typeOf(descriptor), descriptor)); return true; })).onSuccess(v -> context.respond(HttpStatus.NO_CONTENT)).onFailure(this::handleWriteError); @@ -1695,24 +1761,6 @@ private void handleWriteError(Throwable error) { } } - /** - * Rejects a MODEL/INTERCEPTOR/APPLICATION/TOOL_SET write whose derived short name is already - * claimed by a different deployment (of any of those four types) in the live merged Config — - * the partial-update-path counterpart of {@code ConfigPostProcessor.skipOnDuplicate}, which - * only runs during a full rebuild. See {@link ConfigPostProcessor#isDeploymentIdTaken}. - */ - private void rejectDuplicateDeploymentId(ResourceDescriptor descriptor) { - Config snapshot = mergedConfigStore.get(); - if (snapshot == null) { - return; - } - String canonicalId = MergedConfigStore.canonicalId(descriptor); - if (ConfigPostProcessor.isDeploymentIdTaken(snapshot, canonicalId)) { - throw new HttpException(HttpStatus.CONFLICT, - "Deployment ID '" + path + "' is already used by a different entity"); - } - } - /** * Cross-reference check for Model writes. Strict mode aborts with HTTP 422 carrying a * {@code {"validationWarnings":[...]}} JSON body. Soft mode logs and proceeds — the next diff --git a/server/src/main/java/com/epam/aidial/core/server/controller/DeploymentController.java b/server/src/main/java/com/epam/aidial/core/server/controller/DeploymentController.java index 761ab9e50..5836c031f 100644 --- a/server/src/main/java/com/epam/aidial/core/server/controller/DeploymentController.java +++ b/server/src/main/java/com/epam/aidial/core/server/controller/DeploymentController.java @@ -84,7 +84,7 @@ public DeploymentController(Proxy proxy, ProxyContext context) { ) public Future getDeployment(String deploymentId) { Config config = context.getConfig(); - Model model = config.getModel(deploymentId); + Model model = config.getModels().get(deploymentId); if (model == null) { return context.respond(HttpStatus.NOT_FOUND); diff --git a/server/src/main/java/com/epam/aidial/core/server/controller/ModelController.java b/server/src/main/java/com/epam/aidial/core/server/controller/ModelController.java index 4317558d6..8226fe34f 100644 --- a/server/src/main/java/com/epam/aidial/core/server/controller/ModelController.java +++ b/server/src/main/java/com/epam/aidial/core/server/controller/ModelController.java @@ -47,7 +47,7 @@ public class ModelController { ) public Future getModel(String modelId) { Config config = context.getConfig(); - Model model = config.getModel(modelId); + Model model = config.getModels().get(modelId); if (model == null) { return context.respond(HttpStatus.NOT_FOUND); diff --git a/server/src/main/java/com/epam/aidial/core/server/controller/ResourceController.java b/server/src/main/java/com/epam/aidial/core/server/controller/ResourceController.java index 52fdde33e..ebf423017 100644 --- a/server/src/main/java/com/epam/aidial/core/server/controller/ResourceController.java +++ b/server/src/main/java/com/epam/aidial/core/server/controller/ResourceController.java @@ -678,7 +678,7 @@ private void validateCustomApplication(Application application) { } Config config = context.getConfig(); for (String interceptor : application.getInterceptors()) { - if (config.getInterceptor(interceptor) == null) { + if (!config.getInterceptors().containsKey(interceptor)) { throw new HttpException(BAD_REQUEST, "Unknown interceptor: " + interceptor); } } diff --git a/server/src/main/java/com/epam/aidial/core/server/function/CollectResponseAttachmentsFn.java b/server/src/main/java/com/epam/aidial/core/server/function/CollectResponseAttachmentsFn.java index 44f9e5378..2ec5afc39 100644 --- a/server/src/main/java/com/epam/aidial/core/server/function/CollectResponseAttachmentsFn.java +++ b/server/src/main/java/com/epam/aidial/core/server/function/CollectResponseAttachmentsFn.java @@ -77,7 +77,7 @@ private void processAttachedFile(String url, Map return; } String sourceDeployment = context.getApiKeyData().getSourceDeployment(); - if (context.getConfig().getInterceptor(sourceDeployment) != null) { + if (context.getConfig().getInterceptors().containsKey(sourceDeployment)) { // Note. permission check: make sure that the target deployment has access to the resource only // we don't check other permissions like admin, share or publishing access since we give full permissions to the source deployment Map> result = AccessService.getAppResourceAccess(Set.of(resource), diff --git a/server/src/main/java/com/epam/aidial/core/server/limiter/RateLimiter.java b/server/src/main/java/com/epam/aidial/core/server/limiter/RateLimiter.java index 7a6cdf7b7..011126d4f 100644 --- a/server/src/main/java/com/epam/aidial/core/server/limiter/RateLimiter.java +++ b/server/src/main/java/com/epam/aidial/core/server/limiter/RateLimiter.java @@ -1,7 +1,7 @@ package com.epam.aidial.core.server.limiter; -import com.epam.aidial.core.config.Config; import com.epam.aidial.core.config.CostLimit; +import com.epam.aidial.core.config.Deployment; import com.epam.aidial.core.config.Limit; import com.epam.aidial.core.config.Role; import com.epam.aidial.core.config.RoleBasedEntity; @@ -325,14 +325,14 @@ private Limit getLimitByUser(ProxyContext context, RoleBasedEntity roleBasedEnti // find limits for user roles which match to required roles userRoles = context.getUserRoles().stream().filter(role -> roleBasedEntity.getUserRoles().contains(role)).toList(); } - Config config = context.getConfig(); - Limit defaultUserLimit = getLimit(config, DEFAULT_USER_ROLE, name, DEFAULT_LIMIT); + Map roles = context.getConfig().getRoles(); + Limit defaultUserLimit = getLimit(roles, DEFAULT_USER_ROLE, name, DEFAULT_LIMIT); if (userRoles.isEmpty()) { return defaultUserLimit; } Limit limit = null; for (String userRole : userRoles) { - Limit candidate = getLimit(config, userRole, name, null); + Limit candidate = getLimit(roles, userRole, name, null); if (candidate != null) { if (limit == null) { limit = new Limit(); @@ -357,14 +357,14 @@ private Limit getLimitByUser(ProxyContext context, RoleBasedEntity roleBasedEnti private CostLimit getCostLimitByUser(ProxyContext context) { List userRoles = context.getUserRoles(); - Config config = context.getConfig(); - CostLimit defaultUserCostLimit = getCostLimit(config, DEFAULT_USER_ROLE, DEFAULT_COST_LIMIT); + Map roles = context.getConfig().getRoles(); + CostLimit defaultUserCostLimit = getCostLimit(roles, DEFAULT_USER_ROLE, DEFAULT_COST_LIMIT); if (userRoles.isEmpty()) { return defaultUserCostLimit; } CostLimit costLimit = null; for (String userRole : userRoles) { - CostLimit candidate = getCostLimit(config, userRole, null); + CostLimit candidate = getCostLimit(roles, userRole, null); if (candidate != null) { if (costLimit == null) { costLimit = new CostLimit(); @@ -396,29 +396,14 @@ private static String getPathToCosts() { return "costs"; } - private static Limit getLimit(Config config, String userRole, String name, Limit defaultLimit) { - Role role = config.getRole(userRole); - if (role == null) { - return defaultLimit; - } - Limit limit = role.getLimits().get(name); - if (limit != null) { - return limit; - } - // Compat: a Role.limits entry may still be keyed by the canonical id a previously-shipped - // build exposed for API-managed deployments (e.g. "models/platform/gpt-4") before - // deployment.getName() reverted to the short name. Fall back to a last-segment match so - // those pre-existing entries keep resolving after upgrade. - for (Map.Entry entry : role.getLimits().entrySet()) { - if (entry.getKey().endsWith("/" + name)) { - return entry.getValue(); - } - } - return defaultLimit; + private static Limit getLimit(Map roles, String userRole, String name, Limit defaultLimit) { + return Optional.ofNullable(roles.get(userRole)) + .map(role -> role.getLimits().get(name)) + .orElse(defaultLimit); } - private static CostLimit getCostLimit(Config config, String userRole, CostLimit defaultCostLimit) { - return Optional.ofNullable(config.getRole(userRole)) + private static CostLimit getCostLimit(Map roles, String userRole, CostLimit defaultCostLimit) { + return Optional.ofNullable(roles.get(userRole)) .map(Role::getCostLimit) .orElse(defaultCostLimit); } diff --git a/server/src/main/java/com/epam/aidial/core/server/log/AnalyticsLogContext.java b/server/src/main/java/com/epam/aidial/core/server/log/AnalyticsLogContext.java index e3703076d..b860bfb10 100644 --- a/server/src/main/java/com/epam/aidial/core/server/log/AnalyticsLogContext.java +++ b/server/src/main/java/com/epam/aidial/core/server/log/AnalyticsLogContext.java @@ -26,8 +26,6 @@ import java.util.Scanner; import javax.annotation.Nullable; -import static com.epam.aidial.core.server.util.PlatformCanonicalIdUtil.lastSegment; - @Slf4j @Getter @Builder @@ -150,11 +148,7 @@ public static String getParentDeployment(String sourceDeployment, List i int i = executionPath.size() - 2; for (int j = interceptors.size() - 1; i >= 0 && j >= 0; i--, j--) { String deployment = executionPath.get(i); - // executionPath entries are always the resolved deployment.getName() (short name); - // interceptors holds the raw config reference, which may be a short name or a - // canonical id (e.g. "interceptors/platform/my-interceptor") - normalize before - // comparing so a canonical-id reference doesn't look like a path mismatch. - String interceptor = lastSegment(interceptors.get(j)); + String interceptor = interceptors.get(j); if (!deployment.equals(interceptor)) { log.warn("Can't find parent deployment because interceptor path doesn't match: expected - {}, actual - {}", interceptor, deployment); return null; diff --git a/server/src/main/java/com/epam/aidial/core/server/service/ConsentService.java b/server/src/main/java/com/epam/aidial/core/server/service/ConsentService.java index 38d705558..fc11e1a27 100644 --- a/server/src/main/java/com/epam/aidial/core/server/service/ConsentService.java +++ b/server/src/main/java/com/epam/aidial/core/server/service/ConsentService.java @@ -7,7 +7,6 @@ import com.epam.aidial.core.server.util.BucketBuilder; import com.epam.aidial.core.server.util.ProxyUtil; import com.epam.aidial.core.server.util.ResourceDescriptorFactory; -import com.epam.aidial.core.storage.exception.ResourceNotFoundException; import com.epam.aidial.core.storage.resource.ResourceDescriptor; import com.epam.aidial.core.storage.resource.ResourceTypes; import com.epam.aidial.core.storage.service.ResourceService; @@ -17,12 +16,9 @@ import java.util.ArrayDeque; import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Set; -import static com.epam.aidial.core.server.util.PlatformCanonicalIdUtil.lastSegment; - @Slf4j public class ConsentService { @@ -91,37 +87,17 @@ public void verifyUserConsent(ProxyContext context, Deployment deployment) { } if (executionPath != null) { for (String dep : executionPath) { - if (findConsentDeployment(consent, dep) == null) { + if (!consent.getDeployments().containsKey(dep)) { fail(currentDeploymentId); } } } - Consent.Deployment consentDeployment = findConsentDeployment(consent, currentDeploymentId); + Consent.Deployment consentDeployment = consent.getDeployments().get(currentDeploymentId); if (consentDeployment == null || !consentDeployment.isConsentRequired()) { fail(currentDeploymentId); } } - /** - * Looks up a deployment's consent record by id, falling back to a last-segment match when the - * exact key misses. Compat for consent records persisted before deployment.getName() reverted - * from canonical id (e.g. "models/platform/gpt-4") to short name for API-managed deployments — - * without this, every such record would silently fail re-consent after upgrade. - */ - private static Consent.Deployment findConsentDeployment(Consent consent, String id) { - Consent.Deployment exact = consent.getDeployments().get(id); - if (exact != null) { - return exact; - } - String shortId = lastSegment(id); - for (Map.Entry entry : consent.getDeployments().entrySet()) { - if (lastSegment(entry.getKey()).equals(shortId)) { - return entry.getValue(); - } - } - return null; - } - private String getRootDeploymentId(ProxyContext context, Deployment current) { if (context.getApiKeyData().getPerRequestKey() == null) { return current.getName(); diff --git a/server/src/main/java/com/epam/aidial/core/server/service/ShareService.java b/server/src/main/java/com/epam/aidial/core/server/service/ShareService.java index e31ae91f7..599889af0 100644 --- a/server/src/main/java/com/epam/aidial/core/server/service/ShareService.java +++ b/server/src/main/java/com/epam/aidial/core/server/service/ShareService.java @@ -1,7 +1,6 @@ package com.epam.aidial.core.server.service; import com.epam.aidial.core.config.Application; -import com.epam.aidial.core.config.Config; import com.epam.aidial.core.config.CredentialsLevel; import com.epam.aidial.core.config.ResourceAccessType; import com.epam.aidial.core.config.Role; @@ -293,11 +292,11 @@ private void updateLimits(ProxyContext context, ResourceType resourceType, Share private ShareResourceLimit getLimit(ProxyContext context, ResourceType resourceType) { List userRoles = context.getUserRoles(); - Config config = context.getConfig(); + Map roles = context.getConfig().getRoles(); ShareResourceLimit defaultLimit = DEFAULT_LIMITS.get(resourceType); ShareResourceLimit limit = null; for (String userRole : userRoles) { - ShareResourceLimit candidate = Optional.ofNullable(config.getRole(userRole)).map(Role::getShare).map(limits -> limits.get(resourceType.name())).orElse(null); + ShareResourceLimit candidate = Optional.ofNullable(roles.get(userRole)).map(Role::getShare).map(limits -> limits.get(resourceType.name())).orElse(null); if (candidate != null) { if (limit == null) { limit = new ShareResourceLimit(candidate.getMaxAcceptedUsers(), candidate.getInvitationTtl()); diff --git a/server/src/main/java/com/epam/aidial/core/server/util/PlatformCanonicalIdUtil.java b/server/src/main/java/com/epam/aidial/core/server/util/PlatformCanonicalIdUtil.java index a4f579b36..f0090cfdc 100644 --- a/server/src/main/java/com/epam/aidial/core/server/util/PlatformCanonicalIdUtil.java +++ b/server/src/main/java/com/epam/aidial/core/server/util/PlatformCanonicalIdUtil.java @@ -1,9 +1,7 @@ package com.epam.aidial.core.server.util; /** - * Shared helper for the canonical-id ↔ short-name relationship used across config loading - * ({@code ConfigPostProcessor}, {@code MergedConfigStore}), consent records ({@code ConsentService}), - * and analytics logging ({@code AnalyticsLogContext}). Applies only to {@code platform}-bucket + * Shared helper for the canonical-id ↔ short-name relationship. Applies only to {@code platform}-bucket * canonical ids, which are always exactly {@code type/platform/name} with no further nesting * (enforced at the write boundary — see {@code ConfigResourceController.ENTITY_NAME_PATTERN}), so * the last path segment is unambiguously the short name. A bare (slash-free) name is already its diff --git a/server/src/test/java/com/epam/aidial/core/server/CanonicalIdListingTest.java b/server/src/test/java/com/epam/aidial/core/server/CanonicalIdListingTest.java index 51219dcaa..9ea16cc4c 100644 --- a/server/src/test/java/com/epam/aidial/core/server/CanonicalIdListingTest.java +++ b/server/src/test/java/com/epam/aidial/core/server/CanonicalIdListingTest.java @@ -10,10 +10,9 @@ * HTTP integration tests locking the short-name-addressing contract: the {@code id}/{@code model} * fields on the legacy {@code /openai/models} and {@code /openai/deployments} listings surface * the short name (last path segment) for API-managed entries, matching file-sourced entries — - * never the canonical id. The admin Configuration API ({@code /v1/{type}/{bucket}/...}) GET + - * listing projection is unaffected: it independently projects the canonical ID (map key) - * regardless of entity name, so operators can still copy-paste the identifier verbatim into - * per-entity URLs. + * never the canonical id. The admin Configuration API ({@code /v1/{type}/{bucket}/...}) GET is + * blob-storage-backed directly (not the in-memory map), and also projects the short name — the + * URL's own {@code name} segment — for the same reason. */ public class CanonicalIdListingTest extends ResourceBaseTest { @@ -64,17 +63,17 @@ void testFileSourcedModelStillSurfacedAsSimpleName() { } @Test - void testApiManagedModelAdminGetProjectsCanonicalId() { - // Polish.1 (2026-05-08): admin GET projects the canonical ID for API-managed entries so - // operators can copy-paste the identifier verbatim. File-sourced entries keep their simple - // name. Under U.0 the per-entity GET still projects the canonical ID. + void testApiManagedModelAdminGetProjectsShortName() { + // Admin GET reads blob storage directly by descriptor (not the in-memory map — see + // short-name-keyed-config-maps.md), so it projects the URL's own short-name segment, + // matching how the entity is keyed in Config for runtime resolution. verify(send(HttpMethod.PUT, "/v1/models/platform/admin-listing-projection", null, API_MODEL_BODY, "authorization", "admin", "If-None-Match", "*"), 200); Response single = send(HttpMethod.GET, "/v1/models/platform/admin-listing-projection", null, "", "authorization", "admin"); verify(single, 200); - assertTrue(single.body().contains("\"name\":\"models/platform/admin-listing-projection\""), - () -> "Admin GET must project canonical ID for API entries: " + single.body()); + assertTrue(single.body().contains("\"name\":\"admin-listing-projection\""), + () -> "Admin GET must project the short name for API entries: " + single.body()); } } diff --git a/server/src/test/java/com/epam/aidial/core/server/MergedConfigStoreApiTest.java b/server/src/test/java/com/epam/aidial/core/server/MergedConfigStoreApiTest.java index 316bc08eb..0481ab940 100644 --- a/server/src/test/java/com/epam/aidial/core/server/MergedConfigStoreApiTest.java +++ b/server/src/test/java/com/epam/aidial/core/server/MergedConfigStoreApiTest.java @@ -72,21 +72,20 @@ void testBlobModelSurfacesAfterReload() { assertEquals(200, reload.status()); Config merged = dial.getProxy().getConfigStore().get(); - Model blobModel = merged.getModels().get("models/platform/" + blobName); - assertNotNull(blobModel, () -> "Expected canonical-ID key in merged Config: " + merged.getModels().keySet()); - // Model.name carries the short name for API-managed entries too, so legacy /openai/models, - // /openai/deployments, and rate-limit role-limit lookups see the same short form file - // entries already use. The admin Configuration API GET / listing projection is unaffected - // — it independently projects the canonical ID (map key). - assertEquals(blobName, blobModel.getName(), - "Entity.name carries the short name for API-managed entries"); - assertNotNull(merged.getModels().get("test-model-v1"), "File model must still coexist by simple name"); - + Model blobModel = merged.getModels().get(blobName); + assertNotNull(blobModel, () -> "Expected short-name key in merged Config: " + merged.getModels().keySet()); + // Model.name is the map key — the short name, the same key a file entry for this + // logical entity would use. + assertEquals(blobName, blobModel.getName(), "Entity.name is the short name for API-managed entries too"); + assertNotNull(merged.getModels().get("test-model-v1"), "File model must still coexist by its own short name"); + + // Admin GET reads blob storage directly (not the in-memory map), so the projected name is + // whatever the URL's short-name segment was. Response get = send(HttpMethod.GET, "/v1/models/platform/" + blobName, null, "", "authorization", "admin"); verify(get, 200); - assertTrue(get.body().contains("\"name\":\"models/platform/" + blobName + "\""), - () -> "Expected canonical name in projection: " + get.body()); + assertTrue(get.body().contains("\"name\":\"" + blobName + "\""), + () -> "Expected short name in projection: " + get.body()); assertTrue(get.body().contains("\"endpoint\""), () -> "Expected endpoint field in projection: " + get.body()); } @@ -106,21 +105,19 @@ void testBlobInterceptorSurfacesAfterReload() { assertEquals(200, reload.status()); Config merged = dial.getProxy().getConfigStore().get(); - Interceptor blob = merged.getInterceptors().get("interceptors/platform/" + blobName); - assertNotNull(blob, () -> "Expected canonical-ID key in merged Config: " + merged.getInterceptors().keySet()); - // API-managed entries carry the short name (last path segment), not the canonical ID. + Interceptor blob = merged.getInterceptors().get(blobName); + assertNotNull(blob, () -> "Expected short-name key in merged Config: " + merged.getInterceptors().keySet()); assertEquals(blobName, blob.getName()); assertNotNull(merged.getInterceptors().get("interceptor1"), "File interceptor must still coexist"); } @Test - void testBlobModelShadowsFileEntryByShortNameAfterReload() { - // A blob model written under the SAME short name as an existing file-sourced model must - // replace it in the merged Config, not coexist alongside it: the file entry keyed by the - // bare short name is removed, and resolving that short name (verbatim or via getModel) - // hits the blob entity, which is authoritative once migrated. + void testBlobModelOverwritesFileEntryAtSameShortNameAfterReload() { + // A blob model written under the SAME short name as an existing file-sourced model shares + // that single map key with it — both sources key by short name uniformly, so the blob + // write simply overwrites the slot in place; there's no separate canonical-id slot left + // over to shadow. String shortName = "test-model-v1"; - String canonicalId = "models/platform/" + shortName; String body = """ { "type": "chat", @@ -135,17 +132,17 @@ void testBlobModelShadowsFileEntryByShortNameAfterReload() { assertEquals(200, reload.status()); Config merged = dial.getProxy().getConfigStore().get(); - Model blobModel = merged.getModels().get(canonicalId); - assertNotNull(blobModel, () -> "Expected canonical-ID key in merged Config: " + merged.getModels().keySet()); - assertNull(merged.getModels().get(shortName), - () -> "File entry must be shadowed by the migrated blob entity: " + merged.getModels().keySet()); - assertEquals(blobModel, merged.getModel(shortName), "getModel must resolve the short name to the blob entity"); + Model model = merged.getModels().get(shortName); + assertNotNull(model); + assertEquals("http://localhost:7001/openai/deployments/migrated/chat/completions", model.getEndpoint(), + "Blob entity must win over the file entry sharing its short name"); + assertEquals(model, merged.selectDeployment(shortName), + "selectDeployment must resolve the short name directly to the blob entity"); } @Test - void testBlobInterceptorShadowsFileEntryByShortNameAfterReload() { + void testBlobInterceptorOverwritesFileEntryAtSameShortNameAfterReload() { String shortName = "interceptor1"; - String canonicalId = "interceptors/platform/" + shortName; String body = """ { "endpoint": "http://localhost:9000/migrated-intercept" @@ -158,18 +155,15 @@ void testBlobInterceptorShadowsFileEntryByShortNameAfterReload() { assertEquals(200, reload.status()); Config merged = dial.getProxy().getConfigStore().get(); - Interceptor blob = merged.getInterceptors().get(canonicalId); - assertNotNull(blob, () -> "Expected canonical-ID key in merged Config: " + merged.getInterceptors().keySet()); - assertNull(merged.getInterceptors().get(shortName), - () -> "File entry must be shadowed by the migrated blob entity: " + merged.getInterceptors().keySet()); - assertEquals(blob, merged.getInterceptor(shortName), - "getInterceptor must resolve the short name to the blob entity"); + Interceptor interceptor = merged.getInterceptors().get(shortName); + assertNotNull(interceptor); + assertEquals("http://localhost:9000/migrated-intercept", interceptor.getEndpoint(), + "Blob entity must win over the file entry sharing its short name"); } @Test - void testBlobRoleShadowsFileEntryByShortNameAfterReload() { + void testBlobRoleOverwritesFileEntryAtSameShortNameAfterReload() { String shortName = "default"; - String canonicalId = "roles/platform/" + shortName; String body = """ { "limits": {} @@ -182,18 +176,13 @@ void testBlobRoleShadowsFileEntryByShortNameAfterReload() { assertEquals(200, reload.status()); Config merged = dial.getProxy().getConfigStore().get(); - assertNotNull(merged.getRoles().get(canonicalId), - () -> "Expected canonical-ID key in merged Config: " + merged.getRoles().keySet()); - assertNull(merged.getRoles().get(shortName), - () -> "File entry must be shadowed by the migrated blob entity: " + merged.getRoles().keySet()); - assertEquals(merged.getRoles().get(canonicalId), merged.getRole(shortName), - "getRole must resolve the short name to the blob entity"); + assertNotNull(merged.getRoles().get(shortName)); + assertEquals(shortName, merged.getRoles().get(shortName).getName()); } @Test - void testBlobApplicationShadowsFileEntryByShortNameAfterReload() { + void testBlobApplicationOverwritesFileEntryAtSameShortNameAfterReload() { String shortName = "app"; - String canonicalId = "applications/platform/" + shortName; String body = """ { "endpoint": "http://application1/v1/completions", @@ -207,18 +196,14 @@ void testBlobApplicationShadowsFileEntryByShortNameAfterReload() { assertEquals(200, reload.status()); Config merged = dial.getProxy().getConfigStore().get(); - assertNotNull(merged.getApplications().get(canonicalId), - () -> "Expected canonical-ID key in merged Config: " + merged.getApplications().keySet()); - assertNull(merged.getApplications().get(shortName), - () -> "File entry must be shadowed by the migrated blob entity: " + merged.getApplications().keySet()); - assertEquals(merged.getApplications().get(canonicalId), merged.selectDeployment(shortName), - "selectDeployment must resolve the short name to the blob entity"); + assertNotNull(merged.getApplications().get(shortName)); + assertEquals(merged.getApplications().get(shortName), merged.selectDeployment(shortName), + "selectDeployment must resolve the short name directly to the blob entity"); } @Test - void testBlobToolSetShadowsFileEntryByShortNameAfterReload() { + void testBlobToolSetOverwritesFileEntryAtSameShortNameAfterReload() { String shortName = "git"; - String canonicalId = "toolsets/platform/" + shortName; String body = """ { "endpoint": "http://localhost:9876", @@ -233,12 +218,9 @@ void testBlobToolSetShadowsFileEntryByShortNameAfterReload() { assertEquals(200, reload.status()); Config merged = dial.getProxy().getConfigStore().get(); - assertNotNull(merged.getToolsets().get(canonicalId), - () -> "Expected canonical-ID key in merged Config: " + merged.getToolsets().keySet()); - assertNull(merged.getToolsets().get(shortName), - () -> "File entry must be shadowed by the migrated blob entity: " + merged.getToolsets().keySet()); - assertEquals(merged.getToolsets().get(canonicalId), merged.selectDeployment(shortName), - "selectDeployment must resolve the short name to the blob entity"); + assertNotNull(merged.getToolsets().get(shortName)); + assertEquals(merged.getToolsets().get(shortName), merged.selectDeployment(shortName), + "selectDeployment must resolve the short name directly to the blob entity"); } @Test diff --git a/server/src/test/java/com/epam/aidial/core/server/config/ConfigPostProcessorTest.java b/server/src/test/java/com/epam/aidial/core/server/config/ConfigPostProcessorTest.java index 9d0bc127d..255b42403 100644 --- a/server/src/test/java/com/epam/aidial/core/server/config/ConfigPostProcessorTest.java +++ b/server/src/test/java/com/epam/aidial/core/server/config/ConfigPostProcessorTest.java @@ -104,8 +104,7 @@ void testSemanticKeepsCanonicalIdKeyedToolSet() { ConfigPostProcessor.processSemantic(config, null, Map.of(), Map.of(), null); assertTrue(config.getToolsets().containsKey("toolsets/platform/my-toolset")); - // Name is the short name (last path segment), not the canonical id. - assertEquals("my-toolset", config.getToolsets().get("toolsets/platform/my-toolset").getName()); + assertEquals("toolsets/platform/my-toolset", config.getToolsets().get("toolsets/platform/my-toolset").getName()); } @Test diff --git a/server/src/test/java/com/epam/aidial/core/server/config/MergedConfigStorePartialUpdateTest.java b/server/src/test/java/com/epam/aidial/core/server/config/MergedConfigStorePartialUpdateTest.java index 0ccb3c871..aada17bb2 100644 --- a/server/src/test/java/com/epam/aidial/core/server/config/MergedConfigStorePartialUpdateTest.java +++ b/server/src/test/java/com/epam/aidial/core/server/config/MergedConfigStorePartialUpdateTest.java @@ -98,18 +98,20 @@ public void interceptorDeleteCascadesToModelInvalidEntities() { } @Test - public void cascadeClassifiesSourceByMapKeyShape() { - // Two models reference the same interceptor: one file-defined (bare simple-name key) and one - // API-defined (canonical-id key). Deleting the interceptor cross-ref-invalidates both; the - // recorded source must follow the key shape, mirroring the full rebuild() onSkip classifier. - Model fileModel = new Model(); - fileModel.setInterceptors(List.of(INTERCEPTOR_ID)); - Model apiModel = new Model(); - apiModel.setInterceptors(List.of(INTERCEPTOR_ID)); + public void cascadeClassifiesInvalidatedModelsAsApiSourced() { + // Models are keyed by short name uniformly now (file- and blob-sourced alike, see + // short-name-keyed-config-maps.md), so key shape can no longer tell them apart. The + // partial-update path never touches file config either way, so every survivor this + // cascade walks is classified "api" — regardless of which source the model itself + // originally came from. + Model firstModel = new Model(); + firstModel.setInterceptors(List.of(INTERCEPTOR_ID)); + Model secondModel = new Model(); + secondModel.setInterceptors(List.of(INTERCEPTOR_ID)); Config seeded = newConfig(); Map models = new LinkedHashMap<>(); - models.put("gpt-4-file", fileModel); // bare simple name → file-sourced - models.put(MODEL_ID, apiModel); // contains '/' → api-sourced + models.put("gpt-4-first", firstModel); + models.put("gpt-4-second", secondModel); seeded.setModels(models); seeded.setInterceptors(mutable(INTERCEPTOR_ID, new Interceptor())); MergedConfigStore store = initStore(seeded, MergedConfigStore.MODE_SKIP); @@ -119,13 +121,15 @@ public void cascadeClassifiesSourceByMapKeyShape() { Map invalidModels = store.getInvalidEntities().get(ResourceTypes.MODEL); assertEquals(2, invalidModels.size(), "both cross-ref-invalidated models recorded"); - InvalidEntityRecord fileRecord = invalidModels.get("models/platform/gpt-4-file"); - assertEquals("file", fileRecord.getSource(), "bare-key model attributed to file source"); - assertEquals("gpt-4-file", fileRecord.getSimpleName()); + // invalidEntities is always keyed by (derived) canonical id, regardless of source — + // unaffected by short-name keying, which only concerns Config's live entity maps. + InvalidEntityRecord firstRecord = invalidModels.get("models/platform/gpt-4-first"); + assertEquals("api", firstRecord.getSource()); + assertEquals("gpt-4-first", firstRecord.getSimpleName()); - InvalidEntityRecord apiRecord = invalidModels.get(MODEL_ID); - assertEquals("api", apiRecord.getSource(), "canonical-id-key model attributed to api source"); - assertEquals("gpt-4", apiRecord.getSimpleName()); + InvalidEntityRecord secondRecord = invalidModels.get("models/platform/gpt-4-second"); + assertEquals("api", secondRecord.getSource()); + assertEquals("gpt-4-second", secondRecord.getSimpleName()); } @Test diff --git a/server/src/test/java/com/epam/aidial/core/server/config/MergedConfigStoreReplicaUpdateTest.java b/server/src/test/java/com/epam/aidial/core/server/config/MergedConfigStoreReplicaUpdateTest.java index 85fe46866..a176756f0 100644 --- a/server/src/test/java/com/epam/aidial/core/server/config/MergedConfigStoreReplicaUpdateTest.java +++ b/server/src/test/java/com/epam/aidial/core/server/config/MergedConfigStoreReplicaUpdateTest.java @@ -370,7 +370,7 @@ public void modelCreateFetchesDecryptsAndAppliesEntity() { store.applyReplicaEvent(descriptor, ResourceEvent.Action.CREATE); - Model applied = store.get().getModels().get(MODEL_ID); + Model applied = store.get().getModels().get(descriptor.getName()); assertEquals("GPT-4", applied.getDisplayName().getPlainValue()); verify(secretFieldProcessor).decryptFields(any(Model.class), eq(descriptor)); verifyNoInteractions(apiKeyStore); diff --git a/server/src/test/java/com/epam/aidial/core/server/config/MergedConfigStoreTest.java b/server/src/test/java/com/epam/aidial/core/server/config/MergedConfigStoreTest.java index dc85db3b4..260922f11 100644 --- a/server/src/test/java/com/epam/aidial/core/server/config/MergedConfigStoreTest.java +++ b/server/src/test/java/com/epam/aidial/core/server/config/MergedConfigStoreTest.java @@ -124,7 +124,7 @@ vertx, taskExecutor, resourceService, apiKeyStore, new PlatformEntityLocationStr store.init(fileConfigStore); Config config = store.get(); - Application materialized = config.getApplications().get("applications/platform/my-app"); + Application materialized = config.getApplications().get("my-app"); assertEquals("http://localhost/completions", materialized.getEndpoint()); verify(externalServiceService).decryptSecrets( argThat(d -> "platform".equals(d.getBucketName())), eq(materialized)); @@ -147,7 +147,7 @@ vertx, taskExecutor, resourceService, apiKeyStore, new PlatformEntityLocationStr store.init(fileConfigStore); Config config = store.get(); - ToolSet materialized = config.getToolsets().get("toolsets/platform/my-toolset"); + ToolSet materialized = config.getToolsets().get("my-toolset"); assertEquals("http://localhost:9876", materialized.getEndpoint()); verify(resourceAuthSettingsEncryptionService).decrypt( eq(toolSetDescriptor.getUrl()), any(BucketInfo.class), eq(materialized.getAuthSettings())); From f9f2b9493a662f2e7b8c81e670f5ad1027c91020 Mon Sep 17 00:00:00 2001 From: Kiryl_Kurnosenka Date: Thu, 13 Aug 2026 20:14:41 +0300 Subject: [PATCH 09/15] chore: untrack design docs, drop doc-filename references from comments Design docs under docs/design/ aren't meant to be committed to the repo; remove the three added in the previous commit from git tracking (kept on disk, untracked) and rephrase the code comments that referenced them by filename so they stand on their own. --- docs/design/github-updates-draft.md | 133 --------- docs/design/schema-id-atomic-derivation.md | 274 ------------------ docs/design/short-name-keyed-config-maps.md | 178 ------------ .../controller/ConfigResourceController.java | 18 +- .../core/server/CanonicalIdListingTest.java | 6 +- .../MergedConfigStorePartialUpdateTest.java | 9 +- 6 files changed, 16 insertions(+), 602 deletions(-) delete mode 100644 docs/design/github-updates-draft.md delete mode 100644 docs/design/schema-id-atomic-derivation.md delete mode 100644 docs/design/short-name-keyed-config-maps.md diff --git a/docs/design/github-updates-draft.md b/docs/design/github-updates-draft.md deleted file mode 100644 index 8a07087fd..000000000 --- a/docs/design/github-updates-draft.md +++ /dev/null @@ -1,133 +0,0 @@ -# Draft GitHub Updates — Review Before Pushing - -Drafted text for #1781, #1783, #1784, and PR #1813, reflecting `schema-id-atomic-derivation.md` -and `short-name-keyed-config-maps.md`. **Not pushed to GitHub yet** — for your review. The PR #1813 -draft assumes the code has actually been rewritten to match; don't paste it until that's true, or -it'll describe a PR that doesn't match its own diff. - ---- - -## Issue #1781 (epic) — suggested edits - -**Requirement table** — add a row/footnote under the existing table: - -> **Update:** the "canonical id keeps working as a harmless superset" property in the Inbound row -> no longer holds for models, applications, toolsets, interceptors, roles, or schemas. Once -> `Config`'s maps are keyed by each entity's natural identity (short name, or `$id` for schemas — -> see #1813's follow-up), a caller addressing an entity by its full canonical id -> (`models/platform/gpt-4`) instead of its short name (`gpt-4`) no longer resolves. This is an -> accepted, deliberate regression, not a residual to design around. - -**"Rejected alternatives" section** — replace the "Short-name-keyed `Config` maps" entry: - -> - ~~**Short-name-keyed `Config` maps.** Would avoid derivation but create a canonical-vs-short -> impedance mismatch with the deeply canonical CRUD / pub-sub / partial-update machinery. -> Rejected in favor of canonical-keyed maps + derivation.~~ -> **Reopened.** For models/applications/toolsets/interceptors/roles, the canonical-id-to-map-key -> transform is the same `lastSegment` string operation already computed today for outbound -> naming — not new per-type parsing logic — so the mismatch is narrower than originally assessed. -> Schemas need a bespoke version (body-JSON `$id` extraction) because their natural identity isn't -> a path segment; see `schema-id-atomic-derivation.md` for that case and -> `short-name-keyed-config-maps.md` for the generalization to the other five types. - ---- - -## Issue #1783 (Slice B+C) — suggested rewrite - -Replace **Part B** with: - -> ## Part B — short-name resolution (`Config` maps keyed by short name) -> -> `Config`'s maps for `applications`/`models`/`toolsets`/`interceptors`/`roles` are keyed by short -> name uniformly — file-sourced and blob-sourced entries for the same logical entity share the same -> map key. Canonical id (`{type}/platform/{shortName}`) is derived only where it's actually -> needed: the physical blob address, and the admin CRUD URL. It is never a `Config` map key. -> -> ### `config/.../Config.java` -> - `selectDeployment`, `getModel`, `getRole`, `getInterceptor` are plain `Map.get(shortName)` — no -> derivation step, no verbatim/canonical-id fallback. -> -> ### `server/.../config/ConfigPostProcessor.java` -> - `entity.setName(mapKey)` — the map key already is the short name. -> -> ### `server/.../config/MergedConfigStore.java` -> - Blob-sourced entities are inserted keyed by `lastSegment(canonicalId)`, not the raw canonical -> id — the same key a file entry for that entity already uses. Migrating a file entry to blob is -> an ordinary overwrite of that key; there's no separate "shadow the file entry" removal step. -> -> **Accepted regression:** a caller addressing an entity by canonical id instead of short name no -> longer resolves (see #1781's updated Requirement table). - -Replace **Part C** with: - -> ## Part C — schema `$id` resolution -> -> App-type and catalog schemas are keyed by `$id` in `Config`'s maps — both file- and blob-sourced -> entries, uniformly (file entries already work this way via -> `JsonArrayToSchemaMapDeserializer`). Canonical id for schemas is **derived** from `$id` -> (`schemas/platform/encode($id)` / `catalog_schemas/platform/encode($id)`), used only for the -> physical blob address and the admin CRUD URL — never a map key. No side index -> (`schemaAliasesById`/`catalogSchemaAliasesById`) is needed; `$id` collisions become structurally -> impossible (two schemas can't occupy the same derived blob path). See -> `schema-id-atomic-derivation.md` for the full design, including the non-splitting -> `ResourceDescriptorFactory` addition schemas need (their `$id` is a URI and legitimately contains -> `/`, unlike every other type's short name). - ---- - -## Issue #1784 (Slice D) — suggested edit - -Replace the **"Open item — schema migration naming"** section with: - -> ## Schema migration -> -> Resolved by `schema-id-atomic-derivation.md`: a migrated schema's canonical id is derived -> directly from its `$id` (`schemas/platform/encode($id)`), so there's no separate naming decision -> to make during migration — unlike models/applications/etc., where the file entry's key already -> is the short name to migrate to. - -No other change needed — the migration endpoint's behavior for models/applications/toolsets/ -interceptors/roles/schemas is otherwise unaffected (blob write path and address are unchanged; only -`Config`'s in-memory key changes, which Slice D's migration endpoint doesn't touch directly). - ---- - -## PR #1813 — suggested rewrite (only once the code matches this) - -> Makes every materialized `platform`-bucket config entity short-name (or, for schemas, `$id`) -> addressed by keying `Config`'s in-memory maps directly by that natural identity — not by -> canonical id — and deriving canonical id only where it's actually needed (the blob address, the -> admin CRUD URL). -> -> ### Applicable issues -> - fixes #1783 -> -> ### Description of changes -> - `Config.java`: `selectDeployment`/`getModel`/`getRole`/`getInterceptor` are plain -> `Map.get(shortName)`; `getCustomApplicationSchema`/`getCatalogSchema` are plain -> `Map.get($id)`. No derivation helper, no alias index. -> - `ConfigPostProcessor.java`: `entity.setName(mapKey)` directly. -> - `MergedConfigStore.java`: blob-sourced name-addressed entities are inserted keyed by -> `lastSegment(canonicalId)`; blob-sourced schemas are inserted keyed by their body's `$id` -> (extracted via the existing schema-body parse). No shadow-file-entry step, no schema alias -> index — a migrated entity's blob write is an ordinary overwrite of the same key its file -> predecessor used. -> - `ResourceDescriptorFactory.java`: new atomic (non-splitting) descriptor-factory method for -> schemas, whose `$id` is a URI and legitimately contains `/`. -> - Call-site sweep: unchanged from the original PR — deployment/role/interceptor resolution was -> already funneled through `Config`'s accessors. -> - Tests: [update to match whatever actually lands] -> -> ### Checklist -> - [X] Title of the pull request follows [Conventional Commits specification] - ---- - -## Recommended sequencing for pushing these - -1. Implement `schema-id-atomic-derivation.md` + `short-name-keyed-config-maps.md` on this PR's - branch (or a follow-up branch). -2. Once the diff matches, replace PR #1813's description with the draft above. -3. Update #1783 and #1784 to match (they're still open, no urgency conflict). -4. Update #1781 last, since it's the most "public" (epic) summary and should reflect the settled - state, not an in-flight one. diff --git a/docs/design/schema-id-atomic-derivation.md b/docs/design/schema-id-atomic-derivation.md deleted file mode 100644 index 1b321c462..000000000 --- a/docs/design/schema-id-atomic-derivation.md +++ /dev/null @@ -1,274 +0,0 @@ -# Schema `$id` Resolution: Key `Config` by `$id`, Derive the Canonical Id - -## Status - -Design proposal. Not implemented. Supersedes the alias-index design in `schema-id-derivation.md` -(itself already implemented — see `Config.applicationSchemaAliasesById`/`catalogSchemaAliasesById`, -`MergedConfigStore.putSchemaInPlace`/`recordSchemaAlias`) for `applicationTypeSchemas` and -`catalogSchemas` only. Nothing else in `expose-as-short-name.md` changes. - -This is the option that came out of re-examining Option A ("encode `$id` into the canonical id", -rejected in `schema-id-resolution-design-options.md`) after tracing the actual request flow. It -combines Option A's derivation (`canonicalId = "{type}/platform/" + encode($id)`) with Option D2's -idea of keying the in-memory map by `$id` — but avoids D2's O(n) canonical-id cost, because -derivation lets admin `GET`/`PUT`/`DELETE` decode the URL segment straight back to `$id` and do one -map lookup, instead of scanning. In the original doc's taxonomy this is a fifth point not -enumerated as its own option: **A's derivation + D2's keying, without D2's scan.** - -## Why revisit this - -The previously-shipped fix (`schema-id-derivation.md`) is real and correct, but it's a second data -structure (`*AliasesById`) with an eviction discipline someone has to remember (`putSchemaInPlace` -using `Map.put`'s return value) and three independent write-time collision checks -(`ConfigResourceController.handlePut`, `AdminApplyController.applySchema`, `.validateOnly`) that all -have to agree. This proposal removes the index and the collision checks entirely by making -collisions structurally impossible — the same free lunch models/apps/toolsets/interceptors get from -D7 (two entities can't share a short name because they can't occupy the same blob path). - -## What changes - -### S1 — `Config`'s schema maps are keyed by `$id`, uniformly - -Today `applicationTypeSchemas`/`catalogSchemas` mix keying: file entries are keyed by `$id` -(`JsonArrayToSchemaMapDeserializer` already does this — confirmed, no change needed there), blob -entries are keyed by **canonical id**, and a separate `*AliasesById` index bridges the two for `$id` -lookups. - -New shape: **both sources key by `$id`, always.** Canonical id stops being a map key for these two -types; it survives only as the admin-facing address (URL / blob path). - -```java -private static String resolveSchema(Map schemas, URI schemaId) { - return schemaId == null ? null : schemas.get(schemaId.toString()); -} -``` - -`schemaAliasesById`/`catalogSchemaAliasesById` fields, `recordSchemaAlias`, `putSchemaInPlace`'s -eviction logic, and `rejectSchemaIdCollision` (`ConfigResourceController.java:1584-1601`) are all -deleted — there's no longer an index to maintain or a collision to detect at write time (see S3). - -### S2 — Canonical id is derived, not admin-chosen - -``` -canonicalId = "schemas/platform/" + encode($id) // app-type schemas -canonicalId = "catalog_schemas/platform/" + encode($id) // catalog schemas -``` - -using the existing `UrlUtil.encodePathSegment` (`storage/.../util/UrlUtil.java:28-33`) — no new -encoder, and it never needs to be reachable from `config` (see S6). This directly resolves the two -open residuals in `expose-as-short-name.md`: schema migration no longer needs a naming decision (the -name **is** `encode($id)`), and `$id` uniqueness becomes storage-structural (D7-style) instead of -"admin error, matches today's file behavior." - -### S3 — Write-time reconciliation mirrors the `name` precedent - -Just as a model's PUT body disagrees with the URL and the stored `name` is silently overridden to -match the URL, a schema PUT whose body's `$id` disagrees with what the URL segment decodes to gets -its stored `$id` silently overwritten to `decode(urlSegment)` before persisting. This is a policy -choice, not a technical requirement — the docs flagged this as "genuinely hard" because `$id` is -`$ref`-able by other tooling, more consequential than a cosmetic `name` — but it's the same choice -already made for `name`, applied consistently, and it's what makes the map key (`$id`) and the -canonical id (`encode($id)`) provably agree after every write, with no separate check needed. - -### S4 — A new, non-splitting `ResourceDescriptorFactory` method for schemas only - -Confirmed from `ControllerSelector.configResourceController` (`:626-641`): the `{path}` URL segment -is decoded (`UrlUtil.decodePath`) **before** it reaches `ConfigResourceController`, matching -`fromDecoded`'s "url decoded relative path" contract. So whatever encoding scheme the canonical id -uses on the wire, `fromDecoded` receives the **decoded** `$id` — literal `/`s and all. `fromDecoded` -(`ResourceDescriptorFactory.java:52-59`) does `path.split("/")`, so a schema `$id` like -`https://dial.epam.com/catalog-schemas/model` would be chopped into five bogus path elements -(including an empty one from `//`) instead of treated as one atomic resource name. This is not -avoidable by choosing a better encoder — the decode happens generically upstream, for every -config-resource route, before schema-specific code ever runs. - -New method, used only by `descriptorFor(APP_TYPE_SCHEMA)`/`descriptorFor(CATALOG_SCHEMA)` -(`ConfigResourceController.java:1146-1148`): - -```java -/** - * Like {@link #fromDecoded}, but treats {@code decodedName} as a single atomic resource name — - * never splits it on '/'. For identifiers (like a JSON-Schema {@code $id}) that are themselves - * URIs and therefore expected to contain '/', not folder-hierarchy separators. - */ -public static ResourceDescriptor fromDecodedAtomicName(ResourceType type, String bucketName, - String bucketLocation, String decodedName) { - verify(bucketLocation.endsWith(PATH_SEPARATOR), "Bucket location must end with /"); - String physicalName = UrlUtil.encodePathSegment(decodedName); // keep the physical blob key flat — see S5 - ResourceDescriptor resource = from(type, bucketName, bucketLocation, List.of(physicalName), false); - verify(resource.getAbsoluteFilePath().getBytes(StandardCharsets.UTF_8).length <= MAX_PATH_SIZE, - "Resource path exceeds max allowed size: " + MAX_PATH_SIZE); - return resource; -} -``` - -(Exact placement/signature TBD during implementation — shown here to fix the shape of the fix: no -`split`, explicit length check reusing `fromEncoded`'s existing `MAX_PATH_SIZE = 900`.) - -### S5 — Double-encoding is accepted, not fixed, for `getUrl()` - -`ResourceDescriptor.name` is used two ways with nothing reconciling them: `getAbsoluteFilePath()` -(`:107+`) embeds it raw for the physical blob key; `getUrl()` (`:51-75`) encodes it once, -unconditionally, for the client-facing address. A schema's `$id` forces a choice between the two -being safe: - -- `name` = raw `$id` → clean, single-encoded `getUrl()`, but the **physical blob key** contains - literal `/`, indistinguishable from nesting to most blob backends (folder listing / GC / prefix - scans could misbehave). -- `name` = `encode($id)` (S4's approach) → flat, atomic physical key, but `getUrl()` encodes it a - second time (`%2F` → `%252F`). - -Take the second option — a flat blob key is the load-bearing property; a cosmetically double-encoded -admin URL is not (nothing round-trips DIAL's own emitted URL by hand; clients just echo it back). -Document this as a known, deliberate quirk on `ResourceDescriptor`/wherever schema descriptors are -built. Revisit only if it becomes a real complaint — e.g. by adding an "already encoded, don't -re-encode" flag to `ResourceDescriptor` — but that's out of scope for this pass. - -### S6 — No encoder needed in `config` - -`Config.resolveSchema` (S1) is a bare `Map.get`. `UrlUtil` stays exactly where it is -(`storage`, which `config` cannot depend on — confirmed via `storage/build.gradle`'s -`implementation project(':config')`, dependency runs one way). All encoding happens in `server` -(S2's write-time derivation, S4's descriptor factory), which already depends on `storage`. This is -the thing that makes this design cheaper than Option A as originally scoped — Option A needed the -encoder reachable from both write and read paths; here the read path (`Config`) needs no encoding at -all. - -### S7 — New pattern for schemas, validated structurally - -Don't reuse `ENTITY_NAME_PATTERN` (`^[A-Za-z0-9._%:-]+$`, `ConfigResourceController.java:86`) as-is — -confirmed it rejects some characters a correctly percent-encoded `$id` can legitimately contain -(`+`, `@`, `!`, `$`, `&`, `'`, `(`, `)`, `*`, `,`, `;`, `=` — all left unescaped by Guava's -`urlPathSegmentEscaper`, none in the current allowlist). Rather than widen the regex and hope it -matches the escaper's actual output space, validate structurally at the schema write path: reject -unless `encode(decode(pathSegment)) == pathSegment` — i.e. the segment round-trips as a validly -encoded string. This is stronger than any fixed character class and stays correct even if the -underlying escaper's exact character set changes. - -## Critical open decision: existing schemas already written under the old scheme - -`/v1/schemas/{bucket}/{path}` and `/v1/catalog_schemas/{bucket}/{path}` (`GET`/`PUT`/`DELETE`) are -**already implemented and shippable** today (confirmed: `ConfigResourceController`'s `saveSchema`/ -`getSchema`/`deleteSchema` operations, `descriptorFor`, `handleSchemaGet` all exist and use -`fromDecoded` with an admin-chosen canonical id, unrelated to `$id`). Any schema an admin has already -written lives at a canonical id that generally does **not** decode to its own `$id`. Under this -design, `Config`'s map is keyed by `$id`, and admin `GET`/`PUT`/`DELETE` by canonical id needs -`decode(urlSegment) == $id` to find it — which breaks for every schema written before this change. - -Three ways to handle it (pick one before implementing): - -1. **One-time migration, admin-triggered** (recommended — matches this codebase's existing pattern - for exactly this kind of transition, see `POST /v1/admin/config/file/migrate` in - `expose-as-short-name.md` §6). Add a step that reads every existing schema blob, computes its - correct new path (`schemas/platform/encode($id)`), writes it there, and deletes the old blob. - Simple, bounded, explicit, no dual-mode code to maintain afterward. -2. **Reject the redesign's premise for already-existing schemas** — keep them resolvable only via - their old canonical id (permanently), and only apply `$id`-keying to schemas created after this - ships. Avoids a migration step but means two schema addressing regimes coexist indefinitely, - which is exactly the kind of permanent special-casing this whole redesign line has been trying to - avoid elsewhere. -3. **Dual-mode transition window** — keep a legacy canonical-id-keyed fallback map alongside the new - `$id`-keyed one, drop the fallback after a deprecation period. More moving parts than (1) for a - supposedly-temporary need. - -This document proceeds assuming **(1)**. If schemas haven't actually been used in production yet -(worth confirming — this is a recently-added surface), this whole section may be moot and can be -dropped. - -## Implementation plan - -### 1. `config/.../Config.java` - -- Delete `applicationSchemaAliasesById`/`catalogSchemaAliasesById` fields and their getters/setters. -- Replace `resolveSchema`'s two-step lookup with the single `Map.get` in S1. -- No change to `getCustomApplicationSchema(URI)`/`getCatalogSchema(URI)` signatures. - -### 2. `config/.../databind/JsonArrayToSchemaMapDeserializer.java` - -No change — file schemas are already keyed by `$id` (confirmed). - -### 3. `server/.../util/ResourceDescriptorFactory.java` - -- Add `fromDecodedAtomicName` (S4): no `split("/")`, explicit `MAX_PATH_SIZE` check, encodes the - decoded name once internally before building the descriptor (S5). - -### 4. `server/.../config/MergedConfigStore.java` - -This is the biggest piece of surgery — `APP_TYPE_SCHEMA`/`CATALOG_SCHEMA` stop fitting the generic -"canonical id is both the map key and the blob address" pattern every other managed type uses -(the same invariant break the original design-options doc flagged for "D3"). - -- **`rebuild()`** (`:1100-1124`, `:1373-1383`): when scanning `platform` blobs, key - `schemas`/`catalogSchemas` insertion by `extractSchemaId(body)` instead of the blob's canonical id. - Delete `applicationSchemaAliasesById`/`catalogSchemaAliasesById` construction entirely (S1). -- **`peekEntity`** (switch around `:955-956`): for schema types, look up by `$id` (extracted from - the incoming body being validated) rather than by canonical id. -- **`putEntityInPlace`** (`:971-973`, `:990-1023`): replace `putSchemaInPlace`/`recordSchemaAlias` - with a schema-specific put that (a) computes `newId = extractSchemaId(newBody)`, (b) if an existing - entry's canonical id/blob address maps to a *different* `$id` currently in the map, removes that - old map entry (this is the one place eviction logic survives, but it's a single `remove` keyed by - the *old* `$id` — no index, no scan, since S3's override guarantees canonical id and `$id` agree - going forward), (c) inserts under `newId`. -- **`removeEntityInPlace`** (`:1046-1056`, `:1434-1435`): resolve the `$id` to remove from the - canonical id being deleted (decode it — S2's derivation makes this direct) rather than doing a - raw-map `.remove(canonicalId)`. -- **`deserializeReplicaEntity`, `cloneTypeMap`, `shallowClone`** (`:888-891`, `:935-940`): drop the - alias-map cloning (S1); schema map cloning itself is unchanged in shape (`Map`). - -### 5. `server/.../controller/ConfigResourceController.java` - -- **`descriptorFor`** (`:1134-1153`): route `APP_TYPE_SCHEMA`/`CATALOG_SCHEMA` through - `fromDecodedAtomicName` (S4) instead of `fromDecoded`. -- **`canonicalId()`** (`:1158`) / schema write path: apply S7's round-trip validation instead of - `ENTITY_NAME_PATTERN` for these two types. -- **`handleSchemaGet`** (`:1162-1207`): change `schemas.get(canonicalId())` to decode the canonical - id back to `$id` (S2's derivation, direct decode — no map involved) and look that up. -- **PUT path**: after parsing the body, apply S3's override (`node.set("$id", decode(pathSegment))` - when they disagree) before computing the physical write. Delete `rejectSchemaIdCollision` - (`:1584-1601`) — collisions are now structurally impossible (S2). -- Delete the `rejectSchemaIdCollision` call sites here and in `AdminApplyController`. - -### 6. `server/.../controller/AdminApplyController.java` - -- **`scratch` setup** (`:273-276`): drop `ApplicationSchemaAliasesById`/`CatalogSchemaAliasesById` - cloning (S1). -- **`mutateScratch`** (`:633-653`): replace `scratch.getApplicationTypeSchemas().put(entry.name(), json)` - + `recordSchemaAlias` with the same schema-specific put logic as `MergedConfigStore.putEntityInPlace` - (§4) — key by `$id`, not `entry.name()`. -- **Precheck/real-apply `"Schema"`/`"CatalogSchema"` cases** (`:358-369`, `:432-451`): drop the - `rejectSchemaIdCollision` calls; apply S3's override before persisting. - -### 7. `storage/.../util/UrlUtil.java` - -No change — `encodePathSegment`/`decodePath` already do exactly what S2/S4 need. - -## Testing - -- **`ConfigTest`**: `resolveSchema` — `$id`-keyed hit (file or blob entry), miss; confirm it no - longer needs a canonical-id-shaped input to work at all. -- **`MergedConfigStoreTest`**: rebuild keys blob schemas by `$id`, not canonical id; updating a - schema's own `$id` in place evicts the old key and inserts the new one (single `remove`, no - scan); a schema's canonical id always decodes back to its own `$id` after any write (S3's - invariant); deleting resolves and removes the right `$id` entry. -- **`ResourceDescriptorFactoryTest`**: `fromDecodedAtomicName` — a `$id` containing `/` produces one - atomic resource (no `parentFolders`), a path exceeding `MAX_PATH_SIZE` is rejected, output is used - consistently for both `getAbsoluteFilePath()` (flat) and `getUrl()` (documented double-encoded). -- **`ConfigResourceControllerTest`/integration**: PUT with a body `$id` disagreeing with the URL - segment succeeds and the stored body's `$id` is silently corrected (S3); PUT/GET/DELETE by the - derived canonical id round-trips; two different schemas can no longer collide on `$id` because - they physically cannot share a blob path (replaces the old 409-based `AdminApplyApiTest` - assertions — collisions now fail as an ordinary "resource already exists at a different identity" - case, not a dedicated check). -- **`AdminApplyApiTest`**: batch apply of two `Schema` entries with the same `$id` — confirm the - outcome (still an error, just surfaced differently now that there's no dedicated collision check). -- **Migration test** (if S6's decision (1) is taken): a schema written under the old scheme is - migrated to `schemas/platform/encode($id)`, old blob removed, resolves correctly afterward by both - its `$id` and its new canonical id. - -## Sequencing - -This is independent of Slices A/B/D in `expose-as-short-name.md` (schemas were always their own -Slice C) but is **not** independently shippable if any schemas already exist in `platform` — the -migration step (see "Critical open decision") must land in the same release, or immediately before, -this change goes live; otherwise existing schemas become unreachable by canonical id the moment this -deploys. diff --git a/docs/design/short-name-keyed-config-maps.md b/docs/design/short-name-keyed-config-maps.md deleted file mode 100644 index 262fd0e42..000000000 --- a/docs/design/short-name-keyed-config-maps.md +++ /dev/null @@ -1,178 +0,0 @@ -# Generalizing `$id`-Style Keying: `Config` Maps Keyed by Short Name, Canonical Id Purely Derived - -## Status - -Design proposal. Not implemented. Extends `schema-id-atomic-derivation.md`'s core idea (key -`Config`'s maps by the entity's natural identity; derive canonical id, don't store it as the map -key) from schemas to the five other short-name-addressed, `platform`-materialized types: **models, -applications, toolsets, interceptors, roles**. `keys`, `routes`, and `settings` are unaffected — -they were never name-addressed (see the "Coverage by entity type" table in -`expose-as-short-name.md`). - -This **reverses** an alternative explicitly rejected in the epic (issue #1781, "Rejected -alternatives"): *"Short-name-keyed `Config` maps. Would avoid derivation but create a -canonical-vs-short impedance mismatch with the deeply canonical CRUD / pub-sub / partial-update -machinery. Rejected in favor of canonical-keyed maps + derivation."* Reopening it is deliberate — -see "Why the earlier rejection doesn't apply the same way" below — not an oversight. - -Supersedes, for these five types: D2 (`Config.resolve`) and D4 (blob-shadows-file) in -`expose-as-short-name.md`; the corresponding implementation already shipped on this branch/in PR -#1813 (`Config.resolve`/`selectDeployment`/`getModel`/`getRole`/`getInterceptor`, -`Config.java:77-159`; `MergedConfigStore.shadowFileEntry`, `:1409-1415`; the `lastSegment(...)` -outbound-naming calls throughout `ConfigPostProcessor.java`). - -## The idea, generalized - -For models/applications/toolsets/interceptors/roles, canonical id is *already* a pure, derivable -function of short name: `canonicalId = "{type}/platform/" + shortName`. Unlike a schema's `$id` -(an opaque URI that needs an atomic-descriptor workaround because it contains `/`), a short name is -already a valid, slash-free path segment — nothing new needs inventing to make it a map key. - -So: key blob entries in `Config`'s maps by **short name directly** — the same key a file-sourced -entry for the same logical entity already uses — instead of by canonical id. Consequences: - -- **`Config.resolve` (`:159`) disappears.** It becomes a plain `map.get(shortName)` — there's no - "verbatim, then derived" two-step to perform, because there's only ever one key shape now. - `selectDeployment`, `getModel`, `getRole`, `getInterceptor` all collapse to direct `Map.get`. -- **`MergedConfigStore.shadowFileEntry` (`:1409-1415`) disappears.** There's nothing to shadow — a - file entry and its migrated blob counterpart share the same map key, so writing the blob entry - during rebuild is an ordinary overwrite of that key, not an additional removal step elsewhere. -- **The `lastSegment(...)` outbound-naming calls throughout `ConfigPostProcessor.java` (currently - ~10 call sites: `:131,154,165,177,188,253,324,378,397,437,461,470`) revert to `entity.setName(mapKey)`.** - The map key already *is* the short name; there's no canonical form to shorten. -- `PlatformCanonicalIdUtil.lastSegment` itself doesn't disappear — it's still needed wherever code - goes the other direction (derives the *storage* canonical id/blob path from a short name for - writes), just not for keying or outbound naming anymore. - -Net: this is a genuine deletion of code #1813 added, not a lateral move — `resolve`'s two-step -lookup, `shadowFileEntry`, and the ten-plus `lastSegment(...)`-for-naming call sites all go away for -these five types, leaving the map key, the blob address derivation (short name → canonical id, for -writes and for admin CRUD), and the outbound name in permanent agreement by construction, the same -way S1-S3 achieve for schemas in `schema-id-atomic-derivation.md`. - -## Why the earlier rejection (#1781) doesn't apply the same way here - -The epic's stated reason — "impedance mismatch with the deeply canonical CRUD / pub-sub / -partial-update machinery" — is real, and it's exactly what makes the schema version of this change -the larger lift in `schema-id-atomic-derivation.md` (`MergedConfigStore.rebuild`/`peekEntity`/ -`putEntityInPlace`/`removeEntityInPlace` all currently assume "canonical id is both the blob address -and the map key," and schemas need bespoke per-call-site logic to break that assumption cleanly, -since deriving a schema's map key from its canonical id requires parsing the JSON body for `$id`). - -For these five types, the mismatch is narrower, because the transform is not type-specific -content-parsing — it's the same string operation (`lastSegment`) `MergedConfigStore` and -`ConfigPostProcessor` already compute today, just applied one layer earlier (as the map key at -write/rebuild time, not only as the outbound `name` after the fact). There's no new per-type parsing -logic to write; every managed-type call site in `MergedConfigStore` that currently does -`map.put(canonicalId, entity)` for these five types does `map.put(lastSegment(canonicalId), entity)` -instead, uniformly. Worth flagging explicitly to reviewers that this reopens a recorded decision, -with this narrower-mismatch argument as the justification — not silently reversing it. - -## Accepted regression: canonical-id-shaped inbound resolution breaks - -Per direction received: **explicitly accepted, not designed around.** Once these five types are -keyed by short name only, any caller — or any stored reference — that addresses a `platform`-bucket -entity by its full canonical id (`models/platform/gpt-4`) instead of its short name (`gpt-4`) stops -resolving. This affects, at minimum: - -- Entities natively created via the `platform`-bucket CRUD API *before* short-name resolution - (#1783/PR #1813) shipped, if anything still holds a reference to them by canonical id. -- The "canonical id keeps working as a harmless superset" property that #1781/#1783 explicitly - designed in (Requirement table, row 1) — this document removes that property for these five types, - matching the same accepted tradeoff already made for schemas (where nothing ever relied on - canonical-id-shaped resolution in the first place). - -No compatibility shim, dual-mode lookup, or migration bridge is planned for this. If a canonical-id -reference needs to keep working somewhere specific, that needs to be raised as an exception before -implementation — not discovered afterward. - -## Implementation plan - -### 1. `config/.../Config.java` - -- Delete `resolve` (`:159`). Change `selectDeployment` (`:77-96`), `getModel` (`:101-104`), - `getRole` (`:106-109`), `getInterceptor` (`:111-113`) to plain `Map.get(id)` on - `applications`/`models`/`toolsets`/`interceptors`/`roles`. -- No change to map field types (`Map` etc.) — only what's used as the key changes, - at the write/rebuild side (§3), not here. - -### 2. `server/.../config/ConfigPostProcessor.java` - -- Revert every `entity.setName(lastSegment(...))` call (the ~10 sites listed above) to - `entity.setName(mapKey)` (or the already-short key directly, depending on the call site's local - variable naming). -- `validateCrossReferences`'s resolve-aware fix (already landed, docblock at `:278`) stays — - irrelevant to this change; it was about lookup semantics, not the map key. -- `deploymentIds`/de-duplication logic at `:437,461,470` that currently reasons about - `lastSegment(name)` vs. raw map key can simplify back to comparing map keys directly (they're now - always short names). - -### 3. `server/.../config/MergedConfigStore.java` - -- Delete `shadowFileEntry` (`:1409-1415`) and its call site (`:1200-1205`) and the surrounding - comments describing the shadow mechanism (`:982-987`, `:1034`, `:1112`, `:1140`). -- Every place that currently inserts a blob-sourced entity into `Config`'s maps keyed by its - canonical id, for these five types, keys by `lastSegment(canonicalId)` instead — this is the one - place `lastSegment` is still needed, moved from "outbound naming after the fact" to "the map key, - from the start." Audit `rebuild()`, `putEntityInPlace`, `removeEntityInPlace`, - `deserializeReplicaEntity`, `peekEntity`, `cloneTypeMap` for every `MODEL`/`INTERCEPTOR`/`ROLE`/ - `APPLICATION`/`TOOL_SET` case that currently uses the raw canonical id as the key. -- `:839`'s `simpleName = fromApi ? lastSegment(mapKey) : mapKey` ternary becomes unconditional - (`mapKey` is already short in both branches) — audit whatever this feeds to confirm the `fromApi` - distinction isn't load-bearing for something else first. -- Admin `GET`/`PUT`/`DELETE`-by-canonical-id (`ConfigResourceController`, unaffected by this plan - directly) still derives the *storage* address the same way as always - (`{type}/platform/{shortName}`); only the in-memory materialized key changes. Confirm nothing in - the admin CRUD path was implicitly relying on `Config`'s map being canonical-id-keyed (it - shouldn't be — `ConfigResourceController` addresses blob storage directly by descriptor, not - through `Config`'s maps, for writes). - -### 4. Tests - -- **`ConfigTest`**: delete/replace `resolve`-specific test cases (verbatim vs. derived hit) with - plain `Map.get` coverage; `selectDeployment`/`getRole`/`getInterceptor`/`getModel` — short-name hit, - miss; **explicitly assert a canonical-id-shaped input now misses** (locks in the accepted - regression rather than leaving it to silently regress further/inconsistently later). -- **`MergedConfigStoreTest`**: rebuild with a file entry and its migrated blob counterpart — one map - entry, same key, blob's content wins (whichever insertion-order rule is chosen); delete - `shadowFileEntry`-specific test cases. -- **`ConfigPostProcessorTest`**: `name = mapKey` directly, no `lastSegment`; existing - `validateCrossReferences` coverage unaffected. -- **`CanonicalIdListingTest`**: rename/re-scope — it currently covers exactly the - canonical-id-emits-as-short-name transition; confirm what of it still applies once canonical id is - never a map key at all for these types. -- **`MergedConfigStoreApiTest`, `MergedConfigStorePartialUpdateTest`, - `MergedConfigStoreReplicaUpdateTest`**: update key-shape assertions throughout. - -## Updates needed to already-filed issues and the open PR - -- **Issue #1781 (epic)**: update "Rejected alternatives" — either remove the "short-name-keyed - `Config` maps" rejection and replace with a forward reference to this document, or add a note that - it was reopened and why (the narrower-mismatch argument above). Update the Requirement table's row - 1 ("canonical id keeps working as a harmless superset") to state this no longer holds for - models/applications/toolsets/interceptors/roles/schemas, and is an accepted, deliberate change. -- **Issue #1783 (Slice B+C)**: the "Part B" section (derivation via `resolve`, blob-shadows-file, - `lastSegment` outbound naming) describes exactly the mechanism this document removes. Needs a - rewrite describing short-name-keyed maps instead, dropping the `resolve`/shadow-file - language entirely. "Part C" (schema `$id` index) needs the same rewrite pointed at - `schema-id-atomic-derivation.md` instead of the alias-index approach. -- **Issue #1784 (Slice D, migration endpoint)**: the "Open item — schema migration naming" section - is resolved by `schema-id-atomic-derivation.md` (canonical id derives from `$id`, no naming - decision needed) — update or remove that section. The migration behavior for models/apps/etc. is - otherwise unaffected by this document (blob write path/address is unchanged; only the in-memory - key changes), so Slice D's core behavior stands. -- **PR #1813**: still open, not yet merged. Its description documents exactly the `resolve()`/ - shadow-file/`lastSegment` mechanism this document replaces. Two options once the above lands as - actual commits on this PR (or its branch): (a) rewrite the PR description to describe the final - (short-name-keyed) state directly, since GitHub PR descriptions are editable and don't carry - historical baggage the way commit messages do; (b) at merge time, use squash-merge with a fresh - commit message describing the shipped state, rather than the incremental "add index, then replace - index with a bigger rewrite" history — squash-merge naturally collapses this without needing any - destructive history rewrite before merge. - -## Sequencing - -Independent of Slice A (apps/toolsets → `platform`) and Slice D (migration endpoint) in scope, but -touches the same files Slice B (#1783/PR #1813) already modified — this should land as a follow-up -on top of that work (or be squashed into it before merge, per the PR note above), not as a -separate, later PR that has to un-migrate short-name derivation that was just added. diff --git a/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java b/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java index 3974240bb..3c6d0e2e9 100644 --- a/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java +++ b/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java @@ -1036,9 +1036,9 @@ private Future handleGet() throws JsonProcessingException { /** * Per-entity GET for {@code PROJECT_KEY}/{@code ROUTE} only — these two types still key - * {@code Config}'s in-memory map by canonical id (never migrated to short-name keying, see - * {@code short-name-keyed-config-maps.md}), so file-sourced entries never share a key with a - * blob-sourced one and this lookup can safely stay in-memory. + * {@code Config}'s in-memory map by canonical id (never migrated to short-name keying), so + * file-sourced entries never share a key with a blob-sourced one and this lookup can safely + * stay in-memory. */ private Future handleSingleGet(Map source, ResourceTypes resourceType, @@ -1082,12 +1082,12 @@ private Future handleSingleGet(Map source, /** * Per-entity GET for {@code MODEL}/{@code INTERCEPTOR}/{@code ROLE}/{@code APPLICATION}/ * {@code TOOL_SET} — these five types key {@code Config}'s in-memory map by short name, the - * same key a file-sourced entry for the same logical entity already uses (see - * {@code short-name-keyed-config-maps.md}), so the map can no longer tell a genuinely - * blob-managed entity apart from a file-only one sharing that short name. This reads and - * decrypts blob storage directly by descriptor instead — the same pattern PUT/DELETE already - * use for these types — so this endpoint only ever serves entities that actually exist in the - * {@code platform} bucket. {@code Config}'s map stays purely a runtime-resolution structure. + * same key a file-sourced entry for the same logical entity already uses, so the map can no + * longer tell a genuinely blob-managed entity apart from a file-only one sharing that short + * name. This reads and decrypts blob storage directly by descriptor instead — the same + * pattern PUT/DELETE already use for these types — so this endpoint only ever serves entities + * that actually exist in the {@code platform} bucket. {@code Config}'s map stays purely a + * runtime-resolution structure. */ private Future handleSingleGetFromBlob(ResourceTypes resourceType, BiFunction projector) { if (path == null || path.isEmpty()) { diff --git a/server/src/test/java/com/epam/aidial/core/server/CanonicalIdListingTest.java b/server/src/test/java/com/epam/aidial/core/server/CanonicalIdListingTest.java index 9ea16cc4c..e748d1de7 100644 --- a/server/src/test/java/com/epam/aidial/core/server/CanonicalIdListingTest.java +++ b/server/src/test/java/com/epam/aidial/core/server/CanonicalIdListingTest.java @@ -64,9 +64,9 @@ void testFileSourcedModelStillSurfacedAsSimpleName() { @Test void testApiManagedModelAdminGetProjectsShortName() { - // Admin GET reads blob storage directly by descriptor (not the in-memory map — see - // short-name-keyed-config-maps.md), so it projects the URL's own short-name segment, - // matching how the entity is keyed in Config for runtime resolution. + // Admin GET reads blob storage directly by descriptor (not the in-memory map), so it + // projects the URL's own short-name segment, matching how the entity is keyed in Config + // for runtime resolution. verify(send(HttpMethod.PUT, "/v1/models/platform/admin-listing-projection", null, API_MODEL_BODY, "authorization", "admin", "If-None-Match", "*"), 200); diff --git a/server/src/test/java/com/epam/aidial/core/server/config/MergedConfigStorePartialUpdateTest.java b/server/src/test/java/com/epam/aidial/core/server/config/MergedConfigStorePartialUpdateTest.java index aada17bb2..64ab9cfce 100644 --- a/server/src/test/java/com/epam/aidial/core/server/config/MergedConfigStorePartialUpdateTest.java +++ b/server/src/test/java/com/epam/aidial/core/server/config/MergedConfigStorePartialUpdateTest.java @@ -99,11 +99,10 @@ public void interceptorDeleteCascadesToModelInvalidEntities() { @Test public void cascadeClassifiesInvalidatedModelsAsApiSourced() { - // Models are keyed by short name uniformly now (file- and blob-sourced alike, see - // short-name-keyed-config-maps.md), so key shape can no longer tell them apart. The - // partial-update path never touches file config either way, so every survivor this - // cascade walks is classified "api" — regardless of which source the model itself - // originally came from. + // Models are keyed by short name uniformly now (file- and blob-sourced alike), so key + // shape can no longer tell them apart. The partial-update path never touches file config + // either way, so every survivor this cascade walks is classified "api" — regardless of + // which source the model itself originally came from. Model firstModel = new Model(); firstModel.setInterceptors(List.of(INTERCEPTOR_ID)); Model secondModel = new Model(); From 8b6a267c410fc33d86edfcf15523f57f27955307 Mon Sep 17 00:00:00 2001 From: Kiryl_Kurnosenka Date: Thu, 13 Aug 2026 21:20:28 +0300 Subject: [PATCH 10/15] feat: cross-type deployment-id uniqueness checks and ConfigPostProcessor cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add ConfigPostProcessor.isDeploymentIdTaken: exhaustive switch over MODEL/ APPLICATION/TOOL_SET/INTERCEPTOR, throws IllegalArgumentException for non- deployment types - Wire duplicate-deployment-id guard into ConfigResourceController.handlePut (MODEL/INTERCEPTOR) and handleAppOrToolSetPut (APPLICATION/TOOL_SET) - Wire duplicate check into AdminApplyController validate-only precheck, applyManagedEntity, applyModel, applyApplication, applyToolSet - Rename canonicalId → mapKey in ConfigPostProcessor.validateSingleModel/ Interceptor/Role/Application/ToolSet and cascadeInterceptorDelete; the stored value is a short name, not a canonical id - Simplify isValidToolSetKey to delegate to isValidResourceKey (slash- stripping dead code since toolsets are now short-name-keyed) - Drop the legacy canonical-id fallback in ExternalServiceCredentialsController and ExternalServiceManagementController; platform-bucket apps are now addressed by short name in external-service URLs (appPart = short name, not "platform/") - Update tests: PlatformAppToolsetApiTest expects short name in GET body and uses short-name external-service URLs; ConfigPostProcessorTest replaces testSemanticKeepsCanonicalIdKeyedToolSet with drop assertion Co-Authored-By: Claude Sonnet 4.6 --- .../server/config/ConfigPostProcessor.java | 85 ++++++++------ .../controller/AdminApplyController.java | 105 ++++++++++++++---- .../controller/ConfigResourceController.java | 22 ++++ .../ExternalServiceCredentialsController.java | 11 -- .../ExternalServiceManagementController.java | 8 -- .../server/PlatformAppToolsetApiTest.java | 14 +-- .../config/ConfigPostProcessorTest.java | 14 +-- 7 files changed, 171 insertions(+), 88 deletions(-) diff --git a/server/src/main/java/com/epam/aidial/core/server/config/ConfigPostProcessor.java b/server/src/main/java/com/epam/aidial/core/server/config/ConfigPostProcessor.java index bdf5a2e22..cbcba6764 100644 --- a/server/src/main/java/com/epam/aidial/core/server/config/ConfigPostProcessor.java +++ b/server/src/main/java/com/epam/aidial/core/server/config/ConfigPostProcessor.java @@ -37,9 +37,10 @@ *

    *
  • Structural — drops file-defined entries whose map key contains * {@code /} (cross-entity reserved path separator). Always run; cannot - * fail per-entity. Only applied to file-sourced maps — - * {@link MergedConfigStore} skips this pass for the merged config because - * blob entries legitimately key by canonical ID.
    • + * fail per-entity. Only applied to file-sourced maps — {@link MergedConfigStore} + * skips this pass for the merged config: blob-sourced models/applications/ + * interceptors/roles/toolsets key by short name (never contains {@code /}), and + * keys/routes/schemas key by canonical id legitimately. *
  • Semantic — name back-fill, deployment-id uniqueness, ToolSet * resource-key validation, route ordering, {@link ApiKeyStore} hookup. * Each per-entity violation either throws (default {@code abort} mode, @@ -120,13 +121,13 @@ private static void rejectSlashKeyedNames(Map map, String typeLab * only — {@code onSkip == null} means cross-refs are not validated (matches file-loaded * abort path in {@link #processModels}). */ - static void validateSingleModel(Config config, String canonicalId, + static void validateSingleModel(Config config, String mapKey, @Nullable BiConsumer onSkip) { - Model model = config.getModels().get(canonicalId); + Model model = config.getModels().get(mapKey); if (model == null) { return; } - model.setName(canonicalId); + model.setName(mapKey); List warnings = new ArrayList<>(); validatePricing(model, warnings); if (onSkip != null) { @@ -136,20 +137,20 @@ static void validateSingleModel(Config config, String canonicalId, return; } if (onSkip == null) { - throw new InvalidEntityException(ResourceTypes.MODEL, canonicalId, warnings); + throw new InvalidEntityException(ResourceTypes.MODEL, mapKey, warnings); } - config.getModels().remove(canonicalId); - onSkip.accept(ResourceTypes.MODEL, new InvalidEntityException(ResourceTypes.MODEL, canonicalId, warnings)); + config.getModels().remove(mapKey); + onSkip.accept(ResourceTypes.MODEL, new InvalidEntityException(ResourceTypes.MODEL, mapKey, warnings)); } /** * Targeted per-type helper. Sets {@code interceptor.name} from the map key. No cross-ref * validation — interceptors have no outbound refs. */ - static void validateSingleInterceptor(Config config, String canonicalId) { - Interceptor interceptor = config.getInterceptors().get(canonicalId); + static void validateSingleInterceptor(Config config, String mapKey) { + Interceptor interceptor = config.getInterceptors().get(mapKey); if (interceptor != null) { - interceptor.setName(canonicalId); + interceptor.setName(mapKey); } } @@ -157,10 +158,10 @@ static void validateSingleInterceptor(Config config, String canonicalId) { * Targeted per-type helper. Sets {@code role.name} from the map key. {@code Role.limits} * keys are loose-refs (warning-only today) so no cross-ref validation runs. */ - static void validateSingleRole(Config config, String canonicalId) { - Role role = config.getRoles().get(canonicalId); + static void validateSingleRole(Config config, String mapKey) { + Role role = config.getRoles().get(mapKey); if (role != null) { - role.setName(canonicalId); + role.setName(mapKey); } } @@ -169,10 +170,10 @@ static void validateSingleRole(Config config, String canonicalId) { * {@code application.name} from the map key. No cross-ref validation — applications have * no outbound refs checked here. */ - static void validateSingleApplication(Config config, String canonicalId) { - Application application = config.getApplications().get(canonicalId); + static void validateSingleApplication(Config config, String mapKey) { + Application application = config.getApplications().get(mapKey); if (application != null) { - application.setName(canonicalId); + application.setName(mapKey); } } @@ -180,10 +181,10 @@ static void validateSingleApplication(Config config, String canonicalId) { * Targeted per-type helper for {@link MergedConfigStore} partial-update path. Sets * {@code toolSet.name} from the map key. */ - static void validateSingleToolSet(Config config, String canonicalId) { - ToolSet toolSet = config.getToolsets().get(canonicalId); + static void validateSingleToolSet(Config config, String mapKey) { + ToolSet toolSet = config.getToolsets().get(mapKey); if (toolSet != null) { - toolSet.setName(canonicalId); + toolSet.setName(mapKey); } } @@ -206,10 +207,10 @@ static void cascadeInterceptorDelete(Config config, List warnings = new ArrayList<>(); validateCrossReferences(model, config, warnings); if (!warnings.isEmpty()) { - String canonicalId = entry.getKey(); + String mapKey = entry.getKey(); iterator.remove(); onSkip.accept(ResourceTypes.MODEL, - new InvalidEntityException(ResourceTypes.MODEL, canonicalId, warnings)); + new InvalidEntityException(ResourceTypes.MODEL, mapKey, warnings)); } } } @@ -271,10 +272,10 @@ private static void processModels(Config config, Set deploymentIds, /** * Validates that every interceptor reference on the supplied model resolves - * within the merged {@code config.interceptors} map. {@link MergedConfigStore} - * keys file entries by simple name and API entries by canonical ID; either - * shape is accepted via {@code containsKey}. Returns {@code true} when every - * reference resolves (no warnings appended). + * within the merged {@code config.interceptors} map — file- and blob-sourced + * interceptors alike key by short name, so a plain {@code containsKey} against + * that one map key shape is enough. Returns {@code true} when every reference + * resolves (no warnings appended). */ public static boolean validateCrossReferences(Model model, Config config, List warnings) { List refs = model.getInterceptors(); @@ -439,6 +440,24 @@ private static boolean skipOnDuplicate(String name, ResourceTypes type, Set config.getApplications().containsKey(shortName) + || config.getToolsets().containsKey(shortName) + || config.getInterceptors().containsKey(shortName); + case APPLICATION -> config.getModels().containsKey(shortName) + || config.getToolsets().containsKey(shortName) + || config.getInterceptors().containsKey(shortName); + case TOOL_SET -> config.getModels().containsKey(shortName) + || config.getApplications().containsKey(shortName) + || config.getInterceptors().containsKey(shortName); + case INTERCEPTOR -> config.getModels().containsKey(shortName) + || config.getApplications().containsKey(shortName) + || config.getToolsets().containsKey(shortName); + default -> throw new IllegalArgumentException("Not a deployment type: " + type); + }; + } + private static boolean isValidResourceKey(String resourceKey) { return RESOURCE_KEY_PATTERN.matcher(resourceKey).matches(); } @@ -448,14 +467,10 @@ public static String resourceKeyPattern() { return RESOURCE_KEY_PATTERN.pattern(); } - // Bare file-sourced toolset names have no '/'; API-managed toolsets are keyed by their canonical - // id ("toolsets/platform/name") — validate only the trailing short-name segment in that case, same - // as the plain-name case would validate the whole (slash-free) string. Only ToolSet map keys are - // ever canonical-id-shaped this way — other RESOURCE_KEY_PATTERN callers (e.g. external-service - // ids) must keep going through the strict isValidResourceKey above. + // ToolSet map keys are always short names now, file- and blob-sourced alike — same check as + // isValidResourceKey. Kept as its own named entry point since ToolSet call sites reason about + // it as "the toolset key check" rather than the generic one. public static boolean isValidToolSetKey(String resourceKey) { - int slash = resourceKey.lastIndexOf('/'); - String candidate = slash < 0 ? resourceKey : resourceKey.substring(slash + 1); - return RESOURCE_KEY_PATTERN.matcher(candidate).matches(); + return isValidResourceKey(resourceKey); } } diff --git a/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java b/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java index 6e9cb6e76..d65a42fee 100644 --- a/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java +++ b/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java @@ -310,8 +310,18 @@ static ValidationResult validateOnly(AdminManifest entry, Config scratch, boolea if (!warnings.isEmpty() && !softValidation) { return new ValidationResult(id, ValidationStatus.FAILED, joinWarnings(warnings)); } + String dupError = duplicateDeploymentIdMessage(scratch, ResourceTypes.MODEL, parsed); + if (dupError != null) { + return new ValidationResult(id, ValidationStatus.FAILED, dupError); + } + } + case "Interceptor" -> { + ConfigResourceController.treeToEntity(entry.spec(), Interceptor.class); + String dupError = duplicateDeploymentIdMessage(scratch, ResourceTypes.INTERCEPTOR, parsed); + if (dupError != null) { + return new ValidationResult(id, ValidationStatus.FAILED, dupError); + } } - case "Interceptor" -> ConfigResourceController.treeToEntity(entry.spec(), Interceptor.class); case "Role" -> ConfigResourceController.treeToEntity(entry.spec(), Role.class); case "Route" -> ConfigResourceController.treeToEntity(entry.spec(), Route.class); case "Key" -> { @@ -327,8 +337,24 @@ static ValidationResult validateOnly(AdminManifest entry, Config scratch, boolea "Invalid key: at least one role must be assigned to the key " + key.getProject()); } } - case "Application" -> ConfigResourceController.treeToEntity(entry.spec(), Application.class); - case "ToolSet" -> ConfigResourceController.treeToEntity(entry.spec(), ToolSet.class); + case "Application" -> { + ConfigResourceController.treeToEntity(entry.spec(), Application.class); + if (ResourceDescriptor.PLATFORM_BUCKET.equals(parsed.bucket())) { + String dupError = duplicateDeploymentIdMessage(scratch, ResourceTypes.APPLICATION, parsed); + if (dupError != null) { + return new ValidationResult(id, ValidationStatus.FAILED, dupError); + } + } + } + case "ToolSet" -> { + ConfigResourceController.treeToEntity(entry.spec(), ToolSet.class); + if (ResourceDescriptor.PLATFORM_BUCKET.equals(parsed.bucket())) { + String dupError = duplicateDeploymentIdMessage(scratch, ResourceTypes.TOOL_SET, parsed); + if (dupError != null) { + return new ValidationResult(id, ValidationStatus.FAILED, dupError); + } + } + } case "Schema" -> { if (!entry.spec().isObject()) { return new ValidationResult(id, ValidationStatus.FAILED, "Schema spec must be a JSON object"); @@ -353,6 +379,19 @@ static ValidationResult validateOnly(AdminManifest entry, Config scratch, boolea return new ValidationResult(id, ValidationStatus.VALID, null); } + /** + * Shared by {@link #validateOnly} (precheck) and the real-apply {@code applyX} methods: + * non-null iff {@code parsed}'s short name is already claimed by a different + * model/application/interceptor/toolset in {@code scratch}. See + * {@link ConfigPostProcessor#isDeploymentIdTaken}. + */ + private static String duplicateDeploymentIdMessage(Config scratch, ResourceTypes type, ParsedName parsed) { + if (ConfigPostProcessor.isDeploymentIdTaken(scratch, type, parsed.name())) { + return "Deployment ID '" + parsed.name() + "' is already used by a different entity"; + } + return null; + } + private EntityResult applySingle(AdminManifest entry, Config scratch, List pending) { String id = entry.name(); ParsedName parsed; @@ -365,13 +404,13 @@ private EntityResult applySingle(AdminManifest entry, Config scratch, List applySettings(entry, id, parsed); case "Schema" -> applySchema(entry, id, parsed, scratch, pending, ResourceTypes.APP_TYPE_SCHEMA); case "CatalogSchema" -> applySchema(entry, id, parsed, scratch, pending, ResourceTypes.CATALOG_SCHEMA); - case "Interceptor" -> applyManagedEntity(entry, id, parsed, ResourceTypes.INTERCEPTOR, Interceptor.class, pending); - case "Role" -> applyManagedEntity(entry, id, parsed, ResourceTypes.ROLE, Role.class, pending); - case "Route" -> applyManagedEntity(entry, id, parsed, ResourceTypes.ROUTE, Route.class, pending); + case "Interceptor" -> applyManagedEntity(entry, id, parsed, ResourceTypes.INTERCEPTOR, Interceptor.class, scratch, pending); + case "Role" -> applyManagedEntity(entry, id, parsed, ResourceTypes.ROLE, Role.class, scratch, pending); + case "Route" -> applyManagedEntity(entry, id, parsed, ResourceTypes.ROUTE, Route.class, scratch, pending); case "Key" -> applyKey(entry, id, parsed, pending); case "Model" -> applyModel(entry, id, parsed, scratch, pending); - case "ToolSet" -> applyToolSet(entry, id, parsed, pending); - case "Application" -> applyApplication(entry, id, parsed, pending); + case "ToolSet" -> applyToolSet(entry, id, parsed, scratch, pending); + case "Application" -> applyApplication(entry, id, parsed, scratch, pending); default -> new EntityResult(id, AdminApplyStatus.FAILED, "Unknown kind: " + entry.kind()); }; } @@ -415,10 +454,19 @@ private EntityResult applySchema(AdminManifest entry, String id, ParsedName pars } private EntityResult applyManagedEntity(AdminManifest entry, String id, ParsedName parsed, - ResourceTypes type, Class entityClass, List pending) { + ResourceTypes type, Class entityClass, Config scratch, + List pending) { T entity = ConfigResourceController.treeToEntity(entry.spec(), entityClass); ResourceDescriptor descriptor = ResourceDescriptorFactory.fromDecoded( type, parsed.bucket(), parsed.location(), parsed.name()); + /// Deployment-id uniqueness only applies to INTERCEPTOR here — ROLE/ROUTE aren't deployments + // resolved through Config.selectDeployment, so they don't share the short-name namespace. + if (type == ResourceTypes.INTERCEPTOR) { + String dupError = duplicateDeploymentIdMessage(scratch, type, parsed); + if (dupError != null) { + return new EntityResult(id, AdminApplyStatus.FAILED, dupError); + } + } String blobBody = ConfigResourceController.serializeForBlob(entity); resourceService.putResource(descriptor, blobBody, EtagHeader.ANY); pending.add(new EntityChange(type, MergedConfigStore.mapKeyFor(type, descriptor), entity)); @@ -484,6 +532,10 @@ private EntityResult applyModel(AdminManifest entry, String id, ParsedName parse } ResourceDescriptor descriptor = ResourceDescriptorFactory.fromDecoded( ResourceTypes.MODEL, parsed.bucket(), parsed.location(), parsed.name()); + String dupError = duplicateDeploymentIdMessage(scratch, ResourceTypes.MODEL, parsed); + if (dupError != null) { + return new EntityResult(id, AdminApplyStatus.FAILED, dupError); + } secretFieldProcessor.encryptFields(model, descriptor); String blobBody = ConfigResourceController.serializeForBlob(model); resourceService.putResource(descriptor, blobBody, EtagHeader.ANY); @@ -493,32 +545,47 @@ private EntityResult applyModel(AdminManifest entry, String id, ParsedName parse return new EntityResult(id, invalid ? AdminApplyStatus.APPLIED_INVALID : AdminApplyStatus.APPLIED, null); } - private EntityResult applyApplication(AdminManifest entry, String id, ParsedName parsed, List pending) { + private EntityResult applyApplication(AdminManifest entry, String id, ParsedName parsed, Config scratch, List pending) { Application application = ConfigResourceController.treeToEntity(entry.spec(), Application.class); ResourceDescriptor descriptor = ResourceDescriptorFactory.fromDecoded( ResourceTypes.APPLICATION, parsed.bucket(), parsed.location(), parsed.name()); + // Only the platform bucket is materialized into MergedConfigStore (see EntityLocationStrategy) — + // public-bucket apps stay outside it and are served lazily by ApplicationService, so they're + // exempt from deployment-id uniqueness and pushing them into `pending` below would spuriously + // duplicate them in config.getApplications()-backed listings (e.g. ApplicationController/ + // DeploymentController) until the next full rebuild. + boolean platform = ResourceDescriptor.PLATFORM_BUCKET.equals(parsed.bucket()); + if (platform) { + String dupError = duplicateDeploymentIdMessage(scratch, ResourceTypes.APPLICATION, parsed); + if (dupError != null) { + return new EntityResult(id, AdminApplyStatus.FAILED, dupError); + } + } // Bulk admin apply is always admin context — preserve forwardAuthToken if the manifest set it. applicationService.putApplication(descriptor, EtagHeader.ANY, null, application, true, AdminManagedFieldsWriteMode.AUTHORITATIVE); - // Only the platform bucket is materialized into MergedConfigStore (see EntityLocationStrategy) — - // public-bucket apps stay outside it and are served lazily by ApplicationService, so pushing - // them into `pending` here would spuriously duplicate them in config.getApplications()-backed - // listings (e.g. ApplicationController/DeploymentController) until the next full rebuild. - if (ResourceDescriptor.PLATFORM_BUCKET.equals(parsed.bucket())) { + if (platform) { Application decrypted = applicationService.getApplicationWithDecryptedSecrets(descriptor).getValue(); pending.add(new EntityChange(ResourceTypes.APPLICATION, MergedConfigStore.mapKeyFor(ResourceTypes.APPLICATION, descriptor), decrypted)); } return new EntityResult(id, AdminApplyStatus.APPLIED, null); } - private EntityResult applyToolSet(AdminManifest entry, String id, ParsedName parsed, List pending) { + private EntityResult applyToolSet(AdminManifest entry, String id, ParsedName parsed, Config scratch, List pending) { ToolSet toolSet = ConfigResourceController.treeToEntity(entry.spec(), ToolSet.class); ResourceDescriptor descriptor = ResourceDescriptorFactory.fromDecoded( ResourceTypes.TOOL_SET, parsed.bucket(), parsed.location(), parsed.name()); - toolSetService.putToolSet(descriptor, EtagHeader.ANY, null, toolSet, true); // Same rationale as applyApplication above — only platform-bucket toolsets belong in - // MergedConfigStore. - if (ResourceDescriptor.PLATFORM_BUCKET.equals(parsed.bucket())) { + // MergedConfigStore / are subject to deployment-id uniqueness. + boolean platform = ResourceDescriptor.PLATFORM_BUCKET.equals(parsed.bucket()); + if (platform) { + String dupError = duplicateDeploymentIdMessage(scratch, ResourceTypes.TOOL_SET, parsed); + if (dupError != null) { + return new EntityResult(id, AdminApplyStatus.FAILED, dupError); + } + } + toolSetService.putToolSet(descriptor, EtagHeader.ANY, null, toolSet, true); + if (platform) { ToolSet decrypted = toolSetService.getToolSetWithDecryptedAuthSettings(descriptor).getValue(); pending.add(new EntityChange(ResourceTypes.TOOL_SET, MergedConfigStore.mapKeyFor(ResourceTypes.TOOL_SET, descriptor), decrypted)); } diff --git a/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java b/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java index 3c6d0e2e9..d37c838af 100644 --- a/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java +++ b/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java @@ -1379,6 +1379,7 @@ private Future handleAppOrToolSetPut() { throw new HttpException(HttpStatus.BAD_REQUEST, "Request body must be a JSON object"); } return taskExecutor.submit(() -> lockService.underBucketLocks(MergedConfigStore.ADMIN_BUCKET_LOCATIONS, () -> { + rejectDuplicateDeploymentId(type, path); Object decrypted; // The platform bucket requires explicit admin access for every operation (see // AdminRoleAuthorizationService), not just an admin-AND-public-bucket combination like @@ -1522,6 +1523,10 @@ private Future handlePut() { if (entity instanceof Model m) { checkCrossReferences(m); } + ResourceTypes writeType = resourceType(); + if (writeType == ResourceTypes.MODEL || writeType == ResourceTypes.INTERCEPTOR) { + rejectDuplicateDeploymentId(writeType, path); + } if (spec.isKey()) { keyEntity = (Key) entity; validateKeyForApiWrite(keyEntity, "PUT"); @@ -1668,6 +1673,23 @@ static void rejectSchemaIdCollision(Config snapshot, ResourceTypes type, String } } + /** + * Rejects a MODEL/INTERCEPTOR/APPLICATION/TOOL_SET write whose short name is already claimed + * by a different deployment (of any of those four types) in the live merged Config — the + * partial-update-path counterpart of {@code ConfigPostProcessor.skipOnDuplicate}, which only + * runs during a full rebuild. See {@link ConfigPostProcessor#isDeploymentIdTaken}. + */ + private void rejectDuplicateDeploymentId(ResourceTypes type, String shortName) { + Config snapshot = mergedConfigStore.get(); + if (snapshot == null) { + return; + } + if (ConfigPostProcessor.isDeploymentIdTaken(snapshot, type, shortName)) { + throw new HttpException(HttpStatus.CONFLICT, + "Deployment ID '" + shortName + "' is already used by a different entity"); + } + } + private static ApiKeyData apiKeyData(Key key) { ApiKeyData data = new ApiKeyData(); data.setOriginalKey(key); diff --git a/server/src/main/java/com/epam/aidial/core/server/controller/ExternalServiceCredentialsController.java b/server/src/main/java/com/epam/aidial/core/server/controller/ExternalServiceCredentialsController.java index cd458e9f1..ec2880c8a 100644 --- a/server/src/main/java/com/epam/aidial/core/server/controller/ExternalServiceCredentialsController.java +++ b/server/src/main/java/com/epam/aidial/core/server/controller/ExternalServiceCredentialsController.java @@ -365,21 +365,10 @@ private ResolvedExternalService resolveExternalService(String scopeId, @Nullable /** Resolves only the application for a scope's app part — never the owner's bucket — so the OBO gate can run first. */ private ResolvedApplication resolveApplication(String appPart) { - // appPart carries no "applications/" type prefix (the URL is already under /v1/applications/), - // whereas materialized Config keys do. Resolve verbatim first (config-file bare names, and — - // once short-name addressing lands — short-name platform apps via derivation), then by - // canonical id so a platform-bucket reference like "platform/my-app" hits its materialized - // in-memory entry ("applications/platform/my-app"), which already carries decrypted secrets, - // instead of being read back from blob. A config-managed (platform) app is access-controlled - // by its userRoles, so it resolves as a static app and verifyAccess uses hasAccess(userRoles). Deployment deployment = context.getConfig().selectDeployment(appPart); - if (deployment == null) { - deployment = context.getConfig().selectDeployment(CredentialsLocatorFactory.APPLICATIONS_PREFIX + appPart); - } if (deployment instanceof Application configApp) { return new ResolvedApplication(configApp, null, true); } - // Dynamic (public / user-bucket) app — not materialized; read from blob, rule-based access. ResourceDescriptor appDescriptor; try { appDescriptor = ResourceDescriptorFactory.fromAnyUrl( diff --git a/server/src/main/java/com/epam/aidial/core/server/controller/ExternalServiceManagementController.java b/server/src/main/java/com/epam/aidial/core/server/controller/ExternalServiceManagementController.java index f0b46a1fb..40a120a76 100644 --- a/server/src/main/java/com/epam/aidial/core/server/controller/ExternalServiceManagementController.java +++ b/server/src/main/java/com/epam/aidial/core/server/controller/ExternalServiceManagementController.java @@ -266,15 +266,7 @@ private static String scopeId(String appId, String serviceId) { } private ResolvedApp resolveApp(String appId) { - // See ExternalServiceCredentialsController.resolveApplication: appId omits the "applications/" - // type prefix, so resolve verbatim first, then by canonical id so a platform-bucket app hits - // its materialized in-memory entry ("applications/platform/my-app") — with decrypted secrets — - // instead of being read from blob. A config-managed (platform) app resolves as a static app, - // access-controlled by its userRoles like a config-file app rather than by folder rules. Deployment deployment = context.getConfig().selectDeployment(appId); - if (deployment == null) { - deployment = context.getConfig().selectDeployment(CredentialsLocatorFactory.APPLICATIONS_PREFIX + appId); - } if (deployment instanceof Application configApp) { return new ResolvedApp(configApp, null, null, true); } diff --git a/server/src/test/java/com/epam/aidial/core/server/PlatformAppToolsetApiTest.java b/server/src/test/java/com/epam/aidial/core/server/PlatformAppToolsetApiTest.java index 5744de508..0b17b8588 100644 --- a/server/src/test/java/com/epam/aidial/core/server/PlatformAppToolsetApiTest.java +++ b/server/src/test/java/com/epam/aidial/core/server/PlatformAppToolsetApiTest.java @@ -42,8 +42,8 @@ void testApplicationPutGetDeleteRoundTrip() { Response get = send(HttpMethod.GET, "/v1/applications/platform/my-platform-app", null, "", "authorization", "admin"); verify(get, 200); - assertTrue(get.body().contains("\"name\":\"applications/platform/my-platform-app\""), - () -> "Expected canonical name in body: " + get.body()); + assertTrue(get.body().contains("\"name\":\"my-platform-app\""), + () -> "Expected short name in body: " + get.body()); assertTrue(get.body().contains("\"endpoint\":\"http://application1/v1/completions\""), () -> "Expected endpoint in body: " + get.body()); @@ -66,8 +66,8 @@ void testToolSetPutGetDeleteRoundTrip() { Response get = send(HttpMethod.GET, "/v1/toolsets/platform/my-platform-toolset", null, "", "authorization", "admin"); verify(get, 200); - assertTrue(get.body().contains("\"name\":\"toolsets/platform/my-platform-toolset\""), - () -> "Expected canonical name in body: " + get.body()); + assertTrue(get.body().contains("\"name\":\"my-platform-toolset\""), + () -> "Expected short name in body: " + get.body()); assertTrue(get.body().contains("\"endpoint\":\"http://localhost:9876\""), () -> "Expected endpoint in body: " + get.body()); @@ -193,7 +193,7 @@ void testApplicationExternalServiceSecretNeverLeaksOnGet() { } @Test - void testPlatformAppExternalServiceAccessAllowedByUserRoles() throws Exception { + void testPlatformAppExternalServiceAccessAllowedByUserRoles() { // Regression (#1773 review): external-service access on a platform app must go through the // app's userRoles like a config app, not folder rules. This app is open (no user_roles), so a // non-admin user can sign in; before the fix the folder-rules branch returned 403. @@ -225,7 +225,7 @@ void testPlatformAppExternalServiceAccessAllowedByUserRoles() throws Exception { try (TestWebServer ignore = new TestWebServer(9876, handler)) { Response signIn = send(HttpMethod.POST, "/v1/ops/external-service/signin", null, """ { - "url": "applications/platform/extsvc-open-app/external_services/svc1", + "url": "applications/extsvc-open-app/external_services/svc1", "credentials_level": "USER", "authentication_type": "OAUTH", "code": "auth-code" @@ -264,7 +264,7 @@ void testPlatformAppExternalServiceDeniedWithoutUserRole() { Response signIn = send(HttpMethod.POST, "/v1/ops/external-service/signin", null, """ { - "url": "applications/platform/extsvc-restricted-app/external_services/svc1", + "url": "applications/extsvc-restricted-app/external_services/svc1", "credentials_level": "USER", "authentication_type": "OAUTH", "code": "auth-code" diff --git a/server/src/test/java/com/epam/aidial/core/server/config/ConfigPostProcessorTest.java b/server/src/test/java/com/epam/aidial/core/server/config/ConfigPostProcessorTest.java index 255b42403..8d0ea89f4 100644 --- a/server/src/test/java/com/epam/aidial/core/server/config/ConfigPostProcessorTest.java +++ b/server/src/test/java/com/epam/aidial/core/server/config/ConfigPostProcessorTest.java @@ -94,23 +94,21 @@ void testSemanticSkipRoutesDuplicateToCallback() { } @Test - void testSemanticKeepsCanonicalIdKeyedToolSet() { - // Materialized platform toolsets are keyed by canonical id ("toolsets/platform/name"), unlike - // file-sourced ones (bare "name") — processToolSets must validate only the trailing short-name - // segment, not reject the whole key for containing '/'. + void testSemanticDropsCanonicalIdKeyedToolSet() { + // Toolsets in Config are now keyed by short name only; a canonical-id-shaped key + // ("toolsets/platform/name") fails isValidToolSetKey and is dropped. Config config = newMutableConfig(); config.getToolsets().put("toolsets/platform/my-toolset", new ToolSet()); ConfigPostProcessor.processSemantic(config, null, Map.of(), Map.of(), null); - assertTrue(config.getToolsets().containsKey("toolsets/platform/my-toolset")); - assertEquals("toolsets/platform/my-toolset", config.getToolsets().get("toolsets/platform/my-toolset").getName()); + assertTrue(config.getToolsets().isEmpty()); } @Test - void testSemanticDropsCanonicalIdToolSetWithInvalidShortName() { + void testSemanticDropsToolSetWithInvalidName() { Config config = newMutableConfig(); - config.getToolsets().put("toolsets/platform/bad name", new ToolSet()); + config.getToolsets().put("bad name", new ToolSet()); ConfigPostProcessor.processSemantic(config, null, Map.of(), Map.of(), null); From e4af8056a8d5431cb4a4acb89f4cb57707041f23 Mon Sep 17 00:00:00 2001 From: Kiryl_Kurnosenka Date: Thu, 13 Aug 2026 23:13:48 +0300 Subject: [PATCH 11/15] feat: key Config schema maps by \$id, serve schema GET from blob MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - UrlUtil: add escapeSchemaId/unescapeSchemaId for flat blob keys - ResourceDescriptorFactory: add fromDecodedAtomicName for atomic $id paths - Config: remove applicationSchemaAliasesById/catalogSchemaAliasesById; schema maps are now keyed directly by \$id (no alias index needed) - MergedConfigStore: rename isNameAddressed -> isLocallyKeyed, add APP_TYPE_SCHEMA/CATALOG_SCHEMA; add localKey() helper that unescapes blob name to \$id for schema types; remove schemaIdFromCanonicalId — all partial-update helpers (putEntityInPlace, peekEntity, removeEntityInPlace) now receive mapKey (\$id) directly; rename EntityChange.canonicalId -> mapKey throughout - AdminApplyController: pass mapKeyFor(type, descriptor) for schemas instead of canonicalId(descriptor) so all EntityChange callers are consistent; drop stale scratch arg from applySchema call sites - ConfigResourceController: use fromDecodedAtomicName for schema descriptors; serve schema GET via handleSingleGetFromBlob (same as models/apps) instead of Config-map lookup; override body.\$id to match URL segment on PUT Co-Authored-By: Claude Sonnet 4.6 --- .../com/epam/aidial/core/config/Config.java | 42 +-- .../server/config/ConfigPostProcessor.java | 31 ++ .../core/server/config/EntityChange.java | 6 +- .../core/server/config/MergedConfigStore.java | 294 +++++++----------- .../controller/AdminApplyController.java | 58 ++-- .../controller/ConfigResourceController.java | 168 ++++------ .../util/ResourceDescriptorFactory.java | 17 + .../aidial/core/storage/util/UrlUtil.java | 19 ++ 8 files changed, 285 insertions(+), 350 deletions(-) diff --git a/config/src/main/java/com/epam/aidial/core/config/Config.java b/config/src/main/java/com/epam/aidial/core/config/Config.java index cd4110c99..89a7a4270 100644 --- a/config/src/main/java/com/epam/aidial/core/config/Config.java +++ b/config/src/main/java/com/epam/aidial/core/config/Config.java @@ -56,23 +56,6 @@ public class Config { private List globalInterceptors = List.of(); - /** - * $id → canonical-id index for {@link #applicationTypeSchemas}, built at rebuild time from - * blob bodies: each key is a schema's own {@code $id} (as declared in its body), each value - * is the canonical id of the blob entry storing that schema. Bridges $id-keyed file entries - * and canonical-id-keyed blob entries, since a schema's $id is not derivable from its path. - * - *

    For example, given a blob entry stored under canonical id - * {@code schemas/platform/my-schema} whose body declares - * {@code "$id": "https://example.com/schemas/my-schema.json"}, this map holds - * {@code "https://example.com/schemas/my-schema.json" → "schemas/platform/my-schema"}. - */ - @JsonIgnore - private Map applicationSchemaAliasesById = Map.of(); - - @JsonIgnore - private Map catalogSchemaAliasesById = Map.of(); - @JsonIgnore public Deployment selectDeployment(String deploymentId) { Application application = applications.get(deploymentId); @@ -102,7 +85,10 @@ public boolean isDeploymentExists(String deploymentId) { */ @JsonIgnore public String getCustomApplicationSchema(URI schemaId) { - return resolveSchema(applicationTypeSchemas, applicationSchemaAliasesById, schemaId); + if (schemaId == null) { + return null; + } + return applicationTypeSchemas.get(schemaId.toString()); } /** @@ -110,27 +96,9 @@ public String getCustomApplicationSchema(URI schemaId) { */ @JsonIgnore public String getCatalogSchema(URI schemaId) { - return resolveSchema(catalogSchemas, catalogSchemaAliasesById, schemaId); - } - - /** - * Resolves a schema by its $id: verbatim lookup first (canonical-id callers, and file entries - * already keyed by $id), then falls back through the $id → canonical-id alias index for a - * migrated blob entry. A schema's $id is not derivable from its path, so the alias index must - * be maintained explicitly (see {@code MergedConfigStore}). - * - * @return the schema body, or {@code null} if {@code schemaId} is null or unresolved - */ - private static String resolveSchema(Map schemas, Map aliasesById, URI schemaId) { if (schemaId == null) { return null; } - String id = schemaId.toString(); - String body = schemas.get(id); - if (body != null) { - return body; - } - String canonicalId = aliasesById.get(id); - return canonicalId == null ? null : schemas.get(canonicalId); + return catalogSchemas.get(schemaId.toString()); } } diff --git a/server/src/main/java/com/epam/aidial/core/server/config/ConfigPostProcessor.java b/server/src/main/java/com/epam/aidial/core/server/config/ConfigPostProcessor.java index cbcba6764..7658aa27d 100644 --- a/server/src/main/java/com/epam/aidial/core/server/config/ConfigPostProcessor.java +++ b/server/src/main/java/com/epam/aidial/core/server/config/ConfigPostProcessor.java @@ -16,7 +16,11 @@ import com.epam.aidial.core.credentials.validation.AuthSettingsValidator; import com.epam.aidial.core.credentials.validation.AuthSettingsValidatorFactory; import com.epam.aidial.core.server.security.ApiKeyStore; +import com.epam.aidial.core.server.util.ProxyUtil; import com.epam.aidial.core.storage.resource.ResourceTypes; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import lombok.extern.slf4j.Slf4j; import java.util.ArrayList; @@ -177,6 +181,33 @@ static void validateSingleApplication(Config config, String mapKey) { } } + /** + * Targeted per-type helper for {@link MergedConfigStore} partial-update path. Normalises + * the {@code $id} field in the stored schema body so it always matches the map key + * ({@code schemaId}), mirroring the S3 override applied at write time. + */ + public static void validateSingleSchema(Config config, ResourceTypes type, String schemaId) { + Map schemas = switch (type) { + case APP_TYPE_SCHEMA -> config.getApplicationTypeSchemas(); + case CATALOG_SCHEMA -> config.getCatalogSchemas(); + default -> throw new IllegalArgumentException("Unexpected schema type: " + type); + }; + String body = schemas.get(schemaId); + if (body == null) { + return; + } + try { + JsonNode node = ProxyUtil.BLOB_MAPPER.readTree(body); + JsonNode idNode = node.get("$id"); + if (idNode == null || !schemaId.equals(idNode.asText())) { + ((ObjectNode) node).put("$id", schemaId); + schemas.put(schemaId, ProxyUtil.BLOB_MAPPER.writeValueAsString(node)); + } + } catch (JsonProcessingException e) { + // Malformed body — leave as-is; schema validators will report it on next reload + } + } + /** * Targeted per-type helper for {@link MergedConfigStore} partial-update path. Sets * {@code toolSet.name} from the map key. diff --git a/server/src/main/java/com/epam/aidial/core/server/config/EntityChange.java b/server/src/main/java/com/epam/aidial/core/server/config/EntityChange.java index 8e9bc30db..cadb2fbcb 100644 --- a/server/src/main/java/com/epam/aidial/core/server/config/EntityChange.java +++ b/server/src/main/java/com/epam/aidial/core/server/config/EntityChange.java @@ -9,6 +9,10 @@ * {@code decryptedEntity} signals delete. For non-null entities the caller is * responsible for decrypting in-place before the call (same contract as the * single-entity {@code applyEntityWrite} path). + * + *

    {@code mapKey} is the key used in {@code Config}'s in-memory type-map — the short name + * for models/interceptors/roles/applications/toolsets, the decoded {@code $id} for schema types, + * and the canonical id for all other types (keys, routes). */ -public record EntityChange(ResourceTypes type, String canonicalId, @Nullable Object decryptedEntity) { +public record EntityChange(ResourceTypes type, String mapKey, @Nullable Object decryptedEntity) { } diff --git a/server/src/main/java/com/epam/aidial/core/server/config/MergedConfigStore.java b/server/src/main/java/com/epam/aidial/core/server/config/MergedConfigStore.java index 54b4f9404..d6ebf4c6b 100644 --- a/server/src/main/java/com/epam/aidial/core/server/config/MergedConfigStore.java +++ b/server/src/main/java/com/epam/aidial/core/server/config/MergedConfigStore.java @@ -23,6 +23,7 @@ import com.epam.aidial.core.storage.resource.ResourceTypes; import com.epam.aidial.core.storage.service.LockService; import com.epam.aidial.core.storage.service.ResourceService; +import com.epam.aidial.core.storage.util.UrlUtil; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import io.micrometer.core.instrument.Counter; @@ -597,7 +598,7 @@ private void cancelPendingRebuild() { /** * Partial-update fast path for the API write controller (slice 4S.4 / OQ-32). Mutates the - * single entity at {@code canonicalId} of the given {@code type} in the merged {@link Config} + * single entity at {@code mapKey} of the given {@code type} in the merged {@link Config} * without re-scanning blob storage. {@code decryptedEntity} is the post-decryption Java entity * (or the JSON-string body for {@code APP_TYPE_SCHEMA}) — the controller has already validated * cross-references in strict mode and called {@code apiKeyStore.addOrUpdateKey} for keys. @@ -606,10 +607,10 @@ private void cancelPendingRebuild() { * resurrected when the new interceptor satisfies their references. For other types no transitive * effect runs. */ - public Config applyEntityWrite(ResourceTypes type, String canonicalId, Object decryptedEntity) { + public Config applyEntityWrite(ResourceTypes type, String mapKey, Object decryptedEntity) { rebuildLock.lock(); try { - return applyEntityWriteLocked(type, canonicalId, decryptedEntity); + return applyEntityWriteLocked(type, mapKey, decryptedEntity); } finally { rebuildLock.unlock(); } @@ -620,10 +621,10 @@ public Config applyEntityWrite(ResourceTypes type, String canonicalId, Object de * cross-reference revalidation across the model map and routes newly-orphaned models through * the {@code invalidEntities} sibling store. Other types have no transitive effect. */ - public Config applyEntityDelete(ResourceTypes type, String canonicalId) { + public Config applyEntityDelete(ResourceTypes type, String mapKey) { rebuildLock.lock(); try { - return applyEntityDeleteLocked(type, canonicalId); + return applyEntityDeleteLocked(type, mapKey); } finally { rebuildLock.unlock(); } @@ -637,7 +638,7 @@ public Config applyEntityDelete(ResourceTypes type, String canonicalId) { *

    Each touched type-map is cloned once at the top of the batch and mutated * in place across all entries — avoids the {@code O(batch × map)} clone cost that would arise * from cloning the map per entry. Per-entry failures roll back the entity slot at - * {@code (type, canonicalId)} so subsequent entries observe the pre-change state, matching + * {@code (type, mapKey)} so subsequent entries observe the pre-change state, matching * single-entity atomicity. */ public Map applyBatch(List changes) { @@ -666,7 +667,7 @@ public Map applyBatch(List changes) { try { applyChangeInPlace(next, nextInvalid, change, onSkip); } catch (Exception error) { - failures.put(change.canonicalId(), error.getMessage()); + failures.put(change.mapKey(), error.getMessage()); } } @@ -680,7 +681,7 @@ public Map applyBatch(List changes) { /** * In-place per-entry apply for {@link #applyBatch}. Snapshots the entity slot at - * {@code (type, canonicalId)} and the matching invalid-entity record so a validation/coercion + * {@code (type, mapKey)} and the matching invalid-entity record so a validation/coercion * throw rolls back the per-entry mutation, leaving prior-entry effects intact in {@code next}. * Cross-type mutations (resurrection / cascade) cannot partial-fail — the validate helper * throws before the cross-type step runs. @@ -690,44 +691,46 @@ private void applyChangeInPlace(Config next, EntityChange change, BiConsumer onSkip) { ResourceTypes type = change.type(); - String canonicalId = change.canonicalId(); + String mapKey = change.mapKey(); Object decryptedEntity = change.decryptedEntity(); - Object previousEntity = peekEntity(next, type, canonicalId); + Object previousEntity = peekEntity(next, type, mapKey); Map perType = nextInvalid.get(type); - InvalidEntityRecord previousInvalid = perType == null ? null : perType.get(canonicalId); + InvalidEntityRecord previousInvalid = perType == null ? null : perType.get(mapKey); try { - clearInvalid(nextInvalid, type, canonicalId); + clearInvalid(nextInvalid, type, mapKey); if (decryptedEntity == null) { - removeEntityInPlace(next, type, canonicalId); + removeEntityInPlace(next, type, mapKey); if (type == ResourceTypes.INTERCEPTOR) { ConfigPostProcessor.cascadeInterceptorDelete(next, onSkip); } return; } - putEntityInPlace(next, type, canonicalId, decryptedEntity); + putEntityInPlace(next, type, mapKey, decryptedEntity); switch (type) { - case MODEL -> ConfigPostProcessor.validateSingleModel(next, canonicalId, onSkip); + case MODEL -> ConfigPostProcessor.validateSingleModel(next, mapKey, onSkip); case INTERCEPTOR -> { - ConfigPostProcessor.validateSingleInterceptor(next, canonicalId); + ConfigPostProcessor.validateSingleInterceptor(next, mapKey); resurrectInvalidModels(next, nextInvalid); } - case ROLE -> ConfigPostProcessor.validateSingleRole(next, canonicalId); - case APPLICATION -> ConfigPostProcessor.validateSingleApplication(next, canonicalId); - case TOOL_SET -> ConfigPostProcessor.validateSingleToolSet(next, canonicalId); - case PROJECT_KEY, APP_TYPE_SCHEMA, CATALOG_SCHEMA -> { /* no post-processing */ } + case ROLE -> ConfigPostProcessor.validateSingleRole(next, mapKey); + case APPLICATION -> ConfigPostProcessor.validateSingleApplication(next, mapKey); + case TOOL_SET -> ConfigPostProcessor.validateSingleToolSet(next, mapKey); + case PROJECT_KEY -> { /* no post-processing */ } + case APP_TYPE_SCHEMA -> ConfigPostProcessor.validateSingleSchema(next, ResourceTypes.APP_TYPE_SCHEMA, mapKey); + case CATALOG_SCHEMA -> ConfigPostProcessor.validateSingleSchema(next, ResourceTypes.CATALOG_SCHEMA, mapKey); case ROUTE -> ConfigPostProcessor.sortRoutesInPlace(next); default -> throw new IllegalArgumentException("Unsupported type for partial update: " + type); } } catch (RuntimeException error) { if (previousEntity == null) { - removeEntityInPlace(next, type, canonicalId); + removeEntityInPlace(next, type, mapKey); } else { - putEntityInPlace(next, type, canonicalId, previousEntity); + putEntityInPlace(next, type, mapKey, previousEntity); } if (previousInvalid != null) { - nextInvalid.computeIfAbsent(type, k -> new HashMap<>()).put(canonicalId, previousInvalid); + nextInvalid.computeIfAbsent(type, k -> new HashMap<>()).put(mapKey, previousInvalid); } throw error; } @@ -774,7 +777,7 @@ public Config applySettingsDelete() { } } - private Config applyEntityWriteLocked(ResourceTypes type, String canonicalId, Object decryptedEntity) { + private Config applyEntityWriteLocked(ResourceTypes type, String mapKey, Object decryptedEntity) { Config next = shallowClone(this.config); Map> nextInvalid = cloneInvalidDeep(this.invalidEntities); @@ -787,7 +790,7 @@ private Config applyEntityWriteLocked(ResourceTypes type, String canonicalId, Ob } BiConsumer onSkip = skipRouter(nextInvalid); - EntityChange change = new EntityChange(type, canonicalId, decryptedEntity); + EntityChange change = new EntityChange(type, mapKey, decryptedEntity); applyChangeInPlace(next, nextInvalid, change, onSkip); this.config = next; @@ -795,7 +798,7 @@ private Config applyEntityWriteLocked(ResourceTypes type, String canonicalId, Ob return next; } - private Config applyEntityDeleteLocked(ResourceTypes type, String canonicalId) { + private Config applyEntityDeleteLocked(ResourceTypes type, String mapKey) { Config next = shallowClone(this.config); Map> nextInvalid = cloneInvalidDeep(this.invalidEntities); @@ -806,7 +809,7 @@ private Config applyEntityDeleteLocked(ResourceTypes type, String canonicalId) { } BiConsumer onSkip = skipRouter(nextInvalid); - EntityChange change = new EntityChange(type, canonicalId, null); + EntityChange change = new EntityChange(type, mapKey, null); applyChangeInPlace(next, nextInvalid, change, onSkip); this.config = next; @@ -890,8 +893,6 @@ private static Config shallowClone(Config base) { next.setRoutes(base.getRoutes()); next.setApplicationTypeSchemas(base.getApplicationTypeSchemas()); next.setCatalogSchemas(base.getCatalogSchemas()); - next.setApplicationSchemaAliasesById(base.getApplicationSchemaAliasesById()); - next.setCatalogSchemaAliasesById(base.getCatalogSchemaAliasesById()); next.setApplications(base.getApplications()); next.setToolsets(base.getToolsets()); next.setRetriableErrorCodes(base.getRetriableErrorCodes()); @@ -909,10 +910,10 @@ private static Map> cloneInvalid } private static void clearInvalid(Map> invalid, - ResourceTypes type, String canonicalId) { + ResourceTypes type, String mapKey) { Map perType = invalid.get(type); if (perType != null) { - perType.remove(canonicalId); + perType.remove(mapKey); } } @@ -934,117 +935,55 @@ private static void cloneTypeMap(Config config, ResourceTypes type) { case ROLE -> config.setRoles(new HashMap<>(config.getRoles())); case PROJECT_KEY -> config.setKeys(new HashMap<>(config.getKeys())); case ROUTE -> config.setRoutes(new LinkedHashMap<>(config.getRoutes())); - case APP_TYPE_SCHEMA -> { - config.setApplicationTypeSchemas(new LinkedHashMap<>(config.getApplicationTypeSchemas())); - config.setApplicationSchemaAliasesById(new HashMap<>(config.getApplicationSchemaAliasesById())); - } - case CATALOG_SCHEMA -> { - config.setCatalogSchemas(new LinkedHashMap<>(config.getCatalogSchemas())); - config.setCatalogSchemaAliasesById(new HashMap<>(config.getCatalogSchemaAliasesById())); - } + case APP_TYPE_SCHEMA -> config.setApplicationTypeSchemas(new LinkedHashMap<>(config.getApplicationTypeSchemas())); + case CATALOG_SCHEMA -> config.setCatalogSchemas(new LinkedHashMap<>(config.getCatalogSchemas())); case APPLICATION -> config.setApplications(new LinkedHashMap<>(config.getApplications())); case TOOL_SET -> config.setToolsets(new LinkedHashMap<>(config.getToolsets())); default -> throw new IllegalArgumentException("Unsupported type for partial update: " + type); } } - private static Object peekEntity(Config config, ResourceTypes type, String canonicalId) { + private static Object peekEntity(Config config, ResourceTypes type, String mapKey) { return switch (type) { - case MODEL -> config.getModels().get(canonicalId); - case INTERCEPTOR -> config.getInterceptors().get(canonicalId); - case ROLE -> config.getRoles().get(canonicalId); - case PROJECT_KEY -> config.getKeys().get(canonicalId); - case ROUTE -> config.getRoutes().get(canonicalId); - case APP_TYPE_SCHEMA -> config.getApplicationTypeSchemas().get(canonicalId); - case CATALOG_SCHEMA -> config.getCatalogSchemas().get(canonicalId); - case APPLICATION -> config.getApplications().get(canonicalId); - case TOOL_SET -> config.getToolsets().get(canonicalId); + case MODEL -> config.getModels().get(mapKey); + case INTERCEPTOR -> config.getInterceptors().get(mapKey); + case ROLE -> config.getRoles().get(mapKey); + case PROJECT_KEY -> config.getKeys().get(mapKey); + case ROUTE -> config.getRoutes().get(mapKey); + case APP_TYPE_SCHEMA -> config.getApplicationTypeSchemas().get(mapKey); + case CATALOG_SCHEMA -> config.getCatalogSchemas().get(mapKey); + case APPLICATION -> config.getApplications().get(mapKey); + case TOOL_SET -> config.getToolsets().get(mapKey); default -> throw new IllegalArgumentException("Unsupported type for partial update: " + type); }; } - private static void putEntityInPlace(Config config, ResourceTypes type, String canonicalId, Object entity) { + private static void putEntityInPlace(Config config, ResourceTypes type, String mapKey, Object entity) { switch (type) { - case MODEL -> config.getModels().put(canonicalId, (Model) entity); - case INTERCEPTOR -> config.getInterceptors().put(canonicalId, (Interceptor) entity); - case ROLE -> config.getRoles().put(canonicalId, (Role) entity); - case PROJECT_KEY -> config.getKeys().put(canonicalId, (Key) entity); - case ROUTE -> config.getRoutes().put(canonicalId, (Route) entity); - case APP_TYPE_SCHEMA -> - putSchemaInPlace(config.getApplicationTypeSchemas(), config.getApplicationSchemaAliasesById(), canonicalId, entity); - case CATALOG_SCHEMA -> - putSchemaInPlace(config.getCatalogSchemas(), config.getCatalogSchemaAliasesById(), canonicalId, entity); - case APPLICATION -> config.getApplications().put(canonicalId, (Application) entity); - case TOOL_SET -> config.getToolsets().put(canonicalId, (ToolSet) entity); + case MODEL -> config.getModels().put(mapKey, (Model) entity); + case INTERCEPTOR -> config.getInterceptors().put(mapKey, (Interceptor) entity); + case ROLE -> config.getRoles().put(mapKey, (Role) entity); + case PROJECT_KEY -> config.getKeys().put(mapKey, (Key) entity); + case ROUTE -> config.getRoutes().put(mapKey, (Route) entity); + case APP_TYPE_SCHEMA -> config.getApplicationTypeSchemas().put(mapKey, schemaBody(entity)); + case CATALOG_SCHEMA -> config.getCatalogSchemas().put(mapKey, schemaBody(entity)); + case APPLICATION -> config.getApplications().put(mapKey, (Application) entity); + case TOOL_SET -> config.getToolsets().put(mapKey, (ToolSet) entity); default -> throw new IllegalArgumentException("Unsupported type for partial update: " + type); } } - private static void putSchemaInPlace(Map schemas, Map aliasesById, - String canonicalId, Object entity) { - String body = schemaBody(entity); - String previousBody = schemas.put(canonicalId, body); - try { - recordSchemaAlias(schemas, aliasesById, canonicalId, previousBody, ProxyUtil.BLOB_MAPPER.readTree(body)); - } catch (JsonProcessingException e) { - log.warn("Failed to parse schema body for $id alias index: {} ({})", canonicalId, e.getMessage()); - } - } - - /** - * Records the {@code $id → canonicalId} alias for a schema body, and removes the file-defined - * entry keyed by that same $id so a migrated schema doesn't appear twice in $id-keyed listings. - * - *

    If this canonical id previously held a different $id ({@code previousBody}), that stale - * alias is evicted first — otherwise it would keep pointing here after the $id changed. - */ - public static void recordSchemaAlias(Map schemas, Map aliasesById, - String canonicalId, String previousBody, JsonNode node) { - JsonNode idNode = node.get("$id"); - String newId = idNode != null && idNode.isTextual() ? idNode.asText() : null; - String oldId = previousBody == null ? null : extractSchemaId(previousBody); - if (oldId != null && !oldId.equals(newId)) { - aliasesById.remove(oldId); - } - if (newId != null) { - schemas.remove(newId); - aliasesById.put(newId, canonicalId); - } - } - - @Nullable - private static String extractSchemaId(String body) { - try { - JsonNode idNode = ProxyUtil.BLOB_MAPPER.readTree(body).get("$id"); - return idNode != null && idNode.isTextual() ? idNode.asText() : null; - } catch (JsonProcessingException e) { - return null; - } - } - - private static void removeEntityInPlace(Config config, ResourceTypes type, String canonicalId) { + private static void removeEntityInPlace(Config config, ResourceTypes type, String mapKey) { switch (type) { - case MODEL -> config.getModels().remove(canonicalId); - case INTERCEPTOR -> config.getInterceptors().remove(canonicalId); - case ROLE -> config.getRoles().remove(canonicalId); - case PROJECT_KEY -> config.getKeys().remove(canonicalId); - case ROUTE -> config.getRoutes().remove(canonicalId); - case APP_TYPE_SCHEMA -> { - String removedBody = config.getApplicationTypeSchemas().remove(canonicalId); - String id = removedBody == null ? null : extractSchemaId(removedBody); - if (id != null) { - config.getApplicationSchemaAliasesById().remove(id); - } - } - case CATALOG_SCHEMA -> { - String removedBody = config.getCatalogSchemas().remove(canonicalId); - String id = removedBody == null ? null : extractSchemaId(removedBody); - if (id != null) { - config.getCatalogSchemaAliasesById().remove(id); - } - } - case APPLICATION -> config.getApplications().remove(canonicalId); - case TOOL_SET -> config.getToolsets().remove(canonicalId); + case MODEL -> config.getModels().remove(mapKey); + case INTERCEPTOR -> config.getInterceptors().remove(mapKey); + case ROLE -> config.getRoles().remove(mapKey); + case PROJECT_KEY -> config.getKeys().remove(mapKey); + case ROUTE -> config.getRoutes().remove(mapKey); + case APP_TYPE_SCHEMA -> config.getApplicationTypeSchemas().remove(mapKey); + case CATALOG_SCHEMA -> config.getCatalogSchemas().remove(mapKey); + case APPLICATION -> config.getApplications().remove(mapKey); + case TOOL_SET -> config.getToolsets().remove(mapKey); default -> throw new IllegalArgumentException("Unsupported type for partial update: " + type); } } @@ -1088,10 +1027,6 @@ private Config rebuild() { Map catalogSchemas = new LinkedHashMap<>(base.getCatalogSchemas()); Map applications = new LinkedHashMap<>(base.getApplications()); Map toolsets = new LinkedHashMap<>(base.getToolsets()); - // $id -> canonicalId index, built fresh each rebuild from the blob scan below; file - // entries need no alias since they're already keyed by $id. - Map applicationSchemaAliasesById = new HashMap<>(); - Map catalogSchemaAliasesById = new HashMap<>(); merged.setRetriableErrorCodes(base.getRetriableErrorCodes()); merged.setGlobalInterceptors(base.getGlobalInterceptors()); // Wire the (still-being-populated) local maps onto merged now rather than after the blob @@ -1107,8 +1042,6 @@ private Config rebuild() { merged.setCatalogSchemas(catalogSchemas); merged.setApplications(applications); merged.setToolsets(toolsets); - merged.setApplicationSchemaAliasesById(applicationSchemaAliasesById); - merged.setCatalogSchemaAliasesById(catalogSchemaAliasesById); Map blobBodies = new HashMap<>(); Map> pendingInvalid = new EnumMap<>(ResourceTypes.class); @@ -1146,7 +1079,7 @@ private Config rebuild() { // key a file-sourced entry for the same logical entity already uses — no // derivation, no shadowing. Every other managed type keeps using the canonical // id as its map key. - String mapKey = isNameAddressed(type) ? name : canonicalId; + String mapKey = isLocallyKeyed(type) ? localKey(type, name) : canonicalId; JsonNode node; try { // Parse once into JsonNode; reused below for typed deserialization and as the @@ -1189,7 +1122,7 @@ private Config rebuild() { if (type == ResourceTypes.PROJECT_KEY) { apiKeysByCanonicalId.put(canonicalId, (Key) added); } - if (isNameAddressed(type)) { + if (isLocallyKeyed(type)) { apiSourcedKeys.computeIfAbsent(type, t -> new HashSet<>()).add(mapKey); } } @@ -1227,10 +1160,9 @@ private Config rebuild() { ConfigPostProcessor.processSemantic(merged, apiKeyStore, base.getKeys(), apiKeysByCanonicalId, onSkip); // ConfigPostProcessor sets entity.name = mapKey. For models/interceptors/roles/ - // applications/toolsets the map key is now always the short name ("gpt-4"), whether the - // entry is file- or blob-sourced, so /openai/models, /openai/deployments, and rate-limit - // role-limit lookups see the same short form either way. Keys/routes/schemas are unaffected - // — their map key stays the canonical id for API entries, as before. + // applications/toolsets the map key is the short name ("gpt-4"); for schemas it is + // the decoded $id — in both cases file- and blob-sourced entries share the same map key. + // Keys and routes are unaffected — their map key stays the canonical id, as before. boolean overlayFromApi = applySettingsOverlay(merged, pendingInvalid); Map> finalInvalid = @@ -1329,80 +1261,96 @@ public static String canonicalId(ResourceDescriptor descriptor) { } /** - * The key {@code Config}'s in-memory maps use for {@code descriptor}'s entity: the short name - * (last path segment) for models/interceptors/roles/applications/toolsets, so blob-sourced and - * file-sourced entries of the same logical entity share one map key — no derivation, no - * shadowing, no separate canonical-id keying to reconcile. Every other managed type (keys, - * routes, schemas) keeps using the canonical id as its map key, unchanged. Callers that already - * have a {@link ResourceDescriptor} should use this instead of building the canonical id and - * then parsing the short name back out of it. + * The key {@code Config}'s in-memory maps use for {@code descriptor}'s entity. + *

      + *
    • Models/interceptors/roles/applications/toolsets → short name (last path segment), so + * blob-sourced and file-sourced entries of the same logical entity share one map key.
      • + *
    • Schemas ({@code APP_TYPE_SCHEMA}, {@code CATALOG_SCHEMA}) → the decoded JSON-Schema + * {@code $id}, recovered from the blob name via {@link UrlUtil#unescapeSchemaId}.
      • + *
    • Every other type → canonical id (unchanged).
      • + *
    + * Callers that already have a {@link ResourceDescriptor} should use this instead of building + * the canonical id and then parsing the key back out of it. */ public static String mapKeyFor(ResourceTypes type, ResourceDescriptor descriptor) { - return isNameAddressed(type) ? descriptor.getName() : canonicalId(descriptor); + return isLocallyKeyed(type) ? localKey(type, descriptor.getName()) : canonicalId(descriptor); } - private static boolean isNameAddressed(ResourceTypes type) { + /** + * Returns {@code true} for types whose {@code Config} map key is derived from the blob file + * name rather than from the full canonical id. For schema types the blob name is the + * {@link UrlUtil#escapeSchemaId escaped} {@code $id}; for the remaining types it is the short + * entity name directly. + */ + private static boolean isLocallyKeyed(ResourceTypes type) { return switch (type) { - case MODEL, INTERCEPTOR, ROLE, APPLICATION, TOOL_SET -> true; + case MODEL, INTERCEPTOR, ROLE, APPLICATION, TOOL_SET, + APP_TYPE_SCHEMA, CATALOG_SCHEMA -> true; default -> false; }; } + /** + * Derives the {@code Config} map key from a blob file {@code name} for a locally-keyed type. + * Schema blob names are {@link UrlUtil#escapeSchemaId escaped} {@code $id} values and must be + * unescaped; all other locally-keyed types use the blob name verbatim. + */ + private static String localKey(ResourceTypes type, String blobName) { + return switch (type) { + case APP_TYPE_SCHEMA, CATALOG_SCHEMA -> UrlUtil.unescapeSchemaId(blobName); + default -> blobName; + }; + } + /** * Reads the type-map to mutate off {@code config} rather than taking one map parameter per * managed type — {@code config}'s maps are wired up by {@link #rebuild} before the blob scan * that calls this runs, so they're already the right (still being populated) instances. */ - private static Object addBlobEntity(Config config, ResourceTypes type, String canonicalId, JsonNode node) + private static Object addBlobEntity(Config config, ResourceTypes type, String mapKey, JsonNode node) throws JsonProcessingException { switch (type) { case MODEL -> { Model entity = ProxyUtil.BLOB_MAPPER.treeToValue(node, Model.class); - warnIfReplaced(type, canonicalId, config.getModels().put(canonicalId, entity)); + warnIfReplaced(type, mapKey, config.getModels().put(mapKey, entity)); return entity; } case INTERCEPTOR -> { Interceptor entity = ProxyUtil.BLOB_MAPPER.treeToValue(node, Interceptor.class); - warnIfReplaced(type, canonicalId, config.getInterceptors().put(canonicalId, entity)); + warnIfReplaced(type, mapKey, config.getInterceptors().put(mapKey, entity)); return entity; } case ROLE -> { Role entity = ProxyUtil.BLOB_MAPPER.treeToValue(node, Role.class); - warnIfReplaced(type, canonicalId, config.getRoles().put(canonicalId, entity)); + warnIfReplaced(type, mapKey, config.getRoles().put(mapKey, entity)); return entity; } case PROJECT_KEY -> { Key entity = ProxyUtil.BLOB_MAPPER.treeToValue(node, Key.class); - warnIfReplaced(type, canonicalId, config.getKeys().put(canonicalId, entity)); + warnIfReplaced(type, mapKey, config.getKeys().put(mapKey, entity)); return entity; } case ROUTE -> { Route entity = ProxyUtil.BLOB_MAPPER.treeToValue(node, Route.class); - warnIfReplaced(type, canonicalId, config.getRoutes().put(canonicalId, entity)); + warnIfReplaced(type, mapKey, config.getRoutes().put(mapKey, entity)); return entity; } case APP_TYPE_SCHEMA -> { - Map schemas = config.getApplicationTypeSchemas(); - String previousBody = schemas.put(canonicalId, node.toString()); - warnIfReplaced(type, canonicalId, previousBody); - recordSchemaAlias(schemas, config.getApplicationSchemaAliasesById(), canonicalId, previousBody, node); + warnIfReplaced(type, mapKey, config.getApplicationTypeSchemas().put(mapKey, node.toString())); return null; } case CATALOG_SCHEMA -> { - Map catalogSchemas = config.getCatalogSchemas(); - String previousBody = catalogSchemas.put(canonicalId, node.toString()); - warnIfReplaced(type, canonicalId, previousBody); - recordSchemaAlias(catalogSchemas, config.getCatalogSchemaAliasesById(), canonicalId, previousBody, node); + warnIfReplaced(type, mapKey, config.getCatalogSchemas().put(mapKey, node.toString())); return null; } case APPLICATION -> { Application entity = ProxyUtil.BLOB_MAPPER.treeToValue(node, Application.class); - warnIfReplaced(type, canonicalId, config.getApplications().put(canonicalId, entity)); + warnIfReplaced(type, mapKey, config.getApplications().put(mapKey, entity)); return entity; } case TOOL_SET -> { ToolSet entity = ProxyUtil.BLOB_MAPPER.treeToValue(node, ToolSet.class); - warnIfReplaced(type, canonicalId, config.getToolsets().put(canonicalId, entity)); + warnIfReplaced(type, mapKey, config.getToolsets().put(mapKey, entity)); return entity; } default -> { @@ -1419,17 +1367,17 @@ private static void warnIfReplaced(ResourceTypes type, String canonicalId, Objec } } - private static void removeAddedEntity(Config config, ResourceTypes type, String canonicalId) { + private static void removeAddedEntity(Config config, ResourceTypes type, String mapKey) { switch (type) { - case MODEL -> config.getModels().remove(canonicalId); - case INTERCEPTOR -> config.getInterceptors().remove(canonicalId); - case ROLE -> config.getRoles().remove(canonicalId); - case PROJECT_KEY -> config.getKeys().remove(canonicalId); - case ROUTE -> config.getRoutes().remove(canonicalId); - case APP_TYPE_SCHEMA -> config.getApplicationTypeSchemas().remove(canonicalId); - case CATALOG_SCHEMA -> config.getCatalogSchemas().remove(canonicalId); - case APPLICATION -> config.getApplications().remove(canonicalId); - case TOOL_SET -> config.getToolsets().remove(canonicalId); + case MODEL -> config.getModels().remove(mapKey); + case INTERCEPTOR -> config.getInterceptors().remove(mapKey); + case ROLE -> config.getRoles().remove(mapKey); + case PROJECT_KEY -> config.getKeys().remove(mapKey); + case ROUTE -> config.getRoutes().remove(mapKey); + case APP_TYPE_SCHEMA -> config.getApplicationTypeSchemas().remove(mapKey); + case CATALOG_SCHEMA -> config.getCatalogSchemas().remove(mapKey); + case APPLICATION -> config.getApplications().remove(mapKey); + case TOOL_SET -> config.getToolsets().remove(mapKey); default -> { /* no-op */ } } } diff --git a/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java b/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java index d65a42fee..448655acc 100644 --- a/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java +++ b/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java @@ -43,8 +43,11 @@ import com.epam.aidial.core.storage.service.LockService; import com.epam.aidial.core.storage.service.ResourceService; import com.epam.aidial.core.storage.util.EtagHeader; +import com.epam.aidial.core.storage.util.UrlUtil; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import io.vertx.core.Future; import io.vertx.core.buffer.Buffer; import lombok.extern.slf4j.Slf4j; @@ -53,6 +56,8 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Comparator; + +import static com.epam.aidial.core.server.util.PlatformCanonicalIdUtil.lastSegment; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -272,8 +277,6 @@ static Config newScratch(MergedConfigStore mergedConfigStore) { scratch.setInterceptors(new HashMap<>(live.getInterceptors())); scratch.setApplicationTypeSchemas(new HashMap<>(live.getApplicationTypeSchemas())); scratch.setCatalogSchemas(new HashMap<>(live.getCatalogSchemas())); - scratch.setApplicationSchemaAliasesById(new HashMap<>(live.getApplicationSchemaAliasesById())); - scratch.setCatalogSchemaAliasesById(new HashMap<>(live.getCatalogSchemaAliasesById())); scratch.setApplications(new HashMap<>(live.getApplications())); scratch.setToolsets(new HashMap<>(live.getToolsets())); scratch.setRoles(new HashMap<>(live.getRoles())); @@ -359,15 +362,11 @@ static ValidationResult validateOnly(AdminManifest entry, Config scratch, boolea if (!entry.spec().isObject()) { return new ValidationResult(id, ValidationStatus.FAILED, "Schema spec must be a JSON object"); } - ConfigResourceController.rejectSchemaIdCollision(scratch, ResourceTypes.APP_TYPE_SCHEMA, - entry.name(), entry.spec()); } case "CatalogSchema" -> { if (!entry.spec().isObject()) { return new ValidationResult(id, ValidationStatus.FAILED, "CatalogSchema spec must be a JSON object"); } - ConfigResourceController.rejectSchemaIdCollision(scratch, ResourceTypes.CATALOG_SCHEMA, - entry.name(), entry.spec()); } default -> { return new ValidationResult(id, ValidationStatus.FAILED, "Unknown kind: " + entry.kind()); @@ -402,8 +401,8 @@ private EntityResult applySingle(AdminManifest entry, Config scratch, List applySettings(entry, id, parsed); - case "Schema" -> applySchema(entry, id, parsed, scratch, pending, ResourceTypes.APP_TYPE_SCHEMA); - case "CatalogSchema" -> applySchema(entry, id, parsed, scratch, pending, ResourceTypes.CATALOG_SCHEMA); + case "Schema" -> applySchema(entry, id, parsed, pending, ResourceTypes.APP_TYPE_SCHEMA); + case "CatalogSchema" -> applySchema(entry, id, parsed, pending, ResourceTypes.CATALOG_SCHEMA); case "Interceptor" -> applyManagedEntity(entry, id, parsed, ResourceTypes.INTERCEPTOR, Interceptor.class, scratch, pending); case "Role" -> applyManagedEntity(entry, id, parsed, ResourceTypes.ROLE, Role.class, scratch, pending); case "Route" -> applyManagedEntity(entry, id, parsed, ResourceTypes.ROUTE, Route.class, scratch, pending); @@ -427,29 +426,32 @@ private EntityResult applySettings(AdminManifest entry, String id, ParsedName pa return new EntityResult(id, AdminApplyStatus.APPLIED, null); } - private EntityResult applySchema(AdminManifest entry, String id, ParsedName parsed, Config scratch, + private EntityResult applySchema(AdminManifest entry, String id, ParsedName parsed, List pending, ResourceTypes type) { if (!entry.spec().isObject()) { return new EntityResult(id, AdminApplyStatus.FAILED, "Schema spec must be a JSON object"); } - ResourceDescriptor descriptor = ResourceDescriptorFactory.fromDecoded( - type, parsed.bucket(), parsed.location(), parsed.name()); - try { - ConfigResourceController.rejectSchemaIdCollision(scratch, type, - MergedConfigStore.canonicalId(descriptor), entry.spec()); - } catch (HttpException e) { - return new EntityResult(id, AdminApplyStatus.FAILED, e.getMessage()); + // parsed.name() is the percent-encoded $id segment from the manifest name; decode once. + String decodedSchemaId = UrlUtil.decodePath(parsed.name()); + ResourceDescriptor descriptor = ResourceDescriptorFactory.fromDecodedAtomicName( + type, parsed.bucket(), parsed.location(), decodedSchemaId); + // S3: ensure body.$id matches the derived $id from the canonical path. + JsonNode spec = entry.spec(); + JsonNode idNode = spec.get("$id"); + if (idNode == null || !decodedSchemaId.equals(idNode.asText())) { + spec = spec.deepCopy(); + ((ObjectNode) spec).put("$id", decodedSchemaId); } String blobBody; try { - blobBody = ProxyUtil.BLOB_MAPPER.writeValueAsString(entry.spec()); + blobBody = ProxyUtil.BLOB_MAPPER.writeValueAsString(spec); } catch (JsonProcessingException e) { // Drop e.getOriginalMessage() — it can echo verbatim schema content (potentially // submitted secrets). Surface a generic failure tied to the entity id. return new EntityResult(id, AdminApplyStatus.FAILED, "Failed to serialize schema for " + id); } resourceService.putResource(descriptor, blobBody, EtagHeader.ANY); - pending.add(new EntityChange(type, MergedConfigStore.canonicalId(descriptor), entry.spec())); + pending.add(new EntityChange(type, MergedConfigStore.mapKeyFor(type, descriptor), spec)); return new EntityResult(id, AdminApplyStatus.APPLIED, null); } @@ -629,26 +631,18 @@ static void mutateScratch(Config scratch, AdminManifest entry) { scratch.getToolsets().put(entry.name(), toolSet); } case "Schema" -> { - String json; + String schemaId = UrlUtil.decodePath(lastSegment(entry.name())); try { - json = ProxyUtil.BLOB_MAPPER.writeValueAsString(entry.spec()); - } catch (JsonProcessingException e) { - return; + scratch.getApplicationTypeSchemas().put(schemaId, ProxyUtil.BLOB_MAPPER.writeValueAsString(entry.spec())); + } catch (JsonProcessingException ignored) { } - String previousJson = scratch.getApplicationTypeSchemas().put(entry.name(), json); - MergedConfigStore.recordSchemaAlias(scratch.getApplicationTypeSchemas(), - scratch.getApplicationSchemaAliasesById(), entry.name(), previousJson, entry.spec()); } case "CatalogSchema" -> { - String json; + String schemaId = UrlUtil.decodePath(lastSegment(entry.name())); try { - json = ProxyUtil.BLOB_MAPPER.writeValueAsString(entry.spec()); - } catch (JsonProcessingException e) { - return; + scratch.getCatalogSchemas().put(schemaId, ProxyUtil.BLOB_MAPPER.writeValueAsString(entry.spec())); + } catch (JsonProcessingException ignored) { } - String previousJson = scratch.getCatalogSchemas().put(entry.name(), json); - MergedConfigStore.recordSchemaAlias(scratch.getCatalogSchemas(), - scratch.getCatalogSchemaAliasesById(), entry.name(), previousJson, entry.spec()); } default -> { /* unknown kinds never reach this code path */ } } diff --git a/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java b/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java index d37c838af..add95330a 100644 --- a/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java +++ b/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java @@ -46,6 +46,7 @@ import com.epam.aidial.core.storage.service.LockService; import com.epam.aidial.core.storage.service.ResourceService; import com.epam.aidial.core.storage.util.EtagHeader; +import com.epam.aidial.core.storage.util.UrlUtil; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ArrayNode; @@ -951,7 +952,11 @@ public Future handle() throws Exception { if (method == HttpMethod.GET || method == HttpMethod.HEAD) { return handleGet(); } + ResourceTypes nameValidationType = resourceType(); + boolean isSchemaType = nameValidationType == ResourceTypes.APP_TYPE_SCHEMA + || nameValidationType == ResourceTypes.CATALOG_SCHEMA; if ((method == HttpMethod.PUT || method == HttpMethod.DELETE) + && !isSchemaType && !ENTITY_NAME_PATTERN.matcher(path == null ? "" : path).matches()) { context.respond(HttpStatus.BAD_REQUEST, "Invalid entity name segment: must match " + ENTITY_NAME_PATTERN.pattern()); @@ -1003,9 +1008,8 @@ public Future handle() throws Exception { return respondMethodNotAllowed(); } - private Future handleGet() throws JsonProcessingException { + private Future handleGet() { Config config = context.getConfig(); - boolean admin = authorizationService.isAdmin(context); // Per-entity GET is blob-only (slice U.1): only canonical-ID lookups resolve here. // File-sourced entries are inspected via /v1/admin/config/file/{type}[/{name}]. // Slice U.4: secret fields drop on response via @JsonProperty(WRITE_ONLY) — there is no @@ -1023,8 +1027,10 @@ private Future handleGet() throws JsonProcessingException { case ROUTE -> handleSingleGet( config.getRoutes(), ResourceTypes.ROUTE, (key, route) -> projectItem(route, key)); - case APP_TYPE_SCHEMA -> handleSchemaGet(config.getApplicationTypeSchemas(), ResourceTypes.APP_TYPE_SCHEMA, admin); - case CATALOG_SCHEMA -> handleSchemaGet(config.getCatalogSchemas(), ResourceTypes.CATALOG_SCHEMA, admin); + case APP_TYPE_SCHEMA -> handleSingleGetFromBlob(ResourceTypes.APP_TYPE_SCHEMA, + (schemaId, node) -> projectSchemaItem(schemaId, (JsonNode) node)); + case CATALOG_SCHEMA -> handleSingleGetFromBlob(ResourceTypes.CATALOG_SCHEMA, + (schemaId, node) -> projectSchemaItem(schemaId, (JsonNode) node)); case APPLICATION -> handleSingleGetFromBlob(ResourceTypes.APPLICATION, (key, application) -> redactExternalServiceSecrets(projectItem(application, key))); case TOOL_SET -> handleSingleGetFromBlob(ResourceTypes.TOOL_SET, @@ -1080,14 +1086,15 @@ private Future handleSingleGet(Map source, } /** - * Per-entity GET for {@code MODEL}/{@code INTERCEPTOR}/{@code ROLE}/{@code APPLICATION}/ - * {@code TOOL_SET} — these five types key {@code Config}'s in-memory map by short name, the - * same key a file-sourced entry for the same logical entity already uses, so the map can no - * longer tell a genuinely blob-managed entity apart from a file-only one sharing that short - * name. This reads and decrypts blob storage directly by descriptor instead — the same - * pattern PUT/DELETE already use for these types — so this endpoint only ever serves entities - * that actually exist in the {@code platform} bucket. {@code Config}'s map stays purely a - * runtime-resolution structure. + * Per-entity GET for locally-keyed types ({@code MODEL}, {@code INTERCEPTOR}, {@code ROLE}, + * {@code APPLICATION}, {@code TOOL_SET}, {@code APP_TYPE_SCHEMA}, {@code CATALOG_SCHEMA}) — + * these types key {@code Config}'s in-memory map by a local identifier (short name or + * {@code $id}), the same key a file-sourced entry for the same logical entity already uses, so + * the map can no longer tell a genuinely blob-managed entity apart from a file-only one sharing + * that key. This reads blob storage directly by descriptor instead — the same pattern PUT/DELETE + * already use for these types — so this endpoint only ever serves entities that actually exist + * in the {@code platform} bucket. {@code Config}'s map stays purely a runtime-resolution + * structure. */ private Future handleSingleGetFromBlob(ResourceTypes resourceType, BiFunction projector) { if (path == null || path.isEmpty()) { @@ -1138,6 +1145,14 @@ private Object readAndDecrypt(ResourceTypes type, String body, ResourceDescripto return switch (type) { case APPLICATION -> applicationService.getApplicationWithDecryptedSecrets(descriptor).getValue(); case TOOL_SET -> toolSetService.getToolSetWithDecryptedAuthSettings(descriptor).getValue(); + case APP_TYPE_SCHEMA, CATALOG_SCHEMA -> { + try { + yield ProxyUtil.BLOB_MAPPER.readTree(body); + } catch (JsonProcessingException e) { + throw new HttpException(HttpStatus.INTERNAL_SERVER_ERROR, + "Stored schema is malformed at " + locationOf(e)); + } + } default -> { Object entity; try { @@ -1217,9 +1232,9 @@ private ResourceDescriptor descriptorFor(ResourceTypes type) { ResourceDescriptor.PLATFORM_BUCKET, ResourceDescriptor.PLATFORM_LOCATION, path); case ROUTE -> ResourceDescriptorFactory.fromDecoded(ResourceTypes.ROUTE, ResourceDescriptor.PLATFORM_BUCKET, ResourceDescriptor.PLATFORM_LOCATION, path); - case APP_TYPE_SCHEMA -> ResourceDescriptorFactory.fromDecoded(ResourceTypes.APP_TYPE_SCHEMA, + case APP_TYPE_SCHEMA -> ResourceDescriptorFactory.fromDecodedAtomicName(ResourceTypes.APP_TYPE_SCHEMA, ResourceDescriptor.PLATFORM_BUCKET, ResourceDescriptor.PLATFORM_LOCATION, path); - case CATALOG_SCHEMA -> ResourceDescriptorFactory.fromDecoded(ResourceTypes.CATALOG_SCHEMA, + case CATALOG_SCHEMA -> ResourceDescriptorFactory.fromDecodedAtomicName(ResourceTypes.CATALOG_SCHEMA, ResourceDescriptor.PLATFORM_BUCKET, ResourceDescriptor.PLATFORM_LOCATION, path); case APPLICATION -> ResourceDescriptorFactory.fromDecoded(ResourceTypes.APPLICATION, ResourceDescriptor.PLATFORM_BUCKET, ResourceDescriptor.PLATFORM_LOCATION, path); @@ -1230,41 +1245,11 @@ private ResourceDescriptor descriptorFor(ResourceTypes type) { } private String canonicalId() { - return entityType + "/" + bucket + "/" + path; - } - - private Future handleSchemaGet(Map schemas, ResourceTypes resourceType, boolean admin) throws JsonProcessingException { - Map invalid = mergedConfigStore.getInvalidEntities() - .getOrDefault(resourceType, Map.of()); - if (path == null || path.isEmpty()) { - context.respond(HttpStatus.NOT_FOUND); - return Future.succeededFuture(); - } - // Per-entity GET is blob-only (U.1): canonical-ID lookup only; file-defined schemas - // are inspected via /v1/admin/config/file/schemas/{name}. - String schemaJson = schemas.get(canonicalId()); - if (schemaJson != null) { - final String matched = canonicalId(); - final String json = schemaJson; - return respondNotModifiedIfMatched(matched) - .onSuccess(notModified -> { - if (!notModified) { - try { - context.respond(HttpStatus.OK, projectSchemaItem(matched, json)); - } catch (JsonProcessingException e) { - context.respond(HttpStatus.INTERNAL_SERVER_ERROR, e.getMessage()); - } - } - }) - .onFailure(this::handleWriteError); - } - InvalidEntityRecord invalidRecord = invalid.get(canonicalId()); - if (invalidRecord != null) { - context.respond(HttpStatus.OK, projectInvalidItem(invalidRecord, admin)); - return Future.succeededFuture(); + ResourceTypes type = resourceType(); + if (type == ResourceTypes.APP_TYPE_SCHEMA || type == ResourceTypes.CATALOG_SCHEMA) { + return entityType + "/" + bucket + "/" + UrlUtil.encodePathSegment(path); } - context.respond(HttpStatus.NOT_FOUND); - return Future.succeededFuture(); + return entityType + "/" + bucket + "/" + path; } private Future handleSettingsGet(Config config) { @@ -1380,20 +1365,23 @@ private Future handleAppOrToolSetPut() { } return taskExecutor.submit(() -> lockService.underBucketLocks(MergedConfigStore.ADMIN_BUCKET_LOCATIONS, () -> { rejectDuplicateDeploymentId(type, path); - Object decrypted; // The platform bucket requires explicit admin access for every operation (see // AdminRoleAuthorizationService), not just an admin-AND-public-bucket combination like // ResourceController's adminPublicWrite — so, same as AdminApplyController's bulk apply, // this path is always admin context and may preserve forwardAuthToken. - if (type == ResourceTypes.APPLICATION) { - Application application = treeToEntity(requestNode, Application.class); - applicationService.putApplication(descriptor, etag, author, application, true, AdminManagedFieldsWriteMode.AUTHORITATIVE); - decrypted = applicationService.getApplicationWithDecryptedSecrets(descriptor).getValue(); - } else { - ToolSet toolSet = treeToEntity(requestNode, ToolSet.class); - toolSetService.putToolSet(descriptor, etag, author, toolSet, true); - decrypted = toolSetService.getToolSetWithDecryptedAuthSettings(descriptor).getValue(); - } + Object decrypted = switch (type) { + case APPLICATION -> { + Application application = treeToEntity(requestNode, Application.class); + applicationService.putApplication(descriptor, etag, author, application, true, AdminManagedFieldsWriteMode.AUTHORITATIVE); + yield applicationService.getApplicationWithDecryptedSecrets(descriptor).getValue(); + } + case TOOL_SET -> { + ToolSet toolSet = treeToEntity(requestNode, ToolSet.class); + toolSetService.putToolSet(descriptor, etag, author, toolSet, true); + yield toolSetService.getToolSetWithDecryptedAuthSettings(descriptor).getValue(); + } + default -> throw new IllegalArgumentException("Unexpected resource type: " + type); + }; mergedConfigStore.applyEntityWrite(type, MergedConfigStore.mapKeyFor(type, descriptor), decrypted); return resourceService.getResourceMetadata(descriptor); })); @@ -1416,13 +1404,15 @@ private Future handleAppOrToolSetDelete() { EtagHeader etag = ProxyUtil.etag(context.getRequest()); taskExecutor.submit(() -> lockService.underBucketLocks(MergedConfigStore.ADMIN_BUCKET_LOCATIONS, () -> { - if (type == ResourceTypes.APPLICATION) { - applicationService.deleteApplication(descriptor, etag); - } else { - boolean deleted = toolSetService.deleteToolset(context, descriptor, etag); - if (!deleted) { - throw new HttpException(HttpStatus.NOT_FOUND, "Resource not found: " + descriptor.getUrl()); + switch (type) { + case APPLICATION -> applicationService.deleteApplication(descriptor, etag); + case TOOL_SET -> { + boolean deleted = toolSetService.deleteToolset(context, descriptor, etag); + if (!deleted) { + throw new HttpException(HttpStatus.NOT_FOUND, "Resource not found: " + descriptor.getUrl()); + } } + default -> throw new IllegalArgumentException("Unexpected resource type: " + type); } mergedConfigStore.applyEntityDelete(type, MergedConfigStore.mapKeyFor(type, descriptor)); return true; @@ -1475,8 +1465,12 @@ private Future handlePut() { if (spec.entityClass() == null) { ResourceTypes schemaType = resourceType(); if (schemaType == ResourceTypes.APP_TYPE_SCHEMA || schemaType == ResourceTypes.CATALOG_SCHEMA) { - rejectSchemaIdCollision(mergedConfigStore.get(), schemaType, - MergedConfigStore.canonicalId(descriptor), requestNode); + // S3: override body.$id to match the decoded URL segment so the blob always + // round-trips cleanly — canonicalId = type/platform/encode($id), $id = path. + JsonNode idNode = requestNode.get("$id"); + if (idNode == null || !path.equals(idNode.asText())) { + ((ObjectNode) requestNode).put("$id", path); + } } blobBody = requestNode.toString(); } else { @@ -1636,43 +1630,6 @@ private static ResourceTypes typeOf(ResourceDescriptor descriptor) { return (ResourceTypes) descriptor.getType(); } - /** - * App-type/catalog schemas are looked up by their body-embedded {@code $id} - * ({@link Config#getCustomApplicationSchema}/{@link Config#getCatalogSchema}) via - * {@code MergedConfigStore}'s {@code $id -> canonicalId} alias index, which only ever holds - * one canonical id per $id. Reject a write whose $id is already claimed by a *different* - * canonical id, rather than let the index silently pick whichever write landed most recently. - * The alias index only ever contains blob-sourced entries (file-sourced schemas are keyed - * directly by $id and never added to it), so a file-schema sharing this $id — the expected - * pre-migration predecessor a blob write is meant to shadow — is never mistaken for a - * collision here. - * - *

    Package-visible: shared with {@link AdminApplyController#applySchema}/ - * {@link AdminApplyController#validateOnly}, the other write/precheck paths for these types. - * Takes a {@link Config} snapshot rather than {@link MergedConfigStore} so callers can pass - * either the live merged config or a batch-apply {@code scratch} clone. - */ - static void rejectSchemaIdCollision(Config snapshot, ResourceTypes type, String thisCanonicalId, JsonNode requestNode) { - if (snapshot == null) { - return; - } - JsonNode idNode = requestNode.get("$id"); - if (idNode == null || !idNode.isTextual()) { - return; - } - String id = idNode.asText(); - Map aliasesById = switch (type) { - case APP_TYPE_SCHEMA -> snapshot.getApplicationSchemaAliasesById(); - case CATALOG_SCHEMA -> snapshot.getCatalogSchemaAliasesById(); - default -> throw new IllegalArgumentException("Unsupported type for schema $id check: " + type); - }; - String owner = aliasesById.get(id); - if (owner != null && !owner.equals(thisCanonicalId)) { - throw new HttpException(HttpStatus.CONFLICT, - "Schema $id '" + id + "' is already used by '" + owner + "'"); - } - } - /** * Rejects a MODEL/INTERCEPTOR/APPLICATION/TOOL_SET write whose short name is already claimed * by a different deployment (of any of those four types) in the live merged Config — the @@ -1877,10 +1834,7 @@ private static ObjectNode redactExternalServiceSecrets(ObjectNode node) { return node; } - private ObjectNode projectSchemaItem(String name, String json) - throws JsonProcessingException { - // applicationTypeSchemas stores raw JSON strings; parse for projection. - JsonNode schema = ProxyUtil.MAPPER.readTree(json); + private ObjectNode projectSchemaItem(String name, JsonNode schema) { ObjectNode node = ProxyUtil.MAPPER.createObjectNode(); if (schema.isObject()) { node.setAll((ObjectNode) schema); diff --git a/server/src/main/java/com/epam/aidial/core/server/util/ResourceDescriptorFactory.java b/server/src/main/java/com/epam/aidial/core/server/util/ResourceDescriptorFactory.java index cb4222df2..5d95f4014 100644 --- a/server/src/main/java/com/epam/aidial/core/server/util/ResourceDescriptorFactory.java +++ b/server/src/main/java/com/epam/aidial/core/server/util/ResourceDescriptorFactory.java @@ -58,6 +58,23 @@ public static ResourceDescriptor fromDecoded(ResourceType type, String bucketNam return from(type, bucketName, bucketLocation, elements, ResourceUtil.isFolder(path)); } + /** + * Like {@link #fromDecoded}, but treats {@code decodedName} as a single atomic resource name — + * never splits it on '/'. Use for identifiers (like a JSON-Schema {@code $id}) that are + * themselves URIs and therefore expected to contain '/', not folder-hierarchy separators. + * The name is escaped via {@link UrlUtil#escapeSchemaId} so the physical blob key is flat + * while keeping human-readable URI characters like ':' intact. + */ + public static ResourceDescriptor fromDecodedAtomicName(ResourceType type, String bucketName, + String bucketLocation, String decodedName) { + verify(bucketLocation.endsWith(ResourceDescriptor.PATH_SEPARATOR), "Bucket location must end with /"); + String physicalName = UrlUtil.escapeSchemaId(decodedName); + ResourceDescriptor resource = from(type, bucketName, bucketLocation, List.of(physicalName), false); + verify(resource.getAbsoluteFilePath().getBytes(StandardCharsets.UTF_8).length <= MAX_PATH_SIZE, + "Resource path exceeds max allowed size: " + MAX_PATH_SIZE); + return resource; + } + public static ResourceDescriptor fromPublicUrl(String url) { return fromUrl(url, ResourceDescriptor.PUBLIC_BUCKET, ResourceDescriptor.PUBLIC_LOCATION, null); } diff --git a/storage/src/main/java/com/epam/aidial/core/storage/util/UrlUtil.java b/storage/src/main/java/com/epam/aidial/core/storage/util/UrlUtil.java index b5a87f965..ca1d6a4b6 100644 --- a/storage/src/main/java/com/epam/aidial/core/storage/util/UrlUtil.java +++ b/storage/src/main/java/com/epam/aidial/core/storage/util/UrlUtil.java @@ -32,6 +32,25 @@ public String encodePathSegment(@Nullable String segment) { return ENCODER.escape(segment); } + /** + * Escapes a JSON Schema {@code $id} so it can be stored as a flat blob-storage key: replaces + * {@code %} with {@code %25} first, then {@code /} with {@code %2F}. The result contains no + * literal {@code /} characters while preserving all other URI characters (e.g. {@code :}). + * Inverse: {@link #unescapeSchemaId}. + */ + public static String escapeSchemaId(String schemaId) { + return schemaId.replace("%", "%25").replace("/", "%2F"); + } + + /** + * Reverses {@link #escapeSchemaId}: replaces {@code %2F} with {@code /} first, then + * {@code %25} with {@code %}. The reversed order ensures a literal {@code %2F} in the + * original value (stored as {@code %252F}) is restored correctly. + */ + public static String unescapeSchemaId(String escaped) { + return escaped.replace("%2F", "/").replace("%25", "%"); + } + public String encodePath(@Nullable String path) { if (path == null) { return null; From 26fb62609ed3e80c065902b794a5928d1f36e40e Mon Sep 17 00:00:00 2001 From: Kiryl_Kurnosenka Date: Thu, 13 Aug 2026 23:15:15 +0300 Subject: [PATCH 12/15] fix: remove comments --- .../src/main/java/com/epam/aidial/core/config/Config.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/config/src/main/java/com/epam/aidial/core/config/Config.java b/config/src/main/java/com/epam/aidial/core/config/Config.java index 89a7a4270..2842af901 100644 --- a/config/src/main/java/com/epam/aidial/core/config/Config.java +++ b/config/src/main/java/com/epam/aidial/core/config/Config.java @@ -80,9 +80,6 @@ public boolean isDeploymentExists(String deploymentId) { return selectDeployment(deploymentId) != null; } - /** - * @return the schema body, or {@code null} if {@code schemaId} is null or unresolved - */ @JsonIgnore public String getCustomApplicationSchema(URI schemaId) { if (schemaId == null) { @@ -91,9 +88,6 @@ public String getCustomApplicationSchema(URI schemaId) { return applicationTypeSchemas.get(schemaId.toString()); } - /** - * @return the schema body, or {@code null} if {@code schemaId} is null or unresolved - */ @JsonIgnore public String getCatalogSchema(URI schemaId) { if (schemaId == null) { From 277a4a98bdea46120d3b7d9e83d8adbb45c81b11 Mon Sep 17 00:00:00 2001 From: Kiryl_Kurnosenka Date: Thu, 13 Aug 2026 23:17:38 +0300 Subject: [PATCH 13/15] =?UTF-8?q?fix:=20correct=20ConfigPostProcessor=20co?= =?UTF-8?q?mment=20=E2=80=94=20schemas=20key=20by=20\$id=20not=20canonical?= =?UTF-8?q?=20id?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 4.6 --- .../epam/aidial/core/server/config/ConfigPostProcessor.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/com/epam/aidial/core/server/config/ConfigPostProcessor.java b/server/src/main/java/com/epam/aidial/core/server/config/ConfigPostProcessor.java index 7658aa27d..55e961733 100644 --- a/server/src/main/java/com/epam/aidial/core/server/config/ConfigPostProcessor.java +++ b/server/src/main/java/com/epam/aidial/core/server/config/ConfigPostProcessor.java @@ -43,8 +43,9 @@ * {@code /} (cross-entity reserved path separator). Always run; cannot * fail per-entity. Only applied to file-sourced maps — {@link MergedConfigStore} * skips this pass for the merged config: blob-sourced models/applications/ - * interceptors/roles/toolsets key by short name (never contains {@code /}), and - * keys/routes/schemas key by canonical id legitimately.

    • + * interceptors/roles/toolsets key by short name (never contains {@code /}), schemas + * key by decoded {@code $id} (which contains {@code /} legitimately), and keys/routes + * key by canonical id legitimately. *
  • Semantic — name back-fill, deployment-id uniqueness, ToolSet * resource-key validation, route ordering, {@link ApiKeyStore} hookup. * Each per-entity violation either throws (default {@code abort} mode, From ecf5800b4e896525d510231b1af6a98b01b07cd2 Mon Sep 17 00:00:00 2001 From: Kiryl_Kurnosenka Date: Thu, 13 Aug 2026 23:41:48 +0300 Subject: [PATCH 14/15] fix: adjust comments, remove redundant method param --- .../core/server/config/MergedConfigStore.java | 17 ++++++++--------- .../server/controller/AdminApplyController.java | 10 +++++----- .../controller/ConfigResourceController.java | 8 ++++---- 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/server/src/main/java/com/epam/aidial/core/server/config/MergedConfigStore.java b/server/src/main/java/com/epam/aidial/core/server/config/MergedConfigStore.java index d6ebf4c6b..47b9be2e2 100644 --- a/server/src/main/java/com/epam/aidial/core/server/config/MergedConfigStore.java +++ b/server/src/main/java/com/epam/aidial/core/server/config/MergedConfigStore.java @@ -338,7 +338,7 @@ static boolean isManagedEventUrl(String url) { void applyReplicaEvent(ResourceDescriptor descriptor, ResourceEvent.Action action) { try { ResourceTypes type = (ResourceTypes) descriptor.getType(); - String mapKey = mapKeyFor(type, descriptor); + String mapKey = mapKeyFor(descriptor); if (action == ResourceEvent.Action.DELETE) { applyReplicaDelete(type, mapKey); return; @@ -1052,9 +1052,11 @@ private Config rebuild() { Map apiKeysByCanonicalId = new HashMap<>(); // For models/interceptors/roles/applications/toolsets, a blob entry's map key (its short // name) is indistinguishable in shape from a file entry's — both are bare, slash-free names. - // Track which (type, short name) pairs actually came from a blob this rebuild, so the - // semantic pass below can classify a skipped entity as "api" vs "file" correctly instead of - // guessing from key shape (which only works for the still-canonical-id-keyed types). + // Schemas are locally-keyed too (by decoded $id); their $id always contains '/' so the + // shape-based check in the skip lambda already classifies them as "api", but they are + // tracked here for completeness. Track which (type, mapKey) pairs came from a blob this + // rebuild so the semantic pass can classify a skipped entity as "api" vs "file" correctly + // instead of guessing from key shape (which only works for the still-canonical-id-keyed types). Map> apiSourcedKeys = new EnumMap<>(ResourceTypes.class); for (ResourceTypes type : MANAGED_TYPES) { @@ -1075,10 +1077,6 @@ private Config rebuild() { continue; } String canonicalId = canonicalId(type, bucket, name); - // Blob entries for these five types key Config's map by short name, the same - // key a file-sourced entry for the same logical entity already uses — no - // derivation, no shadowing. Every other managed type keeps using the canonical - // id as its map key. String mapKey = isLocallyKeyed(type) ? localKey(type, name) : canonicalId; JsonNode node; try { @@ -1272,7 +1270,8 @@ public static String canonicalId(ResourceDescriptor descriptor) { * Callers that already have a {@link ResourceDescriptor} should use this instead of building * the canonical id and then parsing the key back out of it. */ - public static String mapKeyFor(ResourceTypes type, ResourceDescriptor descriptor) { + public static String mapKeyFor(ResourceDescriptor descriptor) { + ResourceTypes type = (ResourceTypes) descriptor.getType(); return isLocallyKeyed(type) ? localKey(type, descriptor.getName()) : canonicalId(descriptor); } diff --git a/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java b/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java index 448655acc..f36650464 100644 --- a/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java +++ b/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java @@ -451,7 +451,7 @@ private EntityResult applySchema(AdminManifest entry, String id, ParsedName pars return new EntityResult(id, AdminApplyStatus.FAILED, "Failed to serialize schema for " + id); } resourceService.putResource(descriptor, blobBody, EtagHeader.ANY); - pending.add(new EntityChange(type, MergedConfigStore.mapKeyFor(type, descriptor), spec)); + pending.add(new EntityChange(type, MergedConfigStore.mapKeyFor(descriptor), spec)); return new EntityResult(id, AdminApplyStatus.APPLIED, null); } @@ -471,7 +471,7 @@ private EntityResult applyManagedEntity(AdminManifest entry, String id, Pars } String blobBody = ConfigResourceController.serializeForBlob(entity); resourceService.putResource(descriptor, blobBody, EtagHeader.ANY); - pending.add(new EntityChange(type, MergedConfigStore.mapKeyFor(type, descriptor), entity)); + pending.add(new EntityChange(type, MergedConfigStore.mapKeyFor(descriptor), entity)); return new EntityResult(id, AdminApplyStatus.APPLIED, null); } @@ -543,7 +543,7 @@ private EntityResult applyModel(AdminManifest entry, String id, ParsedName parse resourceService.putResource(descriptor, blobBody, EtagHeader.ANY); // Slice 4S.4: decrypt-in-place so partial-update receives plaintext upstream secrets. secretFieldProcessor.decryptFields(model, descriptor); - pending.add(new EntityChange(ResourceTypes.MODEL, MergedConfigStore.mapKeyFor(ResourceTypes.MODEL, descriptor), model)); + pending.add(new EntityChange(ResourceTypes.MODEL, MergedConfigStore.mapKeyFor(descriptor), model)); return new EntityResult(id, invalid ? AdminApplyStatus.APPLIED_INVALID : AdminApplyStatus.APPLIED, null); } @@ -568,7 +568,7 @@ private EntityResult applyApplication(AdminManifest entry, String id, ParsedName AdminManagedFieldsWriteMode.AUTHORITATIVE); if (platform) { Application decrypted = applicationService.getApplicationWithDecryptedSecrets(descriptor).getValue(); - pending.add(new EntityChange(ResourceTypes.APPLICATION, MergedConfigStore.mapKeyFor(ResourceTypes.APPLICATION, descriptor), decrypted)); + pending.add(new EntityChange(ResourceTypes.APPLICATION, MergedConfigStore.mapKeyFor(descriptor), decrypted)); } return new EntityResult(id, AdminApplyStatus.APPLIED, null); } @@ -589,7 +589,7 @@ private EntityResult applyToolSet(AdminManifest entry, String id, ParsedName par toolSetService.putToolSet(descriptor, EtagHeader.ANY, null, toolSet, true); if (platform) { ToolSet decrypted = toolSetService.getToolSetWithDecryptedAuthSettings(descriptor).getValue(); - pending.add(new EntityChange(ResourceTypes.TOOL_SET, MergedConfigStore.mapKeyFor(ResourceTypes.TOOL_SET, descriptor), decrypted)); + pending.add(new EntityChange(ResourceTypes.TOOL_SET, MergedConfigStore.mapKeyFor(descriptor), decrypted)); } return new EntityResult(id, AdminApplyStatus.APPLIED, null); } diff --git a/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java b/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java index add95330a..fe7b30621 100644 --- a/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java +++ b/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java @@ -1382,7 +1382,7 @@ private Future handleAppOrToolSetPut() { } default -> throw new IllegalArgumentException("Unexpected resource type: " + type); }; - mergedConfigStore.applyEntityWrite(type, MergedConfigStore.mapKeyFor(type, descriptor), decrypted); + mergedConfigStore.applyEntityWrite(type, MergedConfigStore.mapKeyFor(descriptor), decrypted); return resourceService.getResourceMetadata(descriptor); })); }).onSuccess(meta -> context.putHeader(HttpHeaders.ETAG, meta.getEtag()) @@ -1414,7 +1414,7 @@ private Future handleAppOrToolSetDelete() { } default -> throw new IllegalArgumentException("Unexpected resource type: " + type); } - mergedConfigStore.applyEntityDelete(type, MergedConfigStore.mapKeyFor(type, descriptor)); + mergedConfigStore.applyEntityDelete(type, MergedConfigStore.mapKeyFor(descriptor)); return true; })).onSuccess(v -> context.respond(HttpStatus.NO_CONTENT)).onFailure(this::handleWriteError); @@ -1552,7 +1552,7 @@ private Future handlePut() { secretFieldProcessor.decryptFields(entity, descriptor); } mergedConfigStore.applyEntityWrite(typeOf(descriptor), - MergedConfigStore.mapKeyFor(typeOf(descriptor), descriptor), + MergedConfigStore.mapKeyFor(descriptor), entity != null ? entity : requestNode); return meta; })); @@ -1617,7 +1617,7 @@ private Future handleDelete() { if (deletedSecret != null) { apiKeyStore.removeKey(deletedSecret); } - mergedConfigStore.applyEntityDelete(typeOf(descriptor), MergedConfigStore.mapKeyFor(typeOf(descriptor), descriptor)); + mergedConfigStore.applyEntityDelete(typeOf(descriptor), MergedConfigStore.mapKeyFor(descriptor)); return true; })).onSuccess(v -> context.respond(HttpStatus.NO_CONTENT)).onFailure(this::handleWriteError); From dfc5bb3904d95f028532a966cc3df3ae300a97ec Mon Sep 17 00:00:00 2001 From: Kiryl_Kurnosenka Date: Fri, 14 Aug 2026 00:19:15 +0300 Subject: [PATCH 15/15] refactor: remove double-fetch and redundant decrypt in Admin blob GET APPLICATION and TOOL_SET are now deserialized via treeToEntity like all other entity types; the previous code re-fetched them through their service's decrypting read, causing a second blob round-trip for every Admin GET. The decrypt was also unnecessary: @EncryptedField fields are all @JsonProperty(WRITE_ONLY) and suppressed by Jackson on serialization, and APPLICATION/TOOL_SET secrets are additionally redacted by the projector. Rename readAndDecrypt -> deserializeBlob to reflect the new semantics. Co-Authored-By: Claude Sonnet 4.6 --- .../controller/ConfigResourceController.java | 21 +++++++------------ 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java b/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java index fe7b30621..ac05e50d5 100644 --- a/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java +++ b/server/src/main/java/com/epam/aidial/core/server/controller/ConfigResourceController.java @@ -1113,7 +1113,7 @@ private Future handleSingleGetFromBlob(ResourceTypes resourceType, BiFunction if (existing == null) { return null; } - Object entity = readAndDecrypt(resourceType, existing.getValue(), descriptor); + Object entity = deserializeBlob(resourceType, existing.getValue()); return Pair.of(existing.getKey().getEtag(), projector.apply(path, entity)); }).onSuccess(result -> { if (result == null) { @@ -1135,16 +1135,14 @@ private Future handleSingleGetFromBlob(ResourceTypes resourceType, BiFunction } /** - * Deserializes and decrypts a blob body for {@link #handleSingleGetFromBlob}. Applications and - * toolsets encrypt secrets outside the {@code @EncryptedField}/{@link SecretFieldProcessor} - * path (per-resource, via {@link ApplicationService}/{@link ToolSetService}), so they're - * re-fetched through those services' own decrypting reads rather than decoded from - * {@code body} directly — mirrors {@code MergedConfigStore.decryptManagedEntity}. + * Deserializes a blob body for {@link #handleSingleGetFromBlob}. No decryption is performed: + * secret fields on entity types are {@code @JsonProperty(WRITE_ONLY)} and are therefore + * suppressed by Jackson on serialization regardless of their content; APPLICATION and TOOL_SET + * secrets are additionally redacted by the projector ({@link #redactExternalServiceSecrets}/ + * {@link #redactAuthSettingsSecrets}). */ - private Object readAndDecrypt(ResourceTypes type, String body, ResourceDescriptor descriptor) { + private Object deserializeBlob(ResourceTypes type, String body) { return switch (type) { - case APPLICATION -> applicationService.getApplicationWithDecryptedSecrets(descriptor).getValue(); - case TOOL_SET -> toolSetService.getToolSetWithDecryptedAuthSettings(descriptor).getValue(); case APP_TYPE_SCHEMA, CATALOG_SCHEMA -> { try { yield ProxyUtil.BLOB_MAPPER.readTree(body); @@ -1154,15 +1152,12 @@ private Object readAndDecrypt(ResourceTypes type, String body, ResourceDescripto } } default -> { - Object entity; try { - entity = treeToEntity(ProxyUtil.BLOB_MAPPER.readTree(body), entityClassFor(entityType)); + yield treeToEntity(ProxyUtil.BLOB_MAPPER.readTree(body), entityClassFor(entityType)); } catch (JsonProcessingException e) { throw new HttpException(HttpStatus.INTERNAL_SERVER_ERROR, "Stored entity is malformed at " + locationOf(e)); } - secretFieldProcessor.decryptFields(entity, descriptor); - yield entity; } }; }