diff --git a/server/src/main/java/com/epam/aidial/core/server/config/SchemaMigrationNaming.java b/server/src/main/java/com/epam/aidial/core/server/config/SchemaMigrationNaming.java new file mode 100644 index 000000000..d61fce229 --- /dev/null +++ b/server/src/main/java/com/epam/aidial/core/server/config/SchemaMigrationNaming.java @@ -0,0 +1,72 @@ +package com.epam.aidial.core.server.config; + +import com.epam.aidial.core.metaschemas.CatalogMetaSchemaHolder; +import com.epam.aidial.core.metaschemas.MetaSchemaHolder; +import com.epam.aidial.core.storage.resource.ResourceTypes; +import com.fasterxml.jackson.databind.JsonNode; + +import java.nio.charset.StandardCharsets; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.regex.Pattern; + +/** + * Mints a canonical blob name for a file-sourced app-type/catalog schema being migrated into the + * {@code platform} bucket. Canonical id for schemas is decoupled from {@code $id}, so migration is + * free to invent any valid, unique, deterministic name — deterministic so a dry-run preview matches + * the real run and re-runs don't mint a second name for the same schema. Idempotency itself does not + * depend on this name being reproduced: the caller decides "already migrated" via the {@code $id} + * alias index, not by recomputing this name. + */ +public final class SchemaMigrationNaming { + + private static final int MAX_SLUG_LENGTH = 40; + private static final Pattern DISALLOWED_CHARS = Pattern.compile("[^a-z0-9._-]+"); + private static final Pattern REPEATED_DASHES = Pattern.compile("-{2,}"); + + private SchemaMigrationNaming() { + } + + public static String mintName(ResourceTypes type, JsonNode body) { + String displayNameField = switch (type) { + case APP_TYPE_SCHEMA -> MetaSchemaHolder.APPLICATION_TYPE_DISPLAY_NAME; + case CATALOG_SCHEMA -> CatalogMetaSchemaHolder.CATALOG_DISPLAY_NAME; + default -> throw new IllegalArgumentException("Not a schema resource type: " + type); + }; + String id = body.path("$id").asText(null); + String displayName = body.path(displayNameField).asText(null); + // The full digest (not a truncated prefix) is used so two different $id values cannot + // collide on name — a truncated hash would only be probabilistically unique. + String hash = sha256Hex(id); + String slug = sanitize(displayName); + return slug.isEmpty() ? hash : slug + "-" + hash; + } + + private static String sanitize(String displayName) { + if (displayName == null) { + return ""; + } + String lower = displayName.toLowerCase(); + String replaced = DISALLOWED_CHARS.matcher(lower).replaceAll("-"); + String collapsed = REPEATED_DASHES.matcher(replaced).replaceAll("-"); + // Without this, a display name that starts/ends with a disallowed character (e.g. a leading + // space) leaves a leading/trailing '-' from the replacement above, producing an ugly + // double-dash once the hash suffix is appended (e.g. "-foo-" + "-" + hash). + String trimmed = collapsed.replaceAll("^-+|-+$", ""); + return trimmed.length() > MAX_SLUG_LENGTH ? trimmed.substring(0, MAX_SLUG_LENGTH) : trimmed; + } + + private static String sha256Hex(String value) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + byte[] bytes = digest.digest((value == null ? "" : value).getBytes(StandardCharsets.UTF_8)); + StringBuilder sb = new StringBuilder(bytes.length * 2); + for (byte b : bytes) { + sb.append(String.format("%02x", b)); + } + return sb.toString(); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException("SHA-256 not available", e); + } + } +} diff --git a/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java b/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java index fb6fe95bd..47a311a37 100644 --- a/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java +++ b/server/src/main/java/com/epam/aidial/core/server/controller/AdminApplyController.java @@ -74,6 +74,16 @@ public class AdminApplyController { "ToolSet", 7, "Application", 8); + /** + * Sorts a manifest list so a kind that other kinds can reference (e.g. {@code Interceptor}) + * is always applied before the kind referencing it (e.g. {@code Model}) — required whenever + * more than one kind in the same batch/call can cross-reference another by name. Shared by + * {@link #applyBatch} and {@link ConfigFileMigrateController}, so the one ordering definition + * governs both. + */ + static final Comparator DEPENDENCY_ORDER_COMPARATOR = + Comparator.comparingInt(entry -> DEPENDENCY_ORDER.getOrDefault(entry.kind(), 99)); + static final Map KIND_URL_SEGMENT = Map.of( "Settings", "settings", "Schema", "schemas", @@ -198,12 +208,12 @@ private void process(Buffer body) { private ApplyResponse applyBatch(boolean precheck, List rawEntries) { List entries = new ArrayList<>(rawEntries); - entries.sort(Comparator.comparingInt(e -> DEPENDENCY_ORDER.getOrDefault(e.kind(), 99))); + entries.sort(DEPENDENCY_ORDER_COMPARATOR); Config scratch = newScratch(mergedConfigStore); - List results = new ArrayList<>(); if (precheck) { + List precheckResults = new ArrayList<>(); boolean anyFailure = false; for (AdminManifest entry : entries) { ValidationResult result = validateOnly(entry, scratch, softValidation); @@ -211,27 +221,38 @@ private ApplyResponse applyBatch(boolean precheck, List rawEntrie anyFailure = true; // Mirror /v1/admin/validate: the offending entry stays FAILED (carrying its // error); only the valid siblings collapse to "skipped" below. - results.add(new EntityResult(result.entityId(), AdminApplyStatus.FAILED, result.error())); + precheckResults.add(new EntityResult(result.entityId(), AdminApplyStatus.FAILED, result.error())); } else { // Mutate scratch so subsequent precheck entries see prior ones — even though we // aren't writing yet, reference resolution depends on the cumulative scratch. mutateScratch(scratch, entry); - results.add(new EntityResult(result.entityId(), AdminApplyStatus.SKIPPED, null)); + precheckResults.add(new EntityResult(result.entityId(), AdminApplyStatus.SKIPPED, null)); } } if (anyFailure) { - return buildResponse(HttpStatus.UNPROCESSABLE_ENTITY, results); + return buildResponse(HttpStatus.UNPROCESSABLE_ENTITY, precheckResults); } // Precheck passed — wipe and re-run as real writes. scratch = newScratch(mergedConfigStore); - results.clear(); } - boolean anyApplied = false; + return buildResponse(HttpStatus.OK, applyEntries(entries, scratch)); + } + + /** + * Applies each entry via {@link #applySingle}, mutating {@code scratch} after every success so + * later entries in the same list see earlier ones, then flushes every in-memory change as a + * single partial-update swap. Shared by {@link #applyBatch}'s real-apply phase and + * {@link ConfigFileMigrateController}, which builds its own entry list and scratch (with any + * shadowed file entries already removed) before calling this. + */ + List applyEntries(List entries, Config scratch) { + List results = new ArrayList<>(); // Slice 4S.4: collect partial-update changes per applied entity; flush as one applyBatch // after the apply loop so the merged Config swap happens once, after all blobs are written. List pendingChanges = new ArrayList<>(); GlobalSettings pendingSettings = null; + boolean anyApplied = false; for (AdminManifest entry : entries) { EntityResult result; try { @@ -261,7 +282,7 @@ private ApplyResponse applyBatch(boolean precheck, List rawEntrie mergedConfigStore.applySettingsWrite(pendingSettings); } } - return buildResponse(HttpStatus.OK, results); + return results; } static Config newScratch(MergedConfigStore mergedConfigStore) { @@ -753,7 +774,7 @@ private ApplyResponse buildResponse(HttpStatus status, List result return new ApplyResponse(status, new AdminApplyResponse(applied, failed, responseResults)); } - private record EntityResult(String entityId, AdminApplyStatus status, String error) {} + record EntityResult(String entityId, AdminApplyStatus status, String error) {} @JsonInclude(JsonInclude.Include.NON_NULL) private record ApplyResponse(HttpStatus status, AdminApplyResponse body) {} diff --git a/server/src/main/java/com/epam/aidial/core/server/controller/ConfigFileMigrateController.java b/server/src/main/java/com/epam/aidial/core/server/controller/ConfigFileMigrateController.java new file mode 100644 index 000000000..b4a3ed8d0 --- /dev/null +++ b/server/src/main/java/com/epam/aidial/core/server/controller/ConfigFileMigrateController.java @@ -0,0 +1,316 @@ +package com.epam.aidial.core.server.controller; + +import com.epam.aidial.core.config.Config; +import com.epam.aidial.core.config.GlobalSettings; +import com.epam.aidial.core.openapi.annotations.ApiExtension; +import com.epam.aidial.core.openapi.annotations.ApiOperation; +import com.epam.aidial.core.openapi.annotations.ApiResponse; +import com.epam.aidial.core.openapi.annotations.ApiSchema; +import com.epam.aidial.core.server.ProxyContext; +import com.epam.aidial.core.server.config.MergedConfigStore; +import com.epam.aidial.core.server.config.SchemaMigrationNaming; +import com.epam.aidial.core.server.data.AdminApplyStatus; +import com.epam.aidial.core.server.data.AdminManifest; +import com.epam.aidial.core.server.data.ConfigFileMigrateRequest; +import com.epam.aidial.core.server.data.ConfigFileMigrateResponse; +import com.epam.aidial.core.server.data.ConfigFileMigrateResult; +import com.epam.aidial.core.server.data.ConfigFileMigrateStatus; +import com.epam.aidial.core.server.data.ValidationResult; +import com.epam.aidial.core.server.data.ValidationStatus; +import com.epam.aidial.core.server.security.ConfigAuthorizationService; +import com.epam.aidial.core.server.util.ProxyUtil; +import com.epam.aidial.core.server.vertx.AsyncTaskExecutor; +import com.epam.aidial.core.storage.http.HttpException; +import com.epam.aidial.core.storage.http.HttpStatus; +import com.epam.aidial.core.storage.resource.ResourceDescriptor; +import com.epam.aidial.core.storage.resource.ResourceTypes; +import com.epam.aidial.core.storage.service.LockService; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; +import io.vertx.core.Future; +import io.vertx.core.buffer.Buffer; + +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; + +/** + * Admin-triggered, on-demand copy of file-defined config entities into the {@code platform} blob + * bucket. Migration is never automatic on startup — that would resurrect an API-deleted entity on + * restart. Reuses {@link AdminApplyController}'s per-kind write pipeline. + */ +public class ConfigFileMigrateController { + + private static final List ALL_TYPES = List.of( + "settings", "schemas", "catalog_schemas", "interceptors", "roles", "keys", "routes", + "models", "toolsets", "applications"); + + private static final List MANAGED_TYPE_SPECS = List.of( + new ManagedTypeSpec("interceptors", "Interceptor", ResourceTypes.INTERCEPTOR, Config::getInterceptors), + new ManagedTypeSpec("roles", "Role", ResourceTypes.ROLE, Config::getRoles), + new ManagedTypeSpec("keys", "Key", ResourceTypes.PROJECT_KEY, Config::getKeys), + new ManagedTypeSpec("routes", "Route", ResourceTypes.ROUTE, Config::getRoutes), + new ManagedTypeSpec("models", "Model", ResourceTypes.MODEL, Config::getModels), + new ManagedTypeSpec("toolsets", "ToolSet", ResourceTypes.TOOL_SET, Config::getToolsets), + new ManagedTypeSpec("applications", "Application", ResourceTypes.APPLICATION, Config::getApplications)); + + private static final List SCHEMA_TYPE_SPECS = List.of( + new SchemaTypeSpec("schemas", "Schema", ResourceTypes.APP_TYPE_SCHEMA, + Config::getApplicationTypeSchemas, Config::getApplicationSchemaAliasesById), + new SchemaTypeSpec("catalog_schemas", "CatalogSchema", ResourceTypes.CATALOG_SCHEMA, + Config::getCatalogSchemas, Config::getCatalogSchemaAliasesById)); + + private final ProxyContext context; + private final ConfigAuthorizationService authorizationService; + private final MergedConfigStore mergedConfigStore; + private final AsyncTaskExecutor taskExecutor; + private final LockService lockService; + private final AdminApplyController applier; + + public ConfigFileMigrateController(ProxyContext context, + ConfigAuthorizationService authorizationService, + MergedConfigStore mergedConfigStore, + AsyncTaskExecutor taskExecutor, + LockService lockService, + AdminApplyController applier) { + this.context = context; + this.authorizationService = authorizationService; + this.mergedConfigStore = mergedConfigStore; + this.taskExecutor = taskExecutor; + this.lockService = lockService; + this.applier = applier; + } + + @ApiOperation( + method = "POST", + path = "/v1/admin/config/file/migrate", + operationId = "migrateFileConfig", + tags = {"Admin"}, + requestBody = @ApiSchema(implementation = ConfigFileMigrateRequest.class), + responses = { + @ApiResponse(code = 200, description = "Migration report", body = @ApiSchema(implementation = ConfigFileMigrateResponse.class)), + @ApiResponse(code = 400), + @ApiResponse(code = 403), + @ApiResponse(code = 500) + }, + extensions = { + @ApiExtension(name = "x-preview", value = "true") + } + ) + public Future handle() { + if (!authorizationService.isAdmin(context)) { + context.respond(HttpStatus.FORBIDDEN, "Forbidden"); + return Future.succeededFuture(); + } + context.getRequest().body() + .onSuccess(this::process) + .onFailure(error -> context.respond(HttpStatus.BAD_REQUEST, + "Failed to read request body: " + error.getMessage())); + return Future.succeededFuture(); + } + + private void process(Buffer body) { + ConfigFileMigrateRequest request; + try { + String text = body.toString(StandardCharsets.UTF_8); + request = ProxyUtil.MAPPER.readValue(text.isEmpty() ? "{}" : text, ConfigFileMigrateRequest.class); + } catch (JsonProcessingException e) { + context.respond(HttpStatus.BAD_REQUEST, "Invalid JSON at " + locationOf(e)); + return; + } + + Set requestedTypes; + try { + requestedTypes = resolveTypes(request.types()); + } catch (IllegalArgumentException e) { + context.respond(HttpStatus.BAD_REQUEST, e.getMessage()); + return; + } + boolean dryRun = Boolean.TRUE.equals(request.dryRun()); + + taskExecutor.submit(() -> lockService.underBucketLocks(MergedConfigStore.ADMIN_BUCKET_LOCATIONS, + () -> runMigration(requestedTypes, dryRun))) + .onSuccess(response -> context.respond(HttpStatus.OK, response)) + .onFailure(error -> { + if (error instanceof HttpException ex) { + context.respond(ex); + } else { + context.respond(HttpStatus.INTERNAL_SERVER_ERROR, error.getMessage()); + } + }); + } + + private static Set resolveTypes(List types) { + if (types == null || types.isEmpty() || types.contains("all")) { + return new LinkedHashSet<>(ALL_TYPES); + } + Set result = new LinkedHashSet<>(); + for (String type : types) { + if (!ALL_TYPES.contains(type)) { + throw new IllegalArgumentException("Unsupported type: " + type); + } + result.add(type); + } + return result; + } + + private ConfigFileMigrateResponse runMigration(Set requestedTypes, boolean dryRun) { + Config fileConfig = mergedConfigStore.getFileSourcedConfig(); + if (fileConfig == null) { + return new ConfigFileMigrateResponse(List.of()); + } + Config live = mergedConfigStore.get(); + Config scratch = AdminApplyController.newScratch(mergedConfigStore); + List results = new ArrayList<>(); + // Real (non-dry) run: candidates accumulate here instead of being written immediately, so + // AdminApplyController.applyEntries can apply and flush them as a single partial-update swap. + List toApply = new ArrayList<>(); + + if (requestedTypes.contains("settings")) { + collectSettings(fileConfig, scratch, dryRun, toApply, results); + } + for (SchemaTypeSpec spec : SCHEMA_TYPE_SPECS) { + if (requestedTypes.contains(spec.typeKey())) { + collectSchemas(spec, fileConfig, live, scratch, dryRun, toApply, results); + } + } + for (ManagedTypeSpec spec : MANAGED_TYPE_SPECS) { + if (requestedTypes.contains(spec.typeKey())) { + collectManaged(spec, fileConfig, live, scratch, dryRun, toApply, results); + } + } + + if (!toApply.isEmpty()) { + // A referencing kind (e.g. Model) must apply after a kind it can reference (e.g. + // Interceptor) — see AdminApplyController.DEPENDENCY_ORDER_COMPARATOR. + toApply.sort(AdminApplyController.DEPENDENCY_ORDER_COMPARATOR); + List appliedEntriesResults = applier.applyEntries(toApply, scratch); + for (AdminApplyController.EntityResult result : appliedEntriesResults) { + results.add(toMigrateResult(result)); + } + } + return new ConfigFileMigrateResponse(results); + } + + private static ConfigFileMigrateResult toMigrateResult(AdminApplyController.EntityResult result) { + boolean migrated = AdminApplyStatus.APPLIED.equals(result.status()) + || AdminApplyStatus.APPLIED_INVALID.equals(result.status()); + ConfigFileMigrateStatus status = migrated ? ConfigFileMigrateStatus.MIGRATED : ConfigFileMigrateStatus.FAILED; + return new ConfigFileMigrateResult(result.entityId(), status, result.error()); + } + + private static ConfigFileMigrateStatus skippedStatus(boolean dryRun) { + return dryRun ? ConfigFileMigrateStatus.WOULD_SKIP : ConfigFileMigrateStatus.SKIPPED; + } + + private static ConfigFileMigrateStatus failedStatus(boolean dryRun) { + return dryRun ? ConfigFileMigrateStatus.WOULD_FAIL : ConfigFileMigrateStatus.FAILED; + } + + private void collectManaged(ManagedTypeSpec spec, Config fileConfig, Config live, Config scratch, boolean dryRun, + List toApply, List results) { + Map liveEntities = spec.entities().apply(live); + for (Map.Entry entry : spec.entities().apply(fileConfig).entrySet()) { + String shortName = entry.getKey(); + String canonicalId = MergedConfigStore.canonicalId(spec.resourceType(), ResourceDescriptor.PLATFORM_BUCKET, shortName); + if (liveEntities.containsKey(canonicalId)) { + results.add(new ConfigFileMigrateResult(canonicalId, skippedStatus(dryRun), "already in blob")); + continue; + } + // scratch (cloned from the live merged Config) still holds the file entry we're about to + // migrate, keyed by its own short name — shadow it out first (mirroring what the real + // blob-write path does once the migrated entity lands) so AdminApplyController's dup-id + // check doesn't see this entity's own pre-migration self as a different entity already + // claiming the short name. + spec.entities().apply(scratch).remove(shortName); + JsonNode specNode = ProxyUtil.MAPPER.valueToTree(entry.getValue()); + collect(new AdminManifest(spec.kind(), canonicalId, specNode), scratch, dryRun, toApply, results); + } + } + + private void collectSchemas(SchemaTypeSpec spec, Config fileConfig, Config live, Config scratch, boolean dryRun, + List toApply, List results) { + Map liveAliasesById = spec.aliasesById().apply(live); + for (Map.Entry entry : spec.schemas().apply(fileConfig).entrySet()) { + String id = entry.getKey(); + if (liveAliasesById.containsKey(id)) { + results.add(new ConfigFileMigrateResult(id, skippedStatus(dryRun), "already in blob")); + continue; + } + JsonNode body; + try { + body = ProxyUtil.MAPPER.readTree(entry.getValue()); + } catch (JsonProcessingException e) { + results.add(new ConfigFileMigrateResult(id, failedStatus(dryRun), "Failed to parse schema body")); + continue; + } + String canonicalId = MergedConfigStore.canonicalId(spec.resourceType(), ResourceDescriptor.PLATFORM_BUCKET, + SchemaMigrationNaming.mintName(spec.resourceType(), body)); + collect(new AdminManifest(spec.kind(), canonicalId, body), scratch, dryRun, toApply, results); + } + } + + private void collectSettings(Config fileConfig, Config scratch, boolean dryRun, + List toApply, List results) { + String canonicalId = MergedConfigStore.canonicalId(ResourceTypes.GLOBAL_SETTINGS, + ResourceDescriptor.PLATFORM_BUCKET, "global"); + if (mergedConfigStore.isSettingsFromApi()) { + results.add(new ConfigFileMigrateResult(canonicalId, skippedStatus(dryRun), "already in blob")); + return; + } + GlobalSettings settings = new GlobalSettings(); + settings.setGlobalInterceptors(fileConfig.getGlobalInterceptors()); + settings.setRetriableErrorCodes(fileConfig.getRetriableErrorCodes()); + JsonNode spec = ProxyUtil.MAPPER.valueToTree(settings); + collect(new AdminManifest("Settings", canonicalId, spec), scratch, dryRun, toApply, results); + } + + /** + * dry-run: validates immediately (no write) via {@link AdminApplyController#validateOnly} and + * reports the outcome right away. Real run: defers to {@code toApply}, applied and flushed once + * for the whole batch by {@link AdminApplyController#applyEntries} at the end of {@link + * #runMigration}. + */ + private void collect(AdminManifest manifest, Config scratch, boolean dryRun, + List toApply, List results) { + if (!dryRun) { + toApply.add(manifest); + return; + } + ValidationResult validation = AdminApplyController.validateOnly(manifest, scratch, mergedConfigStore.isSoftValidation()); + if (ValidationStatus.VALID.equals(validation.status())) { + results.add(new ConfigFileMigrateResult(manifest.name(), ConfigFileMigrateStatus.WOULD_MIGRATE, null)); + AdminApplyController.mutateScratch(scratch, manifest); + } else { + results.add(new ConfigFileMigrateResult(manifest.name(), ConfigFileMigrateStatus.WOULD_FAIL, validation.error())); + } + } + + private static String locationOf(JsonProcessingException e) { + return e.getLocation() == null + ? "unknown location" + : "line " + e.getLocation().getLineNr() + ", column " + e.getLocation().getColumnNr(); + } + + /** + * One accessor per managed, name-addressed type, applied to three different {@link Config} + * instances depending on purpose: {@code fileConfig} (entities to migrate), {@code live} + * (idempotency check), and {@code scratch} (shadow removal ahead of the write) — see {@link + * #collectManaged}. + */ + private record ManagedTypeSpec(String typeKey, String kind, ResourceTypes resourceType, + Function> entities) {} + + /** + * One spec per schema type. Unlike {@link ManagedTypeSpec}, presence is checked against the + * {@code $id → canonicalId} alias index, not the raw schema map, and there is no scratch shadow + * to remove (see {@link #collectSchemas}) — so two accessors are needed here, not one. + */ + private record SchemaTypeSpec(String typeKey, String kind, ResourceTypes resourceType, + Function> schemas, + Function> aliasesById) {} +} diff --git a/server/src/main/java/com/epam/aidial/core/server/controller/ControllerSelector.java b/server/src/main/java/com/epam/aidial/core/server/controller/ControllerSelector.java index aa7a2fd07..5f287e187 100644 --- a/server/src/main/java/com/epam/aidial/core/server/controller/ControllerSelector.java +++ b/server/src/main/java/com/epam/aidial/core/server/controller/ControllerSelector.java @@ -463,6 +463,27 @@ public class ControllerSelector { proxy.getLockService()); return controller::handle; }); + post(RouteTemplate.CONFIG_FILE_MIGRATE, (proxy, context, pathMatcher) -> { + ConfigAuthorizationService authService = new AdminRoleAuthorizationService(proxy.getAccessService()); + MergedConfigStore mergedConfigStore = (MergedConfigStore) proxy.getConfigStore(); + // Reuses AdminApplyController's per-kind write pipeline (applySingle/validateOnly/ + // mutateScratch/newScratch) rather than re-implementing per-type writes; `context` here + // is only ever used by AdminApplyController.handle()/process(), neither of which this + // controller calls, so sharing an instance across the two controllers is safe. + AdminApplyController applier = new AdminApplyController( + context, authService, mergedConfigStore, + proxy.getResourceService(), proxy.getTaskExecutor(), + mergedConfigStore.getSecretFieldProcessor(), + mergedConfigStore.isSoftValidation(), + proxy.getApiKeyStore(), + proxy.getApplicationService(), + proxy.getToolSetService(), + proxy.getLockService()); + ConfigFileMigrateController controller = new ConfigFileMigrateController( + context, authService, mergedConfigStore, proxy.getTaskExecutor(), + proxy.getLockService(), applier); + return controller::handle; + }); get(RouteTemplate.CONFIG_HEALTH, (proxy, context, pathMatcher) -> { ConfigAuthorizationService authService = new AdminRoleAuthorizationService(proxy.getAccessService()); MergedConfigStore mergedConfigStore = (MergedConfigStore) proxy.getConfigStore(); diff --git a/server/src/main/java/com/epam/aidial/core/server/data/ConfigFileMigrateRequest.java b/server/src/main/java/com/epam/aidial/core/server/data/ConfigFileMigrateRequest.java new file mode 100644 index 000000000..0cd6d6001 --- /dev/null +++ b/server/src/main/java/com/epam/aidial/core/server/data/ConfigFileMigrateRequest.java @@ -0,0 +1,5 @@ +package com.epam.aidial.core.server.data; + +import java.util.List; + +public record ConfigFileMigrateRequest(List types, Boolean dryRun) {} diff --git a/server/src/main/java/com/epam/aidial/core/server/data/ConfigFileMigrateResponse.java b/server/src/main/java/com/epam/aidial/core/server/data/ConfigFileMigrateResponse.java new file mode 100644 index 000000000..de4f7bee2 --- /dev/null +++ b/server/src/main/java/com/epam/aidial/core/server/data/ConfigFileMigrateResponse.java @@ -0,0 +1,5 @@ +package com.epam.aidial.core.server.data; + +import java.util.List; + +public record ConfigFileMigrateResponse(List results) {} diff --git a/server/src/main/java/com/epam/aidial/core/server/data/ConfigFileMigrateResult.java b/server/src/main/java/com/epam/aidial/core/server/data/ConfigFileMigrateResult.java new file mode 100644 index 000000000..aad294f36 --- /dev/null +++ b/server/src/main/java/com/epam/aidial/core/server/data/ConfigFileMigrateResult.java @@ -0,0 +1,6 @@ +package com.epam.aidial.core.server.data; + +import com.fasterxml.jackson.annotation.JsonInclude; + +@JsonInclude(JsonInclude.Include.NON_NULL) +public record ConfigFileMigrateResult(String id, ConfigFileMigrateStatus status, String reason) {} diff --git a/server/src/main/java/com/epam/aidial/core/server/data/ConfigFileMigrateStatus.java b/server/src/main/java/com/epam/aidial/core/server/data/ConfigFileMigrateStatus.java new file mode 100644 index 000000000..85cb096d2 --- /dev/null +++ b/server/src/main/java/com/epam/aidial/core/server/data/ConfigFileMigrateStatus.java @@ -0,0 +1,17 @@ +package com.epam.aidial.core.server.data; + +import com.fasterxml.jackson.annotation.JsonValue; + +public enum ConfigFileMigrateStatus { + MIGRATED, + WOULD_MIGRATE, + SKIPPED, + WOULD_SKIP, + FAILED, + WOULD_FAIL; + + @JsonValue + public String toJson() { + return name().toLowerCase(); + } +} diff --git a/server/src/main/java/com/epam/aidial/core/server/data/RouteTemplate.java b/server/src/main/java/com/epam/aidial/core/server/data/RouteTemplate.java index 7c6a03324..7bd4afb37 100644 --- a/server/src/main/java/com/epam/aidial/core/server/data/RouteTemplate.java +++ b/server/src/main/java/com/epam/aidial/core/server/data/RouteTemplate.java @@ -149,6 +149,11 @@ public enum RouteTemplate { "/v1/admin/config/file/{type}/{name}" ), + CONFIG_FILE_MIGRATE( + "^/v1/admin/config/file/migrate$", + "/v1/admin/config/file/migrate" + ), + CONFIG_VALIDATE( "^/v1/admin/validate$", "/v1/admin/validate" diff --git a/server/src/test/java/com/epam/aidial/core/server/ConfigFileMigrateApiTest.java b/server/src/test/java/com/epam/aidial/core/server/ConfigFileMigrateApiTest.java new file mode 100644 index 000000000..9b37c183e --- /dev/null +++ b/server/src/test/java/com/epam/aidial/core/server/ConfigFileMigrateApiTest.java @@ -0,0 +1,227 @@ +package com.epam.aidial.core.server; + +import com.epam.aidial.core.server.util.ProxyUtil; +import com.fasterxml.jackson.databind.JsonNode; +import io.vertx.core.http.HttpMethod; +import lombok.SneakyThrows; +import org.junit.jupiter.api.Test; + +import java.util.HashSet; +import java.util.Set; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +public class ConfigFileMigrateApiTest extends ResourceBaseTest { + + @Test + @SneakyThrows + void testMigrateModelAndInterceptor() { + String body = """ + {"types": ["models", "interceptors"]} + """; + Response response = send(HttpMethod.POST, "/v1/admin/config/file/migrate", null, body, + "authorization", "admin"); + verify(response, 200); + JsonNode results = ProxyUtil.MAPPER.readTree(response.body()).get("results"); + assertTrue(idsWithStatus(results, "migrated").contains("models/platform/test-model-v1"), + () -> "Body: " + response.body()); + assertTrue(idsWithStatus(results, "migrated").contains("interceptors/platform/interceptor1"), + () -> "Body: " + response.body()); + + verify(send(HttpMethod.GET, "/v1/models/platform/test-model-v1", null, "", + "authorization", "admin"), 200); + verify(send(HttpMethod.GET, "/v1/interceptors/platform/interceptor1", null, "", + "authorization", "admin"), 200); + + Response models = send(HttpMethod.GET, "/openai/models", null, "", "authorization", "admin"); + verify(models, 200); + JsonNode data = ProxyUtil.MAPPER.readTree(models.body()).get("data"); + int occurrences = 0; + for (JsonNode m : data) { + if ("test-model-v1".equals(m.get("id").asText())) { + occurrences++; + } + } + assertEquals(1, occurrences, () -> "Expected exactly one 'test-model-v1' entry: " + models.body()); + } + + @Test + @SneakyThrows + void testMigrateIsIdempotent() { + String body = """ + {"types": ["models"]} + """; + Response first = send(HttpMethod.POST, "/v1/admin/config/file/migrate", null, body, + "authorization", "admin"); + verify(first, 200); + JsonNode firstResults = ProxyUtil.MAPPER.readTree(first.body()).get("results"); + assertTrue(idsWithStatus(firstResults, "migrated").contains("models/platform/test-model-v1"), + () -> "Body: " + first.body()); + + Response second = send(HttpMethod.POST, "/v1/admin/config/file/migrate", null, body, + "authorization", "admin"); + verify(second, 200); + JsonNode secondResults = ProxyUtil.MAPPER.readTree(second.body()).get("results"); + for (JsonNode r : secondResults) { + assertEquals("skipped", r.get("status").asText(), () -> "Body: " + second.body()); + } + assertTrue(idsWithStatus(secondResults, "skipped").contains("models/platform/test-model-v1"), + () -> "Body: " + second.body()); + } + + @Test + @SneakyThrows + void testDryRunWritesNothing() { + String body = """ + {"types": ["roles"], "dryRun": true} + """; + Response dryRun = send(HttpMethod.POST, "/v1/admin/config/file/migrate", null, body, + "authorization", "admin"); + verify(dryRun, 200); + JsonNode dryRunResults = ProxyUtil.MAPPER.readTree(dryRun.body()).get("results"); + assertTrue(idsWithStatus(dryRunResults, "would_migrate").contains("roles/platform/default"), + () -> "Body: " + dryRun.body()); + + // Nothing was actually written. + verify(send(HttpMethod.GET, "/v1/roles/platform/default", null, "", + "authorization", "admin"), 404); + + String realBody = """ + {"types": ["roles"], "dryRun": false} + """; + Response real = send(HttpMethod.POST, "/v1/admin/config/file/migrate", null, realBody, + "authorization", "admin"); + verify(real, 200); + JsonNode realResults = ProxyUtil.MAPPER.readTree(real.body()).get("results"); + assertTrue(idsWithStatus(realResults, "migrated").contains("roles/platform/default"), + () -> "Body: " + real.body()); + verify(send(HttpMethod.GET, "/v1/roles/platform/default", null, "", + "authorization", "admin"), 200); + } + + @Test + @SneakyThrows + void testMigrateToolSetKeepsSecretEncrypted() { + // The fixture's "oauth-toolset" carries auth_settings.client_secret in plaintext file config. + // ToolSetService.putToolSet performs OAuth protected-resource-metadata discovery against the + // toolset's own endpoint (localhost:9876), so a real listener is required here. + String body = """ + {"types": ["toolsets"]} + """; + try (TestWebServer ignore = new TestWebServer(9876)) { + Response response = send(HttpMethod.POST, "/v1/admin/config/file/migrate", null, body, + "authorization", "admin"); + verify(response, 200); + JsonNode results = ProxyUtil.MAPPER.readTree(response.body()).get("results"); + assertTrue(idsWithStatus(results, "migrated").contains("toolsets/platform/oauth-toolset"), + () -> "Body: " + response.body()); + + Response get = send(HttpMethod.GET, "/v1/toolsets/platform/oauth-toolset", null, "", + "authorization", "admin"); + verify(get, 200); + assertFalse(get.body().contains("test-client-secret"), + () -> "Plaintext client_secret must never appear on GET: " + get.body()); + assertFalse(get.body().contains("\"client_secret\""), + () -> "client_secret field must be absent from GET response: " + get.body()); + } + } + + @Test + @SneakyThrows + void testMigrateSchemasDisambiguatesSameDisplayName() { + // Three fixture schemas share the same dial:applicationTypeDisplayName ("Specific + // Application Type") but have distinct $id values; the hash suffix must keep their minted + // names distinct. + String body = """ + {"types": ["schemas"]} + """; + Response response = send(HttpMethod.POST, "/v1/admin/config/file/migrate", null, body, + "authorization", "admin"); + verify(response, 200); + JsonNode results = ProxyUtil.MAPPER.readTree(response.body()).get("results"); + + Set migratedIds = idsWithStatus(results, "migrated"); + assertEquals(4, migratedIds.size(), () -> "Body: " + response.body()); + + long specificApplicationTypeCount = migratedIds.stream() + .filter(migratedId -> migratedId.startsWith("schemas/platform/specific-application-type-")) + .count(); + // migratedIds is a Set, so 3 distinct entries here already proves the hash suffix + // disambiguated all three same-display-name schemas. + assertEquals(3, specificApplicationTypeCount, () -> "Body: " + response.body()); + + for (String migratedId : migratedIds) { + String name = migratedId.substring("schemas/platform/".length()); + verify(send(HttpMethod.GET, "/v1/schemas/platform/" + name, null, "", + "authorization", "admin"), 200); + } + } + + @Test + @SneakyThrows + void testMigrateSettings() { + String body = """ + {"types": ["settings"]} + """; + Response response = send(HttpMethod.POST, "/v1/admin/config/file/migrate", null, body, + "authorization", "admin"); + verify(response, 200); + JsonNode results = ProxyUtil.MAPPER.readTree(response.body()).get("results"); + assertTrue(idsWithStatus(results, "migrated").contains("settings/platform/global"), + () -> "Body: " + response.body()); + + Response get = send(HttpMethod.GET, "/v1/settings/platform/global", null, "", + "authorization", "admin"); + verify(get, 200); + JsonNode settings = ProxyUtil.MAPPER.readTree(get.body()); + assertTrue(settings.get("globalInterceptors").isArray()); + assertTrue(settings.get("retriableErrorCodes").isArray()); + } + + @Test + @SneakyThrows + void testTypesFilterOnlyMigratesRequestedTypes() { + String body = """ + {"types": ["keys"]} + """; + Response response = send(HttpMethod.POST, "/v1/admin/config/file/migrate", null, body, + "authorization", "admin"); + verify(response, 200); + JsonNode results = ProxyUtil.MAPPER.readTree(response.body()).get("results"); + for (JsonNode r : results) { + assertTrue(r.get("id").asText().startsWith("keys/platform/"), + () -> "Unexpected type migrated: " + response.body()); + } + // Untouched type never migrated. + verify(send(HttpMethod.GET, "/v1/models/platform/test-model-v1", null, "", + "authorization", "admin"), 404); + } + + @Test + @SneakyThrows + void testUnsupportedTypeReturns400() { + String body = """ + {"types": ["not-a-real-type"]} + """; + verify(send(HttpMethod.POST, "/v1/admin/config/file/migrate", null, body, + "authorization", "admin"), 400); + } + + @Test + void testNonAdminReturns403() { + verify(send(HttpMethod.POST, "/v1/admin/config/file/migrate", null, "{}", + "authorization", "user"), 403); + } + + private static Set idsWithStatus(JsonNode results, String status) { + Set ids = new HashSet<>(); + for (JsonNode r : results) { + if (status.equals(r.get("status").asText())) { + ids.add(r.get("id").asText()); + } + } + return ids; + } +}