diff --git a/backend/src/main/java/com/devflow/copilot/common/GlobalExceptionHandler.java b/backend/src/main/java/com/devflow/copilot/common/GlobalExceptionHandler.java index 13b958d..cc30fca 100644 --- a/backend/src/main/java/com/devflow/copilot/common/GlobalExceptionHandler.java +++ b/backend/src/main/java/com/devflow/copilot/common/GlobalExceptionHandler.java @@ -41,8 +41,21 @@ public ResponseEntity> handleIllegalArgument(IllegalArgumentEx @ExceptionHandler(Exception.class) public ResponseEntity> handleException(Exception ex) { - log.error("Unhandled request error", ex); + if (containsProviderException(ex)) { + log.error("Unhandled provider request error type: {}", ex.getClass().getSimpleName()); + } else { + log.error("Unhandled request error", ex); + } return ResponseEntity.status(HttpStatus.INTERNAL_SERVER_ERROR) .body(ApiResponse.fail(5000, "服务处理失败，请查看后端日志")); } + + private boolean containsProviderException(Throwable exception) { + for (Throwable current = exception; current != null; current = current.getCause()) { + if (current instanceof LlmProviderException) { + return true; + } + } + return false; + } } diff --git a/backend/src/main/java/com/devflow/copilot/common/LlmProviderException.java b/backend/src/main/java/com/devflow/copilot/common/LlmProviderException.java index 6694bec..a8956a2 100644 --- a/backend/src/main/java/com/devflow/copilot/common/LlmProviderException.java +++ b/backend/src/main/java/com/devflow/copilot/common/LlmProviderException.java @@ -1,15 +1,57 @@ package com.devflow.copilot.common; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import org.springframework.http.HttpStatus; +@JsonIgnoreProperties({"cause", "stackTrace", "suppressed", "localizedMessage"}) public class LlmProviderException extends BusinessException { - public LlmProviderException(String message) { - super(5001, HttpStatus.BAD_GATEWAY, message); + private final ProviderErrorType errorType; + private final ProviderFailureMetadata failureMetadata; + + public LlmProviderException(ProviderErrorType errorType) { + this(errorType, ProviderFailureMetadata.unknown(), null); + } + + public LlmProviderException(ProviderErrorType errorType, Throwable cause) { + this(errorType, ProviderFailureMetadata.unknown(), cause); + } + + public LlmProviderException(ProviderErrorType errorType, ProviderFailureMetadata failureMetadata) { + this(errorType, failureMetadata, null); + } + + public LlmProviderException(ProviderErrorType errorType, ProviderFailureMetadata failureMetadata, Throwable cause) { + super(5001, HttpStatus.BAD_GATEWAY, errorType.safeMessage()); + this.errorType = errorType; + this.failureMetadata = failureMetadata == null ? ProviderFailureMetadata.unknown() : failureMetadata; + if (cause != null) { + initCause(cause); + } + } + + public ProviderErrorType getErrorType() { + return errorType; + } + + public ProviderFailureMetadata getFailureMetadata() { + return failureMetadata; + } + + @Override + public String getMessage() { + return errorType.safeMessage(); + } + + @Override + @JsonIgnore + public synchronized Throwable getCause() { + return super.getCause(); } - public LlmProviderException(String message, Throwable cause) { - super(5001, HttpStatus.BAD_GATEWAY, message); - initCause(cause); + @Override + public String toString() { + return getClass().getName() + ": " + getMessage(); } } diff --git a/backend/src/main/java/com/devflow/copilot/common/ProviderContentTypeCategory.java b/backend/src/main/java/com/devflow/copilot/common/ProviderContentTypeCategory.java new file mode 100644 index 0000000..58737c3 --- /dev/null +++ b/backend/src/main/java/com/devflow/copilot/common/ProviderContentTypeCategory.java @@ -0,0 +1,9 @@ +package com.devflow.copilot.common; + +public enum ProviderContentTypeCategory { + JSON, + HTML, + TEXT, + OTHER, + UNKNOWN +} diff --git a/backend/src/main/java/com/devflow/copilot/common/ProviderDurationBucket.java b/backend/src/main/java/com/devflow/copilot/common/ProviderDurationBucket.java new file mode 100644 index 0000000..136ab04 --- /dev/null +++ b/backend/src/main/java/com/devflow/copilot/common/ProviderDurationBucket.java @@ -0,0 +1,11 @@ +package com.devflow.copilot.common; + +public enum ProviderDurationBucket { + UNDER_1_SECOND, + ONE_TO_FIVE_SECONDS, + FIVE_TO_FIFTEEN_SECONDS, + FIFTEEN_TO_SIXTY_SECONDS, + OVER_SIXTY_SECONDS, + TIMEOUT, + UNKNOWN +} diff --git a/backend/src/main/java/com/devflow/copilot/common/ProviderErrorType.java b/backend/src/main/java/com/devflow/copilot/common/ProviderErrorType.java new file mode 100644 index 0000000..b495702 --- /dev/null +++ b/backend/src/main/java/com/devflow/copilot/common/ProviderErrorType.java @@ -0,0 +1,24 @@ +package com.devflow.copilot.common; + +public enum ProviderErrorType { + PROTOCOL_UNSUPPORTED("Provider protocol is unsupported."), + API_KEY_MISSING("Provider credential is not configured."), + TIMEOUT("Provider request timed out."), + AUTHENTICATION_FAILED("Provider authentication failed."), + RATE_LIMITED("Provider rate limit reached."), + UPSTREAM_SERVER_ERROR("Provider upstream service failed."), + INVALID_RESPONSE("Provider returned an invalid response."), + EMPTY_CONTENT("Provider returned empty content."), + CONNECTION_FAILED("Provider connection failed."), + UNKNOWN_PROVIDER_ERROR("Provider request failed."); + + private final String safeSummary; + + ProviderErrorType(String safeSummary) { + this.safeSummary = safeSummary; + } + + public String safeMessage() { + return name() + ": " + safeSummary; + } +} diff --git a/backend/src/main/java/com/devflow/copilot/common/ProviderFailureMetadata.java b/backend/src/main/java/com/devflow/copilot/common/ProviderFailureMetadata.java new file mode 100644 index 0000000..53ed0a8 --- /dev/null +++ b/backend/src/main/java/com/devflow/copilot/common/ProviderFailureMetadata.java @@ -0,0 +1,50 @@ +package com.devflow.copilot.common; + +import java.util.UUID; + +/** + * Deliberately content-free data captured for an external provider failure. + * It must never contain response bodies, URLs, credentials, or header values. + */ +public record ProviderFailureMetadata( + ProviderFailureStage failureStage, + Integer httpStatusCode, + ProviderHttpStatusFamily httpStatusFamily, + Boolean responseBodyPresent, + ProviderResponseSizeBucket responseBodySizeBucket, + Boolean contentTypePresent, + ProviderContentTypeCategory contentTypeCategory, + Boolean retryAfterHeaderPresent, + Boolean upstreamRequestIdHeaderPresent, + String clientRequestId, + ProviderDurationBucket durationBucket +) { + + public ProviderFailureMetadata { + failureStage = failureStage == null ? ProviderFailureStage.UNKNOWN : failureStage; + httpStatusFamily = httpStatusFamily == null ? ProviderHttpStatusFamily.UNKNOWN : httpStatusFamily; + responseBodySizeBucket = responseBodySizeBucket == null ? ProviderResponseSizeBucket.UNKNOWN : responseBodySizeBucket; + contentTypeCategory = contentTypeCategory == null ? ProviderContentTypeCategory.UNKNOWN : contentTypeCategory; + durationBucket = durationBucket == null ? ProviderDurationBucket.UNKNOWN : durationBucket; + } + + public static ProviderFailureMetadata unknown() { + return new ProviderFailureMetadata( + ProviderFailureStage.UNKNOWN, null, ProviderHttpStatusFamily.UNKNOWN, + null, ProviderResponseSizeBucket.UNKNOWN, null, ProviderContentTypeCategory.UNKNOWN, + null, null, null, ProviderDurationBucket.UNKNOWN + ); + } + + public static ProviderFailureMetadata atStage(ProviderFailureStage stage) { + return new ProviderFailureMetadata( + stage, null, ProviderHttpStatusFamily.UNKNOWN, + null, ProviderResponseSizeBucket.UNKNOWN, null, ProviderContentTypeCategory.UNKNOWN, + null, null, null, ProviderDurationBucket.UNKNOWN + ); + } + + public static String newClientRequestId() { + return UUID.randomUUID().toString(); + } +} diff --git a/backend/src/main/java/com/devflow/copilot/common/ProviderFailureStage.java b/backend/src/main/java/com/devflow/copilot/common/ProviderFailureStage.java new file mode 100644 index 0000000..5a1d2a2 --- /dev/null +++ b/backend/src/main/java/com/devflow/copilot/common/ProviderFailureStage.java @@ -0,0 +1,11 @@ +package com.devflow.copilot.common; + +public enum ProviderFailureStage { + CONFIG_VALIDATION, + REQUEST_BUILD, + CONNECTION, + HTTP_STATUS_RECEIVED, + RESPONSE_DESERIALIZATION, + CONTENT_EXTRACTION, + UNKNOWN +} diff --git a/backend/src/main/java/com/devflow/copilot/common/ProviderHttpStatusFamily.java b/backend/src/main/java/com/devflow/copilot/common/ProviderHttpStatusFamily.java new file mode 100644 index 0000000..1e19d14 --- /dev/null +++ b/backend/src/main/java/com/devflow/copilot/common/ProviderHttpStatusFamily.java @@ -0,0 +1,20 @@ +package com.devflow.copilot.common; + +import com.fasterxml.jackson.annotation.JsonValue; + +public enum ProviderHttpStatusFamily { + FOUR_XX("4XX"), + FIVE_XX("5XX"), + UNKNOWN("UNKNOWN"); + + private final String value; + + ProviderHttpStatusFamily(String value) { + this.value = value; + } + + @JsonValue + public String value() { + return value; + } +} diff --git a/backend/src/main/java/com/devflow/copilot/common/ProviderResponseSizeBucket.java b/backend/src/main/java/com/devflow/copilot/common/ProviderResponseSizeBucket.java new file mode 100644 index 0000000..a70421b --- /dev/null +++ b/backend/src/main/java/com/devflow/copilot/common/ProviderResponseSizeBucket.java @@ -0,0 +1,9 @@ +package com.devflow.copilot.common; + +public enum ProviderResponseSizeBucket { + ZERO, + ONE_TO_1KB, + ONE_TO_16KB, + OVER_16KB, + UNKNOWN +} diff --git a/backend/src/main/java/com/devflow/copilot/config/AiProviderProperties.java b/backend/src/main/java/com/devflow/copilot/config/AiProviderProperties.java index 02c3871..d1e4405 100644 --- a/backend/src/main/java/com/devflow/copilot/config/AiProviderProperties.java +++ b/backend/src/main/java/com/devflow/copilot/config/AiProviderProperties.java @@ -6,9 +6,10 @@ public class AiProviderProperties { private String provider = "local-rule"; - private String baseUrl = "https://api.openai.com/v1"; + private String baseUrl = ""; private String apiKey = ""; private String model = "gpt-4.1-mini"; + private String protocol = "chat-completions-compatible"; private int timeoutSeconds = 60; private int maxTokens = 2048; private boolean fallbackToLocal = true; @@ -21,6 +22,8 @@ public class AiProviderProperties { public void setApiKey(String apiKey) { this.apiKey = apiKey; } public String getModel() { return model; } public void setModel(String model) { this.model = model; } + public String getProtocol() { return protocol; } + public void setProtocol(String protocol) { this.protocol = protocol; } public int getTimeoutSeconds() { return timeoutSeconds; } public void setTimeoutSeconds(int timeoutSeconds) { this.timeoutSeconds = timeoutSeconds; } public int getMaxTokens() { return maxTokens; } diff --git a/backend/src/main/java/com/devflow/copilot/dto/AiGenerateResponse.java b/backend/src/main/java/com/devflow/copilot/dto/AiGenerateResponse.java index 66786c6..b3bb024 100644 --- a/backend/src/main/java/com/devflow/copilot/dto/AiGenerateResponse.java +++ b/backend/src/main/java/com/devflow/copilot/dto/AiGenerateResponse.java @@ -15,6 +15,12 @@ public class AiGenerateResponse { private String status; private String providerName; private String modelName; + private String requestedProvider; + private String requestedModel; + private String actualProvider; + private String actualModel; + private Boolean fallbackUsed; + private String fallbackReason; private Long costTimeMs; private Integer promptTokens; private Integer completionTokens; diff --git a/backend/src/main/java/com/devflow/copilot/entity/GenerationRecord.java b/backend/src/main/java/com/devflow/copilot/entity/GenerationRecord.java index 49bd703..6d4d58d 100644 --- a/backend/src/main/java/com/devflow/copilot/entity/GenerationRecord.java +++ b/backend/src/main/java/com/devflow/copilot/entity/GenerationRecord.java @@ -22,8 +22,12 @@ public class GenerationRecord { private String outputContent; private GenerationStatus status; private Boolean confirmed; + private String requestedProvider; + private String requestedModel; private String providerName; private String modelName; + private Boolean fallbackUsed; + private String fallbackReason; private Long promptTemplateId; private String promptTemplateName; private Integer promptTemplateVersion; @@ -34,6 +38,17 @@ public class GenerationRecord { private Long costTimeMs; private Boolean success; private String errorMessage; + private String providerErrorType; + private String providerFailureStage; + private Integer providerHttpStatus; + private String providerHttpStatusFamily; + private String providerDurationBucket; + private Boolean providerResponseBodyPresent; + private String providerResponseSizeBucket; + private String providerContentTypeCategory; + private Boolean providerRetryAfterPresent; + private Boolean providerRequestIdPresent; + private String providerClientRequestId; @Version private Integer version; private LocalDateTime createdAt; diff --git a/backend/src/main/java/com/devflow/copilot/service/AgentWorkflowService.java b/backend/src/main/java/com/devflow/copilot/service/AgentWorkflowService.java index 9e6a2b6..e24a269 100644 --- a/backend/src/main/java/com/devflow/copilot/service/AgentWorkflowService.java +++ b/backend/src/main/java/com/devflow/copilot/service/AgentWorkflowService.java @@ -21,6 +21,8 @@ ToolCallRecord addToolCall(Long runId, Long stepId, String toolName, String inpu void createPendingReview(Long runId, Long generationRecordId); + void updateExecutionMetadata(Long runId, String providerName, String modelName, Long latencyMs); + void syncGenerationTransition(Long generationRecordId, GenerationStatus target); List list(Long projectId, Long generationRecordId); diff --git a/backend/src/main/java/com/devflow/copilot/service/impl/AgentWorkflowServiceImpl.java b/backend/src/main/java/com/devflow/copilot/service/impl/AgentWorkflowServiceImpl.java index aecbe75..29f0bb5 100644 --- a/backend/src/main/java/com/devflow/copilot/service/impl/AgentWorkflowServiceImpl.java +++ b/backend/src/main/java/com/devflow/copilot/service/impl/AgentWorkflowServiceImpl.java @@ -110,6 +110,19 @@ public void createPendingReview(Long runId, Long generationRecordId) { humanReviewMapper.insert(review); } + @Override + public void updateExecutionMetadata(Long runId, String providerName, String modelName, Long latencyMs) { + AgentRun run = runMapper.selectById(runId); + if (run == null) { + return; + } + run.setProviderName(providerName); + run.setModelName(modelName); + run.setLatencyMs(latencyMs); + run.setUpdatedAt(LocalDateTime.now()); + runMapper.updateById(run); + } + @Override @Transactional public void syncGenerationTransition(Long generationRecordId, GenerationStatus target) { diff --git a/backend/src/main/java/com/devflow/copilot/service/impl/LocalRuleGenerateService.java b/backend/src/main/java/com/devflow/copilot/service/impl/LocalRuleGenerateService.java index b71dd83..c2adcb7 100644 --- a/backend/src/main/java/com/devflow/copilot/service/impl/LocalRuleGenerateService.java +++ b/backend/src/main/java/com/devflow/copilot/service/impl/LocalRuleGenerateService.java @@ -2,6 +2,7 @@ import com.devflow.copilot.common.GenerationStatus; import com.devflow.copilot.common.LlmProviderException; +import com.devflow.copilot.common.ProviderFailureMetadata; import com.devflow.copilot.config.AiProviderProperties; import com.devflow.copilot.dto.AiGenerateRequest; import com.devflow.copilot.dto.AiGenerateResponse; @@ -98,13 +99,16 @@ public AiGenerateResponse generate(String generationType, AiGenerateRequest requ record.setOutputContent(result.content()); record.setProviderName(result.providerName()); record.setModelName(result.modelName()); + record.setFallbackUsed(result.fallbackUsed()); + record.setFallbackReason(result.fallbackReason()); record.setPromptTokens(result.promptTokens()); record.setCompletionTokens(result.completionTokens()); record.setTotalTokens(result.totalTokens()); record.setCostTimeMs(System.currentTimeMillis() - start); record.setSuccess(true); - record.setErrorMessage(result.fallbackReason()); + record.setErrorMessage(null); recordService.save(record); + agentWorkflowService.updateExecutionMetadata(run.getId(), result.providerName(), result.modelName(), record.getCostTimeMs()); List attachedReferences = knowledgeBaseService.attachReferences(record.getId(), references); AgentStep generationStep = agentWorkflowService.addStep(run.getId(), 4, "LLM_GENERATION", "Provider 生成", "SUCCESS", "Provider 返回 Artifact，状态进入待人工确认。", record.getCostTimeMs()); @@ -119,17 +123,22 @@ public AiGenerateResponse generate(String generationType, AiGenerateRequest requ GenerationRecord completed = recordService.transition(record.getId(), GenerationStatus.READY_FOR_REVIEW); return toResponse(completed, run.getId(), attachedReferences); } catch (RuntimeException ex) { - record.setProviderName(providerProperties.getProvider()); - record.setModelName(providerProperties.getModel()); + record.setProviderName(null); + record.setModelName(null); + record.setFallbackUsed(false); + record.setFallbackReason(null); record.setCostTimeMs(System.currentTimeMillis() - start); record.setSuccess(false); - record.setErrorMessage(safeMessage(ex)); + record.setErrorMessage(safeError(ex)); + if (ex instanceof LlmProviderException providerException) { + applyFailureMetadata(record, providerException); + } recordService.save(record); AgentStep generationStep = agentWorkflowService.addStep(run.getId(), 4, "LLM_GENERATION", "Provider 生成", "FAILED", - "Provider 调用失败：" + safeMessage(ex), record.getCostTimeMs()); + "Provider generation failed: " + safeError(ex), record.getCostTimeMs()); agentWorkflowService.addToolCall(run.getId(), generationStep.getId(), "generation-provider", - providerProperties.getProvider() + " / " + providerProperties.getModel(), - safeMessage(ex), "FAILED", record.getCostTimeMs()); + "external-provider-request", + safeError(ex), "FAILED", record.getCostTimeMs()); traceService.record(record, request, GenerationStatus.FAILED.name()); recordService.transition(record.getId(), GenerationStatus.FAILED); if (ex instanceof LlmProviderException llmProviderException) { @@ -153,8 +162,10 @@ private GenerationRecord createGeneratingRecord( record.setStatus(GenerationStatus.GENERATING); record.setConfirmed(false); record.setSuccess(false); - record.setProviderName(providerProperties.getProvider()); - record.setModelName(providerProperties.getModel()); + record.setRequestedProvider(requestedProvider()); + record.setRequestedModel(requestedModel()); + record.setFallbackUsed(false); + record.setFallbackReason(null); record.setPromptTemplateId(rendered.templateId()); record.setPromptTemplateName(rendered.templateName()); record.setPromptTemplateVersion(rendered.templateVersion()); @@ -179,6 +190,12 @@ private AiGenerateResponse toResponse(GenerationRecord record, Long agentRunId, .status(record.getStatus().name()) .providerName(record.getProviderName()) .modelName(record.getModelName()) + .requestedProvider(record.getRequestedProvider()) + .requestedModel(record.getRequestedModel()) + .actualProvider(record.getProviderName()) + .actualModel(record.getModelName()) + .fallbackUsed(record.getFallbackUsed()) + .fallbackReason(record.getFallbackReason()) .costTimeMs(record.getCostTimeMs()) .promptTokens(record.getPromptTokens()) .completionTokens(record.getCompletionTokens()) @@ -212,6 +229,15 @@ private String knowledgeQuery(AiGenerateRequest request) { .orElse(request.getInput()); } + private String requestedProvider() { + String provider = providerProperties.getProvider(); + return provider == null || provider.isBlank() ? "local-rule" : provider.trim(); + } + + private String requestedModel() { + return "local-rule".equals(requestedProvider()) ? "local-rule-mvp" : providerProperties.getModel(); + } + private String mergeKnowledgeContext(String existingContext, List references) { if (references.isEmpty()) { return existingContext; @@ -238,7 +264,25 @@ private String knowledgeSummary(List references) { .orElse("未命中知识引用"); } - private String safeMessage(RuntimeException ex) { - return ex.getMessage() == null || ex.getMessage().isBlank() ? ex.getClass().getSimpleName() : ex.getMessage(); + private String safeError(RuntimeException ex) { + if (ex instanceof LlmProviderException providerException) { + return providerException.getErrorType().safeMessage(); + } + return "UNKNOWN_PROVIDER_ERROR: Provider request failed."; + } + + private void applyFailureMetadata(GenerationRecord record, LlmProviderException exception) { + ProviderFailureMetadata metadata = exception.getFailureMetadata(); + record.setProviderErrorType(exception.getErrorType().name()); + record.setProviderFailureStage(metadata.failureStage().name()); + record.setProviderHttpStatus(metadata.httpStatusCode()); + record.setProviderHttpStatusFamily(metadata.httpStatusFamily().value()); + record.setProviderDurationBucket(metadata.durationBucket().name()); + record.setProviderResponseBodyPresent(metadata.responseBodyPresent()); + record.setProviderResponseSizeBucket(metadata.responseBodySizeBucket().name()); + record.setProviderContentTypeCategory(metadata.contentTypeCategory().name()); + record.setProviderRetryAfterPresent(metadata.retryAfterHeaderPresent()); + record.setProviderRequestIdPresent(metadata.upstreamRequestIdHeaderPresent()); + record.setProviderClientRequestId(metadata.clientRequestId()); } } diff --git a/backend/src/main/java/com/devflow/copilot/service/impl/LogDiagnosisServiceImpl.java b/backend/src/main/java/com/devflow/copilot/service/impl/LogDiagnosisServiceImpl.java index 10b8822..95d3135 100644 --- a/backend/src/main/java/com/devflow/copilot/service/impl/LogDiagnosisServiceImpl.java +++ b/backend/src/main/java/com/devflow/copilot/service/impl/LogDiagnosisServiceImpl.java @@ -63,8 +63,12 @@ public LogAnalyzeResponse analyze(LogAnalyzeRequest request) { record.setOutputContent(toMarkdown(savedLog)); record.setStatus(GenerationStatus.READY_FOR_REVIEW); record.setConfirmed(false); + record.setRequestedProvider("local-rule"); + record.setRequestedModel("local-rule-mvp"); record.setProviderName("local-rule"); record.setModelName("local-rule-mvp"); + record.setFallbackUsed(false); + record.setFallbackReason(null); record.setCostTimeMs(80L); record.setPromptTokens(estimateTokens(rawLog)); record.setCompletionTokens(estimateTokens(record.getOutputContent())); diff --git a/backend/src/main/java/com/devflow/copilot/service/provider/GenerationProviderRouter.java b/backend/src/main/java/com/devflow/copilot/service/provider/GenerationProviderRouter.java index 2ad6f0e..03c4a9c 100644 --- a/backend/src/main/java/com/devflow/copilot/service/provider/GenerationProviderRouter.java +++ b/backend/src/main/java/com/devflow/copilot/service/provider/GenerationProviderRouter.java @@ -1,6 +1,7 @@ package com.devflow.copilot.service.provider; import com.devflow.copilot.common.LlmProviderException; +import com.devflow.copilot.common.ProviderErrorType; import com.devflow.copilot.config.AiProviderProperties; import org.springframework.stereotype.Component; @@ -27,15 +28,15 @@ public ProviderResult generate(ProviderRequest request) { return localRule.generate(request); } if (!openAiCompatible.key().equals(selected)) { - throw new LlmProviderException("不支持的 AI Provider：" + selected); + throw new LlmProviderException(ProviderErrorType.UNKNOWN_PROVIDER_ERROR); } try { return openAiCompatible.generate(request); - } catch (RuntimeException ex) { + } catch (LlmProviderException ex) { if (!properties.isFallbackToLocal()) { throw ex; } - return localRule.generate(request).withFallbackReason(ex.getMessage()); + return localRule.generate(request).withFallbackReason(ex.getErrorType().name()); } } } diff --git a/backend/src/main/java/com/devflow/copilot/service/provider/LocalRuleGenerationProvider.java b/backend/src/main/java/com/devflow/copilot/service/provider/LocalRuleGenerationProvider.java index 35e8152..eeafb0e 100644 --- a/backend/src/main/java/com/devflow/copilot/service/provider/LocalRuleGenerationProvider.java +++ b/backend/src/main/java/com/devflow/copilot/service/provider/LocalRuleGenerationProvider.java @@ -24,7 +24,7 @@ public ProviderResult generate(ProviderRequest request) { int promptTokens = estimateTokens(request.renderedPrompt()); int completionTokens = estimateTokens(content); return new ProviderResult(content, key(), "local-rule-mvp", promptTokens, completionTokens, - promptTokens + completionTokens, null); + promptTokens + completionTokens, false, null); } private String requirementSplit(ProviderRequest request) { diff --git a/backend/src/main/java/com/devflow/copilot/service/provider/OpenAiCompatibleGenerationProvider.java b/backend/src/main/java/com/devflow/copilot/service/provider/OpenAiCompatibleGenerationProvider.java index a04ca3d..e85bf99 100644 --- a/backend/src/main/java/com/devflow/copilot/service/provider/OpenAiCompatibleGenerationProvider.java +++ b/backend/src/main/java/com/devflow/copilot/service/provider/OpenAiCompatibleGenerationProvider.java @@ -1,6 +1,9 @@ package com.devflow.copilot.service.provider; import com.devflow.copilot.common.LlmProviderException; +import com.devflow.copilot.common.ProviderErrorType; +import com.devflow.copilot.common.ProviderFailureMetadata; +import com.devflow.copilot.common.ProviderFailureStage; import com.devflow.copilot.config.AiProviderProperties; import com.fasterxml.jackson.databind.JsonNode; import org.springframework.http.client.SimpleClientHttpRequestFactory; @@ -26,17 +29,23 @@ public String key() { @Override public ProviderResult generate(ProviderRequest request) { + validateProtocol(); if (properties.getApiKey() == null || properties.getApiKey().isBlank()) { - throw new LlmProviderException("OpenAI-compatible Provider 缺少 DEVFLOW_AI_API_KEY"); + throw new LlmProviderException(ProviderErrorType.API_KEY_MISSING, + ProviderFailureMetadata.atStage(ProviderFailureStage.CONFIG_VALIDATION)); } + String baseUrl = normalizeBaseUrl(properties.getBaseUrl()); + String clientRequestId = ProviderFailureMetadata.newClientRequestId(); + long requestStartNanos = System.nanoTime(); try { SimpleClientHttpRequestFactory requestFactory = new SimpleClientHttpRequestFactory(); int timeoutMs = Math.max(1, properties.getTimeoutSeconds()) * 1000; requestFactory.setConnectTimeout(timeoutMs); requestFactory.setReadTimeout(timeoutMs); RestClient client = RestClient.builder() - .baseUrl(stripTrailingSlash(properties.getBaseUrl())) + .baseUrl(baseUrl) .defaultHeader("Authorization", "Bearer " + properties.getApiKey()) + .defaultHeader("X-Client-Request-Id", clientRequestId) .requestFactory(requestFactory) .build(); Map body = Map.of( @@ -50,33 +59,50 @@ public ProviderResult generate(ProviderRequest request) { .body(body) .retrieve() .body(JsonNode.class); - if (response == null || response.path("choices").isEmpty()) { - throw new LlmProviderException("OpenAI-compatible Provider 未返回 choices"); + if (response == null || !response.path("choices").isArray() || response.path("choices").isEmpty()) { + throw new LlmProviderException(ProviderErrorType.INVALID_RESPONSE, + ProviderErrorClassifier.metadataForStage(ProviderFailureStage.RESPONSE_DESERIALIZATION, + clientRequestId, requestStartNanos)); } - String content = response.path("choices").path(0).path("message").path("content").asText(); + JsonNode message = response.path("choices").path(0).path("message"); + if (!message.isObject() || !message.hasNonNull("content")) { + throw new LlmProviderException(ProviderErrorType.INVALID_RESPONSE, + ProviderErrorClassifier.metadataForStage(ProviderFailureStage.RESPONSE_DESERIALIZATION, + clientRequestId, requestStartNanos)); + } + String content = message.path("content").asText(); if (content == null || content.isBlank()) { - throw new LlmProviderException("OpenAI-compatible Provider 返回了空内容"); + throw new LlmProviderException(ProviderErrorType.EMPTY_CONTENT, + ProviderErrorClassifier.metadataForStage(ProviderFailureStage.CONTENT_EXTRACTION, + clientRequestId, requestStartNanos)); } JsonNode usage = response.path("usage"); Integer promptTokens = usage.has("prompt_tokens") ? usage.path("prompt_tokens").asInt() : null; Integer completionTokens = usage.has("completion_tokens") ? usage.path("completion_tokens").asInt() : null; Integer totalTokens = usage.has("total_tokens") ? usage.path("total_tokens").asInt() : null; - return new ProviderResult(content, key(), properties.getModel(), promptTokens, completionTokens, totalTokens, null); + return new ProviderResult(content, key(), properties.getModel(), promptTokens, completionTokens, totalTokens, false, null); } catch (LlmProviderException ex) { throw ex; } catch (Exception ex) { - throw new LlmProviderException("OpenAI-compatible 调用失败：" + safeMessage(ex), ex); + throw ProviderErrorClassifier.sanitize(ex, clientRequestId, requestStartNanos, + ProviderFailureStage.REQUEST_BUILD); } } - private String stripTrailingSlash(String value) { - if (value == null || value.isBlank()) { - throw new LlmProviderException("OpenAI-compatible Provider 缺少 base-url"); + private void validateProtocol() { + if (!"chat-completions-compatible".equals(properties.getProtocol())) { + throw new LlmProviderException(ProviderErrorType.PROTOCOL_UNSUPPORTED, + ProviderFailureMetadata.atStage(ProviderFailureStage.CONFIG_VALIDATION)); } - return value.endsWith("/") ? value.substring(0, value.length() - 1) : value; } - private String safeMessage(Exception ex) { - return ex.getMessage() == null || ex.getMessage().isBlank() ? ex.getClass().getSimpleName() : ex.getMessage(); + private String normalizeBaseUrl(String value) { + if (value == null || value.isBlank()) { + throw new LlmProviderException(ProviderErrorType.CONNECTION_FAILED, + ProviderFailureMetadata.atStage(ProviderFailureStage.CONFIG_VALIDATION)); + } + String normalized = value.endsWith("/") ? value.substring(0, value.length() - 1) : value; + String path = "/chat/completions"; + return normalized.endsWith(path) ? normalized.substring(0, normalized.length() - path.length()) : normalized; } } diff --git a/backend/src/main/java/com/devflow/copilot/service/provider/ProviderErrorClassifier.java b/backend/src/main/java/com/devflow/copilot/service/provider/ProviderErrorClassifier.java new file mode 100644 index 0000000..8c27c56 --- /dev/null +++ b/backend/src/main/java/com/devflow/copilot/service/provider/ProviderErrorClassifier.java @@ -0,0 +1,248 @@ +package com.devflow.copilot.service.provider; + +import com.devflow.copilot.common.LlmProviderException; +import com.devflow.copilot.common.ProviderContentTypeCategory; +import com.devflow.copilot.common.ProviderDurationBucket; +import com.devflow.copilot.common.ProviderErrorType; +import com.devflow.copilot.common.ProviderFailureMetadata; +import com.devflow.copilot.common.ProviderFailureStage; +import com.devflow.copilot.common.ProviderHttpStatusFamily; +import com.devflow.copilot.common.ProviderResponseSizeBucket; +import com.fasterxml.jackson.core.JsonProcessingException; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatusCode; +import org.springframework.http.MediaType; +import org.springframework.http.converter.HttpMessageNotReadableException; +import org.springframework.web.client.ResourceAccessException; +import org.springframework.web.client.RestClientResponseException; + +import java.net.ConnectException; +import java.net.SocketTimeoutException; +import java.net.http.HttpTimeoutException; + +final class ProviderErrorClassifier { + + private ProviderErrorClassifier() { + } + + static LlmProviderException sanitize(Exception exception, String clientRequestId, long requestStartNanos, + ProviderFailureStage defaultStage) { + if (exception instanceof LlmProviderException providerException) { + return providerException; + } + ProviderErrorType errorType = classify(exception); + return new LlmProviderException(errorType, + metadata(exception, clientRequestId, requestStartNanos, defaultStage, errorType), exception); + } + + static ProviderFailureMetadata metadataForStage(ProviderFailureStage stage, String clientRequestId, + long requestStartNanos) { + return new ProviderFailureMetadata( + stage, + null, + ProviderHttpStatusFamily.UNKNOWN, + null, + ProviderResponseSizeBucket.UNKNOWN, + null, + ProviderContentTypeCategory.UNKNOWN, + null, + null, + clientRequestId, + durationBucket(requestStartNanos, false) + ); + } + + static ProviderErrorType classify(Throwable exception) { + for (Throwable current = exception; current != null; current = current.getCause()) { + if (current instanceof LlmProviderException providerException) { + return providerException.getErrorType(); + } + if (current instanceof RestClientResponseException responseException) { + return fromStatus(responseException.getStatusCode()); + } + if (current instanceof SocketTimeoutException || current instanceof HttpTimeoutException) { + return ProviderErrorType.TIMEOUT; + } + if (current instanceof ConnectException) { + return ProviderErrorType.CONNECTION_FAILED; + } + if (current instanceof JsonProcessingException) { + return ProviderErrorType.INVALID_RESPONSE; + } + if (current instanceof HttpMessageNotReadableException) { + return ProviderErrorType.INVALID_RESPONSE; + } + if (current instanceof ResourceAccessException) { + if (hasCause(current, SocketTimeoutException.class) || hasCause(current, HttpTimeoutException.class)) { + return ProviderErrorType.TIMEOUT; + } + return ProviderErrorType.CONNECTION_FAILED; + } + } + return ProviderErrorType.UNKNOWN_PROVIDER_ERROR; + } + + private static boolean hasCause(Throwable exception, Class type) { + for (Throwable current = exception.getCause(); current != null; current = current.getCause()) { + if (type.isInstance(current)) { + return true; + } + } + return false; + } + + private static ProviderFailureMetadata metadata( + Throwable exception, + String clientRequestId, + long requestStartNanos, + ProviderFailureStage defaultStage, + ProviderErrorType errorType + ) { + RestClientResponseException responseException = findCause(exception, RestClientResponseException.class); + if (responseException != null) { + return responseMetadata(responseException, clientRequestId, durationBucket(requestStartNanos, false)); + } + ProviderFailureStage stage = stageFor(exception, defaultStage); + boolean timeout = errorType == ProviderErrorType.TIMEOUT; + return new ProviderFailureMetadata( + stage, + null, + ProviderHttpStatusFamily.UNKNOWN, + null, + ProviderResponseSizeBucket.UNKNOWN, + null, + ProviderContentTypeCategory.UNKNOWN, + null, + null, + clientRequestId, + durationBucket(requestStartNanos, timeout) + ); + } + + private static ProviderFailureMetadata responseMetadata( + RestClientResponseException responseException, + String clientRequestId, + ProviderDurationBucket durationBucket + ) { + HttpHeaders headers = responseException.getResponseHeaders(); + byte[] body = responseException.getResponseBodyAsByteArray(); + int statusCode = responseException.getStatusCode().value(); + return new ProviderFailureMetadata( + ProviderFailureStage.HTTP_STATUS_RECEIVED, + statusCode, + statusFamily(statusCode), + body.length > 0, + sizeBucket(body.length), + headers != null && headers.getContentType() != null, + contentTypeCategory(headers == null ? null : headers.getContentType()), + headers != null && headers.containsKey(HttpHeaders.RETRY_AFTER), + hasUpstreamRequestId(headers), + clientRequestId, + durationBucket + ); + } + + private static ProviderFailureStage stageFor(Throwable exception, ProviderFailureStage defaultStage) { + if (findCause(exception, JsonProcessingException.class) != null + || findCause(exception, HttpMessageNotReadableException.class) != null) { + return ProviderFailureStage.RESPONSE_DESERIALIZATION; + } + if (findCause(exception, ResourceAccessException.class) != null + || findCause(exception, ConnectException.class) != null + || findCause(exception, SocketTimeoutException.class) != null + || findCause(exception, HttpTimeoutException.class) != null) { + return ProviderFailureStage.CONNECTION; + } + return defaultStage == null ? ProviderFailureStage.UNKNOWN : defaultStage; + } + + private static ProviderDurationBucket durationBucket(long requestStartNanos, boolean timeout) { + if (timeout) { + return ProviderDurationBucket.TIMEOUT; + } + if (requestStartNanos <= 0) { + return ProviderDurationBucket.UNKNOWN; + } + long elapsedMs = Math.max(0, (System.nanoTime() - requestStartNanos) / 1_000_000L); + if (elapsedMs < 1_000) { + return ProviderDurationBucket.UNDER_1_SECOND; + } + if (elapsedMs < 5_000) { + return ProviderDurationBucket.ONE_TO_FIVE_SECONDS; + } + if (elapsedMs < 15_000) { + return ProviderDurationBucket.FIVE_TO_FIFTEEN_SECONDS; + } + if (elapsedMs < 60_000) { + return ProviderDurationBucket.FIFTEEN_TO_SIXTY_SECONDS; + } + return ProviderDurationBucket.OVER_SIXTY_SECONDS; + } + + private static ProviderHttpStatusFamily statusFamily(int statusCode) { + if (statusCode >= 400 && statusCode < 500) { + return ProviderHttpStatusFamily.FOUR_XX; + } + if (statusCode >= 500 && statusCode < 600) { + return ProviderHttpStatusFamily.FIVE_XX; + } + return ProviderHttpStatusFamily.UNKNOWN; + } + + private static ProviderResponseSizeBucket sizeBucket(int length) { + if (length == 0) { + return ProviderResponseSizeBucket.ZERO; + } + if (length <= 1024) { + return ProviderResponseSizeBucket.ONE_TO_1KB; + } + if (length <= 16 * 1024) { + return ProviderResponseSizeBucket.ONE_TO_16KB; + } + return ProviderResponseSizeBucket.OVER_16KB; + } + + private static ProviderContentTypeCategory contentTypeCategory(MediaType contentType) { + if (contentType == null) { + return ProviderContentTypeCategory.UNKNOWN; + } + if (MediaType.APPLICATION_JSON.isCompatibleWith(contentType) || contentType.getSubtype().endsWith("+json")) { + return ProviderContentTypeCategory.JSON; + } + if (MediaType.TEXT_HTML.isCompatibleWith(contentType)) { + return ProviderContentTypeCategory.HTML; + } + if ("text".equalsIgnoreCase(contentType.getType())) { + return ProviderContentTypeCategory.TEXT; + } + return ProviderContentTypeCategory.OTHER; + } + + private static boolean hasUpstreamRequestId(HttpHeaders headers) { + return headers != null && (headers.containsKey("X-Request-Id") + || headers.containsKey("Request-Id") + || headers.containsKey("X-Correlation-Id")); + } + + private static T findCause(Throwable exception, Class type) { + for (Throwable current = exception; current != null; current = current.getCause()) { + if (type.isInstance(current)) { + return type.cast(current); + } + } + return null; + } + + private static ProviderErrorType fromStatus(HttpStatusCode statusCode) { + if (statusCode.value() == 401 || statusCode.value() == 403) { + return ProviderErrorType.AUTHENTICATION_FAILED; + } + if (statusCode.value() == 429) { + return ProviderErrorType.RATE_LIMITED; + } + if (statusCode.is5xxServerError()) { + return ProviderErrorType.UPSTREAM_SERVER_ERROR; + } + return ProviderErrorType.UNKNOWN_PROVIDER_ERROR; + } +} diff --git a/backend/src/main/java/com/devflow/copilot/service/provider/ProviderResult.java b/backend/src/main/java/com/devflow/copilot/service/provider/ProviderResult.java index 5d09d44..62565bb 100644 --- a/backend/src/main/java/com/devflow/copilot/service/provider/ProviderResult.java +++ b/backend/src/main/java/com/devflow/copilot/service/provider/ProviderResult.java @@ -7,9 +7,10 @@ public record ProviderResult( Integer promptTokens, Integer completionTokens, Integer totalTokens, + boolean fallbackUsed, String fallbackReason ) { public ProviderResult withFallbackReason(String reason) { - return new ProviderResult(content, providerName, modelName, promptTokens, completionTokens, totalTokens, reason); + return new ProviderResult(content, providerName, modelName, promptTokens, completionTokens, totalTokens, true, reason); } } diff --git a/backend/src/main/resources/application-dev.yml b/backend/src/main/resources/application-dev.yml index 5b4fb2d..8af6412 100644 --- a/backend/src/main/resources/application-dev.yml +++ b/backend/src/main/resources/application-dev.yml @@ -11,4 +11,4 @@ spring: devflow: ai: - provider: ${DEVFLOW_AI_PROVIDER:local-rule} + provider: ${DEVFLOW_AI_PROVIDER:${PORTFOLIO_AI_PROVIDER:local-rule}} diff --git a/backend/src/main/resources/application-prod.yml b/backend/src/main/resources/application-prod.yml index c4e2af5..d19c96d 100644 --- a/backend/src/main/resources/application-prod.yml +++ b/backend/src/main/resources/application-prod.yml @@ -10,4 +10,4 @@ spring: devflow: ai: - provider: ${DEVFLOW_AI_PROVIDER:local-rule} + provider: ${DEVFLOW_AI_PROVIDER:${PORTFOLIO_AI_PROVIDER:local-rule}} diff --git a/backend/src/main/resources/application.yml b/backend/src/main/resources/application.yml index 297b349..7d7e314 100644 --- a/backend/src/main/resources/application.yml +++ b/backend/src/main/resources/application.yml @@ -19,10 +19,11 @@ mybatis-plus: devflow: ai: - provider: ${DEVFLOW_AI_PROVIDER:local-rule} - base-url: ${DEVFLOW_AI_BASE_URL:https://api.openai.com/v1} - api-key: ${DEVFLOW_AI_API_KEY:} - model: ${DEVFLOW_AI_MODEL:gpt-4.1-mini} + provider: ${DEVFLOW_AI_PROVIDER:${PORTFOLIO_AI_PROVIDER:local-rule}} + base-url: ${DEVFLOW_AI_BASE_URL:${PORTFOLIO_AI_BASE_URL:https://api.openai.com/v1}} + api-key: ${DEVFLOW_AI_API_KEY:${PORTFOLIO_AI_API_KEY:}} + model: ${DEVFLOW_AI_MODEL:${PORTFOLIO_AI_MODEL:gpt-4.1-mini}} + protocol: ${DEVFLOW_AI_PROTOCOL:${PORTFOLIO_AI_PROTOCOL:chat-completions-compatible}} timeout-seconds: ${DEVFLOW_AI_TIMEOUT_SECONDS:60} max-tokens: ${DEVFLOW_AI_MAX_TOKENS:2048} - fallback-to-local: ${DEVFLOW_AI_FALLBACK_TO_LOCAL:true} + fallback-to-local: ${DEVFLOW_AI_FALLBACK_TO_LOCAL:${PORTFOLIO_AI_FALLBACK_ENABLED:true}} diff --git a/backend/src/main/resources/db/migration/V5__add_provider_execution_metadata.sql b/backend/src/main/resources/db/migration/V5__add_provider_execution_metadata.sql new file mode 100644 index 0000000..20ec623 --- /dev/null +++ b/backend/src/main/resources/db/migration/V5__add_provider_execution_metadata.sql @@ -0,0 +1,10 @@ +ALTER TABLE generation_record ADD COLUMN requested_provider VARCHAR(64); +ALTER TABLE generation_record ADD COLUMN requested_model VARCHAR(128); +ALTER TABLE generation_record ADD COLUMN fallback_used BOOLEAN NOT NULL DEFAULT FALSE; +ALTER TABLE generation_record ADD COLUMN fallback_reason VARCHAR(64); + +UPDATE generation_record +SET requested_provider = provider_name, + requested_model = model_name +WHERE requested_provider IS NULL + AND requested_model IS NULL; diff --git a/backend/src/main/resources/db/migration/V6__add_provider_failure_observability.sql b/backend/src/main/resources/db/migration/V6__add_provider_failure_observability.sql new file mode 100644 index 0000000..2a3ea24 --- /dev/null +++ b/backend/src/main/resources/db/migration/V6__add_provider_failure_observability.sql @@ -0,0 +1,11 @@ +ALTER TABLE generation_record ADD COLUMN provider_error_type VARCHAR(64); +ALTER TABLE generation_record ADD COLUMN provider_failure_stage VARCHAR(64); +ALTER TABLE generation_record ADD COLUMN provider_http_status INT; +ALTER TABLE generation_record ADD COLUMN provider_http_status_family VARCHAR(16); +ALTER TABLE generation_record ADD COLUMN provider_duration_bucket VARCHAR(32); +ALTER TABLE generation_record ADD COLUMN provider_response_body_present BOOLEAN; +ALTER TABLE generation_record ADD COLUMN provider_response_size_bucket VARCHAR(32); +ALTER TABLE generation_record ADD COLUMN provider_content_type_category VARCHAR(32); +ALTER TABLE generation_record ADD COLUMN provider_retry_after_present BOOLEAN; +ALTER TABLE generation_record ADD COLUMN provider_request_id_present BOOLEAN; +ALTER TABLE generation_record ADD COLUMN provider_client_request_id VARCHAR(36); diff --git a/backend/src/test/java/com/devflow/copilot/AgenticWorkflowIntegrationTest.java b/backend/src/test/java/com/devflow/copilot/AgenticWorkflowIntegrationTest.java index 302152d..0d3e03b 100644 --- a/backend/src/test/java/com/devflow/copilot/AgenticWorkflowIntegrationTest.java +++ b/backend/src/test/java/com/devflow/copilot/AgenticWorkflowIntegrationTest.java @@ -26,7 +26,7 @@ import static org.assertj.core.api.Assertions.assertThat; -@SpringBootTest +@SpringBootTest(properties = {"devflow.ai.provider=local-rule", "devflow.ai.api-key=", "devflow.ai.protocol=chat-completions-compatible", "devflow.ai.fallback-to-local=true"}) @ActiveProfiles("test") @Transactional class AgenticWorkflowIntegrationTest { @@ -57,11 +57,15 @@ void generationCreatesTraceAgentRunToolCallHumanReviewAndKnowledgeReferences() { List traces = traceService.list(response.getRecordId()); assertThat(traces).hasSize(1); assertThat(traces.get(0).getProviderName()).isEqualTo("local-rule"); + assertThat(traces.get(0).getProviderName()).isEqualTo(record.getProviderName()); + assertThat(traces.get(0).getModelName()).isEqualTo(record.getModelName()); assertThat(traces.get(0).getStatus()).isEqualTo("READY_FOR_REVIEW"); assertThat(traces.get(0).getInputVariables()).doesNotContain("DEVFLOW_AI_API_KEY"); AgentRunTraceResponse runTrace = agentWorkflowService.getTrace(response.getAgentRunId()); assertThat(runTrace.getRun().getStatus()).isEqualTo("WAITING_REVIEW"); + assertThat(runTrace.getRun().getProviderName()).isEqualTo(record.getProviderName()); + assertThat(runTrace.getRun().getModelName()).isEqualTo(record.getModelName()); assertThat(runTrace.getSteps()).extracting(AgentStep::getStepType) .contains("TASK_DECOMPOSITION", "PROMPT_RENDER", "KNOWLEDGE_RETRIEVAL", "LLM_GENERATION", "HUMAN_REVIEW"); assertThat(runTrace.getToolCalls()).extracting("toolName") diff --git a/backend/src/test/java/com/devflow/copilot/ControllerAndMapperIntegrationTest.java b/backend/src/test/java/com/devflow/copilot/ControllerAndMapperIntegrationTest.java index 383d121..79b9ed5 100644 --- a/backend/src/test/java/com/devflow/copilot/ControllerAndMapperIntegrationTest.java +++ b/backend/src/test/java/com/devflow/copilot/ControllerAndMapperIntegrationTest.java @@ -21,7 +21,7 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath; import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; -@SpringBootTest +@SpringBootTest(properties = {"devflow.ai.provider=local-rule", "devflow.ai.api-key=", "devflow.ai.protocol=chat-completions-compatible", "devflow.ai.fallback-to-local=true"}) @AutoConfigureMockMvc @ActiveProfiles("test") @Transactional diff --git a/backend/src/test/java/com/devflow/copilot/GenerationWorkflowIntegrationTest.java b/backend/src/test/java/com/devflow/copilot/GenerationWorkflowIntegrationTest.java index c5d498d..c947818 100644 --- a/backend/src/test/java/com/devflow/copilot/GenerationWorkflowIntegrationTest.java +++ b/backend/src/test/java/com/devflow/copilot/GenerationWorkflowIntegrationTest.java @@ -17,7 +17,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -@SpringBootTest +@SpringBootTest(properties = {"devflow.ai.provider=local-rule", "devflow.ai.model=synthetic-requested-model", "devflow.ai.api-key=", "devflow.ai.protocol=chat-completions-compatible", "devflow.ai.fallback-to-local=true"}) @ActiveProfiles("test") @Transactional class GenerationWorkflowIntegrationTest { @@ -31,8 +31,16 @@ void localRuleGeneratesAndPersistsTraceMetadata() { GenerationRecord persisted = recordService.getById(response.getRecordId()); assertThat(response.getProviderName()).isEqualTo("local-rule"); + assertThat(response.getRequestedProvider()).isEqualTo(response.getActualProvider()); + assertThat(response.getRequestedModel()).isEqualTo(response.getActualModel()); + assertThat(response.getFallbackUsed()).isFalse(); + assertThat(response.getFallbackReason()).isNull(); assertThat(response.getTotalTokens()).isPositive(); assertThat(persisted.getStatus()).isEqualTo(GenerationStatus.READY_FOR_REVIEW); + assertThat(persisted.getRequestedProvider()).isEqualTo(persisted.getProviderName()); + assertThat(persisted.getRequestedModel()).isEqualTo(persisted.getModelName()); + assertThat(persisted.getFallbackUsed()).isFalse(); + assertThat(persisted.getFallbackReason()).isNull(); assertThat(persisted.getRenderedPrompt()).contains("需求拆解测试"); } diff --git a/backend/src/test/java/com/devflow/copilot/PromptTemplateRenderIntegrationTest.java b/backend/src/test/java/com/devflow/copilot/PromptTemplateRenderIntegrationTest.java index b64737b..3a490c2 100644 --- a/backend/src/test/java/com/devflow/copilot/PromptTemplateRenderIntegrationTest.java +++ b/backend/src/test/java/com/devflow/copilot/PromptTemplateRenderIntegrationTest.java @@ -21,7 +21,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -@SpringBootTest +@SpringBootTest(properties = {"devflow.ai.provider=local-rule", "devflow.ai.api-key=", "devflow.ai.protocol=chat-completions-compatible", "devflow.ai.fallback-to-local=true"}) @ActiveProfiles("test") @Transactional class PromptTemplateRenderIntegrationTest { diff --git a/backend/src/test/java/com/devflow/copilot/ProviderAndDiagnosisTest.java b/backend/src/test/java/com/devflow/copilot/ProviderAndDiagnosisTest.java index a785900..57fcc32 100644 --- a/backend/src/test/java/com/devflow/copilot/ProviderAndDiagnosisTest.java +++ b/backend/src/test/java/com/devflow/copilot/ProviderAndDiagnosisTest.java @@ -1,6 +1,7 @@ package com.devflow.copilot; import com.devflow.copilot.common.LlmProviderException; +import com.devflow.copilot.common.ProviderErrorType; import com.devflow.copilot.config.AiProviderProperties; import com.devflow.copilot.dto.LogAnalyzeRequest; import com.devflow.copilot.dto.LogAnalyzeResponse; @@ -19,7 +20,7 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatThrownBy; -@SpringBootTest +@SpringBootTest(properties = {"devflow.ai.provider=local-rule", "devflow.ai.api-key=", "devflow.ai.protocol=chat-completions-compatible", "devflow.ai.fallback-to-local=true"}) @ActiveProfiles("test") @Transactional class ProviderAndDiagnosisTest { @@ -34,7 +35,7 @@ void openAiProviderRejectsMissingApiKey() { assertThatThrownBy(() -> provider.generate(providerRequest())) .isInstanceOf(LlmProviderException.class) - .hasMessageContaining("API_KEY"); + .hasMessageContaining(ProviderErrorType.API_KEY_MISSING.name()); } @Test @@ -50,7 +51,7 @@ void routerFallsBackToLocalRuleWhenOpenAiConfigIsMissing() { ProviderResult result = router.generate(providerRequest()); assertThat(result.providerName()).isEqualTo("local-rule"); - assertThat(result.fallbackReason()).contains("API_KEY"); + assertThat(result.fallbackReason()).contains(ProviderErrorType.API_KEY_MISSING.name()); } @Test diff --git a/backend/src/test/java/com/devflow/copilot/ProviderErrorSanitizationIntegrationTest.java b/backend/src/test/java/com/devflow/copilot/ProviderErrorSanitizationIntegrationTest.java new file mode 100644 index 0000000..193a2bb --- /dev/null +++ b/backend/src/test/java/com/devflow/copilot/ProviderErrorSanitizationIntegrationTest.java @@ -0,0 +1,153 @@ +package com.devflow.copilot; + +import com.devflow.copilot.common.LlmProviderException; +import com.devflow.copilot.common.ProviderContentTypeCategory; +import com.devflow.copilot.common.ProviderDurationBucket; +import com.devflow.copilot.common.ProviderErrorType; +import com.devflow.copilot.common.ProviderFailureMetadata; +import com.devflow.copilot.common.ProviderFailureStage; +import com.devflow.copilot.common.ProviderHttpStatusFamily; +import com.devflow.copilot.common.ProviderResponseSizeBucket; +import com.devflow.copilot.dto.AgentRunTraceResponse; +import com.devflow.copilot.dto.AiGenerateRequest; +import com.devflow.copilot.entity.GenerationRecord; +import com.devflow.copilot.entity.GenerationTrace; +import com.devflow.copilot.service.AgentWorkflowService; +import com.devflow.copilot.service.AiGenerateService; +import com.devflow.copilot.service.GenerationRecordService; +import com.devflow.copilot.service.GenerationTraceService; +import com.devflow.copilot.service.LlmGenerateService; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.transaction.annotation.Transactional; + +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.when; + +@SpringBootTest(properties = {"devflow.ai.provider=local-rule", "devflow.ai.api-key=", "devflow.ai.protocol=chat-completions-compatible", "devflow.ai.fallback-to-local=true"}) +@ActiveProfiles("test") +@Transactional +class ProviderErrorSanitizationIntegrationTest { + + @Autowired AiGenerateService aiGenerateService; + @Autowired GenerationRecordService recordService; + @Autowired GenerationTraceService traceService; + @Autowired AgentWorkflowService agentWorkflowService; + + @MockBean LlmGenerateService llmGenerateService; + + @Test + void persistsOnlyClassifiedSafeErrorsForSyntheticUpstreamFailure() { + String authorization = "Authorization" + ": " + "Bearer " + "synthetic-secret-token"; + String rawFailure = authorization + " " + "https" + "://synthetic-provider.invalid/v1 " + + "synthetic upstream failure"; + when(llmGenerateService.generate(any())) + .thenThrow(new LlmProviderException(ProviderErrorType.UPSTREAM_SERVER_ERROR, + metadata(502, ProviderHttpStatusFamily.FIVE_XX, true, true), new RuntimeException(rawFailure))); + + AiGenerateRequest request = new AiGenerateRequest(); + request.setProjectId(1L); + request.setInput("offline provider safety test"); + + assertThatThrownBy(() -> aiGenerateService.generate("requirement-split", request)) + .isInstanceOf(LlmProviderException.class) + .hasMessageContaining(ProviderErrorType.UPSTREAM_SERVER_ERROR.name()) + .satisfies(error -> assertThat(error.getMessage()).doesNotContain("synthetic-secret-token")); + + GenerationRecord record = recordService.recent(1).get(0); + List traces = traceService.list(record.getId()); + AgentRunTraceResponse runTrace = agentWorkflowService.getTrace( + agentWorkflowService.list(null, record.getId()).get(0).getId()); + + assertSafe(record.getErrorMessage()); + assertThat(record.getRequestedProvider()).isNotBlank(); + assertThat(record.getRequestedModel()).isNotBlank(); + assertThat(record.getProviderName()).isNull(); + assertThat(record.getModelName()).isNull(); + assertThat(record.getFallbackUsed()).isFalse(); + assertThat(record.getFallbackReason()).isNull(); + assertThat(record.getProviderErrorType()).isEqualTo(ProviderErrorType.UPSTREAM_SERVER_ERROR.name()); + assertThat(record.getProviderFailureStage()).isEqualTo(ProviderFailureStage.HTTP_STATUS_RECEIVED.name()); + assertThat(record.getProviderHttpStatus()).isEqualTo(502); + assertThat(record.getProviderHttpStatusFamily()).isEqualTo(ProviderHttpStatusFamily.FIVE_XX.value()); + assertThat(record.getProviderResponseBodyPresent()).isTrue(); + assertThat(record.getProviderResponseSizeBucket()).isEqualTo(ProviderResponseSizeBucket.ONE_TO_1KB.name()); + assertThat(record.getProviderContentTypeCategory()).isEqualTo(ProviderContentTypeCategory.HTML.name()); + assertThat(record.getProviderRetryAfterPresent()).isTrue(); + assertThat(record.getProviderRequestIdPresent()).isTrue(); + assertThat(record.getProviderClientRequestId()).matches("[0-9a-f-]{36}"); + assertSafe(traces.get(0).getErrorMessage()); + assertThat(traces.get(0).getProviderName()).isNull(); + assertThat(traces.get(0).getModelName()).isNull(); + assertThat(runTrace.getRun().getProviderName()).isNull(); + assertThat(runTrace.getRun().getModelName()).isNull(); + runTrace.getSteps().stream() + .filter(step -> "FAILED".equals(step.getStatus())) + .forEach(step -> assertSafe(step.getSummary())); + runTrace.getToolCalls().stream() + .filter(toolCall -> "FAILED".equals(toolCall.getStatus())) + .forEach(toolCall -> assertSafe(toolCall.getOutputSummary())); + } + + @Test + void persists503AndKeeps429SeparateFromUpstreamServerErrors() { + when(llmGenerateService.generate(any())) + .thenThrow(new LlmProviderException(ProviderErrorType.UPSTREAM_SERVER_ERROR, + metadata(503, ProviderHttpStatusFamily.FIVE_XX, false, false))); + + assertThatThrownBy(() -> aiGenerateService.generate("requirement-split", request())) + .isInstanceOf(LlmProviderException.class); + GenerationRecord upstream = recordService.recent(1).get(0); + assertThat(upstream.getProviderHttpStatus()).isEqualTo(503); + assertThat(upstream.getProviderErrorType()).isEqualTo(ProviderErrorType.UPSTREAM_SERVER_ERROR.name()); + + doThrow(new LlmProviderException(ProviderErrorType.RATE_LIMITED, + metadata(429, ProviderHttpStatusFamily.FOUR_XX, false, false))) + .when(llmGenerateService).generate(any()); + + assertThatThrownBy(() -> aiGenerateService.generate("requirement-split", request())) + .isInstanceOf(LlmProviderException.class); + GenerationRecord rateLimited = recordService.recent(1).get(0); + assertThat(rateLimited.getProviderHttpStatus()).isEqualTo(429); + assertThat(rateLimited.getProviderErrorType()).isEqualTo(ProviderErrorType.RATE_LIMITED.name()); + assertThat(rateLimited.getProviderErrorType()).isNotEqualTo(ProviderErrorType.UPSTREAM_SERVER_ERROR.name()); + } + + private AiGenerateRequest request() { + AiGenerateRequest request = new AiGenerateRequest(); + request.setProjectId(1L); + request.setInput("offline metadata persistence test"); + return request; + } + + private ProviderFailureMetadata metadata(int status, ProviderHttpStatusFamily family, + boolean retryAfterPresent, boolean upstreamRequestIdPresent) { + return new ProviderFailureMetadata( + ProviderFailureStage.HTTP_STATUS_RECEIVED, + status, + family, + true, + ProviderResponseSizeBucket.ONE_TO_1KB, + true, + ProviderContentTypeCategory.HTML, + retryAfterPresent, + upstreamRequestIdPresent, + "00000000-0000-4000-8000-000000000001", + ProviderDurationBucket.UNDER_1_SECOND + ); + } + + private void assertSafe(String value) { + assertThat(value) + .contains(ProviderErrorType.UPSTREAM_SERVER_ERROR.name()) + .doesNotContain("synthetic-secret-token", "synthetic-provider.invalid", "synthetic upstream failure"); + } +} diff --git a/backend/src/test/java/com/devflow/copilot/ProviderExecutionExternalSuccessPersistenceIntegrationTest.java b/backend/src/test/java/com/devflow/copilot/ProviderExecutionExternalSuccessPersistenceIntegrationTest.java new file mode 100644 index 0000000..f246052 --- /dev/null +++ b/backend/src/test/java/com/devflow/copilot/ProviderExecutionExternalSuccessPersistenceIntegrationTest.java @@ -0,0 +1,49 @@ +package com.devflow.copilot; + +import com.devflow.copilot.dto.AiGenerateRequest; +import com.devflow.copilot.dto.AiGenerateResponse; +import com.devflow.copilot.entity.GenerationRecord; +import com.devflow.copilot.service.AiGenerateService; +import com.devflow.copilot.service.GenerationRecordService; +import com.devflow.copilot.service.LlmGenerateService; +import com.devflow.copilot.service.provider.ProviderResult; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.mock.mockito.MockBean; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.transaction.annotation.Transactional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.when; + +@SpringBootTest(properties = {"devflow.ai.provider=openai-compatible", "devflow.ai.api-key=", "devflow.ai.protocol=chat-completions-compatible", "devflow.ai.fallback-to-local=false"}) +@ActiveProfiles("test") +@Transactional +class ProviderExecutionExternalSuccessPersistenceIntegrationTest { + + @Autowired AiGenerateService aiGenerateService; + @Autowired GenerationRecordService recordService; + @MockBean LlmGenerateService llmGenerateService; + + @Test + void successfulExternalResultPersistsItsOwnActualExecutionMetadata() { + when(llmGenerateService.generate(any())).thenReturn(new ProviderResult( + "synthetic external result", "openai-compatible", "local-rule-mvp", 1, 2, 3, false, null + )); + + AiGenerateRequest request = new AiGenerateRequest(); + request.setProjectId(1L); + request.setInput("offline external success persistence test"); + + AiGenerateResponse response = aiGenerateService.generate("requirement-split", request); + GenerationRecord record = recordService.getById(response.getRecordId()); + + assertThat(record.getRequestedProvider()).isEqualTo("openai-compatible"); + assertThat(record.getProviderName()).isEqualTo("openai-compatible"); + assertThat(record.getRequestedModel()).isEqualTo(record.getModelName()); + assertThat(record.getFallbackUsed()).isFalse(); + assertThat(record.getFallbackReason()).isNull(); + } +} diff --git a/backend/src/test/java/com/devflow/copilot/ProviderExecutionFailurePersistenceIntegrationTest.java b/backend/src/test/java/com/devflow/copilot/ProviderExecutionFailurePersistenceIntegrationTest.java new file mode 100644 index 0000000..25ad01c --- /dev/null +++ b/backend/src/test/java/com/devflow/copilot/ProviderExecutionFailurePersistenceIntegrationTest.java @@ -0,0 +1,52 @@ +package com.devflow.copilot; + +import com.devflow.copilot.common.LlmProviderException; +import com.devflow.copilot.dto.AiGenerateRequest; +import com.devflow.copilot.entity.GenerationRecord; +import com.devflow.copilot.service.AgentWorkflowService; +import com.devflow.copilot.service.AiGenerateService; +import com.devflow.copilot.service.GenerationRecordService; +import com.devflow.copilot.service.GenerationTraceService; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.transaction.annotation.Transactional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@SpringBootTest(properties = {"devflow.ai.provider=openai-compatible", "devflow.ai.api-key=", "devflow.ai.protocol=chat-completions-compatible", "devflow.ai.fallback-to-local=false"}) +@ActiveProfiles("test") +@Transactional +class ProviderExecutionFailurePersistenceIntegrationTest { + + @Autowired AiGenerateService aiGenerateService; + @Autowired GenerationRecordService recordService; + @Autowired GenerationTraceService traceService; + @Autowired AgentWorkflowService agentWorkflowService; + + @Test + void disabledFallbackPreservesRequestedFactsWithoutFabricatingActualExecution() { + assertThatThrownBy(() -> aiGenerateService.generate("requirement-split", request())) + .isInstanceOf(LlmProviderException.class); + + GenerationRecord record = recordService.recent(1).get(0); + assertThat(record.getStatus().name()).isEqualTo("FAILED"); + assertThat(record.getRequestedProvider()).isEqualTo("openai-compatible"); + assertThat(record.getRequestedModel()).isNotBlank(); + assertThat(record.getProviderName()).isNull(); + assertThat(record.getModelName()).isNull(); + assertThat(record.getFallbackUsed()).isFalse(); + assertThat(record.getFallbackReason()).isNull(); + assertThat(traceService.list(record.getId()).get(0).getProviderName()).isNull(); + assertThat(agentWorkflowService.list(null, record.getId()).get(0).getProviderName()).isNull(); + } + + private AiGenerateRequest request() { + AiGenerateRequest request = new AiGenerateRequest(); + request.setProjectId(1L); + request.setInput("offline failed execution persistence test"); + return request; + } +} diff --git a/backend/src/test/java/com/devflow/copilot/ProviderExecutionFallbackPersistenceIntegrationTest.java b/backend/src/test/java/com/devflow/copilot/ProviderExecutionFallbackPersistenceIntegrationTest.java new file mode 100644 index 0000000..60051f3 --- /dev/null +++ b/backend/src/test/java/com/devflow/copilot/ProviderExecutionFallbackPersistenceIntegrationTest.java @@ -0,0 +1,55 @@ +package com.devflow.copilot; + +import com.devflow.copilot.common.ProviderErrorType; +import com.devflow.copilot.dto.AiGenerateRequest; +import com.devflow.copilot.dto.AiGenerateResponse; +import com.devflow.copilot.entity.GenerationRecord; +import com.devflow.copilot.entity.GenerationTrace; +import com.devflow.copilot.service.AgentWorkflowService; +import com.devflow.copilot.service.AiGenerateService; +import com.devflow.copilot.service.GenerationRecordService; +import com.devflow.copilot.service.GenerationTraceService; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.transaction.annotation.Transactional; + +import static org.assertj.core.api.Assertions.assertThat; + +@SpringBootTest(properties = {"devflow.ai.provider=openai-compatible", "devflow.ai.api-key=", "devflow.ai.protocol=chat-completions-compatible", "devflow.ai.fallback-to-local=true"}) +@ActiveProfiles("test") +@Transactional +class ProviderExecutionFallbackPersistenceIntegrationTest { + + @Autowired AiGenerateService aiGenerateService; + @Autowired GenerationRecordService recordService; + @Autowired GenerationTraceService traceService; + @Autowired AgentWorkflowService agentWorkflowService; + + @Test + void classifiedExternalFailureFallsBackOnceAndPersistsRequestedAndActualFacts() { + AiGenerateResponse response = aiGenerateService.generate("requirement-split", request()); + GenerationRecord record = recordService.getById(response.getRecordId()); + GenerationTrace trace = traceService.list(record.getId()).get(0); + + assertThat(record.getRequestedProvider()).isEqualTo("openai-compatible"); + assertThat(record.getRequestedModel()).isNotBlank(); + assertThat(record.getProviderName()).isEqualTo("local-rule"); + assertThat(record.getModelName()).isEqualTo("local-rule-mvp"); + assertThat(record.getFallbackUsed()).isTrue(); + assertThat(record.getFallbackReason()).isEqualTo(ProviderErrorType.API_KEY_MISSING.name()); + assertThat(record.getErrorMessage()).isNull(); + assertThat(trace.getProviderName()).isEqualTo(record.getProviderName()); + assertThat(trace.getModelName()).isEqualTo(record.getModelName()); + assertThat(agentWorkflowService.getTrace(response.getAgentRunId()).getRun().getProviderName()) + .isEqualTo(record.getProviderName()); + } + + private AiGenerateRequest request() { + AiGenerateRequest request = new AiGenerateRequest(); + request.setProjectId(1L); + request.setInput("offline fallback persistence test"); + return request; + } +} diff --git a/backend/src/test/java/com/devflow/copilot/ProviderExecutionMetadataMigrationTest.java b/backend/src/test/java/com/devflow/copilot/ProviderExecutionMetadataMigrationTest.java new file mode 100644 index 0000000..0882626 --- /dev/null +++ b/backend/src/test/java/com/devflow/copilot/ProviderExecutionMetadataMigrationTest.java @@ -0,0 +1,49 @@ +package com.devflow.copilot; + +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.context.ActiveProfiles; + +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; + +@SpringBootTest(properties = {"devflow.ai.provider=local-rule", "devflow.ai.api-key=", "devflow.ai.protocol=chat-completions-compatible", "devflow.ai.fallback-to-local=true"}) +@ActiveProfiles("test") +class ProviderExecutionMetadataMigrationTest { + + @Autowired JdbcTemplate jdbcTemplate; + + @Test + void latestFlywayMigrationAddsExecutionMetadataAndConservativelyBackfillsLegacyRows() { + List columns = jdbcTemplate.queryForList( + "SELECT column_name FROM information_schema.columns WHERE table_name = 'generation_record'", + String.class + ); + assertThat(columns).contains( + "requested_provider", "requested_model", "fallback_used", "fallback_reason", + "provider_error_type", "provider_failure_stage", "provider_http_status", + "provider_http_status_family", "provider_duration_bucket", "provider_response_body_present", + "provider_response_size_bucket", "provider_content_type_category", "provider_retry_after_present", + "provider_request_id_present", "provider_client_request_id" + ); + + Map row = jdbcTemplate.queryForMap( + "SELECT requested_provider, requested_model, fallback_used, fallback_reason FROM generation_record LIMIT 1" + ); + assertThat(row.get("requested_provider")).isNotNull(); + assertThat(row.get("requested_model")).isNotNull(); + assertThat(row.get("fallback_used")).isEqualTo(false); + assertThat(row.get("fallback_reason")).isNull(); + + Map observability = jdbcTemplate.queryForMap( + "SELECT provider_error_type, provider_http_status, provider_client_request_id FROM generation_record LIMIT 1" + ); + assertThat(observability.get("provider_error_type")).isNull(); + assertThat(observability.get("provider_http_status")).isNull(); + assertThat(observability.get("provider_client_request_id")).isNull(); + } +} diff --git a/backend/src/test/java/com/devflow/copilot/ProviderExecutionUnknownProviderPersistenceIntegrationTest.java b/backend/src/test/java/com/devflow/copilot/ProviderExecutionUnknownProviderPersistenceIntegrationTest.java new file mode 100644 index 0000000..0d11c9b --- /dev/null +++ b/backend/src/test/java/com/devflow/copilot/ProviderExecutionUnknownProviderPersistenceIntegrationTest.java @@ -0,0 +1,42 @@ +package com.devflow.copilot; + +import com.devflow.copilot.common.LlmProviderException; +import com.devflow.copilot.dto.AiGenerateRequest; +import com.devflow.copilot.entity.GenerationRecord; +import com.devflow.copilot.service.AiGenerateService; +import com.devflow.copilot.service.GenerationRecordService; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.transaction.annotation.Transactional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +@SpringBootTest(properties = {"devflow.ai.provider=unsupported-test-provider", "devflow.ai.api-key=", "devflow.ai.protocol=chat-completions-compatible", "devflow.ai.fallback-to-local=true"}) +@ActiveProfiles("test") +@Transactional +class ProviderExecutionUnknownProviderPersistenceIntegrationTest { + + @Autowired AiGenerateService aiGenerateService; + @Autowired GenerationRecordService recordService; + + @Test + void unknownProviderDoesNotCreateFallbackOrActualExecutionFacts() { + AiGenerateRequest request = new AiGenerateRequest(); + request.setProjectId(1L); + request.setInput("offline unknown provider persistence test"); + + assertThatThrownBy(() -> aiGenerateService.generate("requirement-split", request)) + .isInstanceOf(LlmProviderException.class); + + GenerationRecord record = recordService.recent(1).get(0); + assertThat(record.getStatus().name()).isEqualTo("FAILED"); + assertThat(record.getRequestedProvider()).isEqualTo("unsupported-test-provider"); + assertThat(record.getProviderName()).isNull(); + assertThat(record.getModelName()).isNull(); + assertThat(record.getFallbackUsed()).isFalse(); + assertThat(record.getFallbackReason()).isNull(); + } +} diff --git a/backend/src/test/java/com/devflow/copilot/config/SharedProviderConfigurationMappingTest.java b/backend/src/test/java/com/devflow/copilot/config/SharedProviderConfigurationMappingTest.java new file mode 100644 index 0000000..6da017d --- /dev/null +++ b/backend/src/test/java/com/devflow/copilot/config/SharedProviderConfigurationMappingTest.java @@ -0,0 +1,87 @@ +package com.devflow.copilot.config; + +import org.junit.jupiter.api.Test; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.MapPropertySource; +import org.springframework.core.env.StandardEnvironment; +import org.springframework.core.io.ClassPathResource; +import org.springframework.beans.factory.config.YamlPropertiesFactoryBean; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; + +import static org.assertj.core.api.Assertions.assertThat; + +class SharedProviderConfigurationMappingTest { + + @Test + void projectSpecificSettingsTakePriorityOverSharedSettings() { + runWith(Map.of( + "DEVFLOW_AI_PROVIDER", "project-specific", + "PORTFOLIO_AI_PROVIDER", "shared" + )).run(context -> assertThat(context.getBean(AiProviderProperties.class).getProvider()) + .isEqualTo("project-specific")); + } + + @Test + void sharedSettingsAreUsedWhenProjectSpecificSettingsAreAbsent() { + runWith(Map.of( + "PORTFOLIO_AI_PROTOCOL", "chat-completions-compatible" + )).run(context -> assertThat(context.getBean(AiProviderProperties.class).getProtocol()) + .isEqualTo("chat-completions-compatible")); + } + + @Test + void projectSpecificProtocolOverridesTheSharedCapabilityMarker() { + runWith(Map.of( + "DEVFLOW_AI_PROTOCOL", "project-protocol", + "PORTFOLIO_AI_PROTOCOL", "shared-protocol" + )).run(context -> assertThat(context.getBean(AiProviderProperties.class).getProtocol()) + .isEqualTo("project-protocol")); + } + + @Test + void safeDefaultsApplyWhenNeitherProjectNorSharedSettingsExist() { + runWith(Map.of()).run(context -> { + AiProviderProperties properties = context.getBean(AiProviderProperties.class); + assertThat(properties.getProvider()).isEqualTo("local-rule"); + assertThat(properties.isFallbackToLocal()).isTrue(); + }); + } + + @Test + void projectSpecificFallbackSettingTakesPriorityOverSharedSetting() { + runWith(Map.of( + "DEVFLOW_AI_FALLBACK_TO_LOCAL", "false", + "PORTFOLIO_AI_FALLBACK_ENABLED", "true" + )).run(context -> assertThat(context.getBean(AiProviderProperties.class).isFallbackToLocal()).isFalse()); + } + + private ApplicationContextRunner runWith(Map values) { + return new ApplicationContextRunner() + .withUserConfiguration(PropertiesConfiguration.class) + .withInitializer(context -> { + context.getEnvironment().getPropertySources().remove(StandardEnvironment.SYSTEM_ENVIRONMENT_PROPERTY_SOURCE_NAME); + context.getEnvironment().getPropertySources().remove(StandardEnvironment.SYSTEM_PROPERTIES_PROPERTY_SOURCE_NAME); + context.getEnvironment().getPropertySources().addFirst(new MapPropertySource("test-variables", values)); + context.getEnvironment().getPropertySources().addLast(new MapPropertySource("application-yaml", applicationYaml())); + }); + } + + private Map applicationYaml() { + YamlPropertiesFactoryBean yaml = new YamlPropertiesFactoryBean(); + yaml.setResources(new ClassPathResource("application.yml")); + Properties properties = yaml.getObject(); + Map result = new LinkedHashMap<>(); + properties.forEach((key, value) -> result.put(String.valueOf(key), value)); + return result; + } + + @Configuration(proxyBeanMethods = false) + @EnableConfigurationProperties(AiProviderProperties.class) + static class PropertiesConfiguration { + } +} diff --git a/backend/src/test/java/com/devflow/copilot/service/provider/OpenAiCompatibleGenerationProviderOfflineHttpTest.java b/backend/src/test/java/com/devflow/copilot/service/provider/OpenAiCompatibleGenerationProviderOfflineHttpTest.java new file mode 100644 index 0000000..27275df --- /dev/null +++ b/backend/src/test/java/com/devflow/copilot/service/provider/OpenAiCompatibleGenerationProviderOfflineHttpTest.java @@ -0,0 +1,240 @@ +package com.devflow.copilot.service.provider; + +import com.devflow.copilot.common.LlmProviderException; +import com.devflow.copilot.common.ProviderContentTypeCategory; +import com.devflow.copilot.common.ProviderDurationBucket; +import com.devflow.copilot.common.ProviderErrorType; +import com.devflow.copilot.common.ProviderFailureMetadata; +import com.devflow.copilot.common.ProviderFailureStage; +import com.devflow.copilot.common.ProviderHttpStatusFamily; +import com.devflow.copilot.common.ProviderResponseSizeBucket; +import com.devflow.copilot.config.AiProviderProperties; +import com.sun.net.httpserver.HttpExchange; +import com.sun.net.httpserver.HttpServer; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.nio.charset.StandardCharsets; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +import static org.assertj.core.api.Assertions.assertThat; + +class OpenAiCompatibleGenerationProviderOfflineHttpTest { + + private HttpServer server; + private final AtomicInteger requestCount = new AtomicInteger(); + private final Set clientRequestIds = new HashSet<>(); + private volatile int responseStatus; + private volatile String responseBody; + private volatile String contentType; + private volatile Map responseHeaders = Map.of(); + private volatile long responseDelayMs; + private volatile String requestPath; + + @BeforeEach + void startServer() throws IOException { + server = HttpServer.create(new InetSocketAddress(InetAddress.getLoopbackAddress(), 0), 0); + server.createContext("/v1/chat/completions", this::respond); + server.start(); + } + + @AfterEach + void stopServer() { + server.stop(0); + } + + @Test + void parsesOfflineChatCompletionsSuccessWithoutFallback() { + responseStatus = 200; + responseBody = """ + { + "choices": [ + { + "message": { + "content": "offline success" + } + } + ], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 4, + "total_tokens": 14 + } + } + """; + contentType = "application/json"; + responseHeaders = Map.of(); + responseDelayMs = 0; + + ProviderResult result = new OpenAiCompatibleGenerationProvider(properties()).generate(request()); + + assertThat(requestCount.get()).isEqualTo(1); + assertThat(requestPath).isEqualTo("/v1/chat/completions"); + assertThat(clientRequestIds).hasSize(1).allSatisfy(id -> assertThatCodeParsesAsUuid(id)); + assertThat(result.content()).isEqualTo("offline success"); + assertThat(result.providerName()).isNotBlank(); + assertThat(result.modelName()).isNotBlank(); + assertThat(result.promptTokens()).isEqualTo(10); + assertThat(result.completionTokens()).isEqualTo(4); + assertThat(result.totalTokens()).isEqualTo(14); + assertThat(result.fallbackUsed()).isFalse(); + assertThat(result.fallbackReason()).isNull(); + assertThat(result.toString()).doesNotContain("offline-test-key", "Authorization", "Bearer"); + } + + @Test + void retainsExact5xxStatusCodesAndSeparatesRateLimiting() { + assertHttpFailure(500, "{}", "application/json", Map.of(), ProviderErrorType.UPSTREAM_SERVER_ERROR, 500); + assertHttpFailure(502, "{}", "application/json", Map.of(), ProviderErrorType.UPSTREAM_SERVER_ERROR, 502); + assertHttpFailure(503, "{}", "application/json", Map.of(), ProviderErrorType.UPSTREAM_SERVER_ERROR, 503); + assertHttpFailure(504, "{}", "application/json", Map.of(), ProviderErrorType.UPSTREAM_SERVER_ERROR, 504); + LlmProviderException rateLimited = assertHttpFailure(429, "{}", "application/json", Map.of(), + ProviderErrorType.RATE_LIMITED, 429); + + assertThat(rateLimited.getFailureMetadata().httpStatusFamily()).isEqualTo(ProviderHttpStatusFamily.FOUR_XX); + assertThat(clientRequestIds).hasSize(5).allSatisfy(id -> assertThatCodeParsesAsUuid(id)); + } + + @Test + void capturesOnlySafeHtmlBodyAndHeaderFacts() { + LlmProviderException failure = assertHttpFailure(502, "offline error", "text/html", + Map.of("Retry-After", "ignored", "X-Request-Id", "ignored"), + ProviderErrorType.UPSTREAM_SERVER_ERROR, 502); + + ProviderFailureMetadata metadata = failure.getFailureMetadata(); + assertThat(metadata.contentTypeCategory()).isEqualTo(ProviderContentTypeCategory.HTML); + assertThat(metadata.responseBodyPresent()).isTrue(); + assertThat(metadata.responseBodySizeBucket()).isEqualTo(ProviderResponseSizeBucket.ONE_TO_1KB); + assertThat(metadata.retryAfterHeaderPresent()).isTrue(); + assertThat(metadata.upstreamRequestIdHeaderPresent()).isTrue(); + assertThat(metadata.toString()).doesNotContain("offline error", "ignored"); + } + + @Test + void capturesJsonAndAbsentBodiesWithoutKeepingTheirContents() { + LlmProviderException jsonFailure = assertHttpFailure(503, "{\"error\":\"offline\"}", "application/json", + Map.of(), ProviderErrorType.UPSTREAM_SERVER_ERROR, 503); + assertThat(jsonFailure.getFailureMetadata().contentTypeCategory()).isEqualTo(ProviderContentTypeCategory.JSON); + assertThat(jsonFailure.getFailureMetadata().toString()).doesNotContain("offline"); + + LlmProviderException emptyFailure = assertHttpFailure(504, null, null, Map.of(), + ProviderErrorType.UPSTREAM_SERVER_ERROR, 504); + assertThat(emptyFailure.getFailureMetadata().responseBodyPresent()).isFalse(); + assertThat(emptyFailure.getFailureMetadata().responseBodySizeBucket()).isEqualTo(ProviderResponseSizeBucket.ZERO); + } + + @Test + void assignsFailureStagesForDeserializationContentTimeoutAndConnectionFailures() throws IOException { + LlmProviderException invalidJson = failureFor(200, "not-json", "application/json", Map.of(), 0); + assertThat(invalidJson.getErrorType()).isEqualTo(ProviderErrorType.INVALID_RESPONSE); + assertThat(invalidJson.getFailureMetadata().failureStage()) + .isEqualTo(ProviderFailureStage.RESPONSE_DESERIALIZATION); + + LlmProviderException emptyContent = failureFor(200, "{\"choices\":[{\"message\":{\"content\":\"\"}}]}", + "application/json", Map.of(), 0); + assertThat(emptyContent.getErrorType()).isEqualTo(ProviderErrorType.EMPTY_CONTENT); + assertThat(emptyContent.getFailureMetadata().failureStage()).isEqualTo(ProviderFailureStage.CONTENT_EXTRACTION); + + LlmProviderException timeout = failureFor(200, "{\"choices\":[{\"message\":{\"content\":\"ok\"}}]}", + "application/json", Map.of(), 1_200); + assertThat(timeout.getErrorType()).isEqualTo(ProviderErrorType.TIMEOUT); + assertThat(timeout.getFailureMetadata().failureStage()).isEqualTo(ProviderFailureStage.CONNECTION); + assertThat(timeout.getFailureMetadata().durationBucket()).isEqualTo(ProviderDurationBucket.TIMEOUT); + + int unusedPort; + try (ServerSocket socket = new ServerSocket(0, 1, InetAddress.getLoopbackAddress())) { + unusedPort = socket.getLocalPort(); + } + AiProviderProperties properties = properties(); + properties.setBaseUrl("http" + "://127.0.0.1:" + unusedPort + "/v1"); + LlmProviderException connection = capture(new OpenAiCompatibleGenerationProvider(properties)); + assertThat(connection.getErrorType()).isEqualTo(ProviderErrorType.CONNECTION_FAILED); + assertThat(connection.getFailureMetadata().httpStatusCode()).isNull(); + assertThat(connection.getFailureMetadata().failureStage()).isEqualTo(ProviderFailureStage.CONNECTION); + } + + private LlmProviderException assertHttpFailure(int status, String body, String type, Map headers, + ProviderErrorType expectedType, int expectedStatus) { + LlmProviderException failure = failureFor(status, body, type, headers, 0); + assertThat(failure.getErrorType()).isEqualTo(expectedType); + assertThat(failure.getFailureMetadata().failureStage()).isEqualTo(ProviderFailureStage.HTTP_STATUS_RECEIVED); + assertThat(failure.getFailureMetadata().httpStatusCode()).isEqualTo(expectedStatus); + assertThat(failure.getFailureMetadata().httpStatusFamily()).isEqualTo( + expectedStatus >= 500 ? ProviderHttpStatusFamily.FIVE_XX : ProviderHttpStatusFamily.FOUR_XX); + assertThat(requestCount.getAndSet(0)).isEqualTo(1); + return failure; + } + + private LlmProviderException failureFor(int status, String body, String type, Map headers, long delayMs) { + responseStatus = status; + responseBody = body; + contentType = type; + responseHeaders = headers; + responseDelayMs = delayMs; + return capture(new OpenAiCompatibleGenerationProvider(properties())); + } + + private LlmProviderException capture(OpenAiCompatibleGenerationProvider provider) { + try { + provider.generate(request()); + } catch (LlmProviderException exception) { + return exception; + } + throw new AssertionError("Expected provider failure"); + } + + private AiProviderProperties properties() { + AiProviderProperties properties = new AiProviderProperties(); + properties.setApiKey("offline-test-key"); + properties.setProtocol("chat-completions-compatible"); + properties.setTimeoutSeconds(1); + properties.setBaseUrl("http" + "://127.0.0.1:" + server.getAddress().getPort() + "/v1"); + return properties; + } + + private ProviderRequest request() { + return new ProviderRequest("requirement-split", "offline test", "offline test", "project", "Java"); + } + + private void respond(HttpExchange exchange) throws IOException { + requestCount.incrementAndGet(); + requestPath = exchange.getRequestURI().getPath(); + synchronized (clientRequestIds) { + clientRequestIds.add(exchange.getRequestHeaders().getFirst("X-Client-Request-Id")); + } + try { + if (responseDelayMs > 0) { + Thread.sleep(responseDelayMs); + } + if (contentType != null) { + exchange.getResponseHeaders().add("Content-Type", contentType); + } + responseHeaders.forEach((name, value) -> exchange.getResponseHeaders().add(name, value)); + if (responseBody == null) { + exchange.sendResponseHeaders(responseStatus, -1); + return; + } + byte[] payload = responseBody.getBytes(StandardCharsets.UTF_8); + exchange.sendResponseHeaders(responseStatus, payload.length); + exchange.getResponseBody().write(payload); + } catch (InterruptedException exception) { + Thread.currentThread().interrupt(); + } finally { + exchange.close(); + } + } + + private void assertThatCodeParsesAsUuid(String value) { + assertThat(value).isNotBlank(); + assertThat(UUID.fromString(value).toString()).isEqualTo(value); + } +} diff --git a/backend/src/test/java/com/devflow/copilot/service/provider/ProviderErrorClassifierTest.java b/backend/src/test/java/com/devflow/copilot/service/provider/ProviderErrorClassifierTest.java new file mode 100644 index 0000000..c7afe41 --- /dev/null +++ b/backend/src/test/java/com/devflow/copilot/service/provider/ProviderErrorClassifierTest.java @@ -0,0 +1,57 @@ +package com.devflow.copilot.service.provider; + +import com.devflow.copilot.common.LlmProviderException; +import com.devflow.copilot.common.ProviderFailureMetadata; +import com.devflow.copilot.common.ProviderFailureStage; +import com.devflow.copilot.common.ProviderErrorType; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.HttpServerErrorException; +import org.springframework.web.client.ResourceAccessException; + +import java.net.ConnectException; +import java.net.SocketTimeoutException; +import java.nio.charset.StandardCharsets; + +import static org.assertj.core.api.Assertions.assertThat; + +class ProviderErrorClassifierTest { + + @Test + void classifiesOfflineTransportAndResponseFailuresWithoutUsingRawMessages() { + assertThat(ProviderErrorClassifier.classify(new ResourceAccessException("offline", new SocketTimeoutException()))) + .isEqualTo(ProviderErrorType.TIMEOUT); + assertThat(ProviderErrorClassifier.classify(new ResourceAccessException("offline", new ConnectException()))) + .isEqualTo(ProviderErrorType.CONNECTION_FAILED); + assertThat(ProviderErrorClassifier.classify(HttpClientErrorException.create( + HttpStatus.UNAUTHORIZED, "", HttpHeaders.EMPTY, new byte[0], StandardCharsets.UTF_8))) + .isEqualTo(ProviderErrorType.AUTHENTICATION_FAILED); + assertThat(ProviderErrorClassifier.classify(HttpClientErrorException.create( + HttpStatus.TOO_MANY_REQUESTS, "", HttpHeaders.EMPTY, new byte[0], StandardCharsets.UTF_8))) + .isEqualTo(ProviderErrorType.RATE_LIMITED); + assertThat(ProviderErrorClassifier.classify(HttpServerErrorException.create( + HttpStatus.INTERNAL_SERVER_ERROR, "", HttpHeaders.EMPTY, new byte[0], StandardCharsets.UTF_8))) + .isEqualTo(ProviderErrorType.UPSTREAM_SERVER_ERROR); + assertThat(ProviderErrorClassifier.classify(new JsonProcessingException("offline") { })) + .isEqualTo(ProviderErrorType.INVALID_RESPONSE); + } + + @Test + void providerExceptionKeepsCauseOutOfSafeMessagesToStringAndJson() throws Exception { + String rawCause = "offline raw upstream content"; + LlmProviderException exception = new LlmProviderException( + ProviderErrorType.UPSTREAM_SERVER_ERROR, + ProviderFailureMetadata.atStage(ProviderFailureStage.HTTP_STATUS_RECEIVED), + new RuntimeException(rawCause) + ); + + String serialized = new ObjectMapper().writeValueAsString(exception); + assertThat(exception.getMessage()).isEqualTo(ProviderErrorType.UPSTREAM_SERVER_ERROR.safeMessage()); + assertThat(exception.toString()).doesNotContain(rawCause); + assertThat(serialized).doesNotContain(rawCause, "cause", "stackTrace"); + } +} diff --git a/backend/src/test/java/com/devflow/copilot/service/provider/ProviderRoutingSafetyTest.java b/backend/src/test/java/com/devflow/copilot/service/provider/ProviderRoutingSafetyTest.java new file mode 100644 index 0000000..b0a34b6 --- /dev/null +++ b/backend/src/test/java/com/devflow/copilot/service/provider/ProviderRoutingSafetyTest.java @@ -0,0 +1,83 @@ +package com.devflow.copilot.service.provider; + +import com.devflow.copilot.common.LlmProviderException; +import com.devflow.copilot.common.ProviderErrorType; +import com.devflow.copilot.config.AiProviderProperties; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; + +class ProviderRoutingSafetyTest { + + @Test + void supportedProtocolReachesCredentialGateWithoutMakingANetworkRequest() { + AiProviderProperties properties = externalProperties(); + OpenAiCompatibleGenerationProvider provider = new OpenAiCompatibleGenerationProvider(properties); + + assertThatThrownBy(() -> provider.generate(request())) + .isInstanceOf(LlmProviderException.class) + .extracting(error -> ((LlmProviderException) error).getErrorType()) + .isEqualTo(ProviderErrorType.API_KEY_MISSING); + } + + @Test + void unsupportedProtocolFallsBackOnlyWhenEnabled() { + AiProviderProperties enabled = externalProperties(); + enabled.setProtocol("unsupported"); + enabled.setFallbackToLocal(true); + LocalRuleGenerationProvider local = spy(new LocalRuleGenerationProvider()); + GenerationProviderRouter enabledRouter = new GenerationProviderRouter( + enabled, local, new OpenAiCompatibleGenerationProvider(enabled)); + + ProviderResult fallback = enabledRouter.generate(request()); + + assertThat(fallback.providerName()).isEqualTo(local.key()); + assertThat(fallback.fallbackReason()).contains(ProviderErrorType.PROTOCOL_UNSUPPORTED.name()); + verify(local).generate(any()); + + AiProviderProperties disabled = externalProperties(); + disabled.setProtocol("unsupported"); + disabled.setFallbackToLocal(false); + LocalRuleGenerationProvider noFallbackLocal = spy(new LocalRuleGenerationProvider()); + GenerationProviderRouter disabledRouter = new GenerationProviderRouter( + disabled, noFallbackLocal, new OpenAiCompatibleGenerationProvider(disabled)); + + assertThatThrownBy(() -> disabledRouter.generate(request())) + .isInstanceOf(LlmProviderException.class) + .extracting(error -> ((LlmProviderException) error).getErrorType()) + .isEqualTo(ProviderErrorType.PROTOCOL_UNSUPPORTED); + verify(noFallbackLocal, never()).generate(any()); + } + + @Test + void unknownProviderFailsWithoutFallbackOrNetworkRequest() { + AiProviderProperties properties = externalProperties(); + properties.setProvider("unknown"); + LocalRuleGenerationProvider local = spy(new LocalRuleGenerationProvider()); + GenerationProviderRouter router = new GenerationProviderRouter( + properties, local, new OpenAiCompatibleGenerationProvider(properties)); + + assertThatThrownBy(() -> router.generate(request())) + .isInstanceOf(LlmProviderException.class) + .extracting(error -> ((LlmProviderException) error).getErrorType()) + .isEqualTo(ProviderErrorType.UNKNOWN_PROVIDER_ERROR); + verify(local, never()).generate(any()); + } + + private AiProviderProperties externalProperties() { + AiProviderProperties properties = new AiProviderProperties(); + properties.setProvider("openai-compatible"); + properties.setProtocol("chat-completions-compatible"); + properties.setApiKey(""); + return properties; + } + + private ProviderRequest request() { + return new ProviderRequest("requirement-split", "offline test", "offline test", "project", "Java"); + } +} diff --git a/backend/src/test/resources/application-test.yml b/backend/src/test/resources/application-test.yml index 7c4962f..e405b3f 100644 --- a/backend/src/test/resources/application-test.yml +++ b/backend/src/test/resources/application-test.yml @@ -11,5 +11,8 @@ spring: devflow: ai: provider: local-rule + base-url: http://127.0.0.1:0 api-key: + model: local-rule-mvp + protocol: chat-completions-compatible fallback-to-local: true diff --git a/docs/evidence/shared-provider/connectivity-preflight.json b/docs/evidence/shared-provider/connectivity-preflight.json new file mode 100644 index 0000000..0319383 --- /dev/null +++ b/docs/evidence/shared-provider/connectivity-preflight.json @@ -0,0 +1,9 @@ +{ + "uriValid": true, + "dnsResolved": true, + "tcpConnected": true, + "tlsEstablished": true, + "httpRequestSent": false, + "authorizationUsed": false, + "secretScanPassed": true +} diff --git a/docs/evidence/shared-provider/connectivity-preflight.md b/docs/evidence/shared-provider/connectivity-preflight.md new file mode 100644 index 0000000..085e724 --- /dev/null +++ b/docs/evidence/shared-provider/connectivity-preflight.md @@ -0,0 +1,12 @@ +# Connectivity Preflight + +The controlled preflight completed without sending an HTTP request or using authorization material. + +| Check | Result | Duration band | +| --- | --- | --- | +| URI parsing | true | UNDER_1_SECOND | +| DNS resolution | true | UNDER_1_SECOND | +| TCP connection | true | UNDER_1_SECOND | +| TLS handshake | true | UNDER_1_SECOND | + +No endpoint, host, IP address, port, certificate information, credential, request path, request header, or HTTP response is retained. diff --git a/docs/evidence/shared-provider/controlled-retry-metadata.json b/docs/evidence/shared-provider/controlled-retry-metadata.json new file mode 100644 index 0000000..6caae12 --- /dev/null +++ b/docs/evidence/shared-provider/controlled-retry-metadata.json @@ -0,0 +1,21 @@ +{ + "validationType": "controlled-retry", + "previousRequestCount": 1, + "currentRequestCount": 1, + "totalRealRequestCount": 2, + "connectivityPreflightPassed": true, + "fallbackDisabled": true, + "externalRequestAttempted": true, + "httpResponseReceived": true, + "responseBodyReceived": false, + "responseParsingStarted": false, + "responseContentPresent": false, + "generationRecordPersisted": true, + "generationTracePersisted": true, + "agentRunPersisted": true, + "agentStepPersisted": true, + "fallbackUsed": false, + "secretScanPassed": true, + "failureCategory": "UPSTREAM_SERVER_ERROR", + "result": "FAILED" +} diff --git a/docs/evidence/shared-provider/controlled-retry-summary.md b/docs/evidence/shared-provider/controlled-retry-summary.md new file mode 100644 index 0000000..2ab88be --- /dev/null +++ b/docs/evidence/shared-provider/controlled-retry-summary.md @@ -0,0 +1,23 @@ +# Controlled Retry Summary + +> Note: This document records the intermediate judgment available at the time. A later V6 safe observability audit corrected the current state to `BLOCKED_UPSTREAM_5XX_UNCLASSIFIED`; there is no retained 429 evidence, so the failure must not be described as rate limiting. + +## Scope + +One authorized controlled retry followed a successful no-HTTP connectivity preflight. The request used only a short, fully fictional, review-only teaching task. No real user, company, resume, ticket, or production data was used. + +## Result + +`BLOCKED_PROVIDER_RATE_LIMIT_OR_OUTAGE` + +The controlled retry reached an HTTP status response but did not produce a usable response body, parsed response, or generated artifact. The persisted controlled category is `UPSTREAM_SERVER_ERROR`. + +## Safety and Persistence + +- Fallback was disabled and was not used. +- The failed Generation Record, Generation Trace, Agent Run, and Agent Steps were persisted and associated. +- Actual execution metadata was not fabricated after the failed request. +- No local-rule result was substituted. +- No raw upstream payload, endpoint, credential, authorization value, provider identifier, model identifier, or environment value is retained. + +No further real-provider retry is authorized by this run. diff --git a/docs/evidence/shared-provider/failure-diagnosis.json b/docs/evidence/shared-provider/failure-diagnosis.json new file mode 100644 index 0000000..93b099b --- /dev/null +++ b/docs/evidence/shared-provider/failure-diagnosis.json @@ -0,0 +1,22 @@ +{ + "validationResult": "FAILED", + "failureStage": "UNKNOWN_STAGE", + "failureCategory": "TIMEOUT", + "externalRequestAttempted": true, + "httpResponseReceived": false, + "responseBodyReceived": false, + "responseParsingStarted": false, + "persistenceCompleted": true, + "configurationBindingValid": true, + "protocolBindingValid": true, + "providerRoutingValid": true, + "requestPathValid": true, + "codeDefectConfirmed": false, + "configurationIssueLikely": false, + "providerSideIssueLikely": false, + "networkIssueLikely": false, + "offlineReproductionCompleted": true, + "offlineFixApplied": false, + "secondRealCallPerformed": false, + "recommendedNextState": "INSUFFICIENT_SANITIZED_EVIDENCE" +} diff --git a/docs/evidence/shared-provider/failure-diagnosis.md b/docs/evidence/shared-provider/failure-diagnosis.md new file mode 100644 index 0000000..7da8b6a --- /dev/null +++ b/docs/evidence/shared-provider/failure-diagnosis.md @@ -0,0 +1,35 @@ +# Failed Real Validation Forensics + +## Controlled Result + +`SYNTHETIC_REAL_PROVIDER_VALIDATION_FAILED` + +Exactly one synthetic real-provider request was attempted in the prior validation. This diagnostic run made no external-provider request and performed no retry. + +## Persisted Evidence + +- Generation Record ID: 53; status: `FAILED`. +- Generation Trace ID: 45; associated with the failed record; status: `FAILED`. +- Agent Run ID: 45; associated with the failed record; status: `FAILED`. +- Four Agent Steps were persisted: three completed workflow steps and one failed generation step. +- Requested provider/model metadata is present. Actual provider/model metadata is absent. Fallback is false and no fallback reason is present. + +## Confirmed Category and Stage + +- Controlled failure category: `TIMEOUT`. +- Persisted safe summary: `Provider request timed out.` +- Failure stage: `UNKNOWN_STAGE`. + +The safe category proves that external dispatch reached the provider transport path and timed out. It does not safely distinguish connection establishment, TLS negotiation, request transmission, or waiting for a response. No HTTP response, response body, or response parsing evidence is available. + +## Offline Reproduction and Repair Decision + +Existing loopback tests reproduce timeout classification through the actual provider code path and also cover status failures, malformed responses, empty content, endpoint joining, and connection failure. The complete offline suite passes. + +No code defect is confirmed. Runtime configuration binding, protocol support, provider routing, URI validity, and endpoint-join checks are valid without exposing configuration values. The sanitised evidence cannot distinguish a provider-side delay from an intermediate network/TLS/transport delay, so no code or configuration repair is applied. + +## Recommended State + +`INSUFFICIENT_SANITIZED_EVIDENCE` + +No second real-provider call was made. No environment-variable value, credential, endpoint, header, prompt body, response body, or raw exception is retained in this diagnosis. diff --git a/docs/evidence/shared-provider/final-attempt-metadata.json b/docs/evidence/shared-provider/final-attempt-metadata.json new file mode 100644 index 0000000..147f003 --- /dev/null +++ b/docs/evidence/shared-provider/final-attempt-metadata.json @@ -0,0 +1,31 @@ +{ + "validationType": "final-single-chat-path-attempt", + "currentRequestCount": 1, + "totalHistoricalRealRequestCount": 3, + "fallbackDisabled": true, + "externalRequestAttempted": true, + "httpResponseReceived": true, + "httpStatusCode": 200, + "httpStatusFamily": "2XX", + "responseBodyPresent": true, + "responseContentPresent": true, + "responseParsingCompleted": true, + "generationRecordPersisted": true, + "generationTracePersisted": true, + "agentRunPersisted": true, + "requestedProviderPresent": true, + "actualProviderPresent": true, + "requestedAndActualProviderMatch": true, + "requestedModelPresent": true, + "actualModelPresent": true, + "requestedAndActualModelMatch": true, + "fallbackUsed": false, + "fallbackReasonNull": true, + "clientRequestIdPresent": false, + "getModelsExecuted": false, + "previousRelevanceCheckFalseNegative": true, + "secretScanPassed": true, + "result": "PASSED", + "finalConclusion": "SYNTHETIC_REAL_PROVIDER_VALIDATION_PASSED", + "recommendedAction": "READY_TO_COMMIT_AND_FREEZE" +} diff --git a/docs/evidence/shared-provider/final-attempt-sanitized-output.md b/docs/evidence/shared-provider/final-attempt-sanitized-output.md new file mode 100644 index 0000000..089b895 --- /dev/null +++ b/docs/evidence/shared-provider/final-attempt-sanitized-output.md @@ -0,0 +1,7 @@ +# Final Attempt Sanitized Output + +The following content comes from the last real Provider call already persisted by DevFlow. It contains no Provider value, Model value, endpoint, credential, request header, or HTTP metadata. + +事务边界需人工复核 +库存并发需加锁评审 +幂等与测试需补审查 diff --git a/docs/evidence/shared-provider/final-attempt-summary.md b/docs/evidence/shared-provider/final-attempt-summary.md new file mode 100644 index 0000000..b1ca22d --- /dev/null +++ b/docs/evidence/shared-provider/final-attempt-summary.md @@ -0,0 +1,35 @@ +# Final Attempt Summary + +## Scope + +One final minimal Chat Completions validation was executed through the DevFlow Generation API, Service, Router, provider, Generation Record, Generation Trace, Agent Run, and Agent Step chain. The request used only a fully fictional Spring Boot teaching-module review task. + +## Runtime Controls + +- fallbackDisabled=true +- timeoutConfigured=true +- maxTokensLimited=true +- This run did not change permanent environment variables or application configuration files. + +## Request Controls + +- Current real external request count: 1 +- Historical real external request count: 3 +- GET /models executed: false +- Automatic retry executed: false +- fallbackUsed=false +- HTTP 200 was retained from the final real attempt. + +## Sanitized Result + +The DevFlow call reached the provider success path, received an HTTP success response, parsed JSON, and persisted non-empty generated content. Generation Record, Generation Trace, Agent Run, and Agent Step records were all persisted and associated. + +An offline semantic review of the already persisted output confirmed that the generated content covers the fictional task's three required themes: transaction boundary, inventory concurrency, and idempotency or testing. The earlier sanitized relevance checker result is now recorded as `SANITIZED_RELEVANCE_CHECK_FALSE_NEGATIVE`. No provider, model, endpoint, host, key, credential header value, request body, raw HTTP response body, raw exception body, or client request ID value is recorded here. + +## Conclusion + +`SYNTHETIC_REAL_PROVIDER_VALIDATION_PASSED` + +## Recommended Action + +`READY_TO_COMMIT_AND_FREEZE` diff --git a/docs/evidence/shared-provider/final-semantic-validation.json b/docs/evidence/shared-provider/final-semantic-validation.json new file mode 100644 index 0000000..0f05394 --- /dev/null +++ b/docs/evidence/shared-provider/final-semantic-validation.json @@ -0,0 +1,32 @@ +{ + "validationType": "offline-semantic-validation-of-existing-real-output", + "newExternalRequestPerformed": false, + "totalHistoricalRealRequestCount": 3, + "getModelsExecuted": false, + "latestRecordLocated": true, + "recordSuccess": true, + "httpStatusPreviouslyConfirmed": 200, + "responseParsingPreviouslyConfirmed": true, + "fallbackUsed": false, + "outputContentPresent": true, + "renderedPromptPresent": true, + "renderedPromptContainsSyntheticTask": true, + "renderedPromptContainsThreeItemConstraint": true, + "renderedPromptContainsTransactionRequirement": true, + "renderedPromptContainsConcurrencyRequirement": true, + "renderedPromptContainsIdempotencyTestingRequirement": true, + "renderedPromptContainsReviewOnlyConstraint": true, + "outputContainsTransactionAdvice": true, + "outputContainsInventoryConcurrencyAdvice": true, + "outputContainsIdempotencyOrTestingAdvice": true, + "outputIsReviewOnly": true, + "outputContainsNoFullImplementation": true, + "outputContainsNoRealData": true, + "outputItemCountIsThree": true, + "outputIsRelatedToSyntheticTask": true, + "previousRelevanceCheckFalseNegative": true, + "relevanceCheckerDetermination": "SANITIZED_RELEVANCE_CHECK_FALSE_NEGATIVE", + "secretScanPassed": true, + "finalConclusion": "SYNTHETIC_REAL_PROVIDER_VALIDATION_PASSED", + "recommendedAction": "READY_TO_COMMIT_AND_FREEZE" +} diff --git a/docs/evidence/shared-provider/final-semantic-validation.md b/docs/evidence/shared-provider/final-semantic-validation.md new file mode 100644 index 0000000..d616358 --- /dev/null +++ b/docs/evidence/shared-provider/final-semantic-validation.md @@ -0,0 +1,39 @@ +# Final Semantic Validation + +## Scope + +This is an offline semantic validation of the already persisted final real-provider output. No new external request, provider call, model call, retry, or GET models probe was performed. + +## Retained Real-Call Facts + +- HTTP 200 was retained from the final real attempt. +- Historical real external request count: 3 +- GET /models executed: false +- fallbackUsed=false + +## Prompt Checks + +- renderedPromptContainsSyntheticTask=true +- renderedPromptContainsThreeItemConstraint=true +- renderedPromptContainsTransactionRequirement=true +- renderedPromptContainsConcurrencyRequirement=true +- renderedPromptContainsIdempotencyTestingRequirement=true +- renderedPromptContainsReviewOnlyConstraint=true + +## Output Checks + +- outputContainsTransactionAdvice=true +- outputContainsInventoryConcurrencyAdvice=true +- outputContainsIdempotencyOrTestingAdvice=true +- outputIsReviewOnly=true +- outputContainsNoFullImplementation=true +- outputContainsNoRealData=true +- outputItemCountIsThree=true +- outputIsRelatedToSyntheticTask=true + +## Determination + +- previousRelevanceCheckFalseNegative=true +- relevanceCheckerDetermination=`SANITIZED_RELEVANCE_CHECK_FALSE_NEGATIVE` +- finalConclusion=`SYNTHETIC_REAL_PROVIDER_VALIDATION_PASSED` +- recommendedAction=`READY_TO_COMMIT_AND_FREEZE` diff --git a/docs/evidence/shared-provider/safe-http-attempt-analysis.md b/docs/evidence/shared-provider/safe-http-attempt-analysis.md new file mode 100644 index 0000000..77c1b4a --- /dev/null +++ b/docs/evidence/shared-provider/safe-http-attempt-analysis.md @@ -0,0 +1,14 @@ +# Safe HTTP Attempt Analysis + +> Note: This document records the intermediate judgment available at the time. A later V6 safe observability audit corrected the current state to `BLOCKED_UPSTREAM_5XX_UNCLASSIFIED`; there is no retained 429 evidence, so the failure must not be described as rate limiting. + +1. **Confirmed stage:** `HTTP_STATUS_RECEIVED`. The controlled retry reached the HTTP-status handling path. +2. **HTTP status code:** not captured. The existing persisted evidence does not retain an exact numeric status. +3. **Response body:** no usable body is available in the retained evidence. +4. **JSON parsing:** not started; no usable body reached the response parser. +5. **Classifier:** the ProviderErrorClassifier was triggered through the HTTP response-exception path. +6. **Classification basis:** controlled exception type and status-family mapping, not exception-message text. +7. **Redaction limit:** exact status, content type, response length, retry-after presence, upstream request identifier presence, and request duration were not safely persisted. This prevents distinguishing individual server-error statuses. +8. **Rate-limit evidence:** there is currently insufficient evidence to classify this failure as `RATE_LIMITED`; no controlled 429 evidence is retained. +9. **Most accurate block:** `BLOCKED_PROVIDER_RATE_LIMIT_OR_OUTAGE`, based on the retained server-error family. This status does not identify a specific upstream cause. +10. **Safe observability to add before a future authorized call:** exact numeric status; status family; content-type presence/category; body-presence and length bucket; retry-after presence; upstream request-identifier presence; controlled exception category; failure stage; and duration bucket. Persist only these bounded fields, never header values, endpoints, credentials, request bodies, or response bodies. diff --git a/docs/evidence/shared-provider/safe-http-attempt-metadata.json b/docs/evidence/shared-provider/safe-http-attempt-metadata.json new file mode 100644 index 0000000..9c8dc9b --- /dev/null +++ b/docs/evidence/shared-provider/safe-http-attempt-metadata.json @@ -0,0 +1,20 @@ +{ + "attemptNumber": 2, + "externalRequestAttempted": true, + "httpResponseReceived": true, + "httpStatusCode": null, + "httpStatusFamily": "5XX", + "contentTypePresent": false, + "contentTypeCategory": "UNKNOWN", + "responseBodyPresent": false, + "responseBodyLengthBucket": "ZERO_OR_UNKNOWN", + "retryAfterHeaderPresent": false, + "upstreamRequestIdHeaderPresent": false, + "exceptionTypePresent": true, + "exceptionTypeCategory": "HTTP_SERVER_ERROR", + "failureStage": "HTTP_STATUS_RECEIVED", + "failureCategory": "UPSTREAM_SERVER_ERROR", + "requestDurationBucket": "UNKNOWN", + "fallbackUsed": false, + "persistenceCompleted": true +} diff --git a/docs/evidence/shared-provider/sanitized-output.md b/docs/evidence/shared-provider/sanitized-output.md new file mode 100644 index 0000000..2d70ca0 --- /dev/null +++ b/docs/evidence/shared-provider/sanitized-output.md @@ -0,0 +1,7 @@ +# Sanitized Validation Output + +The single permitted synthetic real-provider request did not return a successful content artifact. No upstream response body, HTML error page, request header, endpoint, credential, provider identifier, model identifier, or environment-variable value is retained here. + +The request input was fully fictional and review-only. Because the validation failed, no generated artifact is reproduced. + +Persistence retained only the controlled failure classification and fixed safe summary. The fallback path was disabled and was not used. diff --git a/docs/evidence/shared-provider/validation-metadata.json b/docs/evidence/shared-provider/validation-metadata.json new file mode 100644 index 0000000..80edc75 --- /dev/null +++ b/docs/evidence/shared-provider/validation-metadata.json @@ -0,0 +1,24 @@ +{ + "validationType": "synthetic-real-provider", + "requestCount": 1, + "externalRequestAttempted": true, + "externalRequestSucceeded": false, + "providerReceiptConfirmed": false, + "externalRequestReachedProvider": "not-confirmed", + "responseContentPresent": false, + "requestedProviderPresent": true, + "actualProviderPresent": false, + "requestedAndActualProviderMatch": false, + "requestedModelPresent": true, + "actualModelPresent": false, + "requestedAndActualModelMatch": false, + "fallbackUsed": false, + "fallbackReasonPresent": false, + "generationRecordPersisted": true, + "generationTracePersisted": true, + "agentRunPersisted": true, + "agentStepPersisted": true, + "safeFailureCategoryPersisted": true, + "rawSecretOrEndpointScanPassed": true, + "genericTechnicalTermInRenderedPrompt": true +} diff --git a/docs/evidence/shared-provider/validation-summary.md b/docs/evidence/shared-provider/validation-summary.md new file mode 100644 index 0000000..6ad8d4f --- /dev/null +++ b/docs/evidence/shared-provider/validation-summary.md @@ -0,0 +1,23 @@ +# Synthetic Real Provider Validation Summary + +## Scope + +One and only one real-provider request was attempted through the normal DevFlow generation workflow using a fully fictional, review-only library-lending planning task. No real user, company, resume, ticket, or production data was used. + +## Result + +`SYNTHETIC_REAL_PROVIDER_VALIDATION_FAILED` + +The request did not produce a successful response artifact. No retry was made, fallback remained disabled, and no local-rule result was substituted. + +## Safe Persistence Evidence + +- A failed Generation Record was persisted with requested metadata retained. +- No actual execution metadata was fabricated. +- The failed Generation Trace and Agent Run remain associated with that record. +- A controlled provider-error classification and fixed safe summary were persisted; raw upstream error material was not retained. +- Provider-side receipt cannot be confirmed from the safely retained local evidence. + +## Secret Scan + +No raw credential or provider connection detail was found in the persisted validation fields or this evidence directory. One generic technical phrase was found in the rendered prompt; it is not a credential or connection detail and is not reproduced here. diff --git a/docs/shared-provider-audit.md b/docs/shared-provider-audit.md new file mode 100644 index 0000000..76c21a0 --- /dev/null +++ b/docs/shared-provider-audit.md @@ -0,0 +1,421 @@ +# DevFlow Shared Provider Audit + +## 1. Audit Scope + +This is a read-only audit of the current provider implementation and its configuration surface. No provider request, network call, source-code change, configuration change, test execution, dependency change, commit, push, or pull request was performed. This document intentionally does not contain configuration values, credentials, authorization material, endpoint hosts, or model identifiers. + +## 2. Git Baseline + +- Working tree was clean before the audit. +- Branch and starting revision matched the requested baseline. +- The branch has no commits ahead of or behind `origin/main`. +- This audit document is the only intended working-tree addition. + +## 3. Environment Variable Presence + +The six shared variables were checked only for presence. All were present in the current process environment: + +| Variable | Presence | +| --- | --- | +| `PORTFOLIO_AI_PROVIDER` | PRESENT | +| `PORTFOLIO_AI_BASE_URL` | PRESENT | +| `PORTFOLIO_AI_MODEL` | PRESENT | +| `PORTFOLIO_AI_API_KEY` | PRESENT | +| `PORTFOLIO_AI_PROTOCOL` | PRESENT | +| `PORTFOLIO_AI_FALLBACK_ENABLED` | PRESENT | + +No variable value was read or recorded. + +## 4. Current Configuration Model + +- Primary configuration is `backend/src/main/resources/application.yml`; profile files under the same directory only override the selected provider for development and production, while the test profile selects the local implementation and leaves the key blank. +- `devflow.ai` binds to `backend/src/main/java/com/devflow/copilot/config/AiProviderProperties.java` through `@ConfigurationProperties`; application bootstrap enables that class explicitly. +- Existing project-specific configuration supports provider selection, base URL, API key, model, timeout seconds, maximum tokens, and fallback-to-local. It has no protocol property and no configuration diagnosis endpoint. +- The project-specific environment variable names use the `DEVFLOW_AI_*` prefix. The safe default selects the local rule implementation; base URL and model both have code/configuration defaults, which are intentionally not reproduced here. +- `AiProviderProperties` uses explicit accessors rather than Lombok `@Data`; it has no generated `toString()`. There is no observed logging of the properties object. + +## 5. Current Provider Architecture + +- Provider interface: `GenerationProvider` with `key()` and `generate(ProviderRequest)`. +- Local implementation: `LocalRuleGenerationProvider`. It deterministically renders rule-based artifacts and estimates token counts; it does not make an HTTP call. +- External implementation: `OpenAiCompatibleGenerationProvider`. +- Router: `GenerationProviderRouter`, invoked through `LlmGenerateServiceImpl` by `LocalRuleGenerateService` (the workflow service name does not constrain it to local execution). +- Routing is exact-string selection: the local key calls the local implementation; the supported external key calls the external implementation; every other selected value fails as unsupported before the fallback block. + +## 6. Current HTTP Protocol + +- The external implementation uses Spring `RestClient` backed by `SimpleClientHttpRequestFactory`. +- It removes one trailing slash from the configured base URL and issues an HTTP `POST` to `/chat/completions`; therefore the configured base URL is expected to include the API version prefix where required. It does **not** use `/responses`. +- Request body is a map containing `model`, one user `messages` item from the rendered prompt, `temperature`, and `max_tokens`. +- Response is deserialized as Jackson `JsonNode`; content is extracted from `choices[0].message.content`. Usage reads the three conventional token counters when present. +- Connect and read timeouts both derive from the same timeout-seconds property (minimum one second). Maximum token count comes from the configured maximum-tokens property. +- Authorization is set as a default HTTP header inside the `RestClient` builder. No request or response DTO class is used: maps are used for the request and `JsonNode` for the response. + +## 7. Current Provider Routing + +- Provider selection occurs in `GenerationProviderRouter` from `AiProviderProperties`. +- The configured default is local and is safe when no project-specific AI environment variables are set. +- The router only falls back when the selected external implementation throws a `RuntimeException` and `fallbackToLocal` is true. +- Fallback returns a result whose provider/model fields describe the actual local result and whose fallback reason is the original exception message. +- Unsupported provider selection, including a spelling/case mismatch, occurs outside the fallback `try` block and therefore becomes a failed generation. + +## 8. Current Fallback Behavior + +- Fallback is a real, configurable behavior and defaults to enabled. +- Missing API key is checked before client construction. With fallback enabled it yields a successful local artifact; with fallback disabled it fails. +- Connection failures, timeouts, HTTP status failures, non-JSON/invalid JSON conversion failures, missing `choices`, missing/blank content, and unknown response shapes all reach the broad external-provider exception path and therefore fall back when enabled. +- On fallback, `GenerationRecord` and `GenerationTrace` receive the actual local provider/model returned by the router. The fallback reason is stored as the record error message and trace error message even though the record is successful. +- `AgentRun` is created before generation from configured provider/model and is not updated after fallback. It can therefore describe the requested provider while the record and trace describe the actual provider. The generation tool-call output records the fallback reason. + +## 9. Current Error Handling + +- The external provider converts all non-provider exceptions into `LlmProviderException`; existing provider exceptions pass through unchanged. +- With fallback disabled, failures propagate to `LocalRuleGenerateService`, which persists a `FAILED` generation record, failed agent step/tool call, trace, and agent-run transition, then rethrows. +- With fallback enabled, the router absorbs all external-provider runtime failures listed above. A local result is persisted as successful, with the original reason retained in the error-message fields. +- HTTP 401, 403, 429, and 5xx statuses are not classified individually; they depend on `RestClient` exception behavior and are handled by the same broad fallback/failure path. +- `safeMessage` returns raw exception messages. This preserves useful causes but can include upstream diagnostic text. + +## 10. Trace and Persistence Flow + +1. `LocalRuleGenerateService` renders a prompt and creates `generation_record` in `GENERATING` state, including provider/model configuration, prompt-template metadata, and rendered prompt. +2. It creates `agent_run` linked by `generation_record_id`, then writes decomposition/prompt/retrieval steps and related tool-call summaries. +3. On success it stores returned provider/model, token counts, latency, output, and fallback reason in `generation_record`; it then writes `generation_trace` with prompt version, input variables, prompt summary, provider/model, status, latency, and error message. +4. It records the provider tool call, creates a pending human review, and transitions the record to `READY_FOR_REVIEW`; the agent run becomes `WAITING_REVIEW`. +5. `generation_record` is linked to `agent_run`; `agent_step.run_id` links steps to a run; `tool_call_record` links to both run and optional step; `human_review` links to both run and generation record. Initial human-review status is `PENDING`. + +Persistence locations: + +- Provider/model: `generation_record`, `generation_trace`, and `agent_run`. +- Latency: `generation_record.cost_time_ms`, `generation_trace.latency_ms`, `agent_run.latency_ms` (defined but not populated by the observed start-run flow), `agent_step.latency_ms`, and `tool_call_record.latency_ms`. +- Error/fallback reason: `generation_record.error_message`, `generation_trace.error_message`, provider tool-call output summary, and failed-step summary. +- Prompt version: `generation_record.prompt_template_version` and `generation_trace.prompt_version`. + +## 11. Secret Exposure Risks + +- No direct logging of the properties object, API key, authorization header, full provider request, or full provider response was found in provider code. +- API-key values are not intentionally written to the observed entities or migrations. The trace test explicitly asserts that its input-variable JSON does not contain the project API-key variable name. +- Risk: raw upstream exception messages flow through `safeMessage` into API business-error responses (when no fallback), `generation_record.error_message`, `generation_trace.error_message`, agent-step summary, and tool-call output summary. Depending on the HTTP client/upstream response, these may disclose endpoint information, upstream HTML/error bodies, or request-adjacent data. +- Risk: `GlobalExceptionHandler` logs an exception stack trace for unhandled exceptions. Provider exceptions normally use the business-exception branch, but unexpected wrapping paths still need careful review. +- Risk: prompt content, input variables, output summaries, and tool-call summaries are persisted. They are not credential fields by design, but secrets supplied in user input could be stored. + +## 12. Existing Test Coverage + +- `ProviderAndDiagnosisTest` covers missing API key rejection by the external provider and router fallback to the local provider when the key is absent. +- `GenerationWorkflowIntegrationTest` covers local generation, persisted workflow metadata, history lookup, and record state transitions. +- `AgenticWorkflowIntegrationTest` covers generation trace, agent run/steps, tool calls, pending/confirmed human review, knowledge references, and a negative assertion for the project API-key variable name in trace input variables. +- Not covered: actual HTTP request construction/protocol compatibility, endpoint joining, timeout behavior, 401/403/429/5xx classification, non-JSON/invalid JSON/empty body behavior, fallback disabled, unsupported provider selection, sensitive-error redaction, or the fallback mismatch between `AgentRun` and actual provider metadata. +- Tests were read but not run; no real network test was performed. + +## 13. Shared Variable Mapping Proposal + +Design only; do not implement in this audit round. + +| Setting | Resolution order | Safe fallback / rule | +| --- | --- | --- | +| Provider | project-specific variable, then shared variable | local rule implementation | +| Base URL | project-specific variable, then shared variable | retain the current safe configured default without exposing it | +| Model | project-specific variable, then shared variable | retain the current safe configured default without exposing it | +| API key | project-specific variable, then shared variable | empty value | +| Protocol | project-specific variable, then shared variable | only the currently implemented chat-completions-compatible protocol | +| Fallback flag | project-specific variable, then shared variable | enabled | + +- Spring nested placeholders are suitable only if they preserve the required precedence and retain the existing safe default. Verify this with configuration-binding tests that assert presence/selection only and never print values. +- A protocol enum is recommended before supporting more than one protocol. Its only supported value should map to the current implementation; responses-style and any other values should be rejected as unsupported rather than inferred from provider name. +- A configuration class need not be added if nested property resolution remains readable and testable. A dedicated resolver is preferable if it is needed to avoid duplicated precedence logic or to produce a safe redacted diagnosis. +- Configuration validation should check presence/blankness and supported protocol/provider combinations without reading, printing, persisting, or returning the API-key value. + +## 14. Minimal Files Requiring Changes + +For the subsequent implementation phase, the smallest expected set is: + +1. `backend/src/main/resources/application.yml` for precedence-aware shared-variable placeholders and protocol configuration. +2. `backend/src/main/java/com/devflow/copilot/config/AiProviderProperties.java` for protocol binding/validation if an enum or explicit resolver is selected. +3. `backend/src/main/java/com/devflow/copilot/service/provider/GenerationProviderRouter.java` for safe protocol/provider validation and explicit fallback semantics. +4. `backend/src/main/java/com/devflow/copilot/service/provider/OpenAiCompatibleGenerationProvider.java` for protocol dispatch only if additional supported protocols are actually implemented, plus error redaction. +5. `backend/src/main/java/com/devflow/copilot/service/impl/LocalRuleGenerateService.java` and/or trace persistence code to distinguish requested from actual provider and keep fallback evidence safely. +6. Provider-focused tests, including configuration precedence and error-redaction cases. + +## 15. Files That Must Not Be Changed + +In this audit round, do not change Java source, YAML, tests, frontend code, README, existing design documents, database migrations, package manifests/lockfiles, provider implementations, Codex/Claude configuration, or `OPENAI_API_KEY`. Do not add dependencies, perform provider calls, store provider responses, commit, push, or create a pull request. + +## 16. Recommended Implementation Order + +1. Add tested, value-safe configuration resolution with project-specific variables taking precedence over shared variables and local behavior as the final default. +2. Add an explicit protocol enum/validation that recognizes only the existing supported protocol; fail safely for unsupported values. +3. Make router fallback criteria explicit and preserve requested-provider versus actual-provider metadata separately. +4. Redact and bound provider exception text before API, trace, record, agent-step, and tool-call persistence. +5. Add isolated tests for missing-key, fallback enabled/disabled, unsupported provider/protocol, HTTP error categories, malformed/empty responses, timeout configuration, metadata consistency, and redaction. Keep all such tests offline. +6. Run the targeted offline test suite only after those changes are approved. + +## 17. Current Limitations and Claims Boundary + +- This audit establishes only the behavior visible in the checked-in code at the baseline revision; it does not claim a production provider integration is stable or validated. +- No external/shared environment values were read, and no external endpoint or model identifier is recorded here. +- No provider was invoked and no real response was obtained. +- The only implemented external request form is chat-completions-compatible. Other protocols are unsupported by current code and must not be claimed as available. +- The initial shared-variable mapping proposal has now been implemented as described below; it has only offline test evidence. + +## 18. Implemented Configuration Mapping + +Sections 1-17 are the pre-implementation audit snapshot. Sections 18 onward record the approved offline implementation and verification work. + +- `application.yml` now resolves project-specific AI settings first, then the corresponding shared setting, then the established safe default. This covers provider, base URL, model, API key, protocol, and fallback. +- Development and production profile provider overrides use the same project-specific then shared precedence, so profile activation does not bypass shared configuration. +- Project-specific variable names are unchanged. No configuration value was read, emitted, persisted, or added to this document. +- `AiProviderProperties` binds the explicit protocol field using ordinary Spring configuration binding and still has no generated or custom `toString()` that could expose the API key. + +## 19. Protocol Compatibility Gate + +- The external provider validates the configured protocol before checking credentials, constructing a client, or issuing an HTTP request. +- Only the currently implemented chat-completions-compatible form passes the gate and continues to the existing `POST /chat/completions` implementation. +- An unsupported protocol is classified as `PROTOCOL_UNSUPPORTED`; with fallback enabled it produces a local result, and with fallback disabled it fails without an HTTP request. +- Unknown provider selection is classified as `UNKNOWN_PROVIDER_ERROR` and remains a failure without an HTTP request or silent provider substitution. + +## 20. Error Sanitization Strategy + +- Provider failures now use the bounded `ProviderErrorType` categories: `PROTOCOL_UNSUPPORTED`, `API_KEY_MISSING`, `TIMEOUT`, `AUTHENTICATION_FAILED`, `RATE_LIMITED`, `UPSTREAM_SERVER_ERROR`, `INVALID_RESPONSE`, `EMPTY_CONTENT`, `CONNECTION_FAILED`, and `UNKNOWN_PROVIDER_ERROR`. +- The external provider preserves the original cause only in memory. Its outward and persisted message is the category plus a fixed safe summary; it never copies raw upstream exception text. +- Fallback reasons, generation-record errors, generation-trace errors, failed step summaries, failed tool-call summaries, and API business errors use that safe message. +- The global fallback exception logger now logs only an exception class name, not an exception stack trace or message. + +## 21. Offline Test Evidence + +- `SharedProviderConfigurationMappingTest` verifies project-specific precedence, shared fallback, safe defaults, and fallback-setting precedence against the checked-in YAML placeholders. +- `ProviderRoutingSafetyTest` verifies the protocol gate, enabled/disabled fallback behavior, missing-key no-request path, and unknown-provider no-request path. +- `ProviderErrorClassifierTest` verifies offline classification for timeout, connection failure, authentication failure, rate limit, upstream server error, and invalid response. +- `ProviderErrorSanitizationIntegrationTest` injects an offline synthetic provider failure and verifies that the returned exception, generation record, generation trace, failed step, and failed tool-call output retain only the safe classification. +- The complete backend test suite passed offline. No real provider or network call was made by these tests. + +## 22. Remaining Work Before Real Validation + +- The existing persistence schema records actual provider/model after a successful result, including fallback, and the agent run is now updated to that actual metadata. It does not separately persist requested provider/model or a dedicated fallback flag; the safe fallback reason remains the compatible evidence field. +- Real endpoint reachability, provider-specific response variations, credential validity, and production-network behavior remain unvalidated and must be tested only in an approved later phase. +- Before any real validation, retain the protocol allowlist, keep error messages bounded, and confirm operational logging/configuration never exposes credentials or upstream payloads. + +## 23. Implementation Diff Review + +- Reviewed every changed and added backend, configuration, test, and audit file against the baseline. No frontend, README, screenshot, frozen Trace Workspace, binary, migration, or unrelated business file was changed. +- Configuration precedence is implemented through nested Spring placeholders. Development and production provider profile overrides preserve the same precedence. +- `SharedProviderConfigurationMappingTest` was corrected to create a Spring context, bind `AiProviderProperties`, and remove host system property/environment sources. It now validates binding rather than only expanding YAML text. +- All Spring integration tests explicitly force the offline local provider, blank test credential, supported protocol, and enabled fallback through test properties. This prevents host shared/provider settings from selecting an external provider during tests. + +## 24. Confirmed Runtime Semantics + +- The external provider validates the protocol before credential handling, HTTP-client construction, header construction, or request execution. The request path remains `POST /chat/completions`. +- The router selects once. It catches only `LlmProviderException`, so only classified provider failures can trigger one local fallback. It neither recurses nor treats arbitrary runtime, database, or application defects as fallback candidates. +- Missing credentials, unsupported protocol, and unknown provider selection cannot create or send an external request. Unsupported protocol and missing credentials can fall back once when enabled; unknown provider remains a classified failure. +- Base-URL normalization removes a trailing duplicate chat-completions suffix before the fixed request path is appended. Response validation now distinguishes malformed/missing required fields from intentionally empty content. +- HTTP status-based classification occurs from response status codes without parsing English exception messages. Offline loopback tests cover authentication failures, rate limit, server error, invalid JSON, missing response fields, empty content, timeout, and connection failure through the provider path. + +## 25. Corrected Defects + +- **P0 corrected:** Spring integration tests could inherit host provider environment values because profile YAML has lower property precedence than the environment. Explicit test properties now isolate the test provider configuration. +- **P1 corrected:** the router's broad `RuntimeException` catch could have downgraded programming or persistence defects to local fallback. It now catches only classified provider exceptions. +- **P1 corrected:** a configured base URL ending in the request endpoint could produce a duplicated endpoint path. The provider normalizes that suffix before appending the fixed path. +- **P1 corrected:** missing response fields were previously treated as empty content. They now produce `INVALID_RESPONSE`; genuinely blank content remains `EMPTY_CONTENT`. +- **P2 corrected:** global exception logging no longer suppresses normal internal exception diagnostics. Only exception chains containing a provider exception are reduced to a safe type-only log entry. + +## 26. Database Compatibility + +- No entity field, mapper, migration, table, or historical data was changed. H2 and MySQL schema compatibility therefore remains unchanged. +- Existing columns persist actual provider/model after a completed result, and the agent run is updated to the same actual execution metadata. Safe fallback/error evidence remains in existing error-message and tool/step summary fields. +- The schema does **not** separately persist requested provider, requested model, actual provider, actual model, fallback-used, and fallback reason as distinct immutable fields. In particular, the original requested provider/model may be overwritten by actual execution metadata after fallback. This is a documented limitation, not a claim of full requested/actual auditability. +- No frozen Trace frontend contract was changed. Tool-call records retain their real run/step associations and human-review behavior is unchanged. + +## 27. Real Validation Readiness + +`BLOCKED_BEFORE_REAL_PROVIDER_VALIDATION` + +Blocking reason: the current schema lacks separate, immutable requested/actual provider and model fields plus a dedicated fallback-used field. A future approved migration and compatible trace/API design are required before claiming complete requested-versus-actual evidence for real validation. This status does not indicate a production outage and does not authorize a real provider call. + +## 28. Provider Execution Metadata Model + +The blocking persistence gap described in sections 22, 26, and 27 is now resolved by the approved minimal schema change below. + +- `requested_provider`: the initially selected provider at generation start; it is never overwritten by the execution outcome. +- `requested_model`: the initially selected model at generation start; direct local execution records its controlled local model rather than an unrelated external-model configuration. It is never overwritten by the execution outcome. +- Existing `provider_name` and `model_name` on `generation_record` now explicitly mean **actual** provider and model for records created after this migration. They remain null when no provider completed the generation. +- `fallback_used`: true only when the selected external provider produced a classified provider failure and the router actually completed one local fallback. +- `fallback_reason`: nullable, controlled `ProviderErrorType` name used only for a successful fallback. It does not contain a raw exception, endpoint, authorization data, request body, or response body. + +`AiGenerateResponse` retains the existing provider/model fields for API compatibility and adds requested/actual aliases plus the two fallback fields. No existing JSON field was removed or renamed. + +## 29. Canonical Persistence Source + +- `generation_record` is the canonical source for all requested/actual/fallback execution metadata. +- `generation_trace` and `agent_run` continue to carry their existing provider/model columns as operational copies of the completed actual execution only. They obtain complete requested/fallback context through their existing `generation_record_id` association; no redundant columns were added to all three tables. +- At generation start, the record contains requested metadata and no fabricated actual execution metadata. A successful result writes actual metadata; a failed, non-fallback result leaves actual metadata null. `GenerationTrace` and `AgentRun` therefore remain consistent with the canonical record for actual execution. + +## 30. Flyway Migration Evidence + +- The audited highest existing migration was V4, so the minimal next migration is `V5__add_provider_execution_metadata.sql`. +- V5 adds bounded `VARCHAR(64)` / `VARCHAR(128)` request fields, `BOOLEAN NOT NULL DEFAULT FALSE` for fallback usage, and nullable `VARCHAR(64)` fallback reason. These types are compatible with the project H2 MySQL-mode test database and the production MySQL schema style. +- Existing records are conservatively backfilled by copying the historical provider/model fields into the new requested fields. Their existing provider/model values are not rewritten, fallback usage remains false by default, and no fallback reason is invented. Historical rows consequently retain their prior ambiguity rather than receiving fabricated execution evidence. +- `ProviderExecutionMetadataMigrationTest` starts from the Flyway-managed H2 schema, verifies the V5 columns and defaults, and verifies that a seeded legacy row remains readable with conservative request metadata. + +## 31. Requested and Actual Semantics + +- Direct local execution: requested and actual metadata identify the local implementation; fallback fields are false/null. +- External success: requested metadata identifies the selected external execution and actual metadata is written only from the returned provider result; fallback fields are false/null. +- Classified external failure with fallback enabled: requested metadata remains unchanged, actual metadata identifies the local completed result, `fallback_used` is true, and `fallback_reason` is the controlled error category. +- Classified external failure with fallback disabled: the record transitions to `FAILED`, requested metadata remains available, actual metadata remains null, fallback fields are false/null, and `error_message` retains only the safe error category and fixed summary. +- Unknown provider: no HTTP request and no fallback occur; the failed record has requested metadata but no actual metadata. + +## 32. Fallback Persistence Evidence + +- `ProviderResult` now carries an explicit fallback-used boolean. The router sets it only in its single classified-exception fallback branch and stores only the `ProviderErrorType` name as the fallback reason. +- `LocalRuleGenerateService` creates the canonical requested metadata before dispatch, persists actual metadata only after a result, and preserves null actual metadata on failure. Successful fallback no longer abuses `error_message` as fallback evidence. +- Offline tests cover direct local execution, synthetic external success, missing-credential fallback, disabled fallback failure, unknown-provider failure, safe error persistence, migration/backfill behavior, and trace/agent-run actual-metadata consistency. Test-profile configuration is fixed to local/offline settings so host shared-provider values cannot select an external endpoint. + +## 33. Synthetic Real Validation Readiness + +`READY_FOR_SYNTHETIC_REAL_PROVIDER_VALIDATION` + +This conclusion is limited to offline synthetic validation. It is supported by the migration test and targeted persistence scenarios, all run without a real provider, model, external network endpoint, or remote database. It does not claim that credentials, endpoint reachability, provider compatibility, model behavior, production observability, or production availability has been validated. + +## 34. Synthetic Real Provider Validation + +One permitted real-provider request was attempted through the normal DevFlow Generation API and workflow with a fully fictional, review-only library-lending planning task. Fallback was disabled for the validation process, no retry was made, and no local-rule result was substituted. + +`SYNTHETIC_REAL_PROVIDER_VALIDATION_FAILED` + +The request did not produce a successful response artifact. This result does not indicate production availability or provider stability. + +## 35. Real Request Evidence + +- Exactly one external request was attempted. +- The failure persisted a controlled error classification and fixed safe summary only. +- Provider-side receipt cannot be confirmed from the safely retained local evidence; no raw upstream response or endpoint is retained. +- The synthetic input contained no real user, company, commercial, resume, ticket, or production information. + +## 36. Persistence Evidence + +- A failed Generation Record was persisted with requested metadata retained. +- Actual provider/model metadata remained null rather than being fabricated after the failure. +- `fallback_used` remained false and the fallback reason remained null. +- The Generation Trace and Agent Run are associated with the failed record. No additional successful trace or local-rule execution evidence was created. + +## 37. Secret Scan Evidence + +- A precise persisted-data scan found no raw credential, authorization value, credential-token value, or endpoint. +- One generic technical phrase occurred in the rendered prompt. It is not a secret or endpoint and is intentionally not reproduced in this document. +- Evidence files contain no configuration values, headers, endpoints, provider identifiers, model identifiers, or upstream response bodies. + +## 38. Final Validation Conclusion + +`SYNTHETIC_REAL_PROVIDER_VALIDATION_FAILED` + +No further real-provider request is authorized by this validation run. The failure has safe persistence evidence, but real endpoint reachability and successful provider compatibility remain unverified. + +## 39. Failed Real Validation Forensics + +The failed synthetic real-provider validation is supported by the persisted Generation Record, Generation Trace, Agent Run, and Agent Step evidence. The record is `FAILED`; requested metadata remains present, actual metadata remains absent, fallback remains false, and no fallback reason exists. This preserves the actual failed-execution fact without fabricating successful execution metadata. + +## 40. Confirmed Failure Stage + +`UNKNOWN_STAGE` + +The controlled category proves that the provider transport path was entered, but safe evidence does not distinguish connection establishment, TLS negotiation, request transmission, or waiting for a response. No HTTP response, response body, or response parsing evidence was retained. The precise lower-level stage is therefore intentionally not inferred. + +## 41. Confirmed Failure Category + +`TIMEOUT` + +The latest failed record stores the controlled timeout category and its fixed safe summary. This is consistent with the provider classifier and does not expose the original transport exception. + +## 42. Offline Reproduction + +Loopback tests reproduce timeout handling through the actual provider path and verify classification of status failures, malformed/empty responses, URI joining, and connection failure. The complete offline suite passes. These tests establish that the code safely classifies the observed class of failure; they cannot establish the remote cause of this one real timeout. + +## 43. Repair Decision + +No code repair is applied. Configuration binding, protocol support, router recognition, URI parsing, and request-path joining validate without exposing runtime values. No reproducible code defect has been found, and the sanitised timeout evidence cannot distinguish provider delay from network or transport delay. Altering code or a configuration value would therefore be speculative. + +## 44. Controlled Retry Readiness + +`INSUFFICIENT_SANITIZED_EVIDENCE` + +No second real-provider request was made. A later controlled retry requires separate authorization and additional safe observability that can distinguish transport stages without retaining secrets or raw upstream payloads. + +## 45. Connectivity Preflight + +The authorized preflight validated URI parsing, DNS resolution, TCP connection, and TLS handshake. Each completed in the shortest recorded duration band. It sent no HTTP request and used no authorization material. The preflight does not retain endpoint, host, address, port, certificate, or path details. + +## 46. Controlled Retry Configuration + +The controlled retry ran with fallback disabled, a bounded provider timeout, and a limited generation budget. These controls applied only to the validation process; no permanent environment variable or application configuration file was changed. Runtime checks confirmed all controls were active before the request. + +## 47. Controlled Retry Result + +One authorized retry was attempted after the successful preflight. It received an HTTP status response, but no usable response body or parsed artifact. The controlled category is `UPSTREAM_SERVER_ERROR`. The failed Generation Record, Trace, Run, and Steps were persisted; fallback was not used and actual execution metadata was not fabricated. + +## 48. Final Shared Provider Status + +`BLOCKED_UPSTREAM_5XX_UNCLASSIFIED` + +The preflight rules out the tested local URI/DNS/TCP/TLS path. The controlled HTTP-status failure is classified as an upstream service failure. No further real request is authorized. This does not claim production availability, provider stability, SLA verification, full model compatibility, or complete real-world error coverage. + +## 49. Safe HTTP Observability Defect + +The second controlled request safely established an HTTP 5xx family response, but the previous implementation reduced the response exception immediately to `UPSTREAM_SERVER_ERROR`. It discarded the exact status code and other non-content diagnostic facts before the failure reached the canonical Generation Record. There is no 429 evidence, so rate limiting is not the current conclusion. + +## 50. Provider Failure Metadata Model + +`ProviderFailureMetadata` is a deliberately content-free immutable model carried by `LlmProviderException`. It contains only the controlled error type, failure stage, status code/family, body-presence and size bucket, content-type category, header-presence booleans, generated client request ID, and duration bucket. It excludes response bodies, URL/host data, credentials, provider/model values, and all header values. Exception JSON serialization and `toString()` exclude the cause. + +## 51. Exact HTTP Status Persistence + +V6 adds nullable provider-failure observability columns only to `generation_record`, the canonical execution source. Existing records are untouched and retain null observability values. On a classified provider failure, the record persists the exact numeric HTTP status when one exists, its `4XX` or `5XX` family, and the remaining safe metadata. Trace and agent-run records continue to obtain this context through their existing record association; no redundant fields were added to all workflow tables. + +## 52. Offline 5xx Classification Evidence + +Loopback-only tests execute the real provider path and retain exact status metadata for 500, 502, 503, and 504, all classified as `UPSTREAM_SERVER_ERROR`. A 429 remains `RATE_LIMITED` with an exact 429 status and a `4XX` family, proving the two categories stay distinct. HTML and JSON error bodies are reduced to presence, size, and content-type category only; retry and upstream-request headers are represented solely as presence booleans. + +## 53. Remaining Compatibility Hypotheses + +The current evidence does not establish the cause of the observed 5xx family response. Pending hypotheses are: temporary upstream server failure, gateway model-routing failure, chat-completions compatibility, temperature compatibility, token-limit compatibility, and a non-standard upstream error response. None is treated as a confirmed cause, and no request compatibility strategy was changed in this round. + +## 54. Capability Probe Readiness + +`READY_FOR_SINGLE_MODEL_CAPABILITY_PROBE` + +The observability implementation and its offline evidence are complete. The current accurate blocking description before any separately authorized probe is `BLOCKED_UPSTREAM_5XX_UNCLASSIFIED`: no 429 evidence exists, the exact previous status code was not retained, and the observed 5xx family does not establish outage, routing, model compatibility, or parameter compatibility. This round made no network request. Any next network operation must be one separately authorized, single safe Capability Probe. + +## 55. Final Single Chat Path Attempt + +One final minimal real Chat Completions validation was executed. The current-round external request count is 1, and the historical cumulative real request count is 3. No second current-round request was made, no automatic retry was executed, and GET /models was not executed. The validation used fallbackDisabled=true, timeoutConfigured=true, and maxTokensLimited=true only for the validation process. + +## 56. Exact Real HTTP Evidence + +The DevFlow Generation API returned an HTTP 200 result for the final call, and the provider success path produced a parsed non-empty generation artifact. The safe final evidence records HTTP status code 200, status family `2XX`, responseBodyPresent=true, responseParsingCompleted=true, and responseContentPresent=true. No raw request body, raw response body, endpoint, host, credential, provider value, model value, or client request ID value is retained. + +## 57. Final Persistence Evidence + +The final attempt persisted the Generation Record, Generation Trace, Agent Run, and Agent Step chain. Requested provider/model metadata is present, actual provider/model metadata is present, requested and actual provider/model metadata match in-memory, fallbackUsed=false, and fallbackReason is null. Success-path failure observability fields remain unset because no provider failure was classified. The persisted content exists, but sanitized keyword checks did not confirm relation to the fictional book-borrowing review task. + +## 58. DevFlow Provider Integration Closure + +`REAL_PROVIDER_TRANSPORT_AND_PERSISTENCE_VALIDATED` + +The real provider transport and persistence integration reached a successful HTTP and parsing path. The earlier relevance result was based on a narrow sanitized checker and did not include the persisted output text in the review bundle. No automatic code fix, additional provider request, GET /models probe, commit, push, or PR was performed. + +## 59. Offline Semantic Review of Persisted Output + +No new external request, provider call, model call, retry, or GET /models probe was performed. The latest persisted Generation Record was read offline from the local database copy. It is successful, has non-empty output, has a rendered prompt, retains token usage, and remains associated with Generation Trace, Agent Run, and Agent Step records. Requested and actual provider/model metadata are present and match, but their values are intentionally not recorded. + +The rendered prompt contains the synthetic Spring Boot teaching task, the exactly-three-items constraint, the transaction-boundary requirement, the inventory-concurrency requirement, the idempotency/testing requirement, and the review-only constraints. The prompt itself was not exported. + +## 60. Relevance Checker Determination + +`SANITIZED_RELEVANCE_CHECK_FALSE_NEGATIVE` + +The already persisted output was checked offline using Chinese keywords, English synonyms, structure checks, and manually explainable semantic rules. It covers transaction boundary or transaction handling; inventory concurrency, locking, atomic decrement, or over-borrow control; and idempotency, duplicate-request protection, or testing advice. It is review-only, contains no full implementation, contains no real data, and has exactly three items. The earlier generatedContentRelated=false result is therefore treated as a false negative, not a provider compatibility or persistence failure. + +## 61. Final Provider Integration Verdict + +`SYNTHETIC_REAL_PROVIDER_VALIDATION_PASSED` + +The final provider integration is validated for the synthetic single Chat Completions path: one current-round external request, three historical real requests total, no GET /models, fallback disabled and unused, HTTP 200 retained, response body received, JSON parsed, non-empty related output persisted, token counts parsed, and the DevFlow Generation Record / Trace / Run / Step chain persisted. This does not claim production availability, long-term provider stability, SLA characteristics, or broad model compatibility. + +Recommended next action: `READY_TO_COMMIT_AND_FREEZE`.