diff --git a/packages/agent/prompts/independent-solver.md b/packages/agent/prompts/independent-solver.md index 4928954..9f40a65 100644 --- a/packages/agent/prompts/independent-solver.md +++ b/packages/agent/prompts/independent-solver.md @@ -1,6 +1,6 @@ --- id: independent-solver -version: 0.1.0 +version: 0.2.0 model: gpt-4o temperature: 0.0 max_tokens: 1500 @@ -8,7 +8,7 @@ schema: SolveAttemptSchema variables: - candidate owner: 비할당 -updated: 2026-05-18 +updated: 2026-06-11 --- # Role @@ -35,6 +35,7 @@ updated: 2026-05-18 - `derived_answer: string` (문제 유형에 맞는 최종 정답. 방정식 해는 `2, -5`, 확률은 `3/8`, 통계값은 `평균 12`, 기하는 `60도`처럼 간결히 쓴다) - `trace: string` - `confidence: "high" | "medium" | "low"` +- `verification_expression: string` (선택) — 당신의 풀이가 정답에 도달하는 *계산식 하나*. SymPy 평가 가능 표기만: `factorial(4)`, `binomial(10, 3)`, `factorial(5)/factorial(5-2)`, `*`, `/`, `+`, `-`, `**`. 이 식의 값이 `derived_answer`와 일치해야 한다. 정답이 단일 수치/수식 값이 아니면 생략. `4!` 표기, 한글, 단위 금지. # Constraints diff --git a/packages/agent/prompts/problem-generator.md b/packages/agent/prompts/problem-generator.md index b893bd7..fc5bc2b 100644 --- a/packages/agent/prompts/problem-generator.md +++ b/packages/agent/prompts/problem-generator.md @@ -1,6 +1,6 @@ --- id: problem-generator -version: 0.1.2 +version: 0.2.0 model: gpt-4o temperature: 0.35 max_tokens: 2000 @@ -14,7 +14,7 @@ variables: - counterexample - schemaError owner: 비할당 -updated: 2026-06-10 +updated: 2026-06-11 --- # Role @@ -105,8 +105,9 @@ JSON으로만 응답. - `expected_answer`는 정답만 간결하게 쓴다. 방정식 해는 `2, 5`, 통계값은 `평균 12`, 확률은 `3/8`, 기하는 `60도`처럼 쓴다. - `techniques_used`는 실제 사용한 풀이 기법 id 배열이다. Strategy 어휘에 있는 `factoring`, `quadratic_formula`, `completing_the_square` 같은 snake_case id를 우선 사용하고, 공백·한글 설명 문장은 넣지 않는다. - structural 모드에서는 필수 기법을 모두 포함해야 한다. conceptual 모드에서는 필수·관련 기법 중 실제 사용한 것을 최소 1개 이상 포함한다. -- 예시 출력: `{ "question_text": "다항식 (x + 3)(x - 5)를 전개하시오.", "expected_answer": "x**2 - 2*x - 15", "techniques_used": ["polynomial_expansion"], "proposed_solution_trace": "분배법칙으로 각 항을 곱해 동류항을 정리한다." }` +- 예시 출력: `{ "question_text": "서로 다른 5권의 책 중 3권을 골라 일렬로 꽂는 방법의 수는?", "expected_answer": "60", "techniques_used": ["permutation"], "proposed_solution_trace": "5권 중 3권을 순서 있게 배열하므로 5P3 = 5*4*3 = 60이다.", "verification_expression": "factorial(5)/factorial(5-3)" }` - `proposed_solution_trace`에 풀이 단계와 출제 의도를 한국어로 명시하되, 수식도 JSON 안전한 plain text 표기만 사용한다. +- `verification_expression`은 정답에 도달하는 *계산식 하나*다. SymPy가 평가할 수 있는 표기만 쓴다: 팩토리얼은 `factorial(4)`, 조합은 `binomial(10, 3)`, 순열은 `factorial(5)/factorial(5-2)`, 사칙연산은 `*`, `/`, `+`, `-`, 지수는 `**`. 이 식을 평가한 값이 `expected_answer`와 정확히 일치해야 한다. 예: 정답이 `864`이고 풀이가 3!*3!*4!이면 `"verification_expression": "factorial(3)*factorial(3)*factorial(4)"`. 정답이 단일 수치/수식 값이 아닐 때(서술형, 참/거짓 등)만 생략한다. `4!` 같은 느낌표 표기, 한글, 단위는 절대 넣지 않는다. - 결과 문제는 source problem과 달라야 하며, structural/conceptual 모드 차이가 풀이 설명에 드러나야 한다. # TODO diff --git a/packages/agent/src/agents/generator-agent.ts b/packages/agent/src/agents/generator-agent.ts index 2e7e27f..d1bbfaa 100644 --- a/packages/agent/src/agents/generator-agent.ts +++ b/packages/agent/src/agents/generator-agent.ts @@ -54,6 +54,13 @@ export const LlmGeneratedCandidateSchema = z.object({ .string() .min(1) .describe("Korean solution trace explaining the structural/conceptual transform"), + verification_expression: z + .string() + .min(1) + .optional() + .describe( + "SymPy-evaluable arithmetic expression whose value equals expected_answer, e.g. factorial(3)*factorial(3)*factorial(4). Omit only when the answer is not a single numeric/symbolic value", + ), }); export type LlmGeneratedCandidate = z.infer; @@ -79,6 +86,9 @@ export function assembleGeneratedProblem(input: { expected_answer: input.object.expected_answer, techniques_used: input.object.techniques_used ?? [], proposed_solution_trace: input.object.proposed_solution_trace, + ...(input.object.verification_expression === undefined + ? {} + : { verification_expression: input.object.verification_expression }), source_refs: input.refs.map((ref) => ref.item_id), inferred_intent: input.intent, generation_metadata: { diff --git a/packages/agent/src/schemas/generated-problem.schema.ts b/packages/agent/src/schemas/generated-problem.schema.ts index c4b5cd1..7e642ac 100644 --- a/packages/agent/src/schemas/generated-problem.schema.ts +++ b/packages/agent/src/schemas/generated-problem.schema.ts @@ -40,6 +40,10 @@ export const GeneratedProblemSchema = z.object({ expected_choices: z.array(z.string()).optional(), // objective일 때 techniques_used: z.array(z.string()).default([]), proposed_solution_trace: z.string(), + /** 정답에 도달하는 SymPy 평가식 (예: "factorial(3)*factorial(3)*factorial(4)"). + * sympy_verify가 math-engine /evaluate로 평가해 expected_answer와 대조한다 — + * equation이 아닌 kind(경우의 수 등)도 결정론 검증이 가능해진다. */ + verification_expression: z.string().min(1).optional(), source_refs: z.array(z.string()), // SourceProblem.item_id[] inferred_intent: IntentSchema, diff --git a/packages/agent/src/schemas/solve-attempt.schema.ts b/packages/agent/src/schemas/solve-attempt.schema.ts index 72fb796..786d0b7 100644 --- a/packages/agent/src/schemas/solve-attempt.schema.ts +++ b/packages/agent/src/schemas/solve-attempt.schema.ts @@ -6,6 +6,8 @@ export const SolveAttemptSchema = z.object({ derived_answer: z.string().min(1), trace: z.string().min(1), confidence: z.enum(["high", "medium", "low"]), + /** 재풀이가 도달한 정답의 SymPy 평가식 — 생성기 식과 독립인 두 번째 결정론 대조용. */ + verification_expression: z.string().min(1).optional(), }); export type SolveAttempt = z.infer; diff --git a/packages/agent/src/server/sse/wire-adapter.ts b/packages/agent/src/server/sse/wire-adapter.ts index 5c6a272..71e4a9d 100644 --- a/packages/agent/src/server/sse/wire-adapter.ts +++ b/packages/agent/src/server/sse/wire-adapter.ts @@ -123,6 +123,9 @@ function sympySummary( return `기호 검증 불가${kindLabel} — 독립 재풀이로 확인`; } const sympyAnswer = asString(evidence["sympy_answer"]); + if (evidence["expression_check"] === true) { + return sympyAnswer === null ? "검증식 평가 일치" : `검증식 평가 일치 (값: ${sympyAnswer})`; + } if (sympyAnswer === null) return "SymPy 검산 일치"; return `SymPy 검산 일치 (해: ${sympyAnswer})`; } diff --git a/packages/agent/src/steps/sympy-verification.ts b/packages/agent/src/steps/sympy-verification.ts index 3b6c9f5..a423a25 100644 --- a/packages/agent/src/steps/sympy-verification.ts +++ b/packages/agent/src/steps/sympy-verification.ts @@ -41,10 +41,6 @@ type VerificationCheck = { readonly evidence?: Record; }; -type SymbolicCheckability = - | { readonly checkable: true } - | { readonly checkable: false; readonly reason: string; readonly evidence?: Record }; - const NO_EXTRACTABLE_EQUATION_REASON = "Question text does not contain an extractable equation for SymPy"; const NO_EXTRACTABLE_EQUATION_EVIDENCE: Record = { @@ -71,15 +67,18 @@ export async function verifyWithSympy( }, }; } catch (err) { + // 엔진 장애/타임아웃은 수학적 판정이 아니다 — failed로 두면 acceptance가 + // 인프라 장애를 "틀린 문제"로 reject한다. unverified로 강등해 warning에 머문다. + const message = err instanceof Error ? err.message : String(err); return { gate: { step: "sympy_verify", - status: "failed", + status: "unverified", duration_ms: Date.now() - started, - evidence: { engine: "sympy" }, + evidence: { engine: "sympy", reason: `math-engine error: ${message}` }, failure_detail: { code: "sympy_error", - message: err instanceof Error ? err.message : String(err), + message, }, }, }; @@ -90,52 +89,121 @@ async function verifyCandidate( mathEngine: MathEngineClient, candidate: GeneratedProblem, ): Promise { - const checkability = classifySymbolicCheckability(candidate); - if (!checkability.checkable) { - return unverifiedCheck(candidate, checkability.reason, checkability.evidence); - } - if (candidate.generation_kind === "equation") { - if (isChoiceStyleCandidate(candidate)) { - return verifyChoiceCandidate(mathEngine, candidate); + const equation = extractEquationText(candidate.question_text); + if (equation !== null) { + if (isChoiceStyleCandidate(candidate)) { + if (resolveChoiceOptions(candidate).length === 0) { + return unverifiedCheck( + candidate, + "Multiple-choice equation has no parseable expected_choices/options", + { equation }, + ); + } + return verifyChoiceCandidate(mathEngine, candidate); + } + return verifyEquationCandidate(mathEngine, candidate); } - return verifyEquationCandidate(mathEngine, candidate); + // 식 추출 실패 (서술형 방정식 문제 등) — 검증식이 있으면 평가 경로로 폴백. + const byExpression = await verifyByExpression(mathEngine, candidate); + if (byExpression !== null) return byExpression; + return unverifiedCheck( + candidate, + NO_EXTRACTABLE_EQUATION_REASON, + NO_EXTRACTABLE_EQUATION_EVIDENCE, + ); } + const byExpression = await verifyByExpression(mathEngine, candidate); + if (byExpression !== null) return byExpression; return unverifiedCheck( candidate, - `No deterministic SymPy verifier is implemented for generation_kind=${candidate.generation_kind}`, + "SymPy verification requires a checkable equation; non-equation candidates rely on independent re-solve", + { generation_kind: candidate.generation_kind }, ); } -function classifySymbolicCheckability(candidate: GeneratedProblem): SymbolicCheckability { - if (candidate.generation_kind !== "equation") { - return { - checkable: false, - reason: - "SymPy verification requires a checkable equation; non-equation candidates rely on independent re-solve", - evidence: { generation_kind: candidate.generation_kind }, - }; +/** 후보의 verification_expression을 /evaluate로 평가해 선언 정답과 대조한다. + * 식이 없으면 null (호출자가 기존 unverified 폴백 유지). 평가 실패는 unverified — + * 식은 LLM 산출물이라 구문 오류가 수학 오류를 뜻하지 않는다. */ +async function verifyByExpression( + mathEngine: MathEngineClient, + candidate: GeneratedProblem, +): Promise { + const expression = candidate.verification_expression; + if (expression === undefined) return null; + + const declared = declaredAnswerBody(candidate); + if (declared === null) { + return unverifiedCheck( + candidate, + "Declared expected_answer does not identify one of the provided choices", + { verification_expression: expression, expected_answer: candidate.expected_answer }, + ); } - const equation = extractEquationText(candidate.question_text); - if (equation === null) { - return { - checkable: false, - reason: NO_EXTRACTABLE_EQUATION_REASON, - evidence: NO_EXTRACTABLE_EQUATION_EVIDENCE, - }; + let evaluatedValue: string; + try { + evaluatedValue = (await mathEngine.evaluate({ expr: expression })).value; + } catch (err) { + return unverifiedCheck( + candidate, + `math-engine could not evaluate the verification expression: ${err instanceof Error ? err.message : String(err)}`, + { verification_expression: expression }, + ); } - if (isChoiceStyleCandidate(candidate) && resolveChoiceOptions(candidate).length === 0) { + const decision = await decideAnswerEquivalence(mathEngine, declared, evaluatedValue); + if (decision.status === "undecidable") { + return unverifiedCheck( + candidate, + `Could not compare declared answer with evaluated verification expression: ${decision.reason ?? "undecidable"}`, + { expression_check: true, verification_expression: expression, evaluated_value: evaluatedValue }, + declared, + evaluatedValue, + ); + } + if (decision.status === "not_equivalent") { return { - checkable: false, - reason: "Multiple-choice equation has no parseable expected_choices/options", - evidence: { equation }, + status: "failed", + verificationKind: candidate.generation_kind, + expectedAnswer: declared, + sympyAnswer: evaluatedValue, + failureCode: "expression_value_mismatch", + failureMessage: `Verification expression evaluates to ${evaluatedValue}, which does not match the declared answer ${declared}`, + evidence: { + expression_check: true, + verification_expression: expression, + evaluated_value: evaluatedValue, + equivalence: decision, + }, }; } + return { + status: "passed", + verificationKind: candidate.generation_kind, + expectedAnswer: declared, + sympyAnswer: evaluatedValue, + evidence: { + expression_check: true, + verification_expression: expression, + evaluated_value: evaluatedValue, + }, + }; +} - return { checkable: true }; +/** 검증식 대조 대상이 되는 선언 정답 본문. 객관식이면 정답 보기의 본문, + * 아니면 choice prefix(`① ` 등)를 벗긴 expected_answer. */ +function declaredAnswerBody(candidate: GeneratedProblem): string | null { + if (isChoiceStyleCandidate(candidate)) { + const choices = resolveChoiceOptions(candidate); + if (choices.length > 0) { + const correct = selectDeclaredCorrectChoice(candidate, choices); + return correct === null ? null : correct.body; + } + } + const stripped = stripChoicePrefix(candidate.expected_answer); + return stripped.length > 0 ? stripped : candidate.expected_answer; } async function verifyEquationCandidate( diff --git a/packages/agent/src/tools/math-engine-client.ts b/packages/agent/src/tools/math-engine-client.ts index ce05fea..08e1dd7 100644 --- a/packages/agent/src/tools/math-engine-client.ts +++ b/packages/agent/src/tools/math-engine-client.ts @@ -32,6 +32,16 @@ export interface SimplifyResponse { simplified: string; } +export interface EvaluateRequest { + expr: string; +} +export interface EvaluateResponse { + /** 정확값 (SymPy canonical 문자열, 예: "864", "3/8", "2*sqrt(2)") */ + value: string; + /** 십진 근사값 (evalf) */ + numeric: string; +} + export interface DifferentiateRequest { expr: string; variable?: string; @@ -59,6 +69,7 @@ export interface MathEngineClient { solve(req: SolveRequest): Promise; verify(req: VerifyRequest): Promise; simplify(req: SimplifyRequest): Promise; + evaluate(req: EvaluateRequest): Promise; differentiate(req: DifferentiateRequest): Promise; limit(req: LimitRequest): Promise; } @@ -134,6 +145,8 @@ export function createMathEngineClient( ), simplify: (req) => post("/simplify", { expr: toSympyInput(req.expr) }, parseSimplifyResponse), + evaluate: (req) => + post("/evaluate", { expr: toSympyInput(req.expr) }, parseEvaluateResponse), differentiate: (req) => post( "/differentiate", @@ -231,6 +244,11 @@ function parseSimplifyResponse(value: unknown): SimplifyResponse { return { simplified: readString(obj, "simplified") }; } +function parseEvaluateResponse(value: unknown): EvaluateResponse { + const obj = asObject(value); + return { value: readString(obj, "value"), numeric: readString(obj, "numeric") }; +} + function parseDifferentiateResponse(value: unknown): DifferentiateResponse { const obj = asObject(value); return { derivative: readString(obj, "derivative") }; diff --git a/packages/agent/tests/deterministic-fallback.test.ts b/packages/agent/tests/deterministic-fallback.test.ts index 3bc08fb..dbbc8f0 100644 --- a/packages/agent/tests/deterministic-fallback.test.ts +++ b/packages/agent/tests/deterministic-fallback.test.ts @@ -95,6 +95,7 @@ const mathEngine: MathEngineClient = { }, verify: async () => ({ equivalent: true, diff: "0" }), simplify: async ({ expr }) => ({ simplified: expr }), + evaluate: async () => ({ value: "", numeric: "" }), differentiate: async () => ({ derivative: "" }), limit: async () => ({ limit: "" }), }; diff --git a/packages/agent/tests/generator-agent.test.ts b/packages/agent/tests/generator-agent.test.ts index 33f8b8a..298b7e6 100644 --- a/packages/agent/tests/generator-agent.test.ts +++ b/packages/agent/tests/generator-agent.test.ts @@ -89,6 +89,49 @@ describe("createGeneratorAgent", () => { ); }); + it("passes the LLM verification expression through to the generated problem", async () => { + aiMock.generateObject.mockResolvedValue({ + object: { ...modelObject, verification_expression: "factorial(3)*factorial(4)" }, + }); + const agent = createGeneratorAgent({ + model: {} as LanguageModel, + modelId: "test-model", + promptId: "problem-generator", + prompts: promptLoader(vi.fn(() => "rendered prompt")), + }); + + const generated = await agent.generate({ + request, + intent, + refs: [], + strategy: null, + attempt: 1, + }); + + expect(generated.verification_expression).toBe("factorial(3)*factorial(4)"); + }); + + it("omits verification_expression when the LLM does not provide one", async () => { + aiMock.generateObject.mockResolvedValue({ object: modelObject }); + const agent = createGeneratorAgent({ + model: {} as LanguageModel, + modelId: "test-model", + promptId: "problem-generator", + prompts: promptLoader(vi.fn(() => "rendered prompt")), + }); + + const generated = await agent.generate({ + request, + intent, + refs: [], + strategy: null, + attempt: 1, + }); + + expect(generated.verification_expression).toBeUndefined(); + expect("verification_expression" in generated).toBe(false); + }); + it("immediately retries a schema failure once with a schema hint", async () => { const schemaError = new Error("question_text is required"); schemaError.name = "NoObjectGeneratedError"; diff --git a/packages/agent/tests/independent-resolve-choice-labels.test.ts b/packages/agent/tests/independent-resolve-choice-labels.test.ts index 944e021..fce4a3b 100644 --- a/packages/agent/tests/independent-resolve-choice-labels.test.ts +++ b/packages/agent/tests/independent-resolve-choice-labels.test.ts @@ -37,6 +37,7 @@ const mathEngine: MathEngineClient = { solve: async () => ({ solutions: [] }), verify: async () => ({ equivalent: true, diff: "0" }), simplify: async ({ expr }) => ({ simplified: expr.replace(/\s+/g, "") }), + evaluate: async () => ({ value: "", numeric: "" }), differentiate: async () => ({ derivative: "" }), limit: async () => ({ limit: "" }), }; diff --git a/packages/agent/tests/independent-resolve-geometry-function-equivalence.test.ts b/packages/agent/tests/independent-resolve-geometry-function-equivalence.test.ts index c662c78..51b2df5 100644 --- a/packages/agent/tests/independent-resolve-geometry-function-equivalence.test.ts +++ b/packages/agent/tests/independent-resolve-geometry-function-equivalence.test.ts @@ -37,6 +37,7 @@ const mathEngine: MathEngineClient = { solve: async () => ({ solutions: [] }), verify: async () => ({ equivalent: false, diff: "not checked" }), simplify: async ({ expr }) => ({ simplified: expr.replace(/\s+/g, "") }), + evaluate: async () => ({ value: "", numeric: "" }), differentiate: async () => ({ derivative: "" }), limit: async () => ({ limit: "" }), }; diff --git a/packages/agent/tests/independent-resolve-labels.test.ts b/packages/agent/tests/independent-resolve-labels.test.ts index e139852..8195084 100644 --- a/packages/agent/tests/independent-resolve-labels.test.ts +++ b/packages/agent/tests/independent-resolve-labels.test.ts @@ -37,6 +37,7 @@ const mathEngine: MathEngineClient = { solve: async () => ({ solutions: [] }), verify: async () => ({ equivalent: true, diff: "0" }), simplify: async ({ expr }) => ({ simplified: expr.replace(/\s+/g, "") }), + evaluate: async () => ({ value: "", numeric: "" }), differentiate: async () => ({ derivative: "" }), limit: async () => ({ limit: "" }), }; diff --git a/packages/agent/tests/independent-resolve-radicals-decimals.test.ts b/packages/agent/tests/independent-resolve-radicals-decimals.test.ts index 0aeab58..2489209 100644 --- a/packages/agent/tests/independent-resolve-radicals-decimals.test.ts +++ b/packages/agent/tests/independent-resolve-radicals-decimals.test.ts @@ -37,6 +37,7 @@ const mathEngine: MathEngineClient = { solve: async () => ({ solutions: [] }), verify: async () => ({ equivalent: true, diff: "0" }), simplify: async ({ expr }) => ({ simplified: expr.replace(/\s+/g, "") }), + evaluate: async () => ({ value: "", numeric: "" }), differentiate: async () => ({ derivative: "" }), limit: async () => ({ limit: "" }), }; diff --git a/packages/agent/tests/independent-resolve-text-equivalence.test.ts b/packages/agent/tests/independent-resolve-text-equivalence.test.ts index 1edff00..0d02893 100644 --- a/packages/agent/tests/independent-resolve-text-equivalence.test.ts +++ b/packages/agent/tests/independent-resolve-text-equivalence.test.ts @@ -37,6 +37,7 @@ const mathEngine: MathEngineClient = { solve: async () => ({ solutions: [] }), verify: async () => ({ equivalent: true, diff: "0" }), simplify: async ({ expr }) => ({ simplified: expr.replace(/\s+/g, "") }), + evaluate: async () => ({ value: "", numeric: "" }), differentiate: async () => ({ derivative: "" }), limit: async () => ({ limit: "" }), }; diff --git a/packages/agent/tests/independent-resolve.test.ts b/packages/agent/tests/independent-resolve.test.ts index 425ca3b..43af51a 100644 --- a/packages/agent/tests/independent-resolve.test.ts +++ b/packages/agent/tests/independent-resolve.test.ts @@ -38,6 +38,7 @@ const mathEngine: MathEngineClient = { solve: async () => ({ solutions: [] }), verify: async () => ({ equivalent: true, diff: "0" }), simplify: async ({ expr }) => ({ simplified: expr.replace(/\s+/g, "") }), + evaluate: async () => ({ value: "", numeric: "" }), differentiate: async () => ({ derivative: "" }), limit: async () => ({ limit: "" }), }; diff --git a/packages/agent/tests/math-engine-client.test.ts b/packages/agent/tests/math-engine-client.test.ts index 9f57abb..2aac7d1 100644 --- a/packages/agent/tests/math-engine-client.test.ts +++ b/packages/agent/tests/math-engine-client.test.ts @@ -78,6 +78,35 @@ describe("math-engine client request boundary", () => { const body = SolveBodySchema.parse(JSON.parse(receivedBodies[0] ?? "")); expect(body.equation).toBe("(x - 2)*(x + 7) = 3*x + 10"); }); + + it("posts verification expressions to /evaluate and parses the exact value", async () => { + const receivedPaths: string[] = []; + const receivedBodies: string[] = []; + const server = createServer(async (req: IncomingMessage, res: ServerResponse) => { + receivedPaths.push(req.url ?? ""); + receivedBodies.push(await readRequestBody(req)); + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ value: "864", numeric: "864.000000000000" })); + }); + servers.push(server); + await listen(server); + + const address = server.address(); + if (address === null || typeof address === "string") { + throw new Error("test server address must be a TCP address"); + } + const client = createMathEngineClient({ + baseUrl: `http://127.0.0.1:${address.port}`, + }); + + const result = await client.evaluate({ expr: "factorial(3)*factorial(3)*factorial(4)" }); + + expect(receivedPaths).toEqual(["/evaluate"]); + expect(JSON.parse(receivedBodies[0] ?? "")).toEqual({ + expr: "factorial(3)*factorial(3)*factorial(4)", + }); + expect(result).toEqual({ value: "864", numeric: "864.000000000000" }); + }); }); function listen(server: ReturnType): Promise { diff --git a/packages/agent/tests/problem-generation-geometry-statistics-guards.test.ts b/packages/agent/tests/problem-generation-geometry-statistics-guards.test.ts index 8bacf3e..43c9db3 100644 --- a/packages/agent/tests/problem-generation-geometry-statistics-guards.test.ts +++ b/packages/agent/tests/problem-generation-geometry-statistics-guards.test.ts @@ -39,6 +39,7 @@ const mathEngine: MathEngineClient = { solve: async () => ({ solutions: [] }), verify: async () => ({ equivalent: true, diff: "0" }), simplify: async ({ expr }) => ({ simplified: expr }), + evaluate: async () => ({ value: "", numeric: "" }), differentiate: async () => ({ derivative: "" }), limit: async () => ({ limit: "" }), }; diff --git a/packages/agent/tests/problem-generation-topic-guards.test.ts b/packages/agent/tests/problem-generation-topic-guards.test.ts index c644377..4a88174 100644 --- a/packages/agent/tests/problem-generation-topic-guards.test.ts +++ b/packages/agent/tests/problem-generation-topic-guards.test.ts @@ -45,6 +45,7 @@ const mathEngine: MathEngineClient = { solve: async () => ({ solutions: [] }), verify: async () => ({ equivalent: true, diff: "0" }), simplify: async ({ expr }) => ({ simplified: expr }), + evaluate: async () => ({ value: "", numeric: "" }), differentiate: async () => ({ derivative: "" }), limit: async () => ({ limit: "" }), }; diff --git a/packages/agent/tests/problem-generation.test.ts b/packages/agent/tests/problem-generation.test.ts index 5c02888..484e219 100644 --- a/packages/agent/tests/problem-generation.test.ts +++ b/packages/agent/tests/problem-generation.test.ts @@ -55,6 +55,7 @@ const mathEngine: MathEngineClient = { }, verify: async () => ({ equivalent: true, diff: "0" }), simplify: async ({ expr }) => ({ simplified: expr }), + evaluate: async () => ({ value: "", numeric: "" }), differentiate: async () => ({ derivative: "" }), limit: async () => ({ limit: "" }), }; diff --git a/packages/agent/tests/sympy-verification.test.ts b/packages/agent/tests/sympy-verification.test.ts index b18d980..be8e639 100644 --- a/packages/agent/tests/sympy-verification.test.ts +++ b/packages/agent/tests/sympy-verification.test.ts @@ -44,6 +44,7 @@ describe("verifyWithSympy", () => { } return { simplified: expr.replace(/\s+/g, "") }; }, + evaluate: async () => ({ value: "", numeric: "" }), differentiate: async () => ({ derivative: "" }), limit: async () => ({ limit: "" }), }; @@ -68,6 +69,7 @@ describe("verifyWithSympy", () => { if (/[가-힣]/u.test(expr)) throw new Error(`textual answer: ${expr}`); return { simplified: expr }; }, + evaluate: async () => ({ value: "", numeric: "" }), differentiate: async () => ({ derivative: "" }), limit: async () => ({ limit: "" }), }; @@ -115,6 +117,7 @@ describe("verifyWithSympy", () => { }, verify: async () => ({ equivalent: true, diff: "0" }), simplify: async ({ expr }) => ({ simplified: expr.replace(/\s+/g, "") }), + evaluate: async () => ({ value: "", numeric: "" }), differentiate: async () => ({ derivative: "" }), limit: async () => ({ limit: "" }), }; @@ -197,6 +200,159 @@ describe("verifyWithSympy", () => { }); }); +describe("verifyWithSympy expression check", () => { + const countingCandidate: GeneratedProblem = { + ...expressionCandidate, + generation_kind: "probability", + question_text: + "공격수 3명을 공격 자리에, 수비수 3명을 수비 자리에 배치하고 나머지 4명을 남은 자리에 배치하는 방법의 수는?", + expected_answer: "864", + verification_expression: "factorial(3)*factorial(3)*factorial(4)", + }; + + it("passes a counting candidate whose verification expression evaluates to the declared answer", async () => { + const result = await verifyWithSympy( + { mathEngine: createExpressionMathEngine({ value: "864" }) }, + { candidate: countingCandidate }, + ); + const gates = passedGates().map((gate) => + gate.step === "sympy_verify" ? result.gate : gate, + ); + + expect(result.gate.status).toBe("passed"); + expect(result.gate.evidence).toMatchObject({ + expression_check: true, + verification_expression: "factorial(3)*factorial(3)*factorial(4)", + sympy_answer: "864", + }); + expect(createAcceptancePolicy().decide(gates, 1)).toBe("verified"); + }); + + it("fails when the verification expression value mismatches the declared answer", async () => { + const result = await verifyWithSympy( + { mathEngine: createExpressionMathEngine({ value: "868" }) }, + { candidate: countingCandidate }, + ); + const gates = passedGates().map((gate) => + gate.step === "sympy_verify" ? result.gate : gate, + ); + + expect(result.gate.status).toBe("failed"); + expect(result.gate.failure_detail?.code).toBe("expression_value_mismatch"); + expect(createAcceptancePolicy().decide(gates, 1)).toBe("rejected"); + }); + + it("resolves multiple-choice declared answers to the choice body before comparing", async () => { + const result = await verifyWithSympy( + { mathEngine: createExpressionMathEngine({ value: "864" }) }, + { + candidate: { + ...countingCandidate, + expected_answer: "① 864", + expected_choices: ["① 864", "② 868", "③ 872", "④ 876", "⑤ 880"], + }, + }, + ); + + expect(result.gate.status).toBe("passed"); + expect(result.gate.evidence).toMatchObject({ expected_answer: "864", sympy_answer: "864" }); + }); + + it("marks evaluation engine failures as unverified, not failed", async () => { + const result = await verifyWithSympy( + { mathEngine: createExpressionMathEngine({ evaluateError: true }) }, + { candidate: countingCandidate }, + ); + + expect(result.gate.status).toBe("unverified"); + expect(result.gate.evidence).toMatchObject({ + reason: expect.stringContaining("could not evaluate the verification expression"), + }); + }); + + it("keeps candidates without a verification expression on the unverified fallback", async () => { + const result = await verifyWithSympy( + { mathEngine: createExpressionMathEngine({ value: "864" }) }, + { + candidate: { + ...countingCandidate, + verification_expression: undefined, + }, + }, + ); + + expect(result.gate.status).toBe("unverified"); + expect(result.gate.evidence).toMatchObject({ + reason: expect.stringContaining("requires a checkable equation"), + }); + }); + + it("falls back to expression verification for equation word problems without extractable equations", async () => { + const result = await verifyWithSympy( + { mathEngine: createExpressionMathEngine({ value: "6" }) }, + { + candidate: { + ...countingCandidate, + generation_kind: "equation", + question_text: "어떤 수의 두 배에 세 배를 곱하면 얼마인지 구하시오.", + expected_answer: "6", + verification_expression: "2*3", + }, + }, + ); + + expect(result.gate.status).toBe("passed"); + expect(result.gate.evidence).toMatchObject({ expression_check: true }); + }); + + it("degrades engine timeouts to unverified so acceptance keeps a warning verdict", async () => { + const hangingEngine: MathEngineClient = { + ...createExpressionMathEngine({ value: "" }), + solve: () => new Promise(() => {}), + }; + + const result = await verifyWithSympy( + { mathEngine: hangingEngine, perStepTimeoutMs: 10 }, + { + candidate: { + ...expressionCandidate, + generation_kind: "equation", + question_text: "다음 방정식을 풀어라. x - 2 = 0", + expected_answer: "2", + }, + }, + ); + const gates = passedGates().map((gate) => + gate.step === "sympy_verify" ? result.gate : gate, + ); + + expect(result.gate.status).toBe("unverified"); + expect(result.gate.failure_detail?.code).toBe("sympy_error"); + expect(createAcceptancePolicy().decide(gates, 1)).toBe("warning"); + }); +}); + +function createExpressionMathEngine(opts: { + readonly value?: string; + readonly evaluateError?: boolean; +}): MathEngineClient { + return { + health: async () => ({ status: "ok", engine: "sympy" }), + solve: async () => ({ solutions: [] }), + verify: async ({ expr1, expr2 }) => ({ + equivalent: expr1.replace(/\s+/g, "") === expr2.replace(/\s+/g, ""), + diff: "0", + }), + simplify: async ({ expr }) => ({ simplified: expr.replace(/\s+/g, "") }), + evaluate: async () => { + if (opts.evaluateError === true) throw new Error("evaluate exploded"); + return { value: opts.value ?? "", numeric: opts.value ?? "" }; + }, + differentiate: async () => ({ derivative: "" }), + limit: async () => ({ limit: "" }), + }; +} + function createEquationMathEngine(opts: { readonly solutions: string[] }): MathEngineClient { return { health: async () => ({ status: "ok", engine: "sympy" }), @@ -206,6 +362,7 @@ function createEquationMathEngine(opts: { readonly solutions: string[] }): MathE diff: "0", }), simplify: async ({ expr }) => ({ simplified: expr.replace(/\s+/g, "") }), + evaluate: async () => ({ value: "", numeric: "" }), differentiate: async () => ({ derivative: "" }), limit: async () => ({ limit: "" }), }; diff --git a/packages/agent/tests/verification-workflow-generate-failure.test.ts b/packages/agent/tests/verification-workflow-generate-failure.test.ts index 1833059..9bbaea4 100644 --- a/packages/agent/tests/verification-workflow-generate-failure.test.ts +++ b/packages/agent/tests/verification-workflow-generate-failure.test.ts @@ -97,6 +97,7 @@ const mathEngine: MathEngineClient = { solve: async () => ({ solutions: ["2", "3"] }), verify: async () => ({ equivalent: true, diff: "0" }), simplify: async ({ expr }) => ({ simplified: expr }), + evaluate: async () => ({ value: "", numeric: "" }), differentiate: async () => ({ derivative: "" }), limit: async () => ({ limit: "" }), }; diff --git a/packages/agent/tests/verification-workflow-last-resort.test.ts b/packages/agent/tests/verification-workflow-last-resort.test.ts index 6dbc7c3..e6cad11 100644 --- a/packages/agent/tests/verification-workflow-last-resort.test.ts +++ b/packages/agent/tests/verification-workflow-last-resort.test.ts @@ -98,6 +98,7 @@ const mathEngine: MathEngineClient = { solve: async () => ({ solutions: ["2", "3"] }), verify: async () => ({ equivalent: true, diff: "0" }), simplify: async ({ expr }) => ({ simplified: expr }), + evaluate: async () => ({ value: "", numeric: "" }), differentiate: async () => ({ derivative: "" }), limit: async () => ({ limit: "" }), }; diff --git a/packages/math-engine/src/routers/math.py b/packages/math-engine/src/routers/math.py index caaab35..dc0132d 100644 --- a/packages/math-engine/src/routers/math.py +++ b/packages/math-engine/src/routers/math.py @@ -542,6 +542,31 @@ async def simplify_expression(req: SimplifyRequest): return await _run_compute(_compute_simplify, req.expr) +class EvaluateRequest(BaseModel): + expr: str + + +class EvaluateResponse(BaseModel): + value: str + numeric: str + + +def _compute_evaluate(expr: str) -> dict[str, Any]: + parsed = parse_math(expr) + evaluated = simplify(parsed.doit() if hasattr(parsed, "doit") else parsed) + if not evaluated.is_number: + raise HTTPException( + status_code=422, + detail=f"Expression did not evaluate to a number: {evaluated}", + ) + return {"value": str(evaluated), "numeric": str(evaluated.evalf())} + + +@router.post("/evaluate", response_model=EvaluateResponse) +async def evaluate_expression(req: EvaluateRequest): + return await _run_compute(_compute_evaluate, req.expr) + + class DifferentiateRequest(BaseModel): expr: str variable: str = "x" diff --git a/packages/math-engine/tests/test_evaluate.py b/packages/math-engine/tests/test_evaluate.py new file mode 100644 index 0000000..bc6bd6d --- /dev/null +++ b/packages/math-engine/tests/test_evaluate.py @@ -0,0 +1,46 @@ +def test_evaluate_factorial_product(client): + response = client.post( + "/evaluate", json={"expr": "factorial(3)*factorial(3)*1*factorial(4)"} + ) + assert response.status_code == 200 + assert response.json()["value"] == "864" + + +def test_evaluate_binomial(client): + response = client.post("/evaluate", json={"expr": "binomial(10, 3)"}) + assert response.status_code == 200 + assert response.json()["value"] == "120" + + +def test_evaluate_permutation_as_factorial_ratio(client): + response = client.post( + "/evaluate", json={"expr": "factorial(5)/factorial(5-2)"} + ) + assert response.status_code == 200 + assert response.json()["value"] == "20" + + +def test_evaluate_keeps_exact_fraction(client): + response = client.post("/evaluate", json={"expr": "3/8"}) + assert response.status_code == 200 + body = response.json() + assert body["value"] == "3/8" + assert body["numeric"].startswith("0.375") + + +def test_evaluate_keeps_exact_sqrt(client): + response = client.post("/evaluate", json={"expr": "sqrt(8)"}) + assert response.status_code == 200 + assert response.json()["value"] == "2*sqrt(2)" + + +def test_evaluate_rejects_symbolic_expression(client): + response = client.post("/evaluate", json={"expr": "x + 1"}) + assert response.status_code == 422 + + +def test_evaluate_rejects_unparseable_expression(client): + # TokenError 류는 500, SyntaxError 류는 400. + # 클라이언트 계약은 "비정상 응답 → unverified". + response = client.post("/evaluate", json={"expr": "factorial("}) + assert response.status_code >= 400