feat: 검증식 평가로 비-equation kind 결정론 검증 (기호 검증 불가 해소)

packages/agent/prompts/independent-solver.md

@@ -1,14 +1,14 @@
 ---
 id: independent-solver
-version: 0.1.0
+version: 0.2.0
 model: gpt-4o
 temperature: 0.0
 max_tokens: 1500
 schema: SolveAttemptSchema
 variables:
   - candidate
 owner: 비할당
-updated: 2026-05-18
+updated: 2026-06-11
 ---
 
 # Role
@@ -35,6 +35,7 @@ updated: 2026-05-18
 - `derived_answer: string` (문제 유형에 맞는 최종 정답. 방정식 해는 `2, -5`, 확률은 `3/8`, 통계값은 `평균 12`, 기하는 `60도`처럼 간결히 쓴다)
 - `trace: string`
 - `confidence: "high" | "medium" | "low"`
+- `verification_expression: string` (선택) — 당신의 풀이가 정답에 도달하는 *계산식 하나*. SymPy 평가 가능 표기만: `factorial(4)`, `binomial(10, 3)`, `factorial(5)/factorial(5-2)`, `*`, `/`, `+`, `-`, `**`. 이 식의 값이 `derived_answer`와 일치해야 한다. 정답이 단일 수치/수식 값이 아니면 생략. `4!` 표기, 한글, 단위 금지.
 
 # Constraints
 

packages/agent/prompts/problem-generator.md

@@ -1,6 +1,6 @@
 ---
 id: problem-generator
-version: 0.1.2
+version: 0.2.0
 model: gpt-4o
 temperature: 0.35
 max_tokens: 2000
@@ -14,7 +14,7 @@ variables:
   - counterexample
   - schemaError
 owner: 비할당
-updated: 2026-06-10
+updated: 2026-06-11
 ---
 
 # Role
@@ -105,8 +105,9 @@ JSON으로만 응답.
 - `expected_answer`는 정답만 간결하게 쓴다. 방정식 해는 `2, 5`, 통계값은 `평균 12`, 확률은 `3/8`, 기하는 `60도`처럼 쓴다.
 - `techniques_used`는 실제 사용한 풀이 기법 id 배열이다. Strategy 어휘에 있는 `factoring`, `quadratic_formula`, `completing_the_square` 같은 snake_case id를 우선 사용하고, 공백·한글 설명 문장은 넣지 않는다.
 - structural 모드에서는 필수 기법을 모두 포함해야 한다. conceptual 모드에서는 필수·관련 기법 중 실제 사용한 것을 최소 1개 이상 포함한다.
-- 예시 출력: `{ "question_text": "다항식 (x + 3)(x - 5)를 전개하시오.", "expected_answer": "x**2 - 2*x - 15", "techniques_used": ["polynomial_expansion"], "proposed_solution_trace": "분배법칙으로 각 항을 곱해 동류항을 정리한다." }`
+- 예시 출력: `{ "question_text": "서로 다른 5권의 책 중 3권을 골라 일렬로 꽂는 방법의 수는?", "expected_answer": "60", "techniques_used": ["permutation"], "proposed_solution_trace": "5권 중 3권을 순서 있게 배열하므로 5P3 = 5*4*3 = 60이다.", "verification_expression": "factorial(5)/factorial(5-3)" }`
 - `proposed_solution_trace`에 풀이 단계와 출제 의도를 한국어로 명시하되, 수식도 JSON 안전한 plain text 표기만 사용한다.
+- `verification_expression`은 정답에 도달하는 *계산식 하나*다. SymPy가 평가할 수 있는 표기만 쓴다: 팩토리얼은 `factorial(4)`, 조합은 `binomial(10, 3)`, 순열은 `factorial(5)/factorial(5-2)`, 사칙연산은 `*`, `/`, `+`, `-`, 지수는 `**`. 이 식을 평가한 값이 `expected_answer`와 정확히 일치해야 한다. 예: 정답이 `864`이고 풀이가 3!*3!*4!이면 `"verification_expression": "factorial(3)*factorial(3)*factorial(4)"`. 정답이 단일 수치/수식 값이 아닐 때(서술형, 참/거짓 등)만 생략한다. `4!` 같은 느낌표 표기, 한글, 단위는 절대 넣지 않는다.
 - 결과 문제는 source problem과 달라야 하며, structural/conceptual 모드 차이가 풀이 설명에 드러나야 한다.
 
 # TODO

packages/agent/src/agents/generator-agent.ts

@@ -54,6 +54,13 @@ export const LlmGeneratedCandidateSchema = z.object({
     .string()
     .min(1)
     .describe("Korean solution trace explaining the structural/conceptual transform"),
+  verification_expression: z
+    .string()
+    .min(1)
+    .optional()
+    .describe(
+      "SymPy-evaluable arithmetic expression whose value equals expected_answer, e.g. factorial(3)*factorial(3)*factorial(4). Omit only when the answer is not a single numeric/symbolic value",
+    ),
 });
 
 export type LlmGeneratedCandidate = z.infer<typeof LlmGeneratedCandidateSchema>;
@@ -79,6 +86,9 @@ export function assembleGeneratedProblem(input: {
     expected_answer: input.object.expected_answer,
     techniques_used: input.object.techniques_used ?? [],
     proposed_solution_trace: input.object.proposed_solution_trace,
+    ...(input.object.verification_expression === undefined
+      ? {}
+      : { verification_expression: input.object.verification_expression }),
     source_refs: input.refs.map((ref) => ref.item_id),
     inferred_intent: input.intent,
     generation_metadata: {

packages/agent/src/schemas/generated-problem.schema.ts

@@ -40,6 +40,10 @@ export const GeneratedProblemSchema = z.object({
   expected_choices: z.array(z.string()).optional(), // objective일 때
   techniques_used: z.array(z.string()).default([]),
   proposed_solution_trace: z.string(),
+  /** 정답에 도달하는 SymPy 평가식 (예: "factorial(3)*factorial(3)*factorial(4)").
+   *  sympy_verify가 math-engine /evaluate로 평가해 expected_answer와 대조한다 —
+   *  equation이 아닌 kind(경우의 수 등)도 결정론 검증이 가능해진다. */
+  verification_expression: z.string().min(1).optional(),
 
   source_refs: z.array(z.string()), // SourceProblem.item_id[]
   inferred_intent: IntentSchema,

packages/agent/src/schemas/solve-attempt.schema.ts

@@ -6,6 +6,8 @@ export const SolveAttemptSchema = z.object({
   derived_answer: z.string().min(1),
   trace: z.string().min(1),
   confidence: z.enum(["high", "medium", "low"]),
+  /** 재풀이가 도달한 정답의 SymPy 평가식 — 생성기 식과 독립인 두 번째 결정론 대조용. */
+  verification_expression: z.string().min(1).optional(),
 });
 
 export type SolveAttempt = z.infer<typeof SolveAttemptSchema>;

packages/agent/src/server/sse/wire-adapter.ts

@@ -123,6 +123,9 @@ function sympySummary(
     return `기호 검증 불가${kindLabel} — 독립 재풀이로 확인`;
   }
   const sympyAnswer = asString(evidence["sympy_answer"]);
+  if (evidence["expression_check"] === true) {
+    return sympyAnswer === null ? "검증식 평가 일치" : `검증식 평가 일치 (값: ${sympyAnswer})`;
+  }
   if (sympyAnswer === null) return "SymPy 검산 일치";
   return `SymPy 검산 일치 (해: ${sympyAnswer})`;
 }

packages/agent/src/steps/sympy-verification.ts

@@ -41,10 +41,6 @@ type VerificationCheck = {
   readonly evidence?: Record<string, unknown>;
 };
 
-type SymbolicCheckability =
-  | { readonly checkable: true }
-  | { readonly checkable: false; readonly reason: string; readonly evidence?: Record<string, unknown> };
-
 const NO_EXTRACTABLE_EQUATION_REASON =
   "Question text does not contain an extractable equation for SymPy";
 const NO_EXTRACTABLE_EQUATION_EVIDENCE: Record<string, unknown> = {
@@ -71,15 +67,18 @@ export async function verifyWithSympy(
       },
     };
   } catch (err) {
+    // 엔진 장애/타임아웃은 수학적 판정이 아니다 — failed로 두면 acceptance가
+    // 인프라 장애를 "틀린 문제"로 reject한다. unverified로 강등해 warning에 머문다.
+    const message = err instanceof Error ? err.message : String(err);
     return {
       gate: {
         step: "sympy_verify",
-        status: "failed",
+        status: "unverified",
         duration_ms: Date.now() - started,
-        evidence: { engine: "sympy" },
+        evidence: { engine: "sympy", reason: `math-engine error: ${message}` },
         failure_detail: {
           code: "sympy_error",
-          message: err instanceof Error ? err.message : String(err),
+          message,
         },
       },
     };
@@ -90,52 +89,121 @@ async function verifyCandidate(
   mathEngine: MathEngineClient,
   candidate: GeneratedProblem,
 ): Promise<VerificationCheck> {
-  const checkability = classifySymbolicCheckability(candidate);
-  if (!checkability.checkable) {
-    return unverifiedCheck(candidate, checkability.reason, checkability.evidence);
-  }
-
   if (candidate.generation_kind === "equation") {
-    if (isChoiceStyleCandidate(candidate)) {
-      return verifyChoiceCandidate(mathEngine, candidate);
+    const equation = extractEquationText(candidate.question_text);
+    if (equation !== null) {
+      if (isChoiceStyleCandidate(candidate)) {
+        if (resolveChoiceOptions(candidate).length === 0) {
+          return unverifiedCheck(
+            candidate,
+            "Multiple-choice equation has no parseable expected_choices/options",
+            { equation },
+          );
+        }
+        return verifyChoiceCandidate(mathEngine, candidate);
+      }
+      return verifyEquationCandidate(mathEngine, candidate);
     }
-    return verifyEquationCandidate(mathEngine, candidate);
+    // 식 추출 실패 (서술형 방정식 문제 등) — 검증식이 있으면 평가 경로로 폴백.
+    const byExpression = await verifyByExpression(mathEngine, candidate);
+    if (byExpression !== null) return byExpression;
+    return unverifiedCheck(
+      candidate,
+      NO_EXTRACTABLE_EQUATION_REASON,
+      NO_EXTRACTABLE_EQUATION_EVIDENCE,
+    );
   }
 
+  const byExpression = await verifyByExpression(mathEngine, candidate);
+  if (byExpression !== null) return byExpression;
   return unverifiedCheck(
     candidate,
-    `No deterministic SymPy verifier is implemented for generation_kind=${candidate.generation_kind}`,
+    "SymPy verification requires a checkable equation; non-equation candidates rely on independent re-solve",
+    { generation_kind: candidate.generation_kind },
   );
 }
 
-function classifySymbolicCheckability(candidate: GeneratedProblem): SymbolicCheckability {
-  if (candidate.generation_kind !== "equation") {
-    return {
-      checkable: false,
-      reason:
-        "SymPy verification requires a checkable equation; non-equation candidates rely on independent re-solve",
-      evidence: { generation_kind: candidate.generation_kind },
-    };
+/** 후보의 verification_expression을 /evaluate로 평가해 선언 정답과 대조한다.
+ *  식이 없으면 null (호출자가 기존 unverified 폴백 유지). 평가 실패는 unverified —
+ *  식은 LLM 산출물이라 구문 오류가 수학 오류를 뜻하지 않는다. */
+async function verifyByExpression(
+  mathEngine: MathEngineClient,
+  candidate: GeneratedProblem,
+): Promise<VerificationCheck | null> {
+  const expression = candidate.verification_expression;
+  if (expression === undefined) return null;
+
+  const declared = declaredAnswerBody(candidate);
+  if (declared === null) {
+    return unverifiedCheck(
+      candidate,
+      "Declared expected_answer does not identify one of the provided choices",
+      { verification_expression: expression, expected_answer: candidate.expected_answer },
+    );
   }
 
-  const equation = extractEquationText(candidate.question_text);
-  if (equation === null) {
-    return {
-      checkable: false,
-      reason: NO_EXTRACTABLE_EQUATION_REASON,
-      evidence: NO_EXTRACTABLE_EQUATION_EVIDENCE,
-    };
+  let evaluatedValue: string;
+  try {
+    evaluatedValue = (await mathEngine.evaluate({ expr: expression })).value;
+  } catch (err) {
+    return unverifiedCheck(
+      candidate,
+      `math-engine could not evaluate the verification expression: ${err instanceof Error ? err.message : String(err)}`,
+      { verification_expression: expression },
+    );
   }
 
-  if (isChoiceStyleCandidate(candidate) && resolveChoiceOptions(candidate).length === 0) {
+  const decision = await decideAnswerEquivalence(mathEngine, declared, evaluatedValue);
+  if (decision.status === "undecidable") {
+    return unverifiedCheck(
+      candidate,
+      `Could not compare declared answer with evaluated verification expression: ${decision.reason ?? "undecidable"}`,
+      { expression_check: true, verification_expression: expression, evaluated_value: evaluatedValue },
+      declared,
+      evaluatedValue,
+    );
+  }
+  if (decision.status === "not_equivalent") {
     return {
-      checkable: false,
-      reason: "Multiple-choice equation has no parseable expected_choices/options",
-      evidence: { equation },
+      status: "failed",
+      verificationKind: candidate.generation_kind,
+      expectedAnswer: declared,
+      sympyAnswer: evaluatedValue,
+      failureCode: "expression_value_mismatch",
+      failureMessage: `Verification expression evaluates to ${evaluatedValue}, which does not match the declared answer ${declared}`,
+      evidence: {
+        expression_check: true,
+        verification_expression: expression,
+        evaluated_value: evaluatedValue,
+        equivalence: decision,
+      },
     };
   }
+  return {
+    status: "passed",
+    verificationKind: candidate.generation_kind,
+    expectedAnswer: declared,
+    sympyAnswer: evaluatedValue,
+    evidence: {
+      expression_check: true,
+      verification_expression: expression,
+      evaluated_value: evaluatedValue,
+    },
+  };
+}
 
-  return { checkable: true };
+/** 검증식 대조 대상이 되는 선언 정답 본문. 객관식이면 정답 보기의 본문,
+ *  아니면 choice prefix(`① ` 등)를 벗긴 expected_answer. */
+function declaredAnswerBody(candidate: GeneratedProblem): string | null {
+  if (isChoiceStyleCandidate(candidate)) {
+    const choices = resolveChoiceOptions(candidate);
+    if (choices.length > 0) {
+      const correct = selectDeclaredCorrectChoice(candidate, choices);
+      return correct === null ? null : correct.body;
+    }
+  }
+  const stripped = stripChoicePrefix(candidate.expected_answer);
+  return stripped.length > 0 ? stripped : candidate.expected_answer;
 }
 
 async function verifyEquationCandidate(

packages/agent/src/tools/math-engine-client.ts

@@ -32,6 +32,16 @@ export interface SimplifyResponse {
   simplified: string;
 }
 
+export interface EvaluateRequest {
+  expr: string;
+}
+export interface EvaluateResponse {
+  /** 정확값 (SymPy canonical 문자열, 예: "864", "3/8", "2*sqrt(2)") */
+  value: string;
+  /** 십진 근사값 (evalf) */
+  numeric: string;
+}
+
 export interface DifferentiateRequest {
   expr: string;
   variable?: string;
@@ -59,6 +69,7 @@ export interface MathEngineClient {
   solve(req: SolveRequest): Promise<SolveResponse>;
   verify(req: VerifyRequest): Promise<VerifyResponse>;
   simplify(req: SimplifyRequest): Promise<SimplifyResponse>;
+  evaluate(req: EvaluateRequest): Promise<EvaluateResponse>;
   differentiate(req: DifferentiateRequest): Promise<DifferentiateResponse>;
   limit(req: LimitRequest): Promise<LimitResponse>;
 }
@@ -134,6 +145,8 @@ export function createMathEngineClient(
       ),
     simplify: (req) =>
       post("/simplify", { expr: toSympyInput(req.expr) }, parseSimplifyResponse),
+    evaluate: (req) =>
+      post("/evaluate", { expr: toSympyInput(req.expr) }, parseEvaluateResponse),
     differentiate: (req) =>
       post(
         "/differentiate",
@@ -231,6 +244,11 @@ function parseSimplifyResponse(value: unknown): SimplifyResponse {
   return { simplified: readString(obj, "simplified") };
 }
 
+function parseEvaluateResponse(value: unknown): EvaluateResponse {
+  const obj = asObject(value);
+  return { value: readString(obj, "value"), numeric: readString(obj, "numeric") };
+}
+
 function parseDifferentiateResponse(value: unknown): DifferentiateResponse {
   const obj = asObject(value);
   return { derivative: readString(obj, "derivative") };

packages/agent/tests/deterministic-fallback.test.ts

@@ -95,6 +95,7 @@ const mathEngine: MathEngineClient = {
   },
   verify: async () => ({ equivalent: true, diff: "0" }),
   simplify: async ({ expr }) => ({ simplified: expr }),
+  evaluate: async () => ({ value: "", numeric: "" }),
   differentiate: async () => ({ derivative: "" }),
   limit: async () => ({ limit: "" }),
 };

packages/agent/tests/generator-agent.test.ts

@@ -89,6 +89,49 @@ describe("createGeneratorAgent", () => {
     );
   });
 
+  it("passes the LLM verification expression through to the generated problem", async () => {
+    aiMock.generateObject.mockResolvedValue({
+      object: { ...modelObject, verification_expression: "factorial(3)*factorial(4)" },
+    });
+    const agent = createGeneratorAgent({
+      model: {} as LanguageModel,
+      modelId: "test-model",
+      promptId: "problem-generator",
+      prompts: promptLoader(vi.fn(() => "rendered prompt")),
+    });
+
+    const generated = await agent.generate({
+      request,
+      intent,
+      refs: [],
+      strategy: null,
+      attempt: 1,
+    });
+
+    expect(generated.verification_expression).toBe("factorial(3)*factorial(4)");
+  });
+
+  it("omits verification_expression when the LLM does not provide one", async () => {
+    aiMock.generateObject.mockResolvedValue({ object: modelObject });
+    const agent = createGeneratorAgent({
+      model: {} as LanguageModel,
+      modelId: "test-model",
+      promptId: "problem-generator",
+      prompts: promptLoader(vi.fn(() => "rendered prompt")),
+    });
+
+    const generated = await agent.generate({
+      request,
+      intent,
+      refs: [],
+      strategy: null,
+      attempt: 1,
+    });
+
+    expect(generated.verification_expression).toBeUndefined();
+    expect("verification_expression" in generated).toBe(false);
+  });
+
   it("immediately retries a schema failure once with a schema hint", async () => {
     const schemaError = new Error("question_text is required");
     schemaError.name = "NoObjectGeneratedError";