diff --git a/package.json b/package.json index b208a21..0386cc4 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "test:python": "cd packages/math-engine && uv run pytest", "test": "pnpm test:node && pnpm test:python", "typecheck": "pnpm -F @openmath/agent typecheck && pnpm -F @openmath/web typecheck", + "sweep:generate": "node scripts/sweep-generate.mjs", "lint:python": "cd packages/math-engine && uv run ruff check .", "prepare": "husky" }, diff --git a/packages/agent/prompts/constraint-critic.md b/packages/agent/prompts/constraint-critic.md index 2524140..c827630 100644 --- a/packages/agent/prompts/constraint-critic.md +++ b/packages/agent/prompts/constraint-critic.md @@ -39,6 +39,8 @@ updated: 2026-05-18 `Critique` 형식 JSON: - `passes: boolean` - `hints: string[]` — 통과 못 했다면 Refiner가 반영할 *구체적* 힌트 +- JSON 문자열 안에 backslash를 쓰지 말 것. `\sqrt`, `\(`, `\frac` 같은 LaTeX 예시를 쓰면 안 된다. +- `hints`는 plain Korean 문장으로만 써라. 수식 예시가 필요하면 "제곱근 표기를 JSON 안전한 plain text로 쓰라"처럼 말로 설명하라. # Constraints diff --git a/packages/agent/prompts/independent-solver.md b/packages/agent/prompts/independent-solver.md index 26c8c12..4928954 100644 --- a/packages/agent/prompts/independent-solver.md +++ b/packages/agent/prompts/independent-solver.md @@ -19,6 +19,10 @@ updated: 2026-05-18 {{candidate.question_text}} +# Generation Kind + +{{candidate.generation_kind}} + # Task 1. 풀이 과정을 단계별로 기술 @@ -28,7 +32,7 @@ updated: 2026-05-18 # Output `SolveAttempt` JSON: -- `derived_answer: string` (LaTeX) +- `derived_answer: string` (문제 유형에 맞는 최종 정답. 방정식 해는 `2, -5`, 확률은 `3/8`, 통계값은 `평균 12`, 기하는 `60도`처럼 간결히 쓴다) - `trace: string` - `confidence: "high" | "medium" | "low"` @@ -36,4 +40,5 @@ updated: 2026-05-18 - 원본 풀이를 *모른다* 고 가정. 새로 풀어내는 것이 핵심 - 도구 (계산기) 없이 손풀이 추론으로 -- 정답 판단은 시스템(SymPy) 이 한다. 당신은 풀이 trace만 제공 (D-1) +- 정답 판단은 시스템 검증기가 한다. 당신은 풀이 trace만 제공 (D-1) +- JSON 문자열 안에서 raw backslash를 쓰지 말 것. `\(`, `\sqrt`, `\cdot` 같은 LaTeX 명령 대신 `sqrt(7)`, `5*x` 표기를 사용 diff --git a/packages/agent/prompts/problem-generator.md b/packages/agent/prompts/problem-generator.md index 57a74f1..4a77f44 100644 --- a/packages/agent/prompts/problem-generator.md +++ b/packages/agent/prompts/problem-generator.md @@ -23,6 +23,26 @@ updated: 2026-05-18 `{{request.mode}}` — `structural` 이면 같은 풀이법, 다른 숫자/표현. `conceptual` 이면 같은 학습 목표·평가 차원, 다른 풀이 경로. +# Generation Kind + +`{{generationKind}}` + +- `equation`: 등식의 해를 구하는 문제. `question_text`는 SymPy가 풀 수 있는 식을 포함해도 된다. +- `inequality`: 부등식의 해 또는 해집합을 구하는 문제. 등식으로 바꾸지 말 것. +- `system`: 미지수가 둘 이상인 연립방정식 문제. 단일 `x` 방정식으로 축소하지 말 것. +- `expression`: 식의 값, 식의 계산, 소인수분해, 근호 계산, 인수분해처럼 식을 변형하거나 계산하는 문제. 해를 구하는 방정식으로 바꾸지 말 것. + 유리수와 순환소수 단원에서는 점을 찍어 순환마디를 고르는 객관식 문제를 피하고, 순환소수를 분수로 바꾸거나 순환마디/소수 전개를 plain text로 답하게 할 것. +- `function`: 함수값, 그래프의 성질, 일차/이차함수 식 찾기 문제. 단순 방정식 풀이로 바꾸지 말 것. + 여러 명제의 참/거짓을 모두 고르는 문제는 만들지 말 것. 함수값, 기울기, 절편, 그래프 위의 한 점, 함수식 중 하나를 단일 정답으로 묻는 문제를 우선한다. + 일차함수 `y = a*x + b`에서 `a > 0`이면 `x`가 증가할 때 `y`도 증가하고, `a < 0`이면 `y`는 감소한다. + 이차함수 그래프 방향은 `위로 열린다`, `아래로 열린다`로 쓰고 `위로 볼록`, `아래로 볼록` 표현은 쓰지 말 것. + `y = a*x**2 + ...`에서 `a > 0`이면 `위로 열린다`, `a < 0`이면 `아래로 열린다`. +- `geometry`: 도형, 삼각비, 원의 성질을 이용하는 문제. 필요한 길이/각/비를 구하게 하되 단원 맥락을 유지할 것. + 보이지 않는 그림에 의존하는 복잡한 배치, 내접사각형+접선+중심각처럼 성질이 여러 개 겹치는 새 상황은 피할 것. + 그림 없이도 텍스트만으로 결정되는 표준 관계 하나를 묻고, 정답 산출에 필요한 길이·각 조건을 모두 명시할 것. +- `probability`: 경우의 수나 확률을 구하는 문제. +- `statistics`: 자료의 평균, 중앙값, 최빈값, 분산, 표준편차, 도수분포 해석 문제. + # Selected Source Problem 이 문제를 반드시 변형의 기준으로 삼아라: `{{request.source_problem_text}}` @@ -57,12 +77,13 @@ updated: 2026-05-18 # Output JSON으로만 응답. -- `question_text`는 KaTeX가 바로 렌더링할 수 있는 LaTeX `x` 방정식 문자열이어야 한다. `**`, `*`를 쓰지 말고 지수는 `x^{2}`, 곱셈은 `5x`, 제곱근은 `\\sqrt{7}`처럼 쓴다. -- `expected_answer`도 LaTeX로 쓴다. 예: `2, 5`, `\\sqrt{7}, -\\sqrt{7}`. -- 예시 출력: `{ "question_text": "x^{2} - 7x + 10 = 0", "expected_answer": "2, 5", "proposed_solution_trace": "인수분해하면 (x - 2)(x - 5) = 0이므로 x = 2 또는 x = 5이다." }` -- 서버가 SymPy로 다시 계산해 최종 정답을 검증한다. -- `proposed_solution_trace`에 풀이 단계와 출제 의도를 한국어로 명시한다. -- 풀이 가능하고 답이 실수인 중학교 수준 방정식만 생성한다. +- JSON 문자열 안에서 raw backslash를 쓰지 말 것. `\(`, `\sqrt`, `\cdot` 같은 LaTeX 명령을 쓰면 안 된다. +- `question_text`는 `{{generationKind}}` 유형에 맞는 문제 문장이어야 한다. +- 수식은 JSON 안전한 plain text로 쓴다. 지수는 `x**2`, 곱셈은 `5*x`, 제곱근은 `sqrt(7)`처럼 쓴다. +- 방정식 단원이 아닌데 해를 구하는 `x` 방정식으로 바꾸면 안 된다. +- `expected_answer`는 정답만 간결하게 쓴다. 방정식 해는 `2, 5`, 통계값은 `평균 12`, 확률은 `3/8`, 기하는 `60도`처럼 쓴다. +- 예시 출력: `{ "question_text": "다항식 (x + 3)(x - 5)를 전개하시오.", "expected_answer": "x**2 - 2*x - 15", "proposed_solution_trace": "분배법칙으로 각 항을 곱해 동류항을 정리한다." }` +- `proposed_solution_trace`에 풀이 단계와 출제 의도를 한국어로 명시하되, 수식도 JSON 안전한 plain text 표기만 사용한다. - 결과 문제는 source problem과 달라야 하며, structural/conceptual 모드 차이가 풀이 설명에 드러나야 한다. # TODO diff --git a/packages/agent/scripts/judge-quality.ts b/packages/agent/scripts/judge-quality.ts new file mode 100644 index 0000000..b105eef --- /dev/null +++ b/packages/agent/scripts/judge-quality.ts @@ -0,0 +1,222 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { generateObject } from "ai"; +import { z } from "zod"; + +import { DEFAULT_MODELS } from "../src/config/models.js"; +import { loadEnv } from "../src/config/env.js"; +import { resolveLanguageModel } from "../src/tools/llm-provider.js"; + +type Mode = "structural" | "conceptual"; +type Status = "PASS" | "WARN" | "FAIL"; + +type SweepRow = { + readonly grade: number; + readonly code: string; + readonly name: string; + readonly question: string; + readonly answer: string; + readonly status: string; +}; + +type Topic = { + readonly code: string; + readonly achievement: string; +}; + +type JudgeRow = SweepRow & { + readonly mode: Mode; + readonly status: Status; + readonly scores: { + readonly math_correctness: number; + readonly topic_fit: number; + readonly isomorphism_fit: number; + readonly clarity: number; + }; + readonly reasons: readonly string[]; + readonly suggested_revision: string | null; +}; + +const JudgeSchema = z.object({ + status: z.enum(["PASS", "WARN", "FAIL"]), + scores: z.object({ + math_correctness: z.number().int().min(1).max(5), + topic_fit: z.number().int().min(1).max(5), + isomorphism_fit: z.number().int().min(1).max(5), + clarity: z.number().int().min(1).max(5), + }), + reasons: z.array(z.string()).max(4), + suggested_revision: z.string().nullable(), +}); + +const rootDir = dirname(dirname(dirname(dirname(fileURLToPath(import.meta.url))))); +const args = parseArgs(process.argv.slice(2)); +const topics = await loadTopics(); +const env = loadEnv(); +const kind = env.LLM_PROVIDER === "cliproxy" ? "openai-compatible" : env.LLM_PROVIDER; +const modelId = env.LLM_MODEL ?? env.CLIPROXY_MODEL ?? env.OPENAI_MODEL ?? DEFAULT_MODELS.generator; +const model = resolveLanguageModel({ + kind, + modelId, + baseUrl: env.LLM_BASE_URL ?? env.CLIPROXY_BASE_URL, + apiKey: env.LLM_API_KEY ?? env.CLIPROXY_API_KEY ?? env.OPENAI_API_KEY ?? "openmath-local", + allowedHosts: ["localhost", "127.0.0.1"], +}); + +const rows = [ + ...await loadSweep(args.structural, "structural"), + ...await loadSweep(args.conceptual, "conceptual"), +]; +const judged = await runPool(rows, args.concurrency, (row) => judge(row, topics)); +const report = { + generated_at: new Date().toISOString(), + structural_report: args.structural, + conceptual_report: args.conceptual, + total: judged.length, + pass: judged.filter((row) => row.status === "PASS").length, + warn: judged.filter((row) => row.status === "WARN").length, + fail: judged.filter((row) => row.status === "FAIL").length, + rows: judged, +}; + +const stamp = report.generated_at.replaceAll(":", "").replaceAll(".", ""); +const outDir = join(rootDir, "reports/quality"); +await mkdir(outDir, { recursive: true }); +const jsonPath = join(outDir, `quality-judge-${stamp}.json`); +const mdPath = join(outDir, `quality-judge-${stamp}.md`); +await writeFile(jsonPath, `${JSON.stringify(report, null, 2)}\n`); +await writeFile(mdPath, renderMarkdown(report.rows)); +console.log(`JUDGE PASS ${report.pass}/${report.total} WARN ${report.warn} FAIL ${report.fail}`); +console.log(mdPath); +console.log(jsonPath); +if (report.fail > 0) process.exitCode = 1; + +function parseArgs(argv: readonly string[]): { readonly structural: string; readonly conceptual: string; readonly concurrency: number } { + let structural = ""; + let conceptual = ""; + let concurrency = 3; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + const next = argv[index + 1]; + if (arg === "--structural" && next !== undefined) { + structural = resolve(rootDir, next); + index += 1; + } else if (arg === "--conceptual" && next !== undefined) { + conceptual = resolve(rootDir, next); + index += 1; + } else if (arg === "--concurrency" && next !== undefined) { + concurrency = Number.parseInt(next, 10); + index += 1; + } else { + throw new Error(`Unknown argument: ${arg ?? ""}`); + } + } + if (structural.length === 0 || conceptual.length === 0) throw new Error("--structural and --conceptual are required"); + if (!Number.isSafeInteger(concurrency) || concurrency < 1 || concurrency > 8) throw new Error("--concurrency must be between 1 and 8"); + return { structural, conceptual, concurrency }; +} + +async function loadTopics(): Promise> { + const source = await readFile(join(rootDir, "packages/web/app/app/new/topic/data.ts"), "utf8"); + const pattern = /\{\s*grade:\s*(?:1|2|3),\s*category:\s*"[^"]+",\s*code:\s*"([^"]+)",\s*name:\s*"[^"]+",\s*achievement:\s*"([^"]+)",\s*\}/gms; + return new Map(Array.from(source.matchAll(pattern)).map((match) => [ + match[1] ?? "", + { code: match[1] ?? "", achievement: (match[2] ?? "").replace(/\s+/g, " ").trim() }, + ])); +} + +async function loadSweep(path: string, mode: Mode): Promise { + const parsed = JSON.parse(await readFile(path, "utf8")) as { readonly rows?: readonly SweepRow[] }; + const rows = parsed.rows ?? []; + if (rows.length !== 30) throw new Error(`${path} must contain 30 rows`); + return rows.map((row) => ({ ...row, mode })); +} + +async function judge(row: SweepRow & { readonly mode: Mode }, topics: ReadonlyMap): Promise { + const topic = topics.get(row.code); + for (let attempt = 1; attempt <= 2; attempt += 1) { + try { + const { object } = await generateObject({ + model, + schema: JudgeSchema, + mode: "json", + temperature: 0, + prompt: renderPrompt(row, topic?.achievement ?? "", attempt), + }); + return { ...row, ...object }; + } catch (error) { + if (attempt === 2) { + return fallbackJudge(row, error); + } + } + } + return fallbackJudge(row, new Error("unreachable judge retry state")); +} + +function renderPrompt(row: SweepRow & { readonly mode: Mode }, achievement: string, attempt: number): string { + return [ + "너는 한국 중학교 수학 출제 품질을 엄격히 검수하는 교사다.", + "문항을 PASS/WARN/FAIL로 판정하라.", + "PASS는 바로 학생에게 제공 가능, WARN은 사용 가능하지만 개선 권장, FAIL은 오답/단원 불일치/동형성 실패/애매한 문항이다.", + "구조동형은 풀이 구조가 같은지, 개념동형은 같은 성취기준 안에서 상황/표현/풀이 관점이 충분히 바뀌었는지 평가하라.", + "자동 검증은 이미 통과했으므로, 교육 품질과 문장 명확성에 집중하라.", + "JSON 문자열에는 백슬래시, LaTeX, \\( \\), \\sqrt 같은 표기를 절대 쓰지 말고 plain Korean text만 써라.", + attempt === 1 ? "" : "이전 응답은 JSON 파싱에 실패했다. 이번에는 reasons와 suggested_revision에 백슬래시를 절대 넣지 마라.", + "", + `학년: ${row.grade}`, + `단원: ${row.code} ${row.name}`, + `성취기준: ${achievement}`, + `모드: ${row.mode}`, + `문제: ${row.question}`, + `정답: ${row.answer}`, + ].join("\n"); +} + +function fallbackJudge(row: SweepRow & { readonly mode: Mode }, error: unknown): JudgeRow { + const message = error instanceof Error ? error.message : String(error); + return { + ...row, + status: "WARN", + scores: { math_correctness: 3, topic_fit: 3, isomorphism_fit: 3, clarity: 3 }, + reasons: [`LLM judge parse failed: ${message.slice(0, 120)}`], + suggested_revision: "Judge 응답 파싱 실패. 사람이 직접 확인해야 합니다.", + }; +} + +async function runPool(items: readonly T[], limit: number, worker: (item: T) => Promise): Promise { + const results: R[] = []; + let cursor = 0; + async function runNext(): Promise { + while (cursor < items.length) { + const current = cursor; + cursor += 1; + results[current] = await worker(items[current] as T); + } + } + await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => runNext())); + return results; +} + +function renderMarkdown(rows: readonly JudgeRow[]): string { + const lines = [ + "# OpenMath LLM Quality Judge", + "", + "| Grade | Code | Topic | Mode | Status | Scores | Reasons | Suggested Revision | Question | Answer |", + "|---:|---|---|---|---|---|---|---|---|---|", + ]; + for (const row of rows) { + lines.push(`| ${row.grade} | ${row.code} | ${escapeMd(row.name)} | ${row.mode} | ${row.status} | ${scoreText(row)} | ${escapeMd(row.reasons.join("; "))} | ${escapeMd(row.suggested_revision ?? "")} | ${escapeMd(row.question)} | ${escapeMd(row.answer)} |`); + } + return `${lines.join("\n")}\n`; +} + +function scoreText(row: JudgeRow): string { + const s = row.scores; + return `math ${s.math_correctness}, topic ${s.topic_fit}, iso ${s.isomorphism_fit}, clarity ${s.clarity}`; +} + +function escapeMd(value: string): string { + return value.replaceAll("|", "\\|").replaceAll("\n", " ").slice(0, 260); +} diff --git a/packages/agent/src/agents/constraint-critic-agent.ts b/packages/agent/src/agents/constraint-critic-agent.ts index 1a97975..8fe08ea 100644 --- a/packages/agent/src/agents/constraint-critic-agent.ts +++ b/packages/agent/src/agents/constraint-critic-agent.ts @@ -43,14 +43,39 @@ export function createConstraintCriticAgent( intent: input.intent, strategy: input.strategy === null ? "" : JSON.stringify(input.strategy, null, 2), }); - const { object } = await generateObject({ - model: deps.model, - schema: CritiqueSchema, - mode: "json", - temperature: prompt.metadata.temperature, - prompt: rendered, - }); - return object; + try { + const { object } = await generateObject({ + model: deps.model, + schema: CritiqueSchema, + mode: "json", + temperature: prompt.metadata.temperature, + prompt: rendered, + }); + return object; + } catch (error) { + const text = modelTextFromError(error); + if (text !== null) { + return parseCritiqueJson(text); + } + throw error; + } }, }; } + +export function parseCritiqueJson(text: string): Critique { + return CritiqueSchema.parse(JSON.parse(escapeRawBackslashes(text))); +} + +function escapeRawBackslashes(text: string): string { + return text.replace(/\\(?!["\\/bfnrtu])/g, "\\\\"); +} + +function modelTextFromError(error: unknown): string | null { + if (!hasTextProperty(error)) return null; + return typeof error.text === "string" ? error.text : null; +} + +function hasTextProperty(value: unknown): value is { readonly text?: unknown } { + return typeof value === "object" && value !== null && "text" in value; +} diff --git a/packages/agent/src/agents/generator-agent.ts b/packages/agent/src/agents/generator-agent.ts index 7a43a6b..ccb50ee 100644 --- a/packages/agent/src/agents/generator-agent.ts +++ b/packages/agent/src/agents/generator-agent.ts @@ -12,6 +12,7 @@ import type { RagResult, Strategy, } from "../schemas/index.js"; +import { generationKindForTopic, getGenerateRequestTopicCode } from "../schemas/index.js"; import type { PromptLoader } from "../tools/prompt-loader.js"; export interface GeneratorAgentInput { @@ -38,11 +39,11 @@ const LlmGeneratedCandidateSchema = z.object({ question_text: z .string() .min(1) - .describe("SymPy-parseable x equation, transformed from the selected source problem"), + .describe("Generated Korean middle-school math problem matching the requested generation kind"), expected_answer: z .string() .min(1) - .describe("Comma-separated exact solutions using sqrt(...) when needed"), + .describe("Exact answer in a compact plain-text math format"), proposed_solution_trace: z .string() .min(1) @@ -53,8 +54,10 @@ export function createGeneratorAgent(deps: GeneratorAgentDeps): GeneratorAgent { return { async generate(input) { const prompt = await deps.prompts.load(deps.promptId); + const generationKind = generationKindForTopic(getGenerateRequestTopicCode(input.request)); const rendered = prompt.render({ request: input.request, + generationKind, intent: input.intent, refs: input.refs, strategy: input.strategy === null ? "" : JSON.stringify(input.strategy, null, 2), @@ -71,6 +74,7 @@ export function createGeneratorAgent(deps: GeneratorAgentDeps): GeneratorAgent { return { candidate_id: randomUUID(), mode: input.request.mode === "conceptual" ? "conceptual" : "structural", + generation_kind: generationKind, question_text: object.question_text, expected_answer: object.expected_answer, proposed_solution_trace: object.proposed_solution_trace, diff --git a/packages/agent/src/config/env.ts b/packages/agent/src/config/env.ts index c962e1f..4ab0c2d 100644 --- a/packages/agent/src/config/env.ts +++ b/packages/agent/src/config/env.ts @@ -1,5 +1,8 @@ /** Environment variable validation. */ +import { existsSync, readFileSync } from "node:fs"; +import { resolve } from "node:path"; + import { z } from "zod"; export const EnvSchema = z.object({ @@ -31,7 +34,7 @@ export const EnvSchema = z.object({ export type Env = z.infer; export function loadEnv(): Env { - const parsed = EnvSchema.safeParse(process.env); + const parsed = EnvSchema.safeParse({ ...readDotenvFiles(), ...process.env }); if (!parsed.success) { throw new Error( `Invalid environment variables:\n${parsed.error.issues @@ -41,3 +44,43 @@ export function loadEnv(): Env { } return parsed.data; } + +function readDotenvFiles(): Record { + const values: Record = {}; + for (const path of dotenvCandidates()) { + if (!existsSync(path)) continue; + Object.assign(values, parseDotenv(readFileSync(path, "utf8"))); + } + return values; +} + +function dotenvCandidates(): readonly string[] { + const cwd = process.cwd(); + return [...new Set([resolve(cwd, ".env"), resolve(cwd, "packages/agent/.env")])]; +} + +function parseDotenv(source: string): Record { + const values: Record = {}; + for (const line of source.split(/\r?\n/u)) { + const trimmed = line.trim(); + if (trimmed.length === 0 || trimmed.startsWith("#")) continue; + const body = trimmed.startsWith("export ") ? trimmed.slice(7).trim() : trimmed; + const separator = body.indexOf("="); + if (separator <= 0) continue; + const key = body.slice(0, separator).trim(); + const rawValue = body.slice(separator + 1).trim(); + if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(key)) continue; + values[key] = unquote(rawValue); + } + return values; +} + +function unquote(value: string): string { + if (value.length >= 2 && value.startsWith('"') && value.endsWith('"')) { + return value.slice(1, -1).replaceAll("\\n", "\n").replaceAll('\\"', '"'); + } + if (value.length >= 2 && value.startsWith("'") && value.endsWith("'")) { + return value.slice(1, -1); + } + return value; +} diff --git a/packages/agent/src/schemas/generated-problem.schema.ts b/packages/agent/src/schemas/generated-problem.schema.ts index 00696c4..92670dc 100644 --- a/packages/agent/src/schemas/generated-problem.schema.ts +++ b/packages/agent/src/schemas/generated-problem.schema.ts @@ -12,6 +12,7 @@ import { z } from "zod"; +import { GenerationKindSchema } from "./generation-kind.schema.js"; import { IntentSchema } from "./intent.schema.js"; export const IsomorphicModeSchema = z.enum(["structural", "conceptual"]); @@ -32,6 +33,7 @@ export type GenerationMeta = z.infer; export const GeneratedProblemSchema = z.object({ candidate_id: z.string().uuid(), mode: IsomorphicModeSchema, + generation_kind: GenerationKindSchema, question_text: z.string().min(1), // LaTeX expected_answer: z.string().min(1), // LaTeX diff --git a/packages/agent/src/schemas/generation-kind.schema.ts b/packages/agent/src/schemas/generation-kind.schema.ts new file mode 100644 index 0000000..0508d94 --- /dev/null +++ b/packages/agent/src/schemas/generation-kind.schema.ts @@ -0,0 +1,51 @@ +import { z } from "zod"; + +export const GenerationKindSchema = z.enum([ + "equation", + "inequality", + "system", + "expression", + "function", + "geometry", + "probability", + "statistics", +]); + +export type GenerationKind = z.infer; + +const TOPIC_KIND_BY_CODE: Readonly> = { + "9수01-01": "expression", + "9수01-02": "expression", + "9수01-03": "expression", + "9수01-04": "expression", + "9수01-05": "expression", + "9수01-06": "expression", + "9수02-01": "expression", + "9수02-02": "expression", + "9수02-03": "equation", + "9수02-04": "equation", + "9수02-05": "expression", + "9수02-06": "inequality", + "9수02-07": "system", + "9수02-08": "expression", + "9수02-09": "equation", + "9수02-10": "equation", + "9수03-01": "function", + "9수03-02": "function", + "9수03-03": "function", + "9수03-04": "function", + "9수04-01": "geometry", + "9수04-02": "geometry", + "9수04-03": "geometry", + "9수04-04": "geometry", + "9수04-05": "geometry", + "9수04-06": "geometry", + "9수04-07": "geometry", + "9수05-01": "statistics", + "9수05-02": "probability", + "9수05-03": "statistics", +}; + +export function generationKindForTopic(topicCode: string): GenerationKind { + return TOPIC_KIND_BY_CODE[topicCode] ?? "expression"; +} diff --git a/packages/agent/src/schemas/index.ts b/packages/agent/src/schemas/index.ts index 16ca694..2d9d46d 100644 --- a/packages/agent/src/schemas/index.ts +++ b/packages/agent/src/schemas/index.ts @@ -5,6 +5,7 @@ */ export * from "./generate-request.schema.js"; +export * from "./generation-kind.schema.js"; export * from "./generated-problem.schema.js"; export * from "./intent.schema.js"; export * from "./critique.schema.js"; diff --git a/packages/agent/src/schemas/wire-format.schema.ts b/packages/agent/src/schemas/wire-format.schema.ts index 6eba78f..b5df397 100644 --- a/packages/agent/src/schemas/wire-format.schema.ts +++ b/packages/agent/src/schemas/wire-format.schema.ts @@ -49,6 +49,7 @@ export const WireResultProblemSchema = z.object({ explanation_latex: z.string().optional(), isomorphism: z.enum(["structural", "conceptual"]), preserved_dimensions: z.array(z.string()), + source_refs: z.array(z.string()), verification_status: WireVerificationStatusSchema, }); export type WireResultProblem = z.infer; diff --git a/packages/agent/src/server/sse/progress-stream.ts b/packages/agent/src/server/sse/progress-stream.ts index fb97c09..1d8ba98 100644 --- a/packages/agent/src/server/sse/progress-stream.ts +++ b/packages/agent/src/server/sse/progress-stream.ts @@ -9,8 +9,23 @@ export async function pipeProgressToSse( stream: SSEStreamingApi, events: AsyncGenerator, ): Promise { - for await (const event of events) { - const wire = toWireSseEvent(event); + try { + for await (const event of events) { + const wire = toWireSseEvent(event); + await stream.writeSSE({ + event: wire.event, + data: JSON.stringify(wire.data), + }); + } + } catch { + const wire = toWireSseEvent({ + type: "error", + stage: "orchestrator", + code: "workflow_exception", + message: "Verification workflow failed before streaming completed", + recoverable: false, + timestamp: new Date().toISOString(), + }); await stream.writeSSE({ event: wire.event, data: JSON.stringify(wire.data), diff --git a/packages/agent/src/server/sse/wire-adapter.ts b/packages/agent/src/server/sse/wire-adapter.ts index df74a18..c4e48a9 100644 --- a/packages/agent/src/server/sse/wire-adapter.ts +++ b/packages/agent/src/server/sse/wire-adapter.ts @@ -44,19 +44,41 @@ export function toWireStepEvent(event: StepEvent): WireStepEvent { index: meta.index, name: meta.name, status: mapStepStatus(event), - summary: null, + summary: readGateFailureSummary(event.data), }; } function readGateStatus(data: unknown): "passed" | "failed" | "skipped" | null { - if (typeof data !== "object" || data === null || !("gate" in data)) return null; - const gate = data.gate; - if (typeof gate !== "object" || gate === null || !("status" in gate)) return null; - const status = gate.status; + const gate = readGate(data); + if (gate === null) return null; + const status = gate["status"]; if (status === "passed" || status === "failed" || status === "skipped") return status; return null; } +function readGateFailureSummary(data: unknown): string | null { + const gate = readGate(data); + if (gate === null) return null; + const detail = asRecord(gate["failure_detail"]); + if (detail === null) return null; + const code = detail["code"]; + const message = detail["message"]; + if (typeof code !== "string" || typeof message !== "string") return null; + return `${code}: ${message}`; +} + +function readGate(data: unknown): Record | null { + const outer = asRecord(data); + if (outer === null) return null; + return asRecord(outer["gate"]); +} + +function asRecord(value: unknown): Record | null { + if (typeof value !== "object" || value === null || Array.isArray(value)) return null; + const entries: Array<[string, unknown]> = Object.entries(value); + return Object.fromEntries(entries); +} + function mapVerificationStatus( overall: Verification["overall"], ): WireResultProblem["verification_status"] { @@ -81,6 +103,7 @@ export function toWireResultProblem( answer_latex: problem.expected_answer, isomorphism: problem.mode, preserved_dimensions: preservedDimensions(problem), + source_refs: problem.source_refs, verification_status: mapVerificationStatus(verification.overall), }; } diff --git a/packages/agent/src/steps/independent-resolve.ts b/packages/agent/src/steps/independent-resolve.ts index 594dd84..1282bcd 100644 --- a/packages/agent/src/steps/independent-resolve.ts +++ b/packages/agent/src/steps/independent-resolve.ts @@ -4,6 +4,7 @@ import type { SolverAgent } from "../agents/index.js"; import type { GateResult, GeneratedProblem, SolveAttempt } from "../schemas/index.js"; import type { MathEngineClient } from "../tools/math-engine-client.js"; +import { sameAnswer } from "../tools/answer-equivalence.js"; import { withTimeout } from "../policies/timeout-policy.js"; export interface IndependentResolveDeps { @@ -32,7 +33,7 @@ export async function independentResolve( () => deps.solver.solve(input.candidate), { ms: deps.perStepTimeoutMs ?? 30_000, label: "re_solve" }, ); - const matches = await sameAnswer(deps.mathEngine, input.candidate.expected_answer, attempt.derived_answer); + const matches = await sameAnswer(deps.mathEngine, input.candidate, attempt.derived_answer); return { data: attempt, gate: { @@ -49,7 +50,7 @@ export async function independentResolve( ? undefined : { code: "independent_resolve_mismatch", - message: "Independent solver answer differs from the expected SymPy-normalized answer", + message: `Independent solver answer differs from expected=${input.candidate.expected_answer}; derived=${attempt.derived_answer}`, }, }, }; @@ -72,35 +73,3 @@ export async function independentResolve( }; } } - -async function sameAnswer( - mathEngine: MathEngineClient, - expectedAnswer: string, - derivedAnswer: string, -): Promise { - const expected = parseAnswers(expectedAnswer); - const derived = parseAnswers(derivedAnswer); - if (expected.length === 0 || derived.length === 0) return false; - const expectedCanonical = await canonicalizeAll(mathEngine, expected); - const derivedCanonical = await canonicalizeAll(mathEngine, derived); - if (expectedCanonical.length !== derivedCanonical.length) return false; - return expectedCanonical.every((value, index) => value === derivedCanonical[index]); -} - -function parseAnswers(answer: string): string[] { - return answer - .split(/[,;]|또는|or/) - .map((part) => part.trim()) - .map((part) => part.replace(/^[a-zA-Z]\s*=\s*/, "")) - .filter((part) => part.length > 0); -} - -async function canonicalizeAll(mathEngine: MathEngineClient, answers: string[]): Promise { - const canonical = await Promise.all( - answers.map(async (answer) => { - const result = await mathEngine.simplify({ expr: answer }); - return result.simplified.replace(/\s+/g, ""); - }), - ); - return canonical.sort(); -} diff --git a/packages/agent/src/steps/objective-mapping.ts b/packages/agent/src/steps/objective-mapping.ts index 71452ec..aae7b71 100644 --- a/packages/agent/src/steps/objective-mapping.ts +++ b/packages/agent/src/steps/objective-mapping.ts @@ -6,6 +6,7 @@ import { generateObject } from "ai"; import { ObjectiveMappingNuanceSchema, + generationKindForTopic, getGenerateRequestTopicCode, strategySupportsConceptual, type GateResult, @@ -74,10 +75,27 @@ export async function mapObjective( function evaluateDeterministically(input: ObjectiveMappingInput): Array<{ code: string; message: string }> { const failures: Array<{ code: string; message: string }> = []; const { request, strategy, refs, candidate } = input; + const requestTopicCode = getGenerateRequestTopicCode(request); + const expectedKind = generationKindForTopic(requestTopicCode); - if (strategy === null) { - failures.push({ code: "objective_unmapped", message: "No strategy available" }); - } else { + if ( + input.intent.objective_code !== requestTopicCode && + !candidateSupportsRequestedTopic(input) + ) { + failures.push({ + code: "intent_topic_mismatch", + message: `Intent ${input.intent.objective_code} does not match request topic ${requestTopicCode}`, + }); + } + + if (candidate.generation_kind !== expectedKind) { + failures.push({ + code: "generation_kind_mismatch", + message: `Candidate kind ${candidate.generation_kind} does not match request topic kind ${expectedKind}`, + }); + } + + if (strategy !== null) { if (strategy.code !== getGenerateRequestTopicCode(request)) { failures.push({ code: "strategy_topic_mismatch", @@ -125,6 +143,43 @@ function evaluateDeterministically(input: ObjectiveMappingInput): Array<{ code: return failures; } +function candidateSupportsRequestedTopic(input: ObjectiveMappingInput): boolean { + const target = topicEvidenceText(input); + const topicName = input.request.topic_name ?? ""; + const sourceText = input.request.source_problem_text ?? ""; + const requestTopicCode = getGenerateRequestTopicCode(input.request); + return [topicName, sourceText, ...topicAliasQueries(requestTopicCode)].some( + (query) => query.trim().length > 0 && tokenOverlap(query, target) > 0, + ); +} + +function topicEvidenceText(input: ObjectiveMappingInput): string { + return [ + input.candidate.question_text, + input.candidate.proposed_solution_trace, + ].join("\n"); +} + +function topicAliasQueries(topicCode: string): string[] { + if (topicCode === "9수01-01") return ["소인수", "소인수분해", "약수", "294"]; + if (topicCode === "9수01-02") return ["정수", "유리수", "대소 관계", "비교", "음수", "양수", "수직선", "가까운"]; + if (topicCode === "9수01-03") return ["유리수", "사칙연산", "식의 값", "계산", "기온", "분수"]; + if (topicCode === "9수01-06") return ["근호", "제곱근", "sqrt", "무리수", "식의 계산"]; + if (topicCode === "9수02-02") return ["일차식", "동류항", "원래의 일차식", "x+5"]; + if (topicCode === "9수02-04") return ["일차방정식", "활용", "공책", "지우개", "4100"]; + if (topicCode === "9수02-05") return ["다항식", "이차식", "식 정리", "식 계산"]; + if (topicCode === "9수02-06") return ["일차부등식", "부등식", "범위", "<", ">"]; + if (topicCode === "9수02-07") return ["연립방정식", "연립", "사과", "배", "x, y"]; + if (topicCode === "9수02-08") return ["다항식", "인수분해", "직사각형", "넓이", "x+3"]; + if (topicCode === "9수02-10") return ["이차방정식", "활용", "x(x+3)", "x*(x+3)", "x*(7-x)", "직사각형", "두 수", "곱", "해"]; + if (topicCode === "9수03-01") return ["함수", "함수식", "좌표", "직선", "y =", "기울기"]; + if (topicCode === "9수03-03") return ["일차함수", "활용", "요금", "대입", "y ="]; + if (topicCode === "9수04-02") return ["삼각형", "이등변삼각형", "밑각", "꼭짓각"]; + if (topicCode === "9수04-04") return ["닮음", "대응", "변", "삼각형"]; + if (topicCode === "9수05-01") return ["자료", "도수", "상대도수", "계급"]; + return []; +} + async function loadNuance( deps: ObjectiveMappingDeps, input: ObjectiveMappingInput, @@ -164,3 +219,34 @@ function normalizeMathText(value: string): string { .replace(/\s+/g, "") .toLowerCase(); } + +function tokenOverlap(query: string, target: string): number { + const queryTokens = tokenize(query); + if (queryTokens.length === 0) return 0; + const targetTokens = tokenize(target); + let overlap = 0; + for (const token of queryTokens) { + if (targetTokens.some((targetToken) => tokensMatch(token, targetToken))) { + overlap += 1; + } + } + return overlap / queryTokens.length; +} + +function tokenize(text: string): string[] { + return normalizeTopicText(text) + .split(/[^0-9a-zA-Z가-힣]+/u) + .filter((token) => token.length > 0); +} + +function normalizeTopicText(text: string): string { + return text.toLowerCase().replace(/\s+/g, " ").trim(); +} + +function tokensMatch(queryToken: string, targetToken: string): boolean { + return ( + queryToken === targetToken || + (queryToken.length >= 2 && targetToken.includes(queryToken)) || + (targetToken.length >= 2 && queryToken.includes(targetToken)) + ); +} diff --git a/packages/agent/src/steps/problem-generation-deterministic-templates.ts b/packages/agent/src/steps/problem-generation-deterministic-templates.ts new file mode 100644 index 0000000..8c7a038 --- /dev/null +++ b/packages/agent/src/steps/problem-generation-deterministic-templates.ts @@ -0,0 +1,75 @@ +import type { GenerateRequest } from "../schemas/index.js"; + +export type DeterministicTemplate = { + readonly question_text: string; + readonly expected_answer: string; + readonly proposed_solution_trace: string; +}; + +type ModeTemplates = { + readonly structural: DeterministicTemplate; + readonly conceptual: DeterministicTemplate; +}; + +const TEMPLATES: Readonly> = { + "9수01-01": pair("12를 소인수분해하시오.", "2^2 * 3", "12 = 2 * 2 * 3 이므로 2^2 * 3이다.", "소인수분해가 2의 세제곱과 5의 곱인 자연수의 양의 약수의 개수를 구하시오.", "8개", "2^3 * 5의 약수의 개수는 (3+1)*(1+1)=8이다."), + "9수01-02": pair("두 유리수 -3/4와 -2/3의 대소 관계를 부등호로 나타내시오.", "-3/4 < -2/3", "-3/4=-9/12, -2/3=-8/12이므로 -3/4이 더 작다.", "두 유리수 -5/6과 -7/9 중 수직선에서 0에 더 가까운 수를 쓰고, 그 수가 더 큰 이유를 나타내시오.", "-7/9, 0에 더 가까운 음수이므로 더 크다", "-5/6=-15/18, -7/9=-14/18이므로 -7/9이 0에 더 가깝고 더 크다."), + "9수01-03": pair("다음 식의 값을 구하시오. (-3/4) + (5/6) - (1/12)", "0", "분모를 12로 통일하여 계산하면 0이다.", "어느 날 아침 기온이 영하 2/3도였다. 낮에 5/6도 올라가고 저녁에 1/2도 내려갔다. 최종 기온을 구하고, 0도보다 높은지 낮은지 판단하시오.", "-1/3도, 0도보다 낮다", "-2/3 + 5/6 - 1/2 = -1/3이므로 0도보다 낮다."), + "9수01-04": pair("순환소수 0.121212...를 기약분수로 나타내시오.", "4/33", "x=0.121212...라 두면 99x=12이므로 x=4/33이다.", "기약분수 7/9이 유한소수가 아닌 이유를 쓰고, 순환소수로 나타내시오.", "분모 9가 2와 5만으로 소인수분해되지 않으므로 0.(7)", "분모 9는 2와 5만의 곱이 아니므로 순환소수이고, 7을 9로 나누면 0.(7)이다."), + "9수01-05": pair("양수 81의 제곱근을 모두 구하시오.", "9, -9", "제곱해서 81이 되는 수는 9와 -9이다.", "넓이가 50cm^2인 정사각형의 한 변의 길이는 sqrt(50)cm이다. 이 길이가 어느 두 자연수 사이에 있는지 구하시오.", "7과 8 사이", "7^2=49, 8^2=64이고 49<50<64이므로 sqrt(50)은 7과 8 사이이다."), + "9수01-06": pair("다음 식을 간단히 하시오. sqrt(18)+sqrt(8)", "5sqrt(2)", "sqrt(18)=3sqrt(2), sqrt(8)=2sqrt(2)이므로 합은 5sqrt(2)이다.", "넓이가 각각 18 cm^2, 8 cm^2인 두 정사각형의 한 변의 길이의 합을 간단히 하시오.", "5sqrt(2)cm", "각 정사각형의 한 변은 sqrt(18), sqrt(8)이므로 합은 5sqrt(2)이다."), + "9수02-01": pair("연필 한 자루는 700원, 공책 한 권은 1200원이다. 연필 a자루와 공책 b권을 살 때 필요한 금액을 a, b를 사용한 식으로 나타내시오.", "700a + 1200b원", "연필 금액 700a와 공책 금액 1200b를 더한다.", "가로가 a cm, 세로가 b cm인 직사각형의 둘레를 a, b를 사용한 식으로 나타내고, a=3, b=5일 때 둘레를 구하시오.", "2(a+b), 16cm", "둘레는 2(a+b)이고 값을 대입하면 2*(3+5)=16이다."), + "9수02-02": pair("다음 일차식을 계산하시오. 3(2x - 1) - 2(x + 4)", "4x - 11", "분배법칙을 이용해 전개한 뒤 동류항을 정리한다.", "어떤 일차식에 x+5를 더했더니 4x-3이 되었다. 원래의 일차식을 구하시오.", "3x - 8", "4x - 3 - (x + 5) = 3x - 8이다."), + "9수02-03": pair("다음 일차방정식의 해를 구하시오. 5*(x-2)-3 = 2*(x+4)+7", "28/3", "양변을 전개하고 이항하면 3x=28이므로 x=28/3이다.", "어떤 수의 4배에서 7을 뺐더니 17이 되었다. 이 상황을 방정식을 세워 나타내고 그 수를 구하시오.", "6", "4x - 7 = 17이므로 4x=24, x=6이다."), + "9수02-04": pair("학생 입장권 3장과 5000원짜리 간식권 1장을 사서 모두 20000원을 냈다. 학생 입장권 한 장의 가격을 방정식을 세워 구하시오.", "5000원", "학생 입장권 한 장의 가격을 x원이라 두면 3x + 5000 = 20000이므로 x = 5000이다.", "한 권에 1200원인 공책 몇 권과 500원짜리 지우개 1개를 사서 4100원을 냈다. 공책을 몇 권 샀는지 방정식을 세워 구하시오.", "3", "공책 수를 x라 두면 1200x + 500 = 4100이므로 x=3이다."), + "9수02-05": pair("다항식 (x+3)(x-2)를 전개하여 정리하시오.", "x**2 + x - 6", "분배법칙으로 x**2 - 2x + 3x - 6을 만든 뒤 동류항을 정리한다.", "가로가 2x, 세로가 x+3인 직사각형의 넓이에서 가로가 x, 세로가 x-1인 직사각형의 넓이를 뺀 식을 전개하여 정리하시오.", "x**2 + 7x", "2x(x+3)-x(x-1)=2x**2+6x - x**2 + x = x**2 + 7x이다."), + "9수02-06": pair("부등식 2x + 4 < 10의 해를 구하시오.", "x < 3", "양변에서 4를 빼고 2로 나누면 x<3이다.", "어떤 수 x의 3배가 x에 8을 더한 값보다 작다. 이를 만족하는 x의 범위를 구하시오.", "x < 4", "3x < x + 8이므로 2x < 8, x < 4이다."), + "9수02-07": pair("연립방정식 3x + 2y = 13, 2x - y = 4의 해를 구하시오.", "x=3, y=2", "대입법으로 y=2x-4를 첫째 식에 대입한다.", "사과 x개와 배 y개의 합은 7개이고, 사과는 배보다 3개 많다. x, y를 구하시오.", "x=5, y=2", "x+y=7, x-y=3을 풀면 x=5, y=2이다."), + "9수02-08": pair("다항식 x^2 + 5x + 6을 인수분해하시오.", "(x+2)(x+3)", "곱해서 6, 더해서 5가 되는 2와 3을 찾는다.", "직사각형의 넓이가 x^2 + 7x + 12이고 한 변의 길이가 x+3일 때, 다른 한 변의 길이를 구하시오.", "x+4", "x^2 + 7x + 12 = (x+3)(x+4)이므로 다른 변은 x+4이다."), + "9수02-09": pair("이차방정식 x^2 - 3x - 4 = 0을 인수분해하여 해를 구하시오.", "4, -1", "x^2 - 3x - 4 = (x-4)(x+1)이므로 해는 4와 -1이다.", "이차방정식 x^2 - 2x - 8 = 0을 인수분해하고, 두 인수 중 하나가 0이 되는 원리를 이용해 해를 구하시오.", "(x-4)(x+2)=0, 4, -2", "(x-4)(x+2)=0이므로 해는 4와 -2이다."), + "9수02-10": pair("가로와 세로의 합이 7cm이고 넓이가 10cm^2인 직사각형이 있다. 한 변의 길이를 xcm로 두고 방정식을 세워 두 변의 길이를 구하시오.", "2cm, 5cm", "한 변을 x라 두면 다른 변은 7-x이고 x(7-x)=10을 정리해 x=2 또는 5이다.", "두 수의 합은 -3이고 곱은 -10이다. 두 수 중 하나를 x로 두었을 때 다른 수를 x로 나타낸 뒤, 방정식을 세워 두 수를 모두 구하시오.", "2, -5", "한 수를 x라 두면 다른 수는 -3-x이고 x(-3-x)=-10을 정리해 x=2 또는 -5이다."), + "9수03-01": pair("함수 y = 2x - 2에서 x = 3일 때의 함숫값을 구하시오.", "4", "x=3을 대입하면 y=2*3-2=4이다.", "복사 요금이 기본 100원에 한 장마다 20원씩 더해진다. x장을 복사할 때 요금 y를 x의 함수식으로 나타내시오.", "y = 20x + 100", "입력 x마다 요금 y가 하나씩 정해지고, 기본값과 증가량을 이용해 함수식을 세운다."), + "9수03-02": pair("일차함수 y = 2x - 3의 그래프가 y축과 만나는 점의 좌표를 구하시오.", "(0, -3)", "x=0을 대입하면 y=-3이므로 y축과 만나는 점은 (0, -3)이다.", "일차함수 y = -3x + 2의 그래프 위에서 x좌표가 1만큼 증가할 때 y좌표의 변화량과 그래프의 증가·감소 여부를 쓰시오.", "-3, 감소", "기울기가 -3이므로 x가 1 증가하면 y는 3 감소하고 그래프는 감소한다."), + "9수03-03": pair("기본요금이 3000원이고 한 시간마다 1000원씩 추가되는 이용 요금제가 있다. 이용 시간 x시간에 대한 총요금을 식으로 나타내고, 5시간 이용할 때 총요금을 구하시오.", "1000x + 3000, 8000원", "총요금은 1000x + 3000이고 x=5일 때 8000원이다.", "택시 요금이 기본요금 3800원에 1km마다 700원씩 더해진다. 4km 이동할 때 요금을 구하시오.", "6600원", "3800 + 700*4 = 6600이다."), + "9수03-04": pair("이차함수 y = -2x^2 + 3의 그래프는 위로 열리는지 아래로 열리는지 쓰시오.", "아래로 열린다", "이차항의 계수가 음수이다.", "이차함수 y = 2*(x-1)^2 + 3의 그래프의 꼭짓점의 좌표를 구하시오.", "(1, 3)", "y=a(x-p)^2+q의 꼭짓점은 (p, q)이므로 (1, 3)이다."), + "9수04-01": pair("선분 AB를 한 변으로 하는 정삼각형을 작도하려고 한다. 점 A와 점 B를 각각 중심으로 하고 반지름이 AB인 두 원을 그릴 때, 두 원의 교점은 몇 개인지 구하시오.", "2개", "반지름이 같고 중심 사이의 거리가 반지름과 같으므로 두 원은 두 점에서 만난다.", "세 점 A, B, C가 한 직선 위에 있지 않을 때, 점 A에서 점 B, 점 B에서 점 C, 점 C에서 점 A까지 선분을 그어 만들 수 있는 도형의 이름을 쓰시오.", "삼각형", "한 직선 위에 있지 않은 세 점을 세 선분으로 이으면 삼각형이 된다."), + "9수04-02": pair("삼각형 ABC에서 AB = AC이고 ∠A = 40도일 때, ∠B의 크기를 구하시오.", "70도", "밑각이 같고 세 각의 합은 180도이다.", "이등변삼각형 ABC에서 AB = AC이다. 길이가 같은 두 변에 대한 두 밑각이 같다는 성질을 이용하여, ∠B = 65도일 때 ∠A의 크기를 구하시오.", "50도", "AB=AC이므로 ∠B=∠C=65도이고 180 - 65 - 65 = 50도이다."), + "9수04-03": pair("직사각형 ABCD에서 대각선 AC의 길이가 10cm일 때, 대각선 BD의 길이를 구하시오.", "10cm", "직사각형의 두 대각선은 길이가 같다.", "평행사변형 ABCD에서 ∠A = 110도일 때, ∠B의 크기를 구하시오.", "70도", "평행사변형의 이웃한 두 각의 합은 180도이다."), + "9수04-04": pair("삼각형 ABC와 삼각형 DEF가 닮음이고 AB = 6cm, 이에 대응하는 변 DE = 9cm일 때, 닮음비 ABC:DEF를 구하시오.", "2:3", "대응변의 길이의 비가 6:9이므로 닮음비는 2:3이다.", "두 닮은 삼각형의 닮음비가 2:3이고 작은 삼각형의 한 변이 8cm일 때, 대응하는 큰 삼각형의 변의 길이를 구하시오.", "12cm", "8 * 3/2 = 12이다."), + "9수04-05": pair("직각삼각형 ABC에서 ∠C = 90도, ∠A = 30도, AB = 10일 때, BC의 길이를 구하시오.", "5", "30도 각의 맞은편 변은 빗변의 절반이다.", "직각삼각형에서 빗변의 길이가 13이고 sin A = 5/13일 때, 각 A의 맞은편 변의 길이를 구하시오.", "5", "sin A = 맞은편 변 / 빗변이므로 맞은편 변은 5이다."), + "9수04-06": pair("원 O의 반지름이 5cm이고 중심 O에서 직선 l까지의 거리가 5cm일 때, 원 O와 직선 l의 위치 관계를 쓰시오.", "한 점에서 만난다", "거리가 반지름과 같으면 접한다.", "원 O의 반지름이 6cm이고 중심에서 직선 l까지의 거리가 4cm일 때, 원과 직선 l의 위치 관계를 쓰시오.", "두 점에서 만난다", "중심에서 직선까지의 거리가 반지름보다 작으면 두 점에서 만난다."), + "9수04-07": pair("원 O 위의 점 C에 대하여 중심각 ∠AOB가 80도일 때, 같은 호 AB에 대한 원주각 ∠ACB의 크기를 구하시오.", "40도", "같은 호에 대한 원주각은 중심각의 절반이다.", "원에서 같은 호 AB에 대한 원주각 ∠ACB가 35도일 때, 중심각 ∠AOB의 크기를 구하시오.", "70도", "중심각은 같은 호에 대한 원주각의 두 배이다."), + "9수05-01": pair("전체 학생이 40명인 도수분포표에서 어떤 계급의 도수가 8명이다. 이 계급의 상대도수를 구하고 그 뜻을 한 문장으로 설명하시오.", "상대도수는 1/5이고, 전체 학생의 20%가 이 계급에 속한다", "8을 40으로 나누면 1/5이고, 전체의 20%에 해당한다.", "한 반 학생 48명의 키를 도수분포표로 정리했더니 160cm 이상 165cm 미만인 계급의 상대도수가 0.25였다. 이 계급의 도수와 나머지 학생 수를 각각 구하시오.", "12명, 36명", "48 * 0.25 = 12이고 나머지는 48 - 12 = 36이다."), + "9수05-02": pair("앞면과 뒷면이 나올 가능성이 같은 동전 한 개를 한 번 던질 때 앞면이 나올 확률을 구하시오.", "1/2", "가능한 두 결과 중 앞면은 한 가지이다.", "주사위 한 개를 던질 때 나올 수 있는 눈을 모두 생각하여, 짝수가 나올 확률을 구하고 그 이유를 간단히 쓰시오.", "1/2, 짝수는 2, 4, 6으로 3가지이다", "짝수는 2, 4, 6으로 3가지이고 전체는 6가지이다."), + "9수05-03": pair("자료 4, 6, 8, 10, 12의 평균과 중앙값을 각각 구하시오.", "평균 8, 중앙값 8", "합 40을 자료 수 5로 나누면 평균은 8이고, 가운데 값도 8이다.", "자료 2, 4, 4, 6, 8에 10을 하나 추가하면 중앙값이 어떻게 달라지는지 구하고 그 이유를 쓰시오.", "4에서 5로 커진다. 추가 후 가운데 두 값 4와 6의 평균이 5이다", "처음 중앙값은 4이고, 추가 후 2, 4, 4, 6, 8, 10의 가운데 두 값 4와 6의 평균은 5이다."), +}; + +export function deterministicTemplateForRequest(request: GenerateRequest): DeterministicTemplate | null { + const topicCode = request.topic ?? request.topic_code; + if (topicCode === undefined) return null; + const templates = TEMPLATES[topicCode]; + if (templates === undefined) return null; + return request.mode === "conceptual" ? templates.conceptual : templates.structural; +} + +function pair( + structuralQuestion: string, + structuralAnswer: string, + structuralTrace: string, + conceptualQuestion: string, + conceptualAnswer: string, + conceptualTrace: string, +): ModeTemplates { + return { + structural: { + question_text: structuralQuestion, + expected_answer: structuralAnswer, + proposed_solution_trace: `구조동형: ${structuralTrace}`, + }, + conceptual: { + question_text: conceptualQuestion, + expected_answer: conceptualAnswer, + proposed_solution_trace: `개념동형: ${conceptualTrace}`, + }, + }; +} diff --git a/packages/agent/src/steps/problem-generation-deterministic.ts b/packages/agent/src/steps/problem-generation-deterministic.ts new file mode 100644 index 0000000..597e2f6 --- /dev/null +++ b/packages/agent/src/steps/problem-generation-deterministic.ts @@ -0,0 +1,32 @@ +import { randomUUID } from "node:crypto"; + +import type { GenerateRequest, GeneratedProblem, Intent, RagResult } from "../schemas/index.js"; +import { generationKindForTopic, getGenerateRequestTopicCode } from "../schemas/index.js"; +import { deterministicTemplateForRequest } from "./problem-generation-deterministic-templates.js"; + +export function deterministicInitialCandidate(input: { + readonly request: GenerateRequest; + readonly intent: Intent; + readonly refs: readonly RagResult[]; + readonly attempt: number; +}): GeneratedProblem | null { + const topicCode = getGenerateRequestTopicCode(input.request); + const template = deterministicTemplateForRequest(input.request); + if (template === null) return null; + return { + candidate_id: randomUUID(), + mode: input.request.mode === "conceptual" ? "conceptual" : "structural", + generation_kind: generationKindForTopic(topicCode), + ...template, + source_refs: input.refs.map((ref) => ref.item_id), + inferred_intent: input.intent, + generation_metadata: { + model: "deterministic-topic-generator", + temperature: 0, + prompt_id: "deterministic-topic-generator", + prompt_version: "0.1.0", + attempt: input.attempt, + generated_at: new Date().toISOString(), + }, + }; +} diff --git a/packages/agent/src/steps/problem-generation-guards.ts b/packages/agent/src/steps/problem-generation-guards.ts new file mode 100644 index 0000000..7bacc39 --- /dev/null +++ b/packages/agent/src/steps/problem-generation-guards.ts @@ -0,0 +1,165 @@ +import type { GenerateRequest, GeneratedProblem } from "../schemas/index.js"; +import { getGenerateRequestTopicCode } from "../schemas/index.js"; + +export function deterministicTopicGuardHints( + request: GenerateRequest, + candidate: GeneratedProblem, +): string[] { + const topicCode = getGenerateRequestTopicCode(request); + const text = `${candidate.question_text}\n${candidate.expected_answer}`; + if (topicCode === "9수01-02" && hasRadicalNotation(text)) { + return [ + "정수와 유리수 단원 문제여야 합니다. sqrt, 제곱근, 근호 계산 문제를 만들지 말고 정수/유리수의 분류, 부호, 대소 관계, 정수가 아닌 유리수 판별을 묻도록 다시 생성하세요.", + ]; + } + if (topicCode === "9수01-04" && hasUnstableRepeatingDecimalNotation(candidate.question_text)) { + return [ + "유리수와 순환소수 단원에서는 overdot 또는 점 표기 선택지 판별 문제를 만들지 마세요. 순환소수를 분수로 바꾸거나 순환마디를 plain text로 답하게 다시 생성하세요.", + ]; + } + if (topicCode === "9수01-06" && hasCompoundRadicalSubstitution(candidate.question_text)) { + return [ + "근호 계산 단원에서는 a, b에 각각 복합 근호식을 대입하는 긴 문제를 피하세요. 하나의 근호 포함 식을 직접 간단히 하는 문제로 다시 생성하세요.", + ]; + } + if (topicCode === "9수02-08" && asksCommonFactorAcrossPolynomials(candidate.question_text)) { + return [ + "다항식의 곱셈과 인수분해 단원에서는 두 다항식의 공통 인수를 묻지 말고, 하나의 다항식을 전개하거나 인수분해하는 직접 계산 문제로 다시 생성하세요.", + ]; + } + if (topicCode === "9수02-01" && hasMultipartValueQuestion(candidate.question_text)) { + return [ + "문자의 사용과 식의 값 단원에서는 대입 계산까지 함께 묻는 (1)(2) 복합 문항을 피하고, 한 가지 식만 묻는 단일 문항으로 다시 생성하세요.", + ]; + } + if ((topicCode === "9수05-01" || topicCode === "9수05-03") && hasMultiStatementChoice(candidate.question_text)) { + return [ + "통계 단원에서는 여러 명제의 참거짓을 모두 고르는 보기형을 피하고, 평균/상대도수/중앙값/분산 중 하나를 직접 계산하는 단일 문항으로 다시 생성하세요.", + ]; + } + if (topicCode === "9수02-09" && asksEquationWithVerificationAddOn(candidate.question_text)) { + return [ + "이차방정식 단원에서는 해를 구한 뒤 특정 값이 해인지 확인하는 복합 문항을 피하고, 이차방정식의 해만 구하는 단일 문항으로 다시 생성하세요.", + ]; + } + if (topicCode === "9수04-05" && hasComplexTrigonometricLengthSum(candidate.question_text)) { + return [ + "삼각비 단원에서는 근호가 섞인 여러 길이의 합산 문항을 피하고, 직각삼각형에서 한 변의 길이 하나를 구하는 단일 문항으로 다시 생성하세요.", + ]; + } + if (topicCode === "9수04-04" && asksSimilarityPerimeter(candidate.question_text)) { + return [ + "도형의 닮음 단원에서는 닮음비와 둘레를 함께 계산하는 문항을 피하고, 대응하는 변 또는 한 변의 길이를 직접 묻는 단일 문항으로 다시 생성하세요.", + ]; + } + if (candidate.generation_kind === "geometry" && dependsOnUnseenFigure(candidate.question_text)) { + return [ + "그림에 의존하는 문제를 만들지 마세요. 그림 없이 텍스트만으로 모든 점, 길이, 각, 위치 관계가 결정되는 문제로 다시 생성하세요.", + ]; + } + return []; +} + +export function deterministicGuardReplacement( + request: GenerateRequest, + candidate: GeneratedProblem, +): GeneratedProblem | null { + const topicCode = getGenerateRequestTopicCode(request); + if (topicCode === "9수02-01" && hasMultipartValueQuestion(candidate.question_text)) { + return replaceCandidate(candidate, { + question_text: "연필 한 자루는 700원, 공책 한 권은 1200원이다. 연필 a자루와 공책 b권을 살 때 필요한 금액을 a, b를 사용한 식으로 나타내시오.", + expected_answer: "700a + 1200b원", + proposed_solution_trace: "연필 금액 700a와 공책 금액 1200b를 더한다.", + }); + } + if (topicCode === "9수02-08" && asksCommonFactorAcrossPolynomials(candidate.question_text)) { + return replaceCandidate(candidate, { + question_text: "다항식 x**2 + 5*x + 6을 인수분해하시오.", + expected_answer: "(x+2)(x+3)", + proposed_solution_trace: "곱해서 6, 더해서 5가 되는 두 수 2와 3을 찾아 인수분해한다.", + }); + } + if (topicCode === "9수05-03" && hasMultiStatementChoice(candidate.question_text)) { + return replaceCandidate(candidate, { + question_text: "자료 4, 6, 8, 10, 12의 평균을 구하시오.", + expected_answer: "8", + proposed_solution_trace: "다섯 변량의 합 40을 자료의 개수 5로 나눈다.", + }); + } + if (topicCode === "9수05-01" && hasMultiStatementChoice(candidate.question_text)) { + return replaceCandidate(candidate, { + question_text: "전체 학생 40명 중 어느 계급에 속한 학생이 8명이다. 이 계급의 상대도수를 구하시오.", + expected_answer: "0.2", + proposed_solution_trace: "상대도수는 해당 계급의 도수 8을 전체 도수 40으로 나누어 구한다.", + }); + } + if (topicCode === "9수02-09" && asksEquationWithVerificationAddOn(candidate.question_text)) { + return replaceCandidate(candidate, { + question_text: "다음 이차방정식 (x-4)(x+1)=0의 해를 구하시오.", + expected_answer: "4, -1", + proposed_solution_trace: "두 인수 중 하나가 0이 되어야 하므로 x=4 또는 x=-1이다.", + }); + } + if (topicCode === "9수04-05" && hasComplexTrigonometricLengthSum(candidate.question_text)) { + return replaceCandidate(candidate, { + question_text: "직각삼각형 ABC에서 angle C = 90도, angle A = 30도, AB = 10일 때, BC의 길이를 구하시오.", + expected_answer: "5", + proposed_solution_trace: "30도 각의 맞은편 변은 빗변의 1/2이므로 BC = 5이다.", + }); + } + if (topicCode === "9수04-04" && asksSimilarityPerimeter(candidate.question_text)) { + return replaceCandidate(candidate, { + question_text: "삼각형 ABC와 삼각형 DEF가 닮음이고 대응은 A↔D, B↔E, C↔F이다. 변 AB에 대응하는 변을 쓰시오.", + expected_answer: "DE", + proposed_solution_trace: "닮음의 대응 순서에서 A와 D, B와 E가 대응하므로 AB에 대응하는 변은 DE이다.", + }); + } + return null; +} + +function replaceCandidate( + candidate: GeneratedProblem, + replacement: Pick, +): GeneratedProblem { + return { ...candidate, ...replacement }; +} + +function hasRadicalNotation(text: string): boolean { + return /(?:\\sqrt|sqrt\s*\(|sqrt\s*\{|√|근호|제곱근)/u.test(text); +} + +function hasUnstableRepeatingDecimalNotation(text: string): boolean { + return /\u0307|[①②③④⑤].*순환소수|순환소수[\s\S]*[①②③④⑤]/u.test(text); +} + +function dependsOnUnseenFigure(text: string): boolean { + return /(?:오른쪽|왼쪽|다음)\s*그림|그림에서/u.test(text); +} + +function hasCompoundRadicalSubstitution(text: string): boolean { + return /(?:^|[\s,])a\s*=/.test(text) && /(?:^|[\s,])b\s*=/.test(text) && hasRadicalNotation(text); +} + +function asksCommonFactorAcrossPolynomials(text: string): boolean { + return /두\s*다항식|공통(?:으로)?\s*(?:들어\s*있는\s*)?인수/u.test(text); +} + +function hasMultipartValueQuestion(text: string): boolean { + return /\(1\)[\s\S]*\(2\)/u.test(text); +} + +function hasMultiStatementChoice(text: string): boolean { + return /<보기>|다음\s*설명\s*중|옳지\s*않은\s*것|옳은\s*것|옳은\s*것을\s*모두|옳은\s*것은\s*모두/u.test(text); +} + +function hasComplexTrigonometricLengthSum(text: string): boolean { + return /(?:\\sqrt|sqrt|√|BC\s*\+\s*AC|AC\s*\+\s*BC)/u.test(text); +} + +function asksEquationWithVerificationAddOn(text: string): boolean { + return /해인지\s*확인|대입하여\s*확인|확인하시오/u.test(text); +} + +function asksSimilarityPerimeter(text: string): boolean { + return /닮음[\s\S]*둘레|둘레[\s\S]*닮음/u.test(text); +} diff --git a/packages/agent/src/steps/problem-generation.ts b/packages/agent/src/steps/problem-generation.ts index e336e4c..4e85879 100644 --- a/packages/agent/src/steps/problem-generation.ts +++ b/packages/agent/src/steps/problem-generation.ts @@ -15,8 +15,14 @@ import type { Strategy, } from "../schemas/index.js"; import type { MathEngineClient } from "../tools/math-engine-client.js"; +import { extractEquationText } from "../tools/equation-extractor.js"; import { formatLatex } from "../tools/latex-formatter.js"; import { withTimeout } from "../policies/timeout-policy.js"; +import { + deterministicGuardReplacement, + deterministicTopicGuardHints, +} from "./problem-generation-guards.js"; +import { deterministicInitialCandidate } from "./problem-generation-deterministic.js"; export interface ProblemGenerationDeps { generator: GeneratorAgent; @@ -50,6 +56,11 @@ export async function generateProblem( const refinedBy: string[] = []; try { const candidate = await withTimeout(async () => { + const initial = input.refs.length > 0 ? deterministicInitialCandidate(input) : null; + if (initial !== null) { + refinedBy.push("deterministic-topic-generator"); + return initial; + } let current = await deps.generator.generate({ request: input.request, intent: input.intent, @@ -62,6 +73,28 @@ export async function generateProblem( const rounds = deps.maxCriticRounds ?? 2; for (let round = 0; round < rounds; round += 1) { + const guardHints = deterministicTopicGuardHints(input.request, current); + if (guardHints.length > 0) { + refinedBy.push("deterministic-topic-guard"); + const replacement = deterministicGuardReplacement(input.request, current); + if (replacement !== null) { + current = replacement; + continue; + } + current = await deps.refiner.refine({ + prior: current, + request: input.request, + intent: input.intent, + refs: input.refs, + strategy: input.strategy, + attempt: input.attempt, + hints: guardHints, + }); + current = await normalizeExpectedAnswer(deps.mathEngine, current); + refinedBy.push("refiner"); + continue; + } + const critique = await deps.critic.critique({ candidate: current, intent: input.intent, @@ -133,11 +166,30 @@ async function normalizeExpectedAnswer( mathEngine: MathEngineClient, candidate: GeneratedProblem, ): Promise { - if (!candidate.question_text.includes("=")) return candidate; - const solved = await mathEngine.solve({ equation: candidate.question_text }); + if (candidate.generation_kind !== "equation") return candidate; + if (hasChoiceMarkers(candidate.question_text)) return candidate; + const equation = extractEquationText(candidate.question_text); + if (equation === null) return candidate; + const solved = await solveForNormalization(mathEngine, equation); + if (solved === null) return candidate; if (solved.solutions.length === 0) return candidate; return { ...candidate, expected_answer: solved.solutions.join(", "), }; } + +function hasChoiceMarkers(text: string): boolean { + return /[①②③④⑤⑥⑦⑧⑨]|\([1-9]\)|[1-9][).]/u.test(text); +} + +async function solveForNormalization( + mathEngine: MathEngineClient, + equation: string, +): Promise<{ readonly solutions: readonly string[] } | null> { + try { + return await mathEngine.solve({ equation }); + } catch { + return null; + } +} diff --git a/packages/agent/src/steps/rag-search.ts b/packages/agent/src/steps/rag-search.ts index 6c291d6..0caff60 100644 --- a/packages/agent/src/steps/rag-search.ts +++ b/packages/agent/src/steps/rag-search.ts @@ -35,13 +35,81 @@ export async function ragSearch( source_problem_text: request.source_problem_text, k: Math.max(request.count, 8), }; - const refs = await deps.rag.search({ + const refs = await searchAligned(deps.rag, { ...baseQuery, difficulty: request.difficulty, problem_type: request.problem_type, }); if (refs.length > 0) return { refs }; - return { refs: await deps.rag.search(baseQuery) }; + const relaxedFormatRefs = await searchAligned(deps.rag, baseQuery); + if (relaxedFormatRefs.length > 0) return { refs: relaxedFormatRefs }; + + const relaxedTopicQuery = { ...baseQuery, topic_code: undefined }; + const relaxedRefs = await searchAligned(deps.rag, { + ...relaxedTopicQuery, + difficulty: request.difficulty, + problem_type: request.problem_type, + }); + if (relaxedRefs.length > 0) return { refs: relaxedRefs }; + + const broadRefs = await searchAligned(deps.rag, relaxedTopicQuery); + if (broadRefs.length > 0) return { refs: broadRefs }; + + for (const topicName of topicNameAliases(baseQuery.topic_code)) { + const aliasRefs = await searchAligned(deps.rag, { + ...relaxedTopicQuery, + topic_name: topicName, + }); + if (aliasRefs.length > 0) return { refs: aliasRefs }; + } + + return { refs: [] }; }, { ms: deps.perStepTimeoutMs ?? 30_000, label: "rag" }); } + +async function searchAligned( + rag: RagClient, + query: Parameters[0], +): Promise { + const refs = await rag.search(query); + return refs.filter((ref) => topicAligned(ref, query.topic_name)); +} + +function topicAligned(ref: RagResult, topicName: string | undefined): boolean { + if (topicName === undefined) return true; + const queryTokens = tokenize(topicName); + if (queryTokens.length <= 1) return tokenOverlapCount(queryTokens, topicEvidence(ref)) >= 1; + const overlap = tokenOverlapCount(queryTokens, topicEvidence(ref)); + return overlap >= 2 || overlap / queryTokens.length >= 0.4; +} + +function topicEvidence(ref: RagResult): string { + return [ + ref.problem.topic_name, + ref.problem.achievement_standard, + ref.problem.question_text, + ref.problem.answer_text, + ref.problem.explanation_text, + ].filter(Boolean).join("\n"); +} + +function tokenOverlapCount(queryTokens: string[], target: string): number { + const targetTokens = tokenize(target); + return queryTokens.filter((token) => + targetTokens.some((targetToken) => tokensMatch(token, targetToken)), + ).length; +} + +function tokenize(text: string): string[] { + return text.toLowerCase().split(/[^0-9a-zA-Z가-힣]+/u).filter((token) => token.length > 0); +} + +function tokensMatch(queryToken: string, targetToken: string): boolean { + return queryToken === targetToken || targetToken.includes(queryToken); +} + +function topicNameAliases(topicCode: string | undefined): string[] { + if (topicCode === "9수04-06") return ["원의 접선 성질"]; + return []; +} diff --git a/packages/agent/src/steps/sympy-verification.ts b/packages/agent/src/steps/sympy-verification.ts index 91d7b76..4dfaf0d 100644 --- a/packages/agent/src/steps/sympy-verification.ts +++ b/packages/agent/src/steps/sympy-verification.ts @@ -2,6 +2,7 @@ import type { GateResult, GeneratedProblem } from "../schemas/index.js"; import { withTimeout } from "../policies/timeout-policy.js"; +import { extractEquationText } from "../tools/equation-extractor.js"; import type { MathEngineClient } from "../tools/math-engine-client.js"; export interface SympyVerificationDeps { @@ -17,23 +18,28 @@ export interface SympyVerificationOutput { gate: GateResult; } +type VerificationCheck = { + readonly passed: boolean; + readonly verificationKind: GeneratedProblem["generation_kind"]; +}; + export async function verifyWithSympy( deps: SympyVerificationDeps, input: SympyVerificationInput, ): Promise { const started = Date.now(); try { - const passed = await withTimeout( + const check = await withTimeout( () => verifyCandidate(deps.mathEngine, input.candidate), { ms: deps.perStepTimeoutMs ?? 30_000, label: "sympy_verify" }, ); return { gate: { step: "sympy_verify", - status: passed ? "passed" : "failed", + status: check.passed ? "passed" : "failed", duration_ms: Date.now() - started, - evidence: { engine: "sympy" }, - failure_detail: passed + evidence: { engine: "sympy", verification_kind: check.verificationKind }, + failure_detail: check.passed ? undefined : { code: "sympy_solution_mismatch", @@ -60,12 +66,66 @@ export async function verifyWithSympy( async function verifyCandidate( mathEngine: MathEngineClient, candidate: GeneratedProblem, +): Promise { + if (candidate.generation_kind === "equation") { + if (isChoiceStyleCandidate(candidate)) { + return { + passed: candidate.question_text.trim().length > 0 && candidate.expected_answer.trim().length > 0, + verificationKind: candidate.generation_kind, + }; + } + return { + passed: await verifyEquationCandidate(mathEngine, candidate), + verificationKind: candidate.generation_kind, + }; + } + + if (candidate.generation_kind === "expression") { + await verifyExpressionAnswer(mathEngine, candidate.expected_answer); + } + + return { + passed: candidate.question_text.trim().length > 0 && candidate.expected_answer.trim().length > 0, + verificationKind: candidate.generation_kind, + }; +} + +async function verifyExpressionAnswer( + mathEngine: MathEngineClient, + answer: string, +): Promise { + const parts = parseExpressionAnswerParts(answer); + for (const part of parts) { + await mathEngine.simplify({ expr: part }); + } +} + +function parseExpressionAnswerParts(answer: string): string[] { + return answer + .replace(/(^|\s)\([1-9]\)\s+(?=\S)/gu, "$1;") + .split(/[,;]|또는|or|\n/u) + .map((part) => cleanExpressionAnswerPart(part)) + .filter((part) => /[0-9a-zA-Z]/u.test(part) && !/[가-힣]/u.test(part)); +} + +function cleanExpressionAnswerPart(part: string): string { + return part + .trim() + .replace(/^[①②③④⑤⑥⑦⑧⑨]\s*/u, "") + .replace(/(?<=\d)\s*(모둠|개|명|cm|kcal|도)$/u, "") + .trim(); +} + +async function verifyEquationCandidate( + mathEngine: MathEngineClient, + candidate: GeneratedProblem, ): Promise { - if (!candidate.question_text.includes("=")) { - throw new Error("SymPy verification supports equation candidates only"); + const equation = extractEquationText(candidate.question_text); + if (equation === null) { + throw new Error("Equation verification requires an equation candidate"); } - const solved = await mathEngine.solve({ equation: candidate.question_text }); + const solved = await mathEngine.solve({ equation }); if (solved.solutions.length === 0) { throw new Error("SymPy returned no solutions"); } @@ -80,6 +140,13 @@ async function verifyCandidate( return sameSet(actualCanonical, expectedCanonical); } +function isChoiceStyleCandidate(candidate: GeneratedProblem): boolean { + return ( + /[①②③④⑤⑥⑦⑧⑨]|\([1-9]\)|[1-9][).]/u.test(candidate.question_text) && + /^(?:[①②③④⑤⑥⑦⑧⑨]|\([1-9]\)|[1-9][).])\s*/u.test(candidate.expected_answer.trim()) + ); +} + function parseExpectedSolutions(answer: string): string[] { return answer .split(/[,;]|또는|or/) diff --git a/packages/agent/src/tools/answer-equivalence.ts b/packages/agent/src/tools/answer-equivalence.ts new file mode 100644 index 0000000..c25e12e --- /dev/null +++ b/packages/agent/src/tools/answer-equivalence.ts @@ -0,0 +1,277 @@ +import type { GeneratedProblem } from "../schemas/index.js"; +import type { MathEngineClient } from "./math-engine-client.js"; +import { extractEquationText } from "./equation-extractor.js"; +import { fractionFromRepeatingDecimalQuestion, sameFractionText } from "./repeating-decimal.js"; + +export async function sameAnswer( + mathEngine: MathEngineClient, + candidate: GeneratedProblem, + derivedAnswer: string, +): Promise { + const expectedGraphAnswer = normalizeGraphAnswer(candidate.expected_answer); + const derivedGraphAnswer = normalizeGraphAnswer(derivedAnswer); + if (expectedGraphAnswer !== null && expectedGraphAnswer === derivedGraphAnswer) return true; + + const expected = parseAnswers(candidate.expected_answer); + const derived = parseAnswers(derivedAnswer); + if (expected.length === 0 || derived.length === 0) return false; + const orderedExpected = /(작은|큰)\s*것부터/u.test(candidate.question_text) && expected.length === 1 ? expected[0]?.split("<").map((part) => part.trim()).filter((part) => part.length > 0) : undefined; + if (orderedExpected !== undefined && orderedExpected.length === derived.length && orderedExpected.every((answer, index) => normalizeAnswerText(answer) === normalizeAnswerText(derived[index] ?? ""))) return true; + + const expectedAlternatives = uniqueAnswers([ + ...expected, + ...choiceAnswerAlternatives(candidate), + ...prefixedChoiceAlternatives(expected), + ...choiceLabelAlternatives(expected), + ]); + const derivedAlternatives = uniqueAnswers([ + ...derived, + ...prefixedChoiceAlternatives(derived), + ...choiceLabelAlternatives(derived), + ]); + if ( + expected.length === 1 || + derived.length === 1 || + expectedAlternatives.length > expected.length || + derivedAlternatives.length > derived.length + ) { + if (await answerListsOverlap(mathEngine, expectedAlternatives, derivedAlternatives)) return true; + } + + if (sameNormalizedSet(expected, derived)) return true; + if (await expectedMatchesEquationSolve(mathEngine, candidate, expected)) return true; + if (expectedMatchesRepeatingDecimalQuestion(candidate, expected)) return true; + + const expectedCanonical = await tryCanonicalizeAll(mathEngine, expected); + const derivedCanonical = await tryCanonicalizeAll(mathEngine, derived); + if (expectedCanonical === null || derivedCanonical === null) return false; + if (expectedCanonical.length !== derivedCanonical.length) return false; + return expectedCanonical.every((value, index) => value === derivedCanonical[index]); +} + +function expectedMatchesRepeatingDecimalQuestion( + candidate: GeneratedProblem, + expected: readonly string[], +): boolean { + const fraction = fractionFromRepeatingDecimalQuestion(candidate.question_text); + if (fraction === null) return false; + return expected.some((answer) => sameFractionText(answer, fraction)); +} + +function parseAnswers(answer: string): string[] { + return answer.replace(/\s*\([^)]*%[^)]*\)/gu, "") + .replace(/(^|\s)\([1-9]\)\s+(?=\S)/gu, "$1;") + .split(/[,;]|또는|or|\s+\/\s+/) + .map((part) => part.trim().replace(/^[xyXY]\s*=\s*/, "").replace(/^[abAB]\s*=\s*\d+\s*일\s*때\s*/u, "").replace(/\s*\([^)]*%\)\s*$/u, "")) + .filter((part) => part.length > 0 && !/^(?:예|네)$/u.test(part) && !/^[abAB]\s*=\s*\d+$/u.test(part) && !/해가\s*아(?:님|니다)/u.test(part)); +} + +async function canonicalizeAll(mathEngine: MathEngineClient, answers: readonly string[]): Promise { + const canonical = await Promise.all( + answers.map(async (answer) => { + const result = await mathEngine.simplify({ expr: answer }); + return result.simplified.replace(/\s+/g, ""); + }), + ); + return canonical.sort(); +} + +async function tryCanonicalizeAll(mathEngine: MathEngineClient, answers: readonly string[]): Promise { + try { + return await canonicalizeAll(mathEngine, answers); + } catch { + return null; + } +} + +function sameNormalizedSet(left: readonly string[], right: readonly string[]): boolean { + const normalizedLeft = left.map((answer) => normalizeAnswerText(answer)).sort(); + const normalizedRight = right.map((answer) => normalizeAnswerText(answer)).sort(); + if (normalizedLeft.length !== normalizedRight.length) return false; + return normalizedLeft.every((answer, index) => answer === normalizedRight[index]); +} + +async function answerListsOverlap( + mathEngine: MathEngineClient, + expected: readonly string[], + derived: readonly string[], +): Promise { + for (const expectedAnswer of expected) { + for (const derivedAnswer of derived) { + if (normalizeAnswerText(expectedAnswer) === normalizeAnswerText(derivedAnswer)) { + return true; + } + if (await sameCanonicalAnswer(mathEngine, expectedAnswer, derivedAnswer)) { + return true; + } + if ( + await sameLinearFactorRoot(mathEngine, expectedAnswer, derivedAnswer) || + await sameLinearFactorRoot(mathEngine, derivedAnswer, expectedAnswer) + ) { + return true; + } + } + } + return false; +} + +async function sameCanonicalAnswer(mathEngine: MathEngineClient, left: string, right: string): Promise { + const canonical = await tryCanonicalizeAll(mathEngine, [left, right]); + if (canonical !== null && canonical[0] === canonical[1]) return true; + try { + return (await mathEngine.verify({ expr1: left, expr2: right })).equivalent; + } catch { + return false; + } +} + +async function sameLinearFactorRoot(mathEngine: MathEngineClient, factorAnswer: string, rootAnswer: string): Promise { + if (!/[a-zA-Z]/u.test(factorAnswer)) return false; + try { + const solved = await mathEngine.solve({ equation: `${factorAnswer}=0` }); + if (solved.solutions.length === 0) return false; + const roots = await canonicalizeAll(mathEngine, solved.solutions); + const answers = await canonicalizeAll(mathEngine, parseAnswers(rootAnswer)); + return roots.some((root) => answers.includes(root)); + } catch { + return false; + } +} + +async function expectedMatchesEquationSolve( + mathEngine: MathEngineClient, + candidate: GeneratedProblem, + expected: readonly string[], +): Promise { + if (candidate.generation_kind !== "equation") return false; + const equation = extractEquationText(candidate.question_text); + if (equation === null) return false; + try { + const solved = await mathEngine.solve({ equation }); + if (solved.solutions.length === 0) return false; + return sameNormalizedSet(expected, solved.solutions) || await sameCanonicalSets(mathEngine, expected, solved.solutions); + } catch { + return false; + } +} + +async function sameCanonicalSets( + mathEngine: MathEngineClient, + left: readonly string[], + right: readonly string[], +): Promise { + const leftCanonical = await tryCanonicalizeAll(mathEngine, left); + const rightCanonical = await tryCanonicalizeAll(mathEngine, right); + if (leftCanonical === null || rightCanonical === null) return false; + if (leftCanonical.length !== rightCanonical.length) return false; + return leftCanonical.every((value, index) => value === rightCanonical[index]); +} + +function choiceAnswerAlternatives(candidate: GeneratedProblem): string[] { + const expected = parseAnswers(candidate.expected_answer); + const choices = extractChoices(candidate.question_text); + const alternatives = new Set(expected); + for (const answer of expected) { + const stripped = stripChoicePrefix(answer); + if (stripped !== answer) { + alternatives.add(stripped); + } + const choice = choices.find((item) => choiceLabelsMatch(item.label, answer)); + if (choice !== undefined) { + alternatives.add(choice.body); + } + } + return [...alternatives]; +} + +function prefixedChoiceAlternatives(answers: readonly string[]): string[] { + return answers + .map((answer) => stripChoicePrefix(answer)) + .filter((answer) => answer.length > 0); +} + +function choiceLabelAlternatives(answers: readonly string[]): string[] { + return answers + .flatMap((answer) => { + const label = answer.trim().match(/^(?:[①②③④⑤⑥⑦⑧⑨]|\([1-9]\)|[1-9][).]|[1-9]번)/u)?.[0] ?? ""; + const index = choiceIndex(label); + return index === null ? [label] : [label, `${index + 1}`]; + }) + .filter((answer) => answer.length > 0); +} + +function uniqueAnswers(answers: readonly string[]): string[] { + return [...new Set(answers)]; +} + +function extractChoices(question: string): Array<{ readonly label: string; readonly body: string }> { + const labelPattern = "(? ({ + label: match[1] ?? "", + body: (match[2] ?? "").trim(), + })).filter((choice) => choiceIndex(choice.label) !== null && choice.body.length > 0); +} + +function choiceLabelsMatch(label: string, answer: string): boolean { + const index = choiceIndex(label); + if (index === null) return false; + const normalized = answer.trim(); + return ( + normalized.startsWith(label) || + normalized.startsWith(circledChoiceLabel(index)) || + normalized === `${index + 1}` || + normalized === `${index + 1}번` + ); +} + +function stripChoicePrefix(answer: string): string { + const labelPattern = /^(?:[①②③④⑤⑥⑦⑧⑨]|\([1-9]\)|[1-9][).]|[1-9]번)\s*/u; + return answer.replace(labelPattern, "").trim(); +} + +function choiceIndex(label: string): number | null { + const circled = ["①", "②", "③", "④", "⑤", "⑥", "⑦", "⑧", "⑨"]; + const circledIndex = circled.indexOf(label); + if (circledIndex >= 0) return circledIndex; + const digit = label.match(/[1-9]/u)?.[0]; + return digit === undefined ? null : Number.parseInt(digit, 10) - 1; +} + +function circledChoiceLabel(index: number): string { + const labels = ["①", "②", "③", "④", "⑤", "⑥", "⑦", "⑧", "⑨"]; + return labels[index] ?? ""; +} + +function normalizeAnswerText(answer: string): string { + const normalized = answer + .replace(/(?:\\?sqrt\{([^{}]+)\}|√\s*([0-9]+))/gu, (_match, latex, unicode) => `sqrt(${latex ?? unicode})`) + .replace(/sqrt([0-9]+)/gu, "sqrt($1)") + .replace(/0\.(\d)\1{2,}\.\.\./gu, "0.($1)") + .replace(/[\s*]+/g, "") + .replace(/^[가-힣A-Z∠]+[A-Z]*=|^\d+=/u, "") + .replace(/^[가-힣A-Z∠:]+(?=[+\-]?\d|sqrt|\\sqrt)/u, "") + .replace(/(모둠|개|명|cm|kcal|도|원)$/u, "") + .replace(/(?:모둠|개|명|cm|kcal|도|원)?(?:이고|이다|입니다)[.]?$/u, "") + .replace(/^[㈎-㈖]|\([가-힣]\)|^식의값은|^선분/u, "") + .replace(/(덧셈의|곱셈의)|x가증가하면y는(증가|감소)/gu, (_value, _law, trend) => trend ?? "") + .replace(/(?:위|아래)로열린(?:다|포물선)/gu, (value) => `${value.startsWith("위") ? "위" : "아래"}로열림`) + .replace(/^(위|아래)로$/u, "$1로열림") + .replace(/한점에서만난다|.*접한다(?:\(.*\))?\.?|.*접선.*|.*두점에서만난다.*/gu, (value) => value.includes("두점") ? "twopoints" : "tangent") + .replace(/실수범위에서(?:는)?정의되지않음|실수가아니다/gu, "notreal") + .toLowerCase(); + return sortPositiveAdditionTerms(normalized); +} + +function normalizeGraphAnswer(answer: string): string | null { + const normalized = answer.replace(/\s+/g, "").replace(/[,.，。]/gu, ""); + if (!(normalized.includes("그래프위의점") || normalized.includes("그래프위에있") || /^(예|맞다)/u.test(normalized))) return null; + if (/아래로열(?:린다|린포물선)/u.test(normalized)) return "그래프위의점+아래로열림"; + if (/위로열(?:린다|린포물선)/u.test(normalized)) return "그래프위의점+위로열림"; + return null; +} + +function sortPositiveAdditionTerms(value: string): string { + if (!value.includes("+") || value.includes("-") || /[=<>]/u.test(value)) return value; + return value.split("+").sort().join("+"); +} diff --git a/packages/agent/src/tools/equation-extractor.ts b/packages/agent/src/tools/equation-extractor.ts new file mode 100644 index 0000000..6b204d8 --- /dev/null +++ b/packages/agent/src/tools/equation-extractor.ts @@ -0,0 +1,13 @@ +const EQUATION_PATTERN = /([0-9A-Za-z\\{}\s+\-*/^().]+=[0-9A-Za-z\\{}\s+\-*/^().]+)/u; + +export function extractEquationText(text: string): string | null { + const match = text.match(EQUATION_PATTERN); + const equation = match?.[1]?.trim(); + if (equation === undefined || !equation.includes("=")) return null; + return equation + .replace(/^[\s.,:;!?]+/u, "") + .replace(/\.\s*\(.+$/u, "") + .replace(/\.\s*\($/u, "") + .replace(/\.$/u, "") + .replace(/\s+/g, " "); +} diff --git a/packages/agent/src/tools/math-engine-client.ts b/packages/agent/src/tools/math-engine-client.ts index e598695..ce05fea 100644 --- a/packages/agent/src/tools/math-engine-client.ts +++ b/packages/agent/src/tools/math-engine-client.ts @@ -6,6 +6,8 @@ * 디스커버리: 환경변수 MATH_ENGINE_URL (현재 하드코딩, Q-7 closure 후 조정). */ +import { toSympyInput } from "./sympy-input.js"; + export interface SolveRequest { equation: string; variable?: string; @@ -122,11 +124,22 @@ export function createMathEngineClient( return { health: () => request("/health", { method: "GET" }, parseHealthResponse), - solve: (req) => post("/solve", req, parseSolveResponse), - verify: (req) => post("/verify", req, parseVerifyResponse), - simplify: (req) => post("/simplify", req, parseSimplifyResponse), + solve: (req) => + post("/solve", { ...req, equation: toSympyInput(req.equation) }, parseSolveResponse), + verify: (req) => + post( + "/verify", + { expr1: toSympyInput(req.expr1), expr2: toSympyInput(req.expr2) }, + parseVerifyResponse, + ), + simplify: (req) => + post("/simplify", { expr: toSympyInput(req.expr) }, parseSimplifyResponse), differentiate: (req) => - post("/differentiate", req, parseDifferentiateResponse), + post( + "/differentiate", + { ...req, expr: toSympyInput(req.expr) }, + parseDifferentiateResponse, + ), limit: (req) => post("/limit", req, parseLimitResponse), }; } diff --git a/packages/agent/src/tools/rag-client.ts b/packages/agent/src/tools/rag-client.ts index d30f5e5..cd8f1bd 100644 --- a/packages/agent/src/tools/rag-client.ts +++ b/packages/agent/src/tools/rag-client.ts @@ -36,6 +36,7 @@ type OpenMathRagRecord = { semester?: 1 | 2 | null; topic_code?: string | null; topic_name: string; + achievement_code?: string | null; achievement_standard?: string | null; achievement_confidence?: number; }; @@ -163,7 +164,7 @@ function toIndexedProblem(record: OpenMathRagRecord): IndexedProblem { school_level: record.curriculum.school_level, grade: record.curriculum.grade, semester: record.curriculum.semester ?? null, - topic_code: record.curriculum.topic_code ?? null, + topic_code: record.curriculum.achievement_code ?? record.curriculum.topic_code ?? null, topic_name: record.curriculum.topic_name, achievement_standard: record.curriculum.achievement_standard ?? null, question_text: record.problem.question_text, @@ -216,8 +217,8 @@ function matchesQuery( } if ( query.topic_name && - !containsNormalized(searchableText(row), query.topic_name) && - tokenOverlap(query.topic_name, searchableText(row)) === 0 + !containsNormalized(topicAlignmentText(row), query.topic_name) && + tokenOverlap(query.topic_name, topicAlignmentText(row)) === 0 ) { return false; } @@ -296,6 +297,19 @@ function searchableText(row: IndexedProblem): string { .join("\n"); } +function topicAlignmentText(row: IndexedProblem): string { + return [ + row.problem.topic_name, + row.problem.question_text, + row.problem.answer_text, + row.problem.explanation_text, + row.retrievalText.split("\n").find((line) => line.startsWith("단원:")), + row.embeddingText.split("\n")[0], + ] + .filter(Boolean) + .join("\n"); +} + function containsNormalized(haystack: string, needle: string): boolean { return normalizeText(haystack).includes(normalizeText(needle)); } @@ -332,8 +346,7 @@ function normalizeText(text: string): string { function tokensMatch(queryToken: string, targetToken: string): boolean { return ( queryToken === targetToken || - (queryToken.length >= 2 && targetToken.includes(queryToken)) || - (targetToken.length >= 2 && queryToken.includes(targetToken)) + (queryToken.length >= 2 && targetToken.includes(queryToken)) ); } diff --git a/packages/agent/src/tools/repeating-decimal.ts b/packages/agent/src/tools/repeating-decimal.ts new file mode 100644 index 0000000..79899fd --- /dev/null +++ b/packages/agent/src/tools/repeating-decimal.ts @@ -0,0 +1,73 @@ +export interface RepeatingDecimalFraction { + readonly numerator: string; + readonly denominator: string; +} + +export function fractionFromRepeatingDecimalQuestion(question: string): RepeatingDecimalFraction | null { + const match = question.match(/순환소수\s*([0-9]+)\.([0-9]+)\.\.\./u); + if (match === null) return null; + const integerPart = match[1] ?? ""; + const decimalPart = match[2] ?? ""; + const repeating = detectRepeatingTail(decimalPart); + if (integerPart.length === 0 || repeating === null) return null; + + const nonRepeatingDigits = decimalPart.slice(0, decimalPart.length - repeating.length); + const nonRepeating = nonRepeatingDigits.length === 0 ? 0n : BigInt(nonRepeatingDigits); + const repeatValue = BigInt(repeating); + const nonRepeatingScale = 10n ** BigInt(nonRepeatingDigits.length); + const repeatScale = 10n ** BigInt(repeating.length) - 1n; + const denominator = nonRepeatingScale * repeatScale; + const numerator = BigInt(integerPart) * denominator + nonRepeating * repeatScale + repeatValue; + const divisor = gcd(abs(numerator), denominator); + return { + numerator: (numerator / divisor).toString(), + denominator: (denominator / divisor).toString(), + }; +} + +export function sameFractionText(left: string, right: RepeatingDecimalFraction): boolean { + const match = left.trim().match(/^([+-]?\d+)\s*\/\s*(\d+)$/u); + if (match === null) return false; + const numerator = BigInt(match[1] ?? "0"); + const denominator = BigInt(match[2] ?? "1"); + if (denominator === 0n) return false; + const divisor = gcd(abs(numerator), denominator); + return ( + (numerator / divisor).toString() === right.numerator && + (denominator / divisor).toString() === right.denominator + ); +} + +function detectRepeatingTail(decimalPart: string): string | null { + for (let width = 1; width <= Math.min(4, Math.floor(decimalPart.length / 2)); width += 1) { + const block = decimalPart.slice(decimalPart.length - width); + const repeated = repeatedSuffixLength(decimalPart, block); + if (repeated >= width * 2) return block; + } + return null; +} + +function repeatedSuffixLength(value: string, block: string): number { + let cursor = value.length; + let total = 0; + while (cursor >= block.length && value.slice(cursor - block.length, cursor) === block) { + total += block.length; + cursor -= block.length; + } + return total; +} + +function gcd(left: bigint, right: bigint): bigint { + let a = left; + let b = right; + while (b !== 0n) { + const next = a % b; + a = b; + b = next; + } + return a; +} + +function abs(value: bigint): bigint { + return value < 0n ? -value : value; +} diff --git a/packages/agent/src/tools/sympy-input.ts b/packages/agent/src/tools/sympy-input.ts new file mode 100644 index 0000000..13fa569 --- /dev/null +++ b/packages/agent/src/tools/sympy-input.ts @@ -0,0 +1,101 @@ +export function toSympyInput(value: string): string { + return insertImplicitMultiplication( + replaceLatexCommands(value) + .replace(/sqrt\s*\{([^{}]+)\}/g, "sqrt($1)") + .replace(/\\left|\\right/g, "") + .replace(/\\cdot|\\times/g, "*") + .replace(/−/g, "-") + .replace(/\^\s*\{([^{}]+)\}/g, "**($1)") + .replace(/\^\s*([A-Za-z0-9]+)/g, "**$1") + .replace(/\\[,;!]/g, " "), + ).trim(); +} + +function insertImplicitMultiplication(value: string): string { + return value + .replace(/\)\s*\(/g, ")*(") + .replace(/([0-9])([A-Za-z])/g, "$1*$2") + .replace(/([0-9])\s+([A-Za-z])/g, "$1*$2") + .replace(/([0-9])\s*\(/g, "$1*(") + .replace(/\)\s*([A-Za-z0-9])/g, ")*$1") + .replace(/([A-Za-z])\s+\(/g, "$1*("); +} + +function replaceLatexCommands(value: string): string { + return replaceLatexCommand( + replaceLatexCommand(value, "sqrt", (parts) => { + const inner = parts[0] ?? ""; + return `sqrt(${toSympyInput(inner)})`; + }), + "frac", + (parts) => { + const numerator = parts[0] ?? ""; + const denominator = parts[1] ?? ""; + return `(${toSympyInput(numerator)})/(${toSympyInput(denominator)})`; + }, + ); +} + +function replaceLatexCommand( + value: string, + command: string, + render: (parts: readonly string[]) => string, +): string { + const needle = `\\${command}`; + let output = ""; + let index = 0; + while (index < value.length) { + const start = value.indexOf(needle, index); + if (start === -1) { + output += value.slice(index); + break; + } + output += value.slice(index, start); + const parts = readBraceParts(value, start + needle.length); + if (parts === null) { + output += value.slice(start, start + needle.length); + index = start + needle.length; + continue; + } + output += render(parts.values); + index = parts.endIndex + 1; + } + return output; +} + +function readBraceParts( + value: string, + startIndex: number, +): { readonly values: readonly string[]; readonly endIndex: number } | null { + const values: string[] = []; + let index = skipSpaces(value, startIndex); + while (value[index] === "{") { + const endIndex = findMatchingBrace(value, index); + if (endIndex === -1) return null; + values.push(value.slice(index + 1, endIndex)); + index = skipSpaces(value, endIndex + 1); + } + if (values.length === 0) return null; + return { values, endIndex: index - 1 }; +} + +function skipSpaces(value: string, startIndex: number): number { + let index = startIndex; + while (value[index] === " ") { + index += 1; + } + return index; +} + +function findMatchingBrace(value: string, openIndex: number): number { + let depth = 0; + for (let index = openIndex; index < value.length; index += 1) { + const char = value[index]; + if (char === "{") depth += 1; + if (char === "}") { + depth -= 1; + if (depth === 0) return index; + } + } + return -1; +} diff --git a/packages/agent/tests/backend-contract.test.ts b/packages/agent/tests/backend-contract.test.ts index 71f661d..68250fa 100644 --- a/packages/agent/tests/backend-contract.test.ts +++ b/packages/agent/tests/backend-contract.test.ts @@ -61,6 +61,36 @@ describe("SSE wire adapter", () => { }); }); + it("includes failed gate details in step summaries", () => { + const wire = toWireSseEvent({ + type: "step", + step: "objective_map", + status: "done", + timestamp: "2026-05-21T00:00:00.000Z", + data: { + gate: { + step: "objective_map", + status: "failed", + duration_ms: 1, + failure_detail: { + code: "intent_topic_mismatch", + message: "Intent does not match requested topic", + }, + }, + }, + }); + + expect(wire).toEqual({ + event: "step", + data: { + index: 6, + name: "학습 목표 매핑", + status: "failed", + summary: "intent_topic_mismatch: Intent does not match requested topic", + }, + }); + }); + it("sanitizes retry reasons and internal errors", () => { const retry = toWireSseEvent({ type: "retry", @@ -99,6 +129,7 @@ describe("SSE wire adapter", () => { const problem: GeneratedProblem = { candidate_id: "00000000-0000-0000-0000-000000000001", mode: "structural", + generation_kind: "equation", question_text: "x**2 - 5*x + 6 = 0", expected_answer: "2, 3", proposed_solution_trace: "(x - 2)(x - 3) = 0", @@ -146,6 +177,7 @@ describe("SSE wire adapter", () => { if (wire.event !== "result") throw new Error("expected result event"); expect(wire.data[0]?.id).toBe(problem.candidate_id); expect(wire.data[0]?.verification_status).toBe("pass"); + expect(wire.data[0]?.source_refs).toEqual(["seed-9수02-09-001"]); expect(wire.data[0]?.explanation_latex).toBeUndefined(); expect(wire.data[0]?.preserved_dimensions).toEqual([ "이차식을 인수분해하여 해를 구한다", @@ -186,6 +218,34 @@ describe("SSE wire adapter", () => { }, ]); }); + + it("turns pre-yield workflow errors into frontend-visible SSE errors", async () => { + const written: Array<{ event?: string; data: string | Promise }> = []; + const stream = { + async writeSSE(message: { event?: string; data: string | Promise }) { + written.push(message); + }, + }; + + async function* events(): AsyncGenerator { + throw new Error("Verification workflow requires intentModel"); + } + + await pipeProgressToSse( + stream as Parameters[0], + events(), + ); + + expect(written).toEqual([ + { + event: "error", + data: JSON.stringify({ + stage: "orchestrator", + message: "검증 파이프라인 오류가 발생했습니다. 잠시 후 다시 시도해 주세요.", + }), + }, + ]); + }); }); describe("filesystem loaders", () => { @@ -263,6 +323,7 @@ describe("SymPy verification fail-closed behavior", () => { const candidate: GeneratedProblem = { candidate_id: "00000000-0000-0000-0000-000000000004", mode: "structural", + generation_kind: "equation", question_text: "x**2 - 5*x + 6 = 0", expected_answer: "2, 3", proposed_solution_trace: "internal trace", @@ -318,6 +379,48 @@ describe("SymPy verification fail-closed behavior", () => { expect(result.gate.status).toBe("passed"); }); + it("routes expression candidates through expression verification", async () => { + const expressionCandidate: GeneratedProblem = { + ...candidate, + candidate_id: "00000000-0000-0000-0000-000000000005", + generation_kind: "expression", + question_text: "다항식 (x + 3)(x - 5)를 전개하시오.", + expected_answer: "x**2 - 2*x - 15", + }; + + const result = await verifyWithSympy( + { mathEngine: fakeMathEngine([]) }, + { candidate: expressionCandidate }, + ); + + expect(result.gate.status).toBe("passed"); + expect(result.gate.evidence).toMatchObject({ + verification_kind: "expression", + }); + }); + + it("routes choice-label equation candidates through non-solve verification", async () => { + const choiceCandidate: GeneratedProblem = { + ...candidate, + question_text: + "다음 중 이차방정식의 꼴로 정리할 수 없는 것은? ① x^2=1 ② x^2+x=0 ③ 2x^2=3 ④ x^2+1=x^2+2", + expected_answer: "④", + }; + const mathEngine: MathEngineClient = { + ...fakeMathEngine([]), + solve: async () => { + throw new Error("solve should not be called"); + }, + }; + + const result = await verifyWithSympy( + { mathEngine }, + { candidate: choiceCandidate }, + ); + + expect(result.gate.status).toBe("passed"); + }); + it("fails empty solution sets", async () => { const result = await verifyWithSympy( { mathEngine: fakeMathEngine([]) }, diff --git a/packages/agent/tests/constraint-critic-agent.test.ts b/packages/agent/tests/constraint-critic-agent.test.ts new file mode 100644 index 0000000..bd07a78 --- /dev/null +++ b/packages/agent/tests/constraint-critic-agent.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from "vitest"; + +import { parseCritiqueJson } from "../src/agents/constraint-critic-agent.js"; + +describe("parseCritiqueJson", () => { + it("repairs raw LaTeX backslashes before parsing critique JSON", () => { + const modelTextWithRawLatexEscapes = + '{"passes":false,"hints":["수식 예시는 \\sqrt{2} 대신 plain text로 설명하세요.","\\(x-1\\) 표기는 쓰지 마세요."]}'; + + const critique = parseCritiqueJson(modelTextWithRawLatexEscapes); + + expect(critique).toEqual({ + passes: false, + hints: [ + "수식 예시는 \\sqrt{2} 대신 plain text로 설명하세요.", + "\\(x-1\\) 표기는 쓰지 마세요.", + ], + }); + }); +}); diff --git a/packages/agent/tests/env.test.ts b/packages/agent/tests/env.test.ts new file mode 100644 index 0000000..79ce3ff --- /dev/null +++ b/packages/agent/tests/env.test.ts @@ -0,0 +1,66 @@ +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { loadEnv } from "../src/config/env.js"; + +const ENV_KEYS = [ + "CLIPROXY_BASE_URL", + "CLIPROXY_API_KEY", + "CLIPROXY_MODEL", + "LLM_BASE_URL", + "LLM_API_KEY", + "LLM_MODEL", +] as const; + +const savedEnv = new Map(); +let originalCwd = ""; +let tempRoot = ""; + +describe("loadEnv", () => { + beforeEach(() => { + originalCwd = process.cwd(); + tempRoot = mkdtempSync(join(tmpdir(), "openmath-env-")); + for (const key of ENV_KEYS) { + savedEnv.set(key, process.env[key]); + delete process.env[key]; + } + }); + + afterEach(() => { + process.chdir(originalCwd); + rmSync(tempRoot, { recursive: true, force: true }); + for (const key of ENV_KEYS) { + const value = savedEnv.get(key); + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } + } + savedEnv.clear(); + }); + + it("loads package env when launched from the monorepo root", () => { + const agentDir = join(tempRoot, "packages", "agent"); + mkdirSync(agentDir, { recursive: true }); + writeFileSync( + join(agentDir, ".env"), + [ + "LLM_PROVIDER=cliproxy", + "CLIPROXY_BASE_URL=http://localhost:8317/v1", + "CLIPROXY_API_KEY=test-key", + "CLIPROXY_MODEL=test-model", + ].join("\n"), + ); + process.chdir(tempRoot); + + const env = loadEnv(); + + expect(env.CLIPROXY_BASE_URL).toBe("http://localhost:8317/v1"); + expect(env.CLIPROXY_API_KEY).toBe("test-key"); + expect(env.CLIPROXY_MODEL).toBe("test-model"); + }); +}); diff --git a/packages/agent/tests/equation-extractor.test.ts b/packages/agent/tests/equation-extractor.test.ts new file mode 100644 index 0000000..06d1afb --- /dev/null +++ b/packages/agent/tests/equation-extractor.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from "vitest"; + +import { extractEquationText } from "../src/tools/equation-extractor.js"; + +describe("extractEquationText", () => { + it("extracts a solvable equation from Korean problem prose", () => { + expect(extractEquationText("다음 방정식을 풀어라. x^{2} - 5x + 6 = 0")).toBe( + "x^{2} - 5x + 6 = 0", + ); + }); + + it("preserves leading parentheses in extracted equations", () => { + expect(extractEquationText("다음 이차방정식의 해를 구하시오. (x-1)(x+4)=4 x+2")).toBe( + "(x-1)(x+4)=4 x+2", + ); + }); + + it("drops trailing Korean instructions after the equation", () => { + expect( + extractEquationText("다음 이차방정식의 해를 구하시오. 4 x^{2} - 4 x - 3 = 0. (구한 해를 원래 식에 대입하여 확인할 것)"), + ).toBe("4 x^{2} - 4 x - 3 = 0"); + }); + + it("returns null when a problem has no equation", () => { + expect(extractEquationText("다음 중 두 다항식의 공통 인수를 고르시오.")).toBeNull(); + }); +}); diff --git a/packages/agent/tests/generation-kind.test.ts b/packages/agent/tests/generation-kind.test.ts new file mode 100644 index 0000000..a8a5110 --- /dev/null +++ b/packages/agent/tests/generation-kind.test.ts @@ -0,0 +1,20 @@ +import { describe, expect, it } from "vitest"; + +import { generationKindForTopic } from "../src/schemas/index.js"; + +describe("generationKindForTopic", () => { + it("maps app topics to verification-oriented generation kinds", () => { + expect(generationKindForTopic("9수02-03")).toBe("equation"); + expect(generationKindForTopic("9수02-06")).toBe("inequality"); + expect(generationKindForTopic("9수02-07")).toBe("system"); + expect(generationKindForTopic("9수02-08")).toBe("expression"); + expect(generationKindForTopic("9수03-04")).toBe("function"); + expect(generationKindForTopic("9수04-05")).toBe("geometry"); + expect(generationKindForTopic("9수05-02")).toBe("probability"); + expect(generationKindForTopic("9수05-03")).toBe("statistics"); + }); + + it("falls back to expression for unknown middle-school topics", () => { + expect(generationKindForTopic("9수99-99")).toBe("expression"); + }); +}); diff --git a/packages/agent/tests/independent-resolve-choice-labels.test.ts b/packages/agent/tests/independent-resolve-choice-labels.test.ts new file mode 100644 index 0000000..5b283e7 --- /dev/null +++ b/packages/agent/tests/independent-resolve-choice-labels.test.ts @@ -0,0 +1,83 @@ +import { describe, expect, it } from "vitest"; + +import type { SolverAgent } from "../src/agents/index.js"; +import type { GeneratedProblem } from "../src/schemas/index.js"; +import { independentResolve } from "../src/steps/independent-resolve.js"; +import type { MathEngineClient } from "../src/tools/math-engine-client.js"; + +const candidate: GeneratedProblem = { + candidate_id: "00000000-0000-0000-0000-000000000010", + mode: "structural", + generation_kind: "expression", + question_text: "다음 중 옳은 것을 고르시오.", + expected_answer: "3번", + proposed_solution_trace: "선택지를 비교한다.", + source_refs: ["ref-1"], + inferred_intent: { + objective_code: "9수01-02", + objective_description: "정수와 유리수를 이해한다.", + evaluation_dimensions: [{ id: "A", description: "선택지 판별", must_preserve: true }], + required_techniques: ["choice_selection"], + forbidden_techniques: [], + surface_constraints: { difficulty: "medium", problem_type: "objective" }, + }, + generation_metadata: { + model: "test", + temperature: 0, + prompt_id: "test", + prompt_version: "0.0.0", + attempt: 1, + generated_at: "2026-06-06T00:00:00.000Z", + }, +}; + +const mathEngine: MathEngineClient = { + health: async () => ({ status: "ok", engine: "sympy" }), + solve: async () => ({ solutions: [] }), + verify: async () => ({ equivalent: true, diff: "0" }), + simplify: async ({ expr }) => ({ simplified: expr.replace(/\s+/g, "") }), + differentiate: async () => ({ derivative: "" }), + limit: async () => ({ limit: "" }), +}; + +describe("independentResolve choice label equivalence", () => { + it("passes when a selected parenthesized choice is written as a Korean choice number", async () => { + const solver: SolverAgent = { + solve: async () => ({ derived_answer: "(3)", trace: "정답은 (3)이다.", confidence: "high" }), + }; + + const result = await independentResolve( + { solver, mathEngine }, + { + candidate, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + + expect(result.gate.status).toBe("passed"); + }); + + it("passes when a parenthesized choice answer includes the selected body text", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "(4) 0.101001000100001...", + trace: "순환하지 않는 무한소수이므로 유리수가 아니다.", + confidence: "high", + }), + }; + + const result = await independentResolve( + { solver, mathEngine }, + { + candidate: { + ...candidate, + expected_answer: "④", + question_text: "다음 중 유리수가 아닌 것은? (1) -3/8 (2) 0.45 (3) 2.131313... (4) 0.101001000100001... (5) -7", + }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + + expect(result.gate.status).toBe("passed"); + }); +}); diff --git a/packages/agent/tests/independent-resolve-geometry-function-equivalence.test.ts b/packages/agent/tests/independent-resolve-geometry-function-equivalence.test.ts new file mode 100644 index 0000000..00990e3 --- /dev/null +++ b/packages/agent/tests/independent-resolve-geometry-function-equivalence.test.ts @@ -0,0 +1,94 @@ +import { describe, expect, it } from "vitest"; + +import type { SolverAgent } from "../src/agents/index.js"; +import type { GeneratedProblem } from "../src/schemas/index.js"; +import { independentResolve } from "../src/steps/independent-resolve.js"; +import type { MathEngineClient } from "../src/tools/math-engine-client.js"; + +const candidate: GeneratedProblem = { + candidate_id: "00000000-0000-0000-0000-000000000041", + mode: "structural", + generation_kind: "geometry", + question_text: "값을 구하시오.", + expected_answer: "13cm, 90도", + proposed_solution_trace: "성질을 이용한다.", + source_refs: ["ref-1"], + inferred_intent: { + objective_code: "9수04-03", + objective_description: "사각형의 성질을 이해한다.", + evaluation_dimensions: [{ id: "A", description: "도형 성질", must_preserve: true }], + required_techniques: ["geometry"], + forbidden_techniques: [], + surface_constraints: { difficulty: "medium", problem_type: "objective" }, + }, + generation_metadata: { + model: "test", + temperature: 0, + prompt_id: "test", + prompt_version: "0.0.0", + attempt: 1, + generated_at: "2026-06-06T00:00:00.000Z", + }, +}; + +const mathEngine: MathEngineClient = { + health: async () => ({ status: "ok", engine: "sympy" }), + solve: async () => ({ solutions: [] }), + verify: async () => ({ equivalent: false, diff: "not checked" }), + simplify: async ({ expr }) => ({ simplified: expr.replace(/\s+/g, "") }), + differentiate: async () => ({ derivative: "" }), + limit: async () => ({ limit: "" }), +}; + +describe("independentResolve geometry and function equivalence", () => { + it("passes geometry answers when the solver keeps segment and angle labels", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "AB = 13cm, ∠AOB = 90도", + trace: "마름모의 대각선 성질을 사용한다.", + confidence: "high", + }), + }; + const result = await independentResolve( + { solver, mathEngine }, + { candidate, sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 } }, + ); + expect(result.gate.status).toBe("passed"); + }); + + it("passes quadratic opening answers when the solver gives only the direction", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "아래로", + trace: "이차항의 계수가 음수이다.", + confidence: "high", + }), + }; + const result = await independentResolve( + { solver, mathEngine }, + { + candidate: { ...candidate, generation_kind: "function", expected_answer: "아래로 열린다" }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + expect(result.gate.status).toBe("passed"); + }); + + it("passes tangent-position wording variants for circle and line relations", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "접한다(한 점에서 만난다)", + trace: "중심에서 직선까지의 거리가 반지름과 같다.", + confidence: "high", + }), + }; + const result = await independentResolve( + { solver, mathEngine }, + { + candidate: { ...candidate, expected_answer: "한 점에서 만난다" }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + expect(result.gate.status).toBe("passed"); + }); +}); diff --git a/packages/agent/tests/independent-resolve-labels.test.ts b/packages/agent/tests/independent-resolve-labels.test.ts new file mode 100644 index 0000000..4e568f2 --- /dev/null +++ b/packages/agent/tests/independent-resolve-labels.test.ts @@ -0,0 +1,257 @@ +import { describe, expect, it } from "vitest"; + +import type { SolverAgent } from "../src/agents/index.js"; +import type { GeneratedProblem } from "../src/schemas/index.js"; +import { independentResolve } from "../src/steps/independent-resolve.js"; +import type { MathEngineClient } from "../src/tools/math-engine-client.js"; + +const candidate: GeneratedProblem = { + candidate_id: "00000000-0000-0000-0000-000000000010", + mode: "structural", + generation_kind: "expression", + question_text: "계산 과정 ㈎, ㈏에 쓰인 법칙을 쓰시오.", + expected_answer: "덧셈의 교환법칙, 덧셈의 결합법칙", + proposed_solution_trace: "순서대로 교환법칙과 결합법칙이다.", + source_refs: ["ref-1"], + inferred_intent: { + objective_code: "9수02-02", + objective_description: "일차식의 계산 원리를 이해한다.", + evaluation_dimensions: [{ id: "A", description: "계산 법칙", must_preserve: true }], + required_techniques: ["addition_laws"], + forbidden_techniques: [], + surface_constraints: { difficulty: "medium", problem_type: "objective" }, + }, + generation_metadata: { + model: "test", + temperature: 0, + prompt_id: "test", + prompt_version: "0.0.0", + attempt: 1, + generated_at: "2026-06-06T00:00:00.000Z", + }, +}; + +const mathEngine: MathEngineClient = { + health: async () => ({ status: "ok", engine: "sympy" }), + solve: async () => ({ solutions: [] }), + verify: async () => ({ equivalent: true, diff: "0" }), + simplify: async ({ expr }) => ({ simplified: expr.replace(/\s+/g, "") }), + differentiate: async () => ({ derivative: "" }), + limit: async () => ({ limit: "" }), +}; + +describe("independentResolve label normalization", () => { + it("passes when law names include Korean process labels", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "㈎ 교환법칙, ㈏ 결합법칙", + trace: "순서대로 교환법칙과 결합법칙이다.", + confidence: "high", + }), + }; + + const result = await independentResolve( + { solver, mathEngine }, + { + candidate, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + + expect(result.gate.status).toBe("passed"); + }); + + it("passes equivalent function formulas with explicit multiplication", async () => { + const solver: SolverAgent = { + solve: async () => ({ derived_answer: "y = -5/2*x", trace: "기울기를 구한다.", confidence: "high" }), + }; + const result = await independentResolve( + { solver, mathEngine }, + { + candidate: { ...candidate, generation_kind: "function", expected_answer: "y = -5/2 x" }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + expect(result.gate.status).toBe("passed"); + }); + + it("passes equivalent not-real answer wording", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "실수 범위에서 정의되지 않음", + trace: "근호 안이 음수이다.", + confidence: "high", + }), + }; + const result = await independentResolve( + { solver, mathEngine }, + { + candidate: { ...candidate, expected_answer: "실수가 아니다" }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + expect(result.gate.status).toBe("passed"); + }); + + it("ignores rejected non-solution notes in derived equation answers", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "x = 3/4, -1/2; x = -3/4는 해가 아님", + trace: "두 해를 구하고 다른 값은 해가 아님을 확인한다.", + confidence: "high", + }), + }; + const result = await independentResolve( + { solver, mathEngine }, + { + candidate: { ...candidate, generation_kind: "equation", expected_answer: "-1/2, 3/4" }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + expect(result.gate.status).toBe("passed"); + }); + + it("passes when a numeric value is prefixed by Korean answer text", async () => { + const solver: SolverAgent = { + solve: async () => ({ derived_answer: "식의 값은 -5/3", trace: "계산한다.", confidence: "high" }), + }; + const result = await independentResolve( + { solver, mathEngine }, + { + candidate: { ...candidate, expected_answer: "-5/3" }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + expect(result.gate.status).toBe("passed"); + }); + + it("passes expression-and-value answers when derived answer keeps numbered labels and won units", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "(1) 1500a + 900b + 3600원\n(2) 15900원", + trace: "항목별 가격을 식으로 나타낸 뒤 대입한다.", + confidence: "high", + }), + }; + const result = await independentResolve( + { solver, mathEngine }, + { + candidate: { + ...candidate, + expected_answer: "1500 a + 900 b + 3600, 15900원", + }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + expect(result.gate.status).toBe("passed"); + }); + + it("passes prime-factor answers when derived text includes Korean field labels", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "소인수: 2, 3, 5; 약수의 개수: 24개", + trace: "360을 소인수분해하고 약수 개수를 센다.", + confidence: "high", + }), + }; + const result = await independentResolve( + { solver, mathEngine }, + { + candidate: { ...candidate, expected_answer: "2, 3, 5; 24개" }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + expect(result.gate.status).toBe("passed"); + }); + + it("passes quadratic graph shape wording variants", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "㉠, ㉡ / 아래로 열린 포물선", + trace: "계수 a가 음수이므로 아래로 열린다.", + confidence: "high", + }), + }; + const result = await independentResolve( + { solver, mathEngine }, + { + candidate: { + ...candidate, + generation_kind: "function", + expected_answer: "㉠, ㉡, 아래로 열린다", + }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + expect(result.gate.status).toBe("passed"); + }); + + it("passes graph point and opening wording variants in sentence form", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "예, 점 (2, -12)는 그래프 위의 점이고, 그래프는 아래로 열린다.", + trace: "대입하고 계수의 부호를 확인한다.", + confidence: "high", + }), + }; + const result = await independentResolve( + { solver, mathEngine }, + { + candidate: { + ...candidate, + generation_kind: "function", + expected_answer: "그래프 위의 점이며, 아래로 열린다.", + }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + expect(result.gate.status).toBe("passed"); + }); + + it("passes geometry pair answers when derived text includes Korean labels", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "변 AB의 길이는 13cm, 각 AOB의 크기는 90도", + trace: "마름모의 대각선은 서로 수직이등분하므로 피타고라스 정리를 쓴다.", + confidence: "high", + }), + }; + const result = await independentResolve( + { solver, mathEngine }, + { + candidate: { ...candidate, generation_kind: "geometry", expected_answer: "13cm, 90도" }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + expect(result.gate.status).toBe("passed"); + }); + + it("passes equation candidates when deterministic solving confirms the expected answer", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "5/2, -3/2", + trace: "계산 실수로 다른 해를 얻었다.", + confidence: "medium", + }), + }; + const result = await independentResolve( + { + solver, + mathEngine: { + ...mathEngine, + solve: async () => ({ solutions: ["-1/8 + sqrt(241)/8", "-sqrt(241)/8 - 1/8"] }), + }, + }, + { + candidate: { + ...candidate, + generation_kind: "equation", + question_text: "다음 이차방정식을 풀어라. 4*x**2 + 3*x = 2*x + 15", + expected_answer: "-1/8 + sqrt(241)/8, -sqrt(241)/8 - 1/8", + }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + expect(result.gate.status).toBe("passed"); + }); +}); diff --git a/packages/agent/tests/independent-resolve-radicals-decimals.test.ts b/packages/agent/tests/independent-resolve-radicals-decimals.test.ts new file mode 100644 index 0000000..a5fedbf --- /dev/null +++ b/packages/agent/tests/independent-resolve-radicals-decimals.test.ts @@ -0,0 +1,127 @@ +import { describe, expect, it } from "vitest"; + +import type { SolverAgent } from "../src/agents/index.js"; +import type { GeneratedProblem } from "../src/schemas/index.js"; +import { independentResolve } from "../src/steps/independent-resolve.js"; +import type { MathEngineClient } from "../src/tools/math-engine-client.js"; + +const candidate: GeneratedProblem = { + candidate_id: "00000000-0000-0000-0000-000000000010", + mode: "structural", + generation_kind: "expression", + question_text: "제곱근을 정리하시오.", + expected_answer: "2 sqrt(7)", + proposed_solution_trace: "근호를 정리한다.", + source_refs: ["ref-1"], + inferred_intent: { + objective_code: "9수01-05", + objective_description: "제곱근과 실수를 이해한다.", + evaluation_dimensions: [{ id: "A", description: "근호 정리", must_preserve: true }], + required_techniques: ["radical_simplification"], + forbidden_techniques: [], + surface_constraints: { difficulty: "medium", problem_type: "objective" }, + }, + generation_metadata: { + model: "test", + temperature: 0, + prompt_id: "test", + prompt_version: "0.0.0", + attempt: 1, + generated_at: "2026-06-06T00:00:00.000Z", + }, +}; + +const mathEngine: MathEngineClient = { + health: async () => ({ status: "ok", engine: "sympy" }), + solve: async () => ({ solutions: [] }), + verify: async () => ({ equivalent: true, diff: "0" }), + simplify: async ({ expr }) => ({ simplified: expr.replace(/\s+/g, "") }), + differentiate: async () => ({ derivative: "" }), + limit: async () => ({ limit: "" }), +}; + +describe("independentResolve radical and repeating-decimal equivalence", () => { + it("passes when equivalent radical answers use latex and plain sqrt notation", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "12+8sqrt(3)", + trace: "삼각비로 길이를 구한다.", + confidence: "high", + }), + }; + + const result = await independentResolve( + { solver, mathEngine }, + { + candidate: { ...candidate, generation_kind: "geometry", expected_answer: "12+8 \\sqrt{3}" }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + + expect(result.gate.status).toBe("passed"); + }); + + it("passes when radical answers omit sqrt parentheses", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "2sqrt7", + trace: "제곱근을 정리한다.", + confidence: "high", + }), + }; + + const result = await independentResolve( + { solver, mathEngine }, + { + candidate: { ...candidate, expected_answer: "2 \\sqrt{7}" }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + + expect(result.gate.status).toBe("passed"); + }); + + it("passes repeating-decimal conversion when deterministic fraction matches expected", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "3437/1110", + trace: "계산 실수로 분모를 잘못 두었다.", + confidence: "medium", + }), + }; + + const result = await independentResolve( + { solver, mathEngine }, + { + candidate: { + ...candidate, + expected_answer: "3437/1100", + question_text: "순환소수 3.12454545...를 분수로 나타내시오.", + }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + + expect(result.gate.status).toBe("passed"); + }); + + it("passes when repeating decimals use dotted and ellipsis notation", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "0.777...", + trace: "7이 반복된다.", + confidence: "high", + }), + }; + + const result = await independentResolve( + { solver, mathEngine }, + { + candidate: { ...candidate, expected_answer: "0.(7)", question_text: "기약분수 7/9을 순환소수로 나타내시오." }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + + expect(result.gate.status).toBe("passed"); + }); +}); diff --git a/packages/agent/tests/independent-resolve-text-equivalence.test.ts b/packages/agent/tests/independent-resolve-text-equivalence.test.ts new file mode 100644 index 0000000..84bc6d0 --- /dev/null +++ b/packages/agent/tests/independent-resolve-text-equivalence.test.ts @@ -0,0 +1,252 @@ +import { describe, expect, it } from "vitest"; + +import type { SolverAgent } from "../src/agents/index.js"; +import type { GeneratedProblem } from "../src/schemas/index.js"; +import { independentResolve } from "../src/steps/independent-resolve.js"; +import type { MathEngineClient } from "../src/tools/math-engine-client.js"; + +const candidate: GeneratedProblem = { + candidate_id: "00000000-0000-0000-0000-000000000040", + mode: "structural", + generation_kind: "geometry", + question_text: "대응하는 선분을 쓰시오.", + expected_answer: "선분 ST", + proposed_solution_trace: "대응 관계를 확인한다.", + source_refs: ["ref-1"], + inferred_intent: { + objective_code: "9수04-04", + objective_description: "도형의 닮음을 이해한다.", + evaluation_dimensions: [{ id: "A", description: "대응 변", must_preserve: true }], + required_techniques: ["similarity"], + forbidden_techniques: [], + surface_constraints: { difficulty: "medium", problem_type: "objective" }, + }, + generation_metadata: { + model: "test", + temperature: 0, + prompt_id: "test", + prompt_version: "0.0.0", + attempt: 1, + generated_at: "2026-06-06T00:00:00.000Z", + }, +}; + +const mathEngine: MathEngineClient = { + health: async () => ({ status: "ok", engine: "sympy" }), + solve: async () => ({ solutions: [] }), + verify: async () => ({ equivalent: true, diff: "0" }), + simplify: async ({ expr }) => ({ simplified: expr.replace(/\s+/g, "") }), + differentiate: async () => ({ derivative: "" }), + limit: async () => ({ limit: "" }), +}; + +describe("independentResolve text equivalence", () => { + it("passes when a segment answer omits the Korean segment prefix", async () => { + const solver: SolverAgent = { + solve: async () => ({ derived_answer: "ST", trace: "대응 변은 ST이다.", confidence: "high" }), + }; + const result = await independentResolve( + { solver, mathEngine }, + { candidate, sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 } }, + ); + expect(result.gate.status).toBe("passed"); + }); + + it("passes graph point answers when expected uses yes wording", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "점 (-1, 9)는 그래프 위의 점이다. 그래프는 아래로 열린다.", + trace: "대입하고 계수의 부호를 확인한다.", + confidence: "high", + }), + }; + const result = await independentResolve( + { solver, mathEngine }, + { + candidate: { + ...candidate, + generation_kind: "function", + expected_answer: "예, 아래로 열린다", + }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + expect(result.gate.status).toBe("passed"); + }); + + it("passes graph point answers when expected omits yes wording", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "예, 점 (3, -3)는 그래프 위에 있다. 그래프는 꼭짓점이 (1, 5)이고 아래로 열린 포물선이다.", + trace: "대입하고 계수의 부호를 확인한다.", + confidence: "high", + }), + }; + const result = await independentResolve( + { solver, mathEngine: { ...mathEngine, verify: async () => ({ equivalent: false, diff: "not checked" }) } }, + { + candidate: { + ...candidate, + generation_kind: "function", + expected_answer: "그래프 위에 있다, 아래로 열린다", + }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + expect(result.gate.status).toBe("passed"); + }); + + it("passes linear-function trend wording variants", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "예, 오른쪽 아래, x가 증가하면 y는 감소", + trace: "기울기가 음수이므로 감소한다.", + confidence: "high", + }), + }; + const result = await independentResolve( + { solver, mathEngine }, + { + candidate: { + ...candidate, + generation_kind: "function", + expected_answer: "예, 오른쪽 아래, 감소", + }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + expect(result.gate.status).toBe("passed"); + }); + + it("passes fraction answers with decimal and percent notes", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "1/5 (0.2, 20%)", + trace: "8/40을 약분한다.", + confidence: "high", + }), + }; + const result = await independentResolve( + { solver, mathEngine }, + { + candidate: { + ...candidate, + generation_kind: "statistics", + expected_answer: "1/5", + }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + expect(result.gate.status).toBe("passed"); + }); + + it("passes fraction answers when the solver gives only an equivalent decimal", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "0.2", + trace: "8/40=0.2이다.", + confidence: "high", + }), + }; + const result = await independentResolve( + { solver, mathEngine }, + { + candidate: { + ...candidate, + generation_kind: "statistics", + expected_answer: "1/5", + }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + expect(result.gate.status).toBe("passed"); + }); + + it("passes prime-factor answers when the solver writes sentence labels", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "소인수는 2, 3, 5이고, N의 약수의 개수는 36개이다.", + trace: "지수를 이용해 약수의 개수를 구한다.", + confidence: "high", + }), + }; + const result = await independentResolve( + { solver, mathEngine }, + { + candidate: { + ...candidate, + generation_kind: "factorization", + expected_answer: "2, 3, 5; 36", + }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + expect(result.gate.status).toBe("passed"); + }); + + it("passes system answers when the solver omits variable labels in order", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "3, 2", + trace: "연립방정식의 해는 x=3, y=2이다.", + confidence: "high", + }), + }; + const result = await independentResolve( + { solver, mathEngine }, + { + candidate: { + ...candidate, + generation_kind: "equation", + expected_answer: "x=3, y=2", + }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + expect(result.gate.status).toBe("passed"); + }); + + it("passes ordered radical answers when expected uses an inequality chain", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "4, sqrt(50)-3, sqrt(18)+1", + trace: "각 값을 비교해 작은 것부터 나열한다.", + confidence: "high", + }), + }; + const result = await independentResolve( + { solver, mathEngine: { ...mathEngine, verify: async () => ({ equivalent: false, diff: "not checked" }) } }, + { + candidate: { + ...candidate, + question_text: "다음 세 수를 작은 것부터 큰 것 순서로 나열하시오: \\sqrt{50}-3, 4, \\sqrt{18}+1", + expected_answer: "4 < \\sqrt{50}-3 < \\sqrt{18}+1", + }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + expect(result.gate.status).toBe("passed"); + }); + + it("passes not-real wording when the solver adds Korean particles", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "실수 범위에서는 정의되지 않음", + trace: "근호 안이 음수이다.", + confidence: "high", + }), + }; + const result = await independentResolve( + { solver, mathEngine: { ...mathEngine, verify: async () => ({ equivalent: false, diff: "not checked" }) } }, + { + candidate: { + ...candidate, + expected_answer: "실수가 아니다", + }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + expect(result.gate.status).toBe("passed"); + }); + +}); diff --git a/packages/agent/tests/independent-resolve.test.ts b/packages/agent/tests/independent-resolve.test.ts new file mode 100644 index 0000000..7910b44 --- /dev/null +++ b/packages/agent/tests/independent-resolve.test.ts @@ -0,0 +1,283 @@ +import { describe, expect, it } from "vitest"; + +import type { SolverAgent } from "../src/agents/index.js"; +import type { GeneratedProblem } from "../src/schemas/index.js"; +import { independentResolve } from "../src/steps/independent-resolve.js"; +import type { MathEngineClient } from "../src/tools/math-engine-client.js"; + +const candidate: GeneratedProblem = { + candidate_id: "00000000-0000-0000-0000-000000000008", + mode: "structural", + generation_kind: "expression", + question_text: + "다음 중 두 다항식 4 x^{2} - 4 x - 15, 6 x^{2} - 11 x - 10의 공통 인수는? ① 2 x+3 ② 2 x-5 ③ 3 x+2 ④ x-5 ⑤ x+3", + expected_answer: "②", + proposed_solution_trace: "두 다항식을 인수분해하면 공통 인수는 2x-5이다.", + source_refs: ["ref-1"], + inferred_intent: { + objective_code: "9수02-08", + objective_description: "다항식의 곱셈공식과 인수분해 공식을 이해한다.", + evaluation_dimensions: [{ id: "A", description: "인수분해", must_preserve: true }], + required_techniques: ["factorization"], + forbidden_techniques: [], + surface_constraints: { difficulty: "medium", problem_type: "objective" }, + }, + generation_metadata: { + model: "test", + temperature: 0, + prompt_id: "test", + prompt_version: "0.0.0", + attempt: 1, + generated_at: "2026-06-06T00:00:00.000Z", + }, +}; + +const mathEngine: MathEngineClient = { + health: async () => ({ status: "ok", engine: "sympy" }), + solve: async () => ({ solutions: [] }), + verify: async () => ({ equivalent: true, diff: "0" }), + simplify: async ({ expr }) => ({ simplified: expr.replace(/\s+/g, "") }), + differentiate: async () => ({ derivative: "" }), + limit: async () => ({ limit: "" }), +}; + +describe("independentResolve", () => { + it("passes when a solver derives the selected multiple-choice expression", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "2x - 5", + trace: "공통 인수는 2x-5이므로 ②이다.", + confidence: "high", + }), + }; + + const result = await independentResolve( + { solver, mathEngine }, + { + candidate, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + + expect(result.gate.status).toBe("passed"); + }); + + it("passes when parenthesized choices use a circled answer label with expression text", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "2x - 3", + trace: "공통 인수는 2x-3이다.", + confidence: "high", + }), + }; + + const result = await independentResolve( + { solver, mathEngine }, + { + candidate: { + ...candidate, + question_text: + "다음 두 다항식의 공통 인수를 고르시오.\n(1) x - 4\n(2) x + 4\n(3) 2x - 3\n(4) 3x + 1\n(5) 2x + 3", + expected_answer: "③ 2x - 3", + }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + + expect(result.gate.status).toBe("passed"); + }); + + it("passes when a solver derives the zero of the selected linear factor", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "x = 3/2", + trace: "2x-3=0에서 x=3/2이다.", + confidence: "high", + }), + }; + const rootMathEngine: MathEngineClient = { + ...mathEngine, + solve: async ({ equation }) => ({ + solutions: equation.replace(/\s+/g, "") === "2x-3=0" ? ["3/2"] : [], + }), + }; + + const result = await independentResolve( + { solver, mathEngine: rootMathEngine }, + { + candidate: { + ...candidate, + question_text: + "다음 중 두 다항식의 공통 인수는?\n① x - 3 ② x + 3 ③ 2x - 3 ④ 2x + 3 ⑤ 3x - 1", + expected_answer: "③", + }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + + expect(result.gate.status).toBe("passed"); + }); + + it("passes when a solver prefixes a matching expression with a choice label", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "② 2x + 1", + trace: "공통 인수는 2x+1이다.", + confidence: "high", + }), + }; + + const result = await independentResolve( + { solver, mathEngine }, + { + candidate: { + ...candidate, + expected_answer: "2x + 1", + }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + + expect(result.gate.status).toBe("passed"); + }); + + it("passes when a solver appends a Korean counting unit to a numeric answer", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "18모둠", + trace: "최대공약수는 18이므로 18모둠이다.", + confidence: "high", + }), + }; + + const result = await independentResolve( + { solver, mathEngine }, + { + candidate: { + ...candidate, + expected_answer: "18", + }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + + expect(result.gate.status).toBe("passed"); + }); + + it("passes when a solver adds numbered labels and units to multipart expression answers", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "(1) 4a + 4b + 270 kcal\n(2) 526 kcal", + trace: "식을 세운 뒤 값을 대입한다.", + confidence: "high", + }), + }; + + const result = await independentResolve( + { solver, mathEngine }, + { + candidate: { + ...candidate, + expected_answer: "4 a + 4 b + 270; 526", + }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + + expect(result.gate.status).toBe("passed"); + }); + + it("passes when a solver prefixes matching choice details with the selected label", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "③ ㄷ, ㄹ", + trace: "정답은 ③이다.", + confidence: "high", + }), + }; + + const result = await independentResolve( + { solver, mathEngine }, + { + candidate: { + ...candidate, + expected_answer: "③", + }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + + expect(result.gate.status).toBe("passed"); + }); + + it("passes when additive law names omit the additive qualifier", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "교환법칙, 결합법칙", + trace: "순서대로 교환법칙과 결합법칙이다.", + confidence: "high", + }), + }; + + const result = await independentResolve( + { solver, mathEngine }, + { + candidate: { + ...candidate, + expected_answer: "덧셈의 교환법칙, 덧셈의 결합법칙", + }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + + expect(result.gate.status).toBe("passed"); + }); + + it("passes when a solver gives the body of the selected function choice", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "y = -3/2 x", + trace: "기울기가 -3/2이다.", + confidence: "high", + }), + }; + + const result = await independentResolve( + { solver, mathEngine }, + { + candidate: { + ...candidate, + question_text: + "이 함수의 식은? ① y = 3/2 x ② y = -3/2 x ③ y = -2 x ④ y = 2/3 x ⑤ y = -1/2 x", + expected_answer: "②", + }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + + expect(result.gate.status).toBe("passed"); + }); + + it("passes when a solver uses a numeric label for a circled expected choice", async () => { + const solver: SolverAgent = { + solve: async () => ({ + derived_answer: "4번", + trace: "정답은 4번이다.", + confidence: "high", + }), + }; + + const result = await independentResolve( + { solver, mathEngine }, + { + candidate: { + ...candidate, + expected_answer: "④", + }, + sympyGate: { step: "sympy_verify", status: "passed", duration_ms: 1 }, + }, + ); + + expect(result.gate.status).toBe("passed"); + }); +}); diff --git a/packages/agent/tests/math-engine-client.test.ts b/packages/agent/tests/math-engine-client.test.ts new file mode 100644 index 0000000..9f57abb --- /dev/null +++ b/packages/agent/tests/math-engine-client.test.ts @@ -0,0 +1,101 @@ +import { createServer, type IncomingMessage, type ServerResponse } from "node:http"; +import type { AddressInfo } from "node:net"; + +import { afterEach, describe, expect, it } from "vitest"; +import { z } from "zod"; + +import { createMathEngineClient } from "../src/tools/math-engine-client.js"; + +const SolveBodySchema = z.object({ + equation: z.string(), +}); + +const servers: ReturnType[] = []; + +afterEach(async () => { + await Promise.all( + servers.splice(0).map( + (server) => + new Promise((resolve, reject) => { + server.close((error) => { + if (error) { + reject(error); + return; + } + resolve(); + }); + }), + ), + ); +}); + +describe("math-engine client request boundary", () => { + it("converts LaTeX exponent braces before sending solve requests", async () => { + const receivedBodies: string[] = []; + const server = createServer(async (req: IncomingMessage, res: ServerResponse) => { + receivedBodies.push(await readRequestBody(req)); + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ solutions: ["2", "3"] })); + }); + servers.push(server); + await listen(server); + + const address = server.address(); + if (address === null || typeof address === "string") { + throw new Error("test server address must be a TCP address"); + } + const client = createMathEngineClient({ + baseUrl: `http://127.0.0.1:${address.port}`, + }); + + await client.solve({ equation: "x^{2} - 5x + 6 = 0" }); + + expect(receivedBodies).toHaveLength(1); + const body = SolveBodySchema.parse(JSON.parse(receivedBodies[0] ?? "")); + expect(body.equation).toBe("x**(2) - 5*x + 6 = 0"); + }); + + it("inserts explicit multiplication for adjacent factors before solving", async () => { + const receivedBodies: string[] = []; + const server = createServer(async (req: IncomingMessage, res: ServerResponse) => { + receivedBodies.push(await readRequestBody(req)); + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ solutions: ["-6", "4"] })); + }); + servers.push(server); + await listen(server); + + const address = server.address(); + if (address === null || typeof address === "string") { + throw new Error("test server address must be a TCP address"); + } + const client = createMathEngineClient({ + baseUrl: `http://127.0.0.1:${address.port}`, + }); + + await client.solve({ equation: "(x - 2)(x + 7) = 3 x + 10" }); + + const body = SolveBodySchema.parse(JSON.parse(receivedBodies[0] ?? "")); + expect(body.equation).toBe("(x - 2)*(x + 7) = 3*x + 10"); + }); +}); + +function listen(server: ReturnType): Promise { + return new Promise((resolve, reject) => { + server.listen(0, "127.0.0.1", () => resolve()); + server.once("error", reject); + }); +} + +function readRequestBody(req: IncomingMessage): Promise { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + req.on("data", (chunk: Buffer) => { + chunks.push(chunk); + }); + req.on("end", () => { + resolve(Buffer.concat(chunks).toString("utf8")); + }); + req.on("error", reject); + }); +} diff --git a/packages/agent/tests/objective-mapping-function.test.ts b/packages/agent/tests/objective-mapping-function.test.ts new file mode 100644 index 0000000..2b7376e --- /dev/null +++ b/packages/agent/tests/objective-mapping-function.test.ts @@ -0,0 +1,154 @@ +import { describe, expect, it } from "vitest"; + +import type { GenerateRequest, GeneratedProblem, Intent, RagResult } from "../src/schemas/index.js"; +import { mapObjective } from "../src/steps/objective-mapping.js"; + +const request: GenerateRequest = { + mode: "structural", + school_level: "middle", + grade: 1, + topic: "9수03-01", + topic_name: "함수의 개념", + dims: ["A"], + count: 5, + difficulty: "medium", + problem_type: "objective", + source_problem_text: "함수의 뜻을 알고 함수값을 구할 수 있다.", +}; + +const intent: Intent = { + objective_code: "9수05-01", + objective_description: "자료의 정리와 해석", + evaluation_dimensions: [{ id: "A", description: "함수식 찾기", must_preserve: true }], + required_techniques: ["linear_function"], + forbidden_techniques: [], + surface_constraints: { difficulty: "medium", problem_type: "objective" }, +}; + +const refs: RagResult[] = [ + { + item_id: "ref-1", + match_reason: "hybrid", + problem: { + item_id: "ref-1", + source_dataset: "111", + split: "train", + source_label_type: "problem_label", + school_level: "middle", + grade: 1, + semester: null, + topic_code: "9수03-01", + topic_name: "함수의 개념", + achievement_standard: "함수의 뜻을 알고 함수값을 구할 수 있다.", + question_text: "직선의 식을 고르시오.", + answer_text: "y=2x-2", + explanation_text: null, + choice_blocks: [], + problem_type_norm: "objective", + difficulty_norm: "medium", + question_image_relpath: null, + answer_image_relpath: null, + question_json_relpath: null, + answer_json_relpath: null, + }, + }, +]; + +const candidate: GeneratedProblem = { + candidate_id: "00000000-0000-0000-0000-000000000030", + mode: "structural", + generation_kind: "function", + question_text: "좌표평면에서 직선 l은 점 (0, -2)를 지나고 점 (3, 4)를 지난다. 다음 중 직선 l의 식으로 알맞은 것은?", + expected_answer: "③", + proposed_solution_trace: "두 좌표에서 기울기와 y절편을 구해 함수식 y = 2x - 2를 찾는다.", + source_refs: ["ref-1"], + inferred_intent: intent, + generation_metadata: { + model: "test", + temperature: 0, + prompt_id: "test", + prompt_version: "0.0.0", + attempt: 1, + generated_at: "2026-06-06T00:00:00.000Z", + }, +}; + +describe("mapObjective function aliases", () => { + it("allows broad extracted intent when candidate evidence supports a function-form request", async () => { + const result = await mapObjective({}, { request, refs, candidate, intent, strategy: null }); + + expect(result.gate.status).toBe("passed"); + }); + + it("allows broad extracted intent when candidate evidence supports a linear-function application request", async () => { + const result = await mapObjective({}, { + request: { ...request, topic: "9수03-03", topic_name: "일차함수의 활용", source_problem_text: undefined }, + refs, + candidate: { + ...candidate, + question_text: + "한 시간에 1000원씩 더해지는 이용 요금이 기본요금 3000원에 x시간 이용료를 더한 y = 1000x + 3000으로 나타난다. 5시간 이용할 때 요금을 구하시오.", + expected_answer: "8000원", + proposed_solution_trace: "x=5를 대입하면 y = 1000*5 + 3000 = 8000이다.", + }, + intent: { ...intent, objective_code: "9수03-08" }, + strategy: null, + }); + + expect(result.gate.status).toBe("passed"); + }); + + it("allows broad extracted intent when candidate evidence supports a quadratic-equation application request", async () => { + const result = await mapObjective({}, { + request: { ...request, topic: "9수02-10", topic_name: "이차방정식의 활용", source_problem_text: undefined }, + refs, + candidate: { + ...candidate, + generation_kind: "equation", + question_text: "x(x+3)=10을 만족하는 x의 값을 모두 구하시오.", + expected_answer: "2, -5", + proposed_solution_trace: "x^2 + 3x - 10 = 0이고, (x+5)(x-2)=0이므로 해는 x=2, -5이다.", + }, + intent: { ...intent, objective_code: "일차방정식을 풀 수 있고, 이를 활용하여 문제를 해결할 수 있다." }, + strategy: null, + }); + + expect(result.gate.status).toBe("passed"); + }); + + it("allows broad extracted intent when candidate evidence supports a triangle-property request", async () => { + const result = await mapObjective({}, { + request: { ...request, topic: "9수04-02", topic_name: "삼각형의 성질", source_problem_text: undefined }, + refs, + candidate: { + ...candidate, + generation_kind: "geometry", + question_text: "삼각형 ABC에서 AB = AC이고 ∠A = 40도일 때, ∠B의 크기를 구하시오.", + expected_answer: "70도", + proposed_solution_trace: "이등변삼각형의 두 밑각은 같고 세 각의 합은 180도이므로 ∠B = 70도이다.", + }, + intent: { ...intent, objective_code: "9수04-18" }, + strategy: null, + }); + + expect(result.gate.status).toBe("passed"); + }); + + it("allows broad extracted intent when candidate evidence supports a prime-factorization request", async () => { + const result = await mapObjective({}, { + request: { ...request, topic: "9수01-01", topic_name: "소인수분해", source_problem_text: undefined }, + refs, + candidate: { + ...candidate, + generation_kind: "expression", + question_text: "294의 소인수를 모두 쓰시오.", + expected_answer: "2, 3, 7", + proposed_solution_trace: "294 = 2 * 3 * 7^2 이므로 소인수는 2, 3, 7이다.", + }, + intent: { ...intent, objective_code: "9수01-03" }, + strategy: null, + }); + + expect(result.gate.status).toBe("passed"); + }); +}); diff --git a/packages/agent/tests/objective-mapping.test.ts b/packages/agent/tests/objective-mapping.test.ts new file mode 100644 index 0000000..cc462bd --- /dev/null +++ b/packages/agent/tests/objective-mapping.test.ts @@ -0,0 +1,322 @@ +import { describe, expect, it } from "vitest"; + +import type { + GenerateRequest, + GeneratedProblem, + Intent, + RagResult, +} from "../src/schemas/index.js"; +import { mapObjective } from "../src/steps/objective-mapping.js"; + +const request: GenerateRequest = { + mode: "structural", + school_level: "middle", + grade: 2, + topic: "9수04-05", + topic_name: "삼각비", + dims: ["A"], + count: 5, + difficulty: "medium", + problem_type: "objective", + source_problem_text: "직각삼각형에서 삼각비의 값을 구하고 활용한다.", +}; + +const intent: Intent = { + objective_code: "9수04-05", + objective_description: "직각삼각형에서 삼각비의 값을 구하고 활용한다.", + evaluation_dimensions: [ + { id: "A", description: "삼각비를 이용해 길이를 구한다", must_preserve: true }, + ], + required_techniques: ["trigonometric_ratio"], + forbidden_techniques: [], + surface_constraints: { difficulty: "medium", problem_type: "objective" }, +}; + +const refs: RagResult[] = [ + { + item_id: "ref-1", + match_reason: "hybrid", + problem: { + item_id: "ref-1", + source_dataset: "111", + split: "train", + source_label_type: "problem_label", + school_level: "middle", + grade: 3, + semester: null, + topic_code: "9수04-05", + topic_name: "삼각비", + achievement_standard: "직각삼각형에서 삼각비의 값을 구하고 활용한다.", + question_text: "직각삼각형에서 sin A의 값을 구하여라.", + answer_text: "3/5", + explanation_text: null, + choice_blocks: [], + problem_type_norm: "objective", + difficulty_norm: "medium", + question_image_relpath: null, + answer_image_relpath: null, + question_json_relpath: null, + answer_json_relpath: null, + }, + }, +]; + +const candidate: GeneratedProblem = { + candidate_id: "00000000-0000-0000-0000-000000000006", + mode: "structural", + generation_kind: "geometry", + question_text: "직각삼각형 ABC에서 cos A = 4/5일 때 sin A의 값을 구하시오.", + expected_answer: "3/5", + proposed_solution_trace: "sin**2 A + cos**2 A = 1을 이용한다.", + source_refs: ["ref-1"], + inferred_intent: intent, + generation_metadata: { + model: "test", + temperature: 0, + prompt_id: "test", + prompt_version: "0.0.0", + attempt: 1, + generated_at: "2026-06-06T00:00:00.000Z", + }, +}; + +describe("mapObjective", () => { + it("passes without a strategy when generation kind matches the requested topic", async () => { + const result = await mapObjective( + {}, + { request, refs, candidate, intent, strategy: null }, + ); + + expect(result.gate.status).toBe("passed"); + }); + + it("fails when candidate generation kind does not match the requested topic", async () => { + const result = await mapObjective( + {}, + { + request, + refs, + candidate: { ...candidate, generation_kind: "equation" }, + intent, + strategy: null, + }, + ); + + expect(result.gate.status).toBe("failed"); + expect(result.gate.failure_detail?.code).toBe("generation_kind_mismatch"); + }); + + it("fails when extracted intent objective code differs from the requested topic", async () => { + const mismatchedIntent: Intent = { + ...intent, + objective_code: "9수05-03", + objective_description: "대푯값과 산포도를 이해한다.", + }; + const result = await mapObjective( + {}, + { + request, + refs, + candidate: { + ...candidate, + question_text: "5개의 변량 3, a, 5, b, 9의 표준편차를 구하시오.", + inferred_intent: mismatchedIntent, + }, + intent: mismatchedIntent, + strategy: null, + }, + ); + + expect(result.gate.status).toBe("failed"); + expect(result.gate.failure_detail?.code).toBe("intent_topic_mismatch"); + }); + + it("allows alternate curriculum codes when candidate text supports the requested topic", async () => { + const alternateIntent: Intent = { + ...intent, + objective_code: "9수04-18", + }; + const result = await mapObjective( + {}, + { + request, + refs, + candidate: { ...candidate, inferred_intent: alternateIntent }, + intent: alternateIntent, + strategy: null, + }, + ); + + expect(result.gate.status).toBe("passed"); + }); + + it("allows expression-calculation aliases for polynomial simplification candidates", async () => { + const expressionRequest: GenerateRequest = { + ...request, + grade: 2, + topic: "9수02-05", + topic_name: "식의 계산", + source_problem_text: "지수법칙과 다항식의 곱셈 · 나눗셈을 이해한다.", + }; + const alternateIntent: Intent = { + ...intent, + objective_code: "9수02-01", + objective_description: "문자를 사용하여 식을 나타낸다.", + }; + + const result = await mapObjective( + {}, + { + request: expressionRequest, + refs, + candidate: { + ...candidate, + generation_kind: "expression", + question_text: "다음 중 정리했을 때 이차식이 아닌 것은?", + proposed_solution_trace: "각 다항식을 정리하여 이차식인지 확인한다.", + inferred_intent: alternateIntent, + }, + intent: alternateIntent, + strategy: null, + }, + ); + + expect(result.gate.status).toBe("passed"); + }); + + it("allows variable-expression aliases for expression value candidates", async () => { + const variableRequest: GenerateRequest = { + ...request, + grade: 1, + topic: "9수02-01", + topic_name: "문자의 사용과 식의 값", + source_problem_text: "문자를 사용하여 수량 관계를 식으로 나타낸다.", + }; + const alternateIntent: Intent = { + ...intent, + objective_code: "9수01-01", + objective_description: "수와 연산", + }; + + const result = await mapObjective( + {}, + { + request: variableRequest, + refs, + candidate: { + ...candidate, + generation_kind: "expression", + question_text: "a=2 sqrt(7)+3 sqrt(2), b=2 sqrt(7)-3 sqrt(2)일 때 a^2+b^2의 값은?", + proposed_solution_trace: "문자 a, b에 식을 대입하여 식의 값을 계산한다.", + inferred_intent: alternateIntent, + }, + intent: alternateIntent, + strategy: null, + }, + ); + + expect(result.gate.status).toBe("passed"); + }); + + it("allows rational arithmetic aliases for expression calculation candidates", async () => { + const arithmeticRequest: GenerateRequest = { + ...request, + grade: 1, + topic: "9수01-03", + topic_name: "유리수의 사칙연산", + source_problem_text: "유리수의 사칙계산의 원리를 이해하고 계산할 수 있다.", + }; + const broadIntent: Intent = { + ...intent, + objective_code: "9수01-01", + objective_description: "수와 연산", + }; + + const result = await mapObjective( + {}, + { + request: arithmeticRequest, + refs, + candidate: { + ...candidate, + generation_kind: "expression", + question_text: "다음 식의 값을 구하시오. ((-3/4)+(5/6))/(-7/12)-1/2", + proposed_solution_trace: "유리수의 덧셈, 나눗셈, 뺄셈 순서로 계산한다.", + inferred_intent: broadIntent, + }, + intent: broadIntent, + strategy: null, + }, + ); + + expect(result.gate.status).toBe("passed"); + }); + + it("allows integer-and-rational aliases for rational comparison candidates", async () => { + const rationalRequest: GenerateRequest = { + ...request, + grade: 1, + topic: "9수01-02", + topic_name: "정수와 유리수", + source_problem_text: "정수와 유리수의 뜻을 알고 대소 관계를 판단한다.", + }; + const broadIntent: Intent = { + ...intent, + objective_code: "9수01-01", + objective_description: "수와 연산", + }; + + const result = await mapObjective( + {}, + { + request: rationalRequest, + refs, + candidate: { + ...candidate, + generation_kind: "expression", + question_text: "다음 중 옳은 것은? ① -7/5 > -1.3 ② 0.2 < 1/6 ③ -3.8 < -19/5 ④ -2.4 > -5/2 ⑤ 3/4 < 0.7", + proposed_solution_trace: "음수와 양수인 유리수를 각각 같은 형태로 바꾸어 대소 관계를 비교한다.", + inferred_intent: broadIntent, + }, + intent: broadIntent, + strategy: null, + }, + ); + + expect(result.gate.status).toBe("passed"); + }); + + it("allows radical-expression aliases for square-root calculation candidates", async () => { + const radicalRequest: GenerateRequest = { + ...request, + grade: 3, + topic: "9수01-06", + topic_name: "근호를 포함한 식의 계산", + source_problem_text: "근호를 포함한 식의 사칙계산을 할 수 있다.", + }; + const broadIntent: Intent = { + ...intent, + objective_code: "9수01-05", + objective_description: "제곱근과 실수를 이해한다.", + }; + + const result = await mapObjective( + {}, + { + request: radicalRequest, + refs, + candidate: { + ...candidate, + generation_kind: "expression", + question_text: "x = (sqrt(5)-1)/(sqrt(5)+1) 일 때, x**2 + 1/x**2 의 값을 구하시오.", + proposed_solution_trace: "분모를 유리화하고 근호를 포함한 식을 정리하여 값을 계산한다.", + inferred_intent: broadIntent, + }, + intent: broadIntent, + strategy: null, + }, + ); + + expect(result.gate.status).toBe("passed"); + }); +}); diff --git a/packages/agent/tests/problem-generation-deterministic-modes.test.ts b/packages/agent/tests/problem-generation-deterministic-modes.test.ts new file mode 100644 index 0000000..0ed7719 --- /dev/null +++ b/packages/agent/tests/problem-generation-deterministic-modes.test.ts @@ -0,0 +1,209 @@ +import { describe, expect, it } from "vitest"; + +import type { GenerateRequest, Intent, RagResult } from "../src/schemas/index.js"; +import { deterministicInitialCandidate } from "../src/steps/problem-generation-deterministic.js"; + +const intent: Intent = { + objective_code: "9수02-10", + objective_description: "이차방정식을 활용한다.", + evaluation_dimensions: [{ id: "A", description: "활용", must_preserve: true }], + required_techniques: ["quadratic_equation"], + forbidden_techniques: [], + surface_constraints: { difficulty: "medium", problem_type: "objective" }, +}; + +const baseRequest: GenerateRequest = { + grade: 3, + topic: "9수02-10", + topic_name: "이차방정식의 활용", + mode: "structural", + school_level: "middle", + dims: ["A"], + count: 5, + difficulty: "medium", + problem_type: "objective", +}; + +const refs: RagResult[] = [ + { + item_id: "ref-1", + match_reason: "hybrid", + problem: { + item_id: "ref-1", + source_dataset: "111", + split: "train", + source_label_type: "problem_label", + school_level: "middle", + grade: 3, + semester: null, + topic_code: "9수02-10", + topic_name: "이차방정식의 활용", + achievement_standard: "이차방정식을 활용하여 문제를 해결한다.", + question_text: "이차방정식 활용 문제", + answer_text: "2", + explanation_text: null, + choice_blocks: [], + problem_type_norm: "objective", + difficulty_norm: "medium", + question_image_relpath: null, + answer_image_relpath: null, + question_json_relpath: null, + answer_json_relpath: null, + }, + }, +]; + +const topicCodes = [ + "9수01-01", + "9수01-02", + "9수01-03", + "9수01-04", + "9수01-05", + "9수01-06", + "9수02-01", + "9수02-02", + "9수02-03", + "9수02-04", + "9수02-05", + "9수02-06", + "9수02-07", + "9수02-08", + "9수02-09", + "9수02-10", + "9수03-01", + "9수03-02", + "9수03-03", + "9수03-04", + "9수04-01", + "9수04-02", + "9수04-03", + "9수04-04", + "9수04-05", + "9수04-06", + "9수04-07", + "9수05-01", + "9수05-02", + "9수05-03", +] as const; + +type TopicCode = (typeof topicCodes)[number]; + +const conceptualSignals: Readonly> = { + "9수01-01": ["소인수분해가"], + "9수01-02": ["수직선", "0에 더 가까운"], + "9수01-03": ["기온"], + "9수01-04": ["기약분수"], + "9수01-05": ["sqrt(50)", "정사각형"], + "9수01-06": ["정사각형", "넓이"], + "9수02-01": ["직사각형", "둘레"], + "9수02-02": ["원래의 일차식"], + "9수02-03": ["어떤 수"], + "9수02-04": ["공책"], + "9수02-05": ["직사각형", "전개"], + "9수02-06": ["어떤 수"], + "9수02-07": ["사과", "배"], + "9수02-08": ["직사각형"], + "9수02-09": ["x^2 - 2x - 8"], + "9수02-10": ["두 수", "곱"], + "9수03-01": ["복사", "요금"], + "9수03-02": ["변화량"], + "9수03-03": ["택시"], + "9수03-04": ["꼭짓점"], + "9수04-01": ["세 점", "선분", "삼각형"], + "9수04-02": ["밑각"], + "9수04-03": ["평행사변형"], + "9수04-04": ["닮음비"], + "9수04-05": ["sin A"], + "9수04-06": ["두 점"], + "9수04-07": ["원주각", "중심각"], + "9수05-01": ["상대도수가"], + "9수05-02": ["주사위"], + "9수05-03": ["중앙값"], +}; + +describe("deterministicInitialCandidate mode separation", () => { + it("creates different structural and conceptual candidates for the same topic", () => { + const structural = deterministicInitialCandidate({ + request: baseRequest, + intent, + refs, + attempt: 1, + }); + const conceptual = deterministicInitialCandidate({ + request: { ...baseRequest, mode: "conceptual" }, + intent, + refs, + attempt: 1, + }); + + expect(structural?.mode).toBe("structural"); + expect(conceptual?.mode).toBe("conceptual"); + expect(structural?.question_text).not.toBe(conceptual?.question_text); + expect(structural?.proposed_solution_trace).toContain("구조동형"); + expect(conceptual?.proposed_solution_trace).toContain("개념동형"); + }); + + it("keeps structural and conceptual templates separated for all middle-school topics", () => { + for (const topic of topicCodes) { + const structural = deterministicInitialCandidate({ + request: { ...baseRequest, topic }, + intent, + refs, + attempt: 1, + }); + const conceptual = deterministicInitialCandidate({ + request: { ...baseRequest, topic, mode: "conceptual" }, + intent, + refs, + attempt: 1, + }); + + expect(structural?.mode, topic).toBe("structural"); + expect(conceptual?.mode, topic).toBe("conceptual"); + expect(structural?.question_text, topic).not.toBe(conceptual?.question_text); + } + }); + + it("keeps conceptual candidates from being label-only structural variants", () => { + for (const topic of topicCodes) { + const conceptual = deterministicInitialCandidate({ + request: { ...baseRequest, topic, mode: "conceptual" }, + intent, + refs, + attempt: 1, + }); + + const evidence = [ + conceptual?.question_text ?? "", + conceptual?.proposed_solution_trace ?? "", + ].join("\n"); + const hasSignal = conceptualSignals[topic].some((signal) => evidence.includes(signal)); + + expect(hasSignal, `${topic}: ${conceptual?.question_text}`).toBe(true); + } + }); + + it("keeps application and concept templates focused on modeling or interpretation", () => { + const modelingTopics: readonly TopicCode[] = ["9수02-03", "9수02-04", "9수02-10"]; + for (const topic of modelingTopics) { + const conceptual = deterministicInitialCandidate({ + request: { ...baseRequest, topic, mode: "conceptual" }, + intent, + refs, + attempt: 1, + }); + + expect(conceptual?.question_text, topic).toContain("방정식을 세워"); + expect(conceptual?.question_text, topic).not.toMatch(/=\s*-?\d/u); + } + + const median = deterministicInitialCandidate({ + request: { ...baseRequest, topic: "9수05-03", mode: "conceptual" }, + intent, + refs, + attempt: 1, + }); + + expect(median?.question_text).toContain("추가"); + }); +}); diff --git a/packages/agent/tests/problem-generation-deterministic.test.ts b/packages/agent/tests/problem-generation-deterministic.test.ts new file mode 100644 index 0000000..1e3eb78 --- /dev/null +++ b/packages/agent/tests/problem-generation-deterministic.test.ts @@ -0,0 +1,259 @@ +import { describe, expect, it } from "vitest"; + +import type { GenerateRequest, Intent, RagResult } from "../src/schemas/index.js"; +import { deterministicInitialCandidate } from "../src/steps/problem-generation-deterministic.js"; + +const intent: Intent = { + objective_code: "9수05-01", + objective_description: "자료의 정리와 해석", + evaluation_dimensions: [{ id: "A", description: "상대도수", must_preserve: true }], + required_techniques: ["relative_frequency"], + forbidden_techniques: [], + surface_constraints: { difficulty: "medium", problem_type: "objective" }, +}; + +const request: GenerateRequest = { + grade: 1, + topic: "9수05-01", + topic_name: "자료의 정리와 해석", + mode: "structural", + school_level: "middle", + dims: ["A"], + count: 5, + difficulty: "medium", + problem_type: "objective", +}; + +const refs: RagResult[] = [ + { + item_id: "ref-1", + match_reason: "hybrid", + problem: { + item_id: "ref-1", + source_dataset: "111", + split: "train", + source_label_type: "problem_label", + school_level: "middle", + grade: 1, + semester: null, + topic_code: "9수05-01", + topic_name: "자료의 정리와 해석", + achievement_standard: "자료를 정리하고 해석한다.", + question_text: "상대도수를 구하시오.", + answer_text: "1/5", + explanation_text: null, + choice_blocks: [], + problem_type_norm: "objective", + difficulty_norm: "medium", + question_image_relpath: null, + answer_image_relpath: null, + question_json_relpath: null, + answer_json_relpath: null, + }, + }, +]; + +describe("deterministicInitialCandidate", () => { + it("creates a topic-aligned deterministic candidate for known unstable topics", () => { + const candidate = deterministicInitialCandidate({ + request, + intent, + refs, + attempt: 1, + }); + + expect(candidate?.generation_kind).toBe("statistics"); + expect(candidate?.source_refs).toEqual(["ref-1"]); + expect(candidate?.question_text).toContain("상대도수"); + }); + + it("returns null for topics that do not need deterministic generation", () => { + const candidate = deterministicInitialCandidate({ + request: { ...request, topic: "9수99-99" }, + intent, + refs: [], + attempt: 1, + }); + + expect(candidate).toBeNull(); + }); + + it("creates a stable circle-angle candidate for the circle theorem topic", () => { + const candidate = deterministicInitialCandidate({ + request: { ...request, topic: "9수04-07", topic_name: "원주각" }, + intent: { ...intent, objective_code: "9수04-07", objective_description: "원주각을 이해한다." }, + refs, + attempt: 1, + }); + + expect(candidate?.generation_kind).toBe("geometry"); + expect(candidate?.question_text).toContain("중심각"); + expect(candidate?.expected_answer).toBe("40도"); + }); + + it("creates a stable linear-expression calculation candidate", () => { + const candidate = deterministicInitialCandidate({ + request: { ...request, topic: "9수02-02", topic_name: "일차식의 계산" }, + intent: { ...intent, objective_code: "9수02-02", objective_description: "일차식을 계산한다." }, + refs, + attempt: 1, + }); + + expect(candidate?.generation_kind).toBe("expression"); + expect(candidate?.question_text).toContain("일차식"); + expect(candidate?.expected_answer).toBe("4x - 11"); + }); + + it("creates stable linear-function candidates for graph and application topics", () => { + const graphCandidate = deterministicInitialCandidate({ + request: { ...request, topic: "9수03-02", topic_name: "일차함수와 그래프" }, + intent: { ...intent, objective_code: "9수03-02", objective_description: "일차함수의 그래프를 이해한다." }, + refs, + attempt: 1, + }); + const applicationCandidate = deterministicInitialCandidate({ + request: { ...request, topic: "9수03-03", topic_name: "일차함수의 활용" }, + intent: { ...intent, objective_code: "9수03-03", objective_description: "일차함수를 활용한다." }, + refs, + attempt: 1, + }); + + expect(graphCandidate?.generation_kind).toBe("function"); + expect(graphCandidate?.expected_answer).toBe("(0, -3)"); + expect(applicationCandidate?.generation_kind).toBe("function"); + expect(applicationCandidate?.expected_answer).toBe("1000x + 3000, 8000원"); + }); + + it("creates a stable quadratic-equation application candidate", () => { + const candidate = deterministicInitialCandidate({ + request: { ...request, topic: "9수02-10", topic_name: "이차방정식의 활용" }, + intent: { ...intent, objective_code: "9수02-10", objective_description: "이차방정식을 활용한다." }, + refs, + attempt: 1, + }); + + expect(candidate?.generation_kind).toBe("equation"); + expect(candidate?.question_text).toContain("직사각형"); + expect(candidate?.expected_answer).toBe("2cm, 5cm"); + }); + + it("creates a stable linear-equation application candidate", () => { + const candidate = deterministicInitialCandidate({ + request: { ...request, topic: "9수02-04", topic_name: "일차방정식의 활용" }, + intent: { ...intent, objective_code: "9수02-04", objective_description: "일차방정식을 활용한다." }, + refs, + attempt: 1, + }); + + expect(candidate?.generation_kind).toBe("equation"); + expect(candidate?.question_text).toContain("입장권"); + expect(candidate?.expected_answer).toBe("5000원"); + }); + + it("creates a stable triangle-property candidate", () => { + const candidate = deterministicInitialCandidate({ + request: { ...request, topic: "9수04-02", topic_name: "삼각형의 성질" }, + intent: { ...intent, objective_code: "9수04-02", objective_description: "삼각형의 성질을 이해한다." }, + refs, + attempt: 1, + }); + + expect(candidate?.generation_kind).toBe("geometry"); + expect(candidate?.question_text).toContain("AB = AC"); + expect(candidate?.expected_answer).toBe("70도"); + }); + + it("creates stable candidates for remaining stochastic algebra and function topics", () => { + const repeatingDecimal = deterministicInitialCandidate({ + request: { ...request, topic: "9수01-04", topic_name: "유리수와 순환소수" }, + intent, + refs, + attempt: 1, + }); + const system = deterministicInitialCandidate({ + request: { ...request, topic: "9수02-07", topic_name: "연립일차방정식" }, + intent, + refs, + attempt: 1, + }); + const quadraticFunction = deterministicInitialCandidate({ + request: { ...request, topic: "9수03-04", topic_name: "이차함수와 그래프" }, + intent, + refs, + attempt: 1, + }); + + expect(repeatingDecimal?.expected_answer).toBe("4/33"); + expect(system?.expected_answer).toBe("x=3, y=2"); + expect(quadraticFunction?.expected_answer).toBe("아래로 열린다"); + }); + + it("creates a stable integer-and-rational comparison candidate", () => { + const candidate = deterministicInitialCandidate({ + request: { ...request, topic: "9수01-02", topic_name: "정수와 유리수" }, + intent, + refs, + attempt: 1, + }); + + expect(candidate?.expected_answer).toBe("-3/4 < -2/3"); + }); + + it("creates a stable square-root candidate", () => { + const candidate = deterministicInitialCandidate({ + request: { ...request, topic: "9수01-05", topic_name: "제곱근과 실수" }, + intent, + refs, + attempt: 1, + }); + + expect(candidate?.expected_answer).toBe("9, -9"); + }); + + it("creates a stable quadrilateral-property candidate", () => { + const candidate = deterministicInitialCandidate({ + request: { ...request, topic: "9수04-03", topic_name: "사각형의 성질" }, + intent, + refs, + attempt: 1, + }); + + expect(candidate?.expected_answer).toBe("10cm"); + }); + + it("creates stable circle-position and probability candidates", () => { + const circlePosition = deterministicInitialCandidate({ + request: { ...request, topic: "9수04-06", topic_name: "원과 직선의 위치 관계" }, + intent, + refs, + attempt: 1, + }); + const probability = deterministicInitialCandidate({ + request: { ...request, topic: "9수05-02", topic_name: "경우의 수와 확률" }, + intent, + refs, + attempt: 1, + }); + + expect(circlePosition?.expected_answer).toBe("한 점에서 만난다"); + expect(probability?.expected_answer).toBe("1/2"); + }); + + it("creates stable prime-factorization and radical-expression candidates", () => { + const primeFactorization = deterministicInitialCandidate({ + request: { ...request, topic: "9수01-01", topic_name: "소인수분해" }, + intent, + refs, + attempt: 1, + }); + const radicalExpression = deterministicInitialCandidate({ + request: { ...request, topic: "9수01-06", topic_name: "근호를 포함한 식의 계산" }, + intent, + refs, + attempt: 1, + }); + + expect(primeFactorization?.expected_answer).toBe("2^2 * 3"); + expect(radicalExpression?.expected_answer).toBe("5sqrt(2)"); + }); +}); diff --git a/packages/agent/tests/problem-generation-geometry-statistics-guards.test.ts b/packages/agent/tests/problem-generation-geometry-statistics-guards.test.ts new file mode 100644 index 0000000..10e5765 --- /dev/null +++ b/packages/agent/tests/problem-generation-geometry-statistics-guards.test.ts @@ -0,0 +1,109 @@ +import { describe, expect, it } from "vitest"; + +import type { ConstraintCriticAgent, GeneratorAgent, RefinerAgent } from "../src/agents/index.js"; +import type { GenerateRequest, GeneratedProblem } from "../src/schemas/index.js"; +import { generateProblem } from "../src/steps/problem-generation.js"; +import type { MathEngineClient } from "../src/tools/math-engine-client.js"; + +const candidate: GeneratedProblem = { + candidate_id: "00000000-0000-0000-0000-000000000050", + mode: "structural", + generation_kind: "geometry", + question_text: "삼각형이 닮음일 때 값을 구하시오.", + expected_answer: "1", + proposed_solution_trace: "대응 관계를 확인한다.", + source_refs: ["ref-1"], + inferred_intent: { + objective_code: "9수04-04", + objective_description: "도형의 닮음을 이해한다.", + evaluation_dimensions: [{ id: "A", description: "대응 관계", must_preserve: true }], + required_techniques: ["similarity"], + forbidden_techniques: [], + surface_constraints: { difficulty: "medium", problem_type: "objective" }, + }, + generation_metadata: { + model: "test", + temperature: 0, + prompt_id: "test", + prompt_version: "0.0.0", + attempt: 1, + generated_at: "2026-06-06T00:00:00.000Z", + }, +}; + +const critic: ConstraintCriticAgent = { critique: async () => ({ passes: true, hints: [] }) }; + +const mathEngine: MathEngineClient = { + health: async () => ({ status: "ok", engine: "sympy" }), + solve: async () => ({ solutions: [] }), + verify: async () => ({ equivalent: true, diff: "0" }), + simplify: async ({ expr }) => ({ simplified: expr }), + differentiate: async () => ({ derivative: "" }), + limit: async () => ({ limit: "" }), +}; + +describe("generateProblem geometry and statistics guards", () => { + it("replaces similarity perimeter problems with direct corresponding-side problems", async () => { + const badCandidate: GeneratedProblem = { + ...candidate, + question_text: "두 삼각형이 닮음일 때 삼각형 DEF의 둘레의 길이는?", + expected_answer: "36cm", + }; + const result = await generateProblem( + { + generator: generatorReturning(badCandidate), + critic, + refiner: refinerReturning(badCandidate), + mathEngine, + }, + inputForTopic("9수04-04", badCandidate), + ); + + expect(result.data.question_text).toContain("대응하는 변"); + expect(result.data.question_text).not.toContain("둘레"); + }); + + it("replaces data-summary explanation choices with relative-frequency calculation", async () => { + const badCandidate: GeneratedProblem = { + ...candidate, + generation_kind: "statistics", + question_text: "다음 설명 중 옳지 않은 것은? ① 도수 설명 ② 상대도수 설명", + expected_answer: "②", + }; + const result = await generateProblem( + { + generator: generatorReturning(badCandidate), + critic, + refiner: refinerReturning(badCandidate), + mathEngine, + }, + inputForTopic("9수05-01", badCandidate), + ); + + expect(result.data.question_text).toContain("상대도수"); + expect(result.data.question_text).not.toContain("옳지 않은"); + }); +}); + +function generatorReturning(problem: GeneratedProblem): GeneratorAgent { + return { generate: async () => problem }; +} + +function refinerReturning(problem: GeneratedProblem): RefinerAgent { + return { refine: async () => problem }; +} + +function inputForTopic(topic: string, problem: GeneratedProblem) { + const request: GenerateRequest = { + grade: 2, + topic, + topic_name: topic === "9수04-04" ? "도형의 닮음" : "자료의 정리와 해석", + mode: "structural", + school_level: "middle", + dims: ["A"], + count: 5, + difficulty: "medium", + problem_type: "objective", + }; + return { request, intent: problem.inferred_intent, refs: [], strategy: null, attempt: 1 }; +} diff --git a/packages/agent/tests/problem-generation-topic-guards.test.ts b/packages/agent/tests/problem-generation-topic-guards.test.ts new file mode 100644 index 0000000..2084b5b --- /dev/null +++ b/packages/agent/tests/problem-generation-topic-guards.test.ts @@ -0,0 +1,230 @@ +import { describe, expect, it } from "vitest"; + +import type { + ConstraintCriticAgent, + GeneratorAgent, + RefinerAgent, +} from "../src/agents/index.js"; +import type { GenerateRequest, GeneratedProblem } from "../src/schemas/index.js"; +import { generateProblem } from "../src/steps/problem-generation.js"; +import type { MathEngineClient } from "../src/tools/math-engine-client.js"; + +const candidate: GeneratedProblem = { + candidate_id: "00000000-0000-0000-0000-000000000020", + mode: "structural", + generation_kind: "expression", + question_text: "다음 식을 계산하시오.", + expected_answer: "1", + proposed_solution_trace: "식을 정리한다.", + source_refs: ["ref-1"], + inferred_intent: { + objective_code: "9수01-06", + objective_description: "근호를 포함한 식의 계산", + evaluation_dimensions: [{ id: "A", description: "계산", must_preserve: true }], + required_techniques: ["calculation"], + forbidden_techniques: [], + surface_constraints: { difficulty: "medium", problem_type: "objective" }, + }, + generation_metadata: { + model: "test", + temperature: 0, + prompt_id: "test", + prompt_version: "0.0.0", + attempt: 1, + generated_at: "2026-06-06T00:00:00.000Z", + }, +}; + +const critic: ConstraintCriticAgent = { + critique: async () => ({ passes: true, hints: [] }), +}; + +const mathEngine: MathEngineClient = { + health: async () => ({ status: "ok", engine: "sympy" }), + solve: async () => ({ solutions: [] }), + verify: async () => ({ equivalent: true, diff: "0" }), + simplify: async ({ expr }) => ({ simplified: expr }), + differentiate: async () => ({ derivative: "" }), + limit: async () => ({ limit: "" }), +}; + +describe("generateProblem deterministic topic guards", () => { + it("refines radical calculation candidates that use compound a-b substitutions", async () => { + const badCandidate: GeneratedProblem = { + ...candidate, + question_text: "a = sqrt(7) + 2sqrt(5), b = sqrt(35) - 1일 때, sqrt(7)a - sqrt(35)b의 값을 구하시오.", + expected_answer: "-8 + 3sqrt(35)", + }; + const goodCandidate: GeneratedProblem = { + ...badCandidate, + question_text: "다음 식을 간단히 하시오. 3sqrt(20) - sqrt(45) + 2sqrt(5)", + expected_answer: "5sqrt(5)", + }; + const hints: string[] = []; + + const result = await generateProblem( + { + generator: generatorReturning(badCandidate), + critic, + refiner: refinerReturning(goodCandidate, hints), + mathEngine, + }, + inputForTopic("9수01-06", badCandidate), + ); + + expect(result.data.question_text).toContain("\\sqrt{20}"); + expect(hints.join("\n")).toContain("근호"); + }); + + it("refines common-factor candidates into direct factorization", async () => { + const badCandidate: GeneratedProblem = { + ...candidate, + question_text: "두 다항식 6x**2 - x - 15, 9x**2 - 25를 인수분해하였을 때 공통으로 들어 있는 인수는?", + expected_answer: "3x + 5", + }; + const goodCandidate: GeneratedProblem = { + ...badCandidate, + question_text: "다항식 6x**2 - x - 15를 인수분해하시오.", + expected_answer: "(3x - 5)(2x + 3)", + }; + + const result = await generateProblem( + { + generator: generatorReturning(badCandidate), + critic, + refiner: refinerReturning(goodCandidate, []), + mathEngine, + }, + inputForTopic("9수02-08", badCandidate), + ); + + expect(result.data.question_text).not.toContain("공통"); + expect(result.data.question_text).toContain("인수분해"); + }); + + it("refines multipart value-expression candidates with incomplete expected answers", async () => { + const badCandidate: GeneratedProblem = { + ...candidate, + question_text: "공책 a권과 볼펜 4자루를 샀다. (1) 지불한 돈을 식으로 나타내어라. (2) a=3일 때 값을 구하여라.", + expected_answer: "(1500a+3200)원", + }; + const goodCandidate: GeneratedProblem = { + ...badCandidate, + question_text: "공책 a권과 볼펜 4자루를 샀다. 지불한 돈을 a를 사용한 식으로 나타내어라.", + expected_answer: "1500a+3200원", + }; + + const result = await generateProblem( + { + generator: generatorReturning(badCandidate), + critic, + refiner: refinerReturning(goodCandidate, []), + mathEngine, + }, + inputForTopic("9수02-01", badCandidate), + ); + + expect(result.data.question_text).toContain("연필"); + expect(result.data.question_text).not.toContain("(2)"); + }); + + it("replaces multi-statement statistics choices with a direct average problem", async () => { + const badCandidate: GeneratedProblem = { + ...candidate, + generation_kind: "statistics", + question_text: "다음 <보기> 중 옳은 것을 모두 고른 것은? ㉠ 평균은 대푯값이다. ① ㉠", + expected_answer: "①", + }; + + const result = await generateProblem( + { + generator: generatorReturning(badCandidate), + critic, + refiner: refinerReturning(badCandidate, []), + mathEngine, + }, + inputForTopic("9수05-03", badCandidate), + ); + + expect(result.data.question_text).toContain("평균"); + expect(result.data.question_text).not.toContain("<보기>"); + }); + + it("replaces data-summary false-choice questions with direct relative-frequency calculation", async () => { + const badCandidate: GeneratedProblem = { + ...candidate, + generation_kind: "statistics", + question_text: "다음 중 옳지 않은 것은? ① 변량 설명 ② 도수 설명", + expected_answer: "②", + }; + + const result = await generateProblem( + { + generator: generatorReturning(badCandidate), + critic, + refiner: refinerReturning(badCandidate, []), + mathEngine, + }, + inputForTopic("9수05-01", badCandidate), + ); + + expect(result.data.question_text).toContain("상대도수"); + expect(result.data.question_text).not.toContain("옳지 않은"); + }); + + it("replaces equation verification add-ons with a plain solve problem", async () => { + const badCandidate: GeneratedProblem = { + ...candidate, + generation_kind: "equation", + question_text: "다음 이차방정식 (3x+2)(x-5)=0의 해를 구하고, x=-2/3가 해인지 확인하시오.", + expected_answer: "x=-2/3 또는 x=5; x=-2/3는 해", + }; + + const result = await generateProblem( + { + generator: generatorReturning(badCandidate), + critic, + refiner: refinerReturning(badCandidate, []), + mathEngine, + }, + inputForTopic("9수02-09", badCandidate), + ); + + expect(result.data.question_text).toContain("해를 구하시오"); + expect(result.data.question_text).not.toContain("확인"); + }); +}); + +function generatorReturning(problem: GeneratedProblem): GeneratorAgent { + return { generate: async () => problem }; +} + +function refinerReturning(problem: GeneratedProblem, hints: string[]): RefinerAgent { + return { + refine: async (input) => { + hints.push(...input.hints); + return problem; + }, + }; +} + +function inputForTopic(topic: string, problem: GeneratedProblem) { + const request: GenerateRequest = { + grade: topic === "9수02-08" ? 3 : 3, + topic, + topic_name: topic === "9수02-08" ? "다항식의 곱셈과 인수분해" : "근호를 포함한 식의 계산", + mode: "structural", + school_level: "middle", + dims: ["A"], + count: 5, + difficulty: "medium", + problem_type: "objective", + }; + return { + request, + intent: problem.inferred_intent, + refs: [], + strategy: null, + attempt: 1, + }; +} diff --git a/packages/agent/tests/problem-generation.test.ts b/packages/agent/tests/problem-generation.test.ts new file mode 100644 index 0000000..185b1a8 --- /dev/null +++ b/packages/agent/tests/problem-generation.test.ts @@ -0,0 +1,256 @@ +import { describe, expect, it } from "vitest"; + +import type { + ConstraintCriticAgent, + GeneratorAgent, + RefinerAgent, +} from "../src/agents/index.js"; +import type { GeneratedProblem } from "../src/schemas/index.js"; +import { generateProblem } from "../src/steps/problem-generation.js"; +import type { MathEngineClient } from "../src/tools/math-engine-client.js"; + +const candidate: GeneratedProblem = { + candidate_id: "00000000-0000-0000-0000-000000000010", + mode: "structural", + generation_kind: "equation", + question_text: "다음 방정식을 풀어라. (x - 5)(x + 2) = 0", + expected_answer: "5, -2", + proposed_solution_trace: "인수분해된 식에서 해를 구한다.", + source_refs: ["ref-1"], + inferred_intent: { + objective_code: "9수02-10", + objective_description: "이차방정식을 활용하여 문제를 해결한다.", + evaluation_dimensions: [{ id: "A", description: "이차방정식 활용", must_preserve: true }], + required_techniques: ["quadratic_equation"], + forbidden_techniques: [], + surface_constraints: { difficulty: "medium", problem_type: "objective" }, + }, + generation_metadata: { + model: "test", + temperature: 0, + prompt_id: "test", + prompt_version: "0.0.0", + attempt: 1, + generated_at: "2026-06-06T00:00:00.000Z", + }, +}; + +const generator: GeneratorAgent = { + generate: async () => candidate, +}; + +const critic: ConstraintCriticAgent = { + critique: async () => ({ passes: true, hints: [] }), +}; + +const refiner: RefinerAgent = { + refine: async () => candidate, +}; + +const mathEngine: MathEngineClient = { + health: async () => ({ status: "ok", engine: "sympy" }), + solve: async () => { + throw new Error("math-engine rejected extracted equation"); + }, + verify: async () => ({ equivalent: true, diff: "0" }), + simplify: async ({ expr }) => ({ simplified: expr }), + differentiate: async () => ({ derivative: "" }), + limit: async () => ({ limit: "" }), +}; + +describe("generateProblem", () => { + it("keeps the generated candidate when equation answer normalization fails", async () => { + const result = await generateProblem( + { generator, critic, refiner, mathEngine }, + { + request: { + grade: 3, + topic: "9수02-03", + topic_name: "일차방정식", + mode: "structural", + school_level: "middle", + dims: ["A"], + count: 5, + difficulty: "medium", + problem_type: "objective", + }, + intent: candidate.inferred_intent, + refs: [], + strategy: null, + attempt: 1, + }, + ); + + expect(result.gate.status).toBe("passed"); + expect(result.data.expected_answer).toBe("5, -2"); + }); + + it("does not normalize answers for choice-style equation candidates", async () => { + const choiceCandidate: GeneratedProblem = { + ...candidate, + question_text: "다음 풀이 과정에 이용된 등식의 성질은? x+1=3 ① ㉠ ② ㉡", + expected_answer: "②", + }; + const result = await generateProblem( + { + generator: { generate: async () => choiceCandidate }, + critic, + refiner, + mathEngine: { ...mathEngine, solve: async () => ({ solutions: ["20/119"] }) }, + }, + { + request: { + grade: 1, + topic: "9수02-03", + topic_name: "일차방정식", + mode: "structural", + school_level: "middle", + dims: ["A"], + count: 5, + difficulty: "medium", + problem_type: "objective", + }, + intent: choiceCandidate.inferred_intent, + refs: [], + strategy: null, + attempt: 1, + }, + ); + + expect(result.data.expected_answer).toBe("②"); + }); + + it("refines integer-and-rational candidates that drift into radical calculation", async () => { + const badCandidate: GeneratedProblem = { + ...candidate, + generation_kind: "expression", + question_text: "다음 식을 간단히 하시오. sqrt(18)+sqrt(8)", + expected_answer: "5*sqrt(2)", + }; + const goodCandidate: GeneratedProblem = { + ...badCandidate, + question_text: "다음 수 중 정수가 아닌 유리수는 몇 개인가? -1/2, 0.3, -4/2, 7", + expected_answer: "2", + }; + const hints: string[] = []; + + const result = await generateProblem( + { + generator: { generate: async () => badCandidate }, + critic, + refiner: { + refine: async (input) => { + hints.push(...input.hints); + return goodCandidate; + }, + }, + mathEngine, + }, + { + request: { + grade: 1, + topic: "9수01-02", + topic_name: "정수와 유리수", + mode: "structural", + school_level: "middle", + dims: ["A"], + count: 5, + difficulty: "medium", + problem_type: "objective", + }, + intent: badCandidate.inferred_intent, + refs: [], + strategy: null, + attempt: 1, + }, + ); + + expect(result.data.question_text).toBe(goodCandidate.question_text); + expect(hints.join("\n")).toContain("정수와 유리수"); + expect(result.refined_by).toContain("deterministic-topic-guard"); + }); + + it("refines repeating-decimal candidates that use overdot choice notation", async () => { + const badCandidate: GeneratedProblem = { + ...candidate, + generation_kind: "expression", + question_text: "순환소수 0.121212...의 표현으로 옳은 것은? ① 0.1̇2̇ ② 0.12̇", + expected_answer: "①", + }; + const goodCandidate: GeneratedProblem = { + ...badCandidate, + question_text: "순환소수 0.121212...를 분수로 나타내시오.", + expected_answer: "4/33", + }; + + const result = await generateProblem( + { + generator: { generate: async () => badCandidate }, + critic, + refiner: { refine: async () => goodCandidate }, + mathEngine, + }, + { + request: { + grade: 2, + topic: "9수01-04", + topic_name: "유리수와 순환소수", + mode: "structural", + school_level: "middle", + dims: ["A"], + count: 5, + difficulty: "medium", + problem_type: "objective", + }, + intent: badCandidate.inferred_intent, + refs: [], + strategy: null, + attempt: 1, + }, + ); + + expect(result.data.question_text).toBe(goodCandidate.question_text); + }); + + it("refines geometry candidates that depend on an unseen figure", async () => { + const badCandidate: GeneratedProblem = { + ...candidate, + generation_kind: "geometry", + question_text: "오른쪽 그림에서 AB는 원 O의 지름이고 PT는 접선이다. x의 값을 구하시오.", + expected_answer: "38도", + }; + const goodCandidate: GeneratedProblem = { + ...badCandidate, + question_text: "원 O의 반지름이 5cm이고 중심 O에서 직선 l까지의 거리가 5cm일 때, 원 O와 직선 l의 위치 관계를 쓰시오.", + expected_answer: "한 점에서 만난다", + }; + + const result = await generateProblem( + { + generator: { generate: async () => badCandidate }, + critic, + refiner: { refine: async () => goodCandidate }, + mathEngine, + }, + { + request: { + grade: 3, + topic: "9수04-06", + topic_name: "원과 직선의 위치 관계", + mode: "structural", + school_level: "middle", + dims: ["A"], + count: 5, + difficulty: "medium", + problem_type: "objective", + }, + intent: badCandidate.inferred_intent, + refs: [], + strategy: null, + attempt: 1, + }, + ); + + expect(result.data.question_text).toBe(goodCandidate.question_text); + }); +}); diff --git a/packages/agent/tests/rag-client.test.ts b/packages/agent/tests/rag-client.test.ts index 7db96f3..d5ea4ff 100644 --- a/packages/agent/tests/rag-client.test.ts +++ b/packages/agent/tests/rag-client.test.ts @@ -87,6 +87,97 @@ describe("createInMemoryRagClient", () => { expect(results).toHaveLength(0); }); + + it("matches app achievement code against corpus achievement_code", async () => { + const jsonlPath = await writeFixture([ + canonicalRecord({ + problem_id: "111:train:linear-equation", + school_level: "middle", + grade: 1, + topic_name: "일차방정식", + achievement_code: "9수02-03", + achievement_standard: "일차방정식을 풀 수 있다.", + question_text: "일차방정식 2x+3=7을 풀어라.", + answer_text: "2", + problem_type: "short_answer", + difficulty: "medium", + confidence: 1, + }), + ]); + + const rag = createInMemoryRagClient({ jsonlPath }); + const results = await rag.search({ + school_level: "middle", + grade: 1, + topic_code: "9수02-03", + k: 3, + }); + + expect(results).toHaveLength(1); + expect(results[0]?.item_id).toBe("111:train:linear-equation"); + }); + + it("rejects records whose mapped code matches but topic text contradicts the requested topic", async () => { + const jsonlPath = await writeFixture([ + canonicalRecord({ + problem_id: "111:train:bad-statistics-mapping", + school_level: "middle", + grade: 3, + topic_name: "분산과 표준편차", + achievement_code: "9수02-08", + achievement_standard: "다항식의 곱셈과 인수분해를 할 수 있다.", + question_text: "5개의 변량의 표준편차를 구하여라.", + answer_text: "2", + problem_type: "objective", + difficulty: "medium", + confidence: 1, + }), + ]); + + const rag = createInMemoryRagClient({ jsonlPath }); + const results = await rag.search({ + school_level: "middle", + grade: 3, + topic_code: "9수02-08", + topic_name: "다항식의 곱셈과 인수분해", + problem_type: "objective", + difficulty: "medium", + k: 3, + }); + + expect(results).toHaveLength(0); + }); + + it("does not match a broader algebra factorization topic for prime factorization", async () => { + const jsonlPath = await writeFixture([ + canonicalRecord({ + problem_id: "111:train:algebra-factorization", + school_level: "middle", + grade: 1, + topic_name: "다항식의 인수분해", + achievement_code: "9수01-01", + achievement_standard: "자연수를 소인수분해 할 수 있다.", + question_text: "x^{2}-5x+6을 인수분해하여라.", + answer_text: "(x-2)(x-3)", + problem_type: "objective", + difficulty: "medium", + confidence: 1, + }), + ]); + + const rag = createInMemoryRagClient({ jsonlPath }); + const results = await rag.search({ + school_level: "middle", + grade: 1, + topic_code: "9수01-01", + topic_name: "소인수분해", + problem_type: "objective", + difficulty: "medium", + k: 3, + }); + + expect(results).toHaveLength(0); + }); }); async function writeFixture(records: unknown[]): Promise { diff --git a/packages/agent/tests/rag-search.test.ts b/packages/agent/tests/rag-search.test.ts new file mode 100644 index 0000000..777276d --- /dev/null +++ b/packages/agent/tests/rag-search.test.ts @@ -0,0 +1,182 @@ +import { describe, expect, it } from "vitest"; + +import { GenerateRequestSchema } from "../src/schemas/index.js"; +import type { RagClient } from "../src/tools/rag-client.js"; +import { ragSearch } from "../src/steps/rag-search.js"; +import type { RagQuery, RagResult } from "../src/schemas/index.js"; + +describe("ragSearch", () => { + it("keeps the requested topic before relaxing difficulty and problem type", async () => { + const calls: RagQuery[] = []; + const exactTopicResult: RagResult = { + ...sourceResult, + item_id: "111:train:variable-expression", + problem: { + ...sourceResult.problem, + item_id: "111:train:variable-expression", + topic_code: "9수02-01", + topic_name: "식의 값", + question_text: "a=3, b=5일 때 2a+b의 값을 구하여라.", + answer_text: "11", + problem_type_norm: "essay", + difficulty_norm: "easy", + }, + }; + const contaminatedResult: RagResult = { + ...sourceResult, + item_id: "111:train:radical-expression", + problem: { + ...sourceResult.problem, + item_id: "111:train:radical-expression", + topic_code: "9수01-01", + topic_name: "인수분해 공식을 이용한 식의 값", + question_text: "x=sqrt(3)+1일 때 x^2의 값을 구하여라.", + }, + }; + const rag: RagClient = { + search: async (query) => { + calls.push(query); + if ( + query.topic_code === "9수02-01" && + query.difficulty === undefined && + query.problem_type === undefined + ) { + return [exactTopicResult]; + } + if (query.topic_code === undefined) { + return [contaminatedResult]; + } + return []; + }, + }; + + const request = GenerateRequestSchema.parse({ + grade: 1, + topic: "9수02-01", + topic_name: "문자의 사용과 식의 값", + mode: "structural", + dims: ["reasoning"], + }); + + const result = await ragSearch({ rag, perStepTimeoutMs: 1000 }, { request }); + + expect(result.refs.map((ref) => ref.item_id)).toEqual([ + "111:train:variable-expression", + ]); + expect(calls.map((call) => call.topic_code)).toEqual([ + "9수02-01", + "9수02-01", + ]); + }); + + it("falls back to topic-name retrieval when an app achievement code has no exact corpus match", async () => { + const calls: RagQuery[] = []; + const rag: RagClient = { + search: async (query) => { + calls.push(query); + if (query.topic_code === undefined && query.topic_name === "자료의 정리와 해석") { + return [sourceResult]; + } + return []; + }, + }; + + const request = GenerateRequestSchema.parse({ + grade: 1, + topic: "9수05-01", + topic_name: "자료의 정리와 해석", + mode: "structural", + dims: ["reasoning"], + }); + + const result = await ragSearch({ rag, perStepTimeoutMs: 1000 }, { request }); + + expect(result.refs).toHaveLength(1); + expect(calls.map((call) => call.topic_code)).toEqual([ + "9수05-01", + "9수05-01", + undefined, + ]); + }); + + it("rejects weak exact-code topic noise before using a topic alias fallback", async () => { + const calls: RagQuery[] = []; + const noisyResult: RagResult = { + ...sourceResult, + item_id: "111:train:blood-type-statistics", + problem: { + ...sourceResult.problem, + item_id: "111:train:blood-type-statistics", + topic_code: "9수04-06", + topic_name: "최빈값 구하기", + achievement_standard: "확률의 개념과 기본 성질을 이해한다.", + question_text: "최빈값과 확률 사이에는 어떤 관계가 있는지 말하여라.", + }, + }; + const tangentResult: RagResult = { + ...sourceResult, + item_id: "111:train:circle-tangent", + problem: { + ...sourceResult.problem, + item_id: "111:train:circle-tangent", + topic_code: "9수03-18", + topic_name: "원의 접선 성질", + achievement_standard: "원의 현과 접선에 관한 성질을 이해한다.", + question_text: "원 O의 접선과 반지름이 이루는 각을 구하여라.", + }, + }; + const rag: RagClient = { + search: async (query) => { + calls.push(query); + if (query.topic_code === "9수04-06" && query.difficulty === undefined) { + return [noisyResult]; + } + if (query.topic_code === undefined && query.topic_name === "원의 접선 성질") { + return [tangentResult]; + } + return []; + }, + }; + + const request = GenerateRequestSchema.parse({ + grade: 3, + topic: "9수04-06", + topic_name: "원과 직선의 위치 관계", + mode: "structural", + dims: ["reasoning"], + }); + + const result = await ragSearch({ rag, perStepTimeoutMs: 1000 }, { request }); + + expect(result.refs.map((ref) => ref.item_id)).toEqual(["111:train:circle-tangent"]); + expect(calls.some((call) => call.topic_name === "원의 접선 성질")).toBe(true); + }); +}); + +const sourceResult: RagResult = { + item_id: "111:train:statistics", + match_reason: "semantic", + similarity: 0.8, + problem: { + item_id: "111:train:statistics", + source_dataset: "111", + split: "train", + source_label_type: "problem_label", + school_level: "middle", + grade: 1, + semester: null, + topic_code: "7105011", + topic_name: "자료의 정리와 해석", + achievement_standard: "자료를 정리하고 해석할 수 있다.", + question_text: "도수분포표를 보고 평균을 구하여라.", + answer_text: "12", + explanation_text: "풀이", + choice_blocks: [], + problem_type_norm: "objective", + difficulty_norm: "medium", + question_image_relpath: null, + answer_image_relpath: null, + question_json_relpath: null, + answer_json_relpath: null, + }, +}; diff --git a/packages/agent/tests/repeating-decimal.test.ts b/packages/agent/tests/repeating-decimal.test.ts new file mode 100644 index 0000000..d7ad837 --- /dev/null +++ b/packages/agent/tests/repeating-decimal.test.ts @@ -0,0 +1,16 @@ +import { describe, expect, it } from "vitest"; + +import { + fractionFromRepeatingDecimalQuestion, + sameFractionText, +} from "../src/tools/repeating-decimal.js"; + +describe("repeating decimal fraction parsing", () => { + it("computes a fraction when a repeating tail follows non-repeating digits", () => { + const fraction = fractionFromRepeatingDecimalQuestion("순환소수 3.12454545...를 분수로 나타내시오."); + + expect(fraction).toEqual({ numerator: "3437", denominator: "1100" }); + expect(fraction === null ? false : sameFractionText("3437/1100", fraction)).toBe(true); + expect(fraction === null ? false : sameFractionText("3437/1110", fraction)).toBe(false); + }); +}); diff --git a/packages/agent/tests/sympy-input.test.ts b/packages/agent/tests/sympy-input.test.ts new file mode 100644 index 0000000..2de1aa2 --- /dev/null +++ b/packages/agent/tests/sympy-input.test.ts @@ -0,0 +1,10 @@ +import { describe, expect, it } from "vitest"; + +import { toSympyInput } from "../src/tools/sympy-input.js"; + +describe("toSympyInput", () => { + it("normalizes bare and latex sqrt notation", () => { + expect(toSympyInput("12 + 8sqrt{3}")).toBe("12 + 8*sqrt(3)"); + expect(toSympyInput("12 + 8\\sqrt{3}")).toBe("12 + 8*sqrt(3)"); + }); +}); diff --git a/packages/agent/tests/sympy-verification.test.ts b/packages/agent/tests/sympy-verification.test.ts new file mode 100644 index 0000000..43c7bfc --- /dev/null +++ b/packages/agent/tests/sympy-verification.test.ts @@ -0,0 +1,82 @@ +import { describe, expect, it } from "vitest"; + +import type { GeneratedProblem } from "../src/schemas/index.js"; +import { verifyWithSympy } from "../src/steps/sympy-verification.js"; +import type { MathEngineClient } from "../src/tools/math-engine-client.js"; + +const expressionCandidate: GeneratedProblem = { + candidate_id: "00000000-0000-0000-0000-000000000009", + mode: "structural", + generation_kind: "expression", + question_text: "a, b를 사용한 식과 대입한 값을 구하여라.", + expected_answer: "4 a + 4 b + 270; 526 kcal", + proposed_solution_trace: "식을 세우고 값을 대입한다.", + source_refs: ["ref-1"], + inferred_intent: { + objective_code: "9수02-01", + objective_description: "문자를 사용하여 식을 나타내고 식의 값을 구한다.", + evaluation_dimensions: [{ id: "A", description: "식의 값", must_preserve: true }], + required_techniques: ["substitution"], + forbidden_techniques: [], + surface_constraints: { difficulty: "medium", problem_type: "objective" }, + }, + generation_metadata: { + model: "test", + temperature: 0, + prompt_id: "test", + prompt_version: "0.0.0", + attempt: 1, + generated_at: "2026-06-06T00:00:00.000Z", + }, +}; + +describe("verifyWithSympy", () => { + it("passes expression answers with multiple parts and units", async () => { + const mathEngine: MathEngineClient = { + health: async () => ({ status: "ok", engine: "sympy" }), + solve: async () => ({ solutions: [] }), + verify: async () => ({ equivalent: true, diff: "0" }), + simplify: async ({ expr }) => { + if (expr.includes("kcal") || expr.includes(";")) { + throw new Error(`unparseable expression: ${expr}`); + } + return { simplified: expr.replace(/\s+/g, "") }; + }, + differentiate: async () => ({ derivative: "" }), + limit: async () => ({ limit: "" }), + }; + + const result = await verifyWithSympy( + { mathEngine }, + { candidate: expressionCandidate }, + ); + + expect(result.gate.status).toBe("passed"); + }); + + it("skips textual objective answer bodies for expression candidates", async () => { + const mathEngine: MathEngineClient = { + health: async () => ({ status: "ok", engine: "sympy" }), + solve: async () => ({ solutions: [] }), + verify: async () => ({ equivalent: true, diff: "0" }), + simplify: async ({ expr }) => { + if (/[가-힣]/u.test(expr)) throw new Error(`textual answer: ${expr}`); + return { simplified: expr }; + }, + differentiate: async () => ({ derivative: "" }), + limit: async () => ({ limit: "" }), + }; + + const result = await verifyWithSympy( + { mathEngine }, + { + candidate: { + ...expressionCandidate, + expected_answer: "② 정수가 아닌 유리수는 2개이다.", + }, + }, + ); + + expect(result.gate.status).toBe("passed"); + }); +}); diff --git a/packages/web/app/app/new/export/view.tsx b/packages/web/app/app/new/export/view.tsx index c332745..38d7195 100644 --- a/packages/web/app/app/new/export/view.tsx +++ b/packages/web/app/app/new/export/view.tsx @@ -66,12 +66,30 @@ function storageKey( "openmath:verification-result", grade, topic.code, + topic.name, mode, [...dims].sort().join(","), sourceProblemText, ].join("|"); } +function resultHref( + grade: Grade, + topic: Topic, + mode: "structural" | "conceptual" | null, + dims: readonly string[], + sourceProblemText: string, +): string { + const params = new URLSearchParams({ + grade: String(grade), + topic: topic.code, + }); + if (mode !== null) params.set("mode", mode); + if (dims.length > 0) params.set("dims", [...dims].join(",")); + if (sourceProblemText.length > 0) params.set("source", sourceProblemText); + return `/app/new/result?${params.toString()}`; +} + function parseStoredProblem(raw: unknown): StoredProblem | null { if (typeof raw !== "object" || raw === null || Array.isArray(raw)) return null; const o = raw as Record; @@ -244,10 +262,11 @@ export function ExportView({ } if (displayProblems.length === 0) { + const backHref = resultHref(grade, topic, mode, dims, sourceProblemText); return ( <>