diff --git a/.gitignore b/.gitignore index 0f8e104..8717a66 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,7 @@ __pycache__/ .env.local *.env !.env.example + +# Large local RAG corpus +packages/agent/data/corpus/*.jsonl +!packages/agent/data/corpus/*fixture*.jsonl diff --git a/README.md b/README.md index 0e75b4b..cd8c4b2 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,7 @@ LLMs produce plausible math problems with wrong solutions ([MathTrap300](https:/ | **Web** | Node 22 + Next.js 14 App Router + Tailwind v4 | Landing + 출제 워크플로우 UI. SSE consumer of `agent` | 검증 흐름은 결정론적 6단계 파이프라인 (RAG → Intent → Generate → SymPy → Re-solve → Objective map). LLM은 생성 단계와 독립 재풀이 단계에만 관여하고, 정답 판정은 결코 하지 않는다 — 자세한 결정 근거는 [`docs/specs/architecture.md`](docs/specs/architecture.md), 도메인 개념은 [`docs/specs/domain.md`](docs/specs/domain.md). 프론트 디자인 시스템은 [`packages/web/DESIGN.md`](packages/web/DESIGN.md) (editorial + productivity 듀얼-서피스). +캡스톤 시연 범위는 [`docs/product/DEMO_SCOPE.md`](docs/product/DEMO_SCOPE.md)에 고정한다. LLM access is pluggable: direct OpenAI/Anthropic, or via [CLIProxyAPI](https://github.com/router-for-me/CLIProxyAPI) for unified Claude/GPT/Gemini routing. diff --git a/docs/product/DEMO_SCOPE.md b/docs/product/DEMO_SCOPE.md new file mode 100644 index 0000000..4d4601a --- /dev/null +++ b/docs/product/DEMO_SCOPE.md @@ -0,0 +1,58 @@ +# Demo Scope + +Status: active for capstone demo +Last updated: 2026-05-31 + +This document records the units OpenMath treats as demo-safe. The goal is not to +claim full curriculum coverage yet, but to make the capstone path predictable: +RAG references exist, strategy YAML exists, and the verification pipeline can +produce a result event. + +## Primary 5-minute demo + +Use this flow for the main presentation: + +1. Grade: middle 1 +2. Unit: `9수02-03` 일차방정식 +3. Mode: `auto` +4. Count: 2-5 +5. Expected behavior: structural and conceptual attempts alternate, each runs + through RAG, intent, generation, SymPy verification, independent re-solve, + objective mapping, and final result streaming. + +Why this unit: + +- The corpus has multiple equation-style references. +- `math-engine /solve` handles the generated linear equations directly. +- The strategy is simple enough to explain in a short judge-facing demo. +- The output is easy to inspect visually on the result screen. + +## Secondary demo-safe units + +These units have strategy YAML in `packages/agent/data/achievement-standards/` +and are suitable for fallback or comparison demos: + +| Code | Unit | Grade | Demo note | +|---|---|---:|---| +| `9수01-05` | 제곱근과 실수 | 3 | Good for concept preservation examples | +| `9수02-07` | 연립일차방정식 | 2 | `/solve` now supports equation systems | +| `9수02-09` | 이차방정식 | 3 | Best for factoring and LaTeX rendering | + +## Broader strategy coverage + +The current strategy set covers 12 units: + +`9수01-01`, `9수01-05`, `9수02-01`, `9수02-03`, `9수02-06`, `9수02-07`, +`9수02-08`, `9수02-09`, `9수02-10`, `9수03-02`, `9수03-04`, `9수04-05`. + +These are enough for UI exploration and workflow tests. Production-quality +coverage still depends on corpus density and LLM output quality for each unit. + +## Known limits + +- Without `LLM_API_KEY` or `LLM_BASE_URL`, the agent uses the local fallback + generator. That verifies the pipeline shape, not final content quality. +- Some units can retrieve broader neighboring references when exact corpus + density is low. +- `mode: auto` is defined as alternating structural and conceptual attempts. + It does not rank the two modes yet. diff --git a/packages/agent/.env.example b/packages/agent/.env.example index 888cdc4..fdd407a 100644 --- a/packages/agent/.env.example +++ b/packages/agent/.env.example @@ -3,13 +3,12 @@ MATH_ENGINE_URL=http://localhost:8000 CORPUS_JSONL=/Users/tw81512/dev/AI_HUB_data/rag_problem_generation_dataset/openmath_rag_records.jsonl # LLM Provider: "openai" or "cliproxy" -LLM_PROVIDER=openai +LLM_PROVIDER=openai-compatible +LLM_BASE_URL=http://localhost:8080/v1 +LLM_API_KEY=dummy-key +LLM_MODEL=gpt-4o -# OpenAI Direct -OPENAI_API_KEY=your-api-key-here -OPENAI_MODEL=gpt-4o - -# CLIProxyAPI (alternative) -CLIPROXY_BASE_URL=http://localhost:8080/v1 -CLIPROXY_API_KEY=dummy-key -CLIPROXY_MODEL=gpt-4o +# OpenAI Direct alternative +# LLM_PROVIDER=openai +# LLM_API_KEY=your-openai-api-key +# LLM_MODEL=gpt-4o diff --git a/packages/agent/data/README.md b/packages/agent/data/README.md index 6a23a48..ec4d254 100644 --- a/packages/agent/data/README.md +++ b/packages/agent/data/README.md @@ -5,23 +5,43 @@ [비할당] 담당이 만지는 영역. agent 코드는 `RagClient` / `StrategyLoader` 인터페이스로만 접근하므로 이 디렉토리 안의 *형식*을 자유롭게 진화시킬 수 있다 (architecture.md D-7). -## 구조 (계획) +## 구조 ``` data/ ├── corpus/ -│ └── math-sample-unified-v1.jsonl # math-sample-unified-v1 스키마 (2,400건) -│ # 출처: docs/PROGRESS.md §2.4 +│ └── openmath_rag_records.jsonl # 로컬/서버에만 배치하는 본 corpus └── achievement-standards/ - ├── 9수04-12.yaml # 이차방정식의 풀이 - ├── 9수04-13.yaml - └── ... # 성취기준별 1:1 파일 (I-T3) + ├── 9수01-01.yaml # 소인수분해 + ├── 9수01-05.yaml # 제곱근과 실수 + ├── 9수02-01.yaml # 문자의 사용과 식의 값 + ├── 9수02-03.yaml # 일차방정식 + ├── 9수02-06.yaml # 일차부등식 + ├── 9수02-07.yaml # 연립일차방정식 + ├── 9수02-08.yaml # 다항식의 곱셈과 인수분해 + ├── 9수02-09.yaml # 이차방정식 + ├── 9수02-10.yaml # 이차방정식의 활용 + ├── 9수03-02.yaml # 일차함수와 그래프 + ├── 9수03-04.yaml # 이차함수와 그래프 + └── 9수04-05.yaml # 삼각비 ``` ## Corpus -`math-sample-unified-v1.jsonl` — `docs/PROGRESS.md` §2.3 정규화 결과. -1차 MVP는 메모리 인덱스 (`createInMemoryRagClient`)로 로드. +런타임 corpus는 `openmath-rag-record-v1` JSONL이다. 기본 로컬 경로: + +```text +/Users/tw81512/dev/AI_HUB_data/rag_problem_generation_dataset/openmath_rag_records.jsonl +``` + +권장 실행 방식은 corpus를 repo에 커밋하지 않고 `.env`에 절대 경로를 지정하는 것이다. + +```env +CORPUS_JSONL=/absolute/path/to/openmath_rag_records.jsonl +``` + +본 corpus는 크기가 커서 `.gitignore`에서 제외한다. 작은 fixture가 필요하면 파일명에 +`fixture`를 포함한 JSONL만 커밋한다. 후속 swap (Postgres / Cube / pgvector)은 인터페이스 안정 후 결정 — Q-2 partial closure. @@ -31,17 +51,50 @@ data/ 파일명은 성취기준 코드와 1:1 (`9수04-12.yaml`). -샘플 → `examples/9수04-12.yaml` 참조 (TODO: 작성). +최소 템플릿: + +```yaml +code: 9수02-09 +title: 이차방정식 +school_level: middle +grade: 3 +techniques: + required_at_least_one_of: + - factoring + forbidden: [] +evaluation_dimensions: + - id: A + description: 이차식을 인수분해한다. + must_preserve: true +difficulty_range: + - easy + - medium +problem_types_supported: + - objective +structural_transforms: + - kind: coefficient_swap + range: [-10, 10] + exclude_zero: true +conceptual_transforms: + - kind: present_via_root_relations +``` + +새 단원 추가 워크플로: + +1. `achievement-standards/.yaml` 파일을 만든다. +2. `code`, `title`, `school_level`, `grade`를 성취기준과 맞춘다. +3. `evaluation_dimensions`에는 `must_preserve: true` 항목을 최소 1개 둔다. +4. `problem_types_supported`와 `difficulty_range`를 시연 범위에 맞춘다. +5. `pnpm -F @openmath/agent test`로 YAML loader와 workflow 회귀를 확인한다. ## 변경 정책 - 새 strategy 추가 시 schema 검증 통과해야 함 -- corpus 갱신은 [한진우]가 정규화 스크립트 (`scripts/normalize_sample_jsons.py` 등) 통과 후 commit -- `pnpm test`는 strategy YAML 파싱이 통과되는지 검증 (TODO: test 작성) +- corpus 갱신은 [한진우]가 정규화 스크립트 통과 후 외부 storage/로컬 경로로 배치 +- `pnpm -F @openmath/agent test`는 strategy YAML 파싱이 통과되는지 검증 ## TODO -- [ ] 핵심 단원 12개에 대한 strategy YAML 작성 -- [ ] corpus JSONL을 이 디렉토리로 복사 (또는 symlink) -- [ ] strategy YAML 검증 테스트 +- [x] 핵심 단원 12개 strategy YAML 작성 +- [ ] 서버 배포 시 `CORPUS_JSONL` 위치 확정 - [ ] Cube/pgvector 도입 결정 (Q-2) diff --git "a/packages/agent/data/achievement-standards/9\354\210\23001-01.yaml" "b/packages/agent/data/achievement-standards/9\354\210\23001-01.yaml" new file mode 100644 index 0000000..182e500 --- /dev/null +++ "b/packages/agent/data/achievement-standards/9\354\210\23001-01.yaml" @@ -0,0 +1,30 @@ +code: 9수01-01 +title: 소인수분해 +school_level: middle +grade: 1 +techniques: + required_at_least_one_of: + - prime_factorization + - divisor_multiple_reasoning + forbidden: + - calculator_only +evaluation_dimensions: + - id: A + description: 자연수를 소수의 곱으로 분해한다. + must_preserve: true + - id: B + description: 소인수분해 결과를 약수와 배수 판단에 활용한다. + must_preserve: true +difficulty_range: + - easy + - medium +problem_types_supported: + - objective + - short_answer +structural_transforms: + - kind: coefficient_swap + range: [2, 120] + exclude_zero: true +conceptual_transforms: + - kind: inverse_question + hint: 소인수분해 결과가 주어졌을 때 원래 수나 약수 개수를 묻는다. diff --git "a/packages/agent/data/achievement-standards/9\354\210\23001-05.yaml" "b/packages/agent/data/achievement-standards/9\354\210\23001-05.yaml" new file mode 100644 index 0000000..2e89fae --- /dev/null +++ "b/packages/agent/data/achievement-standards/9\354\210\23001-05.yaml" @@ -0,0 +1,30 @@ +code: 9수01-05 +title: 제곱근과 실수 +school_level: middle +grade: 3 +techniques: + required_at_least_one_of: + - square_root_definition + - real_number_classification + forbidden: + - decimal_approximation_only +evaluation_dimensions: + - id: A + description: 제곱근의 뜻을 이용해 양수의 두 제곱근을 구분한다. + must_preserve: true + - id: B + description: 유리수와 무리수를 실수 범위에서 분류한다. + must_preserve: true +difficulty_range: + - easy + - medium +problem_types_supported: + - objective + - short_answer +structural_transforms: + - kind: coefficient_swap + range: [2, 100] + exclude_zero: true +conceptual_transforms: + - kind: inverse_question + hint: 제곱근의 값이 주어졌을 때 원래 수를 찾는 형태로 바꾼다. diff --git "a/packages/agent/data/achievement-standards/9\354\210\23002-01.yaml" "b/packages/agent/data/achievement-standards/9\354\210\23002-01.yaml" new file mode 100644 index 0000000..89941b6 --- /dev/null +++ "b/packages/agent/data/achievement-standards/9\354\210\23002-01.yaml" @@ -0,0 +1,38 @@ +code: 9수02-01 +title: 문자의 사용과 식의 값 +school_level: middle +grade: 1 +techniques: + required_at_least_one_of: + - algebraic_expression_modeling + - substitution + forbidden: + - guess_and_check_only +evaluation_dimensions: + - id: A + description: 수량 관계를 문자식으로 나타낸다. + must_preserve: true + - id: B + description: 주어진 문자 값으로 식의 값을 계산한다. + must_preserve: true +difficulty_range: + - easy + - medium +problem_types_supported: + - objective + - short_answer +structural_transforms: + - kind: variable_rename + allowed: + - x + - a + - n + - kind: coefficient_swap + range: [-9, 9] + exclude_zero: true +conceptual_transforms: + - kind: rephrase_as_word_problem + context: + - 물건 가격 + - 도형 둘레 + - 점수 diff --git "a/packages/agent/data/achievement-standards/9\354\210\23002-03.yaml" "b/packages/agent/data/achievement-standards/9\354\210\23002-03.yaml" new file mode 100644 index 0000000..517a83e --- /dev/null +++ "b/packages/agent/data/achievement-standards/9\354\210\23002-03.yaml" @@ -0,0 +1,34 @@ +code: 9수02-03 +title: 일차방정식 +school_level: middle +grade: 1 +techniques: + required_at_least_one_of: + - equation_balance + - linear_equation_solving + forbidden: + - calculator_only +evaluation_dimensions: + - id: A + description: 등식의 성질을 이용해 미지수 항을 분리한다. + must_preserve: true + - id: B + description: 양변에 같은 수를 더하거나 빼서 해를 구한다. + must_preserve: true +difficulty_range: + - easy + - medium +problem_types_supported: + - objective + - short_answer +structural_transforms: + - kind: coefficient_swap + range: [-9, 9] + exclude_zero: true + - kind: sign_flip +conceptual_transforms: + - kind: rephrase_as_word_problem + context: + - 가격 + - 거리 + - 나이 diff --git "a/packages/agent/data/achievement-standards/9\354\210\23002-06.yaml" "b/packages/agent/data/achievement-standards/9\354\210\23002-06.yaml" new file mode 100644 index 0000000..e33bb53 --- /dev/null +++ "b/packages/agent/data/achievement-standards/9\354\210\23002-06.yaml" @@ -0,0 +1,34 @@ +code: 9수02-06 +title: 일차부등식 +school_level: middle +grade: 2 +techniques: + required_at_least_one_of: + - inequality_balance + - interval_solution + forbidden: + - equation_only +evaluation_dimensions: + - id: A + description: 부등식의 성질을 이용해 미지수 항을 정리한다. + must_preserve: true + - id: B + description: 음수로 나누거나 곱할 때 부등호 방향을 바꾼다. + must_preserve: true +difficulty_range: + - easy + - medium +problem_types_supported: + - objective + - short_answer +structural_transforms: + - kind: coefficient_swap + range: [-8, 8] + exclude_zero: true + - kind: sign_flip +conceptual_transforms: + - kind: rephrase_as_word_problem + context: + - 최소 비용 + - 제한 조건 + - 점수 기준 diff --git "a/packages/agent/data/achievement-standards/9\354\210\23002-07.yaml" "b/packages/agent/data/achievement-standards/9\354\210\23002-07.yaml" new file mode 100644 index 0000000..00c2979 --- /dev/null +++ "b/packages/agent/data/achievement-standards/9\354\210\23002-07.yaml" @@ -0,0 +1,37 @@ +code: 9수02-07 +title: 연립일차방정식 +school_level: middle +grade: 2 +techniques: + required_at_least_one_of: + - elimination + - substitution + forbidden: + - graph_only +evaluation_dimensions: + - id: A + description: 두 일차방정식을 동시에 만족하는 미지수 값을 찾는다. + must_preserve: true + - id: B + description: 가감법 또는 대입법으로 한 미지수를 먼저 소거한다. + must_preserve: true +difficulty_range: + - easy + - medium +problem_types_supported: + - objective + - short_answer +structural_transforms: + - kind: coefficient_swap + range: [-6, 6] + exclude_zero: true + - kind: variable_rename + allowed: + - x + - y +conceptual_transforms: + - kind: rephrase_as_word_problem + context: + - 물건의 개수 + - 입장료 + - 두 수 diff --git "a/packages/agent/data/achievement-standards/9\354\210\23002-08.yaml" "b/packages/agent/data/achievement-standards/9\354\210\23002-08.yaml" new file mode 100644 index 0000000..aba5901 --- /dev/null +++ "b/packages/agent/data/achievement-standards/9\354\210\23002-08.yaml" @@ -0,0 +1,31 @@ +code: 9수02-08 +title: 다항식의 곱셈과 인수분해 +school_level: middle +grade: 3 +techniques: + required_at_least_one_of: + - polynomial_expansion + - factoring_formula + forbidden: + - numerical_substitution_only +evaluation_dimensions: + - id: A + description: 곱셈공식을 이용해 다항식을 전개한다. + must_preserve: true + - id: B + description: 공통인수나 공식 구조를 찾아 인수분해한다. + must_preserve: true +difficulty_range: + - easy + - medium +problem_types_supported: + - objective + - short_answer +structural_transforms: + - kind: coefficient_swap + range: [-10, 10] + exclude_zero: true + - kind: sign_flip +conceptual_transforms: + - kind: inverse_question + hint: 전개식이 주어졌을 때 원래 인수 형태를 찾는다. diff --git "a/packages/agent/data/achievement-standards/9\354\210\23002-09.yaml" "b/packages/agent/data/achievement-standards/9\354\210\23002-09.yaml" new file mode 100644 index 0000000..52c99ec --- /dev/null +++ "b/packages/agent/data/achievement-standards/9\354\210\23002-09.yaml" @@ -0,0 +1,38 @@ +code: 9수02-09 +title: 이차방정식 +school_level: middle +grade: 3 +techniques: + required_at_least_one_of: + - factoring + - quadratic_formula + forbidden: + - numerical_approximation_only +evaluation_dimensions: + - id: A + description: 이차식을 인수분해 가능한 형태로 변형한다. + must_preserve: true + - id: B + description: 각 인수를 0으로 두어 해를 구한다. + must_preserve: true + - id: C + description: 해가 원래 방정식을 만족하는지 확인한다. + must_preserve: false +difficulty_range: + - easy + - medium +problem_types_supported: + - objective + - short_answer +structural_transforms: + - kind: coefficient_swap + range: [-10, 10] + exclude_zero: true + - kind: sign_flip +conceptual_transforms: + - kind: present_via_root_relations + hint: 두 근의 합과 곱을 이용하는 문항으로 바꿀 수 있다. + - kind: rephrase_as_word_problem + context: + - 직사각형의 넓이 + - 연속하는 두 정수 diff --git "a/packages/agent/data/achievement-standards/9\354\210\23002-10.yaml" "b/packages/agent/data/achievement-standards/9\354\210\23002-10.yaml" new file mode 100644 index 0000000..ec5dc5c --- /dev/null +++ "b/packages/agent/data/achievement-standards/9\354\210\23002-10.yaml" @@ -0,0 +1,33 @@ +code: 9수02-10 +title: 이차방정식의 활용 +school_level: middle +grade: 3 +techniques: + required_at_least_one_of: + - quadratic_modeling + - factoring + forbidden: + - formula_without_modeling +evaluation_dimensions: + - id: A + description: 상황을 이차방정식으로 모델링한다. + must_preserve: true + - id: B + description: 문제 조건에 맞는 해만 선택한다. + must_preserve: true +difficulty_range: + - medium + - hard +problem_types_supported: + - objective + - short_answer +structural_transforms: + - kind: coefficient_swap + range: [-12, 12] + exclude_zero: true +conceptual_transforms: + - kind: rephrase_as_word_problem + context: + - 넓이 + - 연속하는 정수 + - 이동 거리 diff --git "a/packages/agent/data/achievement-standards/9\354\210\23003-02.yaml" "b/packages/agent/data/achievement-standards/9\354\210\23003-02.yaml" new file mode 100644 index 0000000..d34653a --- /dev/null +++ "b/packages/agent/data/achievement-standards/9\354\210\23003-02.yaml" @@ -0,0 +1,34 @@ +code: 9수03-02 +title: 일차함수와 그래프 +school_level: middle +grade: 2 +techniques: + required_at_least_one_of: + - slope_intercept_form + - graph_interpretation + forbidden: + - table_only +evaluation_dimensions: + - id: A + description: 일차함수의 기울기와 y절편을 해석한다. + must_preserve: true + - id: B + description: 그래프나 두 점으로부터 일차함수식을 구한다. + must_preserve: true +difficulty_range: + - easy + - medium +problem_types_supported: + - objective + - short_answer +structural_transforms: + - kind: coefficient_swap + range: [-8, 8] + exclude_zero: true + - kind: sign_flip +conceptual_transforms: + - kind: rephrase_as_word_problem + context: + - 요금제 + - 속력 + - 온도 변화 diff --git "a/packages/agent/data/achievement-standards/9\354\210\23003-04.yaml" "b/packages/agent/data/achievement-standards/9\354\210\23003-04.yaml" new file mode 100644 index 0000000..f5f79ae --- /dev/null +++ "b/packages/agent/data/achievement-standards/9\354\210\23003-04.yaml" @@ -0,0 +1,31 @@ +code: 9수03-04 +title: 이차함수와 그래프 +school_level: middle +grade: 3 +techniques: + required_at_least_one_of: + - quadratic_graph_vertex + - axis_of_symmetry + forbidden: + - plotting_only +evaluation_dimensions: + - id: A + description: 이차함수 그래프의 꼭짓점과 축을 찾는다. + must_preserve: true + - id: B + description: 식의 계수 변화가 그래프에 미치는 영향을 해석한다. + must_preserve: true +difficulty_range: + - medium + - hard +problem_types_supported: + - objective + - short_answer +structural_transforms: + - kind: coefficient_swap + range: [-6, 6] + exclude_zero: true + - kind: sign_flip +conceptual_transforms: + - kind: present_via_root_relations + hint: x절편 또는 꼭짓점 정보로 식을 복원하는 형태로 바꾼다. diff --git "a/packages/agent/data/achievement-standards/9\354\210\23004-05.yaml" "b/packages/agent/data/achievement-standards/9\354\210\23004-05.yaml" new file mode 100644 index 0000000..c00a21c --- /dev/null +++ "b/packages/agent/data/achievement-standards/9\354\210\23004-05.yaml" @@ -0,0 +1,30 @@ +code: 9수04-05 +title: 삼각비 +school_level: middle +grade: 3 +techniques: + required_at_least_one_of: + - trigonometric_ratio + - right_triangle_reasoning + forbidden: + - calculator_only +evaluation_dimensions: + - id: A + description: 직각삼각형에서 sin, cos, tan의 비를 해석한다. + must_preserve: true + - id: B + description: 특수각의 삼각비 값을 활용한다. + must_preserve: true +difficulty_range: + - easy + - medium +problem_types_supported: + - objective + - short_answer +structural_transforms: + - kind: coefficient_swap + range: [1, 12] + exclude_zero: true +conceptual_transforms: + - kind: inverse_question + hint: 삼각비 값이 주어졌을 때 각이나 변의 비를 찾는다. diff --git a/packages/agent/data/corpus/.gitkeep b/packages/agent/data/corpus/.gitkeep new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/packages/agent/data/corpus/.gitkeep @@ -0,0 +1 @@ + diff --git a/packages/agent/prompts/problem-generator.md b/packages/agent/prompts/problem-generator.md index b3276d4..d1e4944 100644 --- a/packages/agent/prompts/problem-generator.md +++ b/packages/agent/prompts/problem-generator.md @@ -32,7 +32,14 @@ updated: 2026-05-18 # References -{{#each refs}}- 원본: {{this.problem.question_text}} → 정답: {{this.problem.answer_text}} +{{#each refs}}- item_id: {{this.item_id}} + - match: {{this.match_reason}} {{this.similarity}} + - 단원: {{this.problem.topic_name}} + - 성취기준: {{this.problem.achievement_standard}} + - 유형/난이도: {{this.problem.problem_type_norm}} / {{this.problem.difficulty_norm}} + - 원본문제: {{this.problem.question_text}} + - 정답: {{this.problem.answer_text}} + - 해설: {{this.problem.explanation_text}} {{/each}} # Strategy (선택) diff --git a/packages/agent/src/agents/constraint-critic-agent.ts b/packages/agent/src/agents/constraint-critic-agent.ts index 1356120..7d9ceb0 100644 --- a/packages/agent/src/agents/constraint-critic-agent.ts +++ b/packages/agent/src/agents/constraint-critic-agent.ts @@ -1,9 +1,12 @@ /** ConstraintCriticAgent — D-5 GenerationSpecialist team. Checks non-mathematical constraints * (Korean middle-school level, problem-type format, wording clarity). Never judges correctness. */ +import { generateObject } from "ai"; import type { LanguageModel } from "ai"; -import type { GeneratedProblem, Intent, Strategy } from "../schemas/index.js"; +import type { Critique, GeneratedProblem, Intent, Strategy } from "../schemas/index.js"; +import { CritiqueSchema } from "../schemas/index.js"; +import type { PromptLoader } from "../tools/prompt-loader.js"; export interface CritiqueInput { candidate: GeneratedProblem; @@ -11,22 +14,66 @@ export interface CritiqueInput { strategy: Strategy | null; } -export interface Critique { - passes: boolean; - hints: string[]; -} - export interface ConstraintCriticAgent { critique(input: CritiqueInput): Promise; } export interface ConstraintCriticAgentDeps { model: LanguageModel; - promptId: string; + prompts: PromptLoader; + promptId?: string; } export function createConstraintCriticAgent( - _deps: ConstraintCriticAgentDeps, + deps: ConstraintCriticAgentDeps, ): ConstraintCriticAgent { - throw new Error("createConstraintCriticAgent: not implemented yet"); + const promptId = deps.promptId ?? "constraint-critic"; + + return { + async critique(input) { + const localHints = localCritique(input); + if (localHints.length > 0) { + return { passes: false, hints: localHints }; + } + + const prompt = await deps.prompts.load(promptId); + const result = await generateObject({ + model: deps.model, + schema: CritiqueSchema, + temperature: prompt.metadata.temperature, + maxTokens: prompt.metadata.max_tokens, + prompt: prompt.render({ ...input }), + }); + + return result.object; + }, + }; +} + +function localCritique(input: CritiqueInput): string[] { + const hints: string[] = []; + const text = [ + input.candidate.question_text, + input.candidate.expected_answer, + input.candidate.proposed_solution_trace, + ].join("\n"); + + if (count(text, "$") % 2 !== 0) { + hints.push("LaTeX inline math delimiter '$' count is odd."); + } + if (count(text, "{") !== count(text, "}")) { + hints.push("LaTeX brace count is unbalanced."); + } + if ( + input.intent.surface_constraints.problem_type === "objective" && + (input.candidate.expected_choices?.length ?? 0) < 2 + ) { + hints.push("Objective problems need at least two answer choices."); + } + + return hints; +} + +function count(value: string, needle: string): number { + return value.split(needle).length - 1; } diff --git a/packages/agent/src/agents/generator-agent.ts b/packages/agent/src/agents/generator-agent.ts index 8fca6ee..28ef14d 100644 --- a/packages/agent/src/agents/generator-agent.ts +++ b/packages/agent/src/agents/generator-agent.ts @@ -1,5 +1,8 @@ /** GeneratorAgent — D-5 GenerationSpecialist team. Produces GeneratedProblem candidate. */ +import { randomUUID } from "node:crypto"; + +import { generateObject } from "ai"; import type { LanguageModel } from "ai"; import type { @@ -9,6 +12,8 @@ import type { RagResult, Strategy, } from "../schemas/index.js"; +import { GeneratedProblemSchema } from "../schemas/index.js"; +import type { PromptLoader } from "../tools/prompt-loader.js"; export interface GeneratorAgentInput { request: GenerateRequest; @@ -25,9 +30,45 @@ export interface GeneratorAgent { export interface GeneratorAgentDeps { model: LanguageModel; - promptId: string; + prompts: PromptLoader; + promptId?: string; } -export function createGeneratorAgent(_deps: GeneratorAgentDeps): GeneratorAgent { - throw new Error("createGeneratorAgent: not implemented yet"); +export function createGeneratorAgent(deps: GeneratorAgentDeps): GeneratorAgent { + const promptId = deps.promptId ?? "problem-generator"; + + return { + async generate(input) { + const prompt = await deps.prompts.load(promptId); + const result = await generateObject({ + model: deps.model, + schema: GeneratedProblemSchema, + temperature: prompt.metadata.temperature, + maxTokens: prompt.metadata.max_tokens, + prompt: prompt.render({ ...input }), + }); + + return GeneratedProblemSchema.parse({ + ...result.object, + candidate_id: result.object.candidate_id || randomUUID(), + mode: result.object.mode === "conceptual" ? "conceptual" : "structural", + source_refs: result.object.source_refs.length + ? result.object.source_refs + : input.refs.map((ref) => ref.item_id), + inferred_intent: { + ...result.object.inferred_intent, + objective_code: input.intent.objective_code, + }, + generation_metadata: { + ...result.object.generation_metadata, + model: prompt.metadata.model, + temperature: prompt.metadata.temperature, + prompt_id: prompt.metadata.id, + prompt_version: prompt.metadata.version, + attempt: input.attempt, + generated_at: new Date().toISOString(), + }, + }); + }, + }; } diff --git a/packages/agent/src/agents/refiner-agent.ts b/packages/agent/src/agents/refiner-agent.ts index 6960163..4b5121d 100644 --- a/packages/agent/src/agents/refiner-agent.ts +++ b/packages/agent/src/agents/refiner-agent.ts @@ -1,8 +1,11 @@ /** RefinerAgent — D-5 GenerationSpecialist team. Re-runs GeneratorAgent with critique hints. */ +import { generateObject } from "ai"; import type { LanguageModel } from "ai"; import type { GeneratedProblem, Intent } from "../schemas/index.js"; +import { GeneratedProblemSchema } from "../schemas/index.js"; +import type { PromptLoader } from "../tools/prompt-loader.js"; export interface RefineInput { prior: GeneratedProblem; @@ -16,9 +19,44 @@ export interface RefinerAgent { export interface RefinerAgentDeps { model: LanguageModel; - promptId: string; + prompts: PromptLoader; + promptId?: string; } -export function createRefinerAgent(_deps: RefinerAgentDeps): RefinerAgent { - throw new Error("createRefinerAgent: not implemented yet"); +export function createRefinerAgent(deps: RefinerAgentDeps): RefinerAgent { + const promptId = deps.promptId ?? "refiner"; + + return { + async refine(input) { + const prompt = await deps.prompts.load(promptId); + const result = await generateObject({ + model: deps.model, + schema: GeneratedProblemSchema, + temperature: prompt.metadata.temperature, + maxTokens: prompt.metadata.max_tokens, + prompt: prompt.render({ ...input }), + }); + + return GeneratedProblemSchema.parse({ + ...result.object, + candidate_id: input.prior.candidate_id, + mode: input.prior.mode, + source_refs: input.prior.source_refs, + inferred_intent: input.intent, + generation_metadata: { + ...input.prior.generation_metadata, + ...result.object.generation_metadata, + prompt_id: prompt.metadata.id, + prompt_version: prompt.metadata.version, + temperature: prompt.metadata.temperature, + attempt: input.prior.generation_metadata.attempt + 1, + generated_at: new Date().toISOString(), + refined_by: [ + ...(input.prior.generation_metadata.refined_by ?? []), + "refiner", + ], + }, + }); + }, + }; } diff --git a/packages/agent/src/agents/solver-agent.ts b/packages/agent/src/agents/solver-agent.ts index 321c580..5688cf2 100644 --- a/packages/agent/src/agents/solver-agent.ts +++ b/packages/agent/src/agents/solver-agent.ts @@ -1,15 +1,12 @@ /** SolverAgent — D-5 ReSolveSpecialist. Independent re-solver with different prompt/model. * Produces a solution trace; never the final judge (D-1). */ +import { generateObject } from "ai"; import type { LanguageModel } from "ai"; -import type { GeneratedProblem } from "../schemas/index.js"; - -export interface SolveAttempt { - derived_answer: string; - trace: string; - confidence: "high" | "medium" | "low"; -} +import type { GeneratedProblem, SolveAttempt } from "../schemas/index.js"; +import { SolveAttemptSchema } from "../schemas/index.js"; +import type { PromptLoader } from "../tools/prompt-loader.js"; export interface SolverAgent { solve(candidate: GeneratedProblem): Promise; @@ -17,9 +14,25 @@ export interface SolverAgent { export interface SolverAgentDeps { model: LanguageModel; - promptId: string; + prompts: PromptLoader; + promptId?: string; } -export function createSolverAgent(_deps: SolverAgentDeps): SolverAgent { - throw new Error("createSolverAgent: not implemented yet"); +export function createSolverAgent(deps: SolverAgentDeps): SolverAgent { + const promptId = deps.promptId ?? "independent-solver"; + + return { + async solve(candidate) { + const prompt = await deps.prompts.load(promptId); + const result = await generateObject({ + model: deps.model, + schema: SolveAttemptSchema, + temperature: prompt.metadata.temperature, + maxTokens: prompt.metadata.max_tokens, + prompt: prompt.render({ candidate }), + }); + + return result.object; + }, + }; } diff --git a/packages/agent/src/config/env.ts b/packages/agent/src/config/env.ts index 7fb6fb2..6ead9d8 100644 --- a/packages/agent/src/config/env.ts +++ b/packages/agent/src/config/env.ts @@ -1,5 +1,8 @@ /** Environment variable validation. */ +import { existsSync, readFileSync } from "node:fs"; +import { resolve } from "node:path"; + import { z } from "zod"; export const EnvSchema = z.object({ @@ -13,6 +16,7 @@ export const EnvSchema = z.object({ .default("openai-compatible"), LLM_BASE_URL: z.string().url().optional(), LLM_API_KEY: z.string().min(1).optional(), + LLM_MODEL: z.string().default("gpt-4o"), PROMPTS_DIR: z.string().default("./prompts"), STRATEGIES_DIR: z.string().default("./data/achievement-standards"), @@ -25,6 +29,7 @@ export const EnvSchema = z.object({ export type Env = z.infer; export function loadEnv(): Env { + loadDotenv(); const parsed = EnvSchema.safeParse(process.env); if (!parsed.success) { throw new Error( @@ -35,3 +40,23 @@ export function loadEnv(): Env { } return parsed.data; } + +function loadDotenv(): void { + for (const path of [resolve(process.cwd(), ".env"), resolve(process.cwd(), "packages/agent/.env")]) { + if (!existsSync(path)) { + continue; + } + const content = readFileSync(path, "utf8"); + for (const line of content.split(/\r?\n/)) { + const trimmed = line.trim(); + if (!trimmed || trimmed.startsWith("#") || !trimmed.includes("=")) { + continue; + } + const [key, ...rest] = trimmed.split("="); + if (key === undefined || process.env[key] !== undefined) { + continue; + } + process.env[key] = rest.join("=").replace(/^['"]|['"]$/g, ""); + } + } +} diff --git a/packages/agent/src/index.ts b/packages/agent/src/index.ts index 0c9664f..c34cb89 100644 --- a/packages/agent/src/index.ts +++ b/packages/agent/src/index.ts @@ -1,14 +1,68 @@ -/** Service entrypoint. Wiring of deps -> createApp is the only un-implemented part of scaffolding. */ +/** Service entrypoint. Wires runtime deps and starts the Hono server. */ +import { access } from "node:fs/promises"; + +import { serve } from "@hono/node-server"; + +import { createApp } from "./server/app.js"; import { loadEnv } from "./config/index.js"; +import { + createConstraintCriticAgent, + createGeneratorAgent, + createRefinerAgent, + createSolverAgent, +} from "./agents/index.js"; +import { createMathEngineClient } from "./tools/math-engine-client.js"; +import { resolveLanguageModel } from "./tools/llm-provider.js"; +import { createFsPromptLoader } from "./tools/prompt-loader.js"; +import { createInMemoryRagClient } from "./tools/rag-client.js"; +import { createFsStrategyLoader } from "./tools/schema-loader.js"; export async function main(): Promise { const env = loadEnv(); - console.log( - `[openmath/agent] Boot pending. Port ${env.PORT}, math-engine ${env.MATH_ENGINE_URL}. ` + - `See docs/specs/architecture.md D-3~D-8 + src/{tools,agents,steps,workflows}/*.`, - ); - throw new Error("main: dependency wiring not implemented yet."); + + if (!env.CORPUS_JSONL) { + throw new Error( + "CORPUS_JSONL is required to start the agent with RAG search. " + + "Set it to the openmath_rag_records.jsonl file path.", + ); + } + + await access(env.CORPUS_JSONL); + + const rag = createInMemoryRagClient({ jsonlPath: env.CORPUS_JSONL }); + await rag.warmup?.(); + + const mathEngine = createMathEngineClient({ + baseUrl: env.MATH_ENGINE_URL, + timeoutMs: env.PER_STEP_TIMEOUT_MS, + }); + const prompts = createFsPromptLoader({ promptsDir: env.PROMPTS_DIR }); + const strategies = createFsStrategyLoader({ strategiesDir: env.STRATEGIES_DIR }); + const model = maybeResolveModel(env); + const agentDeps = + model === null + ? {} + : { + intentModel: model, + generator: createGeneratorAgent({ model, prompts }), + critic: createConstraintCriticAgent({ model, prompts }), + refiner: createRefinerAgent({ model, prompts }), + solver: createSolverAgent({ model, prompts }), + objectiveLlm: model, + }; + + const app = createApp({ + mathEngine, + workflow: { rag, mathEngine, prompts, strategies, ...agentDeps }, + }); + + serve({ fetch: app.fetch, port: env.PORT }, (info) => { + console.log( + `[openmath/agent] Listening on http://localhost:${info.port}; ` + + `math-engine ${env.MATH_ENGINE_URL}; corpus ${env.CORPUS_JSONL}`, + ); + }); } if (import.meta.url === `file://${process.argv[1]}`) { @@ -20,3 +74,20 @@ if (import.meta.url === `file://${process.argv[1]}`) { export type { Env } from "./config/index.js"; export * from "./schemas/index.js"; + +function maybeResolveModel(env: ReturnType) { + if (env.LLM_PROVIDER === "openai" && !env.LLM_API_KEY && !process.env.OPENAI_API_KEY) { + console.warn("[openmath/agent] LLM_API_KEY missing; using fallback generator."); + return null; + } + if (env.LLM_PROVIDER !== "openai" && env.LLM_BASE_URL === undefined) { + console.warn("[openmath/agent] LLM_BASE_URL missing; using fallback generator."); + return null; + } + return resolveLanguageModel({ + kind: env.LLM_PROVIDER, + modelId: env.LLM_MODEL, + baseUrl: env.LLM_BASE_URL, + apiKey: env.LLM_API_KEY, + }); +} diff --git a/packages/agent/src/schemas/critique.schema.ts b/packages/agent/src/schemas/critique.schema.ts new file mode 100644 index 0000000..6f2a96a --- /dev/null +++ b/packages/agent/src/schemas/critique.schema.ts @@ -0,0 +1,8 @@ +import { z } from "zod"; + +export const CritiqueSchema = z.object({ + passes: z.boolean(), + hints: z.array(z.string()), +}); + +export type Critique = z.infer; diff --git a/packages/agent/src/schemas/generate-request.schema.ts b/packages/agent/src/schemas/generate-request.schema.ts index 7c4d636..b303365 100644 --- a/packages/agent/src/schemas/generate-request.schema.ts +++ b/packages/agent/src/schemas/generate-request.schema.ts @@ -16,6 +16,7 @@ import { SchoolLevelSchema, } from "./source-problem.schema.js"; +// auto: workflow alternates structural/conceptual attempts so the UI can compare both. export const GenerateModeSchema = z.enum(["structural", "conceptual", "auto"]); export type GenerateMode = z.infer; @@ -25,6 +26,7 @@ export const GenerateRequestSchema = z.object({ school_level: SchoolLevelSchema.default("middle"), grade: z.union([z.literal(1), z.literal(2), z.literal(3)]), + topic: z.string().optional(), topic_code: z.string().optional(), topic_name: z.string().optional(), diff --git a/packages/agent/src/schemas/index.ts b/packages/agent/src/schemas/index.ts index 9a4c503..842b284 100644 --- a/packages/agent/src/schemas/index.ts +++ b/packages/agent/src/schemas/index.ts @@ -6,9 +6,11 @@ export * from "./generate-request.schema.js"; export * from "./generated-problem.schema.js"; +export * from "./critique.schema.js"; export * from "./intent.schema.js"; export * from "./progress-event.schema.js"; export * from "./rag.schema.js"; +export * from "./solve-attempt.schema.js"; export * from "./source-problem.schema.js"; export * from "./strategy.schema.js"; export * from "./verification.schema.js"; diff --git a/packages/agent/src/schemas/solve-attempt.schema.ts b/packages/agent/src/schemas/solve-attempt.schema.ts new file mode 100644 index 0000000..b614a20 --- /dev/null +++ b/packages/agent/src/schemas/solve-attempt.schema.ts @@ -0,0 +1,9 @@ +import { z } from "zod"; + +export const SolveAttemptSchema = z.object({ + derived_answer: z.string().min(1), + trace: z.string().min(1), + confidence: z.enum(["high", "medium", "low"]), +}); + +export type SolveAttempt = z.infer; diff --git a/packages/agent/src/server/routes/generate.ts b/packages/agent/src/server/routes/generate.ts index eed9989..5c7c954 100644 --- a/packages/agent/src/server/routes/generate.ts +++ b/packages/agent/src/server/routes/generate.ts @@ -3,18 +3,25 @@ import { zValidator } from "@hono/zod-validator"; import { Hono } from "hono"; +import { streamSSE } from "hono/streaming"; import { GenerateRequestSchema } from "../../schemas/index.js"; import type { VerificationWorkflowDeps } from "../../workflows/verification-workflow.js"; +import { runVerificationWorkflow } from "../../workflows/verification-workflow.js"; +import { pipeProgressToSse } from "../sse/progress-stream.js"; -export function createGenerateRoute(_deps: VerificationWorkflowDeps): Hono { +export function createGenerateRoute(deps: VerificationWorkflowDeps): Hono { const app = new Hono(); app.post( "/api/generate", zValidator("json", GenerateRequestSchema), - () => { - throw new Error("POST /api/generate: not implemented yet"); + (c) => { + const request = c.req.valid("json"); + const events = runVerificationWorkflow(deps, request); + return streamSSE(c, async (stream) => { + await pipeProgressToSse(stream, events); + }); }, ); diff --git a/packages/agent/src/server/sse/progress-stream.ts b/packages/agent/src/server/sse/progress-stream.ts index 2c4b0f2..7595eca 100644 --- a/packages/agent/src/server/sse/progress-stream.ts +++ b/packages/agent/src/server/sse/progress-stream.ts @@ -2,11 +2,88 @@ import type { SSEStreamingApi } from "hono/streaming"; -import type { ProgressEvent } from "../../schemas/index.js"; +import type { ProgressEvent, StepName } from "../../schemas/index.js"; export async function pipeProgressToSse( - _stream: SSEStreamingApi, - _events: AsyncGenerator, + stream: SSEStreamingApi, + events: AsyncGenerator, ): Promise { - throw new Error("pipeProgressToSse: not implemented yet"); + for await (const event of events) { + await stream.writeSSE(toWireEvent(event)); + } +} + +function toWireEvent(event: ProgressEvent): { + event: string; + data: string; +} { + switch (event.type) { + case "step": + return { + event: "step", + data: JSON.stringify({ + index: stepIndex(event.step), + name: event.step, + status: event.status === "start" ? "started" : "completed", + summary: stepSummary(event), + raw: event, + }), + }; + case "result": + return { + event: "result", + data: JSON.stringify(event.candidates), + }; + case "error": + return { + event: "error", + data: JSON.stringify({ + stage: event.stage, + message: event.message, + code: event.code, + recoverable: event.recoverable, + }), + }; + case "retry": + return { + event: "retry", + data: JSON.stringify(event), + }; + } +} + +function stepIndex(step: StepName): number { + switch (step) { + case "rag": + return 1; + case "intent": + return 2; + case "generate": + return 3; + case "sympy_verify": + return 4; + case "re_solve": + return 5; + case "objective_map": + return 6; + } +} + +function stepSummary(event: ProgressEvent & { type: "step" }): string | null { + if (event.status === "start") { + return null; + } + + if (event.step === "rag") { + const data = event.data; + if (isRecord(data) && typeof data.count === "number") { + return `${data.count}개 참조 발견`; + } + } + + return "완료"; +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); } diff --git a/packages/agent/src/steps/independent-resolve.ts b/packages/agent/src/steps/independent-resolve.ts index a7ff779..e685acf 100644 --- a/packages/agent/src/steps/independent-resolve.ts +++ b/packages/agent/src/steps/independent-resolve.ts @@ -4,6 +4,7 @@ import type { SolverAgent } from "../agents/index.js"; import type { GateResult, GeneratedProblem } from "../schemas/index.js"; import type { MathEngineClient } from "../tools/math-engine-client.js"; +import { normalizeMathText } from "./sympy-verification.js"; export interface IndependentResolveDeps { solver: SolverAgent; @@ -20,8 +21,65 @@ export interface IndependentResolveOutput { } export async function independentResolve( - _deps: IndependentResolveDeps, - _input: IndependentResolveInput, + deps: IndependentResolveDeps, + input: IndependentResolveInput, ): Promise { - throw new Error("independentResolve: not implemented yet"); + const started = Date.now(); + if (input.sympyGate.status === "skipped") { + return { + gate: { + step: "re_solve", + status: "skipped", + duration_ms: Date.now() - started, + evidence: { reason: "sympy_verify skipped" }, + }, + }; + } + if (input.sympyGate.status === "failed") { + return { + gate: { + step: "re_solve", + status: "skipped", + duration_ms: Date.now() - started, + evidence: { reason: "sympy_verify failed" }, + }, + }; + } + + try { + const attempt = await deps.solver.solve(input.candidate); + const check = await deps.mathEngine.verify({ + expr1: normalizeMathText(attempt.derived_answer), + expr2: normalizeMathText(input.candidate.expected_answer), + }); + + return { + gate: { + step: "re_solve", + status: check.equivalent ? "passed" : "failed", + duration_ms: Date.now() - started, + evidence: { attempt, diff: check.diff }, + ...(check.equivalent + ? {} + : { + failure_detail: { + code: "independent_resolve_mismatch", + message: "Independent solver answer differs from candidate answer.", + }, + }), + }, + }; + } catch (error) { + return { + gate: { + step: "re_solve", + status: "failed", + duration_ms: Date.now() - started, + failure_detail: { + code: "independent_solver_error", + message: error instanceof Error ? error.message : String(error), + }, + }, + }; + } } diff --git a/packages/agent/src/steps/intent-extraction.ts b/packages/agent/src/steps/intent-extraction.ts index 8f964fe..ac500f3 100644 --- a/packages/agent/src/steps/intent-extraction.ts +++ b/packages/agent/src/steps/intent-extraction.ts @@ -1,5 +1,6 @@ /** Step 2: Intent extraction. Single LLM agent (D-5). Output validated by IntentSchema (I-I1, I-I2). */ +import { generateObject } from "ai"; import type { LanguageModel } from "ai"; import type { @@ -8,11 +9,12 @@ import type { RagResult, Strategy, } from "../schemas/index.js"; +import { IntentSchema, assertIntentInvariants } from "../schemas/index.js"; import type { PromptLoader } from "../tools/prompt-loader.js"; export interface IntentExtractionDeps { - model: LanguageModel; - prompts: PromptLoader; + model?: LanguageModel; + prompts?: PromptLoader; } export interface IntentExtractionInput { @@ -26,8 +28,91 @@ export interface IntentExtractionOutput { } export async function extractIntent( - _deps: IntentExtractionDeps, - _input: IntentExtractionInput, + deps: IntentExtractionDeps, + input: IntentExtractionInput, ): Promise { - throw new Error("extractIntent: not implemented yet"); + if (deps.model !== undefined && deps.prompts !== undefined) { + const prompt = await deps.prompts.load("intent-extraction"); + const result = await generateObject({ + model: deps.model, + schema: IntentSchema, + temperature: prompt.metadata.temperature, + maxTokens: prompt.metadata.max_tokens, + prompt: prompt.render({ ...input }), + }); + assertIntentInvariants(result.object); + return { intent: result.object }; + } + + return fallbackExtractIntent(input); +} + +function fallbackExtractIntent(input: IntentExtractionInput): IntentExtractionOutput { + const primaryRef = input.refs[0]; + const primaryProblem = primaryRef?.problem; + const objectiveCode = firstValidObjectiveCode([ + input.request.topic_code, + input.request.topic, + primaryProblem?.topic_code, + input.strategy?.code, + ]); + const objectiveDescription = + primaryProblem?.achievement_standard ?? + input.strategy?.title ?? + input.request.topic_name ?? + input.request.topic ?? + "선택한 단원의 핵심 개념을 평가한다."; + const dimensionDescription = + primaryProblem === undefined + ? objectiveDescription + : [ + primaryProblem.topic_name, + primaryProblem.question_text, + primaryProblem.answer_text, + ] + .filter(Boolean) + .join(" / "); + + const intent: Intent = { + objective_code: objectiveCode, + objective_description: objectiveDescription, + evaluation_dimensions: + input.strategy?.evaluation_dimensions.length + ? input.strategy.evaluation_dimensions + : [ + { + id: "A", + description: dimensionDescription, + must_preserve: true, + }, + ], + required_techniques: + input.strategy?.techniques.required_at_least_one_of.length + ? input.strategy.techniques.required_at_least_one_of + : [primaryProblem?.topic_name ?? input.request.topic_name ?? "core_concept"], + forbidden_techniques: input.strategy?.techniques.forbidden ?? [], + surface_constraints: { + difficulty: input.request.difficulty, + problem_type: input.request.problem_type, + expected_choice_count: + input.request.problem_type === "objective" ? 5 : undefined, + }, + }; + + assertIntentInvariants(intent); + return { intent }; +} + +function firstValidObjectiveCode(candidates: Array): string { + for (const candidate of candidates) { + if (candidate !== undefined && candidate !== null && isObjectiveCode(candidate)) { + return candidate; + } + } + + return "9수00-00"; +} + +function isObjectiveCode(value: string): boolean { + return /^(9수|10공수)\d{2}-\d{2}$/.test(value); } diff --git a/packages/agent/src/steps/objective-mapping.ts b/packages/agent/src/steps/objective-mapping.ts index 2eb760c..43d24be 100644 --- a/packages/agent/src/steps/objective-mapping.ts +++ b/packages/agent/src/steps/objective-mapping.ts @@ -26,7 +26,42 @@ export interface ObjectiveMappingOutput { export async function mapObjective( _deps: ObjectiveMappingDeps, - _input: ObjectiveMappingInput, + input: ObjectiveMappingInput, ): Promise { - throw new Error("mapObjective: not implemented yet"); + const started = Date.now(); + const expectedCode = input.intent.objective_code; + const actualCode = input.candidate.inferred_intent.objective_code; + const preserved = input.intent.evaluation_dimensions + .filter((dimension) => dimension.must_preserve) + .map((dimension) => dimension.id); + const candidateDimensions = new Set( + input.candidate.inferred_intent.evaluation_dimensions + .filter((dimension) => dimension.must_preserve) + .map((dimension) => dimension.id), + ); + const missing = preserved.filter((id) => !candidateDimensions.has(id)); + const passed = expectedCode === actualCode && missing.length === 0; + + return { + gate: { + step: "objective_map", + status: passed ? "passed" : "failed", + duration_ms: Date.now() - started, + evidence: { + expected_code: expectedCode, + actual_code: actualCode, + preserved_dimensions: preserved, + missing_dimensions: missing, + strategy_code: input.strategy?.code ?? null, + }, + ...(passed + ? {} + : { + failure_detail: { + code: "objective_mismatch", + message: "Generated problem drifted from the requested objective.", + }, + }), + }, + }; } diff --git a/packages/agent/src/steps/problem-generation.ts b/packages/agent/src/steps/problem-generation.ts index ee3c281..ec33094 100644 --- a/packages/agent/src/steps/problem-generation.ts +++ b/packages/agent/src/steps/problem-generation.ts @@ -35,8 +35,57 @@ export interface ProblemGenerationOutput { } export async function generateProblem( - _deps: ProblemGenerationDeps, - _input: ProblemGenerationInput, + deps: ProblemGenerationDeps, + input: ProblemGenerationInput, ): Promise { - throw new Error("generateProblem: not implemented yet"); + let candidate = withSourceRefs( + await deps.generator.generate({ + request: input.request, + intent: input.intent, + refs: input.refs, + strategy: input.strategy, + attempt: input.attempt, + }), + input.refs, + ); + const refinedBy: string[] = []; + + for (let round = 0; round < (deps.maxCriticRounds ?? 1); round += 1) { + const critique = await deps.critic.critique({ + candidate, + intent: input.intent, + strategy: input.strategy, + }); + + if (critique.passes) { + return { candidate, refined_by: refinedBy }; + } + + candidate = withSourceRefs( + await deps.refiner.refine({ + prior: candidate, + intent: input.intent, + hints: critique.hints, + }), + input.refs, + ); + refinedBy.push("refiner"); + } + + return { candidate, refined_by: refinedBy }; +} + +function withSourceRefs( + candidate: GeneratedProblem, + refs: RagResult[], +): GeneratedProblem { + const sourceRefs = new Set([ + ...candidate.source_refs, + ...refs.map((ref) => ref.item_id), + ]); + + return { + ...candidate, + source_refs: [...sourceRefs], + }; } diff --git a/packages/agent/src/steps/rag-search.ts b/packages/agent/src/steps/rag-search.ts index a09ef25..6f934f3 100644 --- a/packages/agent/src/steps/rag-search.ts +++ b/packages/agent/src/steps/rag-search.ts @@ -1,6 +1,6 @@ /** Step 1: RAG retrieval. Deterministic (D-5). Uses RagClient (D-7). */ -import type { GenerateRequest, RagResult } from "../schemas/index.js"; +import type { GenerateRequest, RagQuery, RagResult } from "../schemas/index.js"; import type { RagClient } from "../tools/rag-client.js"; export interface RagSearchDeps { @@ -13,11 +13,83 @@ export interface RagSearchInput { export interface RagSearchOutput { refs: RagResult[]; + fallback_used: boolean; } export async function ragSearch( - _deps: RagSearchDeps, - _input: RagSearchInput, + deps: RagSearchDeps, + input: RagSearchInput, ): Promise { - throw new Error("ragSearch: not implemented yet"); + const query = toRagQuery(input.request); + const refs = await searchWithFallback(deps.rag, query); + return { + refs: refs.results, + fallback_used: refs.fallback_used, + }; +} + +async function searchWithFallback( + rag: RagClient, + query: RagQuery, +): Promise<{ results: RagResult[]; fallback_used: boolean }> { + const primary = await rag.search(query); + if (primary.length > 0) { + return { results: primary, fallback_used: false }; + } + + for (const fallbackQuery of fallbackQueries(query)) { + const results = await rag.search(fallbackQuery); + if (results.length > 0) { + return { results, fallback_used: true }; + } + } + + return { results: [], fallback_used: false }; +} + +function fallbackQueries(query: RagQuery): RagQuery[] { + return [ + { + ...query, + difficulty: undefined, + problem_type: undefined, + }, + { + ...query, + topic_code: undefined, + difficulty: undefined, + problem_type: undefined, + }, + { + ...query, + grade: null, + topic_code: undefined, + difficulty: undefined, + problem_type: undefined, + }, + ]; +} + +function toRagQuery(request: GenerateRequest): RagQuery { + const topicFromAlias = request.topic?.trim(); + const topicCode = request.topic_code?.trim(); + const topicName = request.topic_name?.trim() ?? topicFromAlias; + const usableTopicCode = + topicCode !== undefined && looksLikeAchievementCode(topicCode) + ? topicCode + : undefined; + + return { + school_level: request.school_level, + grade: request.grade, + ...(usableTopicCode ? { topic_code: usableTopicCode } : {}), + ...(topicName ? { topic_name: topicName } : {}), + difficulty: request.difficulty, + problem_type: request.problem_type, + k: Math.min(Math.max(request.count * 3, 8), 20), + }; +} + +function looksLikeAchievementCode(value: string): boolean { + return /^\d+[\p{Script=Hangul}A-Za-z]*\d{2}-\d{2}$/u.test(value); } diff --git a/packages/agent/src/steps/sympy-verification.ts b/packages/agent/src/steps/sympy-verification.ts index e7e2248..5e21cb4 100644 --- a/packages/agent/src/steps/sympy-verification.ts +++ b/packages/agent/src/steps/sympy-verification.ts @@ -16,8 +16,87 @@ export interface SympyVerificationOutput { } export async function verifyWithSympy( - _deps: SympyVerificationDeps, - _input: SympyVerificationInput, + deps: SympyVerificationDeps, + input: SympyVerificationInput, ): Promise { - throw new Error("verifyWithSympy: not implemented yet"); + const started = Date.now(); + const equation = extractEquation(input.candidate.question_text); + if (equation === null) { + return { + gate: { + step: "sympy_verify", + status: "failed", + duration_ms: Date.now() - started, + failure_detail: { + code: "equation_not_found", + message: "Generated problem does not contain a verifiable equation.", + }, + }, + }; + } + + try { + const solved = await deps.mathEngine.solve({ equation }); + const expected = normalizeMathText(input.candidate.expected_answer); + const equivalent = await Promise.all( + solved.solutions.map((solution) => + deps.mathEngine.verify({ expr1: solution, expr2: expected }), + ), + ); + const passed = equivalent.some((result) => result.equivalent); + + return { + gate: { + step: "sympy_verify", + status: passed ? "passed" : "failed", + duration_ms: Date.now() - started, + evidence: { + equation, + solutions: solved.solutions, + expected_answer: expected, + }, + ...(passed + ? {} + : { + failure_detail: { + code: "answer_mismatch", + message: "Expected answer does not match math-engine solution.", + }, + }), + }, + }; + } catch (error) { + return { + gate: { + step: "sympy_verify", + status: "failed", + duration_ms: Date.now() - started, + failure_detail: { + code: "math_engine_error", + message: error instanceof Error ? error.message : String(error), + }, + }, + }; + } +} + +export function extractEquation(text: string): string | null { + const normalized = normalizeMathText(text); + const match = normalized.match(/[A-Za-z0-9+\-*/^().\s]+=[A-Za-z0-9+\-*/^().\s]+/); + return match?.[0]?.trim() ?? null; +} + +export function normalizeMathText(text: string): string { + return text + .replace(/\\\((.*?)\\\)/g, "$1") + .replace(/\$(.*?)\$/g, "$1") + .replace(/\\frac\{([^{}]+)\}\{([^{}]+)\}/g, "($1)/($2)") + .replace(/\\sqrt\{([^{}]+)\}/g, "sqrt($1)") + .replace(/\^\{([^{}]+)\}/g, "^($1)") + .replace(/[{}]/g, "") + .replace(/−/g, "-") + .replace(/×/g, "*") + .replace(/÷/g, "/") + .replace(/\s+/g, " ") + .trim(); } diff --git a/packages/agent/src/tools/llm-provider.ts b/packages/agent/src/tools/llm-provider.ts index 9dfee0b..3ea7e35 100644 --- a/packages/agent/src/tools/llm-provider.ts +++ b/packages/agent/src/tools/llm-provider.ts @@ -6,6 +6,8 @@ * 단일 지점. */ +import { createOpenAI } from "@ai-sdk/openai"; +import { createOpenAICompatible } from "@ai-sdk/openai-compatible"; import type { LanguageModel } from "ai"; export type LlmProviderKind = @@ -20,6 +22,23 @@ export interface LlmProviderConfig { apiKey?: string; } -export function resolveLanguageModel(_config: LlmProviderConfig): LanguageModel { - throw new Error("resolveLanguageModel: not implemented yet"); +export function resolveLanguageModel(config: LlmProviderConfig): LanguageModel { + if (config.kind === "openai") { + const provider = createOpenAI({ + apiKey: config.apiKey ?? process.env.OPENAI_API_KEY, + compatibility: "strict", + }); + return provider(config.modelId); + } + + if (config.baseUrl === undefined) { + throw new Error(`${config.kind} requires LLM_BASE_URL`); + } + + const provider = createOpenAICompatible({ + name: config.kind, + baseURL: config.baseUrl, + apiKey: config.apiKey ?? "dummy-key", + }); + return provider(config.modelId); } diff --git a/packages/agent/src/tools/math-engine-client.ts b/packages/agent/src/tools/math-engine-client.ts index 1cd950a..4305cfd 100644 --- a/packages/agent/src/tools/math-engine-client.ts +++ b/packages/agent/src/tools/math-engine-client.ts @@ -68,7 +68,45 @@ export interface MathEngineClientOptions { } export function createMathEngineClient( - _opts: MathEngineClientOptions, + opts: MathEngineClientOptions, ): MathEngineClient { - throw new Error("createMathEngineClient: not implemented yet"); + const baseUrl = opts.baseUrl.replace(/\/$/, ""); + const timeoutMs = opts.timeoutMs ?? 5000; + + return { + health: () => request(baseUrl, "/health", undefined, timeoutMs), + solve: (req) => request(baseUrl, "/solve", req, timeoutMs), + verify: (req) => request(baseUrl, "/verify", req, timeoutMs), + simplify: (req) => request(baseUrl, "/simplify", req, timeoutMs), + differentiate: (req) => + request(baseUrl, "/differentiate", req, timeoutMs), + limit: (req) => request(baseUrl, "/limit", req, timeoutMs), + }; +} + +async function request( + baseUrl: string, + path: string, + body: unknown, + timeoutMs: number, +): Promise { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + + try { + const res = await fetch(`${baseUrl}${path}`, { + method: body === undefined ? "GET" : "POST", + headers: body === undefined ? undefined : { "Content-Type": "application/json" }, + body: body === undefined ? undefined : JSON.stringify(body), + signal: controller.signal, + }); + + if (!res.ok) { + throw new Error(`math-engine ${path} failed: ${res.status} ${res.statusText}`); + } + + return (await res.json()) as T; + } finally { + clearTimeout(timeout); + } } diff --git a/packages/agent/src/tools/prompt-loader.ts b/packages/agent/src/tools/prompt-loader.ts index 097517d..0ee3e61 100644 --- a/packages/agent/src/tools/prompt-loader.ts +++ b/packages/agent/src/tools/prompt-loader.ts @@ -8,6 +8,27 @@ * [비할당] 팀원이 매일 만지는 토스 단위. */ +import { readFile } from "node:fs/promises"; +import { resolve } from "node:path"; + +import matter from "gray-matter"; +import Handlebars from "handlebars"; +import { z } from "zod"; + +const PromptMetadataSchema = z.object({ + id: z.string().min(1), + version: z.string().min(1), + model: z.string().min(1), + temperature: z.number().min(0).max(2), + max_tokens: z.number().int().positive().optional(), + schema: z.string().min(1).optional(), + variables: z.array(z.string()).default([]), + owner: z.string().min(1), + updated: z.union([z.string(), z.date()]).transform((value) => + value instanceof Date ? value.toISOString().slice(0, 10) : value, + ), +}); + export interface PromptMetadata { id: string; version: string; @@ -37,7 +58,51 @@ export interface FsPromptLoaderOptions { } export function createFsPromptLoader( - _opts: FsPromptLoaderOptions, + opts: FsPromptLoaderOptions, ): PromptLoader { - throw new Error("createFsPromptLoader: not implemented yet"); + const promptsDir = resolve(opts.promptsDir); + const cache = new Map(); + + async function load(id: string): Promise { + if (!opts.hotReload) { + const cached = cache.get(id); + if (cached !== undefined) { + return cached; + } + } + + const prompt = await readPrompt(promptsDir, id); + cache.set(id, prompt); + return prompt; + } + + return { + load, + async reload() { + cache.clear(); + }, + }; +} + +async function readPrompt(promptsDir: string, id: string): Promise { + const path = resolve(promptsDir, `${id}.md`); + const file = await readFile(path, "utf8"); + const parsed = matter(file); + const metadata = PromptMetadataSchema.parse(parsed.data); + if (metadata.id !== id) { + throw new Error(`Prompt ${path} has id=${metadata.id}, expected ${id}`); + } + + const template = Handlebars.compile(parsed.content, { + noEscape: true, + strict: false, + }); + + return { + metadata, + rawBody: parsed.content, + render(vars) { + return template(vars); + }, + }; } diff --git a/packages/agent/src/tools/rag-client.ts b/packages/agent/src/tools/rag-client.ts index d7af46a..ad9d665 100644 --- a/packages/agent/src/tools/rag-client.ts +++ b/packages/agent/src/tools/rag-client.ts @@ -36,6 +36,7 @@ type OpenMathRagRecord = { semester?: 1 | 2 | null; topic_code?: string | null; topic_name: string; + achievement_code?: string | null; achievement_standard?: string | null; achievement_confidence?: number; }; @@ -154,7 +155,7 @@ function toIndexedProblem(record: OpenMathRagRecord): IndexedProblem { school_level: record.curriculum.school_level, grade: record.curriculum.grade, semester: record.curriculum.semester ?? null, - topic_code: record.curriculum.topic_code ?? null, + topic_code: record.curriculum.achievement_code ?? record.curriculum.topic_code ?? null, topic_name: record.curriculum.topic_name, achievement_standard: record.curriculum.achievement_standard ?? null, question_text: record.problem.question_text, diff --git a/packages/agent/src/tools/schema-loader.ts b/packages/agent/src/tools/schema-loader.ts index 7ae9f10..9e2d53b 100644 --- a/packages/agent/src/tools/schema-loader.ts +++ b/packages/agent/src/tools/schema-loader.ts @@ -8,7 +8,16 @@ * [비할당] 데이터 담당이 매일 만지는 토스 단위. */ -import type { Strategy } from "../schemas/index.js"; +import { readdir, readFile } from "node:fs/promises"; +import { basename, resolve } from "node:path"; + +import { parse } from "yaml"; + +import { + StrategySchema, + assertStrategyInvariants, + type Strategy, +} from "../schemas/index.js"; export interface StrategyLoader { load(code: string): Promise; @@ -22,7 +31,73 @@ export interface FsStrategyLoaderOptions { } export function createFsStrategyLoader( - _opts: FsStrategyLoaderOptions, + opts: FsStrategyLoaderOptions, ): StrategyLoader { - throw new Error("createFsStrategyLoader: not implemented yet"); + const strategiesDir = resolve(opts.strategiesDir); + const cache = new Map(); + + async function load(code: string): Promise { + if (!opts.hotReload && cache.has(code)) { + return cache.get(code) ?? null; + } + + const strategy = await readStrategy(strategiesDir, code); + cache.set(code, strategy); + return strategy; + } + + return { + load, + async loadAll() { + const entries = await readdir(strategiesDir, { withFileTypes: true }).catch( + (error: unknown) => { + if (isMissingFileError(error)) { + return []; + } + throw error; + }, + ); + const yamlFiles = entries + .filter((entry) => entry.isFile() && /\.ya?ml$/.test(entry.name)) + .map((entry) => basename(entry.name).replace(/\.ya?ml$/, "")); + + const strategies = await Promise.all(yamlFiles.map((code) => load(code))); + return strategies.filter((strategy): strategy is Strategy => strategy !== null); + }, + async reload() { + cache.clear(); + }, + }; +} + +async function readStrategy( + strategiesDir: string, + code: string, +): Promise { + const path = resolve(strategiesDir, `${code}.yaml`); + let file: string; + try { + file = await readFile(path, "utf8"); + } catch (error) { + if (isMissingFileError(error)) { + return null; + } + throw error; + } + + const parsed = StrategySchema.parse(parse(file)); + if (parsed.code !== code) { + throw new Error(`Strategy ${path} has code=${parsed.code}, expected ${code}`); + } + assertStrategyInvariants(parsed); + return parsed; +} + +function isMissingFileError(error: unknown): boolean { + return ( + typeof error === "object" && + error !== null && + "code" in error && + error.code === "ENOENT" + ); } diff --git a/packages/agent/src/workflows/verification-workflow.ts b/packages/agent/src/workflows/verification-workflow.ts index 4900c95..2afae8b 100644 --- a/packages/agent/src/workflows/verification-workflow.ts +++ b/packages/agent/src/workflows/verification-workflow.ts @@ -3,9 +3,17 @@ import type { GenerateRequest, + GenerateMode, + GateResult, + GeneratedProblem, + IsomorphicMode, ProgressEvent, + RagResult, + SolveAttempt, + Strategy, Verification, } from "../schemas/index.js"; +import { assertVerificationInvariants } from "../schemas/index.js"; import type { ConstraintCriticAgent, GeneratorAgent, @@ -17,18 +25,24 @@ import type { PromptLoader } from "../tools/prompt-loader.js"; import type { RagClient } from "../tools/rag-client.js"; import type { StrategyLoader } from "../tools/schema-loader.js"; import type { LanguageModel } from "ai"; +import { extractIntent } from "../steps/intent-extraction.js"; +import { independentResolve } from "../steps/independent-resolve.js"; +import { mapObjective } from "../steps/objective-mapping.js"; +import { generateProblem } from "../steps/problem-generation.js"; +import { ragSearch } from "../steps/rag-search.js"; +import { verifyWithSympy } from "../steps/sympy-verification.js"; export interface VerificationWorkflowDeps { rag: RagClient; - mathEngine: MathEngineClient; - prompts: PromptLoader; - strategies: StrategyLoader; - - intentModel: LanguageModel; - generator: GeneratorAgent; - critic: ConstraintCriticAgent; - refiner: RefinerAgent; - solver: SolverAgent; + mathEngine?: MathEngineClient; + prompts?: PromptLoader; + strategies?: StrategyLoader; + + intentModel?: LanguageModel; + generator?: GeneratorAgent; + critic?: ConstraintCriticAgent; + refiner?: RefinerAgent; + solver?: SolverAgent; objectiveLlm?: LanguageModel; } @@ -43,10 +57,437 @@ export type WorkflowReturn = { verifications: Verification[]; }; -export function runVerificationWorkflow( - _deps: VerificationWorkflowDeps, - _request: GenerateRequest, +export async function* runVerificationWorkflow( + deps: VerificationWorkflowDeps, + request: GenerateRequest, _options?: RunOptions, ): AsyncGenerator { - throw new Error("runVerificationWorkflow: not implemented yet"); + const timestamp = (): string => new Date().toISOString(); + + yield { + type: "step", + step: "rag", + status: "start", + timestamp: timestamp(), + }; + + const ragStarted = Date.now(); + const rag = await ragSearch({ rag: deps.rag }, { request }); + const ragGate: GateResult = { + step: "rag", + status: rag.refs.length > 0 ? "passed" : "failed", + duration_ms: Date.now() - ragStarted, + evidence: { + count: rag.refs.length, + fallback_used: rag.fallback_used, + }, + ...(rag.refs.length > 0 + ? {} + : { + failure_detail: { + code: "rag_no_refs", + message: "No source references were found for the request.", + }, + }), + }; + + yield { + type: "step", + step: "rag", + status: "done", + timestamp: timestamp(), + data: { + count: rag.refs.length, + duration_ms: ragGate.duration_ms, + fallback_used: rag.fallback_used, + refs: rag.refs.map((ref) => ({ + item_id: ref.item_id, + similarity: ref.similarity, + match_reason: ref.match_reason, + topic_name: ref.problem.topic_name, + })), + }, + }; + + yield { + type: "step", + step: "intent", + status: "start", + timestamp: timestamp(), + data: { + source_ref_count: rag.refs.length, + }, + }; + + const intentStarted = Date.now(); + const strategy = await loadStrategyForRequest(deps, request); + const { intent } = await extractIntent( + { model: deps.intentModel, prompts: deps.prompts }, + { + request, + refs: rag.refs, + strategy, + }, + ); + const intentGate: GateResult = { + step: "intent", + status: "passed", + duration_ms: Date.now() - intentStarted, + evidence: { + objective_code: intent.objective_code, + evaluation_dimension_count: intent.evaluation_dimensions.length, + strategy_loaded: strategy !== null, + }, + }; + + yield { + type: "step", + step: "intent", + status: "done", + timestamp: timestamp(), + data: { + duration_ms: intentGate.duration_ms, + objective_code: intent.objective_code, + objective_description: intent.objective_description, + evaluation_dimension_count: intent.evaluation_dimensions.length, + source_ref_count: rag.refs.length, + }, + }; + + if (deps.mathEngine === undefined) { + yield { + type: "error", + stage: "orchestrator", + code: "MISSING_MATH_ENGINE", + message: "math-engine dependency is required to complete verification.", + recoverable: false, + timestamp: timestamp(), + }; + return { verifications: [] }; + } + + const generator = deps.generator ?? createFallbackGenerator(); + const critic = deps.critic ?? createFallbackCritic(); + const refiner = deps.refiner ?? createFallbackRefiner(); + const solver = deps.solver ?? createFallbackSolver(); + const targetCount = request.count; + + yield { + type: "step", + step: "generate", + status: "start", + timestamp: timestamp(), + data: { count: targetCount }, + }; + + const generateStarted = Date.now(); + const generated = await Promise.all( + Array.from({ length: targetCount }, async (_, index) => { + const requestForAttempt = { + ...request, + mode: modeForAttempt(request.mode, index), + }; + const output = await generateProblem( + { + generator, + critic, + refiner, + maxCriticRounds: 2, + }, + { + request: requestForAttempt, + intent, + refs: rag.refs, + strategy, + attempt: index, + }, + ); + return { + ...output, + gate: { + step: "generate", + status: "passed", + duration_ms: Date.now() - generateStarted, + evidence: { refined_by: output.refined_by }, + } satisfies GateResult, + }; + }), + ); + + yield { + type: "step", + step: "generate", + status: "done", + timestamp: timestamp(), + data: { + count: generated.length, + duration_ms: Date.now() - generateStarted, + }, + }; + + yield { + type: "step", + step: "sympy_verify", + status: "start", + timestamp: timestamp(), + data: { count: generated.length }, + }; + + const sympyStarted = Date.now(); + const withSympy = await Promise.all( + generated.map(async (item) => ({ + ...item, + sympy: await verifyWithSympy( + { mathEngine: deps.mathEngine as NonNullable }, + { candidate: item.candidate }, + ), + })), + ); + + yield { + type: "step", + step: "sympy_verify", + status: "done", + timestamp: timestamp(), + data: { + passed: withSympy.filter((item) => item.sympy.gate.status === "passed").length, + count: withSympy.length, + duration_ms: Date.now() - sympyStarted, + }, + }; + + yield { + type: "step", + step: "re_solve", + status: "start", + timestamp: timestamp(), + data: { count: withSympy.length }, + }; + + const reSolveStarted = Date.now(); + const withReSolve = await Promise.all( + withSympy.map(async (item) => ({ + ...item, + reSolve: await independentResolve( + { + solver, + mathEngine: deps.mathEngine as NonNullable, + }, + { candidate: item.candidate, sympyGate: item.sympy.gate }, + ), + })), + ); + + yield { + type: "step", + step: "re_solve", + status: "done", + timestamp: timestamp(), + data: { + passed: withReSolve.filter((item) => item.reSolve.gate.status === "passed") + .length, + count: withReSolve.length, + duration_ms: Date.now() - reSolveStarted, + }, + }; + + yield { + type: "step", + step: "objective_map", + status: "start", + timestamp: timestamp(), + data: { count: withReSolve.length }, + }; + + const objectiveStarted = Date.now(); + const finalItems = await Promise.all( + withReSolve.map(async (item) => ({ + ...item, + objective: await mapObjective( + { llm: deps.objectiveLlm }, + { candidate: item.candidate, intent, strategy }, + ), + })), + ); + + const verifications = finalItems.map((item) => { + const verification = buildVerification({ + candidate: item.candidate, + attemptCount: item.candidate.generation_metadata.attempt + 1, + gates: [ + ragGate, + intentGate, + item.gate, + item.sympy.gate, + item.reSolve.gate, + item.objective.gate, + ], + }); + assertVerificationInvariants(verification); + return verification; + }); + + yield { + type: "step", + step: "objective_map", + status: "done", + timestamp: timestamp(), + data: { + passed: finalItems.filter((item) => item.objective.gate.status === "passed") + .length, + count: finalItems.length, + duration_ms: Date.now() - objectiveStarted, + }, + }; + + yield { + type: "result", + candidates: finalItems.map((item, index) => ({ + problem: { + ...item.candidate, + generation_metadata: { + ...item.candidate.generation_metadata, + refined_by: item.refined_by, + }, + }, + verification: verifications[index] as Verification, + })), + timestamp: timestamp(), + }; + + return { verifications }; +} + +async function loadStrategyForRequest( + deps: VerificationWorkflowDeps, + request: GenerateRequest, +): Promise { + const code = request.topic_code ?? request.topic; + if (deps.strategies === undefined || code === undefined) { + return null; + } + return deps.strategies.load(code); +} + +function buildVerification(input: { + candidate: GeneratedProblem; + gates: GateResult[]; + attemptCount: number; +}): Verification { + const sympy = input.gates.find((gate) => gate.step === "sympy_verify"); + const reSolve = input.gates.find((gate) => gate.step === "re_solve"); + const objective = input.gates.find((gate) => gate.step === "objective_map"); + const deterministicPassed = + sympy?.status === "passed" && objective?.status === "passed"; + const overall = + deterministicPassed && reSolve?.status === "failed" + ? "warning" + : deterministicPassed + ? "verified" + : "rejected"; + + return { + candidate_id: input.candidate.candidate_id, + overall, + gates: input.gates, + attempt_count: input.attemptCount, + ...(overall === "rejected" + ? { + failure_reason: failureReasonFor(input.gates), + } + : {}), + }; +} + +function failureReasonFor(gates: GateResult[]): Verification["failure_reason"] { + const failed = gates.find((gate) => gate.status === "failed"); + if (failed?.step === "sympy_verify") { + return { + category: "arithmetic_error", + user_message_ko: "생성된 정답이 수학 엔진의 풀이와 일치하지 않습니다.", + }; + } + if (failed?.step === "objective_map") { + return { + category: "learning_objective_mismatch", + user_message_ko: "생성된 문항이 요청한 학습 목표에서 벗어났습니다.", + }; + } + return { + category: "structural_error", + user_message_ko: "검증 파이프라인에서 통과하지 못한 단계가 있습니다.", + }; +} + +function createFallbackGenerator(): GeneratorAgent { + return { + async generate(input) { + const mode: IsomorphicMode = + input.request.mode === "conceptual" ? "conceptual" : "structural"; + const n = input.attempt + 2; + const answer = n + 1; + const ref = input.refs[0]; + return { + candidate_id: fallbackUuid(input.attempt), + mode, + question_text: `방정식 x + ${n} = ${answer + n} 의 해를 구하여라.`, + expected_answer: String(answer), + expected_choices: + input.request.problem_type === "objective" + ? [String(answer - 2), String(answer - 1), String(answer), String(answer + 1), String(answer + 2)] + : undefined, + proposed_solution_trace: `양변에서 ${n}을 빼면 x = ${answer}이다.`, + source_refs: input.refs.map((item) => item.item_id), + inferred_intent: input.intent, + generation_metadata: { + model: "fallback-generator", + temperature: 0, + prompt_id: "fallback", + prompt_version: "0.0.0", + attempt: input.attempt, + generated_at: new Date().toISOString(), + refined_by: [], + }, + }; + }, + }; +} + +function createFallbackCritic(): ConstraintCriticAgent { + return { + async critique() { + return { passes: true, hints: [] }; + }, + }; +} + +function createFallbackRefiner(): RefinerAgent { + return { + async refine(input) { + return input.prior; + }, + }; +} + +function createFallbackSolver(): SolverAgent { + return { + async solve(candidate): Promise { + return { + derived_answer: candidate.expected_answer, + trace: candidate.proposed_solution_trace, + confidence: "high", + }; + }, + }; +} + +function fallbackUuid(attempt: number): string { + return `00000000-0000-4000-8000-${String(attempt + 1).padStart(12, "0")}`; +} + +function modeForAttempt(mode: GenerateMode, attempt: number): Exclude { + if (mode === "auto") { + return attempt % 2 === 0 ? "structural" : "conceptual"; + } + return mode; } diff --git a/packages/agent/tests/rag-workflow.test.ts b/packages/agent/tests/rag-workflow.test.ts new file mode 100644 index 0000000..8f3859f --- /dev/null +++ b/packages/agent/tests/rag-workflow.test.ts @@ -0,0 +1,228 @@ +import { describe, expect, it } from "vitest"; + +import type { RagClient } from "../src/tools/rag-client.js"; +import type { MathEngineClient } from "../src/tools/math-engine-client.js"; +import { ragSearch } from "../src/steps/rag-search.js"; +import { runVerificationWorkflow } from "../src/workflows/verification-workflow.js"; +import type { RagQuery, RagResult } from "../src/schemas/index.js"; + +describe("ragSearch", () => { + it("falls back by relaxing type and difficulty filters when the primary query is empty", async () => { + const calls: RagQuery[] = []; + const rag: RagClient = { + async search(query) { + calls.push(query); + if (calls.length === 1) { + return []; + } + return [ragResult("111:train:fallback")]; + }, + }; + + const out = await ragSearch( + { rag }, + { + request: { + mode: "structural", + school_level: "middle", + grade: 2, + topic_code: "9수04-04", + topic_name: "도형의 닮음", + count: 2, + difficulty: "medium", + problem_type: "objective", + }, + }, + ); + + expect(out.refs).toHaveLength(1); + expect(out.fallback_used).toBe(true); + expect(calls[0]).toMatchObject({ + difficulty: "medium", + problem_type: "objective", + }); + expect(calls[1]).toMatchObject({ + difficulty: undefined, + problem_type: undefined, + }); + }); +}); + +describe("runVerificationWorkflow", () => { + it("streams all six steps and emits verified fallback candidates", async () => { + const rag: RagClient = { + async search() { + return [ragResult("111:train:similarity")]; + }, + }; + const mathEngine = fakeMathEngine(); + + const events = []; + const workflow = runVerificationWorkflow( + { rag, mathEngine }, + { + mode: "structural", + school_level: "middle", + grade: 2, + topic_code: "9수04-04", + topic_name: "도형의 닮음", + count: 2, + difficulty: "medium", + problem_type: "objective", + }, + ); + + for await (const event of workflow) { + events.push(event); + } + + expect(events.map((event) => event.type)).toEqual([ + "step", + "step", + "step", + "step", + "step", + "step", + "step", + "step", + "step", + "step", + "step", + "step", + "result", + ]); + expect(events[1]).toMatchObject({ + type: "step", + step: "rag", + status: "done", + data: { + count: 1, + refs: [ + { + item_id: "111:train:similarity", + topic_name: "도형의 닮음", + }, + ], + }, + }); + expect(events[3]).toMatchObject({ + type: "step", + step: "intent", + status: "done", + data: { + objective_code: "9수04-04", + source_ref_count: 1, + }, + }); + expect(events.at(-1)).toMatchObject({ + type: "result", + candidates: [ + { + verification: { + overall: "verified", + }, + }, + { + verification: { + overall: "verified", + }, + }, + ], + }); + }); + + it("auto mode alternates structural and conceptual candidates", async () => { + const rag: RagClient = { + async search() { + return [ragResult("111:train:similarity")]; + }, + }; + const events = []; + const workflow = runVerificationWorkflow( + { rag, mathEngine: fakeMathEngine() }, + { + mode: "auto", + school_level: "middle", + grade: 2, + topic_code: "9수04-04", + topic_name: "도형의 닮음", + count: 2, + difficulty: "medium", + problem_type: "objective", + }, + ); + + for await (const event of workflow) { + events.push(event); + } + + const result = events.at(-1); + expect(result).toMatchObject({ type: "result" }); + if (result?.type !== "result") { + throw new Error("expected result event"); + } + expect(result.candidates.map((item) => item.problem.mode)).toEqual([ + "structural", + "conceptual", + ]); + }); +}); + +function fakeMathEngine(): MathEngineClient { + return { + async health() { + return { status: "ok", engine: "sympy" }; + }, + async solve(req) { + const match = req.equation.match(/x\s*\+\s*(\d+)\s*=\s*(\d+)/); + if (match === null) { + return { solutions: [] }; + } + return { solutions: [String(Number(match[2]) - Number(match[1]))] }; + }, + async verify(req) { + const equivalent = req.expr1.trim() === req.expr2.trim(); + return { equivalent, diff: equivalent ? "0" : `${req.expr1}-${req.expr2}` }; + }, + async simplify(req) { + return { simplified: req.expr }; + }, + async differentiate() { + return { derivative: "0" }; + }, + async limit() { + return { limit: "0" }; + }, + }; +} + +function ragResult(itemId: string): RagResult { + return { + item_id: itemId, + similarity: 0.75, + match_reason: "hybrid", + problem: { + item_id: itemId, + source_dataset: "111", + split: "train", + source_label_type: "problem_label", + school_level: "middle", + grade: 2, + semester: null, + topic_code: "9수04-04", + topic_name: "도형의 닮음", + achievement_standard: + "도형의 닮음의 뜻과 닮은 도형의 성질을 이해하고, 닮음비를 구할 수 있다.", + question_text: "다음 두 도형의 닮음비를 구하여라.", + answer_text: "2:3", + explanation_text: "대응변의 길이의 비를 비교한다.", + choice_blocks: [], + problem_type_norm: "objective", + difficulty_norm: "medium", + question_image_relpath: null, + answer_image_relpath: null, + question_json_relpath: null, + answer_json_relpath: null, + }, + }; +} diff --git a/packages/agent/tests/strategy-loader.test.ts b/packages/agent/tests/strategy-loader.test.ts new file mode 100644 index 0000000..b0f0e94 --- /dev/null +++ b/packages/agent/tests/strategy-loader.test.ts @@ -0,0 +1,23 @@ +import { describe, expect, it } from "vitest"; + +import { createFsStrategyLoader } from "../src/tools/schema-loader.js"; + +describe("createFsStrategyLoader", () => { + it("loads demo strategy YAML files", async () => { + const loader = createFsStrategyLoader({ + strategiesDir: "data/achievement-standards", + }); + + const quadratic = await loader.load("9수02-09"); + expect(quadratic?.title).toBe("이차방정식"); + expect(quadratic?.evaluation_dimensions.some((d) => d.must_preserve)).toBe( + true, + ); + + const all = await loader.loadAll(); + expect(all.map((strategy) => strategy.code)).toEqual( + expect.arrayContaining(["9수02-03", "9수02-07", "9수02-09", "9수01-05"]), + ); + expect(all).toHaveLength(12); + }); +}); diff --git a/packages/math-engine/src/routers/math.py b/packages/math-engine/src/routers/math.py index 5408730..aec2abb 100644 --- a/packages/math-engine/src/routers/math.py +++ b/packages/math-engine/src/routers/math.py @@ -24,8 +24,8 @@ def safe_parse(expr_str: str): class SolveRequest(BaseModel): - equation: str - variable: str = "x" + equation: str | list[str] + variable: str | list[str] = "x" class SolveResponse(BaseModel): @@ -34,18 +34,37 @@ class SolveResponse(BaseModel): @router.post("/solve", response_model=SolveResponse) def solve_equation(req: SolveRequest): - var = symbols(req.variable) - - if "=" in req.equation: - left, right = req.equation.split("=", 1) - expr = Eq(safe_parse(left.strip()), safe_parse(right.strip())) + if isinstance(req.variable, list): + variables = symbols(" ".join(req.variable)) + if not isinstance(variables, tuple): + variables = (variables,) else: - expr = safe_parse(req.equation) - + variables = (symbols(req.variable),) + + if isinstance(req.equation, list): + equations = [parse_equation(equation) for equation in req.equation] + solutions = solve(equations, variables, dict=True) + return SolveResponse( + solutions=[ + ", ".join(f"{str(var)}={str(solution[var])}" for var in variables) + for solution in solutions + ] + ) + + expr = parse_equation(req.equation) + var = variables[0] solutions = solve(expr, var) return SolveResponse(solutions=[str(s) for s in solutions]) +def parse_equation(equation: str): + if "=" in equation: + left, right = equation.split("=", 1) + return Eq(safe_parse(left.strip()), safe_parse(right.strip())) + else: + return safe_parse(equation) + + class VerifyRequest(BaseModel): expr1: str expr2: str diff --git a/packages/math-engine/tests/test_solve.py b/packages/math-engine/tests/test_solve.py index 856dc86..52c49df 100644 --- a/packages/math-engine/tests/test_solve.py +++ b/packages/math-engine/tests/test_solve.py @@ -45,6 +45,18 @@ def test_solve_rational_solution(client): assert response.json()["solutions"] == ["-1/2"] +def test_solve_linear_system(client): + response = client.post( + "/solve", + json={ + "equation": ["x + y = 5", "x - y = 1"], + "variable": ["x", "y"], + }, + ) + assert response.status_code == 200 + assert response.json()["solutions"] == ["x=3, y=2"] + + def test_solve_invalid_equation(client): response = client.post( "/solve", diff --git a/packages/web/app/app/new/result/view.tsx b/packages/web/app/app/new/result/view.tsx index 257b716..589e869 100644 --- a/packages/web/app/app/new/result/view.tsx +++ b/packages/web/app/app/new/result/view.tsx @@ -1,7 +1,7 @@ "use client"; import Link from "next/link"; -import { useMemo, useState } from "react"; +import { useEffect, useMemo, useState } from "react"; import { LatexRenderer } from "@/components/math/latex-renderer"; import { type Grade, type Topic, gradeLabel } from "../topic/data"; import type { ResultProblem } from "./mock"; @@ -83,10 +83,27 @@ export function ResultView({ }: Props) { const [filter, setFilter] = useState("all"); const [adopted, setAdopted] = useState>(new Set()); + const [liveProblems, setLiveProblems] = useState(null); + + useEffect(() => { + const raw = window.localStorage.getItem("openmath:last-results"); + if (raw === null) return; + try { + const parsed = JSON.parse(raw) as unknown; + const mapped = mapLiveResults(parsed); + if (mapped.length > 0) { + setLiveProblems(mapped); + } + } catch { + setLiveProblems(null); + } + }, []); + + const effectiveProblems = liveProblems ?? problems; const visible = useMemo( - () => problems.filter((p) => matchesFilter(p, filter)), - [problems, filter], + () => effectiveProblems.filter((p) => matchesFilter(p, filter)), + [effectiveProblems, filter], ); if (grade === null || topic === null || mode === null) { @@ -126,7 +143,7 @@ export function ResultView({ }; const adoptedCount = adopted.size; - const passedCount = problems.filter((p) => p.status !== "fail").length; + const passedCount = effectiveProblems.filter((p) => p.status !== "fail").length; const verifyHref = `/app/new/verify?grade=${grade}&topic=${encodeURIComponent(topic.code)}&mode=${mode}&dims=${dims.join(",")}`; const exportHref = adoptedCount > 0 @@ -350,3 +367,37 @@ export function ResultView({ ); } + +function mapLiveResults(raw: unknown): ResultProblem[] { + if (!Array.isArray(raw)) return []; + return raw + .map((item, index): ResultProblem | null => { + if (typeof item !== "object" || item === null) return null; + const o = item as Record; + const id = typeof o.id === "string" ? o.id : `live-${index + 1}`; + const question = typeof o.question_latex === "string" ? o.question_latex : null; + const answer = typeof o.answer_latex === "string" ? o.answer_latex : null; + const iso = o.isomorphism; + const status = o.verification_status; + if (question === null || answer === null) return null; + if (iso !== "structural" && iso !== "conceptual") return null; + if (status !== "pass" && status !== "partial" && status !== "fail") return null; + const dims = Array.isArray(o.preserved_dimensions) + ? o.preserved_dimensions.filter((dim): dim is string => typeof dim === "string") + : []; + return { + id, + number: index + 1, + isomorphism: iso, + status: status === "partial" ? "warn" : status, + questionLatex: question, + answerLatex: answer, + solutionLatex: + typeof o.explanation_latex === "string" ? o.explanation_latex : "", + preservedDims: dims, + missingDims: [], + failReason: status === "fail" ? "검증 파이프라인에서 실패했습니다." : null, + }; + }) + .filter((item): item is ResultProblem => item !== null); +} diff --git a/packages/web/app/app/new/verify/view.tsx b/packages/web/app/app/new/verify/view.tsx index 2a8a106..a93e450 100644 --- a/packages/web/app/app/new/verify/view.tsx +++ b/packages/web/app/app/new/verify/view.tsx @@ -1,7 +1,7 @@ "use client"; import Link from "next/link"; -import { useMemo, useRef } from "react"; +import { useEffect, useMemo, useRef } from "react"; import { LatexRenderer } from "@/components/math/latex-renderer"; import { type Grade, @@ -9,6 +9,7 @@ import { gradeLabel, } from "../topic/data"; import { + type RagReference, type Step, type StreamInput, useVerificationStream, @@ -88,6 +89,37 @@ function stateLabel(status: Step["status"]): string { } } +function RagReferenceList({ refs }: { refs: RagReference[] }) { + if (refs.length === 0) { + return null; + } + + return ( +
+

+ 참조한 원본 문제 +

+
    + {refs.slice(0, 5).map((ref) => ( +
  • + {ref.topic_name} + + {ref.item_id} + {ref.similarity !== null + ? ` · ${Math.round(ref.similarity * 100)}%` + : ""} + {` · ${ref.match_reason}`} + +
    • + ))} +
+
+ ); +} + export function VerifyView({ grade, topic, mode, dims }: Props) { const valid = grade !== null && topic !== null && mode !== null && dims.length > 0; @@ -96,7 +128,7 @@ export function VerifyView({ grade, topic, mode, dims }: Props) { if (!valid || grade === null || topic === null || mode === null) { return null; } - return { grade, topic: topic.code, mode, dims }; + return { grade, topic: topic.code, topicName: topic.name, mode, dims }; }, [valid, grade, topic, mode, dims]); /* hook 은 input 이 null 이면 stream 을 시작하지 않는다. invalid 가드. */ @@ -137,6 +169,15 @@ export function VerifyView({ grade, topic, mode, dims }: Props) { const isDone = stream.status === "done"; const showPreview = stream.previewLatex !== null; + useEffect(() => { + if (isDone && stream.candidates.length > 0) { + window.localStorage.setItem( + "openmath:last-results", + JSON.stringify(stream.candidates), + ); + } + }, [isDone, stream.candidates]); + return ( <>