diff --git a/README.md b/README.md index cf4e4a2..07df231 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ [![Python](https://img.shields.io/badge/python-3.9%2B-blue.svg)](https://www.python.org/) [![License: MIT](https://img.shields.io/badge/License-MIT-green.svg)](LICENSE) -[![Tests](https://img.shields.io/badge/tests-161%20passing-brightgreen.svg)](tests/) +[![Tests](https://img.shields.io/badge/tests-163%20passing-brightgreen.svg)](tests/) [![OpenAI Agents SDK](https://img.shields.io/badge/OpenAI-Agents%20SDK-black.svg)](https://openai.github.io/openai-agents-python/) [![Docs](https://img.shields.io/badge/docs-website-blue.svg)](https://rrahimi-uci.github.io/agentic-context-engineering/) [![Paper](https://img.shields.io/badge/paper-arXiv%3A2510.04618-b31b1b.svg)](https://arxiv.org/abs/2510.04618) @@ -291,9 +291,9 @@ ace/ ├── integrations/ │ └── openai_agents.py # wrap_agent / ACEAgent: drop-in self-improving memory └── cli.py # `ace demo | run | playbook | version` -cookbook/ # 10 guided recipes (7 need no API key) + tests +cookbook/ # 11 guided recipes (8 need no API key) + tests examples/ # 5 runnable demos (4 need no API key) -tests/ # 161 tests, run in <1s, zero network +tests/ # 163 tests, run in <1s, zero network ``` --- @@ -302,7 +302,7 @@ tests/ # 161 tests, run in <1s, zero network ```bash pip install -e ".[dev]" -pytest # 161 tests, fully offline, ~1s +pytest # 163 tests, fully offline, ~1s python examples/01_quickstart.py python examples/02_context_collapse.py # writes ace_report.html ``` diff --git a/cookbook/11_extract_10k_to_csv.py b/cookbook/11_extract_10k_to_csv.py new file mode 100644 index 0000000..1dd548d --- /dev/null +++ b/cookbook/11_extract_10k_to_csv.py @@ -0,0 +1,300 @@ +"""Recipe 11 — A 10-K extraction agent that ports to CSV, optimized by ACE. + +What you'll learn +----------------- +* How to wrap a *real-world* job — reading pages of a company's annual report + (SEC Form **10-K**) and extracting structured financials into a **CSV** — as an + ACE :class:`~ace.tasks.Task`. +* How ACE turns the agent's mistakes into reusable **extraction rules** (one + bullet per field: *where* the number lives, *which* look-alike line items to + avoid). The agent learns the rules from a couple of filings and applies them to + a **brand-new company's 10-K it has never seen**. +* That the same loop that fixes a multiple-choice benchmark fixes a messy + document-extraction pipeline — the engine is domain-agnostic. + +The story +--------- +A 10-K page is full of look-alike numbers: prior-year columns, segment +subtotals, "Total *current* assets" vs. "Total assets", Basic vs. Diluted EPS, +the 21% statutory rate vs. the real effective rate. A naive extractor grabs the +wrong one. We frame each field as "pick the correct value from the candidates on +the page" so the recipe stays **deterministic and needs no API key** — the +simulated agent is wrong until ACE teaches it the right line item, then it nails +that field across all companies. + + python cookbook/11_extract_10k_to_csv.py # writes a CSV of extractions + +Going real +---------- +Swap the deterministic backend for a real model and this becomes a genuine +report-reading agent — the surrounding ACE code is identical:: + + from ace import ACE, OpenAILLM + ace = ACE(OpenAILLM(model="gpt-4o-mini")) # reads real 10-K text + ace.adapt_offline(train, feedback_fn=my_checker) + +or plug the learned playbook into an OpenAI Agents SDK agent with +``wrap_agent(...)`` (see recipes 08-10) so a tool-using agent improves its +extraction prompt from execution feedback. +""" + +from __future__ import annotations + +import csv +import hashlib +import os +import tempfile +from typing import Dict, List, Tuple + +from ace import ACE, ACEConfig, Sample, SimulatedLLM, Task, TeachingEnvironment +from ace.baselines import StaticAgent + +# --------------------------------------------------------------------------- # +# The fields we want in the CSV, each with the extraction *rule* ACE should learn +# and the trap that makes a naive agent get it wrong. +# --------------------------------------------------------------------------- # +FIELDS: List[Dict[str, str]] = [ + { + "concept": "total_revenue", + "label": "Total revenue (USD millions)", + "rule": "Total revenue is the 'Total net sales' line on the Consolidated " + "Statements of Operations for the most recent fiscal year — not the " + "prior-year column and not a segment subtotal.", + }, + { + "concept": "net_income", + "label": "Net income (USD millions)", + "rule": "Net income is the bottom-line 'Net income' after taxes — not " + "'Operating income' and not 'Income before income taxes'.", + }, + { + "concept": "total_assets", + "label": "Total assets (USD millions)", + "rule": "Total assets is the final total of the assets section on the " + "Consolidated Balance Sheets — not 'Total current assets'.", + }, + { + "concept": "diluted_eps", + "label": "Diluted EPS (USD)", + "rule": "Diluted EPS is reported under 'Earnings per share — Diluted' at " + "the bottom of the income statement, not Basic EPS.", + }, + { + "concept": "fiscal_year_end", + "label": "Fiscal year end", + "rule": "The fiscal year end is on the cover page; many retailers close on " + "the Saturday nearest a month-end rather than December 31.", + }, + { + "concept": "effective_tax_rate", + "label": "Effective tax rate", + "rule": "The effective tax rate comes from the income-tax footnote " + "(provision / pre-tax income); it differs from the 21% federal statutory rate.", + }, +] + +# Per company: the correct value for each field plus the look-alike distractors +# that actually appear elsewhere on the same page. +COMPANIES: Dict[str, Dict[str, object]] = { + "Northwind Retail Corp": { + "fiscal_year": "FY2024", + "values": { + "total_revenue": ("$391,035", ["$365,817", "$201,183", "$94,930"]), + "net_income": ("$24,160", ["$31,510", "$28,213", "$5,140"]), + "total_assets": ("$252,200", ["$76,877", "$135,726", "$45,690"]), + "diluted_eps": ("$2.41", ["$2.45", "$1.98", "$3.10"]), + "fiscal_year_end": ("Feb 1, 2025", ["Dec 31, 2024", "Jan 31, 2025", "Feb 3, 2024"]), + "effective_tax_rate": ("14.7%", ["21.0%", "12.3%", "19.5%"]), + }, + }, + "Cascade Logistics Inc": { + "fiscal_year": "FY2024", + "values": { + "total_revenue": ("$88,432", ["$84,002", "$41,205", "$22,118"]), + "net_income": ("$3,902", ["$6,114", "$4,880", "$1,205"]), + "total_assets": ("$71,540", ["$28,330", "$39,902", "$12,007"]), + "diluted_eps": ("$4.12", ["$4.20", "$3.55", "$5.01"]), + "fiscal_year_end": ("Dec 31, 2024", ["Jan 28, 2025", "Dec 26, 2024", "Dec 31, 2023"]), + "effective_tax_rate": ("22.8%", ["21.0%", "25.1%", "18.0%"]), + }, + }, + "Meridian Software Inc": { + "fiscal_year": "FY2024", + "values": { + "total_revenue": ("$61,271", ["$56,189", "$29,540", "$15,002"]), + "net_income": ("$18,344", ["$21,002", "$19,870", "$4,221"]), + "total_assets": ("$132,880", ["$48,210", "$70,115", "$22,640"]), + "diluted_eps": ("$7.66", ["$7.80", "$6.95", "$8.40"]), + "fiscal_year_end": ("Jun 30, 2024", ["Dec 31, 2024", "Jul 1, 2024", "Jun 30, 2023"]), + "effective_tax_rate": ("16.2%", ["21.0%", "13.9%", "20.4%"]), + }, + }, +} + + +def _letter(options: List[str], value: str) -> str: + return f"{chr(65 + options.index(value))}) {value}" + + +def _value(prediction: str) -> str: + """Strip an 'A) ' option prefix to recover the bare extracted value.""" + return prediction.split(")", 1)[-1].strip() if ")" in prediction else prediction.strip() + + +def _options(sample_id: str, correct: str, distractors: List[str]) -> List[str]: + """Deterministically order the candidate values (correct slot varies per id).""" + opts = [correct, *distractors] + h = int(hashlib.sha256(sample_id.encode()).hexdigest(), 16) + rot = h % len(opts) + return opts[rot:] + opts[:rot] + + +def _excerpt(company: str, fy: str, field: Dict[str, str], options: List[str]) -> str: + lines = [ + f"FORM 10-K excerpt — {company} ({fy})", + f"Extract: {field['label']}", + "Candidate values found on the page:", + ] + lines += [f" {chr(65 + i)}) {opt}" for i, opt in enumerate(options)] + return "\n".join(lines) + + +def _evaluate(prediction: str, sample: Sample) -> bool: + return _value(prediction) == _value(sample.answer) + + +def build_10k_task(companies: List[str], name: str = "10k-extraction") -> Task: + """One Sample per (company, field): extract the right value from the page.""" + samples: List[Sample] = [] + for company in companies: + info = COMPANIES[company] + fy = str(info["fiscal_year"]) + values = info["values"] # type: ignore[assignment] + for field in FIELDS: + concept = field["concept"] + correct, distractors = values[concept] # type: ignore[index] + sid = f"{company.split()[0].lower()}-{concept}" + options = _options(sid, correct, distractors) + samples.append( + Sample( + id=sid, + question=_excerpt(company, fy, field, options), + answer=_letter(options, correct), + concept=concept, # the extraction rule key ACE will learn + rule_text=field["rule"], + metadata={ + "options": options, + "company": company, + "fiscal_year": fy, + "field": field["label"], + }, + ) + ) + return Task( + name=name, + samples=samples, + evaluate=_evaluate, + description="Extract structured financials from 10-K pages into a CSV.", + ) + + +def _write_csv(rows: List[Tuple[str, str, str, str, str, bool]], path: str) -> None: + with open(path, "w", newline="", encoding="utf-8") as f: + w = csv.writer(f) + w.writerow( + ["company", "fiscal_year", "field", "extracted_value", "expected_value", "correct"] + ) + w.writerows(rows) + + +def run(csv_path: str | None = None) -> dict: + cleanup = False + if csv_path is None: + fd, csv_path = tempfile.mkstemp(suffix="_10k_extractions.csv") + os.close(fd) + cleanup = True + + # A base agent that innately knows only some of the line-item rules. + env = TeachingEnvironment(known_fraction=0.25, seed=11) + + # Learn from two companies' filings; extract from a brand-new, held-out one. + train_companies = ["Northwind Retail Corp", "Cascade Logistics Inc"] + held_out = "Meridian Software Inc" + train = build_10k_task(train_companies, name="10k-train") + test = build_10k_task([held_out], name="10k-test") + by_id = {s.id: s for s in test.samples} + + # 1) Baseline: extract from the unseen filing with no learned playbook. + base = StaticAgent(SimulatedLLM(env)).run(test) + + # 2) ACE: learn extraction rules offline from labeled feedback, then extract. + ace = ACE(SimulatedLLM(env), ACEConfig(epochs=4)) + ace.adapt_offline(train) + adapted = ace.evaluate(test) + + # 3) Port the extractions to CSV — one row per (company, field). + rows: List[Tuple[str, str, str, str, str, bool]] = [] + for rec in adapted.history: + s = by_id[rec.sample_id] + rows.append( + ( + str(s.metadata["company"]), + str(s.metadata["fiscal_year"]), + str(s.metadata["field"]), + _value(rec.prediction), + _value(rec.ground_truth or ""), + bool(rec.correct), + ) + ) + rows.sort(key=lambda r: (r[0], r[2])) + _write_csv(rows, csv_path) + + learned = [b.content for b in ace.playbook] + result = { + "held_out_company": held_out, + "base_accuracy": base.accuracy, + "ace_accuracy": adapted.accuracy, + "playbook_bullets": len(ace.playbook), + "rules_learned": learned, + "rows_written": len(rows), + "csv_path": csv_path, + "csv_header": [ + "company", + "fiscal_year", + "field", + "extracted_value", + "expected_value", + "correct", + ], + "sample_rows": rows[:3], + } + if cleanup: + os.remove(csv_path) + return result + + +def main() -> int: + r = run() + print("10-K extraction agent, optimized by ACE") + print("=" * 44) + print(f"Held-out filing extracted : {r['held_out_company']}") + print(f"Extraction accuracy (no learning) : {r['base_accuracy']:.1f}%") + print(f"Extraction accuracy (+ ACE) : {r['ace_accuracy']:.1f}%") + print(f"Extraction rules learned : {r['playbook_bullets']}") + print() + print("The agent wrote itself these extraction rules:") + for rule in r["rules_learned"]: + print(f" • {rule}") + print() + print(f"Ported {r['rows_written']} extractions to CSV → {r['csv_path']}") + print("First rows:") + print(" " + ", ".join(r["csv_header"])) + for row in r["sample_rows"]: + print(" " + ", ".join(str(c) for c in row)) + return 0 + + +if __name__ == "__main__": + import sys + + sys.exit(main()) diff --git a/cookbook/README.md b/cookbook/README.md index 88162b0..8b53750 100644 --- a/cookbook/README.md +++ b/cookbook/README.md @@ -26,6 +26,7 @@ covered by [`tests/test_cookbook.py`](../tests/test_cookbook.py). | 05 | [`05_save_and_resume.py`](05_save_and_resume.py) | Persisting a playbook to JSON and warm-starting a fresh engine from it. | | 06 | [`06_grow_and_refine.py`](06_grow_and_refine.py) | De-duplicating and pruning a playbook with `grow_and_refine` — keeping it compact. | | 07 | [`07_inspect_and_report.py`](07_inspect_and_report.py) | Introspecting bullets/stats and rendering a shareable **HTML report**. | +| 11 | [`11_extract_10k_to_csv.py`](11_extract_10k_to_csv.py) | An end-to-end **10-K extraction agent**: learn field-extraction rules from two filings, apply them to a held-out company's 10-K, and **port the results to CSV**. | ## 🔌 OpenAI Agents SDK recipes (API key) diff --git a/tests/test_cookbook.py b/tests/test_cookbook.py index 5905723..1022d4f 100644 --- a/tests/test_cookbook.py +++ b/tests/test_cookbook.py @@ -66,6 +66,7 @@ def test_cookbook_directory_exists(): "05_save_and_resume.py", "06_grow_and_refine.py", "07_inspect_and_report.py", + "11_extract_10k_to_csv.py", "08_agent_quickstart.py", "09_agent_auto_learn_from_tool_errors.py", "10_agent_streaming_and_sessions.py", @@ -149,6 +150,26 @@ def test_07_inspect_and_report_writes_html(tmp_path): assert pathlib.Path(path).exists() +@_feature("Cookbook — core") +@_story("11 extract 10-K to CSV") +def test_11_extract_10k_to_csv(tmp_path): + import csv as _csv + + path = str(tmp_path / "extractions.csv") + r = _load("11_extract_10k_to_csv.py").run(csv_path=path) + # ACE learns extraction rules and lifts accuracy on a held-out filing. + assert r["playbook_bullets"] >= 1 + assert r["ace_accuracy"] >= r["base_accuracy"] + assert r["ace_accuracy"] > 50.0 + # The CSV is written with one row per extracted field. + assert r["rows_written"] >= 1 + assert pathlib.Path(path).exists() + with open(path, encoding="utf-8") as f: + rows = list(_csv.reader(f)) + assert rows[0] == r["csv_header"] + assert len(rows) - 1 == r["rows_written"] + + # --------------------------------------------------------------------------- # # Agent recipes (08–10): import-clean and graceful without a key # --------------------------------------------------------------------------- #