From f98b85302b79fcdca7998a3e91697c097b4eb314 Mon Sep 17 00:00:00 2001 From: joncrall Date: Mon, 23 Feb 2026 16:43:17 -0500 Subject: [PATCH 01/36] wip --- dev/oneoff/check_results_overlap.py | 452 ++++++++++++++++++++-------- magnet/utils/sankey.py | 249 +++++++++++++-- magnet/utils/sankey_builder.py | 445 +++++++++++++++++++++++++++ 3 files changed, 1003 insertions(+), 143 deletions(-) create mode 100644 magnet/utils/sankey_builder.py diff --git a/dev/oneoff/check_results_overlap.py b/dev/oneoff/check_results_overlap.py index 7b5cf3e..95089e9 100644 --- a/dev/oneoff/check_results_overlap.py +++ b/dev/oneoff/check_results_overlap.py @@ -61,81 +61,212 @@ print(f'len(kwdagger_rows)={len(kwdagger_rows)}') +def make_bucket_fn( + edges_desc, + *, + nan_label="unknown (no comparable core)", + endpoints_as_strict_buckets=False, +): + """ + edges_desc: descending bin edges, e.g. [1.0, 0.90, 0.75, 0.50, 0.0] + + endpoints_as_strict_buckets: + - False (default): bins are intervals like "0.9–<1.0" plus a top open bin ">=1.0" + - True: create exact buckets for the endpoints (e.g. "1.0" and "0.0") + """ + edges = list(edges_desc) + assert len(edges) >= 2, "need at least two edges" + assert all(edges[i] >= edges[i + 1] for i in range(len(edges) - 1)), "edges must be descending" + + def _fmt_edge(v: float) -> str: + s = f"{v:.2f}".rstrip("0").rstrip(".") + return s if s else "0" + + top = edges[0] + bot = edges[-1] + + def bucket(x): + import math + if x is None or (isinstance(x, float) and math.isnan(x)): + return nan_label + + # Strict endpoint buckets + if endpoints_as_strict_buckets: + if x == top: + return _fmt_edge(top) + if x == bot: + return _fmt_edge(bot) + + # Top open bucket (captures >top, and also ==top when not strict) + if x >= top: + return f">={_fmt_edge(top)}" + + # Interior interval buckets: [lo, hi) + for i in range(1, len(edges)): + hi = edges[i - 1] + lo = edges[i] + if x >= lo: + return f"{_fmt_edge(lo)}–<{_fmt_edge(hi)}" + + # Below bottom edge + return f"<{_fmt_edge(bot)}" + + return bucket + + def sankey_stats(rd: HelmRunDiff) -> dict: """ Return a small, stable set of fields intended for building Sankey tables. """ - s = self.summary_dict(level=1) - va = (s.get("value_agreement") or {}) - by_class = (va.get("by_class") or {}) - overall = (va.get("overall") or {}) + s = rd.summary_dict(level=1) + va = s.get('value_agreement') or {} + by_class = va.get('by_class') or {} + overall = va.get('overall') or {} - core = (by_class.get("core") or {}) - book = (by_class.get("bookkeeping") or {}) + core = by_class.get('core') or {} + book = by_class.get('bookkeeping') or {} - core_ratio = core.get("agree_ratio", None) - book_ratio = book.get("agree_ratio", None) - overall_ratio = overall.get("agree_ratio", None) + core_ratio = core.get('agree_ratio', None) + book_ratio = book.get('agree_ratio', None) + overall_ratio = overall.get('agree_ratio', None) - spec_ok = bool(s.get("run_spec_dict_ok", False)) - scen_ok = s.get("scenario_ok", None) # True/False/None + spec_ok = bool(s.get('run_spec_dict_ok', False)) + scen_ok = s.get('scenario_ok', None) # True/False/None - spec_status = "spec match" if spec_ok else "spec mismatch" + spec_status = 'spec match' if spec_ok else 'spec mismatch' if scen_ok is None: - scenario_status = "scenario unknown" + scenario_status = 'scenario unknown' else: - scenario_status = "scenario match" if scen_ok else "scenario mismatch" + scenario_status = 'scenario match' if scen_ok else 'scenario mismatch' # stats_name_status is cheap and very useful as “schema drift” indicator - cov = (s.get("stats_coverage_by_name") or {}) - stats_name_status = "stats names match" if (cov.get("only_a", 0) == 0 and cov.get("only_b", 0) == 0) else "stats names mismatch" + cov = s.get('stats_coverage_by_name') or {} + stats_name_status = ( + 'stats names match' + if (cov.get('only_a', 0) == 0 and cov.get('only_b', 0) == 0) + else 'stats names mismatch' + ) def _bucket_ratio(x: float | None, *, good=0.995, ok=0.95) -> str: if x is None: - return "unknown" + return 'unknown' if x >= good: - return "high" + return 'high' if x >= ok: - return "medium" - return "low" + return 'medium' + return 'low' core_b = _bucket_ratio(core_ratio) book_b = _bucket_ratio(book_ratio) - if core_b == "high" and (book_b in {"high", "unknown"}): - agreement_quality = "match" - elif core_b == "high" and book_b in {"medium", "low"}: - agreement_quality = "core match, bookkeeping differs" - elif core_b == "medium": - agreement_quality = "core partial" - elif core_b == "low": - agreement_quality = "core mismatch" + if core_b == 'high' and (book_b in {'high', 'unknown'}): + agreement_quality = 'match' + elif core_b == 'high' and book_b in {'medium', 'low'}: + agreement_quality = 'core match, bookkeeping differs' + elif core_b == 'medium': + agreement_quality = 'core partial' + elif core_b == 'low': + agreement_quality = 'core mismatch' else: - agreement_quality = "unknown" + agreement_quality = 'unknown' return { # orthogonal statuses - "spec_status": spec_status, - "scenario_status": scenario_status, - "stats_name_status": stats_name_status, - "agreement_quality": agreement_quality, - + 'spec_status': spec_status, + 'scenario_status': scenario_status, + 'stats_name_status': stats_name_status, + 'agreement_quality': agreement_quality, # numeric signals - "run_agree_ratio_core": core_ratio, - "run_agree_ratio_bookkeeping": book_ratio, - "run_agree_ratio_overall": overall_ratio, - "comparable_core": core.get("comparable", None), - "mismatched_core": core.get("mismatched", None), - "comparable_bookkeeping": book.get("comparable", None), - "mismatched_bookkeeping": book.get("mismatched", None), - "comparable_overall": overall.get("comparable", None), - "mismatched_overall": overall.get("mismatched", None), + 'run_agree_ratio_core': core_ratio, + 'run_agree_ratio_bookkeeping': book_ratio, + 'run_agree_ratio_overall': overall_ratio, + 'comparable_core': core.get('comparable', None), + 'mismatched_core': core.get('mismatched', None), + 'comparable_bookkeeping': book.get('comparable', None), + 'mismatched_bookkeeping': book.get('mismatched', None), + 'comparable_overall': overall.get('comparable', None), + 'mismatched_overall': overall.get('mismatched', None), } - return out + return row + + +def modified_wormhole_send(fpath): + import subprocess + import selectors + + class NonBlockingPopenIO: + def __init__(self, p: subprocess.Popen, max_bytes=65536): + if p.stdout is None or p.stderr is None: + raise ValueError("Start Popen with stdout=PIPE and stderr=PIPE") + self.p = p + self.max_bytes = max_bytes + self.sel = selectors.DefaultSelector() + self.sel.register(p.stdout, selectors.EVENT_READ, data="stdout") + self.sel.register(p.stderr, selectors.EVENT_READ, data="stderr") + + def drain(self, timeout=0.0): + """ + Read whatever is available right now (bounded), without blocking. + Returns (stdout_bytes, stderr_bytes). + """ + out = b"" + err = b"" + for key, _ in self.sel.select(timeout): + stream = key.fileobj + name = key.data + # read1() avoids blocking for "more"; fallback to read() + reader = getattr(stream, "read1", stream.read) + data = reader(self.max_bytes) + if not data: + continue + if name == "stdout": + out += data + else: + err += data + return out, err + + cmd = ['wormhole', 'send', '--no-qr', str(fpath)] + + p = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + bufsize=0, + ) + + import time + time.sleep(1) + + io = NonBlockingPopenIO(p) + + # call this whenever you want (e.g., in your main loop) + out, err = io.drain(timeout=0.0) + if out: + out_text = out.decode("utf-8", errors="replace") + print("STDOUT:", out_text, end="") + if err: + err_text = err.decode("utf-8", errors="replace") + print("STDERR:", err_text, end="") + else: + err_text = '' + + code = [p for p in err_text.split('\n') if p][-1].split(' ')[-1] + + print(ub.codeblock( + f""" + # On the host run: + rm -rf {fpath} + # Run the wormhole command + wormhole recieve {code} --accept-file + eog {fpath} + """ + )) + p.communicate() + sankey_rows = [] +rundiffs = [] -rundiff_lut = {} for helm_row in ub.ProgIter(helm_rows, desc='compare runs'): run_dir = ub.Path(helm_row['run_dir']) suite_name = run_dir.parent.name @@ -151,39 +282,36 @@ def _bucket_ratio(x: float | None, *, good=0.995, ok=0.95) -> str: helm_row['reproduced_step1'] = kwrow is not None # Base row (always emitted, even if not attempted) - out = { - "run_spec_name": run_spec_name, - "run_dir": str(run_dir), - "suite_name": suite_name, - "benchmark_name": benchmark_name, - "model_name": helm_row['model'], - + row = { + 'run_spec_name': run_spec_name, + 'run_dir': str(run_dir), + 'suite_name': suite_name, + 'benchmark_name': benchmark_name, + 'model_name': helm_row['model'], # default pipeline fields - "reproduced_step1": False, - "attempt_status": None, - "attempt_error": None, - + 'reproduced_step1': False, + 'attempt_status': None, + 'attempt_error': None, # default agreement fields - "agreement_bucket": None, - "agreement_bucket_instances": None, - "run_agree_ratio_core": None, - "run_agree_ratio_bookkeeping": None, - "run_agree_ratio_overall": None, - "inst_agree_ratio_unperturbed": None, - "inst_agree_ratio_perturbed": None, - "n_instance_mismatched": None, - + 'agreement_bucket': None, + 'agreement_bucket_instances': None, + 'run_agree_ratio_core': None, + 'run_agree_ratio_bookkeeping': None, + 'run_agree_ratio_overall': None, + 'inst_agree_ratio_unperturbed': None, + 'inst_agree_ratio_perturbed': None, + 'n_instance_mismatched': None, # signatures (fill opportunistically) - "sig_run_spec": None, - "sig_scenario": None, - "sig_stats_name": None, + 'sig_run_spec': None, + 'sig_scenario': None, + 'sig_stats_name': None, } - out["reproduced_step1"] = (kwrow is not None) - sankey_rows.append(out) + row['reproduced_step1'] = kwrow is not None + sankey_rows.append(row) if kwrow is None: - out["attempt_status"] = "not attempted" - out["agreement_bucket"] = "not attempted" + row['attempt_status'] = 'not attempted' + row['agreement_bucket'] = 'not attempted' helm_row['agreement_bucket_base_task'] = 'not attempted' continue @@ -191,30 +319,40 @@ def _bucket_ratio(x: float | None, *, good=0.995, ok=0.95) -> str: # raise Exception # Attempt exists: try to compare - out["attempt_status"] = "compared" + row['attempt_status'] = 'compared' try: helm_run = HelmRun.coerce(run_dir) - kwdg_run = kwrow["run"] + kwdg_run = kwrow['run'] - a = HelmRunAnalysis(helm_run, name="HELM") - b = HelmRunAnalysis(kwdg_run, name="KWDG") + a = HelmRunAnalysis(helm_run, name='HELM') + b = HelmRunAnalysis(kwdg_run, name='KWDG') # Light signatures: cheap to compute and very useful for debugging. sa = a.summary_dict(level=0) sb = b.summary_dict(level=0) - out["sig_run_spec"] = f"{sa['signatures'].get('run_spec_sig')}|{sb['signatures'].get('run_spec_sig')}" - out["sig_scenario"] = f"{sa['signatures'].get('scenario_sig')}|{sb['signatures'].get('scenario_sig')}" - out["sig_stats_name"] = f"{sa['signatures'].get('stats_name_sig')}|{sb['signatures'].get('stats_name_sig')}" - - rd = HelmRunDiff(run_a=helm_run, run_b=kwdg_run, a_name="HELM", b_name="KWDG") - rundiff_lut[run_spec_name] = rd # save for later drilldown - out.update(sankey_stats(rd)) + row['sig_run_spec'] = ( + f'{sa["signatures"].get("run_spec_sig")}|{sb["signatures"].get("run_spec_sig")}' + ) + row['sig_scenario'] = ( + f'{sa["signatures"].get("scenario_sig")}|{sb["signatures"].get("scenario_sig")}' + ) + row['sig_stats_name'] = ( + f'{sa["signatures"].get("stats_name_sig")}|{sb["signatures"].get("stats_name_sig")}' + ) + + rd = HelmRunDiff( + run_a=helm_run, run_b=kwdg_run, a_name='HELM', b_name='KWDG' + ) + rundiffs.append(rd) # save for later drilldown + row.update(sankey_stats(rd)) + # Keep the row attached to the rundiff for easy interactive use + rd.row = row except Exception as ex: raise - out["attempt_status"] = "error" - out["attempt_error"] = repr(ex) - out["agreement_bucket"] = "error" + row['attempt_status'] = 'error' + row['attempt_error'] = repr(ex) + row['agreement_bucket'] = 'error' # # raise Exception @@ -261,29 +399,48 @@ def _bucket_ratio(x: float | None, *, good=0.995, ok=0.95) -> str: # helm_stats = helm_run.json.stats() # kwdg_stats = kwdg_run.json.stats() - # # out = compare.compare_run_pair(helm_stats, kwdg_stats, rel_tol=1e-4, abs_tol=1e-8) - # helm_row.update(out) + # # row = compare.compare_run_pair(helm_stats, kwdg_stats, rel_tol=1e-4, abs_tol=1e-8) + # helm_row.update(row) -for rd in rundiff_lut.values(): - rd.summary(level=0) - rd.summary() +for rd in rundiffs: + # rd.summary(level=0) + # rd.summary() a = rd.a b = rd.b - rd = HelmRunDiff(run_a=a, run_b=b, a_name="HELM", b_name="KWDG") - spec_a = rd.a.run_spec() - spec_b = rd.b.run_spec() - print(f'spec_a = {ub.urepr(spec_a, nl=3)}') - print(f'spec_b = {ub.urepr(spec_b, nl=3)}') + rd = HelmRunDiff(run_a=a, run_b=b, a_name='HELM', b_name='KWDG') + + summary = rd.summary_dict() + core_agreement = summary['value_agreement']['by_class']['core'] if 0: + idx = a.stat_index(drop_zero_count=True, require_mean=True) + core_a = pd.DataFrame( + {k: m for k, m in idx.items() if m.metric_class == 'core'}.values() + ) + idx = b.stat_index(drop_zero_count=True, require_mean=True) + core_b = pd.DataFrame( + {k: m for k, m in idx.items() if m.metric_class == 'core'}.values() + ) + spec_a = rd.a.run_spec() + spec_b = rd.b.run_spec() + print(f'spec_a = {ub.urepr(spec_a, nl=3)}') + print(f'spec_b = {ub.urepr(spec_b, nl=3)}') rd.summarize_instances() df = pd.DataFrame(sankey_rows) - -df = df[df['attempt_status'] == 'compared'] +# df = df[df['attempt_status'] == 'compared'] print(df.value_counts(['benchmark_name', 'reproduced_step1'])) -print(df.value_counts(['benchmark_name', 'reproduced_step1', 'spec_status', 'agreement_quality'])) +print( + df.value_counts( + [ + 'benchmark_name', + 'reproduced_step1', + 'spec_status', + 'agreement_quality', + ] + ) +) print(df.value_counts(['attempt_status', 'agreement_bucket']).sort_index()) @@ -303,39 +460,91 @@ def agreement_label(row: dict[str, object]) -> str: return row.get('agreement_bucket_base_task', 'unknown') +df['core_iou'] = (df['comparable_core'] - df['mismatched_core']) / df[ + 'comparable_core' +] + + +CORE_IOU_BINS = [ + 1.0, + # 0.90, + 0.75, + 0.50, + 0.25, + 0.00 +] +core_iou_bucket = make_bucket_fn(CORE_IOU_BINS, endpoints_as_strict_buckets=True) +df['core_iou_bucket'] = df['core_iou'].map(core_iou_bucket) +print(df['core_iou_bucket'].value_counts()) + +df['spec_status'].value_counts() +df['scenario_status'].value_counts() +df['stats_name_status'].value_counts() + +# Sankey plan: same skeleton, but add a core IoU bucket stage + +root = sankey.Root('All Attempts') +# When we get the data if one of the names isn't available, we handle it by +# dynamically adding it, but here we can use the result object to specify +# cases. +splits = root.split(by='attempt_status') +compared = splits.add_case(value='compared') +failcase = splits.add_case(value='not attempted') +failcase.set_label('Failed') + +bench_group = compared.group(by='benchmark_name', name='benchmark') +bench_group.group(by='core_iou_bucket') + + +# Note it should alway be possible to put "benchmarks" before attempt status like: +root = sankey.Root('All Attempts') +bench_group = root.group(by='benchmark_name', name='benchmark') +splits = bench_group.split(by='attempt_status') +compared = splits.cases['compared'] # behaves like a defaultdict +compared.group(by='core_iou_bucket') + +splits['not attempted'].set_label('Failed') + + +# --- +# Does this make sense? + + +from magnet.utils import sankey_builder +sankey_builder.Root(f'Attempted Runs') + + plan = sankey.Plan( - sankey.Root('Attempted Set'), - sankey.Group('benchmark', by='benchmark_name'), - sankey.Bucket('spec', by='spec_status'), - sankey.Bucket('agreement', by='agreement_quality'), + sankey.Root(f'Attempted Runs n={len(df)}'), + sankey.Split('Runs', 'attempt_status', branches={ + 'compared': sankey.Plan( + sankey.Group('benchmark', by='benchmark_name'), + sankey.Bucket('core_iou', by='core_iou_bucket'), + ), + 'not attempted': sankey.Node('Failed') # I want any that meet this condition to be sent to a node called failed. + }) ) print(plan.to_text()) -G = plan.build_sankey(helm_rows, label_fmt='{name}: {value}') +G = plan.build_sankey(df.to_dict('records'), label_fmt='{value}') print(G.summarize(max_edges=150)) fig = G.to_plotly(title='HELM Reproduction Funnel') fpath = 'helm_repro_sankey.jpg' -fig.write_image(fpath) +fig.write_image(fpath, scale=4.0) +import kwplot +kwplot.cropwhite_ondisk(fpath) print(f'Wrote helm_repro_sankey: {fpath}') -if 1: - print(ub.codeblock( - f''' - # On Host - rm -rf {fpath} - # Run the wormhole command - ''')) - ub.cmd(f'wormhole send {fpath}', verbose=3) - """ - !wormhole send helm_repro_sankey.jpg - """ +modified_wormhole_send(fpath) # --- Per-benchmark drilldown sankeys (deeper, but still run-level) --- # Here we drill by model within each benchmark (only if model_name exists). # If model_name is missing, you can swap this to group by run_spec_name prefix, etc. -bench_groups = ub.group_items(sankey_rows, key=lambda r: r.get('benchmark_name', 'unknown')) +bench_groups = ub.group_items( + sankey_rows, key=lambda r: r.get('benchmark_name', 'unknown') +) out_dpath = ub.Path('benchmark_sankeys').ensuredir() for bench, rows in bench_groups.items(): @@ -362,10 +571,13 @@ def agreement_label(row: dict[str, object]) -> str: print(f'Wrote benchmark sankey: {fpath_b}') if 1: - print(ub.codeblock( - f''' + print( + ub.codeblock( + f""" # On Host rm -rf {out_dpath} # Run the wormhole command - ''')) + """ + ) + ) ub.cmd(f'wormhole send {out_dpath}', verbose=3) diff --git a/magnet/utils/sankey.py b/magnet/utils/sankey.py index 118b86d..705b99f 100644 --- a/magnet/utils/sankey.py +++ b/magnet/utils/sankey.py @@ -8,6 +8,7 @@ from __future__ import annotations from dataclasses import dataclass, field + from typing import Any, Callable, Dict, Iterable, List, Optional, Union import typing @@ -29,6 +30,34 @@ def update_layout( @dataclass(frozen=True) class Root: label: str + def builder(self) -> PlanBuilder: + return PlanBuilder(Plan(self)) + def group(self, *, by: Grouper, name: Optional[str] = None) -> PlanBuilder: + return self.builder().group(by=by, name=name) + def split(self, *, by: Grouper, name: Optional[str] = None) -> SplitBuilder: + return self.builder().split(by=by, name=name) + + +# A terminal label convenience (primarily for branch specs) +@dataclass(frozen=True) +class Node: + label: str + + +@dataclass(frozen=True) +class Branch: + """ + A Split branch: + - label: optional override for the *split outcome node label* + - plan: plan to continue after the split outcome node + - terminal: if True, stop at the outcome node (ignore plan) + """ + plan: "Plan" = field(default_factory=lambda: Plan()) + label: Optional[str] = None + terminal: bool = False + + +BranchLike = Union["Plan", Branch, Node, str] # str => terminal label @dataclass(frozen=True) @@ -47,12 +76,103 @@ class Bucket: class Split: name: str by: Grouper - branches: Dict[Any, 'Plan'] - default: Optional['Plan'] = None + branches: Dict[Any, BranchLike] + default: Optional[BranchLike] = None + + def __post_init__(self): + def coerce(x: BranchLike) -> Branch: + if isinstance(x, Branch): + return x + if isinstance(x, Node): + return Branch(label=x.label, terminal=True) + if isinstance(x, str): + return Branch(label=x, terminal=True) + if isinstance(x, Plan): + return Branch(plan=x) + raise TypeError(f"Invalid branch spec: {type(x)}") + + # frozen dataclass: use object.__setattr__ to normalize + object.__setattr__( + self, "branches", {k: coerce(v) for k, v in self.branches.items()} + ) + if self.default is not None: + object.__setattr__(self, "default", coerce(self.default)) + + +class PlanBuilder: + def __init__(self, plan: Optional[Plan] = None): + self.plan = plan or Plan() + + def build(self) -> Plan: + return self.plan + + def group(self, *, by: Grouper, name: Optional[str] = None) -> "PlanBuilder": + if name is None: + name = by if isinstance(by, str) else "group" + self.plan.steps.append(Group(name, by)) + return self + + def bucket(self, *, by: Grouper, name: Optional[str] = None) -> "PlanBuilder": + if name is None: + name = by if isinstance(by, str) else "bucket" + self.plan.steps.append(Bucket(name, by)) + return self + + def split(self, *, by: Grouper, name: Optional[str] = None) -> "SplitBuilder": + if name is None: + name = by if isinstance(by, str) else "split" + sp = Split(name, by, branches={}) + self.plan.steps.append(sp) + return SplitBuilder(sp) + + +class CaseBuilder(PlanBuilder): + def __init__(self, branch_ref: Dict[Any, Branch], key: Any): + # branch_ref is the Split.branches dict (already normalized to Branch) + self._branch_ref = branch_ref + self._key = key + super().__init__(plan=self._branch_ref[key].plan) + def set_label(self, label: str) -> "CaseBuilder": + br = self._branch_ref[self._key] + self._branch_ref[self._key] = Branch(plan=br.plan, label=label, terminal=br.terminal) + return self + + def terminal(self, label: Optional[str] = None) -> "CaseBuilder": + br = self._branch_ref[self._key] + self._branch_ref[self._key] = Branch( + plan=br.plan, + label=(label if label is not None else br.label), + terminal=True, + ) + return self + + +class SplitBuilder: + def __init__(self, split: Split): + self.split = split + # Split.__post_init__ normalized branches to Branch already + self.cases: Dict[Any, CaseBuilder] = {} -def _eval_by(by: Grouper, row: Row): - return by(row) if callable(by) else row.get(by) + def add_case(self, *, value: Any) -> CaseBuilder: + if value not in self.split.branches: + # create an empty branch by default + self.split.branches[value] = Branch(plan=Plan()) + cb = CaseBuilder(self.split.branches, value) + self.cases[value] = cb + return cb + + def __getitem__(self, value: Any) -> CaseBuilder: + # defaultdict-ish + return self.add_case(value=value) + + def set_default(self) -> CaseBuilder: + if self.split.default is None: + # store default as a Branch (not in branches dict) + object.__setattr__(self.split, "default", Branch(plan=Plan())) + # expose via a pseudo key + # (or provide a dedicated DefaultCaseBuilder) + raise NotImplementedError("Implement a DefaultCaseBuilder if you want this.") def _by_repr(by: Grouper) -> str: @@ -64,10 +184,6 @@ def _by_repr(by: Grouper) -> str: return f'' -def _label(stage: str, value: Any, fmt: str): - return fmt.format(name=stage, value=value) - - @dataclass class Plan: """ @@ -128,15 +244,19 @@ def rec(plan: 'Plan', indent: str = ''): f'{indent}BUCKET {st.name!r} by={_by_repr(st.by)}' ) elif isinstance(st, Split): - lines.append( - f'{indent}SPLIT {st.name!r} by={_by_repr(st.by)}' - ) - for k, sub in st.branches.items(): - lines.append(f'{indent} BRANCH {k!r}:') - rec(sub, indent + ' ') + lines.append(f"{indent}SPLIT {st.name!r} by={_by_repr(st.by)}") + for k, br in st.branches.items(): + extra = [] + if br.label is not None: + extra.append(f"label={br.label!r}") + if br.terminal: + extra.append("terminal=True") + extra_s = (" [" + ", ".join(extra) + "]") if extra else "" + lines.append(f"{indent} BRANCH {k!r}:{extra_s}") + rec(br.plan, indent + " ") if st.default is not None: - lines.append(f'{indent} DEFAULT:') - rec(st.default, indent + ' ') + # same idea for default + ... else: lines.append(f'{indent}{type(st).__name__} (?)') @@ -149,6 +269,7 @@ def trace(self, row: Row, *, label_fmt='{name}: {value}') -> List[str]: Example: >>> # Trace the path a row takes through the plan + >>> from magnet.utils import sankey >>> plan = Plan( ... Root("ROOT_NODE"), ... Group("dataset", "dataset"), @@ -170,6 +291,9 @@ def trace(self, row: Row, *, label_fmt='{name}: {value}') -> List[str]: root = self._find_root_label(default='ROOT_NODE') path = [root] + def _eval_by(by: Grouper, row: Row): + return by(row) if callable(by) else row.get(by) + def run(plan: 'Plan', cur: str): node = cur for st in plan.steps: @@ -177,18 +301,30 @@ def run(plan: 'Plan', cur: str): continue elif isinstance(st, (Group, Bucket)): val = _eval_by(st.by, row) - nxt = _label(st.name, val, label_fmt) + nxt = label_fmt.format(name=st.name, value=val) path.append(nxt) node = nxt elif isinstance(st, Split): key = _eval_by(st.by, row) - split_node = _label(st.name, key, label_fmt) - path.append(split_node) - node = split_node - branch = st.branches.get(key) or st.default - if branch is None: + + br = st.branches.get(key) or st.default + if br is None: + return node + + # Decide the split outcome node label + split_label = ( + br.label + if (br.label is not None) + else label_fmt.format(name=st.name, value=key) + ) + path.append(split_label) + node = split_label + + # Terminal branch stops here + if br.terminal: return node - node = run(branch, node) + + node = run(br.plan, node) else: raise TypeError(f'Unknown step type: {type(st)}') return node @@ -453,3 +589,70 @@ def to_plotly(self, *, title: str = 'Sankey') -> PlotlyFigureLike: ) fig.update_layout(title_text=title, font_size=14) return fig + + +def demo(): + """ + """ + import kwutil + import ubelt as ub + rows = kwutil.Yaml.loads(ub.codeblock( + ''' + - {run: run1, suite: suite1, retcode: 1, retmsg: 'unsupported', spec_diagnosis: null, metric_iou: null} + - {run: run2, suite: suite1, retcode: 1, retmsg: 'unsupported', spec_diagnosis: null, metric_iou: null} + - {run: run3, suite: suite1, retcode: 1, retmsg: 'unsupported', spec_diagnosis: null, metric_iou: null} + - {run: run4, suite: suite1, retcode: 1, retmsg: 'unsupported', spec_diagnosis: null, metric_iou: null} + - {run: run5, suite: suite1, retcode: 1, retmsg: 'unsupported', spec_diagnosis: null, metric_iou: null} + + - {run: run1, suite: suite2, retcode: 0, retmsg: '', spec_diagnosis: 'agree', metric_iou: 1.0} + - {run: run2, suite: suite2, retcode: 0, retmsg: '', spec_diagnosis: 'agree', metric_iou: 1.0} + - {run: run3, suite: suite2, retcode: 0, retmsg: '', spec_diagnosis: 'agree', metric_iou: 1.0} + - {run: run4, suite: suite2, retcode: 0, retmsg: '', spec_diagnosis: 'agree', metric_iou: 1.0} + - {run: run5, suite: suite2, retcode: 0, retmsg: '', spec_diagnosis: 'agree', metric_iou: 1.0} + + - {run: run1, suite: suite3, retcode: 0, retmsg: '', spec_diagnosis: 'agree', metric_iou: 0.9} + - {run: run2, suite: suite3, retcode: 0, retmsg: '', spec_diagnosis: 'agree', metric_iou: 0.7} + - {run: run3, suite: suite3, retcode: 0, retmsg: '', spec_diagnosis: 'agree', metric_iou: 0.9} + - {run: run4, suite: suite3, retcode: 0, retmsg: '', spec_diagnosis: 'agree', metric_iou: 1.0} + - {run: run5, suite: suite3, retcode: 1, retmsg: 'oom', spec_diagnosis: null, metric_iou: null} + - {run: run6, suite: suite3, retcode: 1, retmsg: 'oom', spec_diagnosis: null, metric_iou: null} + + - {run: run1, suite: suite4, retcode: 0, retmsg: '', spec_diagnosis: 'disagree-deploy', metric_iou: 0.0} + - {run: run2, suite: suite4, retcode: 0, retmsg: '', spec_diagnosis: 'disagree-deploy', metric_iou: 0.0} + - {run: run3, suite: suite4, retcode: 0, retmsg: '', spec_diagnosis: 'disagree-input', metric_iou: 0.3} + - {run: run4, suite: suite4, retcode: 0, retmsg: '', spec_diagnosis: 'disagree-input', metric_iou: 0.1} + ''')) + + from magnet.utils import sankey_builder + root = sankey_builder.Root(label="All Attempts") + bench = root.group(by="suite", name="benchmark") + + splits = bench.group(by="retcode", name="Ran") + splits[1].label = 'Failed' + splits[0].label = 'Ran' + + # Do something unique on the fail branch + splits[1].group(by='retmsg') + + def iou_grouper(row): + value = row['metric_iou'] + if value == 1: + return '1' + elif value > 0.5: + return '0.5 - 1' + elif value > 0: + return '0 - 0.5' + else: + return '0' + + diagnosis_buckets = splits[0].group(by="agreement") + iou_groups = diagnosis_buckets.group(by=iou_grouper) + + # Connect only some of the underlying buckets + iou_groups['0.5 - 1'].connect('Priority Analysis') + iou_groups['0 - 0.5'].connect('Priority Analysis') + + graph = root.build_sankey(rows) + print(root.to_text()) + import networkx as nx + nx.write_network_text(graph) diff --git a/magnet/utils/sankey_builder.py b/magnet/utils/sankey_builder.py new file mode 100644 index 0000000..66f0994 --- /dev/null +++ b/magnet/utils/sankey_builder.py @@ -0,0 +1,445 @@ +""" +sankey_builder.py + +Fluent Sankey spec builder (mutable DAG), focused on: +- Nodes: Root / Constant / Group +- Group becomes a "split" when cases are configured +- connect() coerces strings to Constant nodes +- trace(row) for debugging +- trace_batch(rows) for fast aggregated execution +- build_sankey(rows) to produce a networkx graph with node+edge flow totals +- to_text() for readable structural representation + +Semantics: +- Each row follows ONE path through the spec (row -> labels). +- The sankey graph is the union of these paths (fan-out and fan-in supported). +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Callable, Dict, Iterable, Iterator, List, Optional, Sequence, Tuple, Union + +import itertools +import networkx as nx + +Row = Dict[str, Any] +By = Union[str, Callable[[Row], Any]] + + +def _eval(by: By, row: Row) -> Any: + return by(row) if callable(by) else row.get(by) + + +def _by_repr(by: By) -> str: + if isinstance(by, str): + return repr(by) + name = getattr(by, "__name__", None) + if name: + return f"" + return f"" + + +class SankeyDiGraph(nx.DiGraph): + """ + nx.DiGraph with configured edge/node attribute names. + + - Edge flow stored in `edge_attr` (default: "value") + - Node flow stored in `node_attr` (default: "count") + """ + + def __init__(self, *args, edge_attr: str = "value", node_attr: str = "count", **kwargs): + super().__init__(*args, **kwargs) + self.edge_attr = edge_attr + self.node_attr = node_attr + + +def _coerce_target(target: Union["Node", str, None]) -> Optional["Node"]: + """ + Coerce a connect() target: + - Node -> itself + - str -> new Constant(label=str) + - None -> None (terminal) + """ + if target is None: + return None + if isinstance(target, Node): + return target + if isinstance(target, str): + return Constant(label=target) + raise TypeError(f"Cannot connect to target of type {type(target)}") + + +BatchItem = Tuple[Row, float] +NodeFlow = Dict[str, float] +EdgeFlow = Dict[Tuple[str, str], float] + + +@dataclass +class BatchTraceResult: + node_flow: NodeFlow + edge_flow: EdgeFlow + + +@dataclass +class Case: + """ + Case config for a parent Group. + + - label: override emitted label for that key + - then: override continuation for that key + """ + label: Optional[str] = None + then: Optional["Node"] = None + + def group(self, *, by: By, name: Optional[str] = None) -> "Group": + g = Group(name=name or (by if isinstance(by, str) else "group"), by=by) + if self.then is not None: + raise RuntimeError("This case already has a `.then` continuation") + self.then = g + return g + + def connect(self, target: Union["Node", str, None]) -> Optional["Node"]: + self.then = _coerce_target(target) + return self.then + + +@dataclass +class CaseSet: + """ + Handle for bulk operations on multiple cases. + """ + cases: List[Case] + + def connect(self, target: Union["Node", str, None]) -> Optional["Node"]: + # If target is a string, create ONE Constant and share it. + tgt = _coerce_target(target) + for c in self.cases: + c.then = tgt + return tgt + + def set_label(self, label: str) -> "CaseSet": + for c in self.cases: + c.label = label + return self + + +@dataclass +class Node: + """ + Base node in the spec. + + Every Node: + - emits exactly one label per row + - chooses a continuation Node (or terminates) + + `next` is the fallback continuation. + """ + next: Optional["Node"] = None + + # ---- fluent building ---- + + def group(self, *, by: By, name: Optional[str] = None) -> "Group": + g = Group(name=name or (by if isinstance(by, str) else "group"), by=by) + if self.next is not None: + raise RuntimeError("This node already has a `.next` continuation") + self.next = g + return g + + def connect(self, target: Union["Node", str, None]) -> Optional["Node"]: + self.next = _coerce_target(target) + return self.next + + # ---- debug execution ---- + + def trace(self, row: Row, *, label_fmt: str = "{name}: {value}", max_steps: int = 10_000) -> List[str]: + """ + Debug trace: realize the label path for a single row. + """ + labels: List[str] = [] + cur: Optional[Node] = self + visited: set[int] = set() + steps = 0 + while cur is not None: + steps += 1 + if steps > max_steps: + raise RuntimeError(f"Exceeded max_steps={max_steps}; possible cycle") + oid = id(cur) + if oid in visited: + raise RuntimeError("Cycle detected while tracing (node revisited)") + visited.add(oid) + + label, cur = cur._step_row(row, label_fmt=label_fmt) + labels.append(label) + return labels + + # ---- fast batch execution ---- + + def trace_batch( + self, + rows: Iterable[Row], + *, + weight: Union[float, Callable[[Row], float]] = 1.0, + label_fmt: str = "{name}: {value}", + max_depth: int = 10_000, + ) -> BatchTraceResult: + """ + Fast batch tracing: aggregate node/edge flows without tracing each row into a full path. + + Returns: + BatchTraceResult(node_flow, edge_flow) + """ + weight_fn = weight if callable(weight) else (lambda r: float(weight)) + batch: List[BatchItem] = [(r, float(weight_fn(r))) for r in rows] + + node_flow: NodeFlow = {} + edge_flow: EdgeFlow = {} + + def bump_node(label: str, wsum: float) -> None: + node_flow[label] = node_flow.get(label, 0.0) + wsum + + def bump_edge(u: str, v: str, wsum: float) -> None: + edge_flow[(u, v)] = edge_flow.get((u, v), 0.0) + wsum + + def walk(node: Node, in_batch: List[BatchItem], parent_label: Optional[str], stack: List[int]) -> None: + if not in_batch: + return + if len(stack) > max_depth: + raise RuntimeError(f"Exceeded max_depth={max_depth}; possible cycle") + + oid = id(node) + if oid in stack: + raise RuntimeError("Cycle detected in spec while batch-tracing") + stack.append(oid) + try: + for label, out_batch, wsum, nxt in node._step_batch(in_batch, label_fmt=label_fmt): + bump_node(label, wsum) + if parent_label is not None: + bump_edge(parent_label, label, wsum) + if nxt is not None: + walk(nxt, out_batch, label, stack) + finally: + stack.pop() + + walk(self, batch, parent_label=None, stack=[]) + return BatchTraceResult(node_flow=node_flow, edge_flow=edge_flow) + + def build_sankey( + self, + rows: Iterable[Row], + *, + weight: Union[float, Callable[[Row], float]] = 1.0, + edge_attr: str = "value", + node_attr: str = "count", + label_fmt: str = "{name}: {value}", + ) -> SankeyDiGraph: + """ + Build a SankeyDiGraph using batch tracing. + """ + res = self.trace_batch(rows, weight=weight, label_fmt=label_fmt) + G = SankeyDiGraph(edge_attr=edge_attr, node_attr=node_attr) + + for n, c in res.node_flow.items(): + G.add_node(n, **{node_attr: c}) + for (u, v), val in res.edge_flow.items(): + # ensure nodes exist even if someone mutates later + if not G.has_node(u): + G.add_node(u, **{node_attr: 0.0}) + if not G.has_node(v): + G.add_node(v, **{node_attr: 0.0}) + G.add_edge(u, v, **{edge_attr: val}) + return G + + # ---- structure printing ---- + + def to_text(self) -> str: + dumper = _TextDumper() + dumper.rec(self, "") + return "\n".join(dumper.lines) + + # ---- subclass hooks ---- + + def _step_row(self, row: Row, *, label_fmt: str) -> Tuple[str, Optional["Node"]]: + raise NotImplementedError + + def _step_batch( + self, + batch: List[BatchItem], + *, + label_fmt: str, + ) -> Iterator[Tuple[str, List[BatchItem], float, Optional["Node"]]]: + """ + Yield (label, out_batch, weight_sum, next_node) for each outgoing route. + """ + raise NotImplementedError + + def _text_head(self, nid: int) -> str: + return f"{type(self).__name__}#{nid}" + + def _text_body(self, dumper: "_TextDumper", indent: str, nid: int) -> None: + if self.next is not None: + # dumper.lines.append(f"{indent} NEXT:") + dumper.rec(self.next, indent + " ") + + +@dataclass +class Constant(Node): + label: str = "CONST" + + def _step_row(self, row: Row, *, label_fmt: str) -> Tuple[str, Optional[Node]]: + return self.label, self.next + + def _step_batch( + self, + batch: List[BatchItem], + *, + label_fmt: str, + ) -> Iterator[Tuple[str, List[BatchItem], float, Optional[Node]]]: + wsum = sum(w for _, w in batch) + yield (self.label, batch, wsum, self.next) + + def _text_head(self, nid: int) -> str: + return f"CONST#{nid} {self.label!r}" + + +@dataclass +class Root(Constant): + def _text_head(self, nid: int) -> str: + return f"ROOT#{nid} {self.label!r}" + + +@dataclass +class Group(Node): + """ + A group computes value = by(row), emits a label, and optionally routes based on that value. + + - cases[value] can override: + - label (Case.label) + - next node (Case.then) + - fallback continuation is Group.next + """ + name: str = "group" + by: By = "" + cases: Dict[Any, Case] = field(default_factory=dict) + default: Optional[Case] = None + + def __getitem__(self, key: Union[Any, Sequence[Any]]) -> Union[Case, CaseSet]: + if isinstance(key, (list, tuple, set)): + vals = list(key) + return CaseSet([self.cases.setdefault(v, Case()) for v in vals]) + return self.cases.setdefault(key, Case()) + + def set_default(self) -> Case: + if self.default is None: + self.default = Case() + return self.default + + def _label_for(self, value: Any, case: Optional[Case], *, label_fmt: str) -> str: + if case is not None and case.label is not None: + return case.label + return label_fmt.format(name=self.name, value=value) + + def _next_for(self, case: Optional[Case]) -> Optional[Node]: + if case is not None and case.then is not None: + return case.then + return self.next + + def _step_row(self, row: Row, *, label_fmt: str) -> Tuple[str, Optional[Node]]: + v = _eval(self.by, row) + case = self.cases.get(v) or self.default + label = self._label_for(v, case, label_fmt=label_fmt) + nxt = self._next_for(case) + return label, nxt + + def _step_batch( + self, + batch: List[BatchItem], + *, + label_fmt: str, + ) -> Iterator[Tuple[str, List[BatchItem], float, Optional[Node]]]: + # Partition incoming batch by computed value + groups: Dict[Any, List[BatchItem]] = {} + sums: Dict[Any, float] = {} + for r, w in batch: + v = _eval(self.by, r) + groups.setdefault(v, []).append((r, w)) + sums[v] = sums.get(v, 0.0) + w + + for v, out_batch in groups.items(): + case = self.cases.get(v) or self.default + label = self._label_for(v, case, label_fmt=label_fmt) + nxt = self._next_for(case) + yield (label, out_batch, sums[v], nxt) + + def _text_head(self, nid: int) -> str: + return f"GROUP#{nid} {self.name!r} by={_by_repr(self.by)}" + + def _text_body(self, dumper: "_TextDumper", indent: str, nid: int) -> None: + has_branches = bool(self.cases) or (self.default is not None) + # Cases first + if self.cases: + for k in sorted(self.cases.keys(), key=lambda x: (str(type(x)), repr(x))): + c = self.cases[k] + extra = [] + if c.label is not None: + extra.append(f"label={c.label!r}") + hdr = f"{indent} CASE {k!r}" + if extra: + hdr += " [" + ", ".join(extra) + "]" + dumper.lines.append(hdr + ":") + if c.then is not None: + dumper.rec(c.then, indent + " ") + else: + dumper.lines.append(f"{indent} (fallthrough to NEXT)") + + if self.default is not None: + c = self.default + extra = [] + if c.label is not None: + extra.append(f"label={c.label!r}") + hdr = f"{indent} DEFAULT" + if extra: + hdr += " [" + ", ".join(extra) + "]" + dumper.lines.append(hdr + ":") + if c.then is not None: + dumper.rec(c.then, indent + " ") + else: + dumper.lines.append(f"{indent} (fallthrough to NEXT)") + + # Fallback continuation + if self.next is not None: + if has_branches: + dumper.lines.append(f"{indent} ELSE:") + dumper.rec(self.next, indent + " ") + else: + dumper.rec(self.next, indent + " ") + + +class _TextDumper: + """ + DAG-aware dumper used by Node.to_text(). + """ + def __init__(self) -> None: + self.lines: List[str] = [] + self._ids: Dict[int, int] = {} + self._expanded: set[int] = set() + self._counter = itertools.count(1) + + def _get_id(self, obj: object) -> int: + oid = id(obj) + if oid not in self._ids: + self._ids[oid] = next(self._counter) + return self._ids[oid] + + def rec(self, node: Node, indent: str) -> None: + nid = self._get_id(node) + head = node._text_head(nid) + self.lines.append(f"{indent}{head}") + + oid = id(node) + if oid in self._expanded: + self.lines[-1] += " (ref)" + return + self._expanded.add(oid) + + node._text_body(self, indent, nid) From 6a7147b7c66563dc7ea75229fe5ca36282c38067 Mon Sep 17 00:00:00 2001 From: joncrall Date: Mon, 23 Feb 2026 17:00:12 -0500 Subject: [PATCH 02/36] wip --- dev/oneoff/check_results_overlap.py | 39 ++-- magnet/utils/sankey_builder.py | 280 ++++++++++++++++++++++++++-- 2 files changed, 288 insertions(+), 31 deletions(-) diff --git a/dev/oneoff/check_results_overlap.py b/dev/oneoff/check_results_overlap.py index 95089e9..3b3c3ab 100644 --- a/dev/oneoff/check_results_overlap.py +++ b/dev/oneoff/check_results_overlap.py @@ -511,23 +511,28 @@ def agreement_label(row: dict[str, object]) -> str: from magnet.utils import sankey_builder -sankey_builder.Root(f'Attempted Runs') - - -plan = sankey.Plan( - sankey.Root(f'Attempted Runs n={len(df)}'), - sankey.Split('Runs', 'attempt_status', branches={ - 'compared': sankey.Plan( - sankey.Group('benchmark', by='benchmark_name'), - sankey.Bucket('core_iou', by='core_iou_bucket'), - ), - 'not attempted': sankey.Node('Failed') # I want any that meet this condition to be sent to a node called failed. - }) -) - -print(plan.to_text()) - -G = plan.build_sankey(df.to_dict('records'), label_fmt='{value}') +root = sankey_builder.Root() +rungroup = root.group(by='attempt_status') +compared_node = rungroup['compared'] +unrun_node = rungroup['not attempted'] +unrun_node.label = 'Failed' +bench_groups = compared_node.group(by='benchmark_name') +bench_groups.group(by='core_iou_bucket') + +# plan = sankey.Plan( +# sankey.Root(f'Attempted Runs n={len(df)}'), +# sankey.Split('Runs', 'attempt_status', branches={ +# 'compared': sankey.Plan( +# sankey.Group('benchmark', by='benchmark_name'), +# sankey.Bucket('core_iou', by='core_iou_bucket'), +# ), +# 'not attempted': sankey.Node('Failed') # I want any that meet this condition to be sent to a node called failed. +# }) +# ) + +print(root.to_text()) + +G = root.build_sankey(df.to_dict('records'), label_fmt='{value}') print(G.summarize(max_edges=150)) fig = G.to_plotly(title='HELM Reproduction Funnel') diff --git a/magnet/utils/sankey_builder.py b/magnet/utils/sankey_builder.py index 66f0994..f803d80 100644 --- a/magnet/utils/sankey_builder.py +++ b/magnet/utils/sankey_builder.py @@ -40,20 +40,6 @@ def _by_repr(by: By) -> str: return f"" -class SankeyDiGraph(nx.DiGraph): - """ - nx.DiGraph with configured edge/node attribute names. - - - Edge flow stored in `edge_attr` (default: "value") - - Node flow stored in `node_attr` (default: "count") - """ - - def __init__(self, *args, edge_attr: str = "value", node_attr: str = "count", **kwargs): - super().__init__(*args, **kwargs) - self.edge_attr = edge_attr - self.node_attr = node_attr - - def _coerce_target(target: Union["Node", str, None]) -> Optional["Node"]: """ Coerce a connect() target: @@ -443,3 +429,269 @@ def rec(self, node: Node, indent: str) -> None: self._expanded.add(oid) node._text_body(self, indent, nid) + + +class SankeyDiGraph(nx.DiGraph): + """ + A DiGraph with convenience methods for Sankey rendering / exporting. + + Notes: + - Flow is stored on edges in `edge_attr` (default: "value") + - Plotly node ordering is topological when possible, otherwise insertion order. + """ + + def __init__(self, *args, edge_attr: str = 'value', **kwargs): + super().__init__(*args, **kwargs) + self.edge_attr = edge_attr + + @classmethod + def demo(cls, n=200, seed=0) -> SankeyDiGraph: + """ + Demodata for tests + """ + import random + + r = random.Random(seed) + + rows = [ + dict( + dataset=r.choice(['coco', 'openimages', 'cityscapes']), + backend=r.choice(['cuda', 'cpu']), + status=('fail' if r.random() < 0.15 else 'ok'), + ) + for _ in range(n) + ] + for row in rows: + row['reason'] = ( + r.choice(['oom', 'timeout']) + if row['status'] == 'fail' + else None + ) + + plan = Plan( + Root('All Runs'), + Group('dataset', 'dataset'), + Split( + 'status', + 'status', + branches={ + 'ok': Plan(Group('backend', 'backend')), + 'fail': Plan( + Bucket('reason', 'reason'), Group('backend', 'backend') + ), + }, + ), + ) + self = plan.build_sankey(rows) + return self + + # ---- light reporting helpers (optional, but nice) ---- + + def summarize( + self, + *, + edge_attr: Optional[str] = None, + max_edges: Optional[int] = 200, + sort: str = 'value_desc', + ) -> str: + """ + Like Plan.graph_to_text, but bound to the graph. + + Example: + >>> # xdoctest: +REQUIRES(module:plotly) + >>> import plotly + >>> from magnet.utils.sankey import * # NOQA + >>> self = SankeyDiGraph.demo() + >>> print(self.summarize()) + Nodes: 10 Edges: 17 + ... + Top nodes by outflow/inflow: + All Runs out=200 in=0 + status: ok out=171 in=171 + dataset: cityscapes out=71 in=71 + ... + Edges: + status: ok -> backend: cuda value=94 + status: ok -> backend: cpu value=77 + All Runs -> dataset: cityscapes value=71 + ... + """ + edge_attr = edge_attr or self.edge_attr + lines: List[str] = [] + lines.append( + f'Nodes: {self.number_of_nodes()} Edges: {self.number_of_edges()}' + ) + lines.append('') + + def outflow(n): + return sum(self[n][v].get(edge_attr, 0) for v in self.successors(n)) + + def inflow(n): + return sum( + self[u][n].get(edge_attr, 0) for u in self.predecessors(n) + ) + + nodes_sorted = sorted( + self.nodes, key=lambda n: (outflow(n), inflow(n)), reverse=True + ) + lines.append('Top nodes by outflow/inflow:') + for n in nodes_sorted[:20]: + lines.append(f' {n} out={outflow(n):g} in={inflow(n):g}') + lines.append('') + + edges = [(u, v, self[u][v].get(edge_attr, 0)) for u, v in self.edges] + if sort == 'value_desc': + edges.sort(key=lambda t: t[2], reverse=True) + elif sort == 'lex': + edges.sort(key=lambda t: (str(t[0]), str(t[1]))) + + lines.append('Edges:') + shown = edges if max_edges is None else edges[:max_edges] + for u, v, val in shown: + lines.append(f' {u} -> {v} {edge_attr}={val:g}') + if max_edges is not None and len(edges) > max_edges: + lines.append(f'... ({len(edges) - max_edges} more edges)') + return '\n'.join(lines) + + # ---- core conversions ---- + + def _to_sankey_data( + self, + ) -> tuple[List[Any], List[int], List[int], List[float]]: + """ + Convert into (nodes, source, target, value) for Plotly Sankey. + + Example: + >>> # Convert nx graph to plotly sankey arrays + >>> from magnet.utils.sankey import * # NOQA + >>> import networkx as nx + >>> G = SankeyDiGraph() + >>> G.add_edge("A", "B", value=2) + >>> G.add_edge("A", "C", value=3) + >>> nodes, source, target, value = G._to_sankey_data() + >>> set(nodes) == {"A", "B", "C"} + True + >>> len(source) == len(target) == len(value) == 2 + True + >>> sorted(value) + [2.0, 3.0] + """ + try: + nodes = list(nx.topological_sort(self)) + except nx.NetworkXUnfeasible: + nodes = list(self.nodes) + + idx = {n: i for i, n in enumerate(nodes)} + source: List[int] = [] + target: List[int] = [] + value: List[float] = [] + + for u, v, data in self.edges(data=True): + source.append(idx[u]) + target.append(idx[v]) + value.append(float(data.get(self.edge_attr, 0))) + + return nodes, source, target, value + + def to_plotly(self, *, title: str = 'Sankey') -> PlotlyFigureLike: + """ + Build a publishable Plotly Sankey figure. + + Example: + >>> # xdoctest: +REQUIRES(module:plotly) + >>> import plotly + >>> from magnet.utils.sankey import * # NOQA + >>> G = SankeyDiGraph.demo(n=20) + >>> fig = G.to_plotly(title='Demo') + >>> assert fig.layout.title.text == 'Demo' + >>> # xdoctest: +REQUIRES(module:kaleido) + >>> # xdoctest: +REQUIRES(module:kwplot) + >>> # xdoctest: +REQUIRES(--show) + >>> import kwplot + >>> kwplot.autompl() + >>> import tempfile + >>> import os + >>> with tempfile.TemporaryDirectory() as d: + ... fpath = os.path.join(d, "sankey_demo.png") + ... fig.write_image(fpath, scale=1) + ... assert os.path.exists(fpath) + ... kwplot.imshow(fpath) + """ + import plotly.graph_objects as go + + nodes, source, target, value = self._to_sankey_data() + fig = go.Figure( + go.Sankey( + node=dict(label=nodes, pad=15, thickness=18), + link=dict(source=source, target=target, value=value), + ) + ) + fig.update_layout(title_text=title, font_size=14) + return fig + + +def demo(): + """ + """ + import kwutil + import ubelt as ub + rows = kwutil.Yaml.loads(ub.codeblock( + ''' + - {run: run1, suite: suite1, retcode: 1, retmsg: 'unsupported', spec_diagnosis: null, metric_iou: null} + - {run: run2, suite: suite1, retcode: 1, retmsg: 'unsupported', spec_diagnosis: null, metric_iou: null} + - {run: run3, suite: suite1, retcode: 1, retmsg: 'unsupported', spec_diagnosis: null, metric_iou: null} + - {run: run4, suite: suite1, retcode: 1, retmsg: 'unsupported', spec_diagnosis: null, metric_iou: null} + - {run: run5, suite: suite1, retcode: 1, retmsg: 'unsupported', spec_diagnosis: null, metric_iou: null} + + - {run: run1, suite: suite2, retcode: 0, retmsg: '', spec_diagnosis: 'agree', metric_iou: 1.0} + - {run: run2, suite: suite2, retcode: 0, retmsg: '', spec_diagnosis: 'agree', metric_iou: 1.0} + - {run: run3, suite: suite2, retcode: 0, retmsg: '', spec_diagnosis: 'agree', metric_iou: 1.0} + - {run: run4, suite: suite2, retcode: 0, retmsg: '', spec_diagnosis: 'agree', metric_iou: 1.0} + - {run: run5, suite: suite2, retcode: 0, retmsg: '', spec_diagnosis: 'agree', metric_iou: 1.0} + + - {run: run1, suite: suite3, retcode: 0, retmsg: '', spec_diagnosis: 'agree', metric_iou: 0.9} + - {run: run2, suite: suite3, retcode: 0, retmsg: '', spec_diagnosis: 'agree', metric_iou: 0.7} + - {run: run3, suite: suite3, retcode: 0, retmsg: '', spec_diagnosis: 'agree', metric_iou: 0.9} + - {run: run4, suite: suite3, retcode: 0, retmsg: '', spec_diagnosis: 'agree', metric_iou: 1.0} + - {run: run5, suite: suite3, retcode: 1, retmsg: 'oom', spec_diagnosis: null, metric_iou: null} + - {run: run6, suite: suite3, retcode: 1, retmsg: 'oom', spec_diagnosis: null, metric_iou: null} + + - {run: run1, suite: suite4, retcode: 0, retmsg: '', spec_diagnosis: 'disagree-deploy', metric_iou: 0.0} + - {run: run2, suite: suite4, retcode: 0, retmsg: '', spec_diagnosis: 'disagree-deploy', metric_iou: 0.0} + - {run: run3, suite: suite4, retcode: 0, retmsg: '', spec_diagnosis: 'disagree-input', metric_iou: 0.3} + - {run: run4, suite: suite4, retcode: 0, retmsg: '', spec_diagnosis: 'disagree-input', metric_iou: 0.1} + ''')) + + from magnet.utils import sankey_builder + root = sankey_builder.Root(label="All Attempts") + bench = root.group(by="suite", name="benchmark") + + splits = bench.group(by="retcode", name="Ran") + splits[1].label = 'Failed' + splits[0].label = 'Ran' + + # Do something unique on the fail branch + splits[1].group(by='retmsg') + + def iou_grouper(row): + value = row['metric_iou'] + if value == 1: + return '1' + elif value > 0.5: + return '0.5 - 1' + elif value > 0: + return '0 - 0.5' + else: + return '0' + + diagnosis_buckets = splits[0].group(by="agreement") + iou_groups = diagnosis_buckets.group(by=iou_grouper) + + # Connect only some of the underlying buckets + iou_groups['0.5 - 1'].connect('Priority Analysis') + iou_groups['0 - 0.5'].connect('Priority Analysis') + + graph = root.build_sankey(rows) + print(root.to_text()) + import networkx as nx + nx.write_network_text(graph) From fd060ed1f88f0a0536ef4925fcaa6a53c38a2f46 Mon Sep 17 00:00:00 2001 From: joncrall Date: Mon, 23 Feb 2026 17:19:55 -0500 Subject: [PATCH 03/36] wip --- dev/oneoff/check_results_overlap.py | 33 +++++++++++++++++++++-------- magnet/utils/sankey_builder.py | 26 +++++++++++++++++------ 2 files changed, 44 insertions(+), 15 deletions(-) diff --git a/dev/oneoff/check_results_overlap.py b/dev/oneoff/check_results_overlap.py index 3b3c3ab..0948b51 100644 --- a/dev/oneoff/check_results_overlap.py +++ b/dev/oneoff/check_results_overlap.py @@ -460,11 +460,6 @@ def agreement_label(row: dict[str, object]) -> str: return row.get('agreement_bucket_base_task', 'unknown') -df['core_iou'] = (df['comparable_core'] - df['mismatched_core']) / df[ - 'comparable_core' -] - - CORE_IOU_BINS = [ 1.0, # 0.90, @@ -473,6 +468,9 @@ def agreement_label(row: dict[str, object]) -> str: 0.25, 0.00 ] +df['core_iou'] = (df['comparable_core'] - df['mismatched_core']) / df[ + 'comparable_core' +] core_iou_bucket = make_bucket_fn(CORE_IOU_BINS, endpoints_as_strict_buckets=True) df['core_iou_bucket'] = df['core_iou'].map(core_iou_bucket) print(df['core_iou_bucket'].value_counts()) @@ -512,12 +510,16 @@ def agreement_label(row: dict[str, object]) -> str: from magnet.utils import sankey_builder root = sankey_builder.Root() -rungroup = root.group(by='attempt_status') + +bench_groups = root.group(by='benchmark_name') + +rungroup = bench_groups.group(by='attempt_status') compared_node = rungroup['compared'] +compared_node.label = 'Run' unrun_node = rungroup['not attempted'] unrun_node.label = 'Failed' -bench_groups = compared_node.group(by='benchmark_name') -bench_groups.group(by='core_iou_bucket') + +compared_node.group(by='core_iou_bucket') # plan = sankey.Plan( # sankey.Root(f'Attempted Runs n={len(df)}'), @@ -535,13 +537,26 @@ def agreement_label(row: dict[str, object]) -> str: G = root.build_sankey(df.to_dict('records'), label_fmt='{value}') print(G.summarize(max_edges=150)) +G.nodes['CONST']['label'] = f'Attempted Runs n={len(df)}' fig = G.to_plotly(title='HELM Reproduction Funnel') + + +import plotly.graph_objects as go + +node_labels, source, target, value = G._to_sankey_data() +sankey = go.Sankey( + node=dict(label=node_labels, pad=15, thickness=18), + link=dict(source=source, target=target, value=value), +) +fig = go.Figure(sankey) +fig.update_layout(title_text=title, font_size=14) + + fpath = 'helm_repro_sankey.jpg' fig.write_image(fpath, scale=4.0) import kwplot kwplot.cropwhite_ondisk(fpath) print(f'Wrote helm_repro_sankey: {fpath}') - modified_wormhole_send(fpath) # --- Per-benchmark drilldown sankeys (deeper, but still run-level) --- diff --git a/magnet/utils/sankey_builder.py b/magnet/utils/sankey_builder.py index f803d80..71267e3 100644 --- a/magnet/utils/sankey_builder.py +++ b/magnet/utils/sankey_builder.py @@ -22,10 +22,23 @@ import itertools import networkx as nx +import typing Row = Dict[str, Any] By = Union[str, Callable[[Row], Any]] +if typing.TYPE_CHECKING: + from typing import Protocol + + Row = Dict[str, Any] + Grouper = Union[str, Callable[[Row], Any]] + + class PlotlyFigureLike(Protocol): + def write_image(self, *args, **kwargs): ... + def update_layout( + self, dict1=None, overwrite=False, **kwargs + ) -> 'PlotlyFigureLike': ... + def _eval(by: By, row: Row) -> Any: return by(row) if callable(by) else row.get(by) @@ -591,7 +604,9 @@ def _to_sankey_data( target.append(idx[v]) value.append(float(data.get(self.edge_attr, 0))) - return nodes, source, target, value + node_labels = [self.node[n].get('labeel', n) for n in nodes] + + return node_labels, source, target, value def to_plotly(self, *, title: str = 'Sankey') -> PlotlyFigureLike: """ @@ -620,12 +635,11 @@ def to_plotly(self, *, title: str = 'Sankey') -> PlotlyFigureLike: import plotly.graph_objects as go nodes, source, target, value = self._to_sankey_data() - fig = go.Figure( - go.Sankey( - node=dict(label=nodes, pad=15, thickness=18), - link=dict(source=source, target=target, value=value), - ) + sankey = go.Sankey( + node=dict(label=nodes, pad=15, thickness=18), + link=dict(source=source, target=target, value=value), ) + fig = go.Figure(sankey) fig.update_layout(title_text=title, font_size=14) return fig From 459e2901643eb1989c5dbb8ebb1c2e645d4722e7 Mon Sep 17 00:00:00 2001 From: joncrall Date: Mon, 23 Feb 2026 17:21:06 -0500 Subject: [PATCH 04/36] wip --- magnet/utils/sankey_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/magnet/utils/sankey_builder.py b/magnet/utils/sankey_builder.py index 71267e3..a55421d 100644 --- a/magnet/utils/sankey_builder.py +++ b/magnet/utils/sankey_builder.py @@ -604,7 +604,7 @@ def _to_sankey_data( target.append(idx[v]) value.append(float(data.get(self.edge_attr, 0))) - node_labels = [self.node[n].get('labeel', n) for n in nodes] + node_labels = [self.nodes[n].get('label', n) for n in nodes] return node_labels, source, target, value From bc966fb036f8e5dd07f5d4168fa6de4824e53545 Mon Sep 17 00:00:00 2001 From: joncrall Date: Mon, 2 Mar 2026 12:37:06 -0500 Subject: [PATCH 05/36] wip --- dev/oneoff/check_results_overlap.py | 126 +++++++++++++++++----------- 1 file changed, 77 insertions(+), 49 deletions(-) diff --git a/dev/oneoff/check_results_overlap.py b/dev/oneoff/check_results_overlap.py index 0948b51..654c4f9 100644 --- a/dev/oneoff/check_results_overlap.py +++ b/dev/oneoff/check_results_overlap.py @@ -1,4 +1,7 @@ """ +Notebook style code the developer is working with interactively by copy/pasting +blocks into IPython. + !uv pip install kaleido plotly """ @@ -37,7 +40,7 @@ self = HelmRunAnalysis(helm_run) finished_jobs = list( - ub.Path('/home/local/KHQ/jon.crall/code/aiq-magnet/results/helm').glob( + ub.Path('~/code/aiq-magnet/results/helm').expand().glob( '*/DONE' ) ) @@ -403,33 +406,35 @@ def drain(self, timeout=0.0): # helm_row.update(row) -for rd in rundiffs: - # rd.summary(level=0) - # rd.summary() - a = rd.a - b = rd.b - rd = HelmRunDiff(run_a=a, run_b=b, a_name='HELM', b_name='KWDG') - - summary = rd.summary_dict() - core_agreement = summary['value_agreement']['by_class']['core'] - - if 0: - idx = a.stat_index(drop_zero_count=True, require_mean=True) - core_a = pd.DataFrame( - {k: m for k, m in idx.items() if m.metric_class == 'core'}.values() - ) - idx = b.stat_index(drop_zero_count=True, require_mean=True) - core_b = pd.DataFrame( - {k: m for k, m in idx.items() if m.metric_class == 'core'}.values() - ) - spec_a = rd.a.run_spec() - spec_b = rd.b.run_spec() - print(f'spec_a = {ub.urepr(spec_a, nl=3)}') - print(f'spec_b = {ub.urepr(spec_b, nl=3)}') - rd.summarize_instances() +DEVELOPER_DETAILED_DIFF_ANALYSIS = True +if DEVELOPER_DETAILED_DIFF_ANALYSIS: + for rd in ub.ProgIter(rundiffs, desc='drill down', verbose=3): + # rd.summary(level=0) + # rd.summary() + a = rd.a + b = rd.b + rd = HelmRunDiff(run_a=a, run_b=b, a_name='HELM', b_name='KWDG') + + summary = rd.summary_dict() + core_agreement = summary['value_agreement']['by_class']['core'] + + if 0: + idx = a.stat_index(drop_zero_count=True, require_mean=True) + core_a = pd.DataFrame( + {k: m for k, m in idx.items() if m.metric_class == 'core'}.values() + ) + idx = b.stat_index(drop_zero_count=True, require_mean=True) + core_b = pd.DataFrame( + {k: m for k, m in idx.items() if m.metric_class == 'core'}.values() + ) + spec_a = rd.a.run_spec() + spec_b = rd.b.run_spec() + print(f'spec_a = {ub.urepr(spec_a, nl=3)}') + print(f'spec_b = {ub.urepr(spec_b, nl=3)}') + rd.summarize_instances() df = pd.DataFrame(sankey_rows) -# df = df[df['attempt_status'] == 'compared'] +df_comp = df[df['attempt_status'] == 'compared'] print(df.value_counts(['benchmark_name', 'reproduced_step1'])) print( df.value_counts( @@ -444,6 +449,26 @@ def drain(self, timeout=0.0): print(df.value_counts(['attempt_status', 'agreement_bucket']).sort_index()) +def diagnose_status(row): + if row['attempt_status'] == 'not attempted': + return None + mismatch = set() + print(f'row={row}') + if row['spec_status'].split(' ')[-1] == 'mismatch': + mismatch |= {'run_spec'} + if row['scenario_status'].split(' ')[-1] == 'mismatch': + mismatch |= {'scenario_spec'} + # if row['scenario_status'].split(' ')[-1] == 'mismatch': + # mismatch |= {'stats_name'} + if not mismatch: + return 'specs match' + else: + return 'mismatch: ' + ', '.join(sorted(mismatch)) + +df['spec_diagnostic'] = [diagnose_status(row) for _, row in df.iterrows()] +df['spec_diagnostic'].value_counts() + + def attempt_status(row: dict[str, object]) -> str: return ( 'attempted' if row.get('reproduced_step1', False) else 'not_attempted' @@ -471,7 +496,7 @@ def agreement_label(row: dict[str, object]) -> str: df['core_iou'] = (df['comparable_core'] - df['mismatched_core']) / df[ 'comparable_core' ] -core_iou_bucket = make_bucket_fn(CORE_IOU_BINS, endpoints_as_strict_buckets=True) +core_iou_bucket = make_bucket_fn(CORE_IOU_BINS, endpoints_as_strict_buckets=True, nan_label='no comparable metrics') df['core_iou_bucket'] = df['core_iou'].map(core_iou_bucket) print(df['core_iou_bucket'].value_counts()) @@ -481,27 +506,28 @@ def agreement_label(row: dict[str, object]) -> str: # Sankey plan: same skeleton, but add a core IoU bucket stage -root = sankey.Root('All Attempts') -# When we get the data if one of the names isn't available, we handle it by -# dynamically adding it, but here we can use the result object to specify -# cases. -splits = root.split(by='attempt_status') -compared = splits.add_case(value='compared') -failcase = splits.add_case(value='not attempted') -failcase.set_label('Failed') +# root = sankey.Root('All Attempts') +# # When we get the data if one of the names isn't available, we handle it by +# # dynamically adding it, but here we can use the result object to specify +# # cases. +# splits = root.split(by='attempt_status') +# compared = splits.add_case(value='compared') +# failcase = splits.add_case(value='not attempted') +# failcase.set_label('Failed') -bench_group = compared.group(by='benchmark_name', name='benchmark') -bench_group.group(by='core_iou_bucket') +# bench_group = compared.group(by='benchmark_name', name='benchmark') +# bench_group.group(by='core_iou_bucket') -# Note it should alway be possible to put "benchmarks" before attempt status like: -root = sankey.Root('All Attempts') -bench_group = root.group(by='benchmark_name', name='benchmark') -splits = bench_group.split(by='attempt_status') -compared = splits.cases['compared'] # behaves like a defaultdict -compared.group(by='core_iou_bucket') +# # Note it should alway be possible to put "benchmarks" before attempt status like: +# root = sankey.Root('All Attempts') +# bench_group = root.group(by='benchmark_name', name='benchmark') +# splits = bench_group.split(by='attempt_status') +# splits['not attempted'].set_label('Failed') +# compared = splits.cases['compared'] # behaves like a defaultdict -splits['not attempted'].set_label('Failed') +# compared.group(by='spec_diagnostic') +# compared.group(by='core_iou_bucket') # --- @@ -510,7 +536,6 @@ def agreement_label(row: dict[str, object]) -> str: from magnet.utils import sankey_builder root = sankey_builder.Root() - bench_groups = root.group(by='benchmark_name') rungroup = bench_groups.group(by='attempt_status') @@ -519,7 +544,9 @@ def agreement_label(row: dict[str, object]) -> str: unrun_node = rungroup['not attempted'] unrun_node.label = 'Failed' -compared_node.group(by='core_iou_bucket') +compared_node \ + .group(by='core_iou_bucket') + # .group(by='spec_diagnostic') \ # plan = sankey.Plan( # sankey.Root(f'Attempted Runs n={len(df)}'), @@ -538,17 +565,18 @@ def agreement_label(row: dict[str, object]) -> str: print(G.summarize(max_edges=150)) G.nodes['CONST']['label'] = f'Attempted Runs n={len(df)}' -fig = G.to_plotly(title='HELM Reproduction Funnel') +# fig = G.to_plotly(title='HELM Reproduction Funnel') import plotly.graph_objects as go - node_labels, source, target, value = G._to_sankey_data() sankey = go.Sankey( + # arrangement='freeform', node=dict(label=node_labels, pad=15, thickness=18), link=dict(source=source, target=target, value=value), ) fig = go.Figure(sankey) +title = 'Title' fig.update_layout(title_text=title, font_size=14) @@ -590,7 +618,7 @@ def agreement_label(row: dict[str, object]) -> str: figb.write_image(str(fpath_b)) print(f'Wrote benchmark sankey: {fpath_b}') -if 1: +if 0: print( ub.codeblock( f""" From fd883a6f00b475d141380efdc50f362f47902a82 Mon Sep 17 00:00:00 2001 From: joncrall Date: Mon, 2 Mar 2026 14:05:30 -0500 Subject: [PATCH 06/36] Fix serialization issue --- dev/oneoff/check_results_overlap.py | 65 ++++++++++------ magnet/backends/helm/helm_run_diff.py | 107 ++++++++++++++++++++------ 2 files changed, 125 insertions(+), 47 deletions(-) diff --git a/dev/oneoff/check_results_overlap.py b/dev/oneoff/check_results_overlap.py index 654c4f9..445fdc4 100644 --- a/dev/oneoff/check_results_overlap.py +++ b/dev/oneoff/check_results_overlap.py @@ -19,6 +19,14 @@ """ helm_rows = kwutil.Yaml.load('run_details.yaml') + +for dupname, dupx in ub.find_duplicates([r['run_spec_name'] for r in helm_rows]).items(): + for idx in dupx: + helm_rows[idx] + ... + + + if 0: # Debug HelmRunAnalysis run_dir = '/data/crfm-helm-public/classic/benchmark_output/runs/v0.3.0/wikifact:k=5,subject=symptoms_and_signs,model=lmsys_vicuna-7b-v1.3' @@ -408,30 +416,39 @@ def drain(self, timeout=0.0): DEVELOPER_DETAILED_DIFF_ANALYSIS = True if DEVELOPER_DETAILED_DIFF_ANALYSIS: - for rd in ub.ProgIter(rundiffs, desc='drill down', verbose=3): - # rd.summary(level=0) - # rd.summary() - a = rd.a - b = rd.b - rd = HelmRunDiff(run_a=a, run_b=b, a_name='HELM', b_name='KWDG') - - summary = rd.summary_dict() - core_agreement = summary['value_agreement']['by_class']['core'] - - if 0: - idx = a.stat_index(drop_zero_count=True, require_mean=True) - core_a = pd.DataFrame( - {k: m for k, m in idx.items() if m.metric_class == 'core'}.values() - ) - idx = b.stat_index(drop_zero_count=True, require_mean=True) - core_b = pd.DataFrame( - {k: m for k, m in idx.items() if m.metric_class == 'core'}.values() - ) - spec_a = rd.a.run_spec() - spec_b = rd.b.run_spec() - print(f'spec_a = {ub.urepr(spec_a, nl=3)}') - print(f'spec_b = {ub.urepr(spec_b, nl=3)}') - rd.summarize_instances() + import json + with open('rundiff.jsonl', mode='a', encoding='utf8') as file: + for rd in ub.ProgIter(rundiffs, desc='drill down', verbose=3): + # rd.summary(level=0) + # rd.summary() + a = rd.a + b = rd.b + rd = HelmRunDiff(run_a=a, run_b=b, a_name='HELM', b_name='KWDG') + + summary = rd.summary_dict(level=100) + # json.dumps(summary, ensure_ascii=False) + + # list(kwutil.Json.find_unserializable(summary)) + # summary = kwutil.Json.ensure_serializable(summary) + file.write(json.dumps(summary, ensure_ascii=False) + "\n") + file.flush() # optional; good if you want progress written even if interrupted + + core_agreement = summary['value_agreement']['by_class']['core'] + + if 0: + idx = a.stat_index(drop_zero_count=True, require_mean=True) + core_a = pd.DataFrame( + {k: m for k, m in idx.items() if m.metric_class == 'core'}.values() + ) + idx = b.stat_index(drop_zero_count=True, require_mean=True) + core_b = pd.DataFrame( + {k: m for k, m in idx.items() if m.metric_class == 'core'}.values() + ) + spec_a = rd.a.run_spec() + spec_b = rd.b.run_spec() + print(f'spec_a = {ub.urepr(spec_a, nl=3)}') + print(f'spec_b = {ub.urepr(spec_b, nl=3)}') + rd.summarize_instances() df = pd.DataFrame(sankey_rows) df_comp = df[df['attempt_status'] == 'compared'] diff --git a/magnet/backends/helm/helm_run_diff.py b/magnet/backends/helm/helm_run_diff.py index c93c57f..e1126b6 100644 --- a/magnet/backends/helm/helm_run_diff.py +++ b/magnet/backends/helm/helm_run_diff.py @@ -295,6 +295,31 @@ def _fmt(x: Any) -> str: return str(x) +def _key_to_serializable(key: Any) -> Any: + """Convert various key types (dataclasses, tuples) into JSON-friendly types. + + - If object has ``as_tuple()``, use that and return a list. + - If it's a tuple, return a list (JSON will accept either but list is explicit). + - Otherwise fallback to string repr. + """ + # dataclass-like keys (InstanceStatKey, InstanceVariantKey) implement as_tuple + try: + if hasattr(key, 'as_tuple') and callable(getattr(key, 'as_tuple')): + return list(key.as_tuple()) + except Exception: + pass + if isinstance(key, tuple): + return list(key) + # lists are already JSON-safe + if isinstance(key, list): + return key + # fallback: use a stable repr + try: + return ub.urepr(key, nl=0, compact=1) + except Exception: + return str(key) + + class HelmRunDiff(ub.NiceRepr): """Compare two HELM runs. @@ -761,9 +786,9 @@ def instance_summary_dict( - agree_ratio: 1 - mismatched / comparable - agree_ratio_unperturbed / agree_ratio_perturbed * top_mismatches_by_group: - Mapping from ``(metric_class, metric_name)`` to a list of mismatch - items (sorted by |Δ|), each containing: - key, a, b, abs_delta, signed_delta + List of objects; each element has fields ``metric_class`` and + ``metric`` and an ``items`` list of mismatches (sorted by |Δ|). + Each mismatch item contains: key, a, b, abs_delta, signed_delta. Notes ----- @@ -834,6 +859,8 @@ def agrees(x: float, y: float) -> bool: 'perturbed': {'comparable': 0, 'mismatched': 0}, } + # use a temporary map for bookkeeping, but the final result will be a + # list of objects so the summary dict is JSON-serializable. grouped: dict[tuple[str, str | None], list[dict[str, Any]]] = {} for k in set_a & set_b: @@ -909,7 +936,7 @@ def agrees(x: float, y: float) -> bool: mismatched += 1 var_stats[variant]['mismatched'] += 1 item = { - 'key': k, + 'key': _key_to_serializable(k), 'a': ma, 'b': mb, 'abs_delta': abs(ma - mb), @@ -936,10 +963,20 @@ def agrees(x: float, y: float) -> bool: ), } + # convert to a JSON-friendly structure: list of group objects + group_list: list[dict[str, Any]] = [] + for (mclass, metric), items in grouped.items(): + group_list.append( + { + 'metric_class': mclass, + 'metric': metric, + 'items': items, + } + ) out = { 'coverage': cov.__dict__, 'means': means, - 'top_mismatches_by_group': grouped, + 'top_mismatches_by_group': group_list, } self._cache[cache_key] = out @@ -984,20 +1021,18 @@ def summarize_instances( f'pert={means["agree_ratio_perturbed"]:.3f})' ) - grouped: dict[tuple[str, str | None], list[dict[str, Any]]] = ( - info.get('top_mismatches_by_group', {}) or {} - ) + grouped: list[dict[str, Any]] = info.get('top_mismatches_by_group', []) or [] + # grouped is now a list of group objects; convert to list for sorting # Choose groups to show: core first, bookkeeping second - def _group_rank( - item: tuple[tuple[str, str | None], list[dict[str, Any]]], - ) -> tuple[int, float]: - (cls, _metric), items = item + def _group_rank(group: dict[str, Any]) -> tuple[int, float]: + cls = group.get('metric_class') + items = group.get('items', []) cls_rank = {'core': 0, 'bookkeeping': 1, 'untracked': 2}.get(cls, 9) max_abs = items[0]['abs_delta'] if items else 0.0 return (cls_rank, -max_abs) - groups_sorted = sorted(grouped.items(), key=_group_rank) + groups_sorted = sorted(grouped, key=_group_rank) # Decide which metric classes are eligible at this level allowed_classes = {'core'} @@ -1007,14 +1042,14 @@ def _group_rank( allowed_classes |= {'bookkeeping'} # Filter groups by allowed class - filtered = [g for g in groups_sorted if g[0][0] in allowed_classes] + filtered = [g for g in groups_sorted if g.get('metric_class') in allowed_classes] # If we filtered everything out (e.g. no core diffs), fall back to showing *something* if not filtered and groups_sorted: # Prefer untracked, then bookkeeping, then whatever exists pref_order = ['untracked', 'bookkeeping', 'core'] for cls in pref_order: - filtered = [g for g in groups_sorted if g[0][0] == cls] + filtered = [g for g in groups_sorted if g.get('metric_class') == cls] if filtered: break if not filtered: @@ -1028,12 +1063,12 @@ def _group_rank( max_groups = 6 if max_groups is not None: # Ensure we show at least some core and some bookkeeping groups when possible. - core = [g for g in groups_sorted if g[0][0] == 'core'] - book = [g for g in groups_sorted if g[0][0] == 'bookkeeping'] + core = [g for g in groups_sorted if g.get('metric_class') == 'core'] + book = [g for g in groups_sorted if g.get('metric_class') == 'bookkeeping'] other = [ g for g in groups_sorted - if g[0][0] not in {'core', 'bookkeeping'} + if g.get('metric_class') not in {'core', 'bookkeeping'} ] keep = [] keep.extend(core[: max_groups // 2]) @@ -1062,8 +1097,13 @@ def _group_rank( ) A_map = getattr(A_join, 'row_by_key', None) B_map = getattr(B_join, 'row_by_key', None) + A_getrow = getattr(A_join, 'get_row', None) + B_getrow = getattr(B_join, 'get_row', None) - for (cls, metric), items in groups_sorted: + for group in groups_sorted: + cls = group.get('metric_class') + metric = group.get('metric') + items = group.get('items', []) writer(f' [bold]top mismatches ({(cls, metric)!r}):[/bold]') for rank, item in enumerate(items[:top_n], start=1): k = item['key'] @@ -1072,9 +1112,9 @@ def _group_rank( abs_d = float(item['abs_delta']) signed_d = float(item['signed_delta']) - # Try to extract split/sub_split info from key if it is tuple-like + # Try to extract split/sub_split info from key if it is tuple/list-like split = None - if isinstance(k, tuple) and len(k) >= 5: + if (isinstance(k, tuple) or isinstance(k, list)) and len(k) >= 5: # (id, tti, pert_id, metric, split, ...) split = k[4] @@ -1097,8 +1137,29 @@ def _group_rank( and A_map is not None and B_map is not None ): - ra = A_map.get(k, None) - rb = B_map.get(k, None) + # Try to resolve row objects from the join tables. + # Item keys were serialized (lists); attempt to use the + # table's `get_row` API with a tuple form, which will + # reconstruct InstanceStatKey when appropriate. + ra = None + rb = None + if A_getrow is not None: + try: + lookup_key = tuple(k) if isinstance(k, list) else k + ra = A_getrow(lookup_key) + except Exception: + ra = None + if ra is None and A_map is not None: + ra = A_map.get(k, None) + + if B_getrow is not None: + try: + lookup_key = tuple(k) if isinstance(k, list) else k + rb = B_getrow(lookup_key) + except Exception: + rb = None + if rb is None and B_map is not None: + rb = B_map.get(k, None) rs_a = ( getattr(ra, 'request_state', None) From 3a62502c7010430711b6d40acedd902e4d764ded Mon Sep 17 00:00:00 2001 From: joncrall Date: Tue, 3 Mar 2026 15:59:08 -0500 Subject: [PATCH 07/36] Better sankey diagnostics --- dev/oneoff/check_results_overlap.py | 81 +- dev/oneoff/diagnose_reproducibility.py | 986 +++++++++++++++++++ magnet/backends/helm/helm_run_diff.py | 1099 ++++++++++++++++++++-- tests/test_helm_output_loader.py | 18 + tests/test_helm_run_diff_heavy.py | 93 ++ tests/test_helm_run_diff_serializable.py | 178 ++++ 6 files changed, 2331 insertions(+), 124 deletions(-) create mode 100644 dev/oneoff/diagnose_reproducibility.py create mode 100644 tests/test_helm_output_loader.py create mode 100644 tests/test_helm_run_diff_heavy.py create mode 100644 tests/test_helm_run_diff_serializable.py diff --git a/dev/oneoff/check_results_overlap.py b/dev/oneoff/check_results_overlap.py index 445fdc4..553ce26 100644 --- a/dev/oneoff/check_results_overlap.py +++ b/dev/oneoff/check_results_overlap.py @@ -20,11 +20,11 @@ helm_rows = kwutil.Yaml.load('run_details.yaml') -for dupname, dupx in ub.find_duplicates([r['run_spec_name'] for r in helm_rows]).items(): - for idx in dupx: - helm_rows[idx] - ... - +# duplicates = dict(ub.find_duplicates([r['run_spec_name'] for r in helm_rows])) +# for dupname, dupx in duplicates: +# for idx in dupx: +# row = helm_rows[idx] +# print(f'row = {ub.urepr(row, nl=1)}') if 0: @@ -68,6 +68,10 @@ } ) kwdagger_lut = {r['run_spec_name']: r for r in kwdagger_rows} + +kwd_duplicates = dict(ub.find_duplicates([r['run_spec_name'] for r in kwdagger_rows])) +assert not len(kwd_duplicates) + print(f'len(helm_rows)={len(helm_rows)}') print(f'len(kwdagger_rows)={len(kwdagger_rows)}') @@ -416,39 +420,40 @@ def drain(self, timeout=0.0): DEVELOPER_DETAILED_DIFF_ANALYSIS = True if DEVELOPER_DETAILED_DIFF_ANALYSIS: - import json - with open('rundiff.jsonl', mode='a', encoding='utf8') as file: - for rd in ub.ProgIter(rundiffs, desc='drill down', verbose=3): - # rd.summary(level=0) - # rd.summary() - a = rd.a - b = rd.b - rd = HelmRunDiff(run_a=a, run_b=b, a_name='HELM', b_name='KWDG') - - summary = rd.summary_dict(level=100) - # json.dumps(summary, ensure_ascii=False) - - # list(kwutil.Json.find_unserializable(summary)) - # summary = kwutil.Json.ensure_serializable(summary) - file.write(json.dumps(summary, ensure_ascii=False) + "\n") - file.flush() # optional; good if you want progress written even if interrupted - - core_agreement = summary['value_agreement']['by_class']['core'] - - if 0: - idx = a.stat_index(drop_zero_count=True, require_mean=True) - core_a = pd.DataFrame( - {k: m for k, m in idx.items() if m.metric_class == 'core'}.values() - ) - idx = b.stat_index(drop_zero_count=True, require_mean=True) - core_b = pd.DataFrame( - {k: m for k, m in idx.items() if m.metric_class == 'core'}.values() - ) - spec_a = rd.a.run_spec() - spec_b = rd.b.run_spec() - print(f'spec_a = {ub.urepr(spec_a, nl=3)}') - print(f'spec_b = {ub.urepr(spec_b, nl=3)}') - rd.summarize_instances() + # import json + # with open('rundiff.jsonl', mode='a', encoding='utf8') as file: + for rd in ub.ProgIter(rundiffs, desc='drill down', verbose=3): + # rd.summary(level=0) + # rd.summary() + a = rd.a + b = rd.b + rd = HelmRunDiff(run_a=a, run_b=b, a_name='HELM', b_name='KWDG') + + summary = rd.summary_dict(level=100) + # json.dumps(summary, ensure_ascii=False) + raise Exception + + # list(kwutil.Json.find_unserializable(summary)) + # summary = kwutil.Json.ensure_serializable(summary) + # file.write(json.dumps(summary, ensure_ascii=False) + "\n") + # file.flush() # optional; good if you want progress written even if interrupted + + core_agreement = summary['value_agreement']['by_class']['core'] + + if 0: + idx = a.stat_index(drop_zero_count=True, require_mean=True) + core_a = pd.DataFrame( + {k: m for k, m in idx.items() if m.metric_class == 'core'}.values() + ) + idx = b.stat_index(drop_zero_count=True, require_mean=True) + core_b = pd.DataFrame( + {k: m for k, m in idx.items() if m.metric_class == 'core'}.values() + ) + spec_a = rd.a.run_spec() + spec_b = rd.b.run_spec() + print(f'spec_a = {ub.urepr(spec_a, nl=3)}') + print(f'spec_b = {ub.urepr(spec_b, nl=3)}') + rd.summarize_instances() df = pd.DataFrame(sankey_rows) df_comp = df[df['attempt_status'] == 'compared'] diff --git a/dev/oneoff/diagnose_reproducibility.py b/dev/oneoff/diagnose_reproducibility.py new file mode 100644 index 0000000..ea062aa --- /dev/null +++ b/dev/oneoff/diagnose_reproducibility.py @@ -0,0 +1,986 @@ +""" +Script-style reproducibility diagnostics for HELM vs KWDG runs. + +This is intentionally notebook-like and hard-coded for local iteration. +It writes a case-by-case JSONL stream as it runs, plus a summary JSON report. +""" + +from __future__ import annotations + +import datetime as datetime_mod +import json +from collections import Counter, defaultdict +from typing import Any + +import kwutil +import ubelt as ub + +from magnet.backends.helm.helm_outputs import HelmOutputs +from magnet.backends.helm.helm_outputs import HelmRun +from magnet.backends.helm.helm_run_diff import HelmRunDiff +from magnet.utils import sankey_builder + + +HELM_DETAILS_FPATH = ub.Path('run_details.yaml') +# Use repository-local results by default to avoid HOME-dependent ambiguity. +KWDG_RESULTS_DPATH = (ub.Path.cwd() / 'results/helm').resolve() +REPORT_DPATH = ub.Path('dev/oneoff/repro_reports').ensuredir() + + +def parse_helm_version(version_text: str) -> tuple[int, ...]: + """ + Parse version strings like "v0.3.0" into sortable tuples. + """ + text = str(version_text).strip() + if text.startswith('v'): + text = text[1:] + parts = [] + for tok in text.split('.'): + if tok.isdigit(): + parts.append(int(tok)) + else: + parts.append(0) + return tuple(parts) + + +def parse_helm_run_dir(run_dir: str) -> dict[str, str]: + """Parse HELM public run_dir path components.""" + p = ub.Path(run_dir) + parts = list(p.parts) + out = { + 'helm_suite_name': 'unknown', + 'helm_version': 'unknown', + 'run_leaf': p.name, + } + try: + idx = parts.index('benchmark_output') + except ValueError: + idx = -1 + if idx >= 1: + out['helm_suite_name'] = str(parts[idx - 1]) + if idx >= 0 and (idx + 2) < len(parts): + # .../benchmark_output/runs// + out['helm_version'] = str(parts[idx + 2]) + else: + out['helm_version'] = str(p.parent.name) + return out + + +def infer_benchmark_group( + run_spec_name: str | None, + scenario_class: str | None = None, +) -> str: + """Infer benchmark/scenario family key (e.g. babi_qa, mmlu, raft).""" + text = (run_spec_name or '').strip() + if text: + idxs = [i for i in [text.find(':'), text.find(',')] if i >= 0] + if idxs: + head = text[: min(idxs)].strip() + else: + head = text + if head: + return head + if scenario_class: + base = str(scenario_class).split('.')[-1] + if base.endswith('Scenario'): + base = base[:-8] + if base: + return base + return 'unknown' + + +def select_latest_helm_rows(helm_rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + by_name = defaultdict(list) + for row in helm_rows: + run_dir = ub.Path(row['run_dir']) + parsed = parse_helm_run_dir(str(run_dir)) + version = parsed['helm_version'] + row = dict(row) + row['helm_version'] = version + row['helm_version_tuple'] = parse_helm_version(version) + row['suite_name'] = parsed['helm_suite_name'] + row['benchmark_name'] = parsed['helm_suite_name'] + row['benchmark_group'] = infer_benchmark_group( + row.get('run_spec_name', None), + row.get('scenario_class', None), + ) + by_name[row['run_spec_name']].append(row) + + latest_rows = [] + for _, items in by_name.items(): + best = max( + items, + key=lambda r: ( + r['helm_version_tuple'], + str(r['run_dir']), + ), + ) + latest_rows.append(best) + + latest_rows = sorted( + latest_rows, + key=lambda r: (r.get('benchmark_name', ''), r.get('run_spec_name', '')), + ) + return latest_rows + + +def load_kwdg_rows(results_dpath: ub.Path) -> tuple[list[dict[str, Any]], dict[str, dict[str, Any]]]: + finished_jobs = sorted(results_dpath.glob('*/DONE')) + rows = [] + for fpath in ub.ProgIter(finished_jobs, desc='load kwdg runs'): + dpath = fpath.parent + try: + config = kwutil.Json.coerce(dpath / 'job_config.json') + run_spec_name = config['helm.run_entry'] + suites = HelmOutputs.coerce(dpath / 'benchmark_output').suites() + if not suites: + continue + runs = suites[0].runs() + if len(runs) != 1: + continue + run = runs[0] + rows.append( + { + 'dpath': str(dpath), + 'run_spec_name': run_spec_name, + 'run': run, + } + ) + except Exception: + continue + + lut = {} + dups = defaultdict(list) + for row in rows: + name = row['run_spec_name'] + if name in lut: + dups[name].append(row['dpath']) + lut[name] = row + if dups: + print(f'WARNING: found {len(dups)} duplicate KWDG run_spec_name entries') + return rows, lut + + +def aggregate_report(rows: list[dict[str, Any]]) -> dict[str, Any]: + status_counter = Counter() + diagnosis_counter = Counter() + reason_counter = Counter() + reason_by_priority = defaultdict(Counter) + + for row in rows: + status = row.get('status', 'unknown') + status_counter[status] += 1 + if status != 'compared': + continue + + diag = row.get('diagnosis', {}) or {} + diagnosis_counter[diag.get('label', 'unknown')] += 1 + + for reason in diag.get('reasons', []) or []: + name = reason.get('name', 'unknown') + priority = reason.get('priority', None) + reason_counter[name] += 1 + reason_by_priority[str(priority)][name] += 1 + + out = { + 'n_rows': len(rows), + 'status_counts': dict(status_counter), + 'diagnosis_label_counts': dict(diagnosis_counter), + 'reason_counts': dict(reason_counter), + 'reason_counts_by_priority': { + p: dict(c) for p, c in reason_by_priority.items() + }, + } + return out + + +def _normalize_primary_reasons(diag: dict[str, Any]) -> list[str]: + raw = diag.get('primary_reason_names', []) or [] + if isinstance(raw, str): + return [raw] + out = [str(x) for x in raw if x is not None] + return sorted(out) + + +def _all_reason_names(diag: dict[str, Any]) -> set[str]: + reasons = diag.get('reasons', []) or [] + out = set() + for r in reasons: + if not isinstance(r, dict): + continue + name = r.get('name', None) + if name is not None: + out.add(str(name)) + # Also include primary reasons for compatibility with abbreviated records. + for name in _normalize_primary_reasons(diag): + out.add(str(name)) + return out + + +def _bucket_execution_state(reason_names: set[str]) -> str: + has_dep = 'deployment_drift' in reason_names + has_exec = 'execution_spec_drift' in reason_names + if has_dep and has_exec: + return 'deployment+execution' + if has_dep: + return 'deployment_only' + if has_exec: + return 'execution_only' + return 'none' + + +def _bucket_dataset_state(reason_names: set[str]) -> str: + has_error = 'dataset_overlap_error' in reason_names + has_membership = bool( + {'dataset_instance_drift', 'dataset_variant_drift'} & reason_names + ) + has_input_prompt = bool( + {'dataset_input_drift', 'request_prompt_drift'} & reason_names + ) + if has_error: + return 'error' + if has_membership and has_input_prompt: + return 'membership+input_prompt' + if has_membership: + return 'membership_only' + if has_input_prompt: + return 'input_prompt_only' + return 'none' + + +def _bucket_eval_state(reason_names: set[str]) -> str: + has_eval = 'evaluation_spec_drift' in reason_names + has_completion = 'completion_content_drift' in reason_names + if has_eval and has_completion: + return 'eval_spec+completion' + if has_eval: + return 'eval_spec_only' + if has_completion: + return 'completion_only' + return 'none' + + +def _bucket_core_state(reason_names: set[str]) -> str: + if 'core_metric_drift' in reason_names: + return 'core_drift' + if 'no_comparable_core_metrics' in reason_names: + return 'no_comparable_core' + if 'bookkeeping_metric_drift' in reason_names: + return 'bookkeeping_only' + return 'core_match' + + +def _benchmark_or_suite(row: dict[str, Any]) -> str: + group = row.get('benchmark_group', None) + bench = row.get('benchmark_name', None) + suite = row.get('suite_name', None) + for cand in [group, bench, suite]: + if cand is None: + continue + text = str(cand).strip() + if text and text.lower() != 'unknown': + return text + return 'unknown' + + +def _performance_bucket(row: dict[str, Any]) -> str: + """Bucket core metric agreement for plot readability.""" + va = row.get('value_agreement', {}) or {} + core = ((va.get('by_class') or {}).get('core') or {}) + agree = core.get('agree_ratio', None) + if agree is None: + return 'n/a' + try: + x = float(agree) + except Exception: + return 'n/a' + if x <= 0.0: + return '0%' + if x < 0.50: + return '0-50%' + if x < 0.75: + return '50-75%' + if x < 1.0: + # Not requested explicitly, but needed to avoid hiding these cases. + return '75-100%' + return '100%' + + +def _varying_keys(rows: list[dict[str, Any]], keys: list[str]) -> list[str]: + """Return keys that vary across rows (ignoring stages with 1 unique value).""" + out = [] + for key in keys: + vals = {r.get(key, None) for r in rows} + if len(vals) > 1: + out.append(key) + return out + + +def build_sankey_rows(case_rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + """ + Build normalized row records for diagnosis Sankey construction. + + Example: + >>> rows = [ + ... {'status': 'compared', 'diagnosis': {'label': 'reproduced', 'primary_priority': 0, 'primary_reason_names': ['no_detected_drift']}}, + ... {'status': 'compared', 'diagnosis': {'label': 'multiple_primary_reasons', 'primary_priority': 0, 'primary_reason_names': ['deployment_drift', 'execution_spec_drift']}}, + ... {'status': 'missing_kwdg_match', 'diagnosis': {'label': 'missing_kwdg_match'}}, + ... ] + >>> sk = build_sankey_rows(rows) + >>> assert len(sk) == 3 + >>> assert sk[0]['repro_outcome'] == 'reproduced_core' + >>> assert sk[1]['execution_state'] == 'deployment+execution' + >>> assert sk[2]['repro_outcome'] == 'not_compared' + >>> assert sk[0]['benchmark_or_suite'] == 'unknown' + >>> assert sk[0]['performance_bucket'] == 'n/a' + """ + sink_rows = [] + for row in case_rows: + status = str(row.get('status', 'unknown')) + diag = row.get('diagnosis', {}) or {} + label = str(diag.get('label', 'unknown')) + reason_names = _all_reason_names(diag) + reasons = _normalize_primary_reasons(diag) + if reasons: + reason_bucket = ' + '.join(reasons) + else: + reason_bucket = label + + p = diag.get('primary_priority', None) + if isinstance(p, int): + priority_bucket = f'P{p}' + else: + priority_bucket = 'P?' + + core_state = _bucket_core_state(reason_names) + execution_state = _bucket_execution_state(reason_names) + dataset_state = _bucket_dataset_state(reason_names) + eval_state = _bucket_eval_state(reason_names) + perf_bucket = _performance_bucket(row) + benchmark_or_suite = _benchmark_or_suite(row) + + if status != 'compared': + repro_outcome = 'not_compared' + # keep non-compared flow compact in the sankey. + priority_bucket = 'n/a' + reason_bucket = label + core_state = 'n/a' + execution_state = 'n/a' + dataset_state = 'n/a' + eval_state = 'n/a' + perf_bucket = 'n/a' + else: + if core_state in {'core_match', 'bookkeeping_only'}: + repro_outcome = 'reproduced_core' + elif core_state == 'core_drift': + repro_outcome = 'non_reproduced_core' + else: + repro_outcome = 'unknown_core' + + sink_rows.append( + { + 'status': status, + 'benchmark_or_suite': benchmark_or_suite, + 'repro_outcome': repro_outcome, + 'performance_bucket': perf_bucket, + 'core_state': core_state, + 'execution_state': execution_state, + 'dataset_state': dataset_state, + 'eval_state': eval_state, + 'diagnosis_label': label, + 'primary_priority': priority_bucket, + 'primary_reasons': reason_bucket, + } + ) + return sink_rows + + +def write_sankey_report( + case_rows: list[dict[str, Any]], + *, + report_dpath: ub.Path, + stamp: str, +) -> dict[str, Any]: + """Build and write diagnosis Sankey artifacts.""" + sankey_rows = build_sankey_rows(case_rows) + plotly_errors: list[str] = [] + stage_defs: dict[str, list[str]] = { + 'status': [ + 'compared: HELM and KWDG pair was compared.', + 'missing_kwdg_match: no matching KWDG run for this HELM run_spec.', + 'error: comparison failed.', + ], + 'bench': [ + 'inferred scenario family from HELM run_spec_name (e.g. math, mmlu, raft).', + ], + 'core%': [ + 'core metric agreement bucket (value_agreement.by_class.core.agree_ratio):', + '0% -> agree_ratio == 0.0', + '0-50% -> 0.0 < agree_ratio < 0.5', + '50-75% -> 0.5 <= agree_ratio < 0.75', + '75-100% -> 0.75 <= agree_ratio < 1.0', + '100% -> agree_ratio == 1.0', + 'n/a -> no comparable core metrics', + ], + 'exec': [ + 'execution-level run-spec drift summary.', + 'deployment+execution: deployment_drift + execution_spec_drift', + 'deployment_only: deployment_drift only', + 'execution_only: execution_spec_drift only', + 'none: no execution/deployment drift detected', + ], + 'data': [ + 'dataset/request drift summary.', + 'none: no detected dataset membership or input/prompt drift', + 'input_prompt_only: dataset_input_drift and/or request_prompt_drift', + 'membership_only: dataset_instance_drift and/or dataset_variant_drift', + 'membership+input_prompt: both membership and input/prompt drift', + 'error: dataset overlap computation failed', + ], + 'eval': [ + 'evaluation/content drift summary.', + 'none: no evaluation schema/content drift detected', + 'eval_spec_only: evaluation_spec_drift only', + 'completion_only: completion_content_drift only', + 'eval_spec+completion: both evaluation_spec_drift and completion_content_drift', + ], + 'primary': [ + 'concatenated primary_reason_names from diagnosis.', + 'Primary means lowest priority value (most upstream stage).', + ], + 'core_outcome': [ + 'high-level core reproducibility outcome.', + 'reproduced_core / non_reproduced_core / unknown_core', + ], + 'core_state': [ + 'core-metric reason bucket from diagnosis reasons.', + 'core_match / core_drift / bookkeeping_only / no_comparable_core', + ], + 'diag': [ + 'diagnosis.label (top-level diagnosis label).', + ], + 'prio': [ + 'diagnosis.primary_priority (0 is most upstream/significant).', + ], + } + + def _graph_key_text(title: str, stage_names: list[str]) -> str: + lines: list[str] = [] + lines.append('Sankey Key') + lines.append('----------') + lines.append(f'Graph: {title}') + lines.append('Stage order: ' + ' -> '.join(stage_names)) + lines.append('') + for stage in stage_names: + lines.append(f'{stage}:') + defs = stage_defs.get(stage, ['(no definition available)']) + for d in defs: + lines.append(f' {d}') + lines.append('') + return '\n'.join(lines).rstrip() + '\n' + + def _emit_graph( + *, + kind: str, + title: str, + rows: list[dict[str, Any]], + root, + stage_names: list[str], + ) -> dict[str, Any]: + graph = root.build_sankey(rows, label_fmt='{name}: {value}') + graph_summary = graph.summarize(max_edges=300) + plan_text = root.to_text() + + stem = report_dpath / f'diagnose_repro_sankey_{stamp}_{kind}' + json_fpath = stem.augment(ext='.json') + txt_fpath = stem.augment(ext='.txt') + key_fpath = stem.augment(stemsuffix='_key', ext='.txt') + html_fpath = stem.augment(ext='.html') + png_fpath = stem.augment(ext='.png') + jpg_fpath = stem.augment(ext='.jpg') + + node_labels, source, target, value = graph._to_sankey_data() + payload = kwutil.Json.ensure_serializable( + { + 'kind': kind, + 'title': title, + 'n_rows': len(rows), + 'rows': rows, + 'node_labels': node_labels, + 'source': source, + 'target': target, + 'value': value, + } + ) + json_fpath.write_text(json.dumps(payload, indent=2, ensure_ascii=False)) + txt_fpath.write_text(plan_text + '\n\n' + graph_summary + '\n') + key_fpath.write_text(_graph_key_text(title, stage_names)) + + out = { + 'json': str(json_fpath), + 'txt': str(txt_fpath), + 'key_txt': str(key_fpath), + 'html': None, + 'png': None, + 'jpg': None, + } + try: + fig = graph.to_plotly(title=title) + fig.write_html(str(html_fpath), include_plotlyjs='cdn') + out['html'] = str(html_fpath) + try: + # Keep interactive HTML defaults; apply readability tuning only + # to static JPG exports. + import plotly.graph_objects as go + + fig_static = go.Figure(fig.to_dict()) + node_labels = payload.get('node_labels', []) + max_label_len = max((len(str(x)) for x in node_labels), default=20) + export_width = min(5200, max(2800, 1700 + max_label_len * 18)) + export_height = min(5200, max(2200, 1000 + len(node_labels) * 50)) + export_scale = 4.5 + fig_static.update_traces( + node=dict(pad=20, thickness=20), + ) + fig_static.update_layout( + font_size=13, + width=export_width, + height=export_height, + margin=dict(l=30, r=30, t=70, b=30), + paper_bgcolor='white', + plot_bgcolor='white', + ) + fig_static.write_image(str(jpg_fpath), scale=export_scale) + out['jpg'] = str(jpg_fpath) + except Exception as ex: + plotly_errors.append( + f'[{kind}] unable to write sankey JPG: {ex!r}' + ) + except Exception as ex: + plotly_errors.append( + f'[{kind}] unable to write sankey HTML/images: {ex!r}' + ) + return out + + # Level 1: all attempts + root_all = sankey_builder.Root(label=f'Run Pairs n={len(sankey_rows)}') + status_group = root_all.group(by='status', name='status') + compared = status_group['compared'] + compared.label = 'status: compared' + status_group['missing_kwdg_match'].label = 'status: missing_kwdg_match' + status_group['error'].label = 'status: error' + status_group['unknown'].label = 'status: unknown' + + compared_rows = [r for r in sankey_rows if r.get('status') == 'compared'] + node = compared + key_to_name = { + 'benchmark_or_suite': 'bench', + 'performance_bucket': 'core%', + 'execution_state': 'exec', + 'dataset_state': 'data', + 'eval_state': 'eval', + 'primary_reasons': 'primary', + } + stage_order = [ + 'benchmark_or_suite', + 'performance_bucket', + 'execution_state', + 'dataset_state', + 'eval_state', + 'primary_reasons', + ] + selected_stage_keys = _varying_keys(compared_rows, stage_order) + for key in selected_stage_keys: + node = node.group(by=key, name=key_to_name[key]) + all_stage_names = ['status'] + [key_to_name[k] for k in selected_stage_keys] + + all_art = _emit_graph( + kind='all_attempts', + title='HELM/KWDG Reproducibility Diagnosis (All Attempts)', + rows=sankey_rows, + root=root_all, + stage_names=all_stage_names, + ) + + # Level 2: compared-only detailed + if compared_rows: + root_comp = sankey_builder.Root( + label=f'Compared Pairs n={len(compared_rows)}' + ) + node2 = root_comp + stage_order2 = [ + 'benchmark_or_suite', + 'performance_bucket', + 'execution_state', + 'dataset_state', + 'eval_state', + 'primary_reasons', + ] + selected_stage_keys2 = _varying_keys(compared_rows, stage_order2) + for key in selected_stage_keys2: + node2 = node2.group(by=key, name=key_to_name[key]) + compared_stage_names = [key_to_name[k] for k in selected_stage_keys2] + compared_art = _emit_graph( + kind='compared_detail', + title='HELM/KWDG Reproducibility Diagnosis (Compared Only)', + rows=compared_rows, + root=root_comp, + stage_names=compared_stage_names, + ) + else: + compared_art = { + 'json': None, + 'txt': None, + 'key_txt': None, + 'html': None, + 'png': None, + 'jpg': None, + } + + artifacts: dict[str, Any] = { + # Backward-compatible keys point to the all-attempts sankey. + 'sankey_json': all_art['json'], + 'sankey_txt': all_art['txt'], + 'sankey_key_txt': all_art['key_txt'], + 'sankey_html': all_art['html'], + 'sankey_png': all_art['png'], + 'sankey_jpg': all_art['jpg'], + # Additional detailed level. + 'sankey_compared_json': compared_art['json'], + 'sankey_compared_txt': compared_art['txt'], + 'sankey_compared_key_txt': compared_art['key_txt'], + 'sankey_compared_html': compared_art['html'], + 'sankey_compared_png': compared_art['png'], + 'sankey_compared_jpg': compared_art['jpg'], + 'sankey_compared_full_json': None, + 'sankey_compared_full_txt': None, + 'sankey_compared_full_key_txt': None, + 'sankey_compared_full_html': None, + 'sankey_compared_full_png': None, + 'sankey_compared_full_jpg': None, + 'plotly_error': (' | '.join(plotly_errors) if plotly_errors else None), + } + + # Level 3: compared-only, full (unpruned) diagnostic pipeline. + if compared_rows: + root_comp_full = sankey_builder.Root( + label=f'Compared Pairs (Full) n={len(compared_rows)}' + ) + node3 = root_comp_full + key_to_name_full = { + 'benchmark_or_suite': 'bench', + 'repro_outcome': 'core_outcome', + 'performance_bucket': 'core%', + 'core_state': 'core_state', + 'execution_state': 'exec', + 'dataset_state': 'data', + 'diagnosis_label': 'diag', + 'eval_state': 'eval', + 'primary_priority': 'prio', + 'primary_reasons': 'primary', + } + stage_order3 = [ + 'benchmark_or_suite', + 'repro_outcome', + 'performance_bucket', + 'core_state', + 'execution_state', + 'dataset_state', + 'diagnosis_label', + 'eval_state', + 'primary_priority', + 'primary_reasons', + ] + for key in stage_order3: + node3 = node3.group(by=key, name=key_to_name_full[key]) + compared_full_stage_names = [key_to_name_full[k] for k in stage_order3] + compared_full_art = _emit_graph( + kind='compared_full', + title='HELM/KWDG Reproducibility Diagnosis (Compared Full Pipeline)', + rows=compared_rows, + root=root_comp_full, + stage_names=compared_full_stage_names, + ) + artifacts['sankey_compared_full_json'] = compared_full_art['json'] + artifacts['sankey_compared_full_txt'] = compared_full_art['txt'] + artifacts['sankey_compared_full_key_txt'] = compared_full_art['key_txt'] + artifacts['sankey_compared_full_html'] = compared_full_art['html'] + artifacts['sankey_compared_full_png'] = compared_full_art['png'] + artifacts['sankey_compared_full_jpg'] = compared_full_art['jpg'] + + return kwutil.Json.ensure_serializable(artifacts) + + +def main(): + if not HELM_DETAILS_FPATH.exists(): + raise FileNotFoundError( + f'Expected HELM detail file at {HELM_DETAILS_FPATH}' + ) + + helm_rows = kwutil.Yaml.load(HELM_DETAILS_FPATH) + latest_helm_rows = select_latest_helm_rows(helm_rows) + + if not latest_helm_rows: + raise RuntimeError('No HELM rows found') + + kwdg_rows, kwdg_lut = load_kwdg_rows(KWDG_RESULTS_DPATH) + + print(f'Loaded HELM rows: all={len(helm_rows)} latest_only={len(latest_helm_rows)}') + print(f'Loaded KWDG rows: {len(kwdg_rows)}') + + stamp = datetime_mod.datetime.now(datetime_mod.UTC).strftime('%Y%m%dT%H%M%SZ') + case_jsonl_fpath = REPORT_DPATH / f'diagnose_repro_cases_{stamp}.jsonl' + summary_json_fpath = REPORT_DPATH / f'diagnose_repro_summary_{stamp}.json' + + all_case_rows = [] + with case_jsonl_fpath.open('w', encoding='utf8') as file: + for idx, helm_row in enumerate( + ub.ProgIter(latest_helm_rows, desc='compare latest helm vs kwdg'), start=1 + ): + run_spec_name = helm_row['run_spec_name'] + kwrow = kwdg_lut.get(run_spec_name, None) + case_row = { + 'index': idx, + 'run_spec_name': run_spec_name, + 'benchmark_name': helm_row.get('benchmark_name', 'unknown'), + 'benchmark_group': helm_row.get('benchmark_group', 'unknown'), + 'suite_name': helm_row.get('suite_name', 'unknown'), + 'model_name': helm_row.get('model', None), + 'helm_version': helm_row.get('helm_version', None), + 'helm_run_dir': str(helm_row['run_dir']), + 'kwdg_run_dir': None if kwrow is None else kwrow['dpath'], + } + + if kwrow is None: + case_row.update( + { + 'status': 'missing_kwdg_match', + 'diagnosis': { + 'label': 'missing_kwdg_match', + 'primary_priority': 0, + 'primary_reason_names': ['missing_kwdg_match'], + 'reasons': [ + { + 'name': 'missing_kwdg_match', + 'priority': 0, + 'details': {}, + } + ], + }, + } + ) + print( + f'[{idx:03d}] {run_spec_name} -> missing_kwdg_match' + ) + else: + try: + helm_run = HelmRun.coerce(helm_row['run_dir']) + kwdg_run = kwrow['run'] + rd = HelmRunDiff( + run_a=helm_run, + run_b=kwdg_run, + a_name='HELM', + b_name='KWDG', + ) + summary = rd.summary_dict(level=20) + diag = summary.get('diagnosis', {}) or {} + + case_row.update( + { + 'status': 'compared', + 'diagnosis': diag, + 'run_spec_semantic': summary.get( + 'run_spec_semantic', None + ), + 'scenario_semantic': summary.get( + 'scenario_semantic', None + ), + 'dataset_overlap': summary.get( + 'dataset_overlap', None + ), + 'stats_coverage_by_name': summary.get( + 'stats_coverage_by_name', None + ), + 'stats_coverage_by_name_count': summary.get( + 'stats_coverage_by_name_count', None + ), + 'value_agreement': summary.get( + 'value_agreement', None + ), + 'instance_value_agreement': summary.get( + 'instance_value_agreement', None + ), + } + ) + primary = diag.get('label', 'unknown') + p = diag.get('primary_priority', None) + primary_names = diag.get('primary_reason_names', []) or [] + print( + f'[{idx:03d}] {run_spec_name} -> {primary} ' + f'(p={p}, primary_reasons={primary_names})' + ) + reasons = diag.get('reasons', []) or [] + for reason in reasons: + if reason.get('name') == 'deployment_drift': + det = reason.get('details', {}) or {} + print( + ' deployment: ' + f'{det.get("a_value", None)!r} -> {det.get("b_value", None)!r}' + ) + except Exception as ex: + case_row.update( + { + 'status': 'error', + 'error': repr(ex), + 'diagnosis': { + 'label': 'comparison_error', + 'primary_priority': 0, + 'primary_reason_names': ['comparison_error'], + 'reasons': [ + { + 'name': 'comparison_error', + 'priority': 0, + 'details': {'error': repr(ex)}, + } + ], + }, + } + ) + print(f'[{idx:03d}] {run_spec_name} -> ERROR: {ex!r}') + + case_row = kwutil.Json.ensure_serializable(case_row) + file.write(json.dumps(case_row, ensure_ascii=False) + '\n') + file.flush() + all_case_rows.append(case_row) + + summary_report = { + 'report_case_jsonl': str(case_jsonl_fpath), + 'report_summary_json': str(summary_json_fpath), + 'generated_utc': stamp, + 'inputs': { + 'helm_detail_fpath': str(HELM_DETAILS_FPATH), + 'kwdg_results_dpath': str(KWDG_RESULTS_DPATH), + 'n_helm_rows_all': len(helm_rows), + 'n_helm_rows_latest': len(latest_helm_rows), + 'n_kwdg_rows': len(kwdg_rows), + }, + 'aggregate': aggregate_report(all_case_rows), + } + try: + sankey_artifacts = write_sankey_report( + all_case_rows, report_dpath=REPORT_DPATH, stamp=stamp + ) + except Exception as ex: + sankey_artifacts = { + 'sankey_json': None, + 'sankey_txt': None, + 'sankey_key_txt': None, + 'sankey_html': None, + 'sankey_png': None, + 'sankey_jpg': None, + 'sankey_compared_json': None, + 'sankey_compared_txt': None, + 'sankey_compared_key_txt': None, + 'sankey_compared_html': None, + 'sankey_compared_png': None, + 'sankey_compared_jpg': None, + 'sankey_compared_full_json': None, + 'sankey_compared_full_txt': None, + 'sankey_compared_full_key_txt': None, + 'sankey_compared_full_html': None, + 'sankey_compared_full_png': None, + 'sankey_compared_full_jpg': None, + 'plotly_error': f'failed to build sankey report: {ex!r}', + } + summary_report['artifacts'] = sankey_artifacts + + summary_report = kwutil.Json.ensure_serializable(summary_report) + summary_json_fpath.write_text( + json.dumps(summary_report, indent=2, ensure_ascii=False) + ) + + print('---') + print(f'Wrote case report: {case_jsonl_fpath}') + print(f'Wrote summary report: {summary_json_fpath}') + if sankey_artifacts.get('sankey_json', None): + print(f'Wrote sankey JSON: {sankey_artifacts["sankey_json"]}') + if sankey_artifacts.get('sankey_txt', None): + print(f'Wrote sankey TXT: {sankey_artifacts["sankey_txt"]}') + if sankey_artifacts.get('sankey_html', None): + print(f'Wrote sankey HTML: {sankey_artifacts["sankey_html"]}') + if sankey_artifacts.get('sankey_png', None): + print(f'Wrote sankey PNG: {sankey_artifacts["sankey_png"]}') + if sankey_artifacts.get('sankey_jpg', None): + print(f'Wrote sankey JPG: {sankey_artifacts["sankey_jpg"]}') + if sankey_artifacts.get('sankey_key_txt', None): + print(f'Wrote sankey key: {sankey_artifacts["sankey_key_txt"]}') + if sankey_artifacts.get('sankey_compared_json', None): + print( + f'Wrote compared-detail sankey JSON: ' + f'{sankey_artifacts["sankey_compared_json"]}' + ) + if sankey_artifacts.get('sankey_compared_txt', None): + print( + f'Wrote compared-detail sankey TXT: ' + f'{sankey_artifacts["sankey_compared_txt"]}' + ) + if sankey_artifacts.get('sankey_compared_html', None): + print( + f'Wrote compared-detail sankey HTML: ' + f'{sankey_artifacts["sankey_compared_html"]}' + ) + if sankey_artifacts.get('sankey_compared_png', None): + print( + f'Wrote compared-detail sankey PNG: ' + f'{sankey_artifacts["sankey_compared_png"]}' + ) + if sankey_artifacts.get('sankey_compared_jpg', None): + print( + f'Wrote compared-detail sankey JPG: ' + f'{sankey_artifacts["sankey_compared_jpg"]}' + ) + if sankey_artifacts.get('sankey_compared_key_txt', None): + print( + f'Wrote compared-detail sankey key: ' + f'{sankey_artifacts["sankey_compared_key_txt"]}' + ) + if sankey_artifacts.get('sankey_compared_full_json', None): + print( + f'Wrote compared-full sankey JSON: ' + f'{sankey_artifacts["sankey_compared_full_json"]}' + ) + if sankey_artifacts.get('sankey_compared_full_txt', None): + print( + f'Wrote compared-full sankey TXT: ' + f'{sankey_artifacts["sankey_compared_full_txt"]}' + ) + if sankey_artifacts.get('sankey_compared_full_html', None): + print( + f'Wrote compared-full sankey HTML: ' + f'{sankey_artifacts["sankey_compared_full_html"]}' + ) + if sankey_artifacts.get('sankey_compared_full_png', None): + print( + f'Wrote compared-full sankey PNG: ' + f'{sankey_artifacts["sankey_compared_full_png"]}' + ) + if sankey_artifacts.get('sankey_compared_full_jpg', None): + print( + f'Wrote compared-full sankey JPG: ' + f'{sankey_artifacts["sankey_compared_full_jpg"]}' + ) + if sankey_artifacts.get('sankey_compared_full_key_txt', None): + print( + f'Wrote compared-full sankey key: ' + f'{sankey_artifacts["sankey_compared_full_key_txt"]}' + ) + if sankey_artifacts.get('plotly_error', None): + print(f'Sankey note: {sankey_artifacts["plotly_error"]}') + print( + 'Diagnosis label counts: ' + + ub.urepr(summary_report['aggregate']['diagnosis_label_counts'], nl=0) + ) + + +if __name__ == '__main__': + main() diff --git a/magnet/backends/helm/helm_run_diff.py b/magnet/backends/helm/helm_run_diff.py index e1126b6..194fe6d 100644 --- a/magnet/backends/helm/helm_run_diff.py +++ b/magnet/backends/helm/helm_run_diff.py @@ -17,86 +17,47 @@ xdoctest -m magnet.backends.helm.helm_run_diff __doc__ Example: - >>> import ubelt as ub - >>> import kwutil - >>> from magnet.backends.helm.helm_outputs import HelmRun + >>> import json + >>> from magnet.backends.helm.helm_run_analysis import HelmRunAnalysis >>> from magnet.backends.helm.helm_run_diff import HelmRunDiff - - >>> run_a = HelmRun.demo() - >>> dpath = ub.Path.appdir('magnet/tests/helm/helm_run_diff').delete().ensuredir() - - >>> # --- Case 1: identical copy -> perfect agreement ----------------- - >>> same_path = dpath / (run_a.path.name + '_same') - >>> run_a.path.copy(same_path) - >>> run_b = HelmRun(same_path) - - >>> rd = HelmRunDiff(run_a, run_b, a_name='orig', b_name='same') - >>> info = rd.summary_dict(level=10) - >>> assert info['run_spec_dict_ok'] is True - >>> assert info['scenario_ok'] in {True, None} - >>> assert info['value_agreement']['overall']['mismatched'] == 0 - >>> assert info['value_agreement']['overall']['agree_ratio'] == 1.0 - - >>> line = rd.summary_text(level=0) - >>> assert 'orig' in line and 'same' in line - >>> rd.summary(level=1000) - - >>> # --- Case 2: perturb a single RUN-level stat mean ---------------- - >>> stats_path = dpath / (run_a.path.name + '_statsmod') - >>> run_a.path.copy(stats_path) - >>> stat_fpath = stats_path / 'stats.json' - >>> stats = kwutil.Json.loads(stat_fpath.read_text()) - >>> old_mean = float(stats[0].get('mean', 0.0)) - >>> stats[0]['mean'] = old_mean + 1.23 - >>> stat_fpath.write_text(kwutil.Json.dumps(stats)) - - >>> run_b2 = HelmRun(stats_path) - >>> rd2 = HelmRunDiff(run_a, run_b2, a_name='orig', b_name='stats+1.23') - >>> info2 = rd2.summary_dict(level=10) - >>> assert info2['value_agreement']['overall']['mismatched'] >= 1 - >>> rd.summary(level=1000) - - >>> # --- Case 3: perturb ONE per-instance stat mean ------------------ - >>> inst_path = dpath / (run_a.path.name + '_perinstmod') - >>> run_a.path.copy(inst_path) - >>> pi_fpath = inst_path / 'per_instance_stats.json' - >>> if pi_fpath.exists(): - ... perinst = kwutil.Json.loads(pi_fpath.read_text()) - ... # deterministically modify the first mean-bearing stat for the first entry - ... ei, sj = 0, None - ... for j, s in enumerate(perinst[ei]['stats']): - ... if int(s.get('count', 0) or 0) and ('mean' in s): - ... sj = j - ... break - ... assert sj is not None - ... old = float(perinst[ei]['stats'][sj]['mean']) - ... perinst[ei]['stats'][sj]['mean'] = old + 9.0 - ... pi_fpath.write_text(kwutil.Json.dumps(perinst)) - ... run_bi = HelmRun(inst_path) - ... rd_i = HelmRunDiff(run_a, run_bi, a_name='orig', b_name='perinst+9') - ... inst_info = rd_i.instance_summary_dict(top_n=5) - ... assert inst_info['means']['mismatched'] >= 1 - ... rd_i.summary(level=1000) - - >>> # --- Case 4: run spec diff ---------------- - >>> new_dpath = dpath / (run_a.path.name + '_runspec_mod') - >>> run_a.path.copy(new_dpath) - >>> spec_fpath = new_dpath / 'run_spec.json' - >>> run_spec = kwutil.Json.loads(spec_fpath.read_text()) - >>> run_spec['adapter_spec']['model_deployment'] = 'someotherdeploy/gpt2' - >>> spec_fpath.write_text(kwutil.Json.dumps(run_spec)) - >>> run_b4 = HelmRun(new_dpath) - >>> rd = HelmRunDiff(run_a, run_b4, a_name='orig', b_name='runspec_mod') - >>> rd.summary(level=1000) - >>> info = rd.summary_dict(level=10) - >>> assert not info['run_spec_dict_ok'] + >>> class _DummyJoined: + ... def __init__(self): + ... self.row_by_key = {} + ... def __iter__(self): + ... return iter(self.row_by_key.values()) + >>> def _ana(run_spec, stats, request_states): + ... a = HelmRunAnalysis.__new__(HelmRunAnalysis) + ... a._raw_cache = {} + ... a._cache = {} + ... a.run = None + ... a.name = None + ... a.run_spec = lambda: run_spec + ... a.scenario = lambda: {'class_name': 'ToyScenario', 'output_path': 'tmp/a'} + ... a.scenario_state = lambda: {'request_states': request_states} + ... a.stats = lambda: stats + ... a.joined_instance_stat_table = lambda *args, **kwargs: _DummyJoined() + ... return a + >>> rs = [{'instance': {'id': 'id1', 'split': 'test', 'input': {'text': 'Q'}}, 'train_trial_index': 0, 'request': {'prompt': 'P'}, 'result': {'completions': [{'text': 'A'}]}}] + >>> stats_a = [{'name': {'name': 'exact_match', 'split': 'test'}, 'count': 1, 'mean': 1.0}] + >>> stats_b = [{'name': {'name': 'exact_match', 'split': 'test'}, 'count': 1, 'mean': 0.0}] + >>> spec_a = {'name': 'toy', 'adapter_spec': {'model': 'm'}, 'metric_specs': [{'class_name': 'M0', 'args': {}}]} + >>> spec_b = {'name': 'toy', 'adapter_spec': {'model': 'm', 'model_deployment': 'huggingface/m'}, 'metric_specs': [{'class_name': 'M1', 'args': {}}]} + >>> rd = HelmRunDiff(_ana(spec_a, stats_a, rs), _ana(spec_b, stats_b, rs), a_name='A', b_name='B') + >>> info = rd.summary_dict(level=20) + >>> assert info['run_spec_name_ok'] is True + >>> assert info['dataset_overlap']['base_iou'] == 1.0 + >>> assert info['value_agreement']['overall']['mismatched'] == 1 + >>> assert isinstance(info['diagnosis']['label'], str) + >>> _ = json.dumps(info, allow_nan=False) """ from __future__ import annotations +import math import ubelt as ub +from collections import Counter from dataclasses import dataclass from magnet.backends.helm.util import helm_hashers from magnet.backends.helm.util import helm_metrics @@ -117,7 +78,7 @@ def _safe_float(x: Any) -> float | None: return None -def _walker_diff(a: Any, b: Any, *, max_paths: int = 12) -> list[str]: +def _walker_diff(a: Any, b: Any, *, max_paths: int = 12) -> dict[str, Any]: """ Return a dict with formatted lines for: @@ -198,6 +159,30 @@ def _truncate(lines: list[str], max_items: int) -> list[str]: return out +def _walker_diff_paths(a: Any, b: Any) -> dict[str, list[str]]: + """Return full path-level differences (untruncated), path-only. + + The output is intentionally JSON-friendly and stable for diagnostics. + """ + walker_a = ub.IndexableWalker(a) + walker_b = ub.IndexableWalker(b) + info = walker_a.diff(walker_b) + + def _format_path(path: Iterable[Any]) -> str: + return '.'.join(map(str, path)) + + unique1 = sorted(_format_path(p) for p in info.get('unique1', [])) + unique2 = sorted(_format_path(p) for p in info.get('unique2', [])) + faillist = sorted( + _format_path(d.path) for d in info.get('faillist', []) + ) + return { + 'unique1': unique1, + 'unique2': unique2, + 'faillist': faillist, + } + + def _default_writer(writer=None) -> Callable[[str], Any]: if writer is not None: return writer @@ -262,6 +247,242 @@ def _short_urepr(obj: Any, max_chars: int = 140) -> str: return _smart_truncate(s, max_chars) +def _coerce_path_token(tok: str) -> str | int: + if tok.isdigit(): + try: + return int(tok) + except Exception: + return tok + return tok + + +def _path_get(obj: Any, path: str) -> tuple[Any, bool]: + """Best-effort dotted-path getter supporting dict/list traversal.""" + cur = obj + for raw_tok in path.split('.'): + tok = _coerce_path_token(raw_tok) + if isinstance(cur, dict): + if tok in cur: + cur = cur[tok] + elif isinstance(tok, int) and str(tok) in cur: + cur = cur[str(tok)] + else: + return None, False + elif isinstance(cur, (list, tuple)): + if isinstance(tok, int) and 0 <= tok < len(cur): + cur = cur[tok] + else: + return None, False + else: + return None, False + return cur, True + + +def _path_value_examples( + a_obj: Any, + b_obj: Any, + paths: list[str], + *, + max_items: int = 20, +) -> list[dict[str, Any]]: + """Return path-level value pairs for selected diff paths.""" + examples: list[dict[str, Any]] = [] + for p in sorted(paths): + rec: dict[str, Any] = {'path': p} + va, oka = _path_get(a_obj, p) + vb, okb = _path_get(b_obj, p) + rec['a'] = va if oka else None + rec['b'] = vb if okb else None + rec['a_found'] = bool(oka) + rec['b_found'] = bool(okb) + examples.append(rec) + if len(examples) >= max_items: + break + return _json_compatible(examples) + + +def _json_compatible(obj: Any) -> Any: + """Recursively coerce to strict JSON-compatible types. + + Notably: + - tuples/sets -> lists + - non-finite floats -> None + - unknown objects -> string repr + """ + if obj is None or isinstance(obj, (str, int, bool)): + return obj + if isinstance(obj, float): + return obj if math.isfinite(obj) else None + if isinstance(obj, dict): + return {str(k): _json_compatible(v) for k, v in obj.items()} + if isinstance(obj, (list, tuple, set)): + return [_json_compatible(v) for v in obj] + try: + # common dataclass / custom key cases + if hasattr(obj, 'as_tuple') and callable(getattr(obj, 'as_tuple')): + return _json_compatible(list(obj.as_tuple())) + except Exception: + pass + try: + return ub.urepr(obj, nl=0, compact=1) + except Exception: + return str(obj) + + +def _preview_list(items: list[str], *, limit: int = 20) -> list[str]: + """Return a stable preview list with an optional '' suffix.""" + if limit <= 0 or len(items) <= limit: + return items + remain = len(items) - limit + return items[:limit] + [f'<{remain} more not shown>'] + + +_RUNSPEC_EXEC_ADAPTER_NOISE_FIELDS = { + # Added in newer HELM formats; often default/no-op in practice. + 'chain_of_thought_prefix', + 'chain_of_thought_suffix', + 'global_suffix', + 'num_trials', +} + + +def _classify_run_spec_path(path: str) -> str: + """Classify run-spec diff paths into semantic buckets.""" + if path.startswith('metric_specs') or path.startswith('groups'): + return 'evaluation' + if path.startswith('adapter_spec.'): + parts = path.split('.') + field = parts[1] if len(parts) > 1 else '' + if field in _RUNSPEC_EXEC_ADAPTER_NOISE_FIELDS: + return 'nonsemantic' + return 'execution' + if path.startswith('scenario_spec') or path.startswith('data_augmenter_spec'): + return 'execution' + if path in {'name'}: + return 'nonsemantic' + return 'other' + + +def _classify_scenario_path(path: str) -> str: + """Classify scenario diff paths into semantic buckets.""" + # scenario.output_path is environment-local and should not affect content + if path == 'output_path' or path.endswith('.output_path'): + return 'nonsemantic' + return 'semantic' + + +def _canonicalize_metric_spec_for_semantic_diff(metric_spec: Any) -> Any: + """Normalize one metric spec for order-insensitive semantic comparison.""" + if not isinstance(metric_spec, dict): + return helm_hashers.canonicalize_for_hashing(metric_spec) + out = { + 'class_name': metric_spec.get('class_name', None), + 'args': helm_hashers.canonicalize_for_hashing( + metric_spec.get('args', None) + ), + } + return out + + +def _canonicalize_run_spec_for_semantic_diff(run_spec: dict[str, Any]) -> dict[str, Any]: + """Canonicalize run_spec with order-insensitive handling for select lists.""" + spec = helm_hashers.canonicalize_for_hashing(run_spec) + if not isinstance(spec, dict): + return {'_invalid_spec': spec} + spec = dict(spec) + + metric_specs = spec.get('metric_specs', None) + if isinstance(metric_specs, list): + canon_items = [ + _canonicalize_metric_spec_for_semantic_diff(ms) + for ms in metric_specs + ] + canon_items = sorted( + canon_items, key=lambda x: helm_hashers.stable_hash36(x) + ) + spec['metric_specs'] = canon_items + + groups = spec.get('groups', None) + if isinstance(groups, list): + spec['groups'] = sorted(groups, key=lambda x: str(x)) + + return spec + + +def _metric_specs_multiset_delta( + metric_specs_a: Any, + metric_specs_b: Any, + *, + short_hash: int = 12, + max_items: int = 20, +) -> dict[str, Any]: + """Order-insensitive multiset delta for run_spec.metric_specs.""" + specs_a = metric_specs_a if isinstance(metric_specs_a, list) else [] + specs_b = metric_specs_b if isinstance(metric_specs_b, list) else [] + + def _make_id(ms: Any) -> tuple[str, dict[str, Any]]: + canon = _canonicalize_metric_spec_for_semantic_diff(ms) + sid = helm_hashers.stable_hash36(canon)[:short_hash] + if isinstance(canon, dict): + class_name = canon.get('class_name', None) + args = canon.get('args', None) + else: + class_name = None + args = canon + rec = { + 'id': sid, + 'class_name': class_name, + 'args': args, + 'preview': _short_urepr(canon, max_chars=160), + } + return sid, rec + + id_to_rec: dict[str, dict[str, Any]] = {} + a_ids: list[str] = [] + b_ids: list[str] = [] + for ms in specs_a: + sid, rec = _make_id(ms) + id_to_rec.setdefault(sid, rec) + a_ids.append(sid) + for ms in specs_b: + sid, rec = _make_id(ms) + id_to_rec.setdefault(sid, rec) + b_ids.append(sid) + + a_counter = Counter(a_ids) + b_counter = Counter(b_ids) + keys = sorted(set(a_counter) | set(b_counter)) + added = [] + removed = [] + for sid in keys: + ca = a_counter.get(sid, 0) + cb = b_counter.get(sid, 0) + if cb > ca: + added.append(id_to_rec[sid] | {'count': cb - ca}) + if ca > cb: + removed.append(id_to_rec[sid] | {'count': ca - cb}) + + added = sorted(added, key=lambda r: (str(r.get('class_name')), r['id'])) + removed = sorted(removed, key=lambda r: (str(r.get('class_name')), r['id'])) + return _json_compatible( + { + 'n_a': len(specs_a), + 'n_b': len(specs_b), + 'n_added': sum(r['count'] for r in added), + 'n_removed': sum(r['count'] for r in removed), + 'added': _preview_list( + [ub.urepr(r, nl=0, compact=1) for r in added], limit=max_items + ), + 'removed': _preview_list( + [ub.urepr(r, nl=0, compact=1) for r in removed], limit=max_items + ), + 'added_structured': added[:max_items], + 'removed_structured': removed[:max_items], + 'equal_as_multiset': (len(added) == 0 and len(removed) == 0), + } + ) + + @dataclass(frozen=True) class Coverage: """Coverage bookkeeping for two key-sets.""" @@ -320,6 +541,204 @@ def _key_to_serializable(key: Any) -> Any: return str(key) +def dataset_overlap_from_request_states( + request_states_a: list[dict[str, Any]], + request_states_b: list[dict[str, Any]], + *, + short_hash: int = 16, + max_examples: int = 5, +) -> dict[str, Any]: + """Compare two request_state lists at dataset/prompt/completion level. + + This is a pure function used by :meth:`HelmRunDiff.dataset_overlap_summary`. + + Example: + >>> rs_a = [ + ... { + ... 'instance': {'id': 'id1', 'split': 'test', 'input': {'text': 'Q1'}}, + ... 'train_trial_index': 0, + ... 'request': {'prompt': 'P1'}, + ... 'result': {'completions': [{'text': 'A1'}]}, + ... }, + ... { + ... 'instance': { + ... 'id': 'id1', 'split': 'test', 'input': {'text': 'Q1'}, + ... 'perturbation': {'name': 'dialect', 'prob': 1.0}, + ... }, + ... 'train_trial_index': 0, + ... 'request': {'prompt': 'P1-d'}, + ... 'result': {'completions': [{'text': 'A1d'}]}, + ... }, + ... ] + >>> rs_b = [ + ... { + ... 'instance': {'id': 'id1', 'split': 'test', 'input': {'text': 'Q1'}}, + ... 'train_trial_index': 0, + ... 'request': {'prompt': 'P1x'}, + ... 'result': {'completions': [{'text': 'A1'}]}, + ... }, + ... ] + >>> info = dataset_overlap_from_request_states(rs_a, rs_b, max_examples=2) + >>> assert info['base_coverage']['n_isect'] == 1 + >>> assert info['variant_coverage']['only_a'] == 1 + >>> assert info['content_equality']['prompt']['equal_ratio'] == 0.0 + >>> assert isinstance(info['mismatch_examples']['prompt'], list) + """ + + def _coerce_int(x: Any) -> int | None: + try: + if x is None: + return None + if isinstance(x, bool): + return int(x) + if isinstance(x, int): + return x + if isinstance(x, float) and x.is_integer(): + return int(x) + if isinstance(x, str) and x.isdigit(): + return int(x) + except Exception: + pass + return None + + def _base_key(rs: dict[str, Any]) -> tuple[Any, ...]: + inst = rs.get('instance') or {} + return ( + inst.get('id', None), + _coerce_int(rs.get('train_trial_index', None)), + inst.get('split', None), + ) + + def _variant_key(rs: dict[str, Any]) -> tuple[Any, ...]: + inst = rs.get('instance') or {} + pid = helm_hashers.perturbation_id( + inst.get('perturbation', None), short_hash=short_hash + ) + return _base_key(rs) + (pid,) + + def _index_unique( + rows: list[dict[str, Any]], key_fn + ) -> tuple[dict[tuple[Any, ...], dict[str, Any]], int]: + out: dict[tuple[Any, ...], dict[str, Any]] = {} + duplicates = 0 + for rs in rows: + k = key_fn(rs) + if k in out: + duplicates += 1 + continue + out[k] = rs + return out, duplicates + + def _extract_input(rs: dict[str, Any]) -> Any: + inst = rs.get('instance') or {} + inp = inst.get('input', None) + if isinstance(inp, dict) and 'text' in inp: + return inp.get('text', None) + return inp + + def _extract_prompt(rs: dict[str, Any]) -> Any: + req = rs.get('request') or {} + return req.get('prompt', None) + + def _extract_completion(rs: dict[str, Any]) -> Any: + res = rs.get('result') or {} + comps = res.get('completions') or [] + if not comps: + return None + first = comps[0] + if isinstance(first, dict): + return first.get('text', None) + return first + + def _summarize( + map_a: dict[tuple[Any, ...], dict[str, Any]], + map_b: dict[tuple[Any, ...], dict[str, Any]], + keys: set[tuple[Any, ...]], + *, + extractor, + ) -> tuple[dict[str, Any], list[dict[str, Any]]]: + def _keysort(k: tuple[Any, ...]) -> str: + try: + return ub.urepr(k, nl=0, compact=1) + except Exception: + return str(k) + + comparable = 0 + mismatched = 0 + examples: list[dict[str, Any]] = [] + for k in sorted(keys, key=_keysort): + va = extractor(map_a[k]) + vb = extractor(map_b[k]) + comparable += 1 + if va != vb: + mismatched += 1 + if len(examples) < max_examples: + examples.append( + { + 'key': _key_to_serializable(k), + 'a': _short_urepr(va, max_chars=180), + 'b': _short_urepr(vb, max_chars=180), + } + ) + return ( + { + 'comparable': comparable, + 'mismatched': mismatched, + 'equal_ratio': ratio(comparable, mismatched), + }, + examples, + ) + + base_a, dup_base_a = _index_unique(request_states_a, _base_key) + base_b, dup_base_b = _index_unique(request_states_b, _base_key) + var_a, dup_var_a = _index_unique(request_states_a, _variant_key) + var_b, dup_var_b = _index_unique(request_states_b, _variant_key) + + cov_base = Coverage.from_sets(set(base_a), set(base_b)) + cov_var = Coverage.from_sets(set(var_a), set(var_b)) + isect_base = set(base_a) & set(base_b) + isect_var = set(var_a) & set(var_b) + + base_iou = ( + cov_base.n_isect / cov_base.n_union if cov_base.n_union else None + ) + variant_iou = ( + cov_var.n_isect / cov_var.n_union if cov_var.n_union else None + ) + + input_eq, ex_input = _summarize( + base_a, base_b, isect_base, extractor=_extract_input + ) + prompt_eq, ex_prompt = _summarize( + var_a, var_b, isect_var, extractor=_extract_prompt + ) + completion_eq, ex_completion = _summarize( + var_a, var_b, isect_var, extractor=_extract_completion + ) + + out = { + 'base_coverage': cov_base.__dict__, + 'variant_coverage': cov_var.__dict__, + 'base_iou': base_iou, + 'variant_iou': variant_iou, + 'content_equality': { + 'input': input_eq, + 'prompt': prompt_eq, + 'completion': completion_eq, + }, + 'duplicates': { + 'a': {'base': dup_base_a, 'variant': dup_var_a}, + 'b': {'base': dup_base_b, 'variant': dup_var_b}, + }, + 'mismatch_examples': { + 'input': ex_input, + 'prompt': ex_prompt, + 'completion': ex_completion, + }, + } + return _json_compatible(out) + + class HelmRunDiff(ub.NiceRepr): """Compare two HELM runs. @@ -407,7 +826,7 @@ def summary_dict(self, *, level: int = 10) -> dict[str, Any]: a_run_name is not None ) - # 2) run spec dict hash + # 2) run spec dict hash (strict) spec_hash_a = helm_hashers.stable_hash36( helm_hashers.canonicalize_for_hashing(a_spec) ) @@ -415,6 +834,14 @@ def summary_dict(self, *, level: int = 10) -> dict[str, Any]: helm_hashers.canonicalize_for_hashing(b_spec) ) run_spec_dict_ok = spec_hash_a == spec_hash_b + if run_spec_dict_ok: + spec_path_info: dict[str, list[str]] = { + 'unique1': [], + 'unique2': [], + 'faillist': [], + } + else: + spec_path_info = _walker_diff_paths(a_spec, b_spec) if level == 0: spec_diff_paths = None else: @@ -422,13 +849,42 @@ def summary_dict(self, *, level: int = 10) -> dict[str, Any]: {} if run_spec_dict_ok else _walker_diff(a_spec, b_spec) ) + # 2b) run spec semantic hash (order-insensitive for metric lists) + a_spec_sem = _canonicalize_run_spec_for_semantic_diff(a_spec) + b_spec_sem = _canonicalize_run_spec_for_semantic_diff(b_spec) + spec_sem_hash_a = helm_hashers.stable_hash36(a_spec_sem) + spec_sem_hash_b = helm_hashers.stable_hash36(b_spec_sem) + run_spec_semantic_dict_ok = spec_sem_hash_a == spec_sem_hash_b + if run_spec_semantic_dict_ok: + spec_sem_path_info: dict[str, list[str]] = { + 'unique1': [], + 'unique2': [], + 'faillist': [], + } + else: + spec_sem_path_info = _walker_diff_paths(a_spec_sem, b_spec_sem) + if level == 0: + spec_sem_diff_paths = None + else: + spec_sem_diff_paths = ( + {} + if run_spec_semantic_dict_ok + else _walker_diff(a_spec_sem, b_spec_sem) + ) + run_spec_semantic = self._run_spec_semantic_summary( + path_info=spec_sem_path_info, + a_spec=a_spec, + b_spec=b_spec, + ) + # 3) scenario check with unknown semantics scen_known = bool(a_scen) and bool(b_scen) if not scen_known: scenario_ok: bool | None = None scenario_hash_a = None scenario_hash_b = None - scen_diff_paths: list[str] = [] + scen_diff_paths = [] + scen_path_info: dict[str, list[str]] | None = None else: scenario_hash_a = helm_hashers.stable_hash36( helm_hashers.canonicalize_for_hashing(a_scen) @@ -437,12 +893,19 @@ def summary_dict(self, *, level: int = 10) -> dict[str, Any]: helm_hashers.canonicalize_for_hashing(b_scen) ) scenario_ok = scenario_hash_a == scenario_hash_b + if scenario_ok: + scen_path_info = {'unique1': [], 'unique2': [], 'faillist': []} + else: + scen_path_info = _walker_diff_paths(a_scen, b_scen) if level == 0: scen_diff_paths = None else: scen_diff_paths = ( {} if scenario_ok else _walker_diff(a_scen, b_scen) ) + scenario_semantic = self._scenario_semantic_summary( + scenario_ok=scenario_ok, path_info=scen_path_info + ) # 4/5) stats coverage a_stats = self.a.stats() or [] @@ -483,6 +946,19 @@ def summary_dict(self, *, level: int = 10) -> dict[str, Any]: # 6) value agreement (means) on intersecting keys value_summary = self._value_agreement_summary() + dataset_summary: dict[str, Any] | None = None + if level >= 5: + try: + dataset_summary = self.dataset_overlap_summary(max_examples=5) + except Exception as ex: # nocover + dataset_summary = {'error': repr(ex)} + diagnosis = self._diagnose_repro( + run_spec_name_ok=run_spec_name_ok, + run_spec_semantic=run_spec_semantic, + scenario_semantic=scenario_semantic, + dataset_overlap=dataset_summary, + value_summary=value_summary, + ) out: dict[str, Any] = { 'a': self._lite_run_dict(self.a), @@ -494,13 +970,21 @@ def summary_dict(self, *, level: int = 10) -> dict[str, Any]: 'run_spec_hash_a': spec_hash_a, 'run_spec_hash_b': spec_hash_b, 'run_spec_diff_paths': spec_diff_paths, + 'run_spec_semantic_dict_ok': run_spec_semantic_dict_ok, + 'run_spec_semantic_hash_a': spec_sem_hash_a, + 'run_spec_semantic_hash_b': spec_sem_hash_b, + 'run_spec_diff_paths_semantic': spec_sem_diff_paths, + 'run_spec_semantic': run_spec_semantic, 'scenario_ok': scenario_ok, 'scenario_hash_a': scenario_hash_a, 'scenario_hash_b': scenario_hash_b, 'scenario_diff_paths': scen_diff_paths, + 'scenario_semantic': scenario_semantic, 'stats_coverage_by_name': cov_name.__dict__, 'stats_coverage_by_name_count': cov_name_count.__dict__, 'value_agreement': value_summary, + 'dataset_overlap': dataset_summary, + 'diagnosis': diagnosis, } if level >= 20: @@ -512,6 +996,7 @@ def summary_dict(self, *, level: int = 10) -> dict[str, Any]: except Exception as ex: # nocover out['instance_value_agreement'] = {'error': repr(ex)} + out = _json_compatible(out) self._cache[cache_key] = out return out @@ -549,7 +1034,7 @@ def summary(self, *, level: int = 10, writer=None) -> None: f'{_format_bool(ok)} {self.a_name} vs {self.b_name} {line_name} ' f'spec={_format_bool(info["run_spec_dict_ok"])} ' f'stats={cov["n_isect"]}/{cov["n_union"]} ' - f'agree={agree:.3f}' + f'agree={_fmt(agree)}' ) if level > 0: @@ -574,8 +1059,21 @@ def summary(self, *, level: int = 10, writer=None) -> None: ) if level >= 15: - if not info['run_spec_dict_ok']: - writer(f' diff: {ub.urepr(info["run_spec_diff_paths"])}') + if not info.get('run_spec_semantic_dict_ok', True): + writer( + f' semantic diff: {ub.urepr(info["run_spec_diff_paths_semantic"])}' + ) + if (not info['run_spec_dict_ok']) and level >= 20: + writer( + f' strict diff: {ub.urepr(info["run_spec_diff_paths"])}' + ) + rs_sem = info.get('run_spec_semantic', {}) or {} + dep = rs_sem.get('deployment', {}) or {} + if dep.get('changed', False): + writer( + f' deployment: A={_short_urepr(dep.get("a", None), 80)} ' + f'B={_short_urepr(dep.get("b", None), 80)}' + ) if info['scenario_ok'] is None: writer( @@ -590,6 +1088,11 @@ def summary(self, *, level: int = 10, writer=None) -> None: writer( f' diff: {ub.urepr(info["scenario_diff_paths"])}' ) + scen_sem = info.get('scenario_semantic', {}) or {} + if scen_sem.get('known', False) and level >= 15: + writer( + f'Scenario semantic: {_format_bool(bool(scen_sem.get("semantic_ok", False)))}' + ) writer('') cov2 = info['stats_coverage_by_name_count'] @@ -608,13 +1111,13 @@ def summary(self, *, level: int = 10, writer=None) -> None: ov = info['value_agreement']['overall'] writer( f' overall: comparable={ov["comparable"]} mismatched={ov["mismatched"]} ' - f'agree_ratio={ov["agree_ratio"]:.3f}' + f'agree_ratio={_fmt(ov["agree_ratio"])}' ) for cls in ('core', 'bookkeeping', 'untracked'): s = info['value_agreement']['by_class'][cls] writer( f' {cls:11s}: comparable={s["comparable"]} mismatched={s["mismatched"]} ' - f'agree_ratio={s["agree_ratio"]:.3f}' + f'agree_ratio={_fmt(s["agree_ratio"])}' ) if level >= 20: @@ -626,6 +1129,34 @@ def summary(self, *, level: int = 10, writer=None) -> None: f' {r["key"]} A={_fmt(r["a"])} B={_fmt(r["b"])} |Δ|={_fmt(r["abs_delta"])}' ) + if level >= 15: + ds = info.get('dataset_overlap', None) + if isinstance(ds, dict) and 'error' not in ds: + writer('') + writer('Dataset overlap:') + writer( + f' base_iou={_fmt(ds.get("base_iou"))} ' + f'variant_iou={_fmt(ds.get("variant_iou"))}' + ) + ce = ds.get('content_equality', {}) or {} + for field in ('input', 'prompt', 'completion'): + row = ce.get(field, {}) or {} + writer( + f' {field:10s}: comparable={row.get("comparable")} ' + f'mismatched={row.get("mismatched")} equal_ratio={_fmt(row.get("equal_ratio"))}' + ) + elif isinstance(ds, dict) and 'error' in ds: + writer(f'Dataset overlap: ⚠️ {ds["error"]}') + + if level >= 10: + diag = info.get('diagnosis', {}) or {} + writer('') + writer(f'Diagnosis: {diag.get("label", "unknown")}') + if level >= 20: + reasons = diag.get('reasons', []) + if reasons: + writer(f' reasons: {ub.urepr(reasons, nl=0)}') + if level >= 30: writer('') try: @@ -636,8 +1167,8 @@ def summary(self, *, level: int = 10, writer=None) -> None: means = inst['means'] writer( f'Instance-level means: comparable={means["comparable"]} mismatched={means["mismatched"]} ' - f'agree={means["agree_ratio"]:.3f} (unpert={means["agree_ratio_unperturbed"]:.3f}, ' - f'pert={means["agree_ratio_perturbed"]:.3f})' + f'agree={_fmt(means["agree_ratio"])} (unpert={_fmt(means["agree_ratio_unperturbed"])}, ' + f'pert={_fmt(means["agree_ratio_perturbed"])})' ) def _analysis_summary_line( @@ -675,6 +1206,400 @@ def _lite_run_dict(self, ana: HelmRunAnalysis) -> dict[str, Any]: spec = ana.run_spec() or {} return {'run_spec_name': spec.get('name', None)} + def _run_spec_semantic_summary( + self, + *, + path_info: dict[str, list[str]], + a_spec: dict[str, Any], + b_spec: dict[str, Any], + ) -> dict[str, Any]: + """Classify run-spec differences into semantic buckets.""" + all_paths = sorted( + set(path_info.get('unique1', [])) + | set(path_info.get('unique2', [])) + | set(path_info.get('faillist', [])) + ) + by_class: dict[str, list[str]] = { + 'execution': [], + 'evaluation': [], + 'nonsemantic': [], + 'other': [], + } + for p in all_paths: + by_class[_classify_run_spec_path(p)].append(p) + + deployment_paths = [ + p + for p in all_paths + if p.startswith('adapter_spec.model_deployment') + ] + deployment_a = ( + (a_spec.get('adapter_spec', {}) or {}).get( + 'model_deployment', None + ) + if isinstance(a_spec, dict) + else None + ) + deployment_b = ( + (b_spec.get('adapter_spec', {}) or {}).get( + 'model_deployment', None + ) + if isinstance(b_spec, dict) + else None + ) + deployment_changed = (deployment_a != deployment_b) or any( + p.startswith('adapter_spec.model_deployment') for p in all_paths + ) + metric_specs_delta = _metric_specs_multiset_delta( + (a_spec or {}).get('metric_specs', None), + (b_spec or {}).get('metric_specs', None), + short_hash=self.short_hash, + max_items=20, + ) + evaluation_changed = bool(by_class['evaluation']) or ( + not bool(metric_specs_delta.get('equal_as_multiset', True)) + ) + execution_ok = len(by_class['execution']) == 0 + evaluation_only = ( + (len(all_paths) > 0) + and execution_ok + and (evaluation_changed or len(by_class['nonsemantic']) > 0) + ) + return _json_compatible( + { + 'n_total_paths': len(all_paths), + 'execution_ok': execution_ok, + 'evaluation_only': evaluation_only, + 'evaluation_changed': evaluation_changed, + 'deployment_changed': deployment_changed, + 'deployment': { + 'a': deployment_a, + 'b': deployment_b, + 'changed': deployment_changed, + }, + 'counts': { + k: len(v) + for k, v in by_class.items() + }, + 'deployment_paths': _preview_list(deployment_paths, limit=20), + 'execution_paths': _preview_list( + by_class['execution'], limit=20 + ), + 'execution_value_examples': _path_value_examples( + a_spec, b_spec, by_class['execution'], max_items=20 + ), + 'evaluation_paths': _preview_list( + by_class['evaluation'], limit=20 + ), + 'metric_specs_multiset_delta': metric_specs_delta, + 'nonsemantic_paths': _preview_list( + by_class['nonsemantic'], limit=20 + ), + 'other_paths': _preview_list(by_class['other'], limit=20), + } + ) + + def _scenario_semantic_summary( + self, + *, + scenario_ok: bool | None, + path_info: dict[str, list[str]] | None, + ) -> dict[str, Any]: + """Classify scenario differences into semantic/nonsemantic buckets.""" + if scenario_ok is None: + return { + 'known': False, + 'strict_ok': None, + 'semantic_ok': None, + 'counts': {'semantic': 0, 'nonsemantic': 0}, + 'semantic_paths': [], + 'nonsemantic_paths': [], + } + + path_info = path_info or {'unique1': [], 'unique2': [], 'faillist': []} + all_paths = sorted( + set(path_info.get('unique1', [])) + | set(path_info.get('unique2', [])) + | set(path_info.get('faillist', [])) + ) + semantic_paths = [ + p for p in all_paths if _classify_scenario_path(p) == 'semantic' + ] + nonsemantic_paths = [ + p for p in all_paths if _classify_scenario_path(p) == 'nonsemantic' + ] + semantic_ok = bool(scenario_ok) or (len(semantic_paths) == 0) + return _json_compatible( + { + 'known': True, + 'strict_ok': bool(scenario_ok), + 'semantic_ok': semantic_ok, + 'counts': { + 'semantic': len(semantic_paths), + 'nonsemantic': len(nonsemantic_paths), + }, + 'semantic_paths': _preview_list(semantic_paths, limit=20), + 'nonsemantic_paths': _preview_list( + nonsemantic_paths, limit=20 + ), + } + ) + + def dataset_overlap_summary(self, *, max_examples: int = 5) -> dict[str, Any]: + """Compare scenario_state request datasets between runs. + + Example: + >>> from magnet.backends.helm.helm_run_analysis import HelmRunAnalysis + >>> ana = HelmRunAnalysis.__new__(HelmRunAnalysis) + >>> ana._raw_cache = {} + >>> ana._cache = {} + >>> ana.run = None + >>> ana.name = None + >>> ana.scenario_state = lambda: {'request_states': [ + ... {'instance': {'id': 'id1', 'split': 'test', 'input': {'text': 'Q1'}}, + ... 'train_trial_index': 0, + ... 'request': {'prompt': 'P1'}, + ... 'result': {'completions': [{'text': 'A1'}]}}, + ... ]} + >>> rd = HelmRunDiff(ana, ana) + >>> ds = rd.dataset_overlap_summary(max_examples=2) + >>> assert ds['base_iou'] == 1.0 + >>> assert ds['variant_iou'] == 1.0 + >>> assert ds['content_equality']['input']['equal_ratio'] == 1.0 + """ + cache_key = ('dataset_overlap_summary', max_examples, self.short_hash) + if cache_key in self._cache: + return self._cache[cache_key] + rs_a = (self.a.scenario_state() or {}).get('request_states', []) or [] + rs_b = (self.b.scenario_state() or {}).get('request_states', []) or [] + out = dataset_overlap_from_request_states( + rs_a, + rs_b, + short_hash=self.short_hash, + max_examples=max_examples, + ) + out = _json_compatible(out) + self._cache[cache_key] = out + return out + + def _diagnose_repro( + self, + *, + run_spec_name_ok: bool, + run_spec_semantic: dict[str, Any], + scenario_semantic: dict[str, Any], + dataset_overlap: dict[str, Any] | None, + value_summary: dict[str, Any], + ) -> dict[str, Any]: + """High-level diagnosis for reproducibility triage. + + Returns a primary label plus a full list of contributing reasons. + Lower ``priority`` is earlier / more significant in the pipeline. + """ + reasons: list[dict[str, Any]] = [] + + def add_reason(name: str, priority: int, details: dict[str, Any]) -> None: + reasons.append( + { + 'name': name, + 'priority': int(priority), + 'details': _json_compatible(details), + } + ) + + # Priority 0: run pairing / spec-level execution blockers + if not run_spec_name_ok: + add_reason( + 'wrong_run_pair', + 0, + {'run_spec_name_ok': False}, + ) + + execution_ok = bool(run_spec_semantic.get('execution_ok', False)) + if not execution_ok: + add_reason( + 'execution_spec_drift', + 0, + { + 'execution_paths': run_spec_semantic.get( + 'execution_paths', [] + ), + 'execution_value_examples': run_spec_semantic.get( + 'execution_value_examples', [] + ), + 'counts': run_spec_semantic.get('counts', {}), + }, + ) + + if bool(run_spec_semantic.get('deployment_changed', False)): + dep = run_spec_semantic.get('deployment', {}) or {} + add_reason( + 'deployment_drift', + 0, + { + 'a_value': dep.get('a', None), + 'b_value': dep.get('b', None), + 'execution_paths': [ + p + for p in ( + run_spec_semantic.get('execution_paths', []) or [] + ) + if str(p).startswith('adapter_spec.model_deployment') + ] + or (run_spec_semantic.get('deployment_paths', []) or []), + }, + ) + + scen_known = bool(scenario_semantic.get('known', False)) + scen_semantic_ok = scenario_semantic.get('semantic_ok', None) + if scen_known and not bool(scen_semantic_ok): + add_reason( + 'scenario_spec_drift', + 0, + { + 'semantic_paths': scenario_semantic.get( + 'semantic_paths', [] + ), + 'counts': scenario_semantic.get('counts', {}), + }, + ) + + # Priority 1: dataset/request-state drift + if isinstance(dataset_overlap, dict): + if 'error' in dataset_overlap: + add_reason( + 'dataset_overlap_error', + 1, + {'error': dataset_overlap.get('error', None)}, + ) + else: + base_iou = dataset_overlap.get('base_iou', None) + variant_iou = dataset_overlap.get('variant_iou', None) + if base_iou is not None and base_iou < 1.0: + add_reason( + 'dataset_instance_drift', + 1, + { + 'base_iou': base_iou, + 'base_coverage': dataset_overlap.get( + 'base_coverage', {} + ), + }, + ) + if variant_iou is not None and variant_iou < 1.0: + add_reason( + 'dataset_variant_drift', + 1, + { + 'variant_iou': variant_iou, + 'variant_coverage': dataset_overlap.get( + 'variant_coverage', {} + ), + }, + ) + + ce = dataset_overlap.get('content_equality', {}) or {} + mex = dataset_overlap.get('mismatch_examples', {}) or {} + for field, reason_name, pr in [ + ('input', 'dataset_input_drift', 1), + ('prompt', 'request_prompt_drift', 1), + ('completion', 'completion_content_drift', 2), + ]: + row = ce.get(field, {}) or {} + eq = row.get('equal_ratio', None) + if eq is not None and eq < 1.0: + details = dict(row) + examples = mex.get(field, None) + if examples: + details['examples'] = examples + add_reason(reason_name, pr, details) + + # Priority 2: evaluation schema / metric set drift + metric_specs_delta = ( + run_spec_semantic.get('metric_specs_multiset_delta', {}) or {} + ) + eval_paths = run_spec_semantic.get('evaluation_paths', []) or [] + evaluation_changed = bool(eval_paths) or ( + not bool(metric_specs_delta.get('equal_as_multiset', True)) + ) + if evaluation_changed: + details = {'evaluation_paths': eval_paths} + if not bool(metric_specs_delta.get('equal_as_multiset', True)): + details['metric_specs_multiset_delta'] = metric_specs_delta + add_reason( + 'evaluation_spec_drift', + 2, + details, + ) + + # Priority 3: value-level drift (may be downstream effect) + core = ((value_summary.get('by_class') or {}).get('core') or {}) + book = ((value_summary.get('by_class') or {}).get('bookkeeping') or {}) + core_ratio = core.get('agree_ratio', None) + book_ratio = book.get('agree_ratio', None) + + if core_ratio is None: + add_reason( + 'no_comparable_core_metrics', + 3, + {'core': core}, + ) + else: + if core_ratio < 0.995: + add_reason( + 'core_metric_drift', + 3, + { + 'core_agree_ratio': core_ratio, + 'core': core, + }, + ) + elif (book_ratio is not None) and (book_ratio < 0.95): + add_reason( + 'bookkeeping_metric_drift', + 3, + { + 'core_agree_ratio': core_ratio, + 'bookkeeping_agree_ratio': book_ratio, + 'bookkeeping': book, + }, + ) + + if not reasons: + add_reason( + 'no_detected_drift', + 0, + { + 'core_agree_ratio': core_ratio, + 'bookkeeping_agree_ratio': book_ratio, + }, + ) + + reasons = sorted( + reasons, + key=lambda r: ( + int(r.get('priority', 999)), + str(r.get('name', '')), + ), + ) + min_priority = min(int(r['priority']) for r in reasons) + primary_reason_names = [ + r['name'] for r in reasons if int(r['priority']) == min_priority + ] + if primary_reason_names == ['no_detected_drift']: + label = 'reproduced' + elif len(primary_reason_names) == 1: + label = primary_reason_names[0] + else: + label = 'multiple_primary_reasons' + + return { + 'label': label, + 'primary_priority': min_priority, + 'primary_reason_names': primary_reason_names, + 'reasons': reasons, + } + # --------------------------------------------------------------------- # Run-level mean agreement @@ -758,6 +1683,7 @@ def agrees(x: float, y: float) -> bool: 'top_mismatches': top, } + out = _json_compatible(out) self._cache[cache_key] = out return out @@ -979,6 +1905,7 @@ def agrees(x: float, y: float) -> bool: 'top_mismatches_by_group': group_list, } + out = _json_compatible(out) self._cache[cache_key] = out return out @@ -1017,8 +1944,8 @@ def summarize_instances( ) writer( f' means: comparable={means["comparable"]} mismatched={means["mismatched"]} ' - f'agree_ratio={means["agree_ratio"]:.3f} (unpert={means["agree_ratio_unperturbed"]:.3f}, ' - f'pert={means["agree_ratio_perturbed"]:.3f})' + f'agree_ratio={_fmt(means["agree_ratio"])} (unpert={_fmt(means["agree_ratio_unperturbed"])}, ' + f'pert={_fmt(means["agree_ratio_perturbed"])})' ) grouped: list[dict[str, Any]] = info.get('top_mismatches_by_group', []) or [] @@ -1271,5 +2198,5 @@ def _completion(rs: Any) -> str: writer('') -def ratio(c: int, m: int) -> float: - return 1.0 - (m / c) if c else float('nan') +def ratio(c: int, m: int) -> float | None: + return (1.0 - (m / c)) if c else None diff --git a/tests/test_helm_output_loader.py b/tests/test_helm_output_loader.py new file mode 100644 index 0000000..5a083ed --- /dev/null +++ b/tests/test_helm_output_loader.py @@ -0,0 +1,18 @@ +def _create_small_helm_output(): + # Do we want a temp dir, or cache outputs for faster test rerun? + # import tempfile + # tempdir = tempfile.TemporaryDirectory(prefix='helm-output-test') + + import ubelt as ub + + dpath = ub.Path.appdir('magnet/tests/helm_output').ensuredir() + + res = ub.cmd( + 'helm-run --run-entries mmlu:subject=philosophy,model=openai/gpt2 --suite my-suite --max-eval-instances 1 --num-threads 1', + cwd=dpath, + verbose=3, + ) + res.check_returncode() + + res = ub.cmd('helm-summarize --suite my-suite', cwd=dpath, verbose=3) + res.check_returncode() diff --git a/tests/test_helm_run_diff_heavy.py b/tests/test_helm_run_diff_heavy.py new file mode 100644 index 0000000..83db77c --- /dev/null +++ b/tests/test_helm_run_diff_heavy.py @@ -0,0 +1,93 @@ +import json +import subprocess + +import kwutil +import pytest +import ubelt as ub + +from magnet.backends.helm.helm_outputs import HelmRun +from magnet.backends.helm.helm_run_diff import HelmRunDiff + + +def _coerce_demo_run() -> HelmRun: + """Best-effort demo run fetch; skip if environment cannot materialize it.""" + try: + return HelmRun.demo() + except subprocess.CalledProcessError as ex: + pytest.skip(f'Unable to materialize HelmRun.demo(): {ex!r}') + + +def test_helm_run_diff_heavy_demo_workflow(): + """Heavyweight regression test for end-to-end HelmRunDiff behavior.""" + run_a = _coerce_demo_run() + dpath = ub.Path.appdir('magnet/tests/helm/helm_run_diff_heavy').delete().ensuredir() + + # Case 1: identical copy + same_path = dpath / (run_a.path.name + '_same') + run_a.path.copy(same_path) + run_b = HelmRun(same_path) + rd = HelmRunDiff(run_a, run_b, a_name='orig', b_name='same') + info = rd.summary_dict(level=20) + assert info['run_spec_dict_ok'] is True + assert info['scenario_ok'] in {True, None} + assert info['value_agreement']['overall']['mismatched'] == 0 + assert info['value_agreement']['overall']['agree_ratio'] == 1.0 + assert info['dataset_overlap']['base_iou'] == 1.0 + assert info['diagnosis']['label'] in {'reproduced', 'core_match_bookkeeping_drift'} + json.dumps(info, allow_nan=False) + + # Case 2: perturb one run-level stat mean + stats_path = dpath / (run_a.path.name + '_statsmod') + run_a.path.copy(stats_path) + stat_fpath = stats_path / 'stats.json' + stats = kwutil.Json.loads(stat_fpath.read_text()) + old_mean = float(stats[0].get('mean', 0.0)) + stats[0]['mean'] = old_mean + 1.23 + stat_fpath.write_text(kwutil.Json.dumps(stats)) + + rd2 = HelmRunDiff(run_a, HelmRun(stats_path), a_name='orig', b_name='stats+1.23') + info2 = rd2.summary_dict(level=20) + assert info2['value_agreement']['overall']['mismatched'] >= 1 + assert info2['diagnosis']['label'] in { + 'core_metric_drift', + 'core_match_bookkeeping_drift', + 'reproduced', + } + json.dumps(info2, allow_nan=False) + + # Case 3: perturb one per-instance stat mean (if file exists) + inst_path = dpath / (run_a.path.name + '_perinstmod') + run_a.path.copy(inst_path) + pi_fpath = inst_path / 'per_instance_stats.json' + if pi_fpath.exists(): + perinst = kwutil.Json.loads(pi_fpath.read_text()) + ei, sj = 0, None + for j, stat in enumerate(perinst[ei]['stats']): + if int(stat.get('count', 0) or 0) and ('mean' in stat): + sj = j + break + assert sj is not None + old = float(perinst[ei]['stats'][sj]['mean']) + perinst[ei]['stats'][sj]['mean'] = old + 9.0 + pi_fpath.write_text(kwutil.Json.dumps(perinst)) + + rd_i = HelmRunDiff(run_a, HelmRun(inst_path), a_name='orig', b_name='perinst+9') + inst_info = rd_i.instance_summary_dict(top_n=5) + assert inst_info['means']['mismatched'] >= 1 + json.dumps(inst_info, allow_nan=False) + + # Case 4: run-spec deployment change + spec_path = dpath / (run_a.path.name + '_runspec_mod') + run_a.path.copy(spec_path) + spec_fpath = spec_path / 'run_spec.json' + run_spec = kwutil.Json.loads(spec_fpath.read_text()) + run_spec.setdefault('adapter_spec', {}) + run_spec['adapter_spec']['model_deployment'] = 'someotherdeploy/gpt2' + spec_fpath.write_text(kwutil.Json.dumps(run_spec)) + + rd4 = HelmRunDiff(run_a, HelmRun(spec_path), a_name='orig', b_name='runspec_mod') + info4 = rd4.summary_dict(level=20) + assert info4['run_spec_dict_ok'] is False + assert info4['run_spec_semantic']['deployment_changed'] is True + assert info4['diagnosis']['label'] in {'deployment_drift', 'execution_spec_drift'} + json.dumps(info4, allow_nan=False) diff --git a/tests/test_helm_run_diff_serializable.py b/tests/test_helm_run_diff_serializable.py new file mode 100644 index 0000000..dd29bf7 --- /dev/null +++ b/tests/test_helm_run_diff_serializable.py @@ -0,0 +1,178 @@ +import json +from typing import Any, Iterable + +from magnet.backends.helm.helm_run_analysis import HelmRunAnalysis +from magnet.backends.helm.helm_run_diff import HelmRunDiff +from magnet.backends.helm.helm_run_diff import dataset_overlap_from_request_states + + +class DummyJoined: + def __init__(self, rows: Iterable[dict[str, Any]]): + # rows should each have a 'key' field + self.row_by_key = {r['key']: r for r in rows} + + def __iter__(self): + return iter(self.row_by_key.values()) + + +class DummyRun: + def __init__(self, joined: DummyJoined): + self._joined = joined + + def joined_instance_stat_table(self, *, assert_assumptions: bool = False, short_hash: int = 0): + return self._joined + + +def _dummy_analysis(run_spec: dict[str, Any], stats: list[dict[str, Any]]): + ana = HelmRunAnalysis.__new__(HelmRunAnalysis) + ana._raw_cache = {} + ana._cache = {} + ana.run = None + ana.name = None + ana.run_spec = lambda: run_spec + ana.scenario = lambda: {'class_name': 'ToyScenario', 'output_path': '/tmp/a'} + ana.scenario_state = lambda: {'request_states': []} + ana.stats = lambda: stats + ana.joined_instance_stat_table = lambda *args, **kwargs: DummyJoined([]) + return ana + + +def test_instance_summary_serializable(): + """The output of ``instance_summary_dict`` must be JSON serializable. + + In particular the previously-used tuple keys were converted to a list of + objects with explicit ``metric_class``/``metric`` fields. Exercise the + computation path using minimal dummy data so we don't depend on HELM or + pandas. + """ + # build two joined tables that share a key but have different means + key = ("id", 0, None, "m", "split", None, None) + row_a = { + 'key': key, + 'stat': {'mean': 1.0, 'count': 1, 'name': {'name': 'm'}}, + 'request_state': {}, + } + row_b = { + 'key': key, + 'stat': {'mean': 2.0, 'count': 1, 'name': {'name': 'm'}}, + 'request_state': {}, + } + + # create bare HelmRunAnalysis instances and override their joiners + ana_a = HelmRunAnalysis.__new__(HelmRunAnalysis) + ana_b = HelmRunAnalysis.__new__(HelmRunAnalysis) + # give them the minimal attributes expected by other methods + for ana in (ana_a, ana_b): + ana._raw_cache = {} + ana._cache = {} + ana.run = None + ana.name = None + ana.scenario = lambda: {} + ana.run_spec = lambda: {} + ana.stats = lambda: [] + ana_a.joined_instance_stat_table = lambda *args, **kwargs: DummyJoined([row_a]) + ana_b.joined_instance_stat_table = lambda *args, **kwargs: DummyJoined([row_b]) + rd = HelmRunDiff(ana_a, ana_b) + info = rd.instance_summary_dict(top_n=1) + # also check that the higher-level summary_dict exposes the same list + sdict = rd.summary_dict(level=20) + iva = sdict.get('instance_value_agreement', {}) + assert isinstance(iva.get('top_mismatches_by_group'), list) + # strict JSON: no NaN / Infinity + json.dumps(sdict, allow_nan=False) + + +def test_dataset_overlap_json_serializable(): + """Pure request_state overlap helper should stay strictly JSON-compatible.""" + rs_a = [ + { + 'instance': {'id': 'id1', 'split': 'test', 'input': {'text': 'Q1'}}, + 'train_trial_index': 0, + 'request': {'prompt': 'P1'}, + 'result': {'completions': [{'text': 'A1'}]}, + }, + { + 'instance': { + 'id': 'id1', + 'split': 'test', + 'input': {'text': 'Q1'}, + 'perturbation': {'name': 'dialect', 'prob': 1.0}, + }, + 'train_trial_index': 0, + 'request': {'prompt': 'P1-d'}, + 'result': {'completions': [{'text': 'A1d'}]}, + }, + ] + rs_b = [ + { + 'instance': {'id': 'id1', 'split': 'test', 'input': {'text': 'Q1'}}, + 'train_trial_index': 0, + 'request': {'prompt': 'P1x'}, + 'result': {'completions': [{'text': 'A1'}]}, + }, + ] + info = dataset_overlap_from_request_states(rs_a, rs_b, max_examples=3) + assert info['base_coverage']['n_isect'] == 1 + assert info['variant_coverage']['only_a'] == 1 + assert info['content_equality']['prompt']['equal_ratio'] == 0.0 + json.dumps(info, allow_nan=False) + + +def test_run_spec_metric_order_semantic_and_deployment_reason_values(): + """Metric list order should not create semantic eval drift; deployment reason should include values.""" + stats = [ + { + 'name': {'name': 'exact_match', 'split': 'test'}, + 'count': 1, + 'mean': 1.0, + } + ] + spec_a = { + 'name': 'toy', + 'adapter_spec': {'model': 'm', 'model_deployment': 'dep/A'}, + 'metric_specs': [ + {'class_name': 'M0', 'args': {'x': 0}}, + {'class_name': 'M1', 'args': {'x': 1}}, + ], + } + spec_b_order_only = { + 'name': 'toy', + 'adapter_spec': {'model': 'm', 'model_deployment': 'dep/A'}, + 'metric_specs': [ + {'class_name': 'M1', 'args': {'x': 1}}, + {'class_name': 'M0', 'args': {'x': 0}}, + ], + } + rd_order = HelmRunDiff( + _dummy_analysis(spec_a, stats), + _dummy_analysis(spec_b_order_only, stats), + a_name='A', + b_name='B', + ) + info_order = rd_order.summary_dict(level=20) + assert info_order['run_spec_dict_ok'] is False + assert info_order['run_spec_semantic_dict_ok'] is True + assert info_order['run_spec_semantic']['metric_specs_multiset_delta']['equal_as_multiset'] is True + reason_names_order = [r['name'] for r in info_order['diagnosis']['reasons']] + assert 'evaluation_spec_drift' not in reason_names_order + json.dumps(info_order, allow_nan=False) + + spec_b_deploy = { + 'name': 'toy', + 'adapter_spec': {'model': 'm', 'model_deployment': 'dep/B'}, + 'metric_specs': spec_a['metric_specs'], + } + rd_dep = HelmRunDiff( + _dummy_analysis(spec_a, stats), + _dummy_analysis(spec_b_deploy, stats), + a_name='A', + b_name='B', + ) + info_dep = rd_dep.summary_dict(level=20) + reasons = {r['name']: r for r in info_dep['diagnosis']['reasons']} + assert 'deployment_drift' in reasons + assert reasons['deployment_drift']['details']['a_value'] == 'dep/A' + assert reasons['deployment_drift']['details']['b_value'] == 'dep/B' + exec_examples = info_dep['run_spec_semantic']['execution_value_examples'] + assert any(ex.get('path') == 'adapter_spec.model_deployment' for ex in exec_examples) + json.dumps(info_dep, allow_nan=False) From 67600fdf21495b14cec1e9934f3675eba489f322 Mon Sep 17 00:00:00 2001 From: joncrall Date: Thu, 19 Mar 2026 12:53:02 -0400 Subject: [PATCH 08/36] wip --- dev/oneoff/diagnose_reproducibility.py | 96 +++++++++++++++++++++++++- 1 file changed, 95 insertions(+), 1 deletion(-) diff --git a/dev/oneoff/diagnose_reproducibility.py b/dev/oneoff/diagnose_reproducibility.py index ea062aa..aea7af2 100644 --- a/dev/oneoff/diagnose_reproducibility.py +++ b/dev/oneoff/diagnose_reproducibility.py @@ -166,6 +166,50 @@ def aggregate_report(rows: list[dict[str, Any]]) -> dict[str, Any]: diagnosis_counter = Counter() reason_counter = Counter() reason_by_priority = defaultdict(Counter) + reason_detail_counter = defaultdict(Counter) + inferred_causal_counter = Counter() + deployment_transition_counter = Counter() + inferred_causal_examples: list[dict[str, Any]] = [] + + def _reason_lut(diag: dict[str, Any]) -> dict[str, dict[str, Any]]: + out = {} + for reason in diag.get('reasons', []) or []: + if not isinstance(reason, dict): + continue + name = reason.get('name', None) + if name is None: + continue + out[str(name)] = reason + return out + + def _summarize_eval_detail(details: dict[str, Any]) -> str: + delta = details.get('metric_specs_multiset_delta', {}) or {} + if not isinstance(delta, dict): + return 'metric_specs_multiset_delta=unknown' + n_added = delta.get('n_added', None) + n_removed = delta.get('n_removed', None) + added_structured = delta.get('added_structured', []) or [] + removed_structured = delta.get('removed_structured', []) or [] + added_classes = sorted( + { + str(x.get('class_name', '?')) + for x in added_structured + if isinstance(x, dict) + } + ) + removed_classes = sorted( + { + str(x.get('class_name', '?')) + for x in removed_structured + if isinstance(x, dict) + } + ) + added_s = ','.join(added_classes[:3]) if added_classes else '-' + removed_s = ','.join(removed_classes[:3]) if removed_classes else '-' + return ( + f'n_added={n_added},n_removed={n_removed},' + f'added_classes={added_s},removed_classes={removed_s}' + ) for row in rows: status = row.get('status', 'unknown') @@ -182,6 +226,47 @@ def aggregate_report(rows: list[dict[str, Any]]) -> dict[str, Any]: reason_counter[name] += 1 reason_by_priority[str(priority)][name] += 1 + reason_lut = _reason_lut(diag) + dep = reason_lut.get('deployment_drift', None) + eval_spec = reason_lut.get('evaluation_spec_drift', None) + dep_transition = None + + if dep is not None: + dep_details = dep.get('details', {}) or {} + a_val = dep_details.get('a_value', None) + b_val = dep_details.get('b_value', None) + dep_transition = f'{a_val!r} -> {b_val!r}' + deployment_transition_counter[dep_transition] += 1 + reason_detail_counter['deployment_drift'][dep_transition] += 1 + + if eval_spec is not None: + eval_details = eval_spec.get('details', {}) or {} + eval_summary = _summarize_eval_detail(eval_details) + reason_detail_counter['evaluation_spec_drift'][eval_summary] += 1 + + if dep is not None and eval_spec is not None: + dep_p = dep.get('priority', None) + eval_p = eval_spec.get('priority', None) + if ( + isinstance(dep_p, int) + and isinstance(eval_p, int) + and dep_p <= eval_p + ): + inferred_causal_counter['deployment_precedes_eval_spec_drift'] += 1 + if dep_transition is not None: + inferred_causal_counter[ + f'deployment_transition::{dep_transition}' + ] += 1 + if len(inferred_causal_examples) < 20: + inferred_causal_examples.append( + { + 'run_spec_name': row.get('run_spec_name', None), + 'deployment_transition': dep_transition, + 'deployment_priority': dep_p, + 'eval_priority': eval_p, + } + ) + out = { 'n_rows': len(rows), 'status_counts': dict(status_counter), @@ -190,6 +275,15 @@ def aggregate_report(rows: list[dict[str, Any]]) -> dict[str, Any]: 'reason_counts_by_priority': { p: dict(c) for p, c in reason_by_priority.items() }, + 'reason_detail_counts': { + name: dict(counter.most_common(20)) + for name, counter in reason_detail_counter.items() + }, + 'deployment_transition_counts': dict( + deployment_transition_counter.most_common(20) + ), + 'inferred_causal_counts': dict(inferred_causal_counter), + 'inferred_causal_examples': inferred_causal_examples, } return out @@ -538,7 +632,7 @@ def _emit_graph( max_label_len = max((len(str(x)) for x in node_labels), default=20) export_width = min(5200, max(2800, 1700 + max_label_len * 18)) export_height = min(5200, max(2200, 1000 + len(node_labels) * 50)) - export_scale = 4.5 + export_scale = 2.25 fig_static.update_traces( node=dict(pad=20, thickness=20), ) From 06e51fdb2f3faac7a5da97f64adaff48b22ad673 Mon Sep 17 00:00:00 2001 From: agent Date: Tue, 24 Mar 2026 19:50:51 +0000 Subject: [PATCH 09/36] Start second helm reproduction attempt --- .../audit-helm-reproduction/README.md | 169 ++++++++ .../configs/manifest_template.yaml | 18 + .../configs/model_deployments_template.yaml | 11 + .../configs/smoke_manifest.yaml | 23 ++ .../examples/example_smoke_commands.sh | 10 + .../audit-helm-reproduction/python/common.py | 68 ++++ .../python/compare_batch.py | 378 ++++++++++++++++++ .../python/make_manifest.py | 95 +++++ .../python/render_schedule_params.py | 71 ++++ .../scripts/check_env.sh | 50 +++ .../audit-helm-reproduction/scripts/common.sh | 33 ++ .../scripts/compare_batch.sh | 13 + .../scripts/make_smoke_manifest.sh | 16 + .../scripts/run_from_manifest.sh | 42 ++ .../scripts/run_smoke.sh | 15 + .../backends/helm/cli/materialize_helm_run.py | 112 +++++- magnet/backends/helm/pipeline.py | 4 + 17 files changed, 1125 insertions(+), 3 deletions(-) create mode 100644 dev/experiments/audit-helm-reproduction/README.md create mode 100644 dev/experiments/audit-helm-reproduction/configs/manifest_template.yaml create mode 100644 dev/experiments/audit-helm-reproduction/configs/model_deployments_template.yaml create mode 100644 dev/experiments/audit-helm-reproduction/configs/smoke_manifest.yaml create mode 100755 dev/experiments/audit-helm-reproduction/examples/example_smoke_commands.sh create mode 100644 dev/experiments/audit-helm-reproduction/python/common.py create mode 100644 dev/experiments/audit-helm-reproduction/python/compare_batch.py create mode 100644 dev/experiments/audit-helm-reproduction/python/make_manifest.py create mode 100644 dev/experiments/audit-helm-reproduction/python/render_schedule_params.py create mode 100755 dev/experiments/audit-helm-reproduction/scripts/check_env.sh create mode 100755 dev/experiments/audit-helm-reproduction/scripts/common.sh create mode 100755 dev/experiments/audit-helm-reproduction/scripts/compare_batch.sh create mode 100755 dev/experiments/audit-helm-reproduction/scripts/make_smoke_manifest.sh create mode 100755 dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh create mode 100755 dev/experiments/audit-helm-reproduction/scripts/run_smoke.sh diff --git a/dev/experiments/audit-helm-reproduction/README.md b/dev/experiments/audit-helm-reproduction/README.md new file mode 100644 index 0000000..63cc34e --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/README.md @@ -0,0 +1,169 @@ +# Audit HELM Reproduction + +This folder contains a reusable, shell-first workflow for reproducing selected +historic HELM runs with the local `kwdagger` pipeline, then comparing the +reproduced outputs against the public HELM bundle. + +The design goals are: + +- work out of the box in the current environment, +- remain portable to a fresh operator via environment variables, +- keep heavyweight run artifacts outside the repo by default, +- start with a small smoke-test batch, +- scale later to larger reproduction attempts without redesign. + +## Defaults + +These defaults are chosen to work in the current environment: + +- `AIQ_MAGNET_ROOT=/home/joncrall/code/aiq-magnet` +- `HELM_EDITABLE_ROOT=/home/joncrall/code/helm` +- `AIQ_PYTHON=python` +- `HELM_PRECOMPUTED_ROOT=/data/crfm-helm-public` +- `AUDIT_RESULTS_ROOT=/data/crfm-helm-audit` + +Assumptions: + +- `magnet` is importable from `AIQ_PYTHON` +- `kwdagger` is on `PATH` +- `helm-run` is on `PATH` + +Optional environment variables: + +- `CUDA_VISIBLE_DEVICES` +- `AUDIT_DEFAULT_MAX_EVAL_INSTANCES` +- `AUDIT_DEFAULT_TMUX_WORKERS` + +## Folder Layout + +- `configs/` + Checked-in manifests and templates. +- `scripts/` + Shell entrypoints for operators. +- `python/` + Python helpers used by the shell scripts. +- `reports/` + Lightweight comparison reports. +- `examples/` + Example command sequences. + +## Quick Start + +1. Validate the environment: + +```bash +dev/experiments/audit-helm-reproduction/scripts/check_env.sh +``` + +2. Materialize a smoke-test manifest: + +```bash +dev/experiments/audit-helm-reproduction/scripts/make_smoke_manifest.sh +``` + +By default this writes: + +```text +dev/experiments/audit-helm-reproduction/configs/generated/smoke_manifest.generated.yaml +``` + +3. Launch the smoke-test batch on the GPU machine: + +```bash +CUDA_VISIBLE_DEVICES=0,1 \ +dev/experiments/audit-helm-reproduction/scripts/run_smoke.sh +``` + +4. Compare the completed batch to the historic HELM bundle: + +```bash +dev/experiments/audit-helm-reproduction/scripts/compare_batch.sh +``` + +Reports are written by default to: + +```text +dev/experiments/audit-helm-reproduction/reports// +``` + +Heavy run outputs are written by default to: + +```text +/data/crfm-helm-audit// +``` + +## Manifest Schema + +The workflow uses a small YAML manifest as the unit of experiment definition. + +Fields: + +- `schema_version` +- `experiment_name` +- `description` +- `run_entries` +- `max_eval_instances` +- `suite` +- `mode` +- `materialize` +- `backend` +- `devices` +- `tmux_workers` +- `local_path` +- `precomputed_root` +- `require_per_instance_stats` +- `model_deployments_fpath` +- `enable_huggingface_models` +- `enable_local_huggingface_models` + +See: + +- `configs/smoke_manifest.yaml` +- `configs/manifest_template.yaml` + +## Smoke-Test Batch + +The checked-in smoke batch is intentionally small and uses only models that +already have built-in Hugging Face deployments in HELM: + +- `eleutherai/pythia-6.9b` +- `lmsys/vicuna-7b-v1.3` + +Tasks: + +- `mmlu:subject=us_foreign_policy,...` +- `boolq:...` +- `narrative_qa:...` + +Total: + +- 6 runs + +Defaults: + +- `max_eval_instances=100` +- low worker count +- no custom deployment override YAML + +## Scaling Up + +For larger experiments: + +1. copy `configs/manifest_template.yaml`, +2. expand `run_entries`, +3. optionally introduce `model_deployments_fpath` for Together-only families, +4. run: + +```bash +dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh /path/to/manifest.yaml +dev/experiments/audit-helm-reproduction/scripts/compare_batch.sh /path/to/manifest.yaml +``` + +## Notes + +- This workflow is intended for execution on an external GPU machine. +- The current development environment does not need a GPU to generate manifests + or comparison commands. +- Comparison relies on `HelmRunDiff` and will emit JSONL, JSON, text, and + optional sankey artifacts when the required plotting dependencies are + available. diff --git a/dev/experiments/audit-helm-reproduction/configs/manifest_template.yaml b/dev/experiments/audit-helm-reproduction/configs/manifest_template.yaml new file mode 100644 index 0000000..cdf3030 --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/configs/manifest_template.yaml @@ -0,0 +1,18 @@ +schema_version: 1 +experiment_name: your-experiment-name +description: Describe the reproduction batch here. +run_entries: + - mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=eleutherai/pythia-6.9b,data_augmentation=canonical +max_eval_instances: 100 +suite: audit-custom +mode: compute_if_missing +materialize: symlink +backend: tmux +devices: 0,1 +tmux_workers: 2 +local_path: prod_env +precomputed_root: null +require_per_instance_stats: true +model_deployments_fpath: null +enable_huggingface_models: [] +enable_local_huggingface_models: [] diff --git a/dev/experiments/audit-helm-reproduction/configs/model_deployments_template.yaml b/dev/experiments/audit-helm-reproduction/configs/model_deployments_template.yaml new file mode 100644 index 0000000..1812858 --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/configs/model_deployments_template.yaml @@ -0,0 +1,11 @@ +model_deployments: + - name: huggingface/example-local-deployment + model_name: creator/example-model + tokenizer_name: huggingface/example-local-deployment + max_sequence_length: 4096 + client_spec: + class_name: "helm.clients.huggingface_client.HuggingFaceClient" + args: + pretrained_model_name_or_path: creator/example-model + device_map: auto + torch_dtype: torch.bfloat16 diff --git a/dev/experiments/audit-helm-reproduction/configs/smoke_manifest.yaml b/dev/experiments/audit-helm-reproduction/configs/smoke_manifest.yaml new file mode 100644 index 0000000..775198b --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/configs/smoke_manifest.yaml @@ -0,0 +1,23 @@ +schema_version: 1 +experiment_name: audit-smoke +description: Small smoke-test batch for HELM reproduction auditing. +run_entries: + - mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=eleutherai/pythia-6.9b,data_augmentation=canonical + - boolq:model=eleutherai/pythia-6.9b,data_augmentation=canonical + - narrative_qa:model=eleutherai/pythia-6.9b,data_augmentation=canonical + - mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical + - boolq:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical + - narrative_qa:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical +max_eval_instances: 100 +suite: audit-smoke +mode: compute_if_missing +materialize: symlink +backend: tmux +devices: 0,1 +tmux_workers: 2 +local_path: prod_env +precomputed_root: null +require_per_instance_stats: true +model_deployments_fpath: null +enable_huggingface_models: [] +enable_local_huggingface_models: [] diff --git a/dev/experiments/audit-helm-reproduction/examples/example_smoke_commands.sh b/dev/experiments/audit-helm-reproduction/examples/example_smoke_commands.sh new file mode 100755 index 0000000..5f897b6 --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/examples/example_smoke_commands.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail + +AUDIT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +"$AUDIT_ROOT/scripts/check_env.sh" +"$AUDIT_ROOT/scripts/make_smoke_manifest.sh" +CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1}" \ + "$AUDIT_ROOT/scripts/run_smoke.sh" +"$AUDIT_ROOT/scripts/compare_batch.sh" diff --git a/dev/experiments/audit-helm-reproduction/python/common.py b/dev/experiments/audit-helm-reproduction/python/common.py new file mode 100644 index 0000000..5d43424 --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/python/common.py @@ -0,0 +1,68 @@ +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any + +import kwutil + + +def env_defaults() -> dict[str, str]: + return { + "AIQ_MAGNET_ROOT": os.environ.get( + "AIQ_MAGNET_ROOT", "/home/joncrall/code/aiq-magnet" + ), + "HELM_EDITABLE_ROOT": os.environ.get( + "HELM_EDITABLE_ROOT", "/home/joncrall/code/helm" + ), + "AIQ_PYTHON": os.environ.get("AIQ_PYTHON", "python"), + "HELM_PRECOMPUTED_ROOT": os.environ.get( + "HELM_PRECOMPUTED_ROOT", "/data/crfm-helm-public" + ), + "AUDIT_RESULTS_ROOT": os.environ.get( + "AUDIT_RESULTS_ROOT", "/data/crfm-helm-audit" + ), + "AUDIT_DEFAULT_MAX_EVAL_INSTANCES": os.environ.get( + "AUDIT_DEFAULT_MAX_EVAL_INSTANCES", "100" + ), + "AUDIT_DEFAULT_TMUX_WORKERS": os.environ.get( + "AUDIT_DEFAULT_TMUX_WORKERS", "2" + ), + } + + +def audit_root() -> Path: + return Path(__file__).resolve().parent.parent + + +def aiq_root() -> Path: + return Path(env_defaults()["AIQ_MAGNET_ROOT"]).expanduser().resolve() + + +def repo_run_specs_fpath() -> Path: + return aiq_root() / "run_specs.yaml" + + +def default_report_root() -> Path: + return audit_root() / "reports" + + +def load_manifest(manifest_fpath: str | os.PathLike[str]) -> dict[str, Any]: + data = kwutil.Yaml.load(Path(manifest_fpath)) + if not isinstance(data, dict): + raise TypeError("Manifest must decode to a dictionary") + return data + + +def dump_yaml(data: Any) -> str: + return kwutil.Yaml.dumps(data) + + +def experiment_result_dpath(manifest: dict[str, Any]) -> Path: + root = Path(env_defaults()["AUDIT_RESULTS_ROOT"]).expanduser().resolve() + return root / str(manifest["experiment_name"]) + + +def experiment_report_dpath(manifest: dict[str, Any]) -> Path: + return default_report_root() / str(manifest["experiment_name"]) + diff --git a/dev/experiments/audit-helm-reproduction/python/compare_batch.py b/dev/experiments/audit-helm-reproduction/python/compare_batch.py new file mode 100644 index 0000000..1bd7b08 --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/python/compare_batch.py @@ -0,0 +1,378 @@ +from __future__ import annotations + +import argparse +import datetime as datetime_mod +import importlib.util +import json +from collections import Counter, defaultdict +from pathlib import Path +from typing import Any + +import kwutil +import ubelt as ub + +from common import ( + env_defaults, + experiment_report_dpath, + experiment_result_dpath, + load_manifest, +) +from magnet.backends.helm.cli.materialize_helm_run import find_best_precomputed_run +from magnet.backends.helm.helm_outputs import HelmOutputs, HelmRun +from magnet.backends.helm.helm_run_diff import HelmRunDiff + + +def parse_helm_run_dir(run_dir: str) -> dict[str, str]: + p = ub.Path(run_dir) + parts = list(p.parts) + out = { + "helm_suite_name": "unknown", + "helm_version": "unknown", + "run_leaf": p.name, + } + try: + idx = parts.index("benchmark_output") + except ValueError: + idx = -1 + if idx >= 1: + out["helm_suite_name"] = str(parts[idx - 1]) + if idx >= 0 and (idx + 2) < len(parts): + out["helm_version"] = str(parts[idx + 2]) + else: + out["helm_version"] = str(p.parent.name) + return out + + +def infer_benchmark_group(run_spec_name: str | None) -> str: + text = (run_spec_name or "").strip() + if not text: + return "unknown" + idxs = [i for i in [text.find(":"), text.find(",")] if i >= 0] + if idxs: + return text[: min(idxs)].strip() + return text + + +def load_kwdg_rows(results_dpath: Path) -> tuple[list[dict[str, Any]], dict[str, dict[str, Any]]]: + finished_jobs = sorted(results_dpath.glob("*/DONE")) + rows = [] + for fpath in ub.ProgIter(finished_jobs, desc="load kwdg runs"): + dpath = fpath.parent + try: + config = kwutil.Json.load(dpath / "job_config.json") + run_spec_name = config["helm.run_entry"] + suites = HelmOutputs.coerce(dpath / "benchmark_output").suites() + if not suites: + continue + runs = suites[0].runs() + if len(runs) != 1: + continue + run = runs[0] + rows.append( + { + "dpath": str(dpath), + "run_spec_name": run_spec_name, + "run": run, + } + ) + except Exception: + continue + + lut = {} + for row in rows: + lut[row["run_spec_name"]] = row + return rows, lut + + +def aggregate_report(rows: list[dict[str, Any]]) -> dict[str, Any]: + status_counter = Counter() + diagnosis_counter = Counter() + reason_counter = Counter() + for row in rows: + status = row.get("status", "unknown") + status_counter[status] += 1 + if status != "compared": + continue + diag = row.get("diagnosis", {}) or {} + diagnosis_counter[diag.get("label", "unknown")] += 1 + for reason in diag.get("reasons", []) or []: + name = reason.get("name", "unknown") + reason_counter[name] += 1 + return { + "n_rows": len(rows), + "status_counts": dict(status_counter), + "diagnosis_label_counts": dict(diagnosis_counter), + "reason_counts": dict(reason_counter), + } + + +def maybe_write_sankey_report( + case_rows: list[dict[str, Any]], report_dpath: Path, stamp: str +) -> dict[str, Any]: + source_fpath = ( + Path(__file__).resolve().parents[3] + / "oneoff" + / "diagnose_reproducibility.py" + ) + if not source_fpath.exists(): + return {"plotly_error": "sankey helper script not found"} + + spec = importlib.util.spec_from_file_location( + "audit_diag_helper", source_fpath + ) + if spec is None or spec.loader is None: + return {"plotly_error": "unable to import sankey helper"} + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.write_sankey_report( + case_rows, report_dpath=ub.Path(report_dpath), stamp=stamp + ) + + +def build_historic_rows( + manifest: dict[str, Any], precomputed_root: str +) -> list[dict[str, Any]]: + rows = [] + for run_entry in manifest["run_entries"]: + match = find_best_precomputed_run( + precomputed_root=precomputed_root, + requested_desc=run_entry, + max_eval_instances=manifest.get("max_eval_instances", None), + require_per_instance_stats=manifest.get( + "require_per_instance_stats", True + ), + ) + row = { + "run_spec_name": run_entry, + "run_dir": None, + "model": None, + "benchmark_group": infer_benchmark_group(run_entry), + "benchmark_name": "unknown", + "suite_name": "unknown", + "helm_version": "unknown", + } + if match is not None: + parsed = parse_helm_run_dir(str(match.run_dir)) + row["run_dir"] = str(match.run_dir) + row["benchmark_name"] = parsed["helm_suite_name"] + row["suite_name"] = parsed["helm_suite_name"] + row["helm_version"] = parsed["helm_version"] + if "model=" in run_entry: + model_text = run_entry.split("model=", 1)[1].split(",", 1)[0] + row["model"] = model_text + rows.append(row) + return rows + + +def write_summary_text( + summary_report: dict[str, Any], out_fpath: Path +) -> None: + lines = [] + lines.append(f"generated_utc: {summary_report['generated_utc']}") + lines.append(f"case_jsonl: {summary_report['report_case_jsonl']}") + lines.append(f"summary_json: {summary_report['report_summary_json']}") + lines.append("") + lines.append("status_counts:") + for key, value in sorted( + summary_report["aggregate"]["status_counts"].items() + ): + lines.append(f" {key}: {value}") + lines.append("") + lines.append("diagnosis_label_counts:") + for key, value in sorted( + summary_report["aggregate"]["diagnosis_label_counts"].items() + ): + lines.append(f" {key}: {value}") + out_fpath.write_text("\n".join(lines) + "\n") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--manifest", required=True) + parser.add_argument("--results-dpath", default=None) + parser.add_argument("--report-dpath", default=None) + parser.add_argument("--precomputed-root", default=None) + args = parser.parse_args() + + manifest = load_manifest(args.manifest) + defaults = env_defaults() + results_dpath = ( + Path(args.results_dpath).expanduser().resolve() + if args.results_dpath + else experiment_result_dpath(manifest) + ) + report_dpath = ( + Path(args.report_dpath).expanduser().resolve() + if args.report_dpath + else experiment_report_dpath(manifest) + ) + precomputed_root = args.precomputed_root or defaults["HELM_PRECOMPUTED_ROOT"] + report_dpath.mkdir(parents=True, exist_ok=True) + + historic_rows = build_historic_rows(manifest, precomputed_root) + kwdg_rows, kwdg_lut = load_kwdg_rows(results_dpath) + + stamp = datetime_mod.datetime.now(datetime_mod.UTC).strftime( + "%Y%m%dT%H%M%SZ" + ) + case_jsonl_fpath = report_dpath / f"compare_cases_{stamp}.jsonl" + summary_json_fpath = report_dpath / f"compare_summary_{stamp}.json" + summary_txt_fpath = report_dpath / f"compare_summary_{stamp}.txt" + + all_case_rows = [] + with case_jsonl_fpath.open("w", encoding="utf8") as file: + for idx, helm_row in enumerate(historic_rows, start=1): + run_spec_name = helm_row["run_spec_name"] + kwrow = kwdg_lut.get(run_spec_name, None) + case_row = { + "index": idx, + "run_spec_name": run_spec_name, + "benchmark_name": helm_row.get("benchmark_name", "unknown"), + "benchmark_group": helm_row.get("benchmark_group", "unknown"), + "suite_name": helm_row.get("suite_name", "unknown"), + "model_name": helm_row.get("model", None), + "helm_version": helm_row.get("helm_version", None), + "helm_run_dir": helm_row["run_dir"], + "kwdg_run_dir": None if kwrow is None else kwrow["dpath"], + } + + if helm_row["run_dir"] is None: + case_row.update( + { + "status": "missing_historic_match", + "diagnosis": { + "label": "missing_historic_match", + "primary_priority": 0, + "primary_reason_names": [ + "missing_historic_match" + ], + "reasons": [ + { + "name": "missing_historic_match", + "priority": 0, + "details": {}, + } + ], + }, + } + ) + elif kwrow is None: + case_row.update( + { + "status": "missing_kwdg_match", + "diagnosis": { + "label": "missing_kwdg_match", + "primary_priority": 0, + "primary_reason_names": ["missing_kwdg_match"], + "reasons": [ + { + "name": "missing_kwdg_match", + "priority": 0, + "details": {}, + } + ], + }, + } + ) + else: + try: + helm_run = HelmRun.coerce(helm_row["run_dir"]) + kwdg_run = kwrow["run"] + rd = HelmRunDiff( + run_a=helm_run, + run_b=kwdg_run, + a_name="HELM", + b_name="KWDG", + ) + summary = rd.summary_dict(level=20) + diag = summary.get("diagnosis", {}) or {} + case_row.update( + { + "status": "compared", + "diagnosis": diag, + "run_spec_semantic": summary.get( + "run_spec_semantic", None + ), + "scenario_semantic": summary.get( + "scenario_semantic", None + ), + "dataset_overlap": summary.get( + "dataset_overlap", None + ), + "stats_coverage_by_name": summary.get( + "stats_coverage_by_name", None + ), + "stats_coverage_by_name_count": summary.get( + "stats_coverage_by_name_count", None + ), + "value_agreement": summary.get( + "value_agreement", None + ), + "instance_value_agreement": summary.get( + "instance_value_agreement", None + ), + } + ) + except Exception as ex: + case_row.update( + { + "status": "error", + "error": repr(ex), + "diagnosis": { + "label": "comparison_error", + "primary_priority": 0, + "primary_reason_names": ["comparison_error"], + "reasons": [ + { + "name": "comparison_error", + "priority": 0, + "details": {"error": repr(ex)}, + } + ], + }, + } + ) + + case_row = kwutil.Json.ensure_serializable(case_row) + file.write(json.dumps(case_row, ensure_ascii=False) + "\n") + file.flush() + all_case_rows.append(case_row) + + summary_report = { + "report_case_jsonl": str(case_jsonl_fpath), + "report_summary_json": str(summary_json_fpath), + "report_summary_txt": str(summary_txt_fpath), + "generated_utc": stamp, + "inputs": { + "manifest": str(Path(args.manifest).expanduser().resolve()), + "kwdg_results_dpath": str(results_dpath), + "precomputed_root": str(precomputed_root), + "n_manifest_run_entries": len(manifest["run_entries"]), + "n_kwdg_rows": len(kwdg_rows), + }, + "aggregate": aggregate_report(all_case_rows), + } + try: + sankey_artifacts = maybe_write_sankey_report( + all_case_rows, report_dpath, stamp + ) + except Exception as ex: + sankey_artifacts = { + "plotly_error": f"failed to build sankey report: {ex!r}" + } + summary_report["artifacts"] = sankey_artifacts + summary_report = kwutil.Json.ensure_serializable(summary_report) + summary_json_fpath.write_text( + json.dumps(summary_report, indent=2, ensure_ascii=False) + ) + write_summary_text(summary_report, summary_txt_fpath) + + print(f"Wrote case report: {case_jsonl_fpath}") + print(f"Wrote summary report: {summary_json_fpath}") + print(f"Wrote summary text: {summary_txt_fpath}") + if sankey_artifacts.get("plotly_error", None): + print(f"Sankey note: {sankey_artifacts['plotly_error']}") + + +if __name__ == "__main__": + main() diff --git a/dev/experiments/audit-helm-reproduction/python/make_manifest.py b/dev/experiments/audit-helm-reproduction/python/make_manifest.py new file mode 100644 index 0000000..aa49191 --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/python/make_manifest.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import argparse +import os +from pathlib import Path + +import kwutil + +from common import dump_yaml, env_defaults, repo_run_specs_fpath + + +SMOKE_RUN_ENTRIES = [ + "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=eleutherai/pythia-6.9b,data_augmentation=canonical", + "boolq:model=eleutherai/pythia-6.9b,data_augmentation=canonical", + "narrative_qa:model=eleutherai/pythia-6.9b,data_augmentation=canonical", + "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical", + "boolq:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical", + "narrative_qa:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical", +] + + +def _validate_entries_exist(run_entries: list[str]) -> list[str]: + fpath = repo_run_specs_fpath() + all_run_specs = set(kwutil.Yaml.load(fpath)) + missing = [entry for entry in run_entries if entry not in all_run_specs] + return missing + + +def build_smoke_manifest(args: argparse.Namespace) -> dict: + defaults = env_defaults() + max_eval_instances = ( + args.max_eval_instances + if args.max_eval_instances is not None + else int(defaults["AUDIT_DEFAULT_MAX_EVAL_INSTANCES"]) + ) + tmux_workers = ( + args.tmux_workers + if args.tmux_workers is not None + else int(defaults["AUDIT_DEFAULT_TMUX_WORKERS"]) + ) + devices = args.devices if args.devices is not None else os.environ.get("CUDA_VISIBLE_DEVICES", "0,1") + + missing = _validate_entries_exist(SMOKE_RUN_ENTRIES) + if missing: + raise RuntimeError( + "Smoke manifest entries were not found in run_specs.yaml: " + + kwutil.Json.dumps(missing) + ) + + return { + "schema_version": 1, + "experiment_name": args.experiment_name, + "description": "Small smoke-test batch for HELM reproduction auditing.", + "run_entries": SMOKE_RUN_ENTRIES, + "max_eval_instances": max_eval_instances, + "suite": args.suite, + "mode": "compute_if_missing", + "materialize": "symlink", + "backend": "tmux", + "devices": devices, + "tmux_workers": tmux_workers, + "local_path": "prod_env", + "precomputed_root": None, + "require_per_instance_stats": True, + "model_deployments_fpath": None, + "enable_huggingface_models": [], + "enable_local_huggingface_models": [], + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--manifest-type", default="smoke", choices=["smoke"] + ) + parser.add_argument("--output", required=True) + parser.add_argument("--experiment-name", default="audit-smoke") + parser.add_argument("--suite", default="audit-smoke") + parser.add_argument("--max-eval-instances", type=int, default=None) + parser.add_argument("--tmux-workers", type=int, default=None) + parser.add_argument("--devices", default=None) + args = parser.parse_args() + + if args.manifest_type != "smoke": + raise NotImplementedError(args.manifest_type) + + manifest = build_smoke_manifest(args) + out_fpath = Path(args.output) + out_fpath.parent.mkdir(parents=True, exist_ok=True) + out_fpath.write_text(dump_yaml(manifest)) + print(out_fpath) + + +if __name__ == "__main__": + main() diff --git a/dev/experiments/audit-helm-reproduction/python/render_schedule_params.py b/dev/experiments/audit-helm-reproduction/python/render_schedule_params.py new file mode 100644 index 0000000..1df3661 --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/python/render_schedule_params.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import argparse + +from common import dump_yaml, experiment_result_dpath, load_manifest + + +def build_schedule_params(manifest: dict) -> dict: + matrix = { + "helm.run_entry": list(manifest["run_entries"]), + "helm.max_eval_instances": [manifest["max_eval_instances"]], + "helm.precomputed_root": [manifest.get("precomputed_root", None)], + "helm.suite": [manifest.get("suite", "audit-smoke")], + "helm.require_per_instance_stats": [ + manifest.get("require_per_instance_stats", True) + ], + "helm.mode": [manifest.get("mode", "compute_if_missing")], + "helm.materialize": [manifest.get("materialize", "symlink")], + "helm.local_path": [manifest.get("local_path", "prod_env")], + } + model_deployments_fpath = manifest.get("model_deployments_fpath", None) + if model_deployments_fpath is not None: + matrix["helm.model_deployments_fpath"] = [model_deployments_fpath] + enable_hf = manifest.get("enable_huggingface_models", []) + if enable_hf: + matrix["helm.enable_huggingface_models"] = [enable_hf] + enable_local_hf = manifest.get("enable_local_huggingface_models", []) + if enable_local_hf: + matrix["helm.enable_local_huggingface_models"] = [enable_local_hf] + return { + "pipeline": "magnet.backends.helm.pipeline.helm_single_run_pipeline()", + "matrix": matrix, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--manifest", required=True) + parser.add_argument( + "--mode", + required=True, + choices=[ + "params", + "experiment_name", + "result_dpath", + "backend", + "tmux_workers", + "devices", + ], + ) + args = parser.parse_args() + manifest = load_manifest(args.manifest) + + if args.mode == "params": + print(dump_yaml(build_schedule_params(manifest)), end="") + elif args.mode == "experiment_name": + print(manifest["experiment_name"]) + elif args.mode == "result_dpath": + print(experiment_result_dpath(manifest)) + elif args.mode == "backend": + print(manifest.get("backend", "tmux")) + elif args.mode == "tmux_workers": + print(manifest.get("tmux_workers", 2)) + elif args.mode == "devices": + print(manifest.get("devices", "0,1")) + else: + raise AssertionError(args.mode) + + +if __name__ == "__main__": + main() diff --git a/dev/experiments/audit-helm-reproduction/scripts/check_env.sh b/dev/experiments/audit-helm-reproduction/scripts/check_env.sh new file mode 100755 index 0000000..d3ba94b --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/scripts/check_env.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/common.sh" + +audit::set_defaults + +printf 'Audit environment\n' +printf '=================\n' +audit::print_env +printf '\n' + +for path_var in AIQ_MAGNET_ROOT HELM_EDITABLE_ROOT HELM_PRECOMPUTED_ROOT; do + path="${!path_var}" + if [[ ! -e "$path" ]]; then + printf '%s does not exist: %s\n' "$path_var" "$path" >&2 + exit 1 + fi +done + +if [[ ! -d "$AUDIT_RESULTS_ROOT" ]]; then + if mkdir -p "$AUDIT_RESULTS_ROOT" 2>/dev/null; then + : + else + printf 'Warning: unable to create AUDIT_RESULTS_ROOT: %s\n' "$AUDIT_RESULTS_ROOT" >&2 + printf 'The external runner should create this path or override AUDIT_RESULTS_ROOT.\n' >&2 + fi +fi +if ! command -v kwdagger >/dev/null 2>&1; then + printf 'kwdagger not found on PATH\n' >&2 + exit 1 +fi + +if ! command -v helm-run >/dev/null 2>&1; then + printf 'helm-run not found on PATH\n' >&2 + exit 1 +fi + +if ! command -v "$AIQ_PYTHON" >/dev/null 2>&1; then + printf 'Configured AIQ_PYTHON not found on PATH: %s\n' "$AIQ_PYTHON" >&2 + exit 1 +fi + +if ! "$AIQ_PYTHON" -c "import magnet" >/dev/null 2>&1; then + printf 'Unable to import magnet from %s\n' "$AIQ_PYTHON" >&2 + exit 1 +fi + +printf 'Environment looks good.\n' diff --git a/dev/experiments/audit-helm-reproduction/scripts/common.sh b/dev/experiments/audit-helm-reproduction/scripts/common.sh new file mode 100755 index 0000000..8cb85f7 --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/scripts/common.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -euo pipefail + +AUDIT_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +AUDIT_ROOT="$(cd "${AUDIT_SCRIPT_DIR}/.." && pwd)" + +audit::set_defaults() { + export AIQ_MAGNET_ROOT="${AIQ_MAGNET_ROOT:-/home/joncrall/code/aiq-magnet}" + export HELM_EDITABLE_ROOT="${HELM_EDITABLE_ROOT:-/home/joncrall/code/helm}" + export AIQ_PYTHON="${AIQ_PYTHON:-python}" + export HELM_PRECOMPUTED_ROOT="${HELM_PRECOMPUTED_ROOT:-/data/crfm-helm-public}" + export AUDIT_RESULTS_ROOT="${AUDIT_RESULTS_ROOT:-/data/crfm-helm-audit}" + export AUDIT_DEFAULT_MAX_EVAL_INSTANCES="${AUDIT_DEFAULT_MAX_EVAL_INSTANCES:-100}" + export AUDIT_DEFAULT_TMUX_WORKERS="${AUDIT_DEFAULT_TMUX_WORKERS:-2}" +} + +audit::print_env() { + printf 'AIQ_MAGNET_ROOT=%s\n' "$AIQ_MAGNET_ROOT" + printf 'HELM_EDITABLE_ROOT=%s\n' "$HELM_EDITABLE_ROOT" + printf 'AIQ_PYTHON=%s\n' "$AIQ_PYTHON" + printf 'HELM_PRECOMPUTED_ROOT=%s\n' "$HELM_PRECOMPUTED_ROOT" + printf 'AUDIT_RESULTS_ROOT=%s\n' "$AUDIT_RESULTS_ROOT" + printf 'AUDIT_DEFAULT_MAX_EVAL_INSTANCES=%s\n' "$AUDIT_DEFAULT_MAX_EVAL_INSTANCES" + printf 'AUDIT_DEFAULT_TMUX_WORKERS=%s\n' "$AUDIT_DEFAULT_TMUX_WORKERS" +} + +audit::require_file() { + local path="$1" + if [[ ! -f "$path" ]]; then + printf 'Missing required file: %s\n' "$path" >&2 + exit 1 + fi +} diff --git a/dev/experiments/audit-helm-reproduction/scripts/compare_batch.sh b/dev/experiments/audit-helm-reproduction/scripts/compare_batch.sh new file mode 100755 index 0000000..fe9209d --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/scripts/compare_batch.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/common.sh" + +audit::set_defaults + +MANIFEST="${1:-${AUDIT_ROOT}/configs/generated/smoke_manifest.generated.yaml}" +audit::require_file "$MANIFEST" + +"$AIQ_PYTHON" "${AUDIT_ROOT}/python/compare_batch.py" \ + --manifest "$MANIFEST" diff --git a/dev/experiments/audit-helm-reproduction/scripts/make_smoke_manifest.sh b/dev/experiments/audit-helm-reproduction/scripts/make_smoke_manifest.sh new file mode 100755 index 0000000..a21636a --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/scripts/make_smoke_manifest.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/common.sh" + +audit::set_defaults + +OUTPUT="${1:-${AUDIT_ROOT}/configs/generated/smoke_manifest.generated.yaml}" +mkdir -p "$(dirname "$OUTPUT")" + +"$AIQ_PYTHON" "${AUDIT_ROOT}/python/make_manifest.py" \ + --manifest-type smoke \ + --output "$OUTPUT" + +printf 'Wrote smoke manifest: %s\n' "$OUTPUT" diff --git a/dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh b/dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh new file mode 100755 index 0000000..f40a4e6 --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/common.sh" + +audit::set_defaults + +MANIFEST="${1:-${AUDIT_ROOT}/configs/smoke_manifest.yaml}" +audit::require_file "$MANIFEST" + +"${AUDIT_ROOT}/scripts/check_env.sh" >/dev/null + +EXPERIMENT_NAME="$("$AIQ_PYTHON" "${AUDIT_ROOT}/python/render_schedule_params.py" \ + --manifest "$MANIFEST" --mode experiment_name)" +RESULT_DPATH="$("$AIQ_PYTHON" "${AUDIT_ROOT}/python/render_schedule_params.py" \ + --manifest "$MANIFEST" --mode result_dpath)" +PARAMS="$("$AIQ_PYTHON" "${AUDIT_ROOT}/python/render_schedule_params.py" \ + --manifest "$MANIFEST" --mode params)" +BACKEND="$("$AIQ_PYTHON" "${AUDIT_ROOT}/python/render_schedule_params.py" \ + --manifest "$MANIFEST" --mode backend)" +TMUX_WORKERS="$("$AIQ_PYTHON" "${AUDIT_ROOT}/python/render_schedule_params.py" \ + --manifest "$MANIFEST" --mode tmux_workers)" +DEVICES="$("$AIQ_PYTHON" "${AUDIT_ROOT}/python/render_schedule_params.py" \ + --manifest "$MANIFEST" --mode devices)" + +mkdir -p "$RESULT_DPATH" + +printf 'Launching experiment: %s\n' "$EXPERIMENT_NAME" +printf 'Results root: %s\n' "$RESULT_DPATH" +printf 'Backend: %s\n' "$BACKEND" +printf 'Devices: %s\n' "$DEVICES" +printf 'tmux_workers: %s\n' "$TMUX_WORKERS" + +kwdagger schedule \ + --params="$PARAMS" \ + --devices="$DEVICES" \ + --tmux_workers="$TMUX_WORKERS" \ + --root_dpath="$RESULT_DPATH" \ + --backend="$BACKEND" \ + --skip_existing=1 \ + --run=1 diff --git a/dev/experiments/audit-helm-reproduction/scripts/run_smoke.sh b/dev/experiments/audit-helm-reproduction/scripts/run_smoke.sh new file mode 100755 index 0000000..a658dee --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/scripts/run_smoke.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/common.sh" + +audit::set_defaults + +MANIFEST="${1:-${AUDIT_ROOT}/configs/generated/smoke_manifest.generated.yaml}" + +if [[ ! -f "$MANIFEST" ]]; then + "${AUDIT_ROOT}/scripts/make_smoke_manifest.sh" "$MANIFEST" +fi + +"${AUDIT_ROOT}/scripts/run_from_manifest.sh" "$MANIFEST" diff --git a/magnet/backends/helm/cli/materialize_helm_run.py b/magnet/backends/helm/cli/materialize_helm_run.py index 898a613..d7d7f00 100755 --- a/magnet/backends/helm/cli/materialize_helm_run.py +++ b/magnet/backends/helm/cli/materialize_helm_run.py @@ -136,6 +136,7 @@ import os import time +import shutil from dataclasses import dataclass from pathlib import Path from typing import Iterable, Iterator, Optional @@ -218,6 +219,43 @@ class MaterializeHelmRunConfig(scfg.DataConfig): tags=['perf_param'], ) + local_path = scfg.Value( + 'prod_env', + type=str, + help='Passed to helm-run --local-path. Relative paths are resolved inside out_dpath.', + tags=['perf_param'], + ) + + model_deployments_fpath = scfg.Value( + None, + type=str, + help=( + 'Optional path to a HELM model_deployments.yaml file that will be copied ' + 'into /model_deployments.yaml before invoking helm-run.' + ), + tags=['algo_param'], + ) + + enable_huggingface_models = scfg.Value( + [], + nargs='*', + help=( + 'Optional passthrough for helm-run --enable-huggingface-models. ' + 'Useful when the Hugging Face repo id is directly usable by HELM.' + ), + tags=['algo_param'], + ) + + enable_local_huggingface_models = scfg.Value( + [], + nargs='*', + help=( + 'Optional passthrough for helm-run --enable-local-huggingface-models. ' + 'Useful when pointing HELM at a local model directory.' + ), + tags=['algo_param'], + ) + # extra_helm_args = scfg.Value( # [], # nargs='*', @@ -260,9 +298,9 @@ def main(cls, argv=None, **kwargs) -> dict: Example: >>> # This doctest is illustrative only; it requires helm-run installed. >>> # xdoctest: +REQUIRES(env:HELM_RUN_AVAILABLE) - >>> from magnet.backends.helm.cli.materialize_helm_run import main + >>> from magnet.backends.helm.cli.materialize_helm_run import MaterializeHelmRunConfig >>> dpath = ub.Path.appdir('magnet/tests/materialize').delete().ensuredir() - >>> main([ + >>> MaterializeHelmRunConfig.main([ ... '--run-entry', 'mmlu:subject=philosophy,model=openai/gpt2', ... '--suite', 'my-suite', ... '--max-eval-instances', '2', @@ -320,6 +358,10 @@ def main(cls, argv=None, **kwargs) -> dict: 'require_per_instance_stats': config.require_per_instance_stats, 'mode': config.mode, 'materialize': config.materialize, + 'local_path': config.local_path, + 'model_deployments_fpath': config.model_deployments_fpath, + 'enable_huggingface_models': list(config.enable_huggingface_models or []), + 'enable_local_huggingface_models': list(config.enable_local_huggingface_models or []), }, 'status': None, 'reuse': None, @@ -389,14 +431,22 @@ def main(cls, argv=None, **kwargs) -> dict: # Ensure benchmark_output exists (helm-run will create, but pre-creating is fine) (out_dpath / 'benchmark_output').mkdir(exist_ok=True) + prepared_local_path = prepare_local_helm_config( + out_dpath=out_dpath, + local_path=config.local_path, + model_deployments_fpath=config.model_deployments_fpath, + ) logger.info('No reusable run found; running helm-run') run_helm( requested_desc=config.run_entry, suite=config.suite, out_dpath=out_dpath, + local_path=prepared_local_path, max_eval_instances=config.max_eval_instances, num_threads=config.num_threads, + enable_huggingface_models=list(config.enable_huggingface_models or []), + enable_local_huggingface_models=list(config.enable_local_huggingface_models or []), extra_args=[], ) @@ -993,12 +1043,56 @@ def ensure_copytree(src: Path, dst: Path) -> None: ub.copytree(src, dst) +def resolve_local_path(out_dpath: Path, local_path: str | os.PathLike[str]) -> Path: + """ + Resolve HELM's local config path. + + HELM defaults to ``prod_env`` relative to its current working directory, so + we mirror that here by resolving relative paths inside ``out_dpath``. + """ + path = Path(local_path) + if path.is_absolute(): + return path + return out_dpath / path + + +def prepare_local_helm_config( + out_dpath: Path, + local_path: str | os.PathLike[str], + model_deployments_fpath: str | os.PathLike[str] | None = None, +) -> Path: + """ + Prepare the local HELM config directory used by ``helm-run``. + + Currently this only materializes an optional ``model_deployments.yaml`` + override file, but keeping it centralized makes future config additions + straightforward. + """ + local_path_abs = resolve_local_path(out_dpath, local_path) + local_path_abs.mkdir(parents=True, exist_ok=True) + + if model_deployments_fpath: + src = Path(model_deployments_fpath).expanduser().resolve() + if not src.exists(): + raise FileNotFoundError( + f'model_deployments_fpath does not exist: {src}' + ) + dst = local_path_abs / 'model_deployments.yaml' + shutil.copy2(src, dst) + logger.info('Copied model deployments override: {} -> {}', src, dst) + + return local_path_abs + + def run_helm( requested_desc: str, suite: str, out_dpath: Path, + local_path: Path, max_eval_instances: Optional[int], num_threads: int, + enable_huggingface_models: Optional[list[str]] = None, + enable_local_huggingface_models: Optional[list[str]] = None, extra_args: Optional[list[str]] = None, ) -> None: """ @@ -1006,11 +1100,23 @@ def run_helm( We do not run helm-summarize by design. """ - cmd = ['helm-run', '--run-entries', requested_desc, '--suite', suite] + cmd = [ + 'helm-run', + '--run-entries', + requested_desc, + '--suite', + suite, + '--local-path', + os.fspath(local_path), + ] if max_eval_instances is not None: cmd += ['--max-eval-instances', str(max_eval_instances)] if num_threads is not None: cmd += ['--num-threads', str(num_threads)] + if enable_huggingface_models: + cmd += ['--enable-huggingface-models', *map(str, enable_huggingface_models)] + if enable_local_huggingface_models: + cmd += ['--enable-local-huggingface-models', *map(str, enable_local_huggingface_models)] cmd += list(extra_args or []) logger.info('Executing: {}', ' '.join(map(str, cmd))) ub.cmd(cmd, cwd=out_dpath, verbose=3, system=True).check_returncode() diff --git a/magnet/backends/helm/pipeline.py b/magnet/backends/helm/pipeline.py index a765633..1c26dd0 100644 --- a/magnet/backends/helm/pipeline.py +++ b/magnet/backends/helm/pipeline.py @@ -84,6 +84,9 @@ class MaterializeHelmRunNode(kwdagger.ProcessNode): 'suite': 'my-suite', 'max_eval_instances': None, 'require_per_instance_stats': True, + 'model_deployments_fpath': None, + 'enable_huggingface_models': [], + 'enable_local_huggingface_models': [], # Behavior toggles that change how/what we materialize 'mode': 'compute_if_missing', # reuse_only | compute_if_missing | force_recompute 'materialize': 'symlink', # symlink | copy @@ -96,6 +99,7 @@ class MaterializeHelmRunNode(kwdagger.ProcessNode): 'precomputed_root': '/data/crfm-helm-public', # helm-run perf knobs: 'num_threads': 1, + 'local_path': 'prod_env', } # Optional: You can define load_result if you want kwdagger aggregate to read From 9b668d547ef42c9295fa446d4f705cba50035cde Mon Sep 17 00:00:00 2001 From: joncrall Date: Tue, 24 Mar 2026 15:53:58 -0400 Subject: [PATCH 10/36] wip --- dev/experiments/audit-helm-reproduction/README.md | 4 ++++ dev/experiments/audit-helm-reproduction/scripts/common.sh | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/dev/experiments/audit-helm-reproduction/README.md b/dev/experiments/audit-helm-reproduction/README.md index 63cc34e..7409b1a 100644 --- a/dev/experiments/audit-helm-reproduction/README.md +++ b/dev/experiments/audit-helm-reproduction/README.md @@ -49,6 +49,10 @@ Optional environment variables: ## Quick Start +```bash +cd $HOME/code/aiq-magnet +``` + 1. Validate the environment: ```bash diff --git a/dev/experiments/audit-helm-reproduction/scripts/common.sh b/dev/experiments/audit-helm-reproduction/scripts/common.sh index 8cb85f7..fd6a90f 100755 --- a/dev/experiments/audit-helm-reproduction/scripts/common.sh +++ b/dev/experiments/audit-helm-reproduction/scripts/common.sh @@ -5,8 +5,8 @@ AUDIT_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" AUDIT_ROOT="$(cd "${AUDIT_SCRIPT_DIR}/.." && pwd)" audit::set_defaults() { - export AIQ_MAGNET_ROOT="${AIQ_MAGNET_ROOT:-/home/joncrall/code/aiq-magnet}" - export HELM_EDITABLE_ROOT="${HELM_EDITABLE_ROOT:-/home/joncrall/code/helm}" + export AIQ_MAGNET_ROOT="${AIQ_MAGNET_ROOT:-$HOME/code/aiq-magnet}" + export HELM_EDITABLE_ROOT="${HELM_EDITABLE_ROOT:-$HOME/code/helm}" export AIQ_PYTHON="${AIQ_PYTHON:-python}" export HELM_PRECOMPUTED_ROOT="${HELM_PRECOMPUTED_ROOT:-/data/crfm-helm-public}" export AUDIT_RESULTS_ROOT="${AUDIT_RESULTS_ROOT:-/data/crfm-helm-audit}" From 5efeedc1110449bf2c1c999c400509e218ae45d0 Mon Sep 17 00:00:00 2001 From: joncrall Date: Tue, 24 Mar 2026 15:55:32 -0400 Subject: [PATCH 11/36] wip --- dev/experiments/audit-helm-reproduction/README.md | 1 - dev/experiments/audit-helm-reproduction/python/common.py | 3 --- dev/experiments/audit-helm-reproduction/scripts/check_env.sh | 2 +- dev/experiments/audit-helm-reproduction/scripts/common.sh | 2 -- 4 files changed, 1 insertion(+), 7 deletions(-) diff --git a/dev/experiments/audit-helm-reproduction/README.md b/dev/experiments/audit-helm-reproduction/README.md index 7409b1a..93a8aa4 100644 --- a/dev/experiments/audit-helm-reproduction/README.md +++ b/dev/experiments/audit-helm-reproduction/README.md @@ -17,7 +17,6 @@ The design goals are: These defaults are chosen to work in the current environment: - `AIQ_MAGNET_ROOT=/home/joncrall/code/aiq-magnet` -- `HELM_EDITABLE_ROOT=/home/joncrall/code/helm` - `AIQ_PYTHON=python` - `HELM_PRECOMPUTED_ROOT=/data/crfm-helm-public` - `AUDIT_RESULTS_ROOT=/data/crfm-helm-audit` diff --git a/dev/experiments/audit-helm-reproduction/python/common.py b/dev/experiments/audit-helm-reproduction/python/common.py index 5d43424..d610414 100644 --- a/dev/experiments/audit-helm-reproduction/python/common.py +++ b/dev/experiments/audit-helm-reproduction/python/common.py @@ -12,9 +12,6 @@ def env_defaults() -> dict[str, str]: "AIQ_MAGNET_ROOT": os.environ.get( "AIQ_MAGNET_ROOT", "/home/joncrall/code/aiq-magnet" ), - "HELM_EDITABLE_ROOT": os.environ.get( - "HELM_EDITABLE_ROOT", "/home/joncrall/code/helm" - ), "AIQ_PYTHON": os.environ.get("AIQ_PYTHON", "python"), "HELM_PRECOMPUTED_ROOT": os.environ.get( "HELM_PRECOMPUTED_ROOT", "/data/crfm-helm-public" diff --git a/dev/experiments/audit-helm-reproduction/scripts/check_env.sh b/dev/experiments/audit-helm-reproduction/scripts/check_env.sh index d3ba94b..f4d9553 100755 --- a/dev/experiments/audit-helm-reproduction/scripts/check_env.sh +++ b/dev/experiments/audit-helm-reproduction/scripts/check_env.sh @@ -11,7 +11,7 @@ printf '=================\n' audit::print_env printf '\n' -for path_var in AIQ_MAGNET_ROOT HELM_EDITABLE_ROOT HELM_PRECOMPUTED_ROOT; do +for path_var in AIQ_MAGNET_ROOT HELM_PRECOMPUTED_ROOT; do path="${!path_var}" if [[ ! -e "$path" ]]; then printf '%s does not exist: %s\n' "$path_var" "$path" >&2 diff --git a/dev/experiments/audit-helm-reproduction/scripts/common.sh b/dev/experiments/audit-helm-reproduction/scripts/common.sh index fd6a90f..8fd51cb 100755 --- a/dev/experiments/audit-helm-reproduction/scripts/common.sh +++ b/dev/experiments/audit-helm-reproduction/scripts/common.sh @@ -6,7 +6,6 @@ AUDIT_ROOT="$(cd "${AUDIT_SCRIPT_DIR}/.." && pwd)" audit::set_defaults() { export AIQ_MAGNET_ROOT="${AIQ_MAGNET_ROOT:-$HOME/code/aiq-magnet}" - export HELM_EDITABLE_ROOT="${HELM_EDITABLE_ROOT:-$HOME/code/helm}" export AIQ_PYTHON="${AIQ_PYTHON:-python}" export HELM_PRECOMPUTED_ROOT="${HELM_PRECOMPUTED_ROOT:-/data/crfm-helm-public}" export AUDIT_RESULTS_ROOT="${AUDIT_RESULTS_ROOT:-/data/crfm-helm-audit}" @@ -16,7 +15,6 @@ audit::set_defaults() { audit::print_env() { printf 'AIQ_MAGNET_ROOT=%s\n' "$AIQ_MAGNET_ROOT" - printf 'HELM_EDITABLE_ROOT=%s\n' "$HELM_EDITABLE_ROOT" printf 'AIQ_PYTHON=%s\n' "$AIQ_PYTHON" printf 'HELM_PRECOMPUTED_ROOT=%s\n' "$HELM_PRECOMPUTED_ROOT" printf 'AUDIT_RESULTS_ROOT=%s\n' "$AUDIT_RESULTS_ROOT" From 55a90dac7d0e0c82a93d9f12643e76b94a12b54b Mon Sep 17 00:00:00 2001 From: joncrall Date: Tue, 24 Mar 2026 16:19:54 -0400 Subject: [PATCH 12/36] Fixup kwdagger usage of key/values --- .../audit-helm-reproduction/README.md | 4 ++ .../python/render_schedule_params.py | 5 +- .../scripts/run_from_manifest.sh | 4 ++ .../backends/helm/cli/materialize_helm_run.py | 48 +++++++++++++++---- 4 files changed, 51 insertions(+), 10 deletions(-) diff --git a/dev/experiments/audit-helm-reproduction/README.md b/dev/experiments/audit-helm-reproduction/README.md index 93a8aa4..6f8b404 100644 --- a/dev/experiments/audit-helm-reproduction/README.md +++ b/dev/experiments/audit-helm-reproduction/README.md @@ -95,6 +95,10 @@ Heavy run outputs are written by default to: /data/crfm-helm-audit// ``` +The runner also derives a distinct `kwdagger` queue name from the experiment +name, which helps avoid interactive tmux collision prompts when multiple audit +batches have been launched on the same machine. + ## Manifest Schema The workflow uses a small YAML manifest as the unit of experiment definition. diff --git a/dev/experiments/audit-helm-reproduction/python/render_schedule_params.py b/dev/experiments/audit-helm-reproduction/python/render_schedule_params.py index 1df3661..15481c2 100644 --- a/dev/experiments/audit-helm-reproduction/python/render_schedule_params.py +++ b/dev/experiments/audit-helm-reproduction/python/render_schedule_params.py @@ -1,6 +1,7 @@ from __future__ import annotations import argparse +import json from common import dump_yaml, experiment_result_dpath, load_manifest @@ -23,10 +24,10 @@ def build_schedule_params(manifest: dict) -> dict: matrix["helm.model_deployments_fpath"] = [model_deployments_fpath] enable_hf = manifest.get("enable_huggingface_models", []) if enable_hf: - matrix["helm.enable_huggingface_models"] = [enable_hf] + matrix["helm.enable_huggingface_models"] = [json.dumps(enable_hf)] enable_local_hf = manifest.get("enable_local_huggingface_models", []) if enable_local_hf: - matrix["helm.enable_local_huggingface_models"] = [enable_local_hf] + matrix["helm.enable_local_huggingface_models"] = [json.dumps(enable_local_hf)] return { "pipeline": "magnet.backends.helm.pipeline.helm_single_run_pipeline()", "matrix": matrix, diff --git a/dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh b/dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh index f40a4e6..39bd336 100755 --- a/dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh +++ b/dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh @@ -32,7 +32,11 @@ printf 'Backend: %s\n' "$BACKEND" printf 'Devices: %s\n' "$DEVICES" printf 'tmux_workers: %s\n' "$TMUX_WORKERS" +QUEUE_NAME="$(printf 'audit-%s' "$EXPERIMENT_NAME" | tr -c 'A-Za-z0-9._-' '-')" +printf 'queue_name: %s\n' "$QUEUE_NAME" + kwdagger schedule \ + --queue_name="$QUEUE_NAME" \ --params="$PARAMS" \ --devices="$DEVICES" \ --tmux_workers="$TMUX_WORKERS" \ diff --git a/magnet/backends/helm/cli/materialize_helm_run.py b/magnet/backends/helm/cli/materialize_helm_run.py index d7d7f00..34ac35a 100755 --- a/magnet/backends/helm/cli/materialize_helm_run.py +++ b/magnet/backends/helm/cli/materialize_helm_run.py @@ -154,6 +154,28 @@ from magnet.helm_outputs import HelmOutputs +def _normalize_optional_pathish(value): + """ + Normalize common "unset" placeholder values emitted by schedulers / CLIs. + + Args: + value: raw parsed CLI/config value + + Returns: + The original value, or ``None`` for empty / null-like placeholders. + """ + if value is None: + return None + if isinstance(value, str): + text = value.strip() + if text == '': + return None + if text.lower() in {'none', 'null'}: + return None + return text + return value + + class MaterializeHelmRunConfig(scfg.DataConfig): """ Materialize HELM results either by computing them directly or pulling them @@ -237,21 +259,21 @@ class MaterializeHelmRunConfig(scfg.DataConfig): ) enable_huggingface_models = scfg.Value( - [], - nargs='*', + None, + type=str, help=( - 'Optional passthrough for helm-run --enable-huggingface-models. ' - 'Useful when the Hugging Face repo id is directly usable by HELM.' + 'Optional YAML-encoded list passed through to helm-run ' + '--enable-huggingface-models. Example: \'[repo-a, repo-b]\'' ), tags=['algo_param'], ) enable_local_huggingface_models = scfg.Value( - [], - nargs='*', + None, + type=str, help=( - 'Optional passthrough for helm-run --enable-local-huggingface-models. ' - 'Useful when pointing HELM at a local model directory.' + 'Optional YAML-encoded list passed through to helm-run ' + '--enable-local-huggingface-models. Example: \'[/models/a, /models/b]\'' ), tags=['algo_param'], ) @@ -311,6 +333,16 @@ def main(cls, argv=None, **kwargs) -> dict: config = MaterializeHelmRunConfig.cli( argv=argv, data=kwargs, verbose='auto' ) + config.precomputed_root = _normalize_optional_pathish(config.precomputed_root) + config.model_deployments_fpath = _normalize_optional_pathish( + config.model_deployments_fpath + ) + config.enable_huggingface_models = kwutil.Yaml.coerce( + config.enable_huggingface_models + ) + config.enable_local_huggingface_models = kwutil.Yaml.coerce( + config.enable_local_huggingface_models + ) if config.run_entry is None: raise SystemExit('Missing required --run-entry') From d17e50d506eb49ab1b3c1b20c95af71c2890ed14 Mon Sep 17 00:00:00 2001 From: joncrall Date: Tue, 24 Mar 2026 16:32:03 -0400 Subject: [PATCH 13/36] wip --- dev/benchmark/benchmark_loaders.py | 2 +- .../audit-helm-reproduction/python/render_schedule_params.py | 2 +- magnet/backends/helm/cli/materialize_helm_run.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/benchmark/benchmark_loaders.py b/dev/benchmark/benchmark_loaders.py index af1e48d..ca62261 100644 --- a/dev/benchmark/benchmark_loaders.py +++ b/dev/benchmark/benchmark_loaders.py @@ -1,4 +1,4 @@ -from magnet.helm_outputs import HelmRun +from magnet.backends.helm.helm_outputs import HelmRun import ubelt as ub import timerit from typing import Iterator diff --git a/dev/experiments/audit-helm-reproduction/python/render_schedule_params.py b/dev/experiments/audit-helm-reproduction/python/render_schedule_params.py index 15481c2..d58572a 100644 --- a/dev/experiments/audit-helm-reproduction/python/render_schedule_params.py +++ b/dev/experiments/audit-helm-reproduction/python/render_schedule_params.py @@ -10,7 +10,7 @@ def build_schedule_params(manifest: dict) -> dict: matrix = { "helm.run_entry": list(manifest["run_entries"]), "helm.max_eval_instances": [manifest["max_eval_instances"]], - "helm.precomputed_root": [manifest.get("precomputed_root", None)], + "helm.precomputed_root": manifest.get("precomputed_root", None), "helm.suite": [manifest.get("suite", "audit-smoke")], "helm.require_per_instance_stats": [ manifest.get("require_per_instance_stats", True) diff --git a/magnet/backends/helm/cli/materialize_helm_run.py b/magnet/backends/helm/cli/materialize_helm_run.py index 34ac35a..7f17f6c 100755 --- a/magnet/backends/helm/cli/materialize_helm_run.py +++ b/magnet/backends/helm/cli/materialize_helm_run.py @@ -151,7 +151,7 @@ # We rely on MAGNET's HELM output exploration helpers. # These are already present in aiq-magnet and know how to load / validate # the standard json files produced by helm-run. -from magnet.helm_outputs import HelmOutputs +from magnet.backends.helm.helm_outputs import HelmOutputs def _normalize_optional_pathish(value): From f7c2845e147fa7e089b8025c184c778cd95734b8 Mon Sep 17 00:00:00 2001 From: joncrall Date: Tue, 24 Mar 2026 16:37:31 -0400 Subject: [PATCH 14/36] wip --- .../scripts/run_from_manifest.sh | 2 + .../scripts/run_smoke.sh | 1 + magnet/backends/helm/pipeline.py | 40 +++++++++++++++++-- 3 files changed, 40 insertions(+), 3 deletions(-) diff --git a/dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh b/dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh index 39bd336..bc9ab67 100755 --- a/dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh +++ b/dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -euo pipefail +set +x SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${SCRIPT_DIR}/common.sh" @@ -34,6 +35,7 @@ printf 'tmux_workers: %s\n' "$TMUX_WORKERS" QUEUE_NAME="$(printf 'audit-%s' "$EXPERIMENT_NAME" | tr -c 'A-Za-z0-9._-' '-')" printf 'queue_name: %s\n' "$QUEUE_NAME" +printf 'Schedule params:\n%s\n' "$PARAMS" kwdagger schedule \ --queue_name="$QUEUE_NAME" \ diff --git a/dev/experiments/audit-helm-reproduction/scripts/run_smoke.sh b/dev/experiments/audit-helm-reproduction/scripts/run_smoke.sh index a658dee..fd37374 100755 --- a/dev/experiments/audit-helm-reproduction/scripts/run_smoke.sh +++ b/dev/experiments/audit-helm-reproduction/scripts/run_smoke.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -euo pipefail +set +x SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${SCRIPT_DIR}/common.sh" diff --git a/magnet/backends/helm/pipeline.py b/magnet/backends/helm/pipeline.py index 1c26dd0..d4e5ef5 100644 --- a/magnet/backends/helm/pipeline.py +++ b/magnet/backends/helm/pipeline.py @@ -35,6 +35,8 @@ from __future__ import annotations +import shlex + import kwdagger @@ -85,8 +87,8 @@ class MaterializeHelmRunNode(kwdagger.ProcessNode): 'max_eval_instances': None, 'require_per_instance_stats': True, 'model_deployments_fpath': None, - 'enable_huggingface_models': [], - 'enable_local_huggingface_models': [], + 'enable_huggingface_models': None, + 'enable_local_huggingface_models': None, # Behavior toggles that change how/what we materialize 'mode': 'compute_if_missing', # reuse_only | compute_if_missing | force_recompute 'materialize': 'symlink', # symlink | copy @@ -96,12 +98,44 @@ class MaterializeHelmRunNode(kwdagger.ProcessNode): # These are recorded, but ideally should not change the “meaning” of outputs. perf_params = { # Your shared precomputed root: - 'precomputed_root': '/data/crfm-helm-public', + 'precomputed_root': None, # helm-run perf knobs: 'num_threads': 1, 'local_path': 'prod_env', } + @property + def command(self) -> str: + """ + Render CLI args while omitting unset optional values. + + This keeps kwdagger-facing params in a clean key/value style and avoids + emitting placeholders such as ``--foo=None`` or bare flags for empty + list defaults. + """ + filtered_config = {} + for key, value in self.final_config.items(): + if value is None: + continue + if isinstance(value, list) and len(value) == 0: + continue + filtered_config[key] = value + parts = [] + for key, value in filtered_config.items(): + if isinstance(value, dict): + from kwutil.util_yaml import Yaml + value_text = shlex.quote(Yaml.dumps(value)) + if '\n' in value_text and value_text[0] == "'": + value_text = "'\n" + value_text[1:] + parts.append(f' --{key}={value_text} \\') + else: + value_text = shlex.quote(str(value)) + parts.append(f' --{key}={value_text} \\') + argstr = '\n'.join(parts).lstrip().rstrip('\\') + if argstr: + return self.executable + ' \\\n ' + argstr + return self.executable + # Optional: You can define load_result if you want kwdagger aggregate to read # something out of this node. Usually this node is an “adapter/materializer” # and a downstream eval node would implement load_result instead. From c46f3aa1a33ee7bc29b68704d4eb0a40bca18868 Mon Sep 17 00:00:00 2001 From: joncrall Date: Tue, 24 Mar 2026 16:58:46 -0400 Subject: [PATCH 15/36] wip --- .../python/compare_batch.py | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/dev/experiments/audit-helm-reproduction/python/compare_batch.py b/dev/experiments/audit-helm-reproduction/python/compare_batch.py index 1bd7b08..f331e6e 100644 --- a/dev/experiments/audit-helm-reproduction/python/compare_batch.py +++ b/dev/experiments/audit-helm-reproduction/python/compare_batch.py @@ -54,20 +54,26 @@ def infer_benchmark_group(run_spec_name: str | None) -> str: def load_kwdg_rows(results_dpath: Path) -> tuple[list[dict[str, Any]], dict[str, dict[str, Any]]]: - finished_jobs = sorted(results_dpath.glob("*/DONE")) + finished_jobs = sorted( + fpath + for fpath in results_dpath.rglob("DONE") + if (fpath.parent / "job_config.json").exists() + ) rows = [] for fpath in ub.ProgIter(finished_jobs, desc="load kwdg runs"): dpath = fpath.parent try: config = kwutil.Json.load(dpath / "job_config.json") - run_spec_name = config["helm.run_entry"] - suites = HelmOutputs.coerce(dpath / "benchmark_output").suites() - if not suites: + run_spec_name = config.get("helm.run_entry", None) + if run_spec_name is None: continue - runs = suites[0].runs() + suites = HelmOutputs.coerce(dpath / "benchmark_output").suites() + runs = [] + for suite in suites: + runs.extend(list(suite.runs())) if len(runs) != 1: continue - run = runs[0] + run = HelmRun.coerce(runs[0]) rows.append( { "dpath": str(dpath), @@ -167,10 +173,16 @@ def build_historic_rows( def write_summary_text( summary_report: dict[str, Any], out_fpath: Path ) -> None: + inputs = summary_report.get("inputs", {}) or {} lines = [] lines.append(f"generated_utc: {summary_report['generated_utc']}") lines.append(f"case_jsonl: {summary_report['report_case_jsonl']}") lines.append(f"summary_json: {summary_report['report_summary_json']}") + if inputs: + lines.append("") + lines.append("inputs:") + for key, value in sorted(inputs.items()): + lines.append(f" {key}: {value}") lines.append("") lines.append("status_counts:") for key, value in sorted( @@ -349,6 +361,7 @@ def main() -> None: "precomputed_root": str(precomputed_root), "n_manifest_run_entries": len(manifest["run_entries"]), "n_kwdg_rows": len(kwdg_rows), + "n_historic_rows": len(historic_rows), }, "aggregate": aggregate_report(all_case_rows), } From c66e59bac2de9ebc4d9629013f845b71267e0f52 Mon Sep 17 00:00:00 2001 From: agent Date: Tue, 24 Mar 2026 21:08:42 +0000 Subject: [PATCH 16/36] wip --- .../python/compare_batch.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/dev/experiments/audit-helm-reproduction/python/compare_batch.py b/dev/experiments/audit-helm-reproduction/python/compare_batch.py index f331e6e..ec911c9 100644 --- a/dev/experiments/audit-helm-reproduction/python/compare_batch.py +++ b/dev/experiments/audit-helm-reproduction/python/compare_batch.py @@ -94,6 +94,7 @@ def aggregate_report(rows: list[dict[str, Any]]) -> dict[str, Any]: status_counter = Counter() diagnosis_counter = Counter() reason_counter = Counter() + primary_reason_counter = Counter() for row in rows: status = row.get("status", "unknown") status_counter[status] += 1 @@ -101,6 +102,8 @@ def aggregate_report(rows: list[dict[str, Any]]) -> dict[str, Any]: continue diag = row.get("diagnosis", {}) or {} diagnosis_counter[diag.get("label", "unknown")] += 1 + for reason_name in diag.get("primary_reason_names", []) or []: + primary_reason_counter[reason_name] += 1 for reason in diag.get("reasons", []) or []: name = reason.get("name", "unknown") reason_counter[name] += 1 @@ -108,6 +111,7 @@ def aggregate_report(rows: list[dict[str, Any]]) -> dict[str, Any]: "n_rows": len(rows), "status_counts": dict(status_counter), "diagnosis_label_counts": dict(diagnosis_counter), + "primary_reason_name_counts": dict(primary_reason_counter), "reason_counts": dict(reason_counter), } @@ -195,6 +199,18 @@ def write_summary_text( summary_report["aggregate"]["diagnosis_label_counts"].items() ): lines.append(f" {key}: {value}") + lines.append("") + lines.append("primary_reason_name_counts:") + for key, value in sorted( + summary_report["aggregate"].get("primary_reason_name_counts", {}).items() + ): + lines.append(f" {key}: {value}") + lines.append("") + lines.append("reason_counts:") + for key, value in sorted( + summary_report["aggregate"].get("reason_counts", {}).items() + ): + lines.append(f" {key}: {value}") out_fpath.write_text("\n".join(lines) + "\n") From c7d995680407f1b46f6e1e30ee1a3b0eb332b040 Mon Sep 17 00:00:00 2001 From: joncrall Date: Tue, 24 Mar 2026 17:29:23 -0400 Subject: [PATCH 17/36] more metadata --- .../python/compare_batch.py | 255 +++++++++++++++++- 1 file changed, 245 insertions(+), 10 deletions(-) diff --git a/dev/experiments/audit-helm-reproduction/python/compare_batch.py b/dev/experiments/audit-helm-reproduction/python/compare_batch.py index ec911c9..304d879 100644 --- a/dev/experiments/audit-helm-reproduction/python/compare_batch.py +++ b/dev/experiments/audit-helm-reproduction/python/compare_batch.py @@ -17,7 +17,8 @@ experiment_result_dpath, load_manifest, ) -from magnet.backends.helm.cli.materialize_helm_run import find_best_precomputed_run +from magnet.backends.helm.cli.materialize_helm_run import discover_benchmark_output_dirs +from magnet.backends.helm.cli.materialize_helm_run import run_dir_matches_requested from magnet.backends.helm.helm_outputs import HelmOutputs, HelmRun from magnet.backends.helm.helm_run_diff import HelmRunDiff @@ -43,6 +44,14 @@ def parse_helm_run_dir(run_dir: str) -> dict[str, str]: return out +def load_run_spec_json(run_dir: str | Path) -> dict[str, Any]: + run_dir = Path(run_dir) + fpath = run_dir / "run_spec.json" + if not fpath.exists(): + return {} + return json.loads(fpath.read_text()) + + def infer_benchmark_group(run_spec_name: str | None) -> str: text = (run_spec_name or "").strip() if not text: @@ -53,6 +62,88 @@ def infer_benchmark_group(run_spec_name: str | None) -> str: return text +def collect_historic_candidates( + precomputed_root: str | Path, + run_entry: str, +) -> list[dict[str, Any]]: + candidates = [] + for bo in discover_benchmark_output_dirs([precomputed_root]): + try: + outputs = HelmOutputs.coerce(bo) + except Exception: + continue + for suite in outputs.suites(pattern="*"): + for run in suite.runs(pattern="*"): + run_dir = Path(run.path) + if not run_dir_matches_requested(run.name, run_entry): + continue + run_spec = load_run_spec_json(run_dir) + adapter_spec = run_spec.get("adapter_spec", {}) or {} + metric_specs = run_spec.get("metric_specs", []) or [] + candidates.append( + { + "run_dir": run_dir, + "run_name": run.name, + "source_root": bo, + "helm_version": run_dir.parent.name, + "requested_max_eval_instances": adapter_spec.get( + "max_eval_instances", None + ), + "model_deployment": adapter_spec.get( + "model_deployment", None + ), + "metric_class_names": [ + m.get("class_name", None) for m in metric_specs + ], + } + ) + return candidates + + +def choose_historic_candidate( + candidates: list[dict[str, Any]], + desired_max_eval_instances: int | None, +) -> tuple[dict[str, Any] | None, dict[str, Any]]: + if not candidates: + return None, { + "candidate_count": 0, + "exact_requested_max_eval_match": False, + "candidate_requested_max_eval_instances": [], + } + exact_matches = [] + if desired_max_eval_instances is not None: + exact_matches = [ + c + for c in candidates + if c.get("requested_max_eval_instances", None) + == desired_max_eval_instances + ] + ranked_pool = exact_matches if exact_matches else candidates + + def sort_key(c: dict[str, Any]): + req = c.get("requested_max_eval_instances", None) + req_dist = ( + abs(req - desired_max_eval_instances) + if req is not None and desired_max_eval_instances is not None + else float("inf") + ) + return (req_dist, str(c.get("helm_version", "")), str(c["run_dir"])) + + chosen = sorted(ranked_pool, key=sort_key)[0] + info = { + "candidate_count": len(candidates), + "exact_requested_max_eval_match": bool(exact_matches), + "candidate_requested_max_eval_instances": sorted( + {c.get("requested_max_eval_instances", None) for c in candidates}, + key=lambda x: (x is None, x), + ), + "chosen_requested_max_eval_instances": chosen.get( + "requested_max_eval_instances", None + ), + } + return chosen, info + + def load_kwdg_rows(results_dpath: Path) -> tuple[list[dict[str, Any]], dict[str, dict[str, Any]]]: finished_jobs = sorted( fpath @@ -116,6 +207,85 @@ def aggregate_report(rows: list[dict[str, Any]]) -> dict[str, Any]: } +def build_high_level_findings(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + findings = [] + n_rows = len(rows) + if n_rows == 0: + return findings + + compared = sum(1 for row in rows if row.get("status") == "compared") + if compared == n_rows: + findings.append( + { + "label": "comparison_pipeline_working", + "severity": "info", + "summary": f"All {compared} requested runs were paired and compared successfully.", + } + ) + + requested_max_mismatch = sum( + 1 + for row in rows + if row.get("historic_requested_max_eval_instances", None) + != row.get("kwdg_requested_max_eval_instances", None) + ) + if requested_max_mismatch: + findings.append( + { + "label": "requested_max_eval_mismatch", + "severity": "high", + "summary": ( + f"{requested_max_mismatch}/{n_rows} cases use a historic run with " + "a different requested max_eval_instances value." + ), + } + ) + + no_exact_match = sum( + 1 + for row in rows + if not row.get("historic_exact_requested_max_eval_match", False) + ) + if no_exact_match: + findings.append( + { + "label": "no_exact_historic_eval_size_match", + "severity": "high", + "summary": ( + f"{no_exact_match}/{n_rows} cases did not have an exact historic match " + "for requested max_eval_instances in the public bundle." + ), + } + ) + + for reason_name, severity in [ + ("deployment_drift", "high"), + ("execution_spec_drift", "high"), + ("evaluation_spec_drift", "medium"), + ("dataset_variant_drift", "medium"), + ("dataset_instance_drift", "medium"), + ("core_metric_drift", "medium"), + ("completion_content_drift", "medium"), + ]: + count = sum( + 1 + for row in rows + if any( + reason.get("name") == reason_name + for reason in ((row.get("diagnosis", {}) or {}).get("reasons", []) or []) + ) + ) + if count: + findings.append( + { + "label": reason_name, + "severity": severity, + "summary": f"{reason_name} appears in {count}/{n_rows} compared cases.", + } + ) + return findings + + def maybe_write_sankey_report( case_rows: list[dict[str, Any]], report_dpath: Path, stamp: str ) -> dict[str, Any]: @@ -144,13 +314,13 @@ def build_historic_rows( ) -> list[dict[str, Any]]: rows = [] for run_entry in manifest["run_entries"]: - match = find_best_precomputed_run( + desired_max_eval_instances = manifest.get("max_eval_instances", None) + candidates = collect_historic_candidates( precomputed_root=precomputed_root, - requested_desc=run_entry, - max_eval_instances=manifest.get("max_eval_instances", None), - require_per_instance_stats=manifest.get( - "require_per_instance_stats", True - ), + run_entry=run_entry, + ) + match, match_info = choose_historic_candidate( + candidates, desired_max_eval_instances ) row = { "run_spec_name": run_entry, @@ -160,13 +330,22 @@ def build_historic_rows( "benchmark_name": "unknown", "suite_name": "unknown", "helm_version": "unknown", + "requested_max_eval_instances": None, + "model_deployment": None, + "metric_class_names": [], + "match_info": match_info, } if match is not None: - parsed = parse_helm_run_dir(str(match.run_dir)) - row["run_dir"] = str(match.run_dir) + parsed = parse_helm_run_dir(str(match["run_dir"])) + row["run_dir"] = str(match["run_dir"]) row["benchmark_name"] = parsed["helm_suite_name"] row["suite_name"] = parsed["helm_suite_name"] - row["helm_version"] = parsed["helm_version"] + row["helm_version"] = str(match.get("helm_version", parsed["helm_version"])) + row["requested_max_eval_instances"] = match.get( + "requested_max_eval_instances", None + ) + row["model_deployment"] = match.get("model_deployment", None) + row["metric_class_names"] = match.get("metric_class_names", []) if "model=" in run_entry: model_text = run_entry.split("model=", 1)[1].split(",", 1)[0] row["model"] = model_text @@ -211,6 +390,39 @@ def write_summary_text( summary_report["aggregate"].get("reason_counts", {}).items() ): lines.append(f" {key}: {value}") + findings = summary_report.get("high_level_findings", []) or [] + if findings: + lines.append("") + lines.append("high_level_findings:") + for item in findings: + lines.append( + f" [{item.get('severity', 'info')}] {item.get('label')}: {item.get('summary')}" + ) + out_fpath.write_text("\n".join(lines) + "\n") + + +def write_management_summary( + summary_report: dict[str, Any], out_fpath: Path +) -> None: + inputs = summary_report.get("inputs", {}) or {} + findings = summary_report.get("high_level_findings", []) or [] + aggregate = summary_report.get("aggregate", {}) or {} + status_counts = aggregate.get("status_counts", {}) or {} + compared = status_counts.get("compared", 0) + total = sum(status_counts.values()) + lines = [] + lines.append("Audit HELM Reproduction: Executive Summary") + lines.append("") + lines.append( + f"Compared {inputs.get('n_manifest_run_entries', '?')} requested runs against " + f"{inputs.get('n_historic_rows', '?')} historic matches and " + f"{inputs.get('n_kwdg_rows', '?')} reproduced runs." + ) + lines.append(f"{compared}/{total} runs completed comparison successfully.") + lines.append("") + lines.append("Key findings:") + for item in findings: + lines.append(f"- [{item.get('severity', 'info').upper()}] {item.get('summary')}") out_fpath.write_text("\n".join(lines) + "\n") @@ -246,6 +458,7 @@ def main() -> None: case_jsonl_fpath = report_dpath / f"compare_cases_{stamp}.jsonl" summary_json_fpath = report_dpath / f"compare_summary_{stamp}.json" summary_txt_fpath = report_dpath / f"compare_summary_{stamp}.txt" + management_txt_fpath = report_dpath / f"management_summary_{stamp}.txt" all_case_rows = [] with case_jsonl_fpath.open("w", encoding="utf8") as file: @@ -262,6 +475,20 @@ def main() -> None: "helm_version": helm_row.get("helm_version", None), "helm_run_dir": helm_row["run_dir"], "kwdg_run_dir": None if kwrow is None else kwrow["dpath"], + "historic_requested_max_eval_instances": helm_row.get( + "requested_max_eval_instances", None + ), + "historic_model_deployment": helm_row.get( + "model_deployment", None + ), + "historic_metric_class_names": helm_row.get( + "metric_class_names", [] + ), + "historic_match_info": helm_row.get("match_info", {}), + "historic_exact_requested_max_eval_match": ( + helm_row.get("match_info", {}) or {} + ).get("exact_requested_max_eval_match", False), + "kwdg_requested_max_eval_instances": None, } if helm_row["run_dir"] is None: @@ -306,6 +533,10 @@ def main() -> None: try: helm_run = HelmRun.coerce(helm_row["run_dir"]) kwdg_run = kwrow["run"] + kwdg_run_spec = kwdg_run.run_spec().iloc[0].to_dict() + case_row["kwdg_requested_max_eval_instances"] = kwdg_run_spec.get( + "adapter_spec.max_eval_instances", None + ) rd = HelmRunDiff( run_a=helm_run, run_b=kwdg_run, @@ -370,6 +601,7 @@ def main() -> None: "report_case_jsonl": str(case_jsonl_fpath), "report_summary_json": str(summary_json_fpath), "report_summary_txt": str(summary_txt_fpath), + "report_management_txt": str(management_txt_fpath), "generated_utc": stamp, "inputs": { "manifest": str(Path(args.manifest).expanduser().resolve()), @@ -380,6 +612,7 @@ def main() -> None: "n_historic_rows": len(historic_rows), }, "aggregate": aggregate_report(all_case_rows), + "high_level_findings": build_high_level_findings(all_case_rows), } try: sankey_artifacts = maybe_write_sankey_report( @@ -395,10 +628,12 @@ def main() -> None: json.dumps(summary_report, indent=2, ensure_ascii=False) ) write_summary_text(summary_report, summary_txt_fpath) + write_management_summary(summary_report, management_txt_fpath) print(f"Wrote case report: {case_jsonl_fpath}") print(f"Wrote summary report: {summary_json_fpath}") print(f"Wrote summary text: {summary_txt_fpath}") + print(f"Wrote management summary: {management_txt_fpath}") if sankey_artifacts.get("plotly_error", None): print(f"Sankey note: {sankey_artifacts['plotly_error']}") From 83b353237110f0fa8e0feee0124f1deaccbc2130 Mon Sep 17 00:00:00 2001 From: joncrall Date: Tue, 24 Mar 2026 18:09:44 -0400 Subject: [PATCH 18/36] Add apples to apples comparison --- .../audit-helm-reproduction/README.md | 25 +++++ .../configs/apples_manifest.yaml | 23 +++++ .../python/make_manifest.py | 99 ++++++++++++++----- .../scripts/make_apples_manifest.sh | 18 ++++ 4 files changed, 140 insertions(+), 25 deletions(-) create mode 100644 dev/experiments/audit-helm-reproduction/configs/apples_manifest.yaml create mode 100644 dev/experiments/audit-helm-reproduction/scripts/make_apples_manifest.sh diff --git a/dev/experiments/audit-helm-reproduction/README.md b/dev/experiments/audit-helm-reproduction/README.md index 6f8b404..ddefa4f 100644 --- a/dev/experiments/audit-helm-reproduction/README.md +++ b/dev/experiments/audit-helm-reproduction/README.md @@ -64,12 +64,24 @@ dev/experiments/audit-helm-reproduction/scripts/check_env.sh dev/experiments/audit-helm-reproduction/scripts/make_smoke_manifest.sh ``` +To materialize the first apples-to-apples control manifest instead: + +```bash +dev/experiments/audit-helm-reproduction/scripts/make_apples_manifest.sh +``` + By default this writes: ```text dev/experiments/audit-helm-reproduction/configs/generated/smoke_manifest.generated.yaml ``` +And the apples-to-apples variant writes: + +```text +dev/experiments/audit-helm-reproduction/configs/generated/apples_manifest.generated.yaml +``` + 3. Launch the smoke-test batch on the GPU machine: ```bash @@ -126,6 +138,7 @@ Fields: See: - `configs/smoke_manifest.yaml` +- `configs/apples_manifest.yaml` - `configs/manifest_template.yaml` ## Smoke-Test Batch @@ -152,6 +165,18 @@ Defaults: - low worker count - no custom deployment override YAML +## Apples-To-Apples Smoke Batch + +The first apples-to-apples control batch reuses the same 6 smoke entries, but +aligns `max_eval_instances` with the historic public bundle for those entries: + +- `max_eval_instances=1000` +- experiment name: `audit-smoke-apples` +- suite: `audit-smoke-apples` + +This is the preferred first batch when the goal is reproduction fidelity rather +than just workflow validation. + ## Scaling Up For larger experiments: diff --git a/dev/experiments/audit-helm-reproduction/configs/apples_manifest.yaml b/dev/experiments/audit-helm-reproduction/configs/apples_manifest.yaml new file mode 100644 index 0000000..e7f20cf --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/configs/apples_manifest.yaml @@ -0,0 +1,23 @@ +schema_version: 1 +experiment_name: audit-smoke-apples +description: Apples-to-apples smoke batch aligned to the historic public HELM requested max_eval_instances for the control entries. +run_entries: + - mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=eleutherai/pythia-6.9b,data_augmentation=canonical + - boolq:model=eleutherai/pythia-6.9b,data_augmentation=canonical + - narrative_qa:model=eleutherai/pythia-6.9b,data_augmentation=canonical + - mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical + - boolq:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical + - narrative_qa:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical +max_eval_instances: 1000 +suite: audit-smoke-apples +mode: compute_if_missing +materialize: symlink +backend: tmux +devices: 0,1 +tmux_workers: 2 +local_path: prod_env +precomputed_root: null +require_per_instance_stats: true +model_deployments_fpath: null +enable_huggingface_models: [] +enable_local_huggingface_models: [] diff --git a/dev/experiments/audit-helm-reproduction/python/make_manifest.py b/dev/experiments/audit-helm-reproduction/python/make_manifest.py index aa49191..7e7a3ce 100644 --- a/dev/experiments/audit-helm-reproduction/python/make_manifest.py +++ b/dev/experiments/audit-helm-reproduction/python/make_manifest.py @@ -26,34 +26,29 @@ def _validate_entries_exist(run_entries: list[str]) -> list[str]: return missing -def build_smoke_manifest(args: argparse.Namespace) -> dict: - defaults = env_defaults() - max_eval_instances = ( - args.max_eval_instances - if args.max_eval_instances is not None - else int(defaults["AUDIT_DEFAULT_MAX_EVAL_INSTANCES"]) - ) - tmux_workers = ( - args.tmux_workers - if args.tmux_workers is not None - else int(defaults["AUDIT_DEFAULT_TMUX_WORKERS"]) - ) - devices = args.devices if args.devices is not None else os.environ.get("CUDA_VISIBLE_DEVICES", "0,1") - - missing = _validate_entries_exist(SMOKE_RUN_ENTRIES) +def _build_manifest( + *, + experiment_name: str, + description: str, + run_entries: list[str], + max_eval_instances: int, + suite: str, + tmux_workers: int, + devices: str, +) -> dict: + missing = _validate_entries_exist(run_entries) if missing: raise RuntimeError( - "Smoke manifest entries were not found in run_specs.yaml: " + "Manifest entries were not found in run_specs.yaml: " + kwutil.Json.dumps(missing) ) - return { "schema_version": 1, - "experiment_name": args.experiment_name, - "description": "Small smoke-test batch for HELM reproduction auditing.", - "run_entries": SMOKE_RUN_ENTRIES, + "experiment_name": experiment_name, + "description": description, + "run_entries": run_entries, "max_eval_instances": max_eval_instances, - "suite": args.suite, + "suite": suite, "mode": "compute_if_missing", "materialize": "symlink", "backend": "tmux", @@ -68,10 +63,62 @@ def build_smoke_manifest(args: argparse.Namespace) -> dict: } +def build_smoke_manifest(args: argparse.Namespace) -> dict: + defaults = env_defaults() + max_eval_instances = ( + args.max_eval_instances + if args.max_eval_instances is not None + else int(defaults["AUDIT_DEFAULT_MAX_EVAL_INSTANCES"]) + ) + tmux_workers = ( + args.tmux_workers + if args.tmux_workers is not None + else int(defaults["AUDIT_DEFAULT_TMUX_WORKERS"]) + ) + devices = args.devices if args.devices is not None else os.environ.get("CUDA_VISIBLE_DEVICES", "0,1") + return _build_manifest( + experiment_name=args.experiment_name, + description="Small smoke-test batch for HELM reproduction auditing.", + run_entries=SMOKE_RUN_ENTRIES, + max_eval_instances=max_eval_instances, + suite=args.suite, + tmux_workers=tmux_workers, + devices=devices, + ) + + +def build_apples_manifest(args: argparse.Namespace) -> dict: + defaults = env_defaults() + # Historic public matches for the current smoke-control entries all use 1000. + max_eval_instances = ( + args.max_eval_instances + if args.max_eval_instances is not None + else 1000 + ) + tmux_workers = ( + args.tmux_workers + if args.tmux_workers is not None + else int(defaults["AUDIT_DEFAULT_TMUX_WORKERS"]) + ) + devices = args.devices if args.devices is not None else os.environ.get("CUDA_VISIBLE_DEVICES", "0,1") + return _build_manifest( + experiment_name=args.experiment_name, + description=( + "Apples-to-apples smoke batch aligned to the historic public HELM " + "requested max_eval_instances for the control entries." + ), + run_entries=SMOKE_RUN_ENTRIES, + max_eval_instances=max_eval_instances, + suite=args.suite, + tmux_workers=tmux_workers, + devices=devices, + ) + + def main() -> None: parser = argparse.ArgumentParser() parser.add_argument( - "--manifest-type", default="smoke", choices=["smoke"] + "--manifest-type", default="smoke", choices=["smoke", "apples"] ) parser.add_argument("--output", required=True) parser.add_argument("--experiment-name", default="audit-smoke") @@ -81,10 +128,12 @@ def main() -> None: parser.add_argument("--devices", default=None) args = parser.parse_args() - if args.manifest_type != "smoke": + if args.manifest_type == "smoke": + manifest = build_smoke_manifest(args) + elif args.manifest_type == "apples": + manifest = build_apples_manifest(args) + else: raise NotImplementedError(args.manifest_type) - - manifest = build_smoke_manifest(args) out_fpath = Path(args.output) out_fpath.parent.mkdir(parents=True, exist_ok=True) out_fpath.write_text(dump_yaml(manifest)) diff --git a/dev/experiments/audit-helm-reproduction/scripts/make_apples_manifest.sh b/dev/experiments/audit-helm-reproduction/scripts/make_apples_manifest.sh new file mode 100644 index 0000000..2258490 --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/scripts/make_apples_manifest.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/common.sh" + +audit::set_defaults + +OUTPUT="${1:-${AUDIT_ROOT}/configs/generated/apples_manifest.generated.yaml}" +mkdir -p "$(dirname "$OUTPUT")" + +"$AIQ_PYTHON" "${AUDIT_ROOT}/python/make_manifest.py" \ + --manifest-type apples \ + --experiment-name audit-smoke-apples \ + --suite audit-smoke-apples \ + --output "$OUTPUT" + +printf 'Wrote apples-to-apples manifest: %s\n' "$OUTPUT" From 12a781a7780013339f98fa2a7f46b2f5b2f6d13c Mon Sep 17 00:00:00 2001 From: joncrall Date: Tue, 24 Mar 2026 18:12:35 -0400 Subject: [PATCH 19/36] apples fixes --- dev/experiments/audit-helm-reproduction/README.md | 12 ++++++++++-- .../examples/example_smoke_commands.sh | 3 +-- .../audit-helm-reproduction/python/make_manifest.py | 4 ++-- .../scripts/make_apples_manifest.sh | 6 +++++- .../scripts/make_smoke_manifest.sh | 6 +++++- 5 files changed, 23 insertions(+), 8 deletions(-) diff --git a/dev/experiments/audit-helm-reproduction/README.md b/dev/experiments/audit-helm-reproduction/README.md index ddefa4f..561eb65 100644 --- a/dev/experiments/audit-helm-reproduction/README.md +++ b/dev/experiments/audit-helm-reproduction/README.md @@ -29,7 +29,6 @@ Assumptions: Optional environment variables: -- `CUDA_VISIBLE_DEVICES` - `AUDIT_DEFAULT_MAX_EVAL_INSTANCES` - `AUDIT_DEFAULT_TMUX_WORKERS` @@ -85,10 +84,18 @@ dev/experiments/audit-helm-reproduction/configs/generated/apples_manifest.genera 3. Launch the smoke-test batch on the GPU machine: ```bash -CUDA_VISIBLE_DEVICES=0,1 \ dev/experiments/audit-helm-reproduction/scripts/run_smoke.sh ``` +To change which GPUs `kwdagger` schedules onto, set the manifest `devices` +field or regenerate the manifest with `--devices`, for example: + +```bash +dev/experiments/audit-helm-reproduction/scripts/make_smoke_manifest.sh \ + dev/experiments/audit-helm-reproduction/configs/generated/smoke_manifest.generated.yaml \ + --devices 2,3 +``` + 4. Compare the completed batch to the historic HELM bundle: ```bash @@ -173,6 +180,7 @@ aligns `max_eval_instances` with the historic public bundle for those entries: - `max_eval_instances=1000` - experiment name: `audit-smoke-apples` - suite: `audit-smoke-apples` +- devices are still controlled by the manifest `devices` field or `--devices` This is the preferred first batch when the goal is reproduction fidelity rather than just workflow validation. diff --git a/dev/experiments/audit-helm-reproduction/examples/example_smoke_commands.sh b/dev/experiments/audit-helm-reproduction/examples/example_smoke_commands.sh index 5f897b6..56a50ee 100755 --- a/dev/experiments/audit-helm-reproduction/examples/example_smoke_commands.sh +++ b/dev/experiments/audit-helm-reproduction/examples/example_smoke_commands.sh @@ -5,6 +5,5 @@ AUDIT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" "$AUDIT_ROOT/scripts/check_env.sh" "$AUDIT_ROOT/scripts/make_smoke_manifest.sh" -CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1}" \ - "$AUDIT_ROOT/scripts/run_smoke.sh" +"$AUDIT_ROOT/scripts/run_smoke.sh" "$AUDIT_ROOT/scripts/compare_batch.sh" diff --git a/dev/experiments/audit-helm-reproduction/python/make_manifest.py b/dev/experiments/audit-helm-reproduction/python/make_manifest.py index 7e7a3ce..c237f91 100644 --- a/dev/experiments/audit-helm-reproduction/python/make_manifest.py +++ b/dev/experiments/audit-helm-reproduction/python/make_manifest.py @@ -75,7 +75,7 @@ def build_smoke_manifest(args: argparse.Namespace) -> dict: if args.tmux_workers is not None else int(defaults["AUDIT_DEFAULT_TMUX_WORKERS"]) ) - devices = args.devices if args.devices is not None else os.environ.get("CUDA_VISIBLE_DEVICES", "0,1") + devices = args.devices if args.devices is not None else "0,1" return _build_manifest( experiment_name=args.experiment_name, description="Small smoke-test batch for HELM reproduction auditing.", @@ -100,7 +100,7 @@ def build_apples_manifest(args: argparse.Namespace) -> dict: if args.tmux_workers is not None else int(defaults["AUDIT_DEFAULT_TMUX_WORKERS"]) ) - devices = args.devices if args.devices is not None else os.environ.get("CUDA_VISIBLE_DEVICES", "0,1") + devices = args.devices if args.devices is not None else "0,1" return _build_manifest( experiment_name=args.experiment_name, description=( diff --git a/dev/experiments/audit-helm-reproduction/scripts/make_apples_manifest.sh b/dev/experiments/audit-helm-reproduction/scripts/make_apples_manifest.sh index 2258490..b037d22 100644 --- a/dev/experiments/audit-helm-reproduction/scripts/make_apples_manifest.sh +++ b/dev/experiments/audit-helm-reproduction/scripts/make_apples_manifest.sh @@ -7,12 +7,16 @@ source "${SCRIPT_DIR}/common.sh" audit::set_defaults OUTPUT="${1:-${AUDIT_ROOT}/configs/generated/apples_manifest.generated.yaml}" +if [[ $# -gt 0 ]]; then + shift +fi mkdir -p "$(dirname "$OUTPUT")" "$AIQ_PYTHON" "${AUDIT_ROOT}/python/make_manifest.py" \ --manifest-type apples \ --experiment-name audit-smoke-apples \ --suite audit-smoke-apples \ - --output "$OUTPUT" + --output "$OUTPUT" \ + "$@" printf 'Wrote apples-to-apples manifest: %s\n' "$OUTPUT" diff --git a/dev/experiments/audit-helm-reproduction/scripts/make_smoke_manifest.sh b/dev/experiments/audit-helm-reproduction/scripts/make_smoke_manifest.sh index a21636a..f959ee0 100755 --- a/dev/experiments/audit-helm-reproduction/scripts/make_smoke_manifest.sh +++ b/dev/experiments/audit-helm-reproduction/scripts/make_smoke_manifest.sh @@ -7,10 +7,14 @@ source "${SCRIPT_DIR}/common.sh" audit::set_defaults OUTPUT="${1:-${AUDIT_ROOT}/configs/generated/smoke_manifest.generated.yaml}" +if [[ $# -gt 0 ]]; then + shift +fi mkdir -p "$(dirname "$OUTPUT")" "$AIQ_PYTHON" "${AUDIT_ROOT}/python/make_manifest.py" \ --manifest-type smoke \ - --output "$OUTPUT" + --output "$OUTPUT" \ + "$@" printf 'Wrote smoke manifest: %s\n' "$OUTPUT" From a775599673d0912c13f723d589953b300f244b85 Mon Sep 17 00:00:00 2001 From: joncrall Date: Tue, 24 Mar 2026 18:13:32 -0400 Subject: [PATCH 20/36] wip --- .../audit-helm-reproduction/scripts/make_apples_manifest.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 dev/experiments/audit-helm-reproduction/scripts/make_apples_manifest.sh diff --git a/dev/experiments/audit-helm-reproduction/scripts/make_apples_manifest.sh b/dev/experiments/audit-helm-reproduction/scripts/make_apples_manifest.sh old mode 100644 new mode 100755 From 63ac4fa7475093e864de3cf66eb674628cc571f2 Mon Sep 17 00:00:00 2001 From: joncrall Date: Thu, 26 Mar 2026 11:24:10 -0400 Subject: [PATCH 21/36] more diff --- .../audit-helm-reproduction/README.md | 46 +++++++++++++++++++ .../python/compare_batch.py | 8 ++-- 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/dev/experiments/audit-helm-reproduction/README.md b/dev/experiments/audit-helm-reproduction/README.md index 561eb65..843d8ba 100644 --- a/dev/experiments/audit-helm-reproduction/README.md +++ b/dev/experiments/audit-helm-reproduction/README.md @@ -185,6 +185,52 @@ aligns `max_eval_instances` with the historic public bundle for those entries: This is the preferred first batch when the goal is reproduction fidelity rather than just workflow validation. +Suggested operator flow: + +```bash +dev/experiments/audit-helm-reproduction/scripts/make_apples_manifest.sh \ + dev/experiments/audit-helm-reproduction/configs/generated/apples_manifest.generated.yaml \ + --devices 0,1 + +dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh \ + dev/experiments/audit-helm-reproduction/configs/generated/apples_manifest.generated.yaml + +dev/experiments/audit-helm-reproduction/scripts/compare_batch.sh \ + dev/experiments/audit-helm-reproduction/configs/generated/apples_manifest.generated.yaml +``` + +Raw reproduced outputs are written to: + +```text +/data/crfm-helm-audit/audit-smoke-apples/ +``` + +Comparison reports are written to: + +```text +dev/experiments/audit-helm-reproduction/reports/audit-smoke-apples/ +``` + +Files to inspect first: + +- `management_summary_.txt` +- `compare_summary_.txt` +- `compare_cases_.jsonl` + +Files to transfer back for local analysis: + +- the entire report directory +- optionally the entire raw results directory if deeper run-by-run inspection is needed + +Example transfer commands depend on your setup, but a simple pattern is: + +```bash +ls -td dev/experiments/audit-helm-reproduction/reports/audit-smoke-apples/* +ls -td /data/crfm-helm-audit/audit-smoke-apples/* +``` + +Then transfer the newest report files and, if needed, the raw experiment root. + ## Scaling Up For larger experiments: diff --git a/dev/experiments/audit-helm-reproduction/python/compare_batch.py b/dev/experiments/audit-helm-reproduction/python/compare_batch.py index 304d879..557f216 100644 --- a/dev/experiments/audit-helm-reproduction/python/compare_batch.py +++ b/dev/experiments/audit-helm-reproduction/python/compare_batch.py @@ -533,9 +533,11 @@ def main() -> None: try: helm_run = HelmRun.coerce(helm_row["run_dir"]) kwdg_run = kwrow["run"] - kwdg_run_spec = kwdg_run.run_spec().iloc[0].to_dict() - case_row["kwdg_requested_max_eval_instances"] = kwdg_run_spec.get( - "adapter_spec.max_eval_instances", None + kwdg_run_spec = load_run_spec_json(kwdg_run.path) + case_row["kwdg_requested_max_eval_instances"] = ( + kwdg_run_spec.get("adapter_spec", {}) or {} + ).get( + "max_eval_instances", None ) rd = HelmRunDiff( run_a=helm_run, From facfed13cd07068e608837ddd407a34edb71453e Mon Sep 17 00:00:00 2001 From: joncrall Date: Thu, 26 Mar 2026 22:00:02 -0400 Subject: [PATCH 22/36] wip --- .../audit-helm-reproduction/README.md | 173 ++++++ .../python/compare_pair.py | 181 ++++++ .../python/core_metric_report.py | 536 ++++++++++++++++++ .../python/make_manifest.py | 37 +- .../python/metric_quantiles_report.py | 181 ++++++ .../python/resolve_run.py | 134 +++++ .../scripts/compare_entry.sh | 62 ++ .../scripts/compare_pair.sh | 16 + .../scripts/core_metric_report.sh | 9 + .../scripts/make_repeat_pair_manifests.sh | 47 ++ .../scripts/metric_quantiles_report.sh | 9 + .../scripts/resolve_run.sh | 9 + magnet/backends/helm/helm_run_diff.py | 446 +++++++++++++++ 13 files changed, 1839 insertions(+), 1 deletion(-) create mode 100644 dev/experiments/audit-helm-reproduction/python/compare_pair.py create mode 100644 dev/experiments/audit-helm-reproduction/python/core_metric_report.py create mode 100644 dev/experiments/audit-helm-reproduction/python/metric_quantiles_report.py create mode 100644 dev/experiments/audit-helm-reproduction/python/resolve_run.py create mode 100755 dev/experiments/audit-helm-reproduction/scripts/compare_entry.sh create mode 100755 dev/experiments/audit-helm-reproduction/scripts/compare_pair.sh create mode 100755 dev/experiments/audit-helm-reproduction/scripts/core_metric_report.sh create mode 100755 dev/experiments/audit-helm-reproduction/scripts/make_repeat_pair_manifests.sh create mode 100755 dev/experiments/audit-helm-reproduction/scripts/metric_quantiles_report.sh create mode 100755 dev/experiments/audit-helm-reproduction/scripts/resolve_run.sh diff --git a/dev/experiments/audit-helm-reproduction/README.md b/dev/experiments/audit-helm-reproduction/README.md index 843d8ba..245d875 100644 --- a/dev/experiments/audit-helm-reproduction/README.md +++ b/dev/experiments/audit-helm-reproduction/README.md @@ -118,6 +118,42 @@ The runner also derives a distinct `kwdagger` queue name from the experiment name, which helps avoid interactive tmux collision prompts when multiple audit batches have been launched on the same machine. +## Reproducibility Checklist + +For any experiment you want to cite later, preserve all of the following: + +- the exact manifest YAML used to launch the run +- the exact results root under: + - `/data/crfm-helm-audit//` +- the generated comparison reports under: + - `dev/experiments/audit-helm-reproduction/reports//` +- the current git commit of `aiq-magnet` +- the Python executable used as `AIQ_PYTHON` +- the value of: + - `AIQ_MAGNET_ROOT` + - `HELM_PRECOMPUTED_ROOT` + - `AUDIT_RESULTS_ROOT` + +Recommended capture commands: + +```bash +git rev-parse HEAD +which "$AIQ_PYTHON" +dev/experiments/audit-helm-reproduction/scripts/check_env.sh +``` + +If you need to transfer the run to another machine for analysis, transfer: + +- the manifest YAML +- the report directory for the experiment +- the raw results directory for the experiment + +Minimum useful transfer set: + +- report directory only, if you only need summaries +- report directory plus 1-2 representative raw job directories, if you need direct run artifact inspection +- full raw experiment directory, if you may need to rerun local comparisons later + ## Manifest Schema The workflow uses a small YAML manifest as the unit of experiment definition. @@ -231,6 +267,104 @@ ls -td /data/crfm-helm-audit/audit-smoke-apples/* Then transfer the newest report files and, if needed, the raw experiment root. +## Exact Reproduction Cases Used In This Research + +The following cases were used as the first reproducibility controls in this +research thread. + +### Apples-To-Apples Control Batch + +Purpose: + +- compare current local kwdagger reproductions against official public HELM with matched `max_eval_instances` + +Manifest generation: + +```bash +dev/experiments/audit-helm-reproduction/scripts/make_apples_manifest.sh \ + dev/experiments/audit-helm-reproduction/configs/generated/apples_manifest.generated.yaml \ + --devices 0,1 +``` + +Run: + +```bash +dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh \ + dev/experiments/audit-helm-reproduction/configs/generated/apples_manifest.generated.yaml +``` + +Compare: + +```bash +dev/experiments/audit-helm-reproduction/scripts/compare_batch.sh \ + dev/experiments/audit-helm-reproduction/configs/generated/apples_manifest.generated.yaml +``` + +Outputs: + +- raw results: + - `/data/crfm-helm-audit/audit-smoke-apples/` +- reports: + - `dev/experiments/audit-helm-reproduction/reports/audit-smoke-apples/` + +### Pairwise Repeatability Control: BoolQ / Pythia + +Purpose: + +- measure ordinary local rerun drift on the same benchmark/model pair + +Manifest files used: + +- `dev/experiments/audit-helm-reproduction/configs/generated/boolq_pythia_r1.yaml` +- `dev/experiments/audit-helm-reproduction/configs/generated/boolq_pythia_r2.yaml` + +Run: + +```bash +dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh \ + dev/experiments/audit-helm-reproduction/configs/generated/boolq_pythia_r1.yaml + +dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh \ + dev/experiments/audit-helm-reproduction/configs/generated/boolq_pythia_r2.yaml +``` + +Direct pairwise compare of the two completed runs: + +```bash +dev/experiments/audit-helm-reproduction/scripts/compare_pair.sh \ + /data/crfm-helm-audit/audit-boolq-pythia-r1/helm/helm_id_13jkx9mm4k4n/benchmark_output/runs/audit-boolq-pythia-r1/boolq:model=eleutherai_pythia-6.9b,data_augmentation=canonical \ + /data/crfm-helm-audit/audit-boolq-pythia-r2/helm/helm_id_12jr5w48kge7/benchmark_output/runs/audit-boolq-pythia-r2/boolq:model=eleutherai_pythia-6.9b,data_augmentation=canonical \ + dev/experiments/audit-helm-reproduction/reports/pairwise/boolq-pythia-repeat +``` + +### Pairwise Official-vs-Local Control: BoolQ / Pythia + +Purpose: + +- compare one local reproduced run directly against the matched public HELM run + +Direct compare: + +```bash +dev/experiments/audit-helm-reproduction/scripts/compare_pair.sh \ + /data/crfm-helm-public/classic/benchmark_output/runs/v0.3.0/boolq:model=eleutherai_pythia-6.9b,data_augmentation=canonical \ + /data/crfm-helm-audit/audit-boolq-pythia-r1/helm/helm_id_13jkx9mm4k4n/benchmark_output/runs/audit-boolq-pythia-r1/boolq:model=eleutherai_pythia-6.9b,data_augmentation=canonical \ + dev/experiments/audit-helm-reproduction/reports/pairwise/boolq-pythia-historic +``` + +### Viewing The Key Reports + +```bash +cat dev/experiments/audit-helm-reproduction/reports/pairwise/boolq-pythia-repeat-wide/pair_report_20260327T011202Z.txt +cat dev/experiments/audit-helm-reproduction/reports/pairwise/boolq-pythia-historic-wide/pair_report_20260327T011202Z.txt +``` + +These two reports are the current best compact illustration of: + +- local repeatability drift being small +- official-vs-local drift being much larger +- global tolerance sweeps needing careful interpretation because metric scales differ across classes + ## Scaling Up For larger experiments: @@ -245,6 +379,45 @@ dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh /path/to/ma dev/experiments/audit-helm-reproduction/scripts/compare_batch.sh /path/to/manifest.yaml ``` +## Pairwise Run Reports + +To compare any two concrete HELM run directories directly, use: + +```bash +dev/experiments/audit-helm-reproduction/scripts/compare_pair.sh \ + /path/to/run_a \ + /path/to/run_b \ + dev/experiments/audit-helm-reproduction/reports/pairwise +``` + +This writes: + +- `pair_report_.json` +- `pair_report_.txt` + +The pairwise report includes: + +- strict diff diagnosis +- raw run-level distance distributions +- raw instance-level distance distributions +- tolerance sweeps across several preset thresholds + +Example using the local repeated `boolq + gpt2` runs: + +```bash +dev/experiments/audit-helm-reproduction/scripts/compare_pair.sh \ + /data/crfm-helm-audit/audit-boolq-gpt2-r1/helm/helm_id_lh2zobnkhuwi/benchmark_output/runs/audit-boolq-gpt2-r1/boolq:model=openai_gpt2,data_augmentation=canonical \ + /data/crfm-helm-audit/audit-boolq-gpt2-r2/helm/helm_id_lvb1vuf32m2g/benchmark_output/runs/audit-boolq-gpt2-r2/boolq:model=openai_gpt2,data_augmentation=canonical \ + dev/experiments/audit-helm-reproduction/reports/pairwise +``` + +Then inspect: + +```bash +ls -td dev/experiments/audit-helm-reproduction/reports/pairwise/* +cat dev/experiments/audit-helm-reproduction/reports/pairwise/pair_report_.txt +``` + ## Notes - This workflow is intended for execution on an external GPU machine. diff --git a/dev/experiments/audit-helm-reproduction/python/compare_pair.py b/dev/experiments/audit-helm-reproduction/python/compare_pair.py new file mode 100644 index 0000000..a1bc6a2 --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/python/compare_pair.py @@ -0,0 +1,181 @@ +from __future__ import annotations + +import argparse +import datetime as datetime_mod +import json +from pathlib import Path +from typing import Any + +import kwutil + +from magnet.backends.helm.helm_outputs import HelmRun +from magnet.backends.helm.helm_run_diff import HelmRunDiff + + +def load_yaml_or_default(text: str | None, default: list[dict[str, Any]]) -> list[dict[str, Any]]: + if text is None: + return default + data = kwutil.Yaml.coerce(text) + if not isinstance(data, list): + raise TypeError('Tolerance config must decode to a list of dictionaries') + return data + + +def default_tolerances() -> list[dict[str, Any]]: + return [ + {'name': 'strict', 'abs_tol': 0.0, 'rel_tol': 0.0}, + {'name': 'tiny', 'abs_tol': 1e-12, 'rel_tol': 1e-6}, + {'name': 'small', 'abs_tol': 1e-9, 'rel_tol': 1e-4}, + {'name': 'medium', 'abs_tol': 1e-6, 'rel_tol': 1e-3}, + {'name': 'loose', 'abs_tol': 1e-3, 'rel_tol': 1e-2}, + {'name': 'xloose', 'abs_tol': 1e-2, 'rel_tol': 1e-1}, + {'name': 'xxloose', 'abs_tol': 1e-1, 'rel_tol': 1.0}, + {'name': 'extreme', 'abs_tol': 1.0, 'rel_tol': 10.0}, + ] + + +def validate_run_dir(run_dpath: Path) -> None: + required_files = [ + 'run_spec.json', + 'scenario_state.json', + 'stats.json', + 'per_instance_stats.json', + ] + missing_files = [name for name in required_files if not (run_dpath / name).exists()] + if missing_files: + missing_text = ', '.join(missing_files) + raise SystemExit( + f'Run artifacts are incomplete for {run_dpath}. ' + f'Missing required files: {missing_text}' + ) + + +def summarize_tolerance_hits(sweep: dict[str, Any]) -> dict[str, Any]: + out = {'run_level': [], 'instance_level': []} + for level_key, target in [('run_level', 'overall'), ('instance_level', 'means')]: + for row in sweep.get(level_key, []): + summary = row.get('summary', {}) or {} + if level_key == 'run_level': + agree = ((summary.get('overall', {}) or {}).get('agree_ratio', None)) + else: + agree = ((summary.get('means', {}) or {}).get('agree_ratio', None)) + out[level_key].append({ + 'name': row.get('name'), + 'abs_tol': row.get('abs_tol'), + 'rel_tol': row.get('rel_tol'), + 'agree_ratio': agree, + }) + return out + + +def write_text_report(report: dict[str, Any], out_fpath: Path) -> None: + strict = report.get('strict_summary', {}) or {} + diag = strict.get('diagnosis', {}) or {} + run_dist = report.get('distance_summary', {}).get('run_level', {}) or {} + inst_dist = report.get('distance_summary', {}).get('instance_level', {}) or {} + sweep_hits = report.get('tolerance_highlights', {}) or {} + + lines = [] + lines.append('Audit Pair Comparison') + lines.append('') + lines.append(f"generated_utc: {report.get('generated_utc')}") + lines.append(f"run_a: {report.get('inputs', {}).get('run_a')}") + lines.append(f"run_b: {report.get('inputs', {}).get('run_b')}") + lines.append('') + lines.append(f"diagnosis_label: {diag.get('label')}") + lines.append(f"primary_reason_names: {diag.get('primary_reason_names')}") + lines.append('') + + lines.append('strict_agreement:') + overall = (strict.get('value_agreement', {}) or {}).get('overall', {}) or {} + lines.append(f" run_level_agree_ratio: {overall.get('agree_ratio')}") + means = (strict.get('instance_value_agreement', {}) or {}).get('means', {}) or {} + lines.append(f" instance_level_agree_ratio: {means.get('agree_ratio')}") + lines.append('') + + lines.append('distance_summary:') + lines.append(f" run_level_count: {(run_dist.get('overall', {}) or {}).get('count')}") + lines.append(f" run_level_abs_p50: {((run_dist.get('overall', {}) or {}).get('abs_delta', {}) or {}).get('p50')}") + lines.append(f" run_level_abs_p90: {((run_dist.get('overall', {}) or {}).get('abs_delta', {}) or {}).get('p90')}") + lines.append(f" run_level_abs_max: {((run_dist.get('overall', {}) or {}).get('abs_delta', {}) or {}).get('max')}") + lines.append(f" instance_level_count: {(inst_dist.get('overall', {}) or {}).get('count')}") + lines.append(f" instance_level_abs_p50: {((inst_dist.get('overall', {}) or {}).get('abs_delta', {}) or {}).get('p50')}") + lines.append(f" instance_level_abs_p90: {((inst_dist.get('overall', {}) or {}).get('abs_delta', {}) or {}).get('p90')}") + lines.append(f" instance_level_abs_max: {((inst_dist.get('overall', {}) or {}).get('abs_delta', {}) or {}).get('max')}") + lines.append('') + + lines.append('tolerance_sweep_run_level:') + for row in sweep_hits.get('run_level', []): + lines.append( + f" {row.get('name')}: abs_tol={row.get('abs_tol')} rel_tol={row.get('rel_tol')} agree_ratio={row.get('agree_ratio')}" + ) + lines.append('') + lines.append('tolerance_sweep_instance_level:') + for row in sweep_hits.get('instance_level', []): + lines.append( + f" {row.get('name')}: abs_tol={row.get('abs_tol')} rel_tol={row.get('rel_tol')} agree_ratio={row.get('agree_ratio')}" + ) + out_fpath.write_text('\n'.join(lines) + '\n') + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('--run-a', required=True) + parser.add_argument('--run-b', required=True) + parser.add_argument('--label-a', default='A') + parser.add_argument('--label-b', default='B') + parser.add_argument('--report-dpath', required=True) + parser.add_argument('--run-tolerances-yaml', default=None) + parser.add_argument('--instance-tolerances-yaml', default=None) + args = parser.parse_args() + + report_dpath = Path(args.report_dpath).expanduser().resolve() + report_dpath.mkdir(parents=True, exist_ok=True) + stamp = datetime_mod.datetime.now(datetime_mod.UTC).strftime('%Y%m%dT%H%M%SZ') + + run_a_dpath = Path(args.run_a).expanduser().resolve() + run_b_dpath = Path(args.run_b).expanduser().resolve() + validate_run_dir(run_a_dpath) + validate_run_dir(run_b_dpath) + + run_a = HelmRun.coerce(run_a_dpath) + run_b = HelmRun.coerce(run_b_dpath) + diff = HelmRunDiff(run_a=run_a, run_b=run_b, a_name=args.label_a, b_name=args.label_b) + + run_tolerances = load_yaml_or_default(args.run_tolerances_yaml, default_tolerances()) + instance_tolerances = load_yaml_or_default(args.instance_tolerances_yaml, default_tolerances()) + + strict_summary = diff.summary_dict(level=20) + distance_summary = { + 'run_level': diff.value_distance_profile(), + 'instance_level': diff.instance_distance_profile(), + } + tolerance_sweep = diff.tolerance_sweep_summary( + run_tolerances=run_tolerances, + instance_tolerances=instance_tolerances, + ) + report = { + 'generated_utc': stamp, + 'inputs': { + 'run_a': str(run_a_dpath), + 'run_b': str(run_b_dpath), + 'label_a': args.label_a, + 'label_b': args.label_b, + }, + 'strict_summary': strict_summary, + 'distance_summary': distance_summary, + 'tolerance_sweep': tolerance_sweep, + 'tolerance_highlights': summarize_tolerance_hits(tolerance_sweep), + } + + json_fpath = report_dpath / f'pair_report_{stamp}.json' + txt_fpath = report_dpath / f'pair_report_{stamp}.txt' + report = kwutil.Json.ensure_serializable(report) + json_fpath.write_text(json.dumps(report, indent=2, ensure_ascii=False)) + write_text_report(report, txt_fpath) + print(f'Wrote pair report: {json_fpath}') + print(f'Wrote pair text: {txt_fpath}') + + +if __name__ == '__main__': + main() diff --git a/dev/experiments/audit-helm-reproduction/python/core_metric_report.py b/dev/experiments/audit-helm-reproduction/python/core_metric_report.py new file mode 100644 index 0000000..5b513f5 --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/python/core_metric_report.py @@ -0,0 +1,536 @@ +from __future__ import annotations + +import argparse +import datetime as datetime_mod +import json +import math +from pathlib import Path +from typing import Any + +import kwutil +import matplotlib.pyplot as plt +import pandas as pd +import seaborn as sns + +from magnet.backends.helm.helm_outputs import HelmRun +from magnet.backends.helm.helm_run_analysis import HelmRunAnalysis +from magnet.backends.helm.helm_run_diff import HelmRunDiff +from magnet.backends.helm.util import helm_metrics + + +def _quantile(values: list[float], q: float) -> float | None: + if not values: + return None + values = sorted(values) + if q <= 0: + return values[0] + if q >= 1: + return values[-1] + pos = (len(values) - 1) * q + lo = math.floor(pos) + hi = math.ceil(pos) + if lo == hi: + return values[lo] + alpha = pos - lo + return values[lo] * (1 - alpha) + values[hi] * alpha + + +def _safe_float(x: Any) -> float | None: + try: + if x is None: + return None + return float(x) + except Exception: + return None + + +def _run_level_core_rows(diff: HelmRunDiff) -> list[dict[str, Any]]: + idx_a = diff.a.stat_index(drop_zero_count=True, require_mean=True, short_hash=diff.short_hash) + idx_b = diff.b.stat_index(drop_zero_count=True, require_mean=True, short_hash=diff.short_hash) + rows = [] + for k in set(idx_a) & set(idx_b): + a = idx_a[k] + b = idx_b[k] + if a.mean is None or b.mean is None: + continue + if a.metric_class != 'core': + continue + abs_delta = abs(a.mean - b.mean) + denom = max(abs(a.mean), abs(b.mean), 1e-12) + rel_delta = abs_delta / denom + rows.append({ + 'key': k, + 'metric': a.metric, + 'metric_class': a.metric_class, + 'a': float(a.mean), + 'b': float(b.mean), + 'abs_delta': abs_delta, + 'rel_delta': rel_delta, + }) + return rows + + +def _iter_joined_rows(joined, row_by_key): + if row_by_key is not None: + return row_by_key.values() + if isinstance(joined, dict): + return joined.values() + if hasattr(joined, '__iter__'): + return joined + return [] + + +def _row_key(row: Any) -> Any: + return ( + getattr(row, 'key', None) + or getattr(row, 'stat_key', None) + or getattr(row, 'row_key', None) + or row + ) + + +def _instance_level_core_rows(diff: HelmRunDiff) -> list[dict[str, Any]]: + joined_a = diff.a.joined_instance_stat_table(assert_assumptions=False, short_hash=diff.short_hash) + joined_b = diff.b.joined_instance_stat_table(assert_assumptions=False, short_hash=diff.short_hash) + map_a = getattr(joined_a, 'row_by_key', None) + map_b = getattr(joined_b, 'row_by_key', None) + if map_a is None: + map_a = {_row_key(r): r for r in _iter_joined_rows(joined_a, map_a)} + if map_b is None: + map_b = {_row_key(r): r for r in _iter_joined_rows(joined_b, map_b)} + + rows = [] + for k in set(map_a) & set(map_b): + ra = map_a[k] + rb = map_b[k] + sa = getattr(ra, 'stat', None) if hasattr(ra, 'stat') else (ra.get('stat') if isinstance(ra, dict) else None) + sb = getattr(rb, 'stat', None) if hasattr(rb, 'stat') else (rb.get('stat') if isinstance(rb, dict) else None) + ma = _safe_float((sa or {}).get('mean') if isinstance(sa, dict) else getattr(sa, 'mean', None)) + mb = _safe_float((sb or {}).get('mean') if isinstance(sb, dict) else getattr(sb, 'mean', None)) + ca = int((sa or {}).get('count', 0) or 0) if isinstance(sa, dict) else int(getattr(sa, 'count', 0) or 0) + cb = int((sb or {}).get('count', 0) or 0) if isinstance(sb, dict) else int(getattr(sb, 'count', 0) or 0) + if ma is None or mb is None or ca == 0 or cb == 0: + continue + name_obj = (sa or {}).get('name') if isinstance(sa, dict) else getattr(sa, 'name_obj', None) + metric = name_obj.get('name') if isinstance(name_obj, dict) else getattr(sa, 'metric', None) + metric_class, _ = helm_metrics.classify_metric(metric) + if metric_class != 'core': + continue + abs_delta = abs(ma - mb) + denom = max(abs(ma), abs(mb), 1e-12) + rel_delta = abs_delta / denom + rows.append({ + 'key': k, + 'metric': metric, + 'metric_class': metric_class, + 'a': ma, + 'b': mb, + 'abs_delta': abs_delta, + 'rel_delta': rel_delta, + }) + return rows + + +def _group_quantiles(rows: list[dict[str, Any]]) -> dict[str, Any]: + values = sorted(float(r['abs_delta']) for r in rows) + return { + 'count': len(values), + 'abs_delta': { + 'min': _quantile(values, 0.0), + 'p50': _quantile(values, 0.5), + 'p90': _quantile(values, 0.9), + 'p95': _quantile(values, 0.95), + 'p99': _quantile(values, 0.99), + 'max': _quantile(values, 1.0), + }, + } + + +def _metric_quantiles(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + by_metric: dict[str, list[dict[str, Any]]] = {} + for row in rows: + by_metric.setdefault(str(row['metric']), []).append(row) + out = [] + for metric, items in sorted(by_metric.items()): + info = _group_quantiles(items) + info['metric'] = metric + out.append(info) + return out + + +def _agreement_curve(rows: list[dict[str, Any]], thresholds: list[float]) -> list[dict[str, Any]]: + if not rows: + return [] + vals = [float(r['abs_delta']) for r in rows] + out = [] + for t in thresholds: + matched = sum(v <= t for v in vals) + out.append({ + 'abs_tol': t, + 'agree_ratio': matched / len(vals), + 'matched': matched, + 'count': len(vals), + }) + return out + + +def _build_pair(run_a: str, run_b: str, label: str, thresholds: list[float]) -> dict[str, Any]: + diff = HelmRunDiff(HelmRun.coerce(run_a), HelmRun.coerce(run_b), a_name=f'{label}:A', b_name=f'{label}:B') + run_rows = _run_level_core_rows(diff) + inst_rows = _instance_level_core_rows(diff) + return { + 'label': label, + 'inputs': { + 'run_a': str(Path(run_a).expanduser().resolve()), + 'run_b': str(Path(run_b).expanduser().resolve()), + }, + 'diagnosis': diff.summary_dict(level=20).get('diagnosis', {}), + 'core_metrics': sorted({str(r['metric']) for r in inst_rows}), + 'run_level': { + 'overall_quantiles': _group_quantiles(run_rows), + 'by_metric': _metric_quantiles(run_rows), + 'agreement_vs_abs_tol': _agreement_curve(run_rows, thresholds), + }, + 'instance_level': { + 'overall_quantiles': _group_quantiles(inst_rows), + 'by_metric': _metric_quantiles(inst_rows), + 'agreement_vs_abs_tol': _agreement_curve(inst_rows, thresholds), + }, + '_instance_rows': inst_rows, + } + + +def _agreement_curve_rows(pair_a: dict[str, Any], pair_b: dict[str, Any], level_key: str) -> list[dict[str, Any]]: + rows = [] + for pair in [pair_a, pair_b]: + for row in pair[level_key]['agreement_vs_abs_tol']: + rows.append({ + 'pair': pair['label'], + 'abs_tol': float(row['abs_tol']), + 'agree_ratio': float(row['agree_ratio']), + }) + return rows + + +def _plot_distribution(ax, pair_a: dict[str, Any], pair_b: dict[str, Any], level_key: str) -> None: + rows = pd.DataFrame(_agreement_curve_rows(pair_a, pair_b, level_key)) + sns.lineplot( + ax=ax, + data=rows, + x='abs_tol', + y='agree_ratio', + hue='pair', + style='pair', + markers=True, + dashes=False, + linewidth=2, + ) + ax.set_xscale('symlog', linthresh=1e-12) + ax.set_ylim(0, 1.02) + ax.set_xlabel('Absolute tolerance') + ax.set_ylabel('Agreement ratio') + ax.legend(title='') + + +def _plot_quantiles(ax, pair_a: dict[str, Any], pair_b: dict[str, Any], level_key: str, title: str) -> None: + labels = ['p50', 'p90', 'p95', 'p99', 'max'] + x = list(range(len(labels))) + a_vals = [pair_a[level_key]['overall_quantiles']['abs_delta'][k] for k in labels] + b_vals = [pair_b[level_key]['overall_quantiles']['abs_delta'][k] for k in labels] + ax.plot(x, a_vals, marker='o', label=pair_a['label']) + ax.plot(x, b_vals, marker='o', label=pair_b['label']) + ax.set_xticks(x, labels) + ax.set_yscale('symlog', linthresh=1e-12) + ax.set_title(title) + ax.set_ylabel('Absolute delta quantile') + ax.legend() + + +def _distribution_rows(pair: dict[str, Any]) -> pd.DataFrame: + rows = [] + for row in pair.get('_instance_rows', []): + rows.append({ + 'pair': pair['label'], + 'metric': row['metric'], + 'side': 'A', + 'value': float(row['a']), + }) + rows.append({ + 'pair': pair['label'], + 'metric': row['metric'], + 'side': 'B', + 'value': float(row['b']), + }) + return pd.DataFrame(rows) + + +def _plot_metric_distributions(fig_dpath: Path, stamp: str, left: dict[str, Any], right: dict[str, Any]) -> Path: + df = pd.concat([ + _distribution_rows(left), + _distribution_rows(right), + ], ignore_index=True) + metrics = sorted(df['metric'].dropna().unique().tolist()) + pair_order = [left['label'], right['label']] + fig, axes = plt.subplots(len(pair_order), len(metrics), figsize=(4 * len(metrics), 3.5 * len(pair_order)), constrained_layout=True) + if len(pair_order) == 1 and len(metrics) == 1: + axes = [[axes]] + elif len(pair_order) == 1: + axes = [axes] + elif len(metrics) == 1: + axes = [[ax] for ax in axes] + for row_idx, pair_label in enumerate(pair_order): + for col_idx, metric in enumerate(metrics): + ax = axes[row_idx][col_idx] + sub = df[(df['pair'] == pair_label) & (df['metric'] == metric)] + sns.histplot( + data=sub, + x='value', + hue='side', + stat='probability', + common_norm=False, + discrete=True, + multiple='dodge', + shrink=0.8, + ax=ax, + ) + ax.set_title(f'{pair_label}\n{metric}') + ax.set_xlabel('Core metric value') + ax.set_ylabel('Probability') + legend = ax.get_legend() + if legend is not None: + legend.set_title('') + out_fpath = fig_dpath / f'core_metric_distributions_{stamp}.png' + fig.savefig(out_fpath, dpi=180) + plt.close(fig) + return out_fpath + + +def _single_run_instance_core_rows(run_path: str, label: str) -> pd.DataFrame: + ana = HelmRunAnalysis(HelmRun.coerce(run_path), name=label) + joined = ana.joined_instance_stat_table(assert_assumptions=False) + row_by_key = getattr(joined, 'row_by_key', None) or {} + rows = [] + for row in row_by_key.values(): + stat = row.stat + mean = _safe_float(stat.get('mean')) + count = int(stat.get('count', 0) or 0) + if mean is None or count == 0: + continue + name_obj = stat.get('name', {}) + metric = name_obj.get('name') + metric_class, _ = helm_metrics.classify_metric(metric) + if metric_class != 'core': + continue + rows.append({ + 'run': label, + 'metric': metric, + 'value': float(mean), + }) + return pd.DataFrame(rows) + + +def _plot_three_run_metric_distributions( + fig_dpath: Path, + stamp: str, + kwdagger_a_run: str, + kwdagger_b_run: str, + official_run: str, +) -> Path: + df = pd.concat([ + _single_run_instance_core_rows(kwdagger_a_run, 'kwdagger A'), + _single_run_instance_core_rows(kwdagger_b_run, 'kwdagger B'), + _single_run_instance_core_rows(official_run, 'official'), + ], ignore_index=True) + metrics = sorted(df['metric'].dropna().unique().tolist()) + run_order = ['kwdagger A', 'kwdagger B', 'official'] + fig, axes = plt.subplots( + len(metrics), + len(run_order), + figsize=(4 * len(run_order), 2.8 * len(metrics)), + constrained_layout=True, + ) + if len(metrics) == 1 and len(run_order) == 1: + axes = [[axes]] + elif len(metrics) == 1: + axes = [axes] + elif len(run_order) == 1: + axes = [[ax] for ax in axes] + for row_idx, metric in enumerate(metrics): + for col_idx, run_label in enumerate(run_order): + ax = axes[row_idx][col_idx] + sub = df[(df['metric'] == metric) & (df['run'] == run_label)] + sns.histplot( + data=sub, + x='value', + stat='probability', + discrete=True, + shrink=0.8, + ax=ax, + color='#4C72B0', + ) + if row_idx == 0: + ax.set_title(run_label) + ax.set_xlabel('Core metric value') + ax.set_ylabel(metric if col_idx == 0 else '') + out_fpath = fig_dpath / f'core_metric_three_run_distributions_{stamp}.png' + fig.savefig(out_fpath, dpi=180) + plt.close(fig) + return out_fpath + + +def _single_run_core_stat_index(run_path: str) -> dict[str, Any]: + ana = HelmRunAnalysis(HelmRun.coerce(run_path)) + idx = ana.stat_index(drop_zero_count=True, require_mean=True) + return {k: v for k, v in idx.items() if v.metric_class == 'core'} + + +def _write_three_run_runlevel_table( + out_dpath: Path, + stamp: str, + kwdagger_a_run: str, + kwdagger_b_run: str, + official_run: str, +) -> tuple[Path, Path]: + idx_a = _single_run_core_stat_index(kwdagger_a_run) + idx_b = _single_run_core_stat_index(kwdagger_b_run) + idx_o = _single_run_core_stat_index(official_run) + keys = sorted(set(idx_a) & set(idx_b) & set(idx_o)) + rows = [] + for key in keys: + a = idx_a[key] + b = idx_b[key] + o = idx_o[key] + rows.append({ + 'stat_key': key, + 'metric': a.metric, + 'kwdagger_a': a.mean, + 'kwdagger_b': b.mean, + 'official': o.mean, + 'delta_official_vs_kwdagger_a': None if a.mean is None or o.mean is None else abs(o.mean - a.mean), + 'delta_kwdagger_a_vs_kwdagger_b': None if a.mean is None or b.mean is None else abs(a.mean - b.mean), + }) + table = pd.DataFrame(rows) + csv_fpath = out_dpath / f'core_runlevel_table_{stamp}.csv' + md_fpath = out_dpath / f'core_runlevel_table_{stamp}.md' + table.to_csv(csv_fpath, index=False) + md_fpath.write_text(table.to_markdown(index=False) + '\n') + return csv_fpath, md_fpath + + +def _strip_private(obj: Any) -> Any: + if isinstance(obj, dict): + return { + k: _strip_private(v) + for k, v in obj.items() + if not str(k).startswith('_') + } + if isinstance(obj, list): + return [_strip_private(v) for v in obj] + return obj + + +def _write_text(report: dict[str, Any], out_fpath: Path) -> None: + left, right = report['pairs'] + lines = [] + lines.append('Core Metric Report') + lines.append('') + lines.append(f"generated_utc: {report['generated_utc']}") + lines.append(f"left_label: {left['label']}") + lines.append(f"right_label: {right['label']}") + lines.append('') + lines.append('core_metrics:') + for metric in left['core_metrics']: + lines.append(f' - {metric}') + lines.append('') + for pair in report['pairs']: + lines.append(f"pair: {pair['label']}") + lines.append(f" diagnosis: {pair['diagnosis'].get('label')}") + lines.append(f" primary_reason_names: {pair['diagnosis'].get('primary_reason_names')}") + lines.append(f" run_level_quantiles: {json.dumps(pair['run_level']['overall_quantiles']['abs_delta'])}") + lines.append(f" instance_level_quantiles: {json.dumps(pair['instance_level']['overall_quantiles']['abs_delta'])}") + lines.append(' by_metric:') + for row in pair['instance_level']['by_metric']: + lines.append( + f" - metric={row['metric']} count={row['count']} " + f"p50={row['abs_delta']['p50']} p90={row['abs_delta']['p90']} " + f"p95={row['abs_delta']['p95']} p99={row['abs_delta']['p99']} " + f"max={row['abs_delta']['max']}" + ) + lines.append(' agreement_vs_abs_tol:') + for row in pair['instance_level']['agreement_vs_abs_tol']: + lines.append( + f" - abs_tol={row['abs_tol']} agree_ratio={row['agree_ratio']}" + ) + lines.append('') + out_fpath.write_text('\n'.join(lines) + '\n') + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('--left-run-a', required=True) + parser.add_argument('--left-run-b', required=True) + parser.add_argument('--left-label', required=True) + parser.add_argument('--right-run-a', required=True) + parser.add_argument('--right-run-b', required=True) + parser.add_argument('--right-label', required=True) + parser.add_argument('--report-dpath', required=True) + args = parser.parse_args() + + thresholds = [0.0, 1e-12, 1e-9, 1e-6, 1e-4, 1e-3, 1e-2, 2e-2, 5e-2, 1e-1, 5e-1, 1.0] + report_dpath = Path(args.report_dpath).expanduser().resolve() + report_dpath.mkdir(parents=True, exist_ok=True) + stamp = datetime_mod.datetime.now(datetime_mod.UTC).strftime('%Y%m%dT%H%M%SZ') + + left = _build_pair(args.left_run_a, args.left_run_b, args.left_label, thresholds) + right = _build_pair(args.right_run_a, args.right_run_b, args.right_label, thresholds) + report = { + 'generated_utc': stamp, + 'thresholds': thresholds, + 'pairs': [left, right], + } + + json_fpath = report_dpath / f'core_metric_report_{stamp}.json' + txt_fpath = report_dpath / f'core_metric_report_{stamp}.txt' + fig_fpath = report_dpath / f'core_metric_report_{stamp}.png' + dist_fig_fpath = _plot_metric_distributions(report_dpath, stamp, left, right) + three_run_dist_fpath = _plot_three_run_metric_distributions( + report_dpath, + stamp, + args.left_run_a, + args.left_run_b, + args.right_run_a, + ) + runlevel_csv_fpath, runlevel_md_fpath = _write_three_run_runlevel_table( + report_dpath, + stamp, + args.left_run_a, + args.left_run_b, + args.right_run_a, + ) + + report = kwutil.Json.ensure_serializable(_strip_private(report)) + json_fpath.write_text(json.dumps(report, indent=2)) + _write_text(report, txt_fpath) + + sns.set_theme(style='whitegrid', context='talk') + fig, axes = plt.subplots(2, 2, figsize=(12, 8), constrained_layout=True) + _plot_quantiles(axes[0, 0], left, right, 'run_level', 'Core Run-Level Abs Delta Quantiles') + _plot_quantiles(axes[0, 1], left, right, 'instance_level', 'Core Instance-Level Abs Delta Quantiles') + _plot_distribution(axes[1, 0], left, right, 'run_level') + axes[1, 0].set_title('Core Run-Level Agreement vs Abs Tol') + _plot_distribution(axes[1, 1], left, right, 'instance_level') + axes[1, 1].set_title('Core Instance-Level Agreement vs Abs Tol') + fig.savefig(fig_fpath, dpi=180) + plt.close(fig) + + print(f'Wrote core metric report: {json_fpath}') + print(f'Wrote core metric text: {txt_fpath}') + print(f'Wrote core metric plot: {fig_fpath}') + print(f'Wrote core metric distributions: {dist_fig_fpath}') + print(f'Wrote core metric three-run distributions: {three_run_dist_fpath}') + print(f'Wrote core run-level table csv: {runlevel_csv_fpath}') + print(f'Wrote core run-level table md: {runlevel_md_fpath}') + + +if __name__ == '__main__': + main() diff --git a/dev/experiments/audit-helm-reproduction/python/make_manifest.py b/dev/experiments/audit-helm-reproduction/python/make_manifest.py index c237f91..50b81f5 100644 --- a/dev/experiments/audit-helm-reproduction/python/make_manifest.py +++ b/dev/experiments/audit-helm-reproduction/python/make_manifest.py @@ -115,10 +115,41 @@ def build_apples_manifest(args: argparse.Namespace) -> dict: ) +def build_single_manifest(args: argparse.Namespace) -> dict: + defaults = env_defaults() + if not args.run_entry: + raise SystemExit('--run-entry is required for --manifest-type single') + max_eval_instances = ( + args.max_eval_instances + if args.max_eval_instances is not None + else int(defaults["AUDIT_DEFAULT_MAX_EVAL_INSTANCES"]) + ) + tmux_workers = ( + args.tmux_workers + if args.tmux_workers is not None + else int(defaults["AUDIT_DEFAULT_TMUX_WORKERS"]) + ) + devices = args.devices if args.devices is not None else "0" + description = ( + args.description + if args.description is not None + else f"Single-run audit manifest for {args.run_entry}" + ) + return _build_manifest( + experiment_name=args.experiment_name, + description=description, + run_entries=[args.run_entry], + max_eval_instances=max_eval_instances, + suite=args.suite, + tmux_workers=tmux_workers, + devices=devices, + ) + + def main() -> None: parser = argparse.ArgumentParser() parser.add_argument( - "--manifest-type", default="smoke", choices=["smoke", "apples"] + "--manifest-type", default="smoke", choices=["smoke", "apples", "single"] ) parser.add_argument("--output", required=True) parser.add_argument("--experiment-name", default="audit-smoke") @@ -126,12 +157,16 @@ def main() -> None: parser.add_argument("--max-eval-instances", type=int, default=None) parser.add_argument("--tmux-workers", type=int, default=None) parser.add_argument("--devices", default=None) + parser.add_argument("--run-entry", default=None) + parser.add_argument("--description", default=None) args = parser.parse_args() if args.manifest_type == "smoke": manifest = build_smoke_manifest(args) elif args.manifest_type == "apples": manifest = build_apples_manifest(args) + elif args.manifest_type == "single": + manifest = build_single_manifest(args) else: raise NotImplementedError(args.manifest_type) out_fpath = Path(args.output) diff --git a/dev/experiments/audit-helm-reproduction/python/metric_quantiles_report.py b/dev/experiments/audit-helm-reproduction/python/metric_quantiles_report.py new file mode 100644 index 0000000..79ae83a --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/python/metric_quantiles_report.py @@ -0,0 +1,181 @@ +from __future__ import annotations + +import argparse +import datetime as datetime_mod +import json +from pathlib import Path +from typing import Any + +import kwutil + +from magnet.backends.helm.helm_outputs import HelmRun +from magnet.backends.helm.helm_run_diff import HelmRunDiff + + +def _pair_report(run_a: str, run_b: str, label: str) -> dict[str, Any]: + diff = HelmRunDiff( + run_a=HelmRun.coerce(run_a), + run_b=HelmRun.coerce(run_b), + a_name=f'{label}:A', + b_name=f'{label}:B', + ) + return { + 'label': label, + 'inputs': { + 'run_a': str(Path(run_a).expanduser().resolve()), + 'run_b': str(Path(run_b).expanduser().resolve()), + }, + 'run_level': { + 'agreement': diff._value_agreement_summary(), + 'distance': diff.value_distance_profile(), + }, + 'instance_level': { + 'agreement': diff.instance_agreement_profile(), + 'distance': diff.instance_distance_profile(), + }, + 'diagnosis': diff.summary_dict(level=20).get('diagnosis', {}), + } + + +def _instance_metric_rows(pair: dict[str, Any]) -> dict[tuple[str, str | None], dict[str, Any]]: + distance_rows = pair['instance_level']['distance'].get('by_metric', []) + agreement_rows = pair['instance_level']['agreement'].get('by_metric', []) + agree_lut = { + (row.get('metric_class'), row.get('metric')): row + for row in agreement_rows + } + rows = {} + for row in distance_rows: + key = (row.get('metric_class'), row.get('metric')) + rows[key] = { + 'metric_class': row.get('metric_class'), + 'metric': row.get('metric'), + 'count': row.get('summary', {}).get('count'), + 'agree_ratio': agree_lut.get(key, {}).get('agree_ratio'), + 'mismatched': agree_lut.get(key, {}).get('mismatched'), + 'abs_delta': row.get('summary', {}).get('abs_delta', {}), + 'rel_delta': row.get('summary', {}).get('rel_delta', {}), + } + return rows + + +def _run_class_rows(pair: dict[str, Any]) -> dict[str, dict[str, Any]]: + agree = pair['run_level']['agreement'].get('by_class', {}) + dist = pair['run_level']['distance'].get('by_class', {}) + rows = {} + for cls in sorted(set(agree) | set(dist)): + rows[cls] = { + 'metric_class': cls, + 'comparable': (agree.get(cls) or {}).get('comparable'), + 'agree_ratio': (agree.get(cls) or {}).get('agree_ratio'), + 'mismatched': (agree.get(cls) or {}).get('mismatched'), + 'abs_delta': (dist.get(cls) or {}).get('abs_delta', {}), + 'rel_delta': (dist.get(cls) or {}).get('rel_delta', {}), + 'count': (dist.get(cls) or {}).get('count'), + } + return rows + + +def _fmt(x: Any) -> str: + if x is None: + return 'NA' + if isinstance(x, float): + return f'{x:.6g}' + return str(x) + + +def _build_text(report: dict[str, Any]) -> str: + left = report['pairs'][0] + right = report['pairs'][1] + left_run = _run_class_rows(left) + right_run = _run_class_rows(right) + left_inst = _instance_metric_rows(left) + right_inst = _instance_metric_rows(right) + + lines = [] + lines.append('Metric Quantiles Comparison') + lines.append('') + lines.append(f"generated_utc: {report['generated_utc']}") + lines.append(f"left_label: {left['label']}") + lines.append(f"right_label: {right['label']}") + lines.append('') + lines.append('pair_diagnoses:') + lines.append(f" {left['label']}: {left['diagnosis'].get('label')}") + lines.append(f" {right['label']}: {right['diagnosis'].get('label')}") + lines.append('') + lines.append('run_level_by_class:') + for cls in sorted(set(left_run) | set(right_run)): + lrow = left_run.get(cls, {}) + rrow = right_run.get(cls, {}) + lines.append(f' class={cls}') + lines.append( + f" {left['label']}: agree={_fmt(lrow.get('agree_ratio'))} count={_fmt(lrow.get('count'))} " + f"abs_p50={_fmt((lrow.get('abs_delta') or {}).get('p50'))} " + f"abs_p90={_fmt((lrow.get('abs_delta') or {}).get('p90'))} " + f"abs_p99={_fmt((lrow.get('abs_delta') or {}).get('p99'))} " + f"abs_max={_fmt((lrow.get('abs_delta') or {}).get('max'))}" + ) + lines.append( + f" {right['label']}: agree={_fmt(rrow.get('agree_ratio'))} count={_fmt(rrow.get('count'))} " + f"abs_p50={_fmt((rrow.get('abs_delta') or {}).get('p50'))} " + f"abs_p90={_fmt((rrow.get('abs_delta') or {}).get('p90'))} " + f"abs_p99={_fmt((rrow.get('abs_delta') or {}).get('p99'))} " + f"abs_max={_fmt((rrow.get('abs_delta') or {}).get('max'))}" + ) + lines.append('') + lines.append('instance_level_by_metric:') + for key in sorted(set(left_inst) | set(right_inst), key=lambda k: (str(k[0]), str(k[1]))): + lrow = left_inst.get(key, {}) + rrow = right_inst.get(key, {}) + lines.append(f' metric_class={key[0]} metric={key[1]}') + lines.append( + f" {left['label']}: agree={_fmt(lrow.get('agree_ratio'))} count={_fmt(lrow.get('count'))} " + f"abs_p50={_fmt((lrow.get('abs_delta') or {}).get('p50'))} " + f"abs_p90={_fmt((lrow.get('abs_delta') or {}).get('p90'))} " + f"abs_p99={_fmt((lrow.get('abs_delta') or {}).get('p99'))} " + f"abs_max={_fmt((lrow.get('abs_delta') or {}).get('max'))}" + ) + lines.append( + f" {right['label']}: agree={_fmt(rrow.get('agree_ratio'))} count={_fmt(rrow.get('count'))} " + f"abs_p50={_fmt((rrow.get('abs_delta') or {}).get('p50'))} " + f"abs_p90={_fmt((rrow.get('abs_delta') or {}).get('p90'))} " + f"abs_p99={_fmt((rrow.get('abs_delta') or {}).get('p99'))} " + f"abs_max={_fmt((rrow.get('abs_delta') or {}).get('max'))}" + ) + return '\n'.join(lines) + '\n' + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('--left-run-a', required=True) + parser.add_argument('--left-run-b', required=True) + parser.add_argument('--left-label', required=True) + parser.add_argument('--right-run-a', required=True) + parser.add_argument('--right-run-b', required=True) + parser.add_argument('--right-label', required=True) + parser.add_argument('--report-dpath', required=True) + args = parser.parse_args() + + report_dpath = Path(args.report_dpath).expanduser().resolve() + report_dpath.mkdir(parents=True, exist_ok=True) + stamp = datetime_mod.datetime.now(datetime_mod.UTC).strftime('%Y%m%dT%H%M%SZ') + + report = { + 'generated_utc': stamp, + 'pairs': [ + _pair_report(args.left_run_a, args.left_run_b, args.left_label), + _pair_report(args.right_run_a, args.right_run_b, args.right_label), + ], + } + report = kwutil.Json.ensure_serializable(report) + + json_fpath = report_dpath / f'metric_quantiles_{stamp}.json' + txt_fpath = report_dpath / f'metric_quantiles_{stamp}.txt' + json_fpath.write_text(json.dumps(report, indent=2)) + txt_fpath.write_text(_build_text(report)) + print(f'Wrote metric quantiles report: {json_fpath}') + print(f'Wrote metric quantiles text: {txt_fpath}') + + +if __name__ == '__main__': + main() diff --git a/dev/experiments/audit-helm-reproduction/python/resolve_run.py b/dev/experiments/audit-helm-reproduction/python/resolve_run.py new file mode 100644 index 0000000..35e9693 --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/python/resolve_run.py @@ -0,0 +1,134 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +import kwutil + +from magnet.backends.helm.cli.materialize_helm_run import ( + discover_benchmark_output_dirs, + run_dir_matches_requested, +) +from magnet.backends.helm.helm_outputs import HelmOutputs + + +def load_json(fpath: Path) -> dict[str, Any]: + return json.loads(fpath.read_text()) + + +def summarize_run_artifacts(run_dir: Path | None) -> dict[str, Any]: + required_files = [ + 'run_spec.json', + 'scenario_state.json', + 'stats.json', + 'per_instance_stats.json', + ] + if run_dir is None: + return { + 'artifact_status': 'missing_run_dir', + 'missing_files': required_files, + } + missing_files = [name for name in required_files if not (run_dir / name).exists()] + return { + 'artifact_status': 'ready' if not missing_files else 'incomplete_run_dir', + 'missing_files': missing_files, + } + + +def resolve_kwdagger_run(results_dpath: Path, run_entry: str) -> dict[str, Any] | None: + helm_root = results_dpath / 'helm' + if not helm_root.exists(): + return None + matches = [] + for job_cfg in helm_root.glob('*/job_config.json'): + data = load_json(job_cfg) + if data.get('helm.run_entry') != run_entry: + continue + job_dpath = job_cfg.parent + run_dirs = [] + try: + suites = HelmOutputs.coerce(job_dpath / 'benchmark_output').suites() + for suite in suites: + for run in suite.runs(): + run_dirs.append(Path(run.path)) + except Exception: + run_dirs = [] + if len(run_dirs) == 1: + run_dir = run_dirs[0] + else: + run_dir = None + for candidate in run_dirs: + if run_dir_matches_requested(candidate.name, run_entry): + run_dir = candidate + break + match = { + 'job_id': job_dpath.name, + 'job_dpath': str(job_dpath), + 'run_dir': None if run_dir is None else str(run_dir), + 'run_entry': run_entry, + } + match.update(summarize_run_artifacts(run_dir)) + matches.append(match) + if not matches: + return None + if len(matches) > 1: + raise RuntimeError( + f'Found multiple kwdagger matches for {run_entry}: {kwutil.Json.dumps(matches, indent=2)}' + ) + return matches[0] + + +def resolve_historic_run(precomputed_root: Path, run_entry: str) -> dict[str, Any] | None: + matches = [] + for bo in discover_benchmark_output_dirs([precomputed_root]): + try: + outputs = HelmOutputs.coerce(bo) + except Exception: + continue + for suite in outputs.suites(): + for run in suite.runs(): + run_dir = Path(run.path) + if not run_dir_matches_requested(run.name, run_entry): + continue + match = { + 'run_dir': str(run_dir), + 'suite': suite.path.name if hasattr(suite, 'path') else None, + 'helm_version': run_dir.parent.name, + 'run_entry': run_entry, + } + match.update(summarize_run_artifacts(run_dir)) + matches.append(match) + if not matches: + return None + return { + 'matches': matches, + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('--mode', required=True, choices=['kwdg', 'historic']) + parser.add_argument('--run-entry', required=True) + parser.add_argument('--results-dpath', default=None) + parser.add_argument('--precomputed-root', default='/data/crfm-helm-public') + args = parser.parse_args() + + if args.mode == 'kwdg': + if args.results_dpath is None: + raise SystemExit('--results-dpath is required in kwdg mode') + info = resolve_kwdagger_run( + results_dpath=Path(args.results_dpath).expanduser().resolve(), + run_entry=args.run_entry, + ) + else: + info = resolve_historic_run( + precomputed_root=Path(args.precomputed_root).expanduser().resolve(), + run_entry=args.run_entry, + ) + print(kwutil.Json.dumps(info, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/dev/experiments/audit-helm-reproduction/scripts/compare_entry.sh b/dev/experiments/audit-helm-reproduction/scripts/compare_entry.sh new file mode 100755 index 0000000..f3f0c91 --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/scripts/compare_entry.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/common.sh" + +audit::set_defaults + +RESULTS_DPATH="${1:?need kwdagger results root}" +RUN_ENTRY="${2:?need run entry}" +REPORT_DPATH="${3:-${AUDIT_ROOT}/reports/pairwise}" + +kwdg_json="$("$AIQ_PYTHON" "${AUDIT_ROOT}/python/resolve_run.py" \ + --mode kwdg \ + --results-dpath "$RESULTS_DPATH" \ + --run-entry "$RUN_ENTRY")" + +historic_json="$("$AIQ_PYTHON" "${AUDIT_ROOT}/python/resolve_run.py" \ + --mode historic \ + --precomputed-root "$HELM_PRECOMPUTED_ROOT" \ + --run-entry "$RUN_ENTRY")" + +kwdg_status="$(python - <<'PY' "$kwdg_json" +import json, sys +obj = json.loads(sys.argv[1]) +print(obj.get('artifact_status', 'unknown')) +PY +)" + +kwdg_run_dir="$(python - <<'PY' "$kwdg_json" +import json, sys +obj = json.loads(sys.argv[1]) +print(obj.get('run_dir') or '') +PY +)" + +if [[ "$kwdg_status" != "ready" ]]; then + echo "Resolved kwdagger job, but HELM run artifacts are not ready." >&2 + python - <<'PY' "$kwdg_json" +import json, sys +obj = json.loads(sys.argv[1]) +print(json.dumps(obj, indent=2)) +PY + exit 1 +fi + +historic_run_dir="$(python - <<'PY' "$historic_json" +import json, sys +obj = json.loads(sys.argv[1]) +matches = obj.get('matches', []) +if not matches: + raise SystemExit('No historic matches found') +ready = [m for m in matches if m.get('artifact_status') == 'ready'] +target = ready[-1] if ready else matches[-1] +print(target['run_dir']) +PY +)" + +"${AUDIT_ROOT}/scripts/compare_pair.sh" \ + "$historic_run_dir" \ + "$kwdg_run_dir" \ + "$REPORT_DPATH" diff --git a/dev/experiments/audit-helm-reproduction/scripts/compare_pair.sh b/dev/experiments/audit-helm-reproduction/scripts/compare_pair.sh new file mode 100755 index 0000000..710306a --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/scripts/compare_pair.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/common.sh" + +audit::set_defaults + +RUN_A="${1:?need run A path}" +RUN_B="${2:?need run B path}" +REPORT_DPATH="${3:-${AUDIT_ROOT}/reports/pairwise}" + +"$AIQ_PYTHON" "${AUDIT_ROOT}/python/compare_pair.py" \ + --run-a "$RUN_A" \ + --run-b "$RUN_B" \ + --report-dpath "$REPORT_DPATH" diff --git a/dev/experiments/audit-helm-reproduction/scripts/core_metric_report.sh b/dev/experiments/audit-helm-reproduction/scripts/core_metric_report.sh new file mode 100755 index 0000000..43e15ee --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/scripts/core_metric_report.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/common.sh" + +audit::set_defaults + +"$AIQ_PYTHON" "${AUDIT_ROOT}/python/core_metric_report.py" "$@" diff --git a/dev/experiments/audit-helm-reproduction/scripts/make_repeat_pair_manifests.sh b/dev/experiments/audit-helm-reproduction/scripts/make_repeat_pair_manifests.sh new file mode 100755 index 0000000..fea3767 --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/scripts/make_repeat_pair_manifests.sh @@ -0,0 +1,47 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/common.sh" + +audit::set_defaults + +RUN_ENTRY="${1:-narrative_qa:model=eleutherai/pythia-6.9b,data_augmentation=canonical}" +BASE_NAME="${2:-audit-narrative-pythia}" +OUTPUT_DIR="${3:-${AUDIT_ROOT}/configs/generated}" +if [[ $# -gt 0 ]]; then + shift +fi +if [[ $# -gt 0 ]]; then + shift +fi +if [[ $# -gt 0 ]]; then + shift +fi + +mkdir -p "$OUTPUT_DIR" + +R1_OUTPUT="${OUTPUT_DIR}/${BASE_NAME}_r1.yaml" +R2_OUTPUT="${OUTPUT_DIR}/${BASE_NAME}_r2.yaml" + +"$AIQ_PYTHON" "${AUDIT_ROOT}/python/make_manifest.py" \ + --manifest-type single \ + --run-entry "$RUN_ENTRY" \ + --experiment-name "${BASE_NAME}-r1" \ + --suite "${BASE_NAME}-r1" \ + --description "Repeat run 1 for ${RUN_ENTRY}" \ + --output "$R1_OUTPUT" \ + "$@" + +"$AIQ_PYTHON" "${AUDIT_ROOT}/python/make_manifest.py" \ + --manifest-type single \ + --run-entry "$RUN_ENTRY" \ + --experiment-name "${BASE_NAME}-r2" \ + --suite "${BASE_NAME}-r2" \ + --description "Repeat run 2 for ${RUN_ENTRY}" \ + --output "$R2_OUTPUT" \ + "$@" + +printf 'Wrote repeat manifests:\n' +printf ' %s\n' "$R1_OUTPUT" +printf ' %s\n' "$R2_OUTPUT" diff --git a/dev/experiments/audit-helm-reproduction/scripts/metric_quantiles_report.sh b/dev/experiments/audit-helm-reproduction/scripts/metric_quantiles_report.sh new file mode 100755 index 0000000..cf12a7b --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/scripts/metric_quantiles_report.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/common.sh" + +audit::set_defaults + +"$AIQ_PYTHON" "${AUDIT_ROOT}/python/metric_quantiles_report.py" "$@" diff --git a/dev/experiments/audit-helm-reproduction/scripts/resolve_run.sh b/dev/experiments/audit-helm-reproduction/scripts/resolve_run.sh new file mode 100755 index 0000000..d0ea3d9 --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/scripts/resolve_run.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/common.sh" + +audit::set_defaults + +"$AIQ_PYTHON" "${AUDIT_ROOT}/python/resolve_run.py" "$@" diff --git a/magnet/backends/helm/helm_run_diff.py b/magnet/backends/helm/helm_run_diff.py index 194fe6d..3a8144e 100644 --- a/magnet/backends/helm/helm_run_diff.py +++ b/magnet/backends/helm/helm_run_diff.py @@ -78,6 +78,22 @@ def _safe_float(x: Any) -> float | None: return None +def _quantile(values: list[float], q: float) -> float | None: + if not values: + return None + if q <= 0: + return values[0] + if q >= 1: + return values[-1] + pos = (len(values) - 1) * q + lo = math.floor(pos) + hi = math.ceil(pos) + if lo == hi: + return values[lo] + alpha = pos - lo + return values[lo] * (1 - alpha) + values[hi] * alpha + + def _walker_diff(a: Any, b: Any, *, max_paths: int = 12) -> dict[str, Any]: """ @@ -1687,6 +1703,93 @@ def agrees(x: float, y: float) -> bool: self._cache[cache_key] = out return out + def value_distance_profile( + self, + *, + top_n: int = 12, + ) -> dict[str, Any]: + """Programmatic raw distance summary for intersecting run-level stats. + + Unlike :meth:`_value_agreement_summary`, this does not threshold values + into matched / mismatched. It reports absolute / relative deltas and + their distributions so tolerance policies can be applied later without + recomputing the underlying joins. + """ + cache_key = ('value_distance_profile', top_n, self.short_hash) + if cache_key in self._cache: + return self._cache[cache_key] + + idx_a = self.a.stat_index( + drop_zero_count=True, require_mean=True, short_hash=self.short_hash + ) + idx_b = self.b.stat_index( + drop_zero_count=True, require_mean=True, short_hash=self.short_hash + ) + keys = set(idx_a.keys()) & set(idx_b.keys()) + + by_class: dict[str, list[dict[str, Any]]] = { + 'core': [], + 'bookkeeping': [], + 'untracked': [], + } + all_rows: list[dict[str, Any]] = [] + for k in keys: + a = idx_a[k] + b = idx_b[k] + if a.mean is None or b.mean is None: + continue + abs_delta = abs(a.mean - b.mean) + denom = max(abs(a.mean), abs(b.mean), 1e-12) + rel_delta = abs_delta / denom + row = { + 'key': k, + 'a': a.mean, + 'b': b.mean, + 'abs_delta': abs_delta, + 'rel_delta': rel_delta, + 'metric_class': a.metric_class, + } + all_rows.append(row) + by_class.setdefault(a.metric_class, []).append(row) + + def summarize(rows: list[dict[str, Any]]) -> dict[str, Any]: + if not rows: + return { + 'count': 0, + 'abs_delta': {'min': None, 'p50': None, 'p90': None, 'p99': None, 'max': None}, + 'rel_delta': {'min': None, 'p50': None, 'p90': None, 'p99': None, 'max': None}, + 'top_abs_deltas': [], + } + abs_vals = sorted(float(r['abs_delta']) for r in rows) + rel_vals = sorted(float(r['rel_delta']) for r in rows) + top = sorted(rows, key=lambda r: r['abs_delta'], reverse=True)[:top_n] + return { + 'count': len(rows), + 'abs_delta': { + 'min': abs_vals[0], + 'p50': _quantile(abs_vals, 0.50), + 'p90': _quantile(abs_vals, 0.90), + 'p99': _quantile(abs_vals, 0.99), + 'max': abs_vals[-1], + }, + 'rel_delta': { + 'min': rel_vals[0], + 'p50': _quantile(rel_vals, 0.50), + 'p90': _quantile(rel_vals, 0.90), + 'p99': _quantile(rel_vals, 0.99), + 'max': rel_vals[-1], + }, + 'top_abs_deltas': top, + } + + out = { + 'overall': summarize(all_rows), + 'by_class': {cls: summarize(rows) for cls, rows in by_class.items()}, + } + out = _json_compatible(out) + self._cache[cache_key] = out + return out + # --------------------------------------------------------------------- # Instance-level agreement / drilldowns @@ -1909,6 +2012,349 @@ def agrees(x: float, y: float) -> bool: self._cache[cache_key] = out return out + def instance_distance_profile( + self, + *, + top_n: int = 10, + ) -> dict[str, Any]: + """Programmatic raw distance summary for per-instance stat means.""" + cache_key = ('instance_distance_profile', top_n, self.short_hash) + if cache_key in self._cache: + return self._cache[cache_key] + + joined_a = self.a.joined_instance_stat_table( + assert_assumptions=False, short_hash=self.short_hash + ) + joined_b = self.b.joined_instance_stat_table( + assert_assumptions=False, short_hash=self.short_hash + ) + map_a = getattr(joined_a, 'row_by_key', None) + map_b = getattr(joined_b, 'row_by_key', None) + + def _iter_rows(joined) -> Iterable[Any]: + if map_a is not None and joined is joined_a: + return map_a.values() + if map_b is not None and joined is joined_b: + return map_b.values() + if isinstance(joined, dict): + return joined.values() + if hasattr(joined, '__iter__'): + return joined + return [] + + def _row_key(row: Any) -> Any: + return ( + getattr(row, 'key', None) + or getattr(row, 'stat_key', None) + or getattr(row, 'row_key', None) + or row + ) + + if map_a is None: + map_a = {_row_key(r): r for r in _iter_rows(joined_a)} + if map_b is None: + map_b = {_row_key(r): r for r in _iter_rows(joined_b)} + + all_rows: list[dict[str, Any]] = [] + by_group: dict[tuple[str, str | None], list[dict[str, Any]]] = {} + for k in set(map_a) & set(map_b): + ra = map_a[k] + rb = map_b[k] + sa = ( + getattr(ra, 'stat', None) + if hasattr(ra, 'stat') + else (ra.get('stat', None) if isinstance(ra, dict) else None) + ) + sb = ( + getattr(rb, 'stat', None) + if hasattr(rb, 'stat') + else (rb.get('stat', None) if isinstance(rb, dict) else None) + ) + ma = _safe_float( + (sa or {}).get('mean', None) + if isinstance(sa, dict) + else getattr(sa, 'mean', None) + ) + mb = _safe_float( + (sb or {}).get('mean', None) + if isinstance(sb, dict) + else getattr(sb, 'mean', None) + ) + ca = ( + int((sa or {}).get('count', 0) or 0) + if isinstance(sa, dict) + else int(getattr(sa, 'count', 0) or 0) + ) + cb = ( + int((sb or {}).get('count', 0) or 0) + if isinstance(sb, dict) + else int(getattr(sb, 'count', 0) or 0) + ) + if ma is None or mb is None or ca == 0 or cb == 0: + continue + name_obj = ( + (sa or {}).get('name', None) + if isinstance(sa, dict) + else getattr(sa, 'name_obj', None) + ) + metric = ( + name_obj.get('name', None) + if isinstance(name_obj, dict) + else None + ) + if metric is None and sa is not None and not isinstance(sa, dict): + metric = getattr(sa, 'metric', None) + metric_class, _ = helm_metrics.classify_metric(metric) + abs_delta = abs(ma - mb) + denom = max(abs(ma), abs(mb), 1e-12) + rel_delta = abs_delta / denom + item = { + 'key': _key_to_serializable(k), + 'a': ma, + 'b': mb, + 'abs_delta': abs_delta, + 'rel_delta': rel_delta, + 'signed_delta': (mb - ma), + 'metric_class': metric_class, + 'metric': metric, + } + all_rows.append(item) + by_group.setdefault((metric_class, metric), []).append(item) + + def summarize(items: list[dict[str, Any]]) -> dict[str, Any]: + if not items: + return { + 'count': 0, + 'abs_delta': {'min': None, 'p50': None, 'p90': None, 'p99': None, 'max': None}, + 'rel_delta': {'min': None, 'p50': None, 'p90': None, 'p99': None, 'max': None}, + } + abs_vals = sorted(float(r['abs_delta']) for r in items) + rel_vals = sorted(float(r['rel_delta']) for r in items) + return { + 'count': len(items), + 'abs_delta': { + 'min': abs_vals[0], + 'p50': _quantile(abs_vals, 0.50), + 'p90': _quantile(abs_vals, 0.90), + 'p99': _quantile(abs_vals, 0.99), + 'max': abs_vals[-1], + }, + 'rel_delta': { + 'min': rel_vals[0], + 'p50': _quantile(rel_vals, 0.50), + 'p90': _quantile(rel_vals, 0.90), + 'p99': _quantile(rel_vals, 0.99), + 'max': rel_vals[-1], + }, + } + + grouped_rows = [] + for (metric_class, metric), items in sorted(by_group.items()): + items = sorted(items, key=lambda r: r['abs_delta'], reverse=True) + grouped_rows.append( + { + 'metric_class': metric_class, + 'metric': metric, + 'summary': summarize(items), + 'top_abs_deltas': items[:top_n], + } + ) + out = { + 'overall': summarize(all_rows), + 'by_metric': grouped_rows, + } + out = _json_compatible(out) + self._cache[cache_key] = out + return out + + def instance_agreement_profile( + self, + *, + abs_tol: float = 0.0, + rel_tol: float = 0.0, + ) -> dict[str, Any]: + """Programmatic agreement summary grouped by per-instance metric.""" + cache_key = ('instance_agreement_profile', abs_tol, rel_tol, self.short_hash) + if cache_key in self._cache: + return self._cache[cache_key] + + joined_a = self.a.joined_instance_stat_table( + assert_assumptions=False, short_hash=self.short_hash + ) + joined_b = self.b.joined_instance_stat_table( + assert_assumptions=False, short_hash=self.short_hash + ) + map_a = getattr(joined_a, 'row_by_key', None) + map_b = getattr(joined_b, 'row_by_key', None) + + def _iter_rows(joined) -> Iterable[Any]: + if map_a is not None and joined is joined_a: + return map_a.values() + if map_b is not None and joined is joined_b: + return map_b.values() + if isinstance(joined, dict): + return joined.values() + if hasattr(joined, '__iter__'): + return joined + return [] + + def _row_key(row: Any) -> Any: + return ( + getattr(row, 'key', None) + or getattr(row, 'stat_key', None) + or getattr(row, 'row_key', None) + or row + ) + + if map_a is None: + map_a = {_row_key(r): r for r in _iter_rows(joined_a)} + if map_b is None: + map_b = {_row_key(r): r for r in _iter_rows(joined_b)} + + def agrees(x: float, y: float) -> bool: + if abs_tol == 0.0 and rel_tol == 0.0: + return x == y + return abs(x - y) <= max(abs_tol, rel_tol * max(abs(x), abs(y))) + + overall = {'comparable': 0, 'mismatched': 0} + by_group: dict[tuple[str, str | None], dict[str, Any]] = {} + + for k in set(map_a) & set(map_b): + ra = map_a[k] + rb = map_b[k] + sa = ( + getattr(ra, 'stat', None) + if hasattr(ra, 'stat') + else (ra.get('stat', None) if isinstance(ra, dict) else None) + ) + sb = ( + getattr(rb, 'stat', None) + if hasattr(rb, 'stat') + else (rb.get('stat', None) if isinstance(rb, dict) else None) + ) + ma = _safe_float( + (sa or {}).get('mean', None) + if isinstance(sa, dict) + else getattr(sa, 'mean', None) + ) + mb = _safe_float( + (sb or {}).get('mean', None) + if isinstance(sb, dict) + else getattr(sb, 'mean', None) + ) + ca = ( + int((sa or {}).get('count', 0) or 0) + if isinstance(sa, dict) + else int(getattr(sa, 'count', 0) or 0) + ) + cb = ( + int((sb or {}).get('count', 0) or 0) + if isinstance(sb, dict) + else int(getattr(sb, 'count', 0) or 0) + ) + if ma is None or mb is None or ca == 0 or cb == 0: + continue + name_obj = ( + (sa or {}).get('name', None) + if isinstance(sa, dict) + else getattr(sa, 'name_obj', None) + ) + metric = ( + name_obj.get('name', None) + if isinstance(name_obj, dict) + else None + ) + if metric is None and sa is not None and not isinstance(sa, dict): + metric = getattr(sa, 'metric', None) + metric_class, _ = helm_metrics.classify_metric(metric) + key = (metric_class, metric) + group = by_group.setdefault( + key, + { + 'metric_class': metric_class, + 'metric': metric, + 'comparable': 0, + 'mismatched': 0, + }, + ) + overall['comparable'] += 1 + group['comparable'] += 1 + if not agrees(ma, mb): + overall['mismatched'] += 1 + group['mismatched'] += 1 + + grouped_rows = [] + for _, group in sorted(by_group.items()): + grouped_rows.append( + { + 'metric_class': group['metric_class'], + 'metric': group['metric'], + 'comparable': group['comparable'], + 'mismatched': group['mismatched'], + 'agree_ratio': ratio(group['comparable'], group['mismatched']), + } + ) + + out = { + 'overall': { + 'comparable': overall['comparable'], + 'mismatched': overall['mismatched'], + 'agree_ratio': ratio(overall['comparable'], overall['mismatched']), + }, + 'by_metric': grouped_rows, + } + out = _json_compatible(out) + self._cache[cache_key] = out + return out + + def tolerance_sweep_summary( + self, + *, + run_tolerances: list[dict[str, Any]] | None = None, + instance_tolerances: list[dict[str, Any]] | None = None, + ) -> dict[str, Any]: + """Evaluate multiple tolerance policies without recomputing run loading.""" + if run_tolerances is None: + run_tolerances = [ + {'name': 'strict', 'abs_tol': 0.0, 'rel_tol': 0.0}, + {'name': 'tiny', 'abs_tol': 1e-12, 'rel_tol': 1e-6}, + {'name': 'small', 'abs_tol': 1e-9, 'rel_tol': 1e-4}, + {'name': 'medium', 'abs_tol': 1e-6, 'rel_tol': 1e-3}, + {'name': 'loose', 'abs_tol': 1e-3, 'rel_tol': 1e-2}, + ] + if instance_tolerances is None: + instance_tolerances = list(run_tolerances) + + run_results = [] + for cfg in run_tolerances: + summary = self._value_agreement_summary( + abs_tol=float(cfg.get('abs_tol', 0.0) or 0.0), + rel_tol=float(cfg.get('rel_tol', 0.0) or 0.0), + ) + run_results.append({ + 'name': cfg.get('name', 'unnamed'), + 'abs_tol': float(cfg.get('abs_tol', 0.0) or 0.0), + 'rel_tol': float(cfg.get('rel_tol', 0.0) or 0.0), + 'summary': summary, + }) + + instance_results = [] + for cfg in instance_tolerances: + summary = self.instance_summary_dict( + abs_tol=float(cfg.get('abs_tol', 0.0) or 0.0), + rel_tol=float(cfg.get('rel_tol', 0.0) or 0.0), + ) + instance_results.append({ + 'name': cfg.get('name', 'unnamed'), + 'abs_tol': float(cfg.get('abs_tol', 0.0) or 0.0), + 'rel_tol': float(cfg.get('rel_tol', 0.0) or 0.0), + 'summary': summary, + }) + return _json_compatible({ + 'run_level': run_results, + 'instance_level': instance_results, + }) + def summarize_instances( self, *, From a602436082a50e2437f5af0e1e05497ed604591b Mon Sep 17 00:00:00 2001 From: agent Date: Fri, 27 Mar 2026 03:15:13 +0000 Subject: [PATCH 23/36] wip --- .../python/core_metric_report.py | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/dev/experiments/audit-helm-reproduction/python/core_metric_report.py b/dev/experiments/audit-helm-reproduction/python/core_metric_report.py index 5b513f5..c5a6785 100644 --- a/dev/experiments/audit-helm-reproduction/python/core_metric_report.py +++ b/dev/experiments/audit-helm-reproduction/python/core_metric_report.py @@ -465,6 +465,53 @@ def _write_text(report: dict[str, Any], out_fpath: Path) -> None: out_fpath.write_text('\n'.join(lines) + '\n') +def _find_curve_value(rows: list[dict[str, Any]], abs_tol: float) -> float | None: + for row in rows: + if float(row.get('abs_tol', float('nan'))) == float(abs_tol): + return row.get('agree_ratio') + return None + + +def _write_management_summary(report: dict[str, Any], out_fpath: Path) -> None: + left, right = report['pairs'] + lines = [] + lines.append('Core Metric Executive Summary') + lines.append('') + lines.append(f"generated_utc: {report['generated_utc']}") + lines.append(f"core_metrics: {', '.join(left.get('core_metrics', []))}") + lines.append('') + lines.append(f"{left['label']}:") + lines.append(f" diagnosis: {left['diagnosis'].get('label')}") + lines.append( + f" instance agreement at abs_tol=0.0: {_find_curve_value(left['instance_level']['agreement_vs_abs_tol'], 0.0)}" + ) + lines.append( + f" run-level abs delta max: {left['run_level']['overall_quantiles']['abs_delta']['max']}" + ) + lines.append( + f" instance-level abs delta max: {left['instance_level']['overall_quantiles']['abs_delta']['max']}" + ) + lines.append('') + lines.append(f"{right['label']}:") + lines.append(f" diagnosis: {right['diagnosis'].get('label')}") + for tol in [0.0, 1e-3, 1e-2, 1e-1, 1.0]: + lines.append( + f" instance agreement at abs_tol={tol}: " + f"{_find_curve_value(right['instance_level']['agreement_vs_abs_tol'], tol)}" + ) + lines.append( + f" run-level abs delta p90/max: " + f"{right['run_level']['overall_quantiles']['abs_delta']['p90']} / " + f"{right['run_level']['overall_quantiles']['abs_delta']['max']}" + ) + lines.append( + f" instance-level abs delta p99/max: " + f"{right['instance_level']['overall_quantiles']['abs_delta']['p99']} / " + f"{right['instance_level']['overall_quantiles']['abs_delta']['max']}" + ) + out_fpath.write_text('\n'.join(lines) + '\n') + + def main() -> None: parser = argparse.ArgumentParser() parser.add_argument('--left-run-a', required=True) @@ -491,6 +538,7 @@ def main() -> None: json_fpath = report_dpath / f'core_metric_report_{stamp}.json' txt_fpath = report_dpath / f'core_metric_report_{stamp}.txt' + mgmt_fpath = report_dpath / f'core_metric_management_summary_{stamp}.txt' fig_fpath = report_dpath / f'core_metric_report_{stamp}.png' dist_fig_fpath = _plot_metric_distributions(report_dpath, stamp, left, right) three_run_dist_fpath = _plot_three_run_metric_distributions( @@ -511,6 +559,7 @@ def main() -> None: report = kwutil.Json.ensure_serializable(_strip_private(report)) json_fpath.write_text(json.dumps(report, indent=2)) _write_text(report, txt_fpath) + _write_management_summary(report, mgmt_fpath) sns.set_theme(style='whitegrid', context='talk') fig, axes = plt.subplots(2, 2, figsize=(12, 8), constrained_layout=True) @@ -525,6 +574,7 @@ def main() -> None: print(f'Wrote core metric report: {json_fpath}') print(f'Wrote core metric text: {txt_fpath}') + print(f'Wrote core metric management summary: {mgmt_fpath}') print(f'Wrote core metric plot: {fig_fpath}') print(f'Wrote core metric distributions: {dist_fig_fpath}') print(f'Wrote core metric three-run distributions: {three_run_dist_fpath}') From 80009d770ed0661b9c8744272536aee591d9bd04 Mon Sep 17 00:00:00 2001 From: joncrall Date: Fri, 27 Mar 2026 20:08:38 -0400 Subject: [PATCH 24/36] Continue audit --- .../helm-reproduction-research-journal.md | 316 +++++++++++++ dev/codex/kwdagger-notes.md | 30 ++ dev/codex/reproduce-helm-session-v2.md | 432 ++++++++++++++++++ .../audit-helm-reproduction/README.md | 160 +++++++ .../python/aggregate_core_reports.py | 165 +++++++ .../python/core_metric_report.py | 316 ++++++++++++- .../python/index_results.py | 201 ++++++++ .../python/inspect_pair_samples.py | 94 ++++ .../rebuild_all_core_reports_from_index.py | 65 +++ .../python/rebuild_core_report_from_index.py | 229 ++++++++++ .../scripts/aggregate_core_reports.sh | 11 + .../scripts/index_results.sh | 16 + .../scripts/inspect_pair_samples.sh | 9 + .../rebuild_all_core_reports_from_index.sh | 11 + .../scripts/rebuild_core_report_from_index.sh | 12 + .../backends/helm/cli/materialize_helm_run.py | 78 ++++ 16 files changed, 2123 insertions(+), 22 deletions(-) create mode 100644 dev/codex/helm-reproduction-research-journal.md create mode 100644 dev/codex/kwdagger-notes.md create mode 100644 dev/codex/reproduce-helm-session-v2.md create mode 100644 dev/experiments/audit-helm-reproduction/python/aggregate_core_reports.py create mode 100644 dev/experiments/audit-helm-reproduction/python/index_results.py create mode 100644 dev/experiments/audit-helm-reproduction/python/inspect_pair_samples.py create mode 100644 dev/experiments/audit-helm-reproduction/python/rebuild_all_core_reports_from_index.py create mode 100644 dev/experiments/audit-helm-reproduction/python/rebuild_core_report_from_index.py create mode 100755 dev/experiments/audit-helm-reproduction/scripts/aggregate_core_reports.sh create mode 100755 dev/experiments/audit-helm-reproduction/scripts/index_results.sh create mode 100755 dev/experiments/audit-helm-reproduction/scripts/inspect_pair_samples.sh create mode 100755 dev/experiments/audit-helm-reproduction/scripts/rebuild_all_core_reports_from_index.sh create mode 100755 dev/experiments/audit-helm-reproduction/scripts/rebuild_core_report_from_index.sh diff --git a/dev/codex/helm-reproduction-research-journal.md b/dev/codex/helm-reproduction-research-journal.md new file mode 100644 index 0000000..1f5c095 --- /dev/null +++ b/dev/codex/helm-reproduction-research-journal.md @@ -0,0 +1,316 @@ +# HELM Reproduction Research Journal + +Date started: 2026-03-27 +Project: `/home/joncrall/code/aiq-magnet` +Scope: reproduce historic public HELM results with local `kwdagger` pipelines, explain observed drift, and build reporting that supports both management summaries and maintainer-grade technical diagnosis. + +## Research Goal + +Determine whether current locally reproduced HELM runs meaningfully differ from official public HELM results, and if so: + +- quantify how different they are +- separate ordinary rerun noise from structural drift +- identify likely causes such as deployment/provider changes, metric-spec changes, and execution-spec changes + +## Data Sources + +- Public historic bundle: + - `/data/crfm-helm-public` +- Local audit runs: + - `/data/crfm-helm-audit` +- Audit experiment workflow: + - `dev/experiments/audit-helm-reproduction` + +## Workflow Built During This Research + +- Created a reusable audit experiment folder under: + - `dev/experiments/audit-helm-reproduction` +- Added shell-first operator scripts for: + - environment validation + - manifest generation + - scheduling runs with `kwdagger` + - comparing completed batches to public HELM runs + - direct pairwise comparison of two concrete run directories +- Added pairwise reporting support that writes: + - JSON reports + - text summaries + - tolerance sweeps + +## Current Direction Shift + +We likely do not need more same-hardware repeat runs right now. + +- The current evidence is already strong that repeated `kwdagger` runs on the + same machine are very stable relative to the official-vs-local gap. +- The more interesting next source of variation is now cross-hardware / + cross-machine drift. +- That means future repeatability work should prioritize: + - different GPU architectures + - different VRAM tiers + - possibly different host environments + +This should reduce time spent waiting on redundant same-machine reruns while +improving the causal story for a paper-quality reproducibility analysis. + +## Important Audit / kwdagger Notes + +- `kwdagger` is the right execution mechanism for these experiments, but it has a few behaviors worth remembering: + - unset optional params may still be rendered into generated CLI commands + - list-valued params are best passed as YAML strings, not `nargs='*'` + - tmux queue name collisions can cause interactive prompts unless queue names are experiment-specific +- Pairwise and entry-based comparison tooling now validates finalized HELM run artifacts before attempting diffs. + - This avoids misleading Python tracebacks when a job exists but its run directory is incomplete. + +See also: +- `dev/codex/kwdagger-notes.md` +- `dev/codex/reproduce-helm-session-v2.md` + +## Provenance / Multi-Machine Goal + +Current status before this note: + +- historical audit results mostly recorded logical run config +- they did **not** explicitly record machine / GPU provenance in a structured, + analysis-friendly way + +This is a problem if we want to: + +- merge runs produced on different machines +- compare same-config runs across hardware +- avoid blocking on one machine just to obtain intermediate analyses + +Implementation direction: + +- record lightweight process provenance per materialized HELM job +- use `kwutil.ProcessContext` to capture: + - host + - user + - cwd + - OS / Python info + - memory / CPU metadata when available +- augment that with best-effort `nvidia-smi` GPU details when available + +Desired downstream behavior: + +- pairwise / aggregate analysis should work against whatever runs exist +- missing runs should reduce coverage, not block intermediate analysis +- when cross-machine drift appears, we should be able to attribute it to a + known machine / GPU context rather than reconstructing provenance manually + +Open goal: + +- teach the audit reporting layer to group / filter by machine provenance once + enough multi-machine runs exist + +## Deployment / Registry Findings + +- Some target models already have built-in HELM Hugging Face deployments: + - `eleutherai/pythia-6.9b` + - `lmsys/vicuna-7b-v1.3` + - `aisingapore/llama3-8b-cpt-sea-lionv2.1-instruct` +- Some newer or different families appear Together-backed in the built-in registry: + - example: `meta/llama-3-8b-chat` +- HELM’s experimental Hugging Face registration flags are not reliable enough by themselves for this workflow. + - robust reproduction work should prefer explicit deployment control, e.g. via custom `model_deployments.yaml` + +## Apples-To-Apples Finding + +The original smoke batch was useful as a workflow check, but not apples-to-apples because public matches were often at `max_eval_instances=1000` while the reproduced smoke runs used `100`. + +We then ran an apples-to-apples control batch aligned to `max_eval_instances=1000`. + +Key result: + +- the eval-size mismatch confounder was removed +- the remaining observed drift still persisted + +That means the disagreement is not explained solely by comparing `100` local examples to `1000` historic examples. + +## Representative Structural Drift + +For apples-to-apples cases, the main remaining execution-path difference was: + +- `adapter_spec.model_deployment` + +Representative examples showed: + +- historic public runs often record: + - `adapter_spec.model_deployment = null` +- local reproduced runs often record: + - `adapter_spec.model_deployment = huggingface/...` + +Metric spec drift was also observed: + +- historic often uses: + - `helm.benchmark.metrics.basic_metrics.BasicMetric` +- local reproduced runs often use: + - `BasicGenerationMetric` + - `BasicReferenceMetric` + - `InstancesPerSplitMetric` + +This is important because it means the disagreement is not just at the output level. The effective evaluation configuration is also changing. + +## Pairwise Comparison Baseline + +To separate structural drift from normal stochastic variation, repeated local runs were compared directly. + +Case studied in detail: + +- `boolq:model=eleutherai/pythia-6.9b,data_augmentation=canonical` + +Compared: + +- local repeat 1: + - `/data/crfm-helm-audit/audit-boolq-pythia-r1/helm/helm_id_13jkx9mm4k4n/benchmark_output/runs/audit-boolq-pythia-r1/boolq:model=eleutherai_pythia-6.9b,data_augmentation=canonical` +- local repeat 2: + - `/data/crfm-helm-audit/audit-boolq-pythia-r2/helm/helm_id_12jr5w48kge7/benchmark_output/runs/audit-boolq-pythia-r2/boolq:model=eleutherai_pythia-6.9b,data_augmentation=canonical` +- official public HELM: + - `/data/crfm-helm-public/classic/benchmark_output/runs/v0.3.0/boolq:model=eleutherai_pythia-6.9b,data_augmentation=canonical` + +### Local Repeatability Result + +`r1` vs `r2`: + +- diagnosis: + - `bookkeeping_metric_drift` +- strict run-level agree ratio: + - `0.9552238805970149` +- strict instance-level agree ratio: + - `0.9523809523809523` +- run-level max abs delta: + - `0.0015118565559387176` +- instance-level max abs delta: + - `0.4418189525604248` + +Interpretation: + +- local reruns are very close +- residual differences are mostly bookkeeping/runtime style noise, not large task-quality drift + +### Official vs Local Result + +Official `v0.3.0` vs local `r1`: + +- diagnosis: + - `multiple_primary_reasons` +- primary reason names: + - `deployment_drift` + - `execution_spec_drift` +- strict run-level agree ratio: + - `0.4626865671641791` +- strict instance-level agree ratio: + - `0.6577333333333333` +- run-level abs p90: + - `4.0` +- run-level abs max: + - `11.985` +- instance-level abs p90: + - `4.0` +- instance-level abs max: + - `75.73884344100952` + +Interpretation: + +- official-vs-local drift is much larger than local-vs-local drift +- that makes it very unlikely that ordinary nondeterminism is the main explanation + +## High-Level Verdict + +For the tested BoolQ/Pythia apples-to-apples case, we now have enough information to say: + +- the current locally reproduced HELM result is significantly different from the official public HELM result +- the difference is much larger than the observed local repeatability noise +- the most likely explanation is structural/configurational drift rather than ordinary stochastic rerun variance + +Important limitation: + +- this claim is solid for the tested case +- it should not automatically be generalized to all HELM benchmarks or model families without more cases + +## Tolerance Sweep Findings + +The pairwise tool supports tolerance sweeps across several preset thresholds. + +Current presets: + +- `strict`: `abs_tol=0.0`, `rel_tol=0.0` +- `tiny`: `abs_tol=1e-12`, `rel_tol=1e-6` +- `small`: `abs_tol=1e-9`, `rel_tol=1e-4` +- `medium`: `abs_tol=1e-6`, `rel_tol=1e-3` +- `loose`: `abs_tol=1e-3`, `rel_tol=1e-2` +- `xloose`: `abs_tol=1e-2`, `rel_tol=1e-1` +- `xxloose`: `abs_tol=1e-1`, `rel_tol=1.0` +- `extreme`: `abs_tol=1.0`, `rel_tol=10.0` + +For local BoolQ/Pythia repeats: + +- similarity becomes nearly perfect by `loose` / `xloose` + +For official vs local BoolQ/Pythia: + +- similarity stays much lower through `loose` and `xloose` +- it only collapses to `1.0` at the very permissive `xxloose` threshold + +Interpretation: + +- official-vs-local mismatch is robust to modest tolerance relaxation +- forcing them to look identical requires very permissive tolerances + +## Important Metric-Scale Caveat + +A single global absolute tolerance is not easy to interpret because the comparison spans different metric families with very different numeric scales. + +Observed for official vs local BoolQ/Pythia: + +- `core` metrics are approximately `[0, 1]` + - examples: + - `exact_match` + - `prefix_exact_match` + - `quasi_exact_match` + - run-level max abs delta in this class was only about: + - `0.021` +- `bookkeeping` metrics are not bounded to `[0, 1]` + - examples: + - `num_bytes` + - `num_output_tokens` + - `logprob` + - observed values included: + - `num_bytes`: `15.432` vs `3.448` + - per-instance `num_bytes`: `16.0` vs `3.0` + - per-instance `num_output_tokens`: `5.0` vs `1.0` + - per-instance `logprob`: values like `-3.64` vs `0.0` + +Implication: + +- `abs_tol=1.0` is extremely loose for bounded core metrics +- but it is not necessarily huge for bookkeeping metrics such as byte counts or token counts + +Therefore: + +- global tolerance sweeps are still useful as diagnostics +- but interpretation should move toward: + - per metric class tolerances + - and eventually per metric family tolerances + +## Current Reporting Files Worth Inspecting + +- Session journal: + - `dev/codex/reproduce-helm-session-v2.md` +- kwdagger notes: + - `dev/codex/kwdagger-notes.md` +- Pairwise report, local repeat: + - `dev/experiments/audit-helm-reproduction/reports/pairwise/boolq-pythia-repeat-wide/pair_report_20260327T011202Z.txt` +- Pairwise report, official vs local: + - `dev/experiments/audit-helm-reproduction/reports/pairwise/boolq-pythia-historic-wide/pair_report_20260327T011202Z.txt` + +## Recommended Next Steps + +- Add class-specific tolerance reporting: + - at least separate `core` from `bookkeeping` +- Add effect-size style summaries: + - compare official-vs-local distance to local-vs-local baseline distance +- Collect more repeat runs if formal significance estimates are desired + - current data is enough for a strong effect-size-style argument + - it is not enough for a stable p-value estimate +- Run a minimal Together-backed control on a representative case + - this should help separate provider/deployment effects from general HELM evolution diff --git a/dev/codex/kwdagger-notes.md b/dev/codex/kwdagger-notes.md new file mode 100644 index 0000000..fc326bb --- /dev/null +++ b/dev/codex/kwdagger-notes.md @@ -0,0 +1,30 @@ +# kwdagger Notes + +Date started: 2026-03-24 +Project: `/home/joncrall/code/aiq-magnet` + +## Important Points + +- `kwdagger schedule` is correctly building the HELM smoke-test DAG as one node per `run_entry`; for the current smoke manifest that means 6 jobs total. +- With `backend=tmux`, `devices=0,1`, and `tmux_workers=2`, the queue is split into two tmux sessions, one per visible GPU, with jobs serialized within each session. +- `kwdagger` may render node defaults into the generated CLI even when the manifest omits them. + - Observed examples: + - `--precomputed_root=None` + - `--model_deployments_fpath=None` + - bare `--enable_huggingface_models` + - bare `--enable_local_huggingface_models` +- Because of that behavior, downstream CLIs should normalize null-like placeholders such as `None`, `null`, and empty strings instead of assuming omitted values stay omitted. +- For the HELM audit node, it is better to omit unset optional params when rendering the final command than to rely only on downstream normalization. + - The custom node `command` property now drops `None` and empty-list values before building the shell command. +- For list-valued params passed through `kwdagger`, prefer a single key/value YAML string over `nargs='*'`. + - Current audit/HELM example: + - `helm.enable_huggingface_models: '["repo-a", "repo-b"]'` + - `helm.enable_local_huggingface_models: '["/models/a"]'` + - The pipeline-facing script can decode that with `kwutil.Yaml.coerce`, then expand it into the downstream CLI format if needed. +- `scriptconfig` emitted a smartcast warning for comma-separated `devices`; this did not block scheduling, but it is a hint that list parsing behavior may change in a future version. +- Existing `cmd_queue` tmux sessions with the same queue name can trigger an interactive prompt asking whether older sessions should be killed. + - This is important for automation and unattended runs. + - Current mitigation in the audit runner: derive `--queue_name` from `experiment_name` instead of reusing the generic `schedule-eval`. +- The audit comparison step needs to discover completed jobs recursively under the kwdagger results root. + - Real layout is nested like: `///DONE` + - A shallow glob like `results/*/DONE` will miss all HELM jobs and incorrectly report `missing_kwdg_match`. diff --git a/dev/codex/reproduce-helm-session-v2.md b/dev/codex/reproduce-helm-session-v2.md new file mode 100644 index 0000000..43525a8 --- /dev/null +++ b/dev/codex/reproduce-helm-session-v2.md @@ -0,0 +1,432 @@ +# Reproduce HELM Session V2 + +Date: 2026-03-24 +Workspace: `/home/joncrall/code/aiq-magnet` +Python env: `/home/agent/.local/uv/envs/uvpy3.13.2/bin/python` +HELM installation: assumed available in the Python environment used by `magnet` +Precomputed HELM root: `/data/crfm-helm-public` + +## Task Summary + +Take a second pass at reproducing historic HELM evaluations with the local `kwdagger` pipelines, with a focus on cases where historic runs used Together-hosted model deployments but could plausibly be rerun locally with HELM's Hugging Face client on the available 4 x 96 GB GPUs. + +We also want to preserve useful discoveries here because rate limits are nearly exhausted for this session. + +## Findings So Far + +- The rough notes at the top of `dev/poc/inspect_historic_helm_runs.py` match the earlier workflow: + - generate candidate run specs from `/data/crfm-helm-public` + - filter them + - schedule `magnet.backends.helm.pipeline.helm_single_run_pipeline()` +- `dev/poc/inspect_historic_helm_runs.py` already contains an important filtering heuristic: + - it registers HELM built-in configs + - inspects model metadata and deployments from HELM + - keeps only models whose resolved deployment client is `helm.clients.huggingface_client.HuggingFaceClient` + - it also limits to open-access text models with `num_parameters <= 10e9` +- That means the previous workflow may have avoided explicit Together-to-HuggingFace rewrites in many cases by selecting only models that HELM already knew how to run through a Hugging Face deployment. +- Concrete registry examples from this session: + - `aisingapore/llama3-8b-cpt-sea-lionv2.1-instruct` + - default deployment available in HELM: `huggingface/llama3-8b-cpt-sea-lionv2.1-instruct` + - client: `helm.clients.huggingface_client.HuggingFaceClient` + - `lmsys/vicuna-7b-v1.3` + - default deployment available in HELM: `huggingface/vicuna-7b-v1.3` + - `eleutherai/pythia-6.9b` + - default deployment available in HELM: `huggingface/pythia-6.9b` + - `meta/llama-3-8b-chat` + - built-in default deployment appears to be `together/llama-3-8b-chat` + - no built-in Hugging Face deployment was present in the registry snapshot checked today + +## Historic Together Cases + +- `rg` over `/data/crfm-helm-public/**/run_spec.json` confirms many newer public bundles explicitly record Together deployments, e.g.: + - `together/deepseek-v3` + - `together/qwen2.5-7b-instruct-turbo` + - `together/qwen2.5-72b-instruct-turbo` + - `together/llama-3.2-11b-vision-instruct-turbo` + - `together/llama-3.2-90b-vision-instruct-turbo` + - `together/gpt-oss-20b` +- So the second-pass reproduction work really does need a principled local-deployment override path for some model families. + +## Pipeline / Code Notes + +- `magnet/backends/helm/pipeline.py` defines a thin kwdagger node around: + - `python -m magnet.backends.helm.cli.materialize_helm_run` +- `magnet/backends/helm/cli/materialize_helm_run.py` behavior today: + - tries to reuse a matching run from `precomputed_root` + - otherwise executes: + - `helm-run --run-entries --suite ...` + - no provider override or deployment rewrite option is exposed yet +- Matching of historic runs is based on run-entry token subset matching, with model normalization `/ -> _` +- Reuse search scans discovered `benchmark_output` trees and returns the best matching run directory + +## Immediate Hypothesis + +- If a historic run spec points at a Together-backed deployment, reproducing it locally likely requires one of: + - rewriting the `model=` token in the run-entry to a HELM deployment name that uses `HuggingFaceClient` + - or adding custom `prod_env/model_deployments.yaml` entries so the same logical model can resolve to a local Hugging Face deployment + - or both + +## Important HELM Behavior + +- `helm-run` calls: + - `register_builtin_configs_from_helm_package()` + - then `register_configs_from_directory(args.local_path)` +- Default `--local-path` is `prod_env` +- Because `materialize_helm_run.py` runs `helm-run` with `cwd=out_dpath`, a relative `local_path=prod_env` naturally maps to: + - `/prod_env` +- This gives us a clean way to inject per-job `model_deployments.yaml` overrides without modifying the shared HELM checkout. + +## New Code Support Added This Session + +- Updated `magnet/backends/helm/cli/materialize_helm_run.py` to support: + - `local_path` + - `model_deployments_fpath` + - `enable_huggingface_models` + - `enable_local_huggingface_models` +- Behavior: + - before computing a run, the wrapper now prepares the HELM local config directory + - if `model_deployments_fpath` is given, it copies that file to `/model_deployments.yaml` + - `helm-run` is now invoked with explicit `--local-path` + - optional HELM Hugging Face registration flags are passed through when specified +- Updated `magnet/backends/helm/pipeline.py` so these parameters are available from kwdagger. +- Updated `materialize_helm_run.py` to defensively normalize optional values that may be rendered by `kwdagger` as CLI placeholders. + - Real-world examples observed in generated commands: + - `--precomputed_root=None` + - `--model_deployments_fpath=None` + - bare `--enable_huggingface_models` + - bare `--enable_local_huggingface_models` + - The wrapper now treats null-like placeholders and empty optional list inputs as truly unset. +- For list-shaped params that are directly exposed to `kwdagger`, prefer a single YAML-encoded string value instead of `nargs='*'`. + - This fits kwdagger's key/value param model better. + - The HELM wrapper now decodes those values with `kwutil.Yaml.coerce` before calling `helm-run`. + +## Caveat About HELM's Experimental HuggingFace Registration + +- HELM's `--enable-huggingface-models` is not a universal solution. +- Observed limitations from quick checks: + - `meta/llama-3-8b-chat` + - failed because the repo id is not directly usable as a public HF model id in this environment + - likely also requires Hugging Face auth / correct repo id for gated models + - `eleutherai/pythia-6.9b` + - failed because HELM could not infer `model_max_length` + - `aisingapore/llama3-8b-cpt-sea-lionv2.1-instruct` + - failed for the same `model_max_length` inference reason + - `lmsys/vicuna-7b-v1.3` + - failed during tokenizer conversion in this environment +- Conclusion: + - custom `model_deployments.yaml` overrides are the more robust path for reproduction work + - pass-through support for HELM's flags is still useful for models where they happen to work + +## Likely Reproduction Strategy + +- Prefer models that already have built-in Hugging Face deployments when possible. +- For Together-only model families that are still locally runnable: + - create a custom HELM `model_deployments.yaml` + - point kwdagger jobs at it via `helm.model_deployments_fpath` + - keep the logical `adapter_spec.model` stable where possible, and change deployment resolution rather than renaming the model everywhere + +## Next Checks + +- Inspect a concrete historic run spec from `/data/crfm-helm-public` +- Inspect HELM registry entries for the corresponding model/deployment +- Determine whether a Hugging Face deployment already exists for those model names +- If not, decide whether to extend `materialize_helm_run.py` and/or add custom model deployment registration support + +## kwdagger Observations + +- The smoke-test audit manifest scheduled as expected: + - 6 total jobs + - 2 tmux workers + - `CUDA_VISIBLE_DEVICES` split across GPU 0 and GPU 1 +- For the HELM audit node, overriding the node `command` renderer is the cleanest way to suppress unset optional args. + - This avoids generated commands like: + - `--precomputed_root=None` + - `--model_deployments_fpath=None` + - bare `--enable_huggingface_models` +- `kwdagger` / `cmd_queue` may stop for an interactive prompt if older tmux queue sessions with the same queue name are still running. + - That behavior matters for unattended reproduction runs and should be accounted for in future script refinements. +- The smoke-test comparison is now working end to end for the 6-run control batch. + - Historic rows found: 6 + - kwdagger rows found: 6 + - current diagnosis label across all six cases: `multiple_primary_reasons` + - primary reason counts: + - `deployment_drift`: 6 + - `execution_spec_drift`: 6 + - full reason counts: + - `completion_content_drift`: 6 + - `core_metric_drift`: 6 + - `dataset_instance_drift`: 6 + - `dataset_variant_drift`: 6 + - `deployment_drift`: 6 + - `evaluation_spec_drift`: 6 + - `execution_spec_drift`: 6 + - `request_prompt_drift`: 1 + - Interpretation: + - this smoke batch validates the audit workflow plumbing + - it does not yet validate close behavioral reproduction of the historic runs + - the dominant differences are consistent with local HF execution diverging materially from the historic HELM deployment / execution configuration +- Representative case inspected: + - `mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=eleutherai/pythia-6.9b,data_augmentation=canonical` + - historic run selected by matcher: + - `/data/crfm-helm-public/classic/benchmark_output/runs/v0.2.4/...` + - primary drift: + - `deployment_drift` + - historic `adapter_spec.model_deployment`: `null` + - reproduced `adapter_spec.model_deployment`: `huggingface/pythia-6.9b` + - `execution_spec_drift` + - historic `adapter_spec.max_eval_instances`: `1000` + - reproduced `adapter_spec.max_eval_instances`: `100` + - additional evidence that this is not a close reproduction yet: + - dataset base coverage differs: historic 111 vs reproduced 100 + - variant coverage differs: historic 327 vs reproduced 294 + - metric spec set differs substantially: + - historic uses `BasicMetric` + - reproduced uses `BasicGenerationMetric`, `BasicReferenceMetric`, and `InstancesPerSplitMetric` + - core metric agreement is effectively zero in this case + - `core_agree_ratio: 0.0` + - completion agreement is only partial + - 294 comparable completions + - 28 mismatches + - equal ratio about `0.905` +- Reporting improvements added after inspecting the smoke batch: + - `compare_batch.py` now writes a management-facing text summary in addition to the technical summary and JSON outputs + - summaries now expose: + - `primary_reason_name_counts` + - `reason_counts` + - high-level findings suitable for executive reporting + - historic candidate selection in the compare step is now more explicit and records whether an exact historic `adapter_spec.max_eval_instances` match exists +- Current important finding: + - for the smoke manifest, the public HELM bundle does not appear to contain exact historic matches with requested `max_eval_instances=100` + - available matched public runs for the inspected `pythia-6.9b` MMLU case record `adapter_spec.max_eval_instances=1000` + - this means the current smoke comparison is useful for auditing drift, but it is not yet an apples-to-apples reproduction check on evaluation size +- Apples-to-apples smoke batch (`audit-smoke-apples`) result: + - transferred report bundle shows: + - historic rows found: 6 + - kwdagger rows found: 6 + - all 6 compared successfully + - all 6 cases report `historic_exact_requested_max_eval_match = True` + - the actual run-spec execution-path drift in the transferred case JSONL is now only: + - `adapter_spec.model_deployment` + - primary reasons across all 6 apples-to-apples cases: + - `deployment_drift` + - `execution_spec_drift` + - recurring secondary reasons across all 6 apples-to-apples cases: + - `evaluation_spec_drift` + - `core_metric_drift` + - `completion_content_drift` + - one case also shows: + - `request_prompt_drift` + - Important correction: + - the initial apples executive summary still claimed a requested `max_eval_instances` mismatch in all 6 cases + - inspection of the case JSONL indicates that was a reporting bug from how the reproduced-side value was extracted, not the real comparison outcome + - `compare_batch.py` was updated to read reproduced `run_spec.json` directly for `adapter_spec.max_eval_instances` + +## Apples-To-Apples Report + +Date: 2026-03-25 +Experiment: `audit-smoke-apples` + +### Executive Summary + +- The apples-to-apples smoke batch compared 6 reproduced runs against 6 matched historic public HELM runs. +- All 6 runs were paired and compared successfully. +- After aligning `max_eval_instances` to the historic public bundle (`1000`), the remaining dominant drift is not evaluation size. +- The main recurring technical differences are: + - explicit Hugging Face `model_deployment` in reproduced runs vs `null` deployment in historic runs + - evaluation-spec drift caused by metric class changes + - completion-content drift + - core-metric drift +- This means the audit workflow is functioning correctly, and the remaining mismatch is now focused on how HELM resolves and evaluates these runs, not on the basic scheduling / pairing machinery. + +### High-Level Conclusion + +- The earlier non-apples smoke batch was confounded by comparing reproduced `max_eval_instances=100` runs against historic `max_eval_instances=1000` runs. +- The apples-to-apples batch removes that confounder. +- In the apples batch, the real remaining execution drift appears to be: + - `adapter_spec.model_deployment` +- The real remaining evaluation drift appears to be: + - `BasicMetric` in historic runs vs newer split metric classes in reproduced runs +- The current evidence suggests that at least part of the reproduction difference is due to HELM config / registry / metric behavior differences across environments or versions, not merely operator error in kwdagger scheduling. + +### Apples Batch Artifacts + +- Report bundle transferred locally to: + - `/home/joncrall/code/aiq-magnet/audit-smoke-apples` +- Representative reproduced job directories transferred locally to: + - `/home/joncrall/code/aiq-magnet/helm_id_10qfn238081w` + - `/home/joncrall/code/aiq-magnet/helm_id_kydvdl7oawex` + +### Apples Batch Technical Findings + +- All 6 case rows in the transferred JSONL indicate: + - `historic_exact_requested_max_eval_match = True` +- The union of reported run-spec execution-path drift across the transferred apples case rows is: + - `adapter_spec.model_deployment` +- Recurring primary reasons in the transferred apples report: + - `deployment_drift` + - `execution_spec_drift` +- Recurring secondary reasons in the transferred apples report: + - `evaluation_spec_drift` + - `core_metric_drift` + - `completion_content_drift` +- One case additionally reported: + - `request_prompt_drift` + +### Important Correction To Earlier Reporting + +- The first apples executive summary incorrectly said all 6 cases had a requested `max_eval_instances` mismatch. +- Inspection of the transferred apples case JSONL and reproduced job directories shows: + - historic `max_eval_instances = 1000` + - reproduced `max_eval_instances = 1000` +- Therefore that headline was a reporting bug, not a real result. +- Root cause of the bug: + - compare-side extraction of reproduced `adapter_spec.max_eval_instances` used the wrong view of the run spec +- Fix: + - `compare_batch.py` now reads reproduced `run_spec.json` directly via `load_run_spec_json(...)` + +### Representative Evidence: MMLU / Pythia-6.9b + +Run entry: +- `mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=eleutherai/pythia-6.9b,data_augmentation=canonical` + +Historic matched run: +- `/data/crfm-helm-public/classic/benchmark_output/runs/v0.2.4/mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=eleutherai_pythia-6.9b,data_augmentation=canonical` + +Reproduced transferred job: +- `/home/joncrall/code/aiq-magnet/helm_id_10qfn238081w` + +Observed config differences: +- historic `adapter_spec.model_deployment`: `null` +- reproduced `adapter_spec.model_deployment`: `huggingface/pythia-6.9b` +- historic `adapter_spec.max_eval_instances`: `1000` +- reproduced `adapter_spec.max_eval_instances`: `1000` + +Observed metric class differences: +- historic: + - `helm.benchmark.metrics.basic_metrics.BasicMetric` +- reproduced: + - `helm.benchmark.metrics.basic_metrics.BasicGenerationMetric` + - `helm.benchmark.metrics.basic_metrics.BasicReferenceMetric` + - `helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric` + +Observed behavior-level consequences from the transferred case JSONL: +- completion equal ratio about `0.896` +- core metric agree ratio about `0.5` + +Interpretation: +- once eval size is aligned, this case still differs materially +- the remaining differences are consistent with deployment resolution and metric/evaluation config changes + +### Representative Evidence: BoolQ / Vicuna-7b-v1.3 + +Run entry: +- `boolq:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical` + +Historic matched run: +- `/data/crfm-helm-public/classic/benchmark_output/runs/v0.3.0/boolq:model=lmsys_vicuna-7b-v1.3,data_augmentation=canonical` + +Reproduced transferred job: +- `/home/joncrall/code/aiq-magnet/helm_id_kydvdl7oawex` + +Observed config differences: +- historic `adapter_spec.model_deployment`: `null` +- reproduced `adapter_spec.model_deployment`: `huggingface/vicuna-7b-v1.3` +- historic `adapter_spec.max_eval_instances`: `1000` +- reproduced `adapter_spec.max_eval_instances`: `1000` + +Observed metric class differences: +- historic: + - `helm.benchmark.metrics.basic_metrics.BasicMetric` + - multiple `helm.benchmark.metrics.bias_metrics.BiasMetric` +- reproduced: + - `helm.benchmark.metrics.basic_metrics.BasicGenerationMetric` + - `helm.benchmark.metrics.basic_metrics.BasicReferenceMetric` + - `helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric` + - the same family of `helm.benchmark.metrics.bias_metrics.BiasMetric` + +Observed behavior-level consequences from the transferred case JSONL: +- completion equal ratio about `0.083` +- core metric agree ratio `0.0` + +Interpretation: +- this is a very strong mismatch even under apples-to-apples eval size +- the local reproduction is not merely “slightly off”; it is behaviorally very different for this case + +### What We Can Say To Management + +- The reproduction audit workflow is working end to end. +- We now have a real apples-to-apples control batch, not just a workflow smoke test. +- Even under apples-to-apples eval size, reproduced results still differ materially from historic public HELM runs. +- The dominant remaining differences are concentrated in deployment resolution and evaluation/metric configuration, with downstream output and metric drift. +- This is enough evidence to justify deeper technical follow-up with HELM maintainers. + +### What We Can Say To HELM Maintainers + +- The reproduced runs use explicit Hugging Face deployments where the matched historic public runs record `adapter_spec.model_deployment = null`. +- The reproduced runs also use a different metric-spec layout: + - `BasicMetric` historically + - `BasicGenerationMetric` + `BasicReferenceMetric` + `InstancesPerSplitMetric` in reproduced runs +- These config-level differences correlate with substantial completion-content and core-metric drift even after aligning requested eval size. +- The next maintainer-facing step should focus on why the same logical run entry resolves to these different `adapter_spec` / `metric_specs` structures across environments or HELM versions. + +### Suggested Next Step + +- Re-run the apples comparison after the `compare_batch.py` bugfix so the management summary no longer falsely reports requested eval-size mismatch. +- Then produce: + - one management-facing memo from the executive summary + - one maintainer-facing technical note with the two representative cases above and direct `run_spec.json` evidence + +### Tooling Note: Pairwise Comparison Artifact Validation + +- On 2026-03-27, `resolve_run.py`, `compare_entry.sh`, and `compare_pair.py` were hardened to validate finalized HELM run artifacts before diffing. +- A kwdagger job can exist with: + - `job_config.json` + - local caches + - scenario assets + while still lacking the finalized run outputs needed for diffing. +- Required files checked now: + - `run_spec.json` + - `scenario_state.json` + - `stats.json` + - `per_instance_stats.json` +- When those are missing, the tooling now reports a structured status such as: + - `artifact_status: incomplete_run_dir` + - `missing_files: [...]` + instead of surfacing a `FileNotFoundError` from deep inside `HelmRunDiff`. + +## Pairwise BoolQ / Pythia Result Snapshot + +Compared: + +- official HELM public run: + - `/data/crfm-helm-public/classic/benchmark_output/runs/v0.3.0/boolq:model=eleutherai_pythia-6.9b,data_augmentation=canonical` +- local kwdagger repeats: + - `/data/crfm-helm-audit/audit-boolq-pythia-r1/helm/helm_id_13jkx9mm4k4n/benchmark_output/runs/audit-boolq-pythia-r1/boolq:model=eleutherai_pythia-6.9b,data_augmentation=canonical` + - `/data/crfm-helm-audit/audit-boolq-pythia-r2/helm/helm_id_12jr5w48kge7/benchmark_output/runs/audit-boolq-pythia-r2/boolq:model=eleutherai_pythia-6.9b,data_augmentation=canonical` + +Repeatability baseline (`r1` vs `r2`): + +- diagnosis: `bookkeeping_metric_drift` +- strict run-level agree ratio: `0.9552238805970149` +- strict instance-level agree ratio: `0.9523809523809523` +- run-level max abs delta: `0.0015118565559387176` +- instance-level max abs delta: `0.4418189525604248` +- interpretation: + - repeated local runs are very close + - residual drift is mostly bookkeeping/runtime noise + +Historic vs local (`v0.3.0` vs `r1`): + +- diagnosis: `multiple_primary_reasons` +- primary reason names: + - `deployment_drift` + - `execution_spec_drift` +- strict run-level agree ratio: `0.4626865671641791` +- strict instance-level agree ratio: `0.6577333333333333` +- run-level abs p90: `4.0` +- run-level abs max: `11.985` +- instance-level abs p90: `4.0` +- instance-level abs max: `75.73884344100952` +- interpretation: + - historic vs local drift is much larger than local repeatability drift + - this strongly suggests the remaining mismatch is structural/configurational rather than ordinary nondeterminism diff --git a/dev/experiments/audit-helm-reproduction/README.md b/dev/experiments/audit-helm-reproduction/README.md index 245d875..8dd7ea3 100644 --- a/dev/experiments/audit-helm-reproduction/README.md +++ b/dev/experiments/audit-helm-reproduction/README.md @@ -114,10 +114,135 @@ Heavy run outputs are written by default to: /data/crfm-helm-audit// ``` +## Refreshing Analysis From Latest Data + +When new raw audit runs have landed under `/data/crfm-helm-audit`, the easiest +way to refresh analysis is: + +1. rebuild the audit index +2. regenerate the specific report you care about from that index + +Rebuild the index: + +```bash +AUDIT_FALLBACK_HOST=aiq-gpu \ +dev/experiments/audit-helm-reproduction/scripts/index_results.sh +``` + +This writes: + +- `dev/experiments/audit-helm-reproduction/reports/indexes/audit_results_index_.jsonl` +- `dev/experiments/audit-helm-reproduction/reports/indexes/audit_results_index_.csv` +- `dev/experiments/audit-helm-reproduction/reports/indexes/audit_results_index_.txt` + +Use `AUDIT_FALLBACK_HOST` for older runs that predate explicit process +provenance capture. Newer runs will record machine context directly in each job +directory via `process_context.json`. + +Then rebuild a core metric report for a specific run entry: + +```bash +dev/experiments/audit-helm-reproduction/scripts/rebuild_core_report_from_index.sh \ + --run-entry 'narrative_qa:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical' +``` + +This will: + +- pick the newest two indexed kwdagger runs for that exact `run_entry` +- find the best matching historic HELM run from `/data/crfm-helm-public` +- regenerate the core metric report bundle automatically + +Each stable report directory now stores timestamped artifacts under: + +- `.history//` + +The top-level user-facing files are symlinks to the newest timestamped +artifacts, for example: + +- `core_metric_management_summary.latest.txt` +- `core_metric_report.latest.txt` +- `core_metric_report.latest.json` +- `core_metric_report.latest.png` +- `core_metric_overlay_distributions.latest.png` +- `core_metric_ecdfs.latest.png` +- `core_runlevel_table.latest.csv` +- `report_selection.latest.json` +- `instance_samples_kwdagger_repeat.latest.txt` +- `instance_samples_official_vs_kwdagger.latest.txt` +- `kwdagger_a.run` +- `kwdagger_b.run` +- `official.run` +- `kwdagger_a.job` +- `kwdagger_b.job` + +This is the preferred way to inspect the newest version of a report without +guessing which timestamp is current. + +The raw-run symlinks make it easy to jump directly from a report bundle back to +the exact local and historic run directories that were selected for analysis. +The instance sample reports are text summaries produced from +`HelmRunDiff.summarize_instances(...)` and are intended as the first stop when +you want to inspect prompts, completions, references, and the largest per-run +instance mismatches in a suspicious case. + +If only one local run exists and you still want a report, allow reuse of that +single run on both sides of the repeatability slot: + +```bash +dev/experiments/audit-helm-reproduction/scripts/rebuild_core_report_from_index.sh \ + --run-entry 'narrative_qa:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical' \ + --allow-single-repeat +``` + +This is the preferred workflow for fast iteration when we want to keep changing +what we measure or how we visualize it without manually reconstructing run +paths. + +To rebuild all currently available core reports from the latest index: + +```bash +bash dev/experiments/audit-helm-reproduction/scripts/rebuild_all_core_reports_from_index.sh +``` + +If you want the tool to emit reports even for entries with only one matching +kwdagger run, use: + +```bash +bash dev/experiments/audit-helm-reproduction/scripts/rebuild_all_core_reports_from_index.sh \ + --allow-single-repeat +``` + +After rebuilding the per-run-spec reports, generate an overall reproducibility +assessment across all currently available core reports: + +```bash +bash dev/experiments/audit-helm-reproduction/scripts/aggregate_core_reports.sh +``` + +This writes: + +- `reports/overall-reproducibility/.history//overall_reproducibility_summary_.*` +- stable symlinks: + - `overall_reproducibility_summary.latest.txt` + - `overall_reproducibility_summary.latest.json` + - `overall_reproducibility_summary.latest.csv` + - `overall_reproducibility_summary.latest.md` + The runner also derives a distinct `kwdagger` queue name from the experiment name, which helps avoid interactive tmux collision prompts when multiple audit batches have been launched on the same machine. +If you want to inspect a specific pair directly without rebuilding the full +core report, you can write an instance-sample inspection report with: + +```bash +bash dev/experiments/audit-helm-reproduction/scripts/inspect_pair_samples.sh \ + --run-a /path/to/run_a \ + --run-b /path/to/run_b \ + --label investigation_pair \ + --report-dpath dev/experiments/audit-helm-reproduction/reports/manual-inspection +``` + ## Reproducibility Checklist For any experiment you want to cite later, preserve all of the following: @@ -154,6 +279,13 @@ Minimum useful transfer set: - report directory plus 1-2 representative raw job directories, if you need direct run artifact inspection - full raw experiment directory, if you may need to rerun local comparisons later +For newer runs, also preserve: + +- `process_context.json` in each kwdagger HELM job directory + +That file records structured host/process provenance and is intended to support +future cross-machine and cross-hardware analysis. + ## Manifest Schema The workflow uses a small YAML manifest as the unit of experiment definition. @@ -418,6 +550,34 @@ ls -td dev/experiments/audit-helm-reproduction/reports/pairwise/* cat dev/experiments/audit-helm-reproduction/reports/pairwise/pair_report_.txt ``` +## Indexing Existing Audit Results + +The indexer scans all current audit outputs and builds a machine-readable +inventory of what exists. + +Use: + +```bash +AUDIT_FALLBACK_HOST=aiq-gpu \ +dev/experiments/audit-helm-reproduction/scripts/index_results.sh +``` + +The index currently records: + +- experiment name +- job id +- status +- run entry +- benchmark / model / method +- max eval instances +- resolved run directory +- machine host +- GPU fields when recorded +- provenance source (`recorded` vs `fallback`) + +This index is the preferred starting point for rebuilding reports against the +latest available runs without manually searching the raw results tree. + ## Notes - This workflow is intended for execution on an external GPU machine. diff --git a/dev/experiments/audit-helm-reproduction/python/aggregate_core_reports.py b/dev/experiments/audit-helm-reproduction/python/aggregate_core_reports.py new file mode 100644 index 0000000..347d7ec --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/python/aggregate_core_reports.py @@ -0,0 +1,165 @@ +from __future__ import annotations + +import argparse +import datetime as datetime_mod +import glob +import json +import os +from collections import Counter +from pathlib import Path +from typing import Any + +import pandas as pd + +from common import default_report_root + + +def _load_json(fpath: Path) -> dict[str, Any]: + return json.loads(fpath.read_text()) + + +def _find_pair(report: dict[str, Any], label: str) -> dict[str, Any]: + for pair in report.get('pairs', []): + if pair.get('label') == label: + return pair + return {} + + +def _find_curve_value(rows: list[dict[str, Any]], abs_tol: float) -> float | None: + for row in rows or []: + try: + if float(row.get('abs_tol')) == float(abs_tol): + return float(row.get('agree_ratio')) + except Exception: + pass + return None + + +def _assessment_label(repeat_agree_0: float | None, official_agree_01: float | None) -> str: + if repeat_agree_0 is None or official_agree_01 is None: + return 'unknown' + if repeat_agree_0 >= 0.99 and official_agree_01 >= 0.95: + return 'close_match' + if repeat_agree_0 >= 0.99 and official_agree_01 >= 0.75: + return 'moderate_drift' + if repeat_agree_0 >= 0.99: + return 'strong_drift' + return 'unstable_local_repeat' + + +def _slugify(text: str) -> str: + return ( + text.replace('/', '-') + .replace(':', '-') + .replace(',', '-') + .replace('=', '-') + .replace('@', '-') + .replace(' ', '-') + ) + + +def _write_latest_alias(src: Path, latest_root: Path, latest_name: str) -> None: + latest_fpath = latest_root / latest_name + if latest_fpath.exists() or latest_fpath.is_symlink(): + latest_fpath.unlink() + rel_src = os.path.relpath(src, start=latest_root) + os.symlink(rel_src, latest_fpath) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('--report-root', default=str(default_report_root())) + parser.add_argument('--out-dpath', default=None) + args = parser.parse_args() + + report_root = Path(args.report_root).expanduser().resolve() + out_dpath = Path(args.out_dpath).expanduser().resolve() if args.out_dpath else (report_root / 'overall-reproducibility') + out_dpath.mkdir(parents=True, exist_ok=True) + + report_paths = sorted(glob.glob(str(report_root / 'core-metrics-*' / 'core_metric_report.latest.json'))) + rows = [] + for p in report_paths: + fpath = Path(p) + report = _load_json(fpath) + repeat = _find_pair(report, 'kwdagger_repeat') + official = _find_pair(report, 'official_vs_kwdagger') + repeat_agree_0 = _find_curve_value(repeat.get('instance_level', {}).get('agreement_vs_abs_tol', []), 0.0) + official_agree_0 = _find_curve_value(official.get('instance_level', {}).get('agreement_vs_abs_tol', []), 0.0) + official_agree_01 = _find_curve_value(official.get('instance_level', {}).get('agreement_vs_abs_tol', []), 0.1) + official_agree_025 = _find_curve_value(official.get('instance_level', {}).get('agreement_vs_abs_tol', []), 0.25) + official_agree_05 = _find_curve_value(official.get('instance_level', {}).get('agreement_vs_abs_tol', []), 0.5) + rows.append({ + 'report_dir': str(fpath.parent), + 'report_json': str(fpath), + 'run_spec_name': report.get('run_spec_name'), + 'generated_utc': report.get('generated_utc'), + 'n_core_metrics': len((_find_pair(report, 'kwdagger_repeat').get('core_metrics') or [])), + 'repeat_instance_agree_0': repeat_agree_0, + 'official_instance_agree_0': official_agree_0, + 'official_instance_agree_01': official_agree_01, + 'official_instance_agree_025': official_agree_025, + 'official_instance_agree_05': official_agree_05, + 'official_runlevel_p90': (((official.get('run_level') or {}).get('overall_quantiles') or {}).get('abs_delta') or {}).get('p90'), + 'official_runlevel_max': (((official.get('run_level') or {}).get('overall_quantiles') or {}).get('abs_delta') or {}).get('max'), + 'assessment_label': _assessment_label(repeat_agree_0, official_agree_01), + }) + + table = pd.DataFrame(rows).sort_values(['assessment_label', 'run_spec_name'], na_position='last') + stamp = datetime_mod.datetime.now(datetime_mod.UTC).strftime('%Y%m%dT%H%M%SZ') + history_dpath = out_dpath / '.history' / stamp[:8] + history_dpath.mkdir(parents=True, exist_ok=True) + + json_fpath = history_dpath / f'overall_reproducibility_summary_{stamp}.json' + csv_fpath = history_dpath / f'overall_reproducibility_summary_{stamp}.csv' + txt_fpath = history_dpath / f'overall_reproducibility_summary_{stamp}.txt' + md_fpath = history_dpath / f'overall_reproducibility_summary_{stamp}.md' + + summary = { + 'generated_utc': stamp, + 'n_reports': len(rows), + 'assessment_counts': dict(Counter(row['assessment_label'] for row in rows)), + 'run_specs': rows, + } + json_fpath.write_text(json.dumps(summary, indent=2)) + table.to_csv(csv_fpath, index=False) + md_fpath.write_text(table.to_markdown(index=False) + '\n') + + lines = [] + lines.append('Overall Reproducibility Assessment') + lines.append('') + lines.append(f'generated_utc: {stamp}') + lines.append(f'n_reports: {len(rows)}') + lines.append('') + lines.append('assessment_counts:') + for key, val in sorted(summary['assessment_counts'].items()): + lines.append(f' {key}: {val}') + lines.append('') + lines.append('per_run_spec:') + for row in rows: + lines.append(f" - run_spec_name: {row['run_spec_name']}") + lines.append(f" assessment_label: {row['assessment_label']}") + lines.append(f" repeat_instance_agree_0: {row['repeat_instance_agree_0']}") + lines.append(f" official_instance_agree_0: {row['official_instance_agree_0']}") + lines.append(f" official_instance_agree_01: {row['official_instance_agree_01']}") + lines.append(f" official_instance_agree_025: {row['official_instance_agree_025']}") + lines.append(f" official_instance_agree_05: {row['official_instance_agree_05']}") + lines.append(f" official_runlevel_p90: {row['official_runlevel_p90']}") + lines.append(f" official_runlevel_max: {row['official_runlevel_max']}") + txt_fpath.write_text('\n'.join(lines) + '\n') + + for src, latest_name in [ + (json_fpath, 'overall_reproducibility_summary.latest.json'), + (csv_fpath, 'overall_reproducibility_summary.latest.csv'), + (txt_fpath, 'overall_reproducibility_summary.latest.txt'), + (md_fpath, 'overall_reproducibility_summary.latest.md'), + ]: + _write_latest_alias(src, out_dpath, latest_name) + + print(f'Wrote summary json: {json_fpath}') + print(f'Wrote summary csv: {csv_fpath}') + print(f'Wrote summary md: {md_fpath}') + print(f'Wrote summary txt: {txt_fpath}') + + +if __name__ == '__main__': + main() diff --git a/dev/experiments/audit-helm-reproduction/python/core_metric_report.py b/dev/experiments/audit-helm-reproduction/python/core_metric_report.py index c5a6785..951bb8a 100644 --- a/dev/experiments/audit-helm-reproduction/python/core_metric_report.py +++ b/dev/experiments/audit-helm-reproduction/python/core_metric_report.py @@ -4,6 +4,8 @@ import datetime as datetime_mod import json import math +import os +import shutil from pathlib import Path from typing import Any @@ -158,6 +160,43 @@ def _metric_quantiles(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: return out +def _metric_descriptor(metric: str) -> dict[str, str]: + if metric in { + 'exact_match', + 'prefix_exact_match', + 'quasi_exact_match', + 'quasi_prefix_exact_match', + 'exact_match@5', + 'prefix_exact_match@5', + 'quasi_exact_match@5', + 'quasi_prefix_exact_match@5', + }: + return { + 'kind': 'binary', + 'range': '0 to 1', + 'direction': 'higher is better', + } + if metric in {'bleu_1', 'bleu_4', 'f1_score', 'rouge_l'}: + return { + 'kind': 'bounded overlap score', + 'range': '0 to 1', + 'direction': 'higher is better', + } + return { + 'kind': 'score', + 'range': 'metric-dependent', + 'direction': 'higher is better unless documented otherwise', + } + + +def _should_treat_as_discrete(values) -> bool: + values = [float(v) for v in values if v is not None] + unique_values = sorted(set(values)) + if not unique_values: + return False + return len(unique_values) <= 8 and all(v in {0.0, 1.0} for v in unique_values) + + def _agreement_curve(rows: list[dict[str, Any]], thresholds: list[float]) -> list[dict[str, Any]]: if not rows: return [] @@ -174,6 +213,17 @@ def _agreement_curve(rows: list[dict[str, Any]], thresholds: list[float]) -> lis return out +def _infer_run_spec_name(*run_paths: str) -> str: + names = [Path(p).name for p in run_paths if p] + names = [n for n in names if n] + if not names: + return 'unknown_run_spec' + unique = sorted(set(names)) + if len(unique) == 1: + return unique[0] + return unique[0] + + def _build_pair(run_a: str, run_b: str, label: str, thresholds: list[float]) -> dict[str, Any]: diff = HelmRunDiff(HelmRun.coerce(run_a), HelmRun.coerce(run_b), a_name=f'{label}:A', b_name=f'{label}:B') run_rows = _run_level_core_rows(diff) @@ -187,11 +237,13 @@ def _build_pair(run_a: str, run_b: str, label: str, thresholds: list[float]) -> 'diagnosis': diff.summary_dict(level=20).get('diagnosis', {}), 'core_metrics': sorted({str(r['metric']) for r in inst_rows}), 'run_level': { + 'n_rows': len(run_rows), 'overall_quantiles': _group_quantiles(run_rows), 'by_metric': _metric_quantiles(run_rows), 'agreement_vs_abs_tol': _agreement_curve(run_rows, thresholds), }, 'instance_level': { + 'n_rows': len(inst_rows), 'overall_quantiles': _group_quantiles(inst_rows), 'by_metric': _metric_quantiles(inst_rows), 'agreement_vs_abs_tol': _agreement_curve(inst_rows, thresholds), @@ -227,8 +279,8 @@ def _plot_distribution(ax, pair_a: dict[str, Any], pair_b: dict[str, Any], level ) ax.set_xscale('symlog', linthresh=1e-12) ax.set_ylim(0, 1.02) - ax.set_xlabel('Absolute tolerance') - ax.set_ylabel('Agreement ratio') + ax.set_xlabel('Absolute Tolerance Threshold for Core Metric Difference') + ax.set_ylabel('Fraction of Core Metric Comparisons in Agreement') ax.legend(title='') @@ -242,7 +294,8 @@ def _plot_quantiles(ax, pair_a: dict[str, Any], pair_b: dict[str, Any], level_ke ax.set_xticks(x, labels) ax.set_yscale('symlog', linthresh=1e-12) ax.set_title(title) - ax.set_ylabel('Absolute delta quantile') + ax.set_xlabel('Quantile') + ax.set_ylabel('Absolute Difference in Core Metric Value') ax.legend() @@ -264,14 +317,19 @@ def _distribution_rows(pair: dict[str, Any]) -> pd.DataFrame: return pd.DataFrame(rows) -def _plot_metric_distributions(fig_dpath: Path, stamp: str, left: dict[str, Any], right: dict[str, Any]) -> Path: +def _plot_metric_distributions(fig_dpath: Path, stamp: str, left: dict[str, Any], right: dict[str, Any], run_spec_name: str) -> Path: df = pd.concat([ _distribution_rows(left), _distribution_rows(right), ], ignore_index=True) metrics = sorted(df['metric'].dropna().unique().tolist()) pair_order = [left['label'], right['label']] - fig, axes = plt.subplots(len(pair_order), len(metrics), figsize=(4 * len(metrics), 3.5 * len(pair_order)), constrained_layout=True) + fig, axes = plt.subplots( + len(pair_order), + len(metrics), + figsize=(5.2 * len(metrics), 4.2 * len(pair_order)), + constrained_layout=True, + ) if len(pair_order) == 1 and len(metrics) == 1: axes = [[axes]] elif len(pair_order) == 1: @@ -282,15 +340,17 @@ def _plot_metric_distributions(fig_dpath: Path, stamp: str, left: dict[str, Any] for col_idx, metric in enumerate(metrics): ax = axes[row_idx][col_idx] sub = df[(df['pair'] == pair_label) & (df['metric'] == metric)] + discrete = _should_treat_as_discrete(sub['value'].tolist()) sns.histplot( data=sub, x='value', hue='side', stat='probability', common_norm=False, - discrete=True, + discrete=discrete, multiple='dodge', shrink=0.8, + bins=None if discrete else 20, ax=ax, ) ax.set_title(f'{pair_label}\n{metric}') @@ -299,6 +359,12 @@ def _plot_metric_distributions(fig_dpath: Path, stamp: str, left: dict[str, Any] legend = ax.get_legend() if legend is not None: legend.set_title('') + fig.suptitle( + 'Core Metric Score Distributions Within Each Comparison Pair\n' + f'Run Spec: {run_spec_name}\n' + 'Each panel shows the per-instance score distribution for side A vs side B.', + fontsize=16, + ) out_fpath = fig_dpath / f'core_metric_distributions_{stamp}.png' fig.savefig(out_fpath, dpi=180) plt.close(fig) @@ -335,6 +401,7 @@ def _plot_three_run_metric_distributions( kwdagger_a_run: str, kwdagger_b_run: str, official_run: str, + run_spec_name: str, ) -> Path: df = pd.concat([ _single_run_instance_core_rows(kwdagger_a_run, 'kwdagger A'), @@ -346,7 +413,7 @@ def _plot_three_run_metric_distributions( fig, axes = plt.subplots( len(metrics), len(run_order), - figsize=(4 * len(run_order), 2.8 * len(metrics)), + figsize=(5.0 * len(run_order), 3.2 * len(metrics)), constrained_layout=True, ) if len(metrics) == 1 and len(run_order) == 1: @@ -359,12 +426,14 @@ def _plot_three_run_metric_distributions( for col_idx, run_label in enumerate(run_order): ax = axes[row_idx][col_idx] sub = df[(df['metric'] == metric) & (df['run'] == run_label)] + discrete = _should_treat_as_discrete(sub['value'].tolist()) sns.histplot( data=sub, x='value', stat='probability', - discrete=True, + discrete=discrete, shrink=0.8, + bins=None if discrete else 20, ax=ax, color='#4C72B0', ) @@ -372,12 +441,128 @@ def _plot_three_run_metric_distributions( ax.set_title(run_label) ax.set_xlabel('Core metric value') ax.set_ylabel(metric if col_idx == 0 else '') + fig.suptitle( + 'Per-Run Instance-Level Core Metric Score Distributions\n' + f'Run Spec: {run_spec_name}\n' + 'Columns are kwdagger repeat A, kwdagger repeat B, and the official HELM run.', + fontsize=16, + ) out_fpath = fig_dpath / f'core_metric_three_run_distributions_{stamp}.png' fig.savefig(out_fpath, dpi=180) plt.close(fig) return out_fpath +def _plot_overlay_metric_distributions( + fig_dpath: Path, + stamp: str, + kwdagger_a_run: str, + kwdagger_b_run: str, + official_run: str, + run_spec_name: str, +) -> Path: + df = pd.concat([ + _single_run_instance_core_rows(kwdagger_a_run, 'kwdagger A'), + _single_run_instance_core_rows(kwdagger_b_run, 'kwdagger B'), + _single_run_instance_core_rows(official_run, 'official'), + ], ignore_index=True) + metrics = sorted(df['metric'].dropna().unique().tolist()) + fig, axes = plt.subplots( + len(metrics), + 1, + figsize=(10, 3.2 * len(metrics)), + constrained_layout=True, + ) + if len(metrics) == 1: + axes = [axes] + for ax, metric in zip(axes, metrics): + sub = df[df['metric'] == metric].copy() + discrete = _should_treat_as_discrete(sub['value'].tolist()) + sns.histplot( + data=sub, + x='value', + hue='run', + stat='probability', + common_norm=False, + element='step', + fill=False, + multiple='layer', + discrete=discrete, + bins=None if discrete else 20, + ax=ax, + ) + desc = _metric_descriptor(metric) + ax.set_title( + f"{metric} ({desc['kind']}, {desc['range']}, {desc['direction']})" + ) + ax.set_xlabel('Instance-level metric value') + ax.set_ylabel('Probability') + legend = ax.get_legend() + if legend is not None: + legend.set_title('') + fig.suptitle( + 'Overlay of Per-Instance Core Metric Score Distributions by Run\n' + f'Run Spec: {run_spec_name}\n' + 'This shows the raw score distributions for each core metric across kwdagger repeats and the official HELM run.', + fontsize=16, + ) + out_fpath = fig_dpath / f'core_metric_overlay_distributions_{stamp}.png' + fig.savefig(out_fpath, dpi=180) + plt.close(fig) + return out_fpath + + +def _plot_overlay_metric_ecdfs( + fig_dpath: Path, + stamp: str, + kwdagger_a_run: str, + kwdagger_b_run: str, + official_run: str, + run_spec_name: str, +) -> Path: + df = pd.concat([ + _single_run_instance_core_rows(kwdagger_a_run, 'kwdagger A'), + _single_run_instance_core_rows(kwdagger_b_run, 'kwdagger B'), + _single_run_instance_core_rows(official_run, 'official'), + ], ignore_index=True) + metrics = sorted(df['metric'].dropna().unique().tolist()) + fig, axes = plt.subplots( + len(metrics), + 1, + figsize=(10, 3.2 * len(metrics)), + constrained_layout=True, + ) + if len(metrics) == 1: + axes = [axes] + for ax, metric in zip(axes, metrics): + sub = df[df['metric'] == metric].copy() + sns.ecdfplot( + data=sub, + x='value', + hue='run', + ax=ax, + ) + desc = _metric_descriptor(metric) + ax.set_title( + f"{metric} ECDF ({desc['kind']}, {desc['range']}, {desc['direction']})" + ) + ax.set_xlabel('Instance-level metric value') + ax.set_ylabel('Cumulative fraction of instances') + legend = ax.get_legend() + if legend is not None: + legend.set_title('') + fig.suptitle( + 'ECDF of Per-Instance Core Metric Scores by Run\n' + f'Run Spec: {run_spec_name}\n' + 'This often communicates sparse or zero-heavy metric distributions more clearly than histograms.', + fontsize=16, + ) + out_fpath = fig_dpath / f'core_metric_ecdfs_{stamp}.png' + fig.savefig(out_fpath, dpi=180) + plt.close(fig) + return out_fpath + + def _single_run_core_stat_index(run_path: str) -> dict[str, Any]: ana = HelmRunAnalysis(HelmRun.coerce(run_path)) idx = ana.stat_index(drop_zero_count=True, require_mean=True) @@ -446,6 +631,8 @@ def _write_text(report: dict[str, Any], out_fpath: Path) -> None: lines.append(f"pair: {pair['label']}") lines.append(f" diagnosis: {pair['diagnosis'].get('label')}") lines.append(f" primary_reason_names: {pair['diagnosis'].get('primary_reason_names')}") + lines.append(f" run_level_n: {pair['run_level']['n_rows']}") + lines.append(f" instance_level_n: {pair['instance_level']['n_rows']}") lines.append(f" run_level_quantiles: {json.dumps(pair['run_level']['overall_quantiles']['abs_delta'])}") lines.append(f" instance_level_quantiles: {json.dumps(pair['instance_level']['overall_quantiles']['abs_delta'])}") lines.append(' by_metric:') @@ -478,10 +665,20 @@ def _write_management_summary(report: dict[str, Any], out_fpath: Path) -> None: lines.append('Core Metric Executive Summary') lines.append('') lines.append(f"generated_utc: {report['generated_utc']}") + lines.append(f"run_spec_name: {report['run_spec_name']}") lines.append(f"core_metrics: {', '.join(left.get('core_metrics', []))}") lines.append('') + lines.append('metric_descriptions:') + for metric in left.get('core_metrics', []): + desc = _metric_descriptor(metric) + lines.append( + f" - {metric}: {desc['kind']}; {desc['range']}; {desc['direction']}" + ) + lines.append('') lines.append(f"{left['label']}:") lines.append(f" diagnosis: {left['diagnosis'].get('label')}") + lines.append(f" run-level N: {left['run_level']['n_rows']}") + lines.append(f" instance-level N: {left['instance_level']['n_rows']}") lines.append( f" instance agreement at abs_tol=0.0: {_find_curve_value(left['instance_level']['agreement_vs_abs_tol'], 0.0)}" ) @@ -494,7 +691,9 @@ def _write_management_summary(report: dict[str, Any], out_fpath: Path) -> None: lines.append('') lines.append(f"{right['label']}:") lines.append(f" diagnosis: {right['diagnosis'].get('label')}") - for tol in [0.0, 1e-3, 1e-2, 1e-1, 1.0]: + lines.append(f" run-level N: {right['run_level']['n_rows']}") + lines.append(f" instance-level N: {right['instance_level']['n_rows']}") + for tol in [0.0, 1e-3, 1e-2, 1e-1, 2.5e-1, 5e-1, 1.0]: lines.append( f" instance agreement at abs_tol={tol}: " f"{_find_curve_value(right['instance_level']['agreement_vs_abs_tol'], tol)}" @@ -512,6 +711,15 @@ def _write_management_summary(report: dict[str, Any], out_fpath: Path) -> None: out_fpath.write_text('\n'.join(lines) + '\n') +def _write_latest_alias(src: Path, latest_root: Path, latest_name: str) -> Path: + latest_fpath = latest_root / latest_name + if latest_fpath.exists() or latest_fpath.is_symlink(): + latest_fpath.unlink() + rel_src = os.path.relpath(src, start=latest_fpath.parent) + os.symlink(rel_src, latest_fpath) + return latest_fpath + + def main() -> None: parser = argparse.ArgumentParser() parser.add_argument('--left-run-a', required=True) @@ -523,33 +731,54 @@ def main() -> None: parser.add_argument('--report-dpath', required=True) args = parser.parse_args() - thresholds = [0.0, 1e-12, 1e-9, 1e-6, 1e-4, 1e-3, 1e-2, 2e-2, 5e-2, 1e-1, 5e-1, 1.0] + thresholds = [0.0, 1e-12, 1e-9, 1e-6, 1e-4, 1e-3, 1e-2, 2e-2, 5e-2, 1e-1, 2.5e-1, 5e-1, 1.0] report_dpath = Path(args.report_dpath).expanduser().resolve() report_dpath.mkdir(parents=True, exist_ok=True) stamp = datetime_mod.datetime.now(datetime_mod.UTC).strftime('%Y%m%dT%H%M%SZ') + history_dpath = report_dpath / '.history' / stamp[:8] + history_dpath.mkdir(parents=True, exist_ok=True) + run_spec_name = _infer_run_spec_name(args.left_run_a, args.left_run_b, args.right_run_a) left = _build_pair(args.left_run_a, args.left_run_b, args.left_label, thresholds) right = _build_pair(args.right_run_a, args.right_run_b, args.right_label, thresholds) report = { 'generated_utc': stamp, + 'run_spec_name': run_spec_name, 'thresholds': thresholds, 'pairs': [left, right], } - json_fpath = report_dpath / f'core_metric_report_{stamp}.json' - txt_fpath = report_dpath / f'core_metric_report_{stamp}.txt' - mgmt_fpath = report_dpath / f'core_metric_management_summary_{stamp}.txt' - fig_fpath = report_dpath / f'core_metric_report_{stamp}.png' - dist_fig_fpath = _plot_metric_distributions(report_dpath, stamp, left, right) + json_fpath = history_dpath / f'core_metric_report_{stamp}.json' + txt_fpath = history_dpath / f'core_metric_report_{stamp}.txt' + mgmt_fpath = history_dpath / f'core_metric_management_summary_{stamp}.txt' + fig_fpath = history_dpath / f'core_metric_report_{stamp}.png' + dist_fig_fpath = _plot_metric_distributions(history_dpath, stamp, left, right, run_spec_name) three_run_dist_fpath = _plot_three_run_metric_distributions( - report_dpath, + history_dpath, + stamp, + args.left_run_a, + args.left_run_b, + args.right_run_a, + run_spec_name, + ) + overlay_dist_fpath = _plot_overlay_metric_distributions( + history_dpath, + stamp, + args.left_run_a, + args.left_run_b, + args.right_run_a, + run_spec_name, + ) + ecdf_fig_fpath = _plot_overlay_metric_ecdfs( + history_dpath, stamp, args.left_run_a, args.left_run_b, args.right_run_a, + run_spec_name, ) runlevel_csv_fpath, runlevel_md_fpath = _write_three_run_runlevel_table( - report_dpath, + history_dpath, stamp, args.left_run_a, args.left_run_b, @@ -562,22 +791,65 @@ def main() -> None: _write_management_summary(report, mgmt_fpath) sns.set_theme(style='whitegrid', context='talk') - fig, axes = plt.subplots(2, 2, figsize=(12, 8), constrained_layout=True) - _plot_quantiles(axes[0, 0], left, right, 'run_level', 'Core Run-Level Abs Delta Quantiles') - _plot_quantiles(axes[0, 1], left, right, 'instance_level', 'Core Instance-Level Abs Delta Quantiles') + fig, axes = plt.subplots(2, 2, figsize=(15, 10), constrained_layout=True) + _plot_quantiles( + axes[0, 0], + left, + right, + 'run_level', + f"Run-Level Core Metric Difference Quantiles\nRun Spec: {run_spec_name}\nN={left['run_level']['n_rows']} vs N={right['run_level']['n_rows']}" + ) + _plot_quantiles( + axes[0, 1], + left, + right, + 'instance_level', + f"Instance-Level Core Metric Difference Quantiles\nRun Spec: {run_spec_name}\nN={left['instance_level']['n_rows']} vs N={right['instance_level']['n_rows']}" + ) _plot_distribution(axes[1, 0], left, right, 'run_level') - axes[1, 0].set_title('Core Run-Level Agreement vs Abs Tol') + axes[1, 0].set_title( + f"Run-Level Core Metric Agreement vs Tolerance\n" + f"Run Spec: {run_spec_name}\n" + f"{left['label']} N={left['run_level']['n_rows']}, {right['label']} N={right['run_level']['n_rows']}" + ) _plot_distribution(axes[1, 1], left, right, 'instance_level') - axes[1, 1].set_title('Core Instance-Level Agreement vs Abs Tol') + axes[1, 1].set_title( + f"Instance-Level Core Metric Agreement vs Tolerance\n" + f"Run Spec: {run_spec_name}\n" + f"{left['label']} N={left['instance_level']['n_rows']}, {right['label']} N={right['instance_level']['n_rows']}" + ) + fig.suptitle( + 'Core Metric Agreement and Difference Summary\n' + f'Run Spec: {run_spec_name}\n' + 'Comparing kwdagger repeatability against official-vs-kwdagger divergence.', + fontsize=18, + ) fig.savefig(fig_fpath, dpi=180) plt.close(fig) + latest_map = { + json_fpath: 'core_metric_report.latest.json', + txt_fpath: 'core_metric_report.latest.txt', + mgmt_fpath: 'core_metric_management_summary.latest.txt', + fig_fpath: 'core_metric_report.latest.png', + dist_fig_fpath: 'core_metric_distributions.latest.png', + three_run_dist_fpath: 'core_metric_three_run_distributions.latest.png', + overlay_dist_fpath: 'core_metric_overlay_distributions.latest.png', + ecdf_fig_fpath: 'core_metric_ecdfs.latest.png', + runlevel_csv_fpath: 'core_runlevel_table.latest.csv', + runlevel_md_fpath: 'core_runlevel_table.latest.md', + } + for src, latest_name in latest_map.items(): + _write_latest_alias(src, report_dpath, latest_name) + print(f'Wrote core metric report: {json_fpath}') print(f'Wrote core metric text: {txt_fpath}') print(f'Wrote core metric management summary: {mgmt_fpath}') print(f'Wrote core metric plot: {fig_fpath}') print(f'Wrote core metric distributions: {dist_fig_fpath}') print(f'Wrote core metric three-run distributions: {three_run_dist_fpath}') + print(f'Wrote core metric overlay distributions: {overlay_dist_fpath}') + print(f'Wrote core metric ecdfs: {ecdf_fig_fpath}') print(f'Wrote core run-level table csv: {runlevel_csv_fpath}') print(f'Wrote core run-level table md: {runlevel_md_fpath}') diff --git a/dev/experiments/audit-helm-reproduction/python/index_results.py b/dev/experiments/audit-helm-reproduction/python/index_results.py new file mode 100644 index 0000000..7cbe7b9 --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/python/index_results.py @@ -0,0 +1,201 @@ +from __future__ import annotations + +import argparse +import datetime as datetime_mod +import json +from collections import Counter +from pathlib import Path +from typing import Any + +import kwutil +import pandas as pd + +from common import default_report_root, env_defaults +from magnet.backends.helm.cli.materialize_helm_run import parse_run_entry_description +from magnet.backends.helm.helm_outputs import HelmOutputs + + +def _safe_json_load(fpath: Path) -> dict[str, Any]: + if not fpath.exists(): + return {} + try: + return json.loads(fpath.read_text()) + except Exception: + return {} + + +def _first_run_dir(job_dpath: Path) -> Path | None: + bo = job_dpath / 'benchmark_output' + if not bo.exists(): + return None + try: + outputs = HelmOutputs.coerce(bo) + except Exception: + return None + runs = [] + for suite in outputs.suites(pattern='*'): + runs.extend(list(suite.runs())) + if not runs: + return None + return Path(runs[0].path) + + +def _process_context_info(process_context: dict[str, Any], fallback_host: str | None) -> dict[str, Any]: + props = process_context.get('properties', {}) if isinstance(process_context, dict) else {} + machine = props.get('machine', {}) if isinstance(props.get('machine', {}), dict) else {} + extra = props.get('extra', {}) if isinstance(props.get('extra', {}), dict) else {} + env = extra.get('env', {}) if isinstance(extra.get('env', {}), dict) else {} + nvidia_smi = extra.get('nvidia_smi', {}) if isinstance(extra.get('nvidia_smi', {}), dict) else {} + gpus = nvidia_smi.get('gpus', []) if isinstance(nvidia_smi.get('gpus', []), list) else [] + + host = machine.get('host') + provenance = 'recorded' + if not host: + host = fallback_host + provenance = 'fallback' if fallback_host else 'unknown' + + return { + 'machine_host': host, + 'machine_user': machine.get('user'), + 'machine_os': machine.get('os_name'), + 'machine_arch': machine.get('arch'), + 'python_version': machine.get('py_version'), + 'cuda_visible_devices': env.get('CUDA_VISIBLE_DEVICES'), + 'gpu_count': len(gpus), + 'gpu_names': [g.get('name') for g in gpus if isinstance(g, dict)], + 'gpu_memory_total_mb': [g.get('memory_total_mb') for g in gpus if isinstance(g, dict)], + 'provenance_source': provenance, + } + + +def _row_for_job(job_config_fpath: Path, fallback_host: str | None) -> dict[str, Any]: + job_dpath = job_config_fpath.parent + adapter_manifest = _safe_json_load(job_dpath / 'adapter_manifest.json') + process_context = _safe_json_load(job_dpath / 'process_context.json') + if not process_context: + process_context = adapter_manifest.get('process_context', {}) if isinstance(adapter_manifest, dict) else {} + run_dir = _first_run_dir(job_dpath) + run_spec = _safe_json_load(run_dir / 'run_spec.json') if run_dir else {} + + job_config = _safe_json_load(job_config_fpath) + run_entry = job_config.get('helm.run_entry') + benchmark = None + model = None + method = None + if run_entry: + try: + benchmark, tokens = parse_run_entry_description(run_entry) + model = tokens.get('model') + method = tokens.get('method') + except Exception: + benchmark = None + + context_info = _process_context_info(process_context, fallback_host) + adapter_spec = run_spec.get('adapter_spec', {}) if isinstance(run_spec, dict) else {} + metric_specs = run_spec.get('metric_specs', []) if isinstance(run_spec, dict) else [] + + row = { + 'experiment_name': job_dpath.parent.parent.name if job_dpath.parent.name == 'helm' else job_dpath.parent.name, + 'job_id': job_dpath.name, + 'job_dpath': str(job_dpath), + 'status': adapter_manifest.get('status'), + 'manifest_timestamp': adapter_manifest.get('timestamp'), + 'run_entry': run_entry, + 'benchmark': benchmark, + 'model': model, + 'method': method, + 'suite': job_config.get('helm.suite'), + 'max_eval_instances': job_config.get('helm.max_eval_instances'), + 'run_dir': str(run_dir) if run_dir else None, + 'has_run_dir': bool(run_dir and run_dir.exists()), + 'has_run_spec': bool(run_dir and (run_dir / 'run_spec.json').exists()), + 'has_stats': bool(run_dir and (run_dir / 'stats.json').exists()), + 'has_per_instance_stats': bool(run_dir and (run_dir / 'per_instance_stats.json').exists()), + 'model_deployment': adapter_spec.get('model_deployment'), + 'metric_class_names': [m.get('class_name') for m in metric_specs if isinstance(m, dict)], + } + row.update(context_info) + return row + + +def _write_summary(rows: list[dict[str, Any]], out_fpath: Path) -> None: + benchmark_counts = Counter(row.get('benchmark') or 'unknown' for row in rows) + model_counts = Counter(row.get('model') or 'unknown' for row in rows) + host_counts = Counter(row.get('machine_host') or 'unknown' for row in rows) + status_counts = Counter(row.get('status') or 'unknown' for row in rows) + + lines = [] + lines.append('Audit Results Index Summary') + lines.append('') + lines.append(f'n_rows: {len(rows)}') + lines.append('') + lines.append('status_counts:') + for key, val in sorted(status_counts.items()): + lines.append(f' {key}: {val}') + lines.append('') + lines.append('machine_host_counts:') + for key, val in sorted(host_counts.items()): + lines.append(f' {key}: {val}') + lines.append('') + lines.append('benchmark_counts:') + for key, val in sorted(benchmark_counts.items()): + lines.append(f' {key}: {val}') + lines.append('') + lines.append('model_counts:') + for key, val in sorted(model_counts.items()): + lines.append(f' {key}: {val}') + out_fpath.write_text('\n'.join(lines) + '\n') + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('--results-root', default=env_defaults()['AUDIT_RESULTS_ROOT']) + parser.add_argument('--report-dpath', default=str(default_report_root() / 'indexes')) + parser.add_argument('--fallback-host', default=None) + args = parser.parse_args() + + results_root = Path(args.results_root).expanduser().resolve() + report_dpath = Path(args.report_dpath).expanduser().resolve() + report_dpath.mkdir(parents=True, exist_ok=True) + stamp = datetime_mod.datetime.now(datetime_mod.UTC).strftime('%Y%m%dT%H%M%SZ') + + rows = [] + for job_config_fpath in sorted(results_root.rglob('job_config.json')): + try: + rows.append(_row_for_job(job_config_fpath, args.fallback_host)) + except Exception as ex: + rows.append({ + 'job_dpath': str(job_config_fpath.parent), + 'status': 'index_error', + 'error': repr(ex), + 'machine_host': args.fallback_host, + 'provenance_source': 'fallback' if args.fallback_host else 'unknown', + }) + + jsonl_fpath = report_dpath / f'audit_results_index_{stamp}.jsonl' + csv_fpath = report_dpath / f'audit_results_index_{stamp}.csv' + summary_fpath = report_dpath / f'audit_results_index_{stamp}.txt' + + with jsonl_fpath.open('w') as file: + for row in rows: + file.write(json.dumps(kwutil.Json.ensure_serializable(row)) + '\n') + + table = pd.DataFrame(rows) + if not table.empty: + preferred = [ + 'experiment_name', 'job_id', 'status', 'benchmark', 'model', 'method', + 'max_eval_instances', 'machine_host', 'gpu_count', 'gpu_names', + 'cuda_visible_devices', 'provenance_source', 'run_dir', + ] + cols = [c for c in preferred if c in table.columns] + [c for c in table.columns if c not in preferred] + table = table[cols] + table.to_csv(csv_fpath, index=False) + _write_summary(rows, summary_fpath) + + print(f'Wrote jsonl index: {jsonl_fpath}') + print(f'Wrote csv index: {csv_fpath}') + print(f'Wrote summary: {summary_fpath}') + + +if __name__ == '__main__': + main() diff --git a/dev/experiments/audit-helm-reproduction/python/inspect_pair_samples.py b/dev/experiments/audit-helm-reproduction/python/inspect_pair_samples.py new file mode 100644 index 0000000..2336b23 --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/python/inspect_pair_samples.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import argparse +import datetime as datetime_mod +import os +from pathlib import Path + +from magnet.backends.helm.helm_outputs import HelmRun +from magnet.backends.helm.helm_run_diff import HelmRunDiff + + +def _safe_unlink(path: Path) -> None: + if path.exists() or path.is_symlink(): + path.unlink() + + +def _write_latest_alias(src: Path, latest_root: Path, latest_name: str) -> Path: + latest_fpath = latest_root / latest_name + _safe_unlink(latest_fpath) + rel_src = os.path.relpath(src, start=latest_fpath.parent) + os.symlink(rel_src, latest_fpath) + return latest_fpath + + +def _slugify(text: str) -> str: + return ( + str(text) + .replace('/', '-') + .replace(':', '-') + .replace(',', '-') + .replace('=', '-') + .replace('@', '-') + .replace(' ', '-') + ) + + +def _infer_run_spec_name(*run_paths: str) -> str: + names = [Path(p).name for p in run_paths if p] + names = [n for n in names if n] + if not names: + return 'unknown_run_spec' + unique = sorted(set(names)) + if len(unique) == 1: + return unique[0] + return unique[0] + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('--run-a', required=True) + parser.add_argument('--run-b', required=True) + parser.add_argument('--label', required=True) + parser.add_argument('--report-dpath', required=True) + parser.add_argument('--top-n', type=int, default=8) + parser.add_argument('--show-details', type=int, default=6) + parser.add_argument('--level', type=int, default=30) + args = parser.parse_args() + + report_dpath = Path(args.report_dpath).expanduser().resolve() + report_dpath.mkdir(parents=True, exist_ok=True) + stamp = datetime_mod.datetime.now(datetime_mod.UTC).strftime('%Y%m%dT%H%M%SZ') + history_dpath = report_dpath / '.history' / stamp[:8] + history_dpath.mkdir(parents=True, exist_ok=True) + + diff = HelmRunDiff( + HelmRun.coerce(args.run_a), + HelmRun.coerce(args.run_b), + a_name=f'{args.label}:A', + b_name=f'{args.label}:B', + ) + run_spec_name = _infer_run_spec_name(args.run_a, args.run_b) + lines: list[str] = [] + lines.append(f'Instance Sample Inspection') + lines.append(f'label: {args.label}') + lines.append(f'run_spec_name: {run_spec_name}') + lines.append(f'run_a: {Path(args.run_a).expanduser().resolve()}') + lines.append(f'run_b: {Path(args.run_b).expanduser().resolve()}') + lines.append('') + diff.summarize_instances( + level=args.level, + top_n=args.top_n, + show_details=args.show_details, + writer=lines.append, + ) + out_fpath = history_dpath / f'instance_samples_{_slugify(args.label)}_{stamp}.txt' + out_fpath.write_text('\n'.join(lines) + '\n') + latest_name = f'instance_samples_{_slugify(args.label)}.latest.txt' + latest_fpath = _write_latest_alias(out_fpath, report_dpath, latest_name) + print(f'Wrote instance sample report: {out_fpath}') + print(f'Updated latest link: {latest_fpath}') + + +if __name__ == '__main__': + main() diff --git a/dev/experiments/audit-helm-reproduction/python/rebuild_all_core_reports_from_index.py b/dev/experiments/audit-helm-reproduction/python/rebuild_all_core_reports_from_index.py new file mode 100644 index 0000000..5ca28c1 --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/python/rebuild_all_core_reports_from_index.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +import argparse +import subprocess +from collections import Counter +from pathlib import Path + +from common import audit_root, default_report_root, env_defaults +from rebuild_core_report_from_index import latest_index_csv, load_rows, matching_rows +from compare_batch import collect_historic_candidates, choose_historic_candidate + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('--index-fpath', default=None) + parser.add_argument('--index-dpath', default=str(default_report_root() / 'indexes')) + parser.add_argument('--report-root', default=str(default_report_root())) + parser.add_argument('--allow-single-repeat', action='store_true') + args = parser.parse_args() + + index_fpath = Path(args.index_fpath) if args.index_fpath else latest_index_csv(Path(args.index_dpath)) + rows = load_rows(index_fpath) + run_entries = sorted({row.get('run_entry') for row in rows if row.get('run_entry')}) + counts = Counter(row.get('run_entry') for row in rows if row.get('run_entry')) + + script = audit_root() / 'python' / 'rebuild_core_report_from_index.py' + built = 0 + skipped = [] + for run_entry in run_entries: + matches = matching_rows(rows, run_entry) + n = len(matches) + if n < 2 and not args.allow_single_repeat: + skipped.append((run_entry, n, 'not_enough_matching_runs')) + continue + desired_max = None + try: + desired_max = int(matches[0].get('max_eval_instances')) if matches and matches[0].get('max_eval_instances') else None + except Exception: + desired_max = None + historic_candidates = collect_historic_candidates(env_defaults()['HELM_PRECOMPUTED_ROOT'], run_entry) + chosen_historic, _info = choose_historic_candidate(historic_candidates, desired_max) + if chosen_historic is None: + skipped.append((run_entry, n, 'no_historic_match')) + continue + cmd = [ + env_defaults()['AIQ_PYTHON'], + str(script), + '--run-entry', str(run_entry), + '--index-fpath', str(index_fpath), + ] + if args.allow_single_repeat: + cmd.append('--allow-single-repeat') + subprocess.run(cmd, check=True) + built += 1 + + print(f'index_fpath={index_fpath}') + print(f'n_unique_run_entries={len(run_entries)}') + print(f'n_reports_built={built}') + print(f'n_skipped={len(skipped)}') + for run_entry, n, reason in skipped: + print(f'skipped run_entry={run_entry!r} matching_runs={n} reason={reason}') + + +if __name__ == '__main__': + main() diff --git a/dev/experiments/audit-helm-reproduction/python/rebuild_core_report_from_index.py b/dev/experiments/audit-helm-reproduction/python/rebuild_core_report_from_index.py new file mode 100644 index 0000000..85b07e1 --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/python/rebuild_core_report_from_index.py @@ -0,0 +1,229 @@ +from __future__ import annotations + +import argparse +import csv +import datetime as datetime_mod +import json +import os +import subprocess +from pathlib import Path +from typing import Any + +from common import audit_root, default_report_root, env_defaults +from compare_batch import collect_historic_candidates, choose_historic_candidate + + +def latest_index_csv(index_dpath: Path) -> Path: + cands = sorted(index_dpath.glob('audit_results_index_*.csv'), reverse=True) + if not cands: + raise FileNotFoundError(f'No index csv files found in {index_dpath}') + return cands[0] + + +def load_rows(index_fpath: Path) -> list[dict[str, Any]]: + with index_fpath.open() as file: + return list(csv.DictReader(file)) + + +def _coerce_float(x): + try: + return float(x) + except Exception: + return float('-inf') + + +def matching_rows(rows: list[dict[str, Any]], run_entry: str) -> list[dict[str, Any]]: + out = [] + for row in rows: + if row.get('run_entry') != run_entry: + continue + if row.get('status') not in {'computed', 'reused', 'unknown', ''}: + continue + if row.get('has_run_spec', '').lower() not in {'true', '1'}: + continue + run_dir = row.get('run_dir') + if not run_dir: + continue + out.append(row) + out.sort(key=lambda r: (_coerce_float(r.get('manifest_timestamp')), r.get('experiment_name', '')), reverse=True) + return out + + +def slugify(text: str) -> str: + return ( + text.replace('/', '-') + .replace(':', '-') + .replace(',', '-') + .replace('=', '-') + .replace('@', '-') + ) + + +def _write_latest_selection(report_dpath: Path, selection: dict[str, Any]) -> Path: + stamp = datetime_mod.datetime.now(datetime_mod.UTC).strftime('%Y%m%dT%H%M%SZ') + history_dpath = report_dpath / '.history' / stamp[:8] + history_dpath.mkdir(parents=True, exist_ok=True) + history_fpath = history_dpath / f'report_selection_{stamp}.json' + history_fpath.write_text(json.dumps(selection, indent=2)) + latest_fpath = report_dpath / 'report_selection.latest.json' + if latest_fpath.exists() or latest_fpath.is_symlink(): + latest_fpath.unlink() + rel_src = os.path.relpath(history_fpath, start=report_dpath) + os.symlink(rel_src, latest_fpath) + return latest_fpath + + +def _safe_unlink(path: Path) -> None: + if path.exists() or path.is_symlink(): + path.unlink() + + +def _symlink(target: str | os.PathLike[str], link_path: Path) -> Path: + target = Path(target).expanduser().resolve() + link_path.parent.mkdir(parents=True, exist_ok=True) + _safe_unlink(link_path) + rel_src = os.path.relpath(target, start=link_path.parent) + os.symlink(rel_src, link_path) + return link_path + + +def _find_kwdagger_job_dpath(run_dpath: str | os.PathLike[str]) -> Path | None: + current = Path(run_dpath).expanduser().resolve() + for cand in [current, *current.parents]: + if (cand / 'job_config.json').exists(): + return cand + return None + + +def _write_selected_run_symlinks(report_dpath: Path, selection: dict[str, Any]) -> dict[str, str]: + created = {} + run_targets = { + 'kwdagger_a.run': selection['left_run_a'], + 'kwdagger_b.run': selection['left_run_b'], + 'official.run': selection['right_run_a'], + } + for name, target in run_targets.items(): + created[name] = str(_symlink(target, report_dpath / name)) + + job_targets = { + 'kwdagger_a.job': _find_kwdagger_job_dpath(selection['left_run_a']), + 'kwdagger_b.job': _find_kwdagger_job_dpath(selection['left_run_b']), + } + for name, target in job_targets.items(): + if target is not None: + created[name] = str(_symlink(target, report_dpath / name)) + return created + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('--run-entry', required=True) + parser.add_argument('--index-fpath', default=None) + parser.add_argument('--index-dpath', default=str(default_report_root() / 'indexes')) + parser.add_argument('--precomputed-root', default=env_defaults()['HELM_PRECOMPUTED_ROOT']) + parser.add_argument('--report-dpath', default=None) + parser.add_argument('--left-label', default='kwdagger_repeat') + parser.add_argument('--right-label', default='official_vs_kwdagger') + parser.add_argument('--allow-single-repeat', action='store_true') + args = parser.parse_args() + + index_fpath = ( + Path(args.index_fpath).expanduser().resolve() + if args.index_fpath else + latest_index_csv(Path(args.index_dpath).expanduser().resolve()) + ) + rows = load_rows(index_fpath) + matches = matching_rows(rows, args.run_entry) + if not matches: + raise SystemExit(f'No indexed kwdagger runs found for run_entry={args.run_entry!r}') + + if len(matches) >= 2: + left_a = matches[0]['run_dir'] + left_b = matches[1]['run_dir'] + elif args.allow_single_repeat: + left_a = matches[0]['run_dir'] + left_b = matches[0]['run_dir'] + else: + raise SystemExit( + f'Need at least 2 matching kwdagger runs for run_entry={args.run_entry!r}; ' + f'found {len(matches)}. Use --allow-single-repeat to duplicate the latest run.' + ) + + desired_max = None + try: + desired_max = int(matches[0].get('max_eval_instances')) if matches[0].get('max_eval_instances') else None + except Exception: + desired_max = None + + historic_candidates = collect_historic_candidates(args.precomputed_root, args.run_entry) + chosen_historic, info = choose_historic_candidate(historic_candidates, desired_max) + if chosen_historic is None: + raise SystemExit(f'No historic HELM candidate found for run_entry={args.run_entry!r}') + + report_dpath = Path(args.report_dpath) if args.report_dpath else ( + default_report_root() / f'core-metrics-{slugify(args.run_entry)}' + ) + report_dpath = report_dpath.expanduser().resolve() + report_dpath.mkdir(parents=True, exist_ok=True) + + script = audit_root() / 'python' / 'core_metric_report.py' + cmd = [ + env_defaults()['AIQ_PYTHON'], + str(script), + '--left-run-a', str(left_a), + '--left-run-b', str(left_b), + '--left-label', args.left_label, + '--right-run-a', str(chosen_historic['run_dir']), + '--right-run-b', str(left_a), + '--right-label', args.right_label, + '--report-dpath', str(report_dpath), + ] + print(f'index_fpath={index_fpath}') + print(f'left_run_a={left_a}') + print(f'left_run_b={left_b}') + print(f'right_run_a={chosen_historic["run_dir"]}') + print(f'report_dpath={report_dpath}') + print(f'historic_info={info}') + selection = { + 'index_fpath': str(index_fpath), + 'run_entry': args.run_entry, + 'left_run_a': str(left_a), + 'left_run_b': str(left_b), + 'right_run_a': str(chosen_historic['run_dir']), + 'left_label': args.left_label, + 'right_label': args.right_label, + 'report_dpath': str(report_dpath), + 'historic_info': info, + } + selection_fpath = _write_latest_selection(report_dpath, selection) + link_info = _write_selected_run_symlinks(report_dpath, selection) + selection['selected_run_links'] = link_info + selection_fpath = _write_latest_selection(report_dpath, selection) + print(f'selection_fpath={selection_fpath}') + subprocess.run(cmd, check=True) + + inspect_script = audit_root() / 'python' / 'inspect_pair_samples.py' + sample_jobs = [ + [ + env_defaults()['AIQ_PYTHON'], + str(inspect_script), + '--run-a', str(left_a), + '--run-b', str(left_b), + '--label', args.left_label, + '--report-dpath', str(report_dpath), + ], + [ + env_defaults()['AIQ_PYTHON'], + str(inspect_script), + '--run-a', str(chosen_historic['run_dir']), + '--run-b', str(left_a), + '--label', args.right_label, + '--report-dpath', str(report_dpath), + ], + ] + for inspect_cmd in sample_jobs: + subprocess.run(inspect_cmd, check=True) + + +if __name__ == '__main__': + main() diff --git a/dev/experiments/audit-helm-reproduction/scripts/aggregate_core_reports.sh b/dev/experiments/audit-helm-reproduction/scripts/aggregate_core_reports.sh new file mode 100755 index 0000000..9511a7f --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/scripts/aggregate_core_reports.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail +set +x + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/common.sh" +audit::set_defaults + +AUDIT_PYTHON_DIR="${AUDIT_ROOT}/python" + +"$AIQ_PYTHON" "$AUDIT_PYTHON_DIR/aggregate_core_reports.py" "$@" diff --git a/dev/experiments/audit-helm-reproduction/scripts/index_results.sh b/dev/experiments/audit-helm-reproduction/scripts/index_results.sh new file mode 100755 index 0000000..5527963 --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/scripts/index_results.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail +set +x + +SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" +source "$SCRIPT_DIR/common.sh" +audit::set_defaults + +fallback_host="${AUDIT_FALLBACK_HOST:-}" +AUDIT_PYTHON_DIR="${AUDIT_ROOT}/python" +AUDIT_REPORT_ROOT="${AUDIT_ROOT}/reports" + +"$AIQ_PYTHON" "$AUDIT_PYTHON_DIR/index_results.py" \ + --results-root "${1:-$AUDIT_RESULTS_ROOT}" \ + --report-dpath "${2:-$AUDIT_REPORT_ROOT/indexes}" \ + ${fallback_host:+--fallback-host "$fallback_host"} diff --git a/dev/experiments/audit-helm-reproduction/scripts/inspect_pair_samples.sh b/dev/experiments/audit-helm-reproduction/scripts/inspect_pair_samples.sh new file mode 100755 index 0000000..8f18224 --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/scripts/inspect_pair_samples.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/common.sh" + +audit::set_defaults + +"$AIQ_PYTHON" "${AUDIT_ROOT}/python/inspect_pair_samples.py" "$@" diff --git a/dev/experiments/audit-helm-reproduction/scripts/rebuild_all_core_reports_from_index.sh b/dev/experiments/audit-helm-reproduction/scripts/rebuild_all_core_reports_from_index.sh new file mode 100755 index 0000000..d1a5a36 --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/scripts/rebuild_all_core_reports_from_index.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail +set +x + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/common.sh" +audit::set_defaults + +AUDIT_PYTHON_DIR="${AUDIT_ROOT}/python" + +"$AIQ_PYTHON" "$AUDIT_PYTHON_DIR/rebuild_all_core_reports_from_index.py" "$@" diff --git a/dev/experiments/audit-helm-reproduction/scripts/rebuild_core_report_from_index.sh b/dev/experiments/audit-helm-reproduction/scripts/rebuild_core_report_from_index.sh new file mode 100755 index 0000000..d9d7604 --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/scripts/rebuild_core_report_from_index.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail +set +x + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/common.sh" +audit::set_defaults + +AUDIT_PYTHON_DIR="${AUDIT_ROOT}/python" +AUDIT_REPORT_ROOT="${AUDIT_ROOT}/reports" + +"$AIQ_PYTHON" "$AUDIT_PYTHON_DIR/rebuild_core_report_from_index.py" "$@" diff --git a/magnet/backends/helm/cli/materialize_helm_run.py b/magnet/backends/helm/cli/materialize_helm_run.py index 7f17f6c..67ca75a 100755 --- a/magnet/backends/helm/cli/materialize_helm_run.py +++ b/magnet/backends/helm/cli/materialize_helm_run.py @@ -176,6 +176,81 @@ def _normalize_optional_pathish(value): return value +def _safe_config_dict(config) -> dict: + try: + return config.asdict() + except Exception: + try: + return dict(config) + except Exception: + return {} + + +def _query_nvidia_smi() -> dict | None: + """ + Best-effort GPU query for reproducibility metadata. + """ + try: + cmd = [ + 'nvidia-smi', + '--query-gpu=index,name,memory.total,driver_version', + '--format=csv,noheader,nounits', + ] + info = ub.cmd(cmd, verbose=0, system=False, check=False) + if info.returncode != 0: + return None + gpus = [] + for line in info.stdout.strip().splitlines(): + parts = [p.strip() for p in line.split(',')] + if len(parts) != 4: + continue + idx, name, mem_total, driver = parts + gpus.append({ + 'index': int(idx), + 'name': name, + 'memory_total_mb': int(mem_total), + 'driver_version': driver, + }) + return {'gpus': gpus} + except FileNotFoundError: + return None + except Exception as ex: + return {'error': repr(ex)} + + +def _capture_process_context(out_dpath: Path, config) -> dict: + from kwutil.process_context import ProcessContext + + process_context_fpath = out_dpath / 'process_context.json' + extra = { + 'env': { + 'CUDA_VISIBLE_DEVICES': os.environ.get('CUDA_VISIBLE_DEVICES'), + 'HOSTNAME': os.environ.get('HOSTNAME'), + } + } + ctx = ProcessContext( + name='materialize_helm_run', + config=_safe_config_dict(config), + extra=extra, + output_fpath=process_context_fpath, + ) + ctx.start() + ctx.stop() + try: + ctx.add_disk_info(out_dpath) + except Exception: + pass + gpu_info = _query_nvidia_smi() + if gpu_info: + ctx.properties.setdefault('extra', {}) + ctx.properties['extra']['nvidia_smi'] = gpu_info + try: + process_context_fpath.write_text(kwutil.Json.dumps(ctx.obj, indent=2)) + except Exception: + pass + return ctx.obj + + class MaterializeHelmRunConfig(scfg.DataConfig): """ Materialize HELM results either by computing them directly or pulling them @@ -401,6 +476,9 @@ def main(cls, argv=None, **kwargs) -> dict: 'out_dpath': str(out_dpath), 'timestamp': time.time(), } + process_context = _capture_process_context(out_dpath, config) + manifest['process_context_fpath'] = str(out_dpath / 'process_context.json') + manifest['process_context'] = process_context # 1) Try reuse match = None From 93cf8887fae643521453e6cd14d2cdefcd644b94 Mon Sep 17 00:00:00 2001 From: joncrall Date: Fri, 27 Mar 2026 20:35:31 -0400 Subject: [PATCH 25/36] wip --- .../helm-reproduction-research-journal.md | 94 +++++++++++++++++++ .../backends/helm/cli/materialize_helm_run.py | 59 ++++++++++++ 2 files changed, 153 insertions(+) diff --git a/dev/codex/helm-reproduction-research-journal.md b/dev/codex/helm-reproduction-research-journal.md index 1f5c095..095b5c7 100644 --- a/dev/codex/helm-reproduction-research-journal.md +++ b/dev/codex/helm-reproduction-research-journal.md @@ -314,3 +314,97 @@ Therefore: - it is not enough for a stable p-value estimate - Run a minimal Together-backed control on a representative case - this should help separate provider/deployment effects from general HELM evolution + +## Direct NarrativeQA/Vicuna Debug Run + +We ran a focused local debug job outside kwdagger scheduling: + +- suite: `debug-narrative-vicuna-direct` +- run entry: `narrative_qa:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical` +- max_eval_instances: `20` + +Main result: + +- The direct run reproduces the same failure mode as the larger local NarrativeQA/Vicuna runs. +- This strongly argues against the issue being a kwdagger scheduling/orchestration bug. + +Observed from the raw run: + +- request count: `100` +- empty completions: `99` +- non-empty completions: `1` +- mean output token count: `0.13` +- token count histogram: + - `0`: `99` + - `13`: `1` + +Observed from `stats.json`: + +- `num_completion_tokens` mean on `test`: `0.0` +- `num_output_tokens` mean on `test`: `0.0` +- `finish_reason_unknown` mean on `test`: `1.0` +- `exact_match`, `quasi_exact_match`, `f1_score`, `rouge_l`: all `0.0` + +Relevant log clues: + +- HELM automatically set `apply_chat_template=True` +- HELM removed 4 in-context examples to fit the context window +- HELM logged stop/truncation warnings: + - `truncate_sequence needs to strip "\\n"` + - `truncate_sequence needs to strip ""` + +Current interpretation: + +- Most likely this is a local HELM Hugging Face Vicuna execution/configuration issue. +- The strongest current suspect is chat-template application on a non-chat-style NarrativeQA prompt. +- Secondary suspects are newline stop-sequence handling and/or immediate EOS/empty-generation behavior in the HF Vicuna path. + +Current follow-up experiment: + +- rerun the same benchmark with a custom `model_deployments.yaml` override that sets: + - `apply_chat_template: false` + +## NarrativeQA/Vicuna Root Cause Update + +The `apply_chat_template: false` rerun strongly supports a root-cause diagnosis. + +Run: + +- suite: `debug-narrative-vicuna-nochat` +- same benchmark/model family as the failing debug run +- same local Hugging Face deployment name +- overridden deployment config: + - `client_spec.args.apply_chat_template: false` + +Observed: + +- request count: `500` +- empty completions: `0` +- non-empty completions: `500` +- mean output token count: `12.894` + +Aggregate test metrics from the corrected local run: + +- `exact_match`: `0.2727` +- `quasi_exact_match`: `0.4026` +- `f1_score`: `0.6422` +- `rouge_l`: `0.6442` +- `bleu_1`: `0.5138` +- `bleu_4`: `0.0722` + +These are now close to the official public HELM run for the same benchmark/model pair. + +Conclusion: + +- The prior NarrativeQA/Vicuna failure was **not** good evidence of irreproducibility. +- It was caused by a local HELM/HuggingFace configuration issue. +- The main culprit appears to be automatic chat-template application for this run. + +Implications for the audit: + +- For local Hugging Face reproductions, `apply_chat_template` must be treated as an explicit controlled setting. +- Some earlier "failed reproductions" may need to be reinterpreted or rerun if they depended on HELM's automatic chat-template inference. +- The audit/reporting system should surface suspicious signals such as: + - high empty-completion rate + - near-zero `num_output_tokens` + - pervasive `finish_reason_unknown` diff --git a/magnet/backends/helm/cli/materialize_helm_run.py b/magnet/backends/helm/cli/materialize_helm_run.py index 67ca75a..c7c252c 100755 --- a/magnet/backends/helm/cli/materialize_helm_run.py +++ b/magnet/backends/helm/cli/materialize_helm_run.py @@ -1194,6 +1194,63 @@ def prepare_local_helm_config( return local_path_abs +def write_helm_log_config(out_dpath: Path) -> Path: + """ + Write a HELM logging config file into ``out_dpath``. + + This keeps the ``helm-run`` console logs and file logs colocated with the + kwdagger node outputs so they survive rsync / artifact transfer. + """ + log_fpath = out_dpath / 'helm-run.log' + debug_log_fpath = out_dpath / 'helm-run.debug.log' + config = { + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'console': { + 'datefmt': '%Y-%m-%dT%H:%M:%S', + 'format': '%(asctime)s %(levelname)-8s %(message)s', + }, + 'detailed': { + 'datefmt': '%Y-%m-%dT%H:%M:%S', + 'format': '%(asctime)s %(levelname)-8s %(name)s %(message)s', + }, + }, + 'handlers': { + 'stdout': { + 'class': 'logging.StreamHandler', + 'stream': 'ext://sys.stdout', + 'formatter': 'console', + 'level': 'INFO', + }, + 'file_info': { + 'class': 'logging.FileHandler', + 'filename': os.fspath(log_fpath), + 'formatter': 'detailed', + 'level': 'INFO', + 'mode': 'w', + }, + 'file_debug': { + 'class': 'logging.FileHandler', + 'filename': os.fspath(debug_log_fpath), + 'formatter': 'detailed', + 'level': 'DEBUG', + 'mode': 'w', + }, + }, + 'loggers': { + 'helm': { + 'handlers': ['stdout', 'file_info', 'file_debug'], + 'level': 'DEBUG', + 'propagate': False, + } + }, + } + config_fpath = out_dpath / 'helm_log_config.yaml' + config_fpath.write_text(kwutil.Yaml.dumps(config)) + return config_fpath + + def run_helm( requested_desc: str, suite: str, @@ -1218,6 +1275,8 @@ def run_helm( suite, '--local-path', os.fspath(local_path), + '--log-config', + os.fspath(write_helm_log_config(out_dpath)), ] if max_eval_instances is not None: cmd += ['--max-eval-instances', str(max_eval_instances)] From fa9ef26ee707f21172bf541dd8c335d85a767f91 Mon Sep 17 00:00:00 2001 From: joncrall Date: Fri, 27 Mar 2026 20:43:39 -0400 Subject: [PATCH 26/36] wip --- .../python/make_manifest.py | 44 ++++++++++++++++++- .../scripts/make_vicuna_nochat_manifest.sh | 19 ++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) create mode 100755 dev/experiments/audit-helm-reproduction/scripts/make_vicuna_nochat_manifest.sh diff --git a/dev/experiments/audit-helm-reproduction/python/make_manifest.py b/dev/experiments/audit-helm-reproduction/python/make_manifest.py index 50b81f5..64fee0f 100644 --- a/dev/experiments/audit-helm-reproduction/python/make_manifest.py +++ b/dev/experiments/audit-helm-reproduction/python/make_manifest.py @@ -18,6 +18,12 @@ "narrative_qa:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical", ] +VICUNA_NOCHAT_RUN_ENTRIES = [ + "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical", + "boolq:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical", + "narrative_qa:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical", +] + def _validate_entries_exist(run_entries: list[str]) -> list[str]: fpath = repo_run_specs_fpath() @@ -146,10 +152,44 @@ def build_single_manifest(args: argparse.Namespace) -> dict: ) +def build_vicuna_nochat_manifest(args: argparse.Namespace) -> dict: + defaults = env_defaults() + max_eval_instances = ( + args.max_eval_instances + if args.max_eval_instances is not None + else 1000 + ) + tmux_workers = ( + args.tmux_workers + if args.tmux_workers is not None + else 1 + ) + devices = args.devices if args.devices is not None else "0" + manifest = _build_manifest( + experiment_name=args.experiment_name, + description=( + "Overnight Vicuna batch with chat templating explicitly disabled " + "via a model_deployments override." + ), + run_entries=VICUNA_NOCHAT_RUN_ENTRIES, + max_eval_instances=max_eval_instances, + suite=args.suite, + tmux_workers=tmux_workers, + devices=devices, + ) + manifest["model_deployments_fpath"] = ( + "dev/experiments/audit-helm-reproduction/configs/debug/" + "vicuna_no_chat_template.yaml" + ) + return manifest + + def main() -> None: parser = argparse.ArgumentParser() parser.add_argument( - "--manifest-type", default="smoke", choices=["smoke", "apples", "single"] + "--manifest-type", + default="smoke", + choices=["smoke", "apples", "single", "vicuna_nochat"], ) parser.add_argument("--output", required=True) parser.add_argument("--experiment-name", default="audit-smoke") @@ -167,6 +207,8 @@ def main() -> None: manifest = build_apples_manifest(args) elif args.manifest_type == "single": manifest = build_single_manifest(args) + elif args.manifest_type == "vicuna_nochat": + manifest = build_vicuna_nochat_manifest(args) else: raise NotImplementedError(args.manifest_type) out_fpath = Path(args.output) diff --git a/dev/experiments/audit-helm-reproduction/scripts/make_vicuna_nochat_manifest.sh b/dev/experiments/audit-helm-reproduction/scripts/make_vicuna_nochat_manifest.sh new file mode 100755 index 0000000..f84643e --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/scripts/make_vicuna_nochat_manifest.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/common.sh" + +audit::set_defaults + +OUTPATH="${1:-${AUDIT_ROOT}/configs/generated/vicuna_nochat_overnight.generated.yaml}" +shift || true + +"$AIQ_PYTHON" "${AUDIT_ROOT}/python/make_manifest.py" \ + --manifest-type vicuna_nochat \ + --output "$OUTPATH" \ + --experiment-name audit-vicuna-nochat-overnight \ + --suite audit-vicuna-nochat-overnight \ + "$@" + +printf 'Wrote Vicuna no-chat manifest: %s\n' "$OUTPATH" From 55bdb38a77509732ef5b4513bb681d0b09455bad Mon Sep 17 00:00:00 2001 From: agent Date: Sat, 28 Mar 2026 04:08:13 +0000 Subject: [PATCH 27/36] wip --- .../audit-helm-reproduction/README.md | 22 +++ .../examples/example_overnight_commands.sh | 24 +++ .../python/analyze_experiment_from_index.py | 137 ++++++++++++++++++ .../scripts/analyze_experiment_from_index.sh | 9 ++ 4 files changed, 192 insertions(+) create mode 100755 dev/experiments/audit-helm-reproduction/examples/example_overnight_commands.sh create mode 100644 dev/experiments/audit-helm-reproduction/python/analyze_experiment_from_index.py create mode 100755 dev/experiments/audit-helm-reproduction/scripts/analyze_experiment_from_index.sh diff --git a/dev/experiments/audit-helm-reproduction/README.md b/dev/experiments/audit-helm-reproduction/README.md index 8dd7ea3..465aae1 100644 --- a/dev/experiments/audit-helm-reproduction/README.md +++ b/dev/experiments/audit-helm-reproduction/README.md @@ -243,6 +243,28 @@ bash dev/experiments/audit-helm-reproduction/scripts/inspect_pair_samples.sh \ --report-dpath dev/experiments/audit-helm-reproduction/reports/manual-inspection ``` +For a newly synced kwdagger experiment, you can rebuild only the reports +relevant to that experiment and write a focused summary with: + +```bash +AUDIT_FALLBACK_HOST=aiq-gpu \ +bash dev/experiments/audit-helm-reproduction/scripts/index_results.sh + +bash dev/experiments/audit-helm-reproduction/scripts/analyze_experiment_from_index.sh \ + --experiment-name audit-vicuna-nochat-overnight \ + --allow-single-repeat +``` + +This writes a compact experiment-level summary under: + +- `reports/experiment-analysis-/` + +Note: + +- this experiment analyzer uses the kwdagger results index +- it is intended for indexed kwdagger experiments +- direct one-off debug runs that were not scheduled as kwdagger jobs will not appear there + ## Reproducibility Checklist For any experiment you want to cite later, preserve all of the following: diff --git a/dev/experiments/audit-helm-reproduction/examples/example_overnight_commands.sh b/dev/experiments/audit-helm-reproduction/examples/example_overnight_commands.sh new file mode 100755 index 0000000..a3aefad --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/examples/example_overnight_commands.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Example overnight control on a richer generation-style task. + +dev/experiments/audit-helm-reproduction/scripts/make_repeat_pair_manifests.sh \ + 'narrative_qa:model=eleutherai/pythia-6.9b,data_augmentation=canonical' \ + audit-narrative-pythia \ + dev/experiments/audit-helm-reproduction/configs/generated \ + --max-eval-instances 1000 \ + --devices 0 \ + --tmux-workers 1 + +dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh \ + dev/experiments/audit-helm-reproduction/configs/generated/audit-narrative-pythia_r1.yaml + +dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh \ + dev/experiments/audit-helm-reproduction/configs/generated/audit-narrative-pythia_r2.yaml + +dev/experiments/audit-helm-reproduction/scripts/compare_batch.sh \ + dev/experiments/audit-helm-reproduction/configs/generated/audit-narrative-pythia_r1.yaml + +dev/experiments/audit-helm-reproduction/scripts/compare_batch.sh \ + dev/experiments/audit-helm-reproduction/configs/generated/audit-narrative-pythia_r2.yaml diff --git a/dev/experiments/audit-helm-reproduction/python/analyze_experiment_from_index.py b/dev/experiments/audit-helm-reproduction/python/analyze_experiment_from_index.py new file mode 100644 index 0000000..f953e42 --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/python/analyze_experiment_from_index.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import argparse +import datetime as datetime_mod +import json +import os +import subprocess +from pathlib import Path +from typing import Any + +import pandas as pd + +from aggregate_core_reports import _find_curve_value, _find_pair, _write_latest_alias +from common import audit_root, default_report_root, env_defaults +from rebuild_core_report_from_index import latest_index_csv, load_rows, slugify + + +def _report_dir_for_run_entry(run_entry: str) -> Path: + return default_report_root() / f'core-metrics-{slugify(run_entry)}' + + +def _load_json(fpath: Path) -> dict[str, Any]: + return json.loads(fpath.read_text()) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('--experiment-name', required=True) + parser.add_argument('--index-fpath', default=None) + parser.add_argument('--index-dpath', default=str(default_report_root() / 'indexes')) + parser.add_argument('--allow-single-repeat', action='store_true') + args = parser.parse_args() + + index_fpath = ( + Path(args.index_fpath).expanduser().resolve() + if args.index_fpath else + latest_index_csv(Path(args.index_dpath).expanduser().resolve()) + ) + rows = load_rows(index_fpath) + experiment_rows = [r for r in rows if r.get('experiment_name') == args.experiment_name] + if not experiment_rows: + raise SystemExit(f'No rows found for experiment_name={args.experiment_name!r}') + run_entries = sorted({r.get('run_entry') for r in experiment_rows if r.get('run_entry')}) + + rebuild_script = audit_root() / 'python' / 'rebuild_core_report_from_index.py' + built_report_paths = [] + for run_entry in run_entries: + cmd = [ + env_defaults()['AIQ_PYTHON'], + str(rebuild_script), + '--run-entry', str(run_entry), + '--index-fpath', str(index_fpath), + ] + if args.allow_single_repeat: + cmd.append('--allow-single-repeat') + subprocess.run(cmd, check=True) + built_report_paths.append(_report_dir_for_run_entry(run_entry) / 'core_metric_report.latest.json') + + summary_rows = [] + for report_json in built_report_paths: + if not report_json.exists(): + continue + report = _load_json(report_json) + repeat = _find_pair(report, 'kwdagger_repeat') + official = _find_pair(report, 'official_vs_kwdagger') + summary_rows.append({ + 'experiment_name': args.experiment_name, + 'run_spec_name': report.get('run_spec_name'), + 'report_dir': str(report_json.parent), + 'generated_utc': report.get('generated_utc'), + 'repeat_instance_agree_0': _find_curve_value(repeat.get('instance_level', {}).get('agreement_vs_abs_tol', []), 0.0), + 'official_instance_agree_0': _find_curve_value(official.get('instance_level', {}).get('agreement_vs_abs_tol', []), 0.0), + 'official_instance_agree_01': _find_curve_value(official.get('instance_level', {}).get('agreement_vs_abs_tol', []), 0.1), + 'official_instance_agree_025': _find_curve_value(official.get('instance_level', {}).get('agreement_vs_abs_tol', []), 0.25), + 'official_instance_agree_05': _find_curve_value(official.get('instance_level', {}).get('agreement_vs_abs_tol', []), 0.5), + 'official_runlevel_p90': (((official.get('run_level') or {}).get('overall_quantiles') or {}).get('abs_delta') or {}).get('p90'), + 'official_runlevel_max': (((official.get('run_level') or {}).get('overall_quantiles') or {}).get('abs_delta') or {}).get('max'), + }) + + out_dpath = default_report_root() / f'experiment-analysis-{slugify(args.experiment_name)}' + out_dpath.mkdir(parents=True, exist_ok=True) + stamp = datetime_mod.datetime.now(datetime_mod.UTC).strftime('%Y%m%dT%H%M%SZ') + history_dpath = out_dpath / '.history' / stamp[:8] + history_dpath.mkdir(parents=True, exist_ok=True) + + table = pd.DataFrame(summary_rows).sort_values('run_spec_name') + json_fpath = history_dpath / f'experiment_summary_{stamp}.json' + csv_fpath = history_dpath / f'experiment_summary_{stamp}.csv' + txt_fpath = history_dpath / f'experiment_summary_{stamp}.txt' + + payload = { + 'generated_utc': stamp, + 'experiment_name': args.experiment_name, + 'index_fpath': str(index_fpath), + 'n_run_entries': len(run_entries), + 'run_entries': run_entries, + 'rows': summary_rows, + } + json_fpath.write_text(json.dumps(payload, indent=2)) + table.to_csv(csv_fpath, index=False) + + lines = [] + lines.append('Experiment Analysis Summary') + lines.append('') + lines.append(f'generated_utc: {stamp}') + lines.append(f'experiment_name: {args.experiment_name}') + lines.append(f'index_fpath: {index_fpath}') + lines.append(f'n_run_entries: {len(run_entries)}') + lines.append('') + lines.append('run_entries:') + for run_entry in run_entries: + lines.append(f' - {run_entry}') + lines.append('') + lines.append('per_run_spec:') + for row in summary_rows: + lines.append(f" - run_spec_name: {row['run_spec_name']}") + lines.append(f" report_dir: {row['report_dir']}") + lines.append(f" repeat_instance_agree_0: {row['repeat_instance_agree_0']}") + lines.append(f" official_instance_agree_0: {row['official_instance_agree_0']}") + lines.append(f" official_instance_agree_01: {row['official_instance_agree_01']}") + lines.append(f" official_instance_agree_025: {row['official_instance_agree_025']}") + lines.append(f" official_instance_agree_05: {row['official_instance_agree_05']}") + lines.append(f" official_runlevel_p90: {row['official_runlevel_p90']}") + lines.append(f" official_runlevel_max: {row['official_runlevel_max']}") + txt_fpath.write_text('\n'.join(lines) + '\n') + + _write_latest_alias(json_fpath, out_dpath, 'experiment_summary.latest.json') + _write_latest_alias(csv_fpath, out_dpath, 'experiment_summary.latest.csv') + _write_latest_alias(txt_fpath, out_dpath, 'experiment_summary.latest.txt') + + print(f'Wrote experiment summary json: {json_fpath}') + print(f'Wrote experiment summary csv: {csv_fpath}') + print(f'Wrote experiment summary txt: {txt_fpath}') + + +if __name__ == '__main__': + main() diff --git a/dev/experiments/audit-helm-reproduction/scripts/analyze_experiment_from_index.sh b/dev/experiments/audit-helm-reproduction/scripts/analyze_experiment_from_index.sh new file mode 100755 index 0000000..0175e2a --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/scripts/analyze_experiment_from_index.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail +set +x + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/common.sh" +audit::set_defaults + +"$AIQ_PYTHON" "${AUDIT_ROOT}/python/analyze_experiment_from_index.py" "$@" From 8b0376fb429f13bf9a3041da549c3e86202d321c Mon Sep 17 00:00:00 2001 From: agent Date: Sat, 28 Mar 2026 20:12:11 +0000 Subject: [PATCH 28/36] wip --- .../helm-reproduction-research-journal.md | 76 ++++++++ .../audit-helm-reproduction/README.md | 171 ++++++++++++++++++ .../python/aggregate_core_reports.py | 10 + .../python/analyze_experiment_from_index.py | 26 ++- .../audit-helm-reproduction/python/common.py | 5 +- .../python/core_metric_report.py | 118 ++++++++++++ .../python/rebuild_core_report_from_index.py | 17 +- dev/poc/inspect_historic_helm_runs.py | 2 +- 8 files changed, 414 insertions(+), 11 deletions(-) diff --git a/dev/codex/helm-reproduction-research-journal.md b/dev/codex/helm-reproduction-research-journal.md index 095b5c7..ebaa97b 100644 --- a/dev/codex/helm-reproduction-research-journal.md +++ b/dev/codex/helm-reproduction-research-journal.md @@ -408,3 +408,79 @@ Implications for the audit: - high empty-completion rate - near-zero `num_output_tokens` - pervasive `finish_reason_unknown` + +## Server Batch Logging Validation + +The latest rsynced server-side experiment artifacts confirmed that the new per-job HELM logging capture is working as intended. + +Observed in: + +- `/data/crfm-helm-audit/audit-vicuna-nochat-server/helm/helm_id_dijo03bfux6g/` +- `/data/crfm-helm-audit/audit-vicuna-nochat-server/helm/helm_id_obr7gu9kxuql/` +- `/data/crfm-helm-audit/audit-vicuna-nochat-server/helm/helm_id_s2ez33ko97jb/` + +Each failed job now preserves: + +- `helm-run.log` +- `helm-run.debug.log` +- `process_context.json` +- `helm_log_config.yaml` +- `job_config.json` + +These logs show that the latest server failures were not new reproducibility failures. All three jobs failed during local Hugging Face model load with the same infrastructure error: + +- `torch.OutOfMemoryError` +- attempted allocation: about `172 MiB` +- GPU `0` free memory at failure: about `143.50 MiB` +- competing process `696468` already using about `89.43 GiB` + +Interpretation: + +- the `audit-vicuna-nochat-server` run failed due to GPU occupancy / infrastructure contention +- this does **not** currently count as evidence for or against HELM reproducibility +- the new logging path is useful and should remain part of all future materialized job outputs + +## Cross-Machine Vicuna No-Chat Confirmation + +After clearing the server-side GPU occupancy issue, the `audit-vicuna-nochat-server` batch completed successfully and matched the earlier `audit-vicuna-nochat-overnight` batch. + +High-level result: + +- for the three fixed-config Vicuna runs we tested, the server results were numerically identical to the earlier run on the other machine at the run-level metric summaries we inspected +- this is a meaningful positive signal for cross-machine reproducibility once the chat-template confounder is removed + +Benchmarks: + +- `boolq:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical` +- `mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical` +- `narrative_qa:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical` + +Observed diagnostics from the successful server run: + +- BoolQ: + - `N = 5000` + - empty completions: `0` + - mean output tokens: `1.0064` +- MMLU: + - `N = 327` + - empty completions: `0` + - mean output tokens: `1.0` +- NarrativeQA: + - `N = 2350` + - empty completions: `0` + - mean output tokens: `11.9498` + - output token quantiles: + - `p50 = 6` + - `p90 = 26` + - `max = 100` + +Comparison to official public HELM: + +- BoolQ correctness metrics match exactly; the remaining visible drift is shorter output length +- MMLU core metrics match exactly +- NarrativeQA remains very close to official on core metrics, with the main residual difference in answer length rather than answer quality + +Interpretation: + +- the fixed Vicuna/HuggingFace path appears independently reproducible on the tested core metrics +- at least for these experiments, the newer server run supports a positive cross-machine reproducibility result rather than revealing a hardware-specific failure mode diff --git a/dev/experiments/audit-helm-reproduction/README.md b/dev/experiments/audit-helm-reproduction/README.md index 465aae1..bc3eb69 100644 --- a/dev/experiments/audit-helm-reproduction/README.md +++ b/dev/experiments/audit-helm-reproduction/README.md @@ -259,6 +259,177 @@ This writes a compact experiment-level summary under: - `reports/experiment-analysis-/` +## Building Larger Historic Reproduction Batches + +The repo-root files: + +- `run_specs.yaml` +- `run_details.yaml` + +are the curated historic candidate list produced from: + +- `dev/poc/inspect_historic_helm_runs.py` + +You can regenerate them with: + +```bash +python dev/poc/inspect_historic_helm_runs.py \ + /data/crfm-helm-public \ + --out_fpath run_specs.yaml \ + --out_detail_fpath run_details.yaml +``` + +To build a larger refreshed kwdagger manifest from those historic candidates: + +```bash +bash dev/experiments/audit-helm-reproduction/scripts/make_historic_grid_manifest.sh \ + dev/experiments/audit-helm-reproduction/configs/generated/historic_grid.generated.yaml \ + --experiment-name audit-historic-grid \ + --suite audit-historic-grid \ + --devices 0,1 \ + --tmux-workers 2 \ + --max-eval-instances 1000 +``` + +This also writes a sidecar selection file: + +- `.../historic_grid.generated.yaml.selection.yaml` + +The selection sidecar records: + +- exactly which `run_entry` values were selected +- machine/shard settings +- any model override file that was applied +- matching metadata from `run_details.yaml` when available + +### Filtering The Historic Grid + +The larger builder supports shell-style pattern filtering and model/benchmark +selection. For example, a Vicuna-only slice over three benchmarks: + +```bash +bash dev/experiments/audit-helm-reproduction/scripts/make_historic_grid_manifest.sh \ + dev/experiments/audit-helm-reproduction/configs/generated/historic_vicuna_focus.generated.yaml \ + --experiment-name audit-historic-vicuna-focus \ + --suite audit-historic-vicuna-focus \ + --model lmsys/vicuna-7b-v1.3 \ + --include-pattern 'boolq:*' \ + --include-pattern 'mmlu:*' \ + --include-pattern 'narrative_qa:*' \ + --devices 0 \ + --tmux-workers 1 \ + --max-eval-instances 1000 +``` + +If any selected entries use `lmsys/vicuna-7b-v1.3`, the builder automatically +applies: + +- `dev/experiments/audit-helm-reproduction/configs/debug/vicuna_no_chat_template.yaml` + +so the fixed no-chat-template configuration is used. + +### Deterministic One-GPU Shards For Multiple Machines + +To split the same filtered candidate set deterministically across multiple +machines, use the shard builder. This is useful for `namek` and `yardrat`, +where only one GPU is available. + +Example: build 2 shards from the same filtered historic set. + +For `namek`: + +```bash +bash dev/experiments/audit-helm-reproduction/scripts/make_machine_shard_manifest.sh \ + namek \ + 0 \ + 2 \ + dev/experiments/audit-helm-reproduction/configs/generated/namek.generated.yaml \ + --model lmsys/vicuna-7b-v1.3 \ + --include-pattern 'boolq:*' \ + --include-pattern 'mmlu:*' \ + --include-pattern 'narrative_qa:*' \ + --max-eval-instances 1000 +``` + +For `yardrat`: + +```bash +bash dev/experiments/audit-helm-reproduction/scripts/make_machine_shard_manifest.sh \ + yardrat \ + 1 \ + 2 \ + dev/experiments/audit-helm-reproduction/configs/generated/yardrat.generated.yaml \ + --model lmsys/vicuna-7b-v1.3 \ + --include-pattern 'boolq:*' \ + --include-pattern 'mmlu:*' \ + --include-pattern 'narrative_qa:*' \ + --max-eval-instances 1000 +``` + +These manifests default to: + +- `devices: 0` +- `tmux_workers: 1` + +so they are safe for single-GPU machines unless explicitly overridden. + +### Same Subset On Multiple Machines For Hardware Comparison + +If the goal is to compare reproducibility across different hardware, both +machines should run the same subset rather than different shards. + +For `namek`: + +```bash +bash dev/experiments/audit-helm-reproduction/scripts/make_machine_subset_manifest.sh \ + namek \ + dev/experiments/audit-helm-reproduction/configs/generated/namek.subset.generated.yaml \ + --model lmsys/vicuna-7b-v1.3 \ + --include-pattern 'boolq:*' \ + --include-pattern 'mmlu:*us_foreign_policy*' \ + --include-pattern 'narrative_qa:*' \ + --max-eval-instances 1000 +``` + +For `yardrat`, use the same filters: + +```bash +bash dev/experiments/audit-helm-reproduction/scripts/make_machine_subset_manifest.sh \ + yardrat \ + dev/experiments/audit-helm-reproduction/configs/generated/yardrat.subset.generated.yaml \ + --model lmsys/vicuna-7b-v1.3 \ + --include-pattern 'boolq:*' \ + --include-pattern 'mmlu:*us_foreign_policy*' \ + --include-pattern 'narrative_qa:*' \ + --max-eval-instances 1000 +``` + +These produce two manifests with the same selected `run_entry` set but distinct +`experiment_name` / `suite` values, which makes later indexing and +cross-machine analysis easier. + +### Running The Larger Batch + +Once a manifest has been generated, launch it the same way as the smaller +batches: + +```bash +bash dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh \ + dev/experiments/audit-helm-reproduction/configs/generated/historic_grid.generated.yaml +``` + +After raw results are synced back, rebuild the index and analyze the specific +experiment: + +```bash +AUDIT_FALLBACK_HOST=aiq-gpu \ +bash dev/experiments/audit-helm-reproduction/scripts/index_results.sh + +bash dev/experiments/audit-helm-reproduction/scripts/analyze_experiment_from_index.sh \ + --experiment-name audit-historic-grid \ + --allow-single-repeat +``` + Note: - this experiment analyzer uses the kwdagger results index diff --git a/dev/experiments/audit-helm-reproduction/python/aggregate_core_reports.py b/dev/experiments/audit-helm-reproduction/python/aggregate_core_reports.py index 347d7ec..8ef8666 100644 --- a/dev/experiments/audit-helm-reproduction/python/aggregate_core_reports.py +++ b/dev/experiments/audit-helm-reproduction/python/aggregate_core_reports.py @@ -94,6 +94,11 @@ def main() -> None: 'run_spec_name': report.get('run_spec_name'), 'generated_utc': report.get('generated_utc'), 'n_core_metrics': len((_find_pair(report, 'kwdagger_repeat').get('core_metrics') or [])), + 'diagnostic_flags': report.get('diagnostic_flags', []), + 'kwdagger_a_empty_completion_rate': (((report.get('run_diagnostics') or {}).get('kwdagger_a') or {}).get('empty_completion_rate')), + 'kwdagger_a_mean_output_tokens': ((((report.get('run_diagnostics') or {}).get('kwdagger_a') or {}).get('output_token_count') or {}).get('mean')), + 'official_empty_completion_rate': (((report.get('run_diagnostics') or {}).get('official') or {}).get('empty_completion_rate')), + 'official_mean_output_tokens': ((((report.get('run_diagnostics') or {}).get('official') or {}).get('output_token_count') or {}).get('mean')), 'repeat_instance_agree_0': repeat_agree_0, 'official_instance_agree_0': official_agree_0, 'official_instance_agree_01': official_agree_01, @@ -138,6 +143,11 @@ def main() -> None: for row in rows: lines.append(f" - run_spec_name: {row['run_spec_name']}") lines.append(f" assessment_label: {row['assessment_label']}") + lines.append(f" diagnostic_flags: {row['diagnostic_flags']}") + lines.append(f" kwdagger_a_empty_completion_rate: {row['kwdagger_a_empty_completion_rate']}") + lines.append(f" kwdagger_a_mean_output_tokens: {row['kwdagger_a_mean_output_tokens']}") + lines.append(f" official_empty_completion_rate: {row['official_empty_completion_rate']}") + lines.append(f" official_mean_output_tokens: {row['official_mean_output_tokens']}") lines.append(f" repeat_instance_agree_0: {row['repeat_instance_agree_0']}") lines.append(f" official_instance_agree_0: {row['official_instance_agree_0']}") lines.append(f" official_instance_agree_01: {row['official_instance_agree_01']}") diff --git a/dev/experiments/audit-helm-reproduction/python/analyze_experiment_from_index.py b/dev/experiments/audit-helm-reproduction/python/analyze_experiment_from_index.py index f953e42..2b7e787 100644 --- a/dev/experiments/audit-helm-reproduction/python/analyze_experiment_from_index.py +++ b/dev/experiments/audit-helm-reproduction/python/analyze_experiment_from_index.py @@ -15,10 +15,6 @@ from rebuild_core_report_from_index import latest_index_csv, load_rows, slugify -def _report_dir_for_run_entry(run_entry: str) -> Path: - return default_report_root() / f'core-metrics-{slugify(run_entry)}' - - def _load_json(fpath: Path) -> dict[str, Any]: return json.loads(fpath.read_text()) @@ -42,19 +38,27 @@ def main() -> None: raise SystemExit(f'No rows found for experiment_name={args.experiment_name!r}') run_entries = sorted({r.get('run_entry') for r in experiment_rows if r.get('run_entry')}) + out_dpath = default_report_root() / f'experiment-analysis-{slugify(args.experiment_name)}' + out_dpath.mkdir(parents=True, exist_ok=True) + reports_dpath = out_dpath / 'core-reports' + reports_dpath.mkdir(parents=True, exist_ok=True) + rebuild_script = audit_root() / 'python' / 'rebuild_core_report_from_index.py' built_report_paths = [] for run_entry in run_entries: + report_dpath = reports_dpath / f'core-metrics-{slugify(run_entry)}' cmd = [ env_defaults()['AIQ_PYTHON'], str(rebuild_script), '--run-entry', str(run_entry), '--index-fpath', str(index_fpath), + '--experiment-name', str(args.experiment_name), + '--report-dpath', str(report_dpath), ] if args.allow_single_repeat: cmd.append('--allow-single-repeat') subprocess.run(cmd, check=True) - built_report_paths.append(_report_dir_for_run_entry(run_entry) / 'core_metric_report.latest.json') + built_report_paths.append(report_dpath / 'core_metric_report.latest.json') summary_rows = [] for report_json in built_report_paths: @@ -68,6 +72,11 @@ def main() -> None: 'run_spec_name': report.get('run_spec_name'), 'report_dir': str(report_json.parent), 'generated_utc': report.get('generated_utc'), + 'diagnostic_flags': report.get('diagnostic_flags', []), + 'kwdagger_a_empty_completion_rate': (((report.get('run_diagnostics') or {}).get('kwdagger_a') or {}).get('empty_completion_rate')), + 'kwdagger_a_mean_output_tokens': ((((report.get('run_diagnostics') or {}).get('kwdagger_a') or {}).get('output_token_count') or {}).get('mean')), + 'official_empty_completion_rate': (((report.get('run_diagnostics') or {}).get('official') or {}).get('empty_completion_rate')), + 'official_mean_output_tokens': ((((report.get('run_diagnostics') or {}).get('official') or {}).get('output_token_count') or {}).get('mean')), 'repeat_instance_agree_0': _find_curve_value(repeat.get('instance_level', {}).get('agreement_vs_abs_tol', []), 0.0), 'official_instance_agree_0': _find_curve_value(official.get('instance_level', {}).get('agreement_vs_abs_tol', []), 0.0), 'official_instance_agree_01': _find_curve_value(official.get('instance_level', {}).get('agreement_vs_abs_tol', []), 0.1), @@ -77,8 +86,6 @@ def main() -> None: 'official_runlevel_max': (((official.get('run_level') or {}).get('overall_quantiles') or {}).get('abs_delta') or {}).get('max'), }) - out_dpath = default_report_root() / f'experiment-analysis-{slugify(args.experiment_name)}' - out_dpath.mkdir(parents=True, exist_ok=True) stamp = datetime_mod.datetime.now(datetime_mod.UTC).strftime('%Y%m%dT%H%M%SZ') history_dpath = out_dpath / '.history' / stamp[:8] history_dpath.mkdir(parents=True, exist_ok=True) @@ -115,6 +122,11 @@ def main() -> None: for row in summary_rows: lines.append(f" - run_spec_name: {row['run_spec_name']}") lines.append(f" report_dir: {row['report_dir']}") + lines.append(f" diagnostic_flags: {row['diagnostic_flags']}") + lines.append(f" kwdagger_a_empty_completion_rate: {row['kwdagger_a_empty_completion_rate']}") + lines.append(f" kwdagger_a_mean_output_tokens: {row['kwdagger_a_mean_output_tokens']}") + lines.append(f" official_empty_completion_rate: {row['official_empty_completion_rate']}") + lines.append(f" official_mean_output_tokens: {row['official_mean_output_tokens']}") lines.append(f" repeat_instance_agree_0: {row['repeat_instance_agree_0']}") lines.append(f" official_instance_agree_0: {row['official_instance_agree_0']}") lines.append(f" official_instance_agree_01: {row['official_instance_agree_01']}") diff --git a/dev/experiments/audit-helm-reproduction/python/common.py b/dev/experiments/audit-helm-reproduction/python/common.py index d610414..7fdfd83 100644 --- a/dev/experiments/audit-helm-reproduction/python/common.py +++ b/dev/experiments/audit-helm-reproduction/python/common.py @@ -40,6 +40,10 @@ def repo_run_specs_fpath() -> Path: return aiq_root() / "run_specs.yaml" +def repo_run_details_fpath() -> Path: + return aiq_root() / "run_details.yaml" + + def default_report_root() -> Path: return audit_root() / "reports" @@ -62,4 +66,3 @@ def experiment_result_dpath(manifest: dict[str, Any]) -> Path: def experiment_report_dpath(manifest: dict[str, Any]) -> Path: return default_report_root() / str(manifest["experiment_name"]) - diff --git a/dev/experiments/audit-helm-reproduction/python/core_metric_report.py b/dev/experiments/audit-helm-reproduction/python/core_metric_report.py index 951bb8a..dd635b1 100644 --- a/dev/experiments/audit-helm-reproduction/python/core_metric_report.py +++ b/dev/experiments/audit-helm-reproduction/python/core_metric_report.py @@ -6,6 +6,7 @@ import math import os import shutil +import statistics from pathlib import Path from typing import Any @@ -72,6 +73,91 @@ def _run_level_core_rows(diff: HelmRunDiff) -> list[dict[str, Any]]: return rows +def _load_json(fpath: Path) -> Any: + return json.loads(fpath.read_text()) + + +def _collect_stat_means(stats: list[dict[str, Any]], metric_name: str) -> dict[str, float]: + found = {} + for row in stats: + name = row.get('name') + if not isinstance(name, dict): + continue + if name.get('name') != metric_name: + continue + split = name.get('split') + found[str(split)] = row.get('mean') + return found + + +def _run_diagnostics(run_path: str) -> dict[str, Any]: + run_path = str(Path(run_path).expanduser().resolve()) + run_dpath = Path(run_path) + scenario_state = _load_json(run_dpath / 'scenario_state.json') + stats = _load_json(run_dpath / 'stats.json') + reqs = scenario_state.get('request_states', []) + + token_counts = [] + empty_completion_count = 0 + nonempty_completion_count = 0 + completion_count = 0 + for rs in reqs: + comps = (rs.get('result') or {}).get('completions') or [] + if not comps: + continue + completion_count += 1 + c0 = comps[0] or {} + text = c0.get('text', '') + toklist = c0.get('tokens') or [] + token_counts.append(len(toklist)) + if text == '': + empty_completion_count += 1 + else: + nonempty_completion_count += 1 + + mean_tokens = statistics.mean(token_counts) if token_counts else None + return { + 'run_path': run_path, + 'run_name': run_dpath.name, + 'n_request_states': len(reqs), + 'n_with_completions': completion_count, + 'empty_completion_count': empty_completion_count, + 'nonempty_completion_count': nonempty_completion_count, + 'empty_completion_rate': ( + empty_completion_count / completion_count if completion_count else None + ), + 'output_token_count': { + 'mean': mean_tokens, + 'p50': _quantile(token_counts, 0.5), + 'p90': _quantile(token_counts, 0.9), + 'max': _quantile(token_counts, 1.0), + }, + 'stats_means': { + 'num_output_tokens': _collect_stat_means(stats, 'num_output_tokens'), + 'num_completion_tokens': _collect_stat_means(stats, 'num_completion_tokens'), + 'finish_reason_unknown': _collect_stat_means(stats, 'finish_reason_unknown'), + }, + } + + +def _diagnostic_flags(run_diagnostics: dict[str, dict[str, Any]]) -> list[str]: + flags = [] + for label, diag in run_diagnostics.items(): + rate = diag.get('empty_completion_rate') + mean_tokens = (diag.get('output_token_count') or {}).get('mean') + if rate is not None and rate > 0.1: + flags.append(f'{label}:high_empty_completion_rate') + if mean_tokens is not None and mean_tokens < 1.0: + flags.append(f'{label}:near_zero_mean_output_tokens') + official = run_diagnostics.get('official', {}) + kwdagger_a = run_diagnostics.get('kwdagger_a', {}) + off_rate = official.get('empty_completion_rate') + kwa_rate = kwdagger_a.get('empty_completion_rate') + if off_rate is not None and kwa_rate is not None and off_rate < 0.01 and kwa_rate > 0.1: + flags.append('official_vs_kwdagger_a:empty_completion_pathology') + return flags + + def _iter_joined_rows(joined, row_by_key): if row_by_key is not None: return row_by_key.values() @@ -620,13 +706,25 @@ def _write_text(report: dict[str, Any], out_fpath: Path) -> None: lines.append('Core Metric Report') lines.append('') lines.append(f"generated_utc: {report['generated_utc']}") + lines.append(f"run_spec_name: {report['run_spec_name']}") lines.append(f"left_label: {left['label']}") lines.append(f"right_label: {right['label']}") + lines.append(f"diagnostic_flags: {report.get('diagnostic_flags', [])}") lines.append('') lines.append('core_metrics:') for metric in left['core_metrics']: lines.append(f' - {metric}') lines.append('') + lines.append('run_diagnostics:') + for label, diag in report.get('run_diagnostics', {}).items(): + lines.append(f' {label}:') + lines.append(f" n_request_states: {diag.get('n_request_states')}") + lines.append(f" n_with_completions: {diag.get('n_with_completions')}") + lines.append(f" empty_completion_count: {diag.get('empty_completion_count')}") + lines.append(f" empty_completion_rate: {diag.get('empty_completion_rate')}") + lines.append(f" output_token_count: {json.dumps(diag.get('output_token_count'))}") + lines.append(f" stats_means: {json.dumps(diag.get('stats_means'))}") + lines.append('') for pair in report['pairs']: lines.append(f"pair: {pair['label']}") lines.append(f" diagnosis: {pair['diagnosis'].get('label')}") @@ -667,6 +765,7 @@ def _write_management_summary(report: dict[str, Any], out_fpath: Path) -> None: lines.append(f"generated_utc: {report['generated_utc']}") lines.append(f"run_spec_name: {report['run_spec_name']}") lines.append(f"core_metrics: {', '.join(left.get('core_metrics', []))}") + lines.append(f"diagnostic_flags: {report.get('diagnostic_flags', [])}") lines.append('') lines.append('metric_descriptions:') for metric in left.get('core_metrics', []): @@ -675,6 +774,18 @@ def _write_management_summary(report: dict[str, Any], out_fpath: Path) -> None: f" - {metric}: {desc['kind']}; {desc['range']}; {desc['direction']}" ) lines.append('') + lines.append('run_diagnostics:') + for label, diag in report.get('run_diagnostics', {}).items(): + lines.append(f' {label}:') + lines.append(f" n_request_states: {diag.get('n_request_states')}") + lines.append(f" n_with_completions: {diag.get('n_with_completions')}") + lines.append(f" empty_completion_count: {diag.get('empty_completion_count')}") + lines.append(f" empty_completion_rate: {diag.get('empty_completion_rate')}") + lines.append(f" mean_output_tokens_from_state: {(diag.get('output_token_count') or {}).get('mean')}") + lines.append(f" p90_output_tokens_from_state: {(diag.get('output_token_count') or {}).get('p90')}") + lines.append(f" num_output_tokens_from_stats: {(diag.get('stats_means') or {}).get('num_output_tokens')}") + lines.append(f" finish_reason_unknown_from_stats: {(diag.get('stats_means') or {}).get('finish_reason_unknown')}") + lines.append('') lines.append(f"{left['label']}:") lines.append(f" diagnosis: {left['diagnosis'].get('label')}") lines.append(f" run-level N: {left['run_level']['n_rows']}") @@ -741,11 +852,18 @@ def main() -> None: left = _build_pair(args.left_run_a, args.left_run_b, args.left_label, thresholds) right = _build_pair(args.right_run_a, args.right_run_b, args.right_label, thresholds) + run_diagnostics = { + 'kwdagger_a': _run_diagnostics(args.left_run_a), + 'kwdagger_b': _run_diagnostics(args.left_run_b), + 'official': _run_diagnostics(args.right_run_a), + } report = { 'generated_utc': stamp, 'run_spec_name': run_spec_name, 'thresholds': thresholds, 'pairs': [left, right], + 'run_diagnostics': run_diagnostics, + 'diagnostic_flags': _diagnostic_flags(run_diagnostics), } json_fpath = history_dpath / f'core_metric_report_{stamp}.json' diff --git a/dev/experiments/audit-helm-reproduction/python/rebuild_core_report_from_index.py b/dev/experiments/audit-helm-reproduction/python/rebuild_core_report_from_index.py index 85b07e1..0274d21 100644 --- a/dev/experiments/audit-helm-reproduction/python/rebuild_core_report_from_index.py +++ b/dev/experiments/audit-helm-reproduction/python/rebuild_core_report_from_index.py @@ -32,11 +32,17 @@ def _coerce_float(x): return float('-inf') -def matching_rows(rows: list[dict[str, Any]], run_entry: str) -> list[dict[str, Any]]: +def matching_rows( + rows: list[dict[str, Any]], + run_entry: str, + experiment_name: str | None = None, +) -> list[dict[str, Any]]: out = [] for row in rows: if row.get('run_entry') != run_entry: continue + if experiment_name is not None and row.get('experiment_name') != experiment_name: + continue if row.get('status') not in {'computed', 'reused', 'unknown', ''}: continue if row.get('has_run_spec', '').lower() not in {'true', '1'}: @@ -125,6 +131,7 @@ def main() -> None: parser.add_argument('--left-label', default='kwdagger_repeat') parser.add_argument('--right-label', default='official_vs_kwdagger') parser.add_argument('--allow-single-repeat', action='store_true') + parser.add_argument('--experiment-name', default=None) args = parser.parse_args() index_fpath = ( @@ -133,8 +140,13 @@ def main() -> None: latest_index_csv(Path(args.index_dpath).expanduser().resolve()) ) rows = load_rows(index_fpath) - matches = matching_rows(rows, args.run_entry) + matches = matching_rows(rows, args.run_entry, experiment_name=args.experiment_name) if not matches: + if args.experiment_name is not None: + raise SystemExit( + f'No indexed kwdagger runs found for run_entry={args.run_entry!r} ' + f'within experiment_name={args.experiment_name!r}' + ) raise SystemExit(f'No indexed kwdagger runs found for run_entry={args.run_entry!r}') if len(matches) >= 2: @@ -193,6 +205,7 @@ def main() -> None: 'left_label': args.left_label, 'right_label': args.right_label, 'report_dpath': str(report_dpath), + 'experiment_name': args.experiment_name, 'historic_info': info, } selection_fpath = _write_latest_selection(report_dpath, selection) diff --git a/dev/poc/inspect_historic_helm_runs.py b/dev/poc/inspect_historic_helm_runs.py index 0d60d4f..30e0840 100644 --- a/dev/poc/inspect_historic_helm_runs.py +++ b/dev/poc/inspect_historic_helm_runs.py @@ -56,7 +56,7 @@ from magnet.backends.helm.helm_outputs import HelmOutputs, HelmRun # Reuse your existing discovery + inference logic -from magnet.backends.helm.materialize_helm_run import ( +from magnet.backends.helm.cli.materialize_helm_run import ( discover_benchmark_output_dirs, infer_num_instances, is_complete_run_dir, From dc5c4230a1099548beeb429fc0bc0446136091ac Mon Sep 17 00:00:00 2001 From: agent Date: Sat, 28 Mar 2026 20:20:21 +0000 Subject: [PATCH 29/36] wip --- .../python/build_repro_manifest.py | 275 ++++++++++++++++++ .../scripts/make_historic_grid_manifest.sh | 19 ++ .../scripts/make_machine_shard_manifest.sh | 29 ++ .../scripts/make_machine_subset_manifest.sh | 23 ++ 4 files changed, 346 insertions(+) create mode 100644 dev/experiments/audit-helm-reproduction/python/build_repro_manifest.py create mode 100755 dev/experiments/audit-helm-reproduction/scripts/make_historic_grid_manifest.sh create mode 100755 dev/experiments/audit-helm-reproduction/scripts/make_machine_shard_manifest.sh create mode 100755 dev/experiments/audit-helm-reproduction/scripts/make_machine_subset_manifest.sh diff --git a/dev/experiments/audit-helm-reproduction/python/build_repro_manifest.py b/dev/experiments/audit-helm-reproduction/python/build_repro_manifest.py new file mode 100644 index 0000000..a750d38 --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/python/build_repro_manifest.py @@ -0,0 +1,275 @@ +from __future__ import annotations + +import argparse +import fnmatch +import math +from pathlib import Path +from typing import Any + +import kwutil + +from common import dump_yaml, env_defaults, repo_run_details_fpath, repo_run_specs_fpath + + +VICUNA_NOCHAT_OVERRIDE = ( + "dev/experiments/audit-helm-reproduction/configs/debug/" + "vicuna_no_chat_template.yaml" +) + + +def _load_run_specs(fpath: str | None) -> list[str]: + path = Path(fpath) if fpath else repo_run_specs_fpath() + data = kwutil.Yaml.load(path) + if not isinstance(data, list): + raise TypeError(f"run specs at {path} must decode to a list") + run_specs = [str(x) for x in data] + return list(dict.fromkeys(run_specs)) + + +def _load_run_details(fpath: str | None) -> list[dict[str, Any]]: + path = Path(fpath) if fpath else repo_run_details_fpath() + if not path.exists(): + return [] + data = kwutil.Yaml.load(path) + if not isinstance(data, list): + raise TypeError(f"run details at {path} must decode to a list") + rows = [row for row in data if isinstance(row, dict)] + return rows + + +def _matches_any(text: str, patterns: list[str]) -> bool: + return any(fnmatch.fnmatch(text, pat) for pat in patterns) + + +def _infer_benchmark(run_entry: str) -> str: + left = run_entry.split(":", 1)[0] + return left.split(",", 1)[0] + + +def _infer_model(run_entry: str) -> str | None: + for part in run_entry.replace(":", ",").split(","): + if part.startswith("model="): + return part.split("=", 1)[1] + return None + + +def _sort_key(run_entry: str) -> tuple[str, str, str]: + model = _infer_model(run_entry) or "" + benchmark = _infer_benchmark(run_entry) + return (model, benchmark, run_entry) + + +def _filter_run_entries( + run_entries: list[str], + *, + include_patterns: list[str], + exclude_patterns: list[str], + models: list[str], + benchmarks: list[str], +) -> list[str]: + filtered = [] + for run_entry in run_entries: + if include_patterns and not _matches_any(run_entry, include_patterns): + continue + if exclude_patterns and _matches_any(run_entry, exclude_patterns): + continue + model = _infer_model(run_entry) + benchmark = _infer_benchmark(run_entry) + if models and model not in set(models): + continue + if benchmarks and benchmark not in set(benchmarks): + continue + filtered.append(run_entry) + return filtered + + +def _shard_entries( + run_entries: list[str], + *, + num_shards: int | None, + shard_index: int | None, +) -> list[str]: + if num_shards is None and shard_index is None: + return run_entries + if num_shards is None or shard_index is None: + raise SystemExit("--num-shards and --shard-index must be provided together") + if num_shards <= 0: + raise SystemExit("--num-shards must be positive") + if shard_index < 0 or shard_index >= num_shards: + raise SystemExit("--shard-index must satisfy 0 <= shard-index < num-shards") + return [entry for idx, entry in enumerate(run_entries) if idx % num_shards == shard_index] + + +def _choose_model_override(run_entries: list[str], force_nochat: bool) -> str | None: + models = {_infer_model(entry) for entry in run_entries} + needs_vicuna = "lmsys/vicuna-7b-v1.3" in models + if force_nochat or needs_vicuna: + return VICUNA_NOCHAT_OVERRIDE + return None + + +def _detail_lut(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: + lut: dict[str, dict[str, Any]] = {} + for row in rows: + key = row.get("run_spec_name") + if isinstance(key, str) and key not in lut: + lut[key] = row + return lut + + +def _build_manifest( + *, + experiment_name: str, + description: str, + suite: str, + run_entries: list[str], + max_eval_instances: int, + tmux_workers: int, + devices: str, + model_deployments_fpath: str | None, +) -> dict[str, Any]: + return { + "schema_version": 1, + "experiment_name": experiment_name, + "description": description, + "run_entries": run_entries, + "max_eval_instances": max_eval_instances, + "suite": suite, + "mode": "compute_if_missing", + "materialize": "symlink", + "backend": "tmux", + "devices": devices, + "tmux_workers": tmux_workers, + "local_path": "prod_env", + "precomputed_root": None, + "require_per_instance_stats": True, + "model_deployments_fpath": model_deployments_fpath, + "enable_huggingface_models": [], + "enable_local_huggingface_models": [], + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--output", required=True) + parser.add_argument("--selection-output", default=None) + parser.add_argument("--run-specs-fpath", default=None) + parser.add_argument("--run-details-fpath", default=None) + parser.add_argument("--experiment-name", required=True) + parser.add_argument("--suite", required=True) + parser.add_argument("--description", default=None) + parser.add_argument("--devices", default=None) + parser.add_argument("--tmux-workers", type=int, default=None) + parser.add_argument("--max-eval-instances", type=int, default=None) + parser.add_argument("--limit", type=int, default=None) + parser.add_argument("--num-shards", type=int, default=None) + parser.add_argument("--shard-index", type=int, default=None) + parser.add_argument("--single-gpu", action="store_true") + parser.add_argument("--sort", default="model_benchmark", choices=["model_benchmark", "input"]) + parser.add_argument("--force-vicuna-nochat", action="store_true") + parser.add_argument("--include-pattern", action="append", default=[]) + parser.add_argument("--exclude-pattern", action="append", default=[]) + parser.add_argument("--model", action="append", default=[]) + parser.add_argument("--benchmark", action="append", default=[]) + args = parser.parse_args() + + defaults = env_defaults() + run_entries = _load_run_specs(args.run_specs_fpath) + run_details = _load_run_details(args.run_details_fpath) + detail_lut = _detail_lut(run_details) + + run_entries = _filter_run_entries( + run_entries, + include_patterns=args.include_pattern, + exclude_patterns=args.exclude_pattern, + models=args.model, + benchmarks=args.benchmark, + ) + if args.sort == "model_benchmark": + run_entries = sorted(run_entries, key=_sort_key) + run_entries = _shard_entries( + run_entries, + num_shards=args.num_shards, + shard_index=args.shard_index, + ) + if args.limit is not None: + run_entries = run_entries[: args.limit] + if not run_entries: + raise SystemExit("No run entries matched the requested filters") + + max_eval_instances = ( + args.max_eval_instances + if args.max_eval_instances is not None + else 1000 + ) + if args.single_gpu: + devices = args.devices if args.devices is not None else "0" + tmux_workers = args.tmux_workers if args.tmux_workers is not None else 1 + else: + devices = args.devices if args.devices is not None else "0,1" + tmux_workers = ( + args.tmux_workers + if args.tmux_workers is not None + else int(defaults["AUDIT_DEFAULT_TMUX_WORKERS"]) + ) + + model_override = _choose_model_override(run_entries, args.force_vicuna_nochat) + description = args.description or ( + f"Historic reproducibility batch with {len(run_entries)} run entries" + ) + manifest = _build_manifest( + experiment_name=args.experiment_name, + description=description, + suite=args.suite, + run_entries=run_entries, + max_eval_instances=max_eval_instances, + tmux_workers=tmux_workers, + devices=devices, + model_deployments_fpath=model_override, + ) + + out_fpath = Path(args.output) + out_fpath.parent.mkdir(parents=True, exist_ok=True) + out_fpath.write_text(dump_yaml(manifest)) + + selection_rows = [] + for idx, run_entry in enumerate(run_entries): + row = { + "index": idx, + "run_entry": run_entry, + "benchmark": _infer_benchmark(run_entry), + "model": _infer_model(run_entry), + "detail": detail_lut.get(run_entry), + } + selection_rows.append(row) + + selection = { + "experiment_name": args.experiment_name, + "suite": args.suite, + "manifest_fpath": str(out_fpath), + "selection_count": len(run_entries), + "num_shards": args.num_shards, + "shard_index": args.shard_index, + "limit": args.limit, + "devices": devices, + "tmux_workers": tmux_workers, + "max_eval_instances": max_eval_instances, + "model_deployments_fpath": model_override, + "include_patterns": args.include_pattern, + "exclude_patterns": args.exclude_pattern, + "models": args.model, + "benchmarks": args.benchmark, + "entries": selection_rows, + } + selection_fpath = ( + Path(args.selection_output) + if args.selection_output + else out_fpath.with_suffix(out_fpath.suffix + ".selection.yaml") + ) + selection_fpath.write_text(dump_yaml(selection)) + print(out_fpath) + print(selection_fpath) + + +if __name__ == "__main__": + main() diff --git a/dev/experiments/audit-helm-reproduction/scripts/make_historic_grid_manifest.sh b/dev/experiments/audit-helm-reproduction/scripts/make_historic_grid_manifest.sh new file mode 100755 index 0000000..750a2f2 --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/scripts/make_historic_grid_manifest.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/common.sh" + +audit::set_defaults + +OUTPATH="${1:-${AUDIT_ROOT}/configs/generated/historic_grid.generated.yaml}" +shift || true + +"$AIQ_PYTHON" "${AUDIT_ROOT}/python/build_repro_manifest.py" \ + --output "$OUTPATH" \ + --experiment-name audit-historic-grid \ + --suite audit-historic-grid \ + "$@" + +printf 'Wrote historic grid manifest: %s\n' "$OUTPATH" +printf 'Wrote selection sidecar: %s.selection.yaml\n' "$OUTPATH" diff --git a/dev/experiments/audit-helm-reproduction/scripts/make_machine_shard_manifest.sh b/dev/experiments/audit-helm-reproduction/scripts/make_machine_shard_manifest.sh new file mode 100755 index 0000000..323746e --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/scripts/make_machine_shard_manifest.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/common.sh" + +audit::set_defaults + +MACHINE_NAME="${1:?machine name required, e.g. namek or yardrat}" +SHARD_INDEX="${2:?shard index required}" +NUM_SHARDS="${3:?num shards required}" +OUTPATH="${4:-${AUDIT_ROOT}/configs/generated/${MACHINE_NAME}.generated.yaml}" +if [[ $# -gt 0 ]]; then shift; fi +if [[ $# -gt 0 ]]; then shift; fi +if [[ $# -gt 0 ]]; then shift; fi +if [[ $# -gt 0 ]]; then shift; fi + +"$AIQ_PYTHON" "${AUDIT_ROOT}/python/build_repro_manifest.py" \ + --output "$OUTPATH" \ + --experiment-name "audit-${MACHINE_NAME}" \ + --suite "audit-${MACHINE_NAME}" \ + --single-gpu \ + --devices 0 \ + --num-shards "$NUM_SHARDS" \ + --shard-index "$SHARD_INDEX" \ + "$@" + +printf 'Wrote machine shard manifest: %s\n' "$OUTPATH" +printf 'Wrote selection sidecar: %s.selection.yaml\n' "$OUTPATH" diff --git a/dev/experiments/audit-helm-reproduction/scripts/make_machine_subset_manifest.sh b/dev/experiments/audit-helm-reproduction/scripts/make_machine_subset_manifest.sh new file mode 100755 index 0000000..2c77eba --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/scripts/make_machine_subset_manifest.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/common.sh" + +audit::set_defaults + +MACHINE_NAME="${1:?machine name required, e.g. namek or yardrat}" +OUTPATH="${2:-${AUDIT_ROOT}/configs/generated/${MACHINE_NAME}.subset.generated.yaml}" +if [[ $# -gt 0 ]]; then shift; fi +if [[ $# -gt 0 ]]; then shift; fi + +"$AIQ_PYTHON" "${AUDIT_ROOT}/python/build_repro_manifest.py" \ + --output "$OUTPATH" \ + --experiment-name "audit-${MACHINE_NAME}-subset" \ + --suite "audit-${MACHINE_NAME}-subset" \ + --single-gpu \ + --devices 0 \ + "$@" + +printf 'Wrote machine subset manifest: %s\n' "$OUTPATH" +printf 'Wrote selection sidecar: %s.selection.yaml\n' "$OUTPATH" From 75977b83740890f1062379bb0a247a54fe49548d Mon Sep 17 00:00:00 2001 From: joncrall Date: Sat, 28 Mar 2026 16:54:23 -0400 Subject: [PATCH 30/36] wip --- dev/experiments/audit-helm-reproduction/README.md | 5 +++++ .../python/render_schedule_params.py | 7 +++++++ .../audit-helm-reproduction/scripts/check_env.sh | 10 +++++++++- .../scripts/run_from_manifest.sh | 8 ++++++++ 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/dev/experiments/audit-helm-reproduction/README.md b/dev/experiments/audit-helm-reproduction/README.md index bc3eb69..0a7458b 100644 --- a/dev/experiments/audit-helm-reproduction/README.md +++ b/dev/experiments/audit-helm-reproduction/README.md @@ -12,6 +12,11 @@ The design goals are: - start with a small smoke-test batch, - scale later to larger reproduction attempts without redesign. +Remote reproduction-only machines do not need `/data/crfm-helm-public` mounted. +The public HELM bundle is only required on the analysis machine that will do +historic comparisons later. If a manifest sets `precomputed_root: null`, the +runner now skips the `HELM_PRECOMPUTED_ROOT` existence check. + ## Defaults These defaults are chosen to work in the current environment: diff --git a/dev/experiments/audit-helm-reproduction/python/render_schedule_params.py b/dev/experiments/audit-helm-reproduction/python/render_schedule_params.py index d58572a..f7bdcec 100644 --- a/dev/experiments/audit-helm-reproduction/python/render_schedule_params.py +++ b/dev/experiments/audit-helm-reproduction/python/render_schedule_params.py @@ -47,6 +47,7 @@ def main() -> None: "backend", "tmux_workers", "devices", + "precomputed_root", ], ) args = parser.parse_args() @@ -64,6 +65,12 @@ def main() -> None: print(manifest.get("tmux_workers", 2)) elif args.mode == "devices": print(manifest.get("devices", "0,1")) + elif args.mode == "precomputed_root": + value = manifest.get("precomputed_root", None) + if value is None: + print("") + else: + print(value) else: raise AssertionError(args.mode) diff --git a/dev/experiments/audit-helm-reproduction/scripts/check_env.sh b/dev/experiments/audit-helm-reproduction/scripts/check_env.sh index f4d9553..bad151f 100755 --- a/dev/experiments/audit-helm-reproduction/scripts/check_env.sh +++ b/dev/experiments/audit-helm-reproduction/scripts/check_env.sh @@ -11,7 +11,7 @@ printf '=================\n' audit::print_env printf '\n' -for path_var in AIQ_MAGNET_ROOT HELM_PRECOMPUTED_ROOT; do +for path_var in AIQ_MAGNET_ROOT; do path="${!path_var}" if [[ ! -e "$path" ]]; then printf '%s does not exist: %s\n' "$path_var" "$path" >&2 @@ -19,6 +19,14 @@ for path_var in AIQ_MAGNET_ROOT HELM_PRECOMPUTED_ROOT; do fi done +REQUIRE_PRECOMPUTED_ROOT="${AUDIT_REQUIRE_PRECOMPUTED_ROOT:-1}" +if [[ "$REQUIRE_PRECOMPUTED_ROOT" == "1" ]]; then + if [[ ! -e "$HELM_PRECOMPUTED_ROOT" ]]; then + printf '%s does not exist: %s\n' "HELM_PRECOMPUTED_ROOT" "$HELM_PRECOMPUTED_ROOT" >&2 + exit 1 + fi +fi + if [[ ! -d "$AUDIT_RESULTS_ROOT" ]]; then if mkdir -p "$AUDIT_RESULTS_ROOT" 2>/dev/null; then : diff --git a/dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh b/dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh index bc9ab67..2bbf3af 100755 --- a/dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh +++ b/dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh @@ -24,6 +24,14 @@ TMUX_WORKERS="$("$AIQ_PYTHON" "${AUDIT_ROOT}/python/render_schedule_params.py" \ --manifest "$MANIFEST" --mode tmux_workers)" DEVICES="$("$AIQ_PYTHON" "${AUDIT_ROOT}/python/render_schedule_params.py" \ --manifest "$MANIFEST" --mode devices)" +PRECOMPUTED_ROOT="$("$AIQ_PYTHON" "${AUDIT_ROOT}/python/render_schedule_params.py" \ + --manifest "$MANIFEST" --mode precomputed_root)" + +if [[ -n "$PRECOMPUTED_ROOT" ]]; then + AUDIT_REQUIRE_PRECOMPUTED_ROOT=1 "${AUDIT_ROOT}/scripts/check_env.sh" >/dev/null +else + AUDIT_REQUIRE_PRECOMPUTED_ROOT=0 "${AUDIT_ROOT}/scripts/check_env.sh" >/dev/null +fi mkdir -p "$RESULT_DPATH" From 0a3a26977e8d7fc164e475dc502590ec0e2777ab Mon Sep 17 00:00:00 2001 From: joncrall Date: Sat, 28 Mar 2026 16:58:31 -0400 Subject: [PATCH 31/36] wip --- .../audit-helm-reproduction/scripts/run_from_manifest.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh b/dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh index 2bbf3af..9f50491 100755 --- a/dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh +++ b/dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh @@ -10,8 +10,6 @@ audit::set_defaults MANIFEST="${1:-${AUDIT_ROOT}/configs/smoke_manifest.yaml}" audit::require_file "$MANIFEST" -"${AUDIT_ROOT}/scripts/check_env.sh" >/dev/null - EXPERIMENT_NAME="$("$AIQ_PYTHON" "${AUDIT_ROOT}/python/render_schedule_params.py" \ --manifest "$MANIFEST" --mode experiment_name)" RESULT_DPATH="$("$AIQ_PYTHON" "${AUDIT_ROOT}/python/render_schedule_params.py" \ From 4eb9194ddf64b001efbe76583771ee5503ccb74a Mon Sep 17 00:00:00 2001 From: joncrall Date: Sat, 28 Mar 2026 17:00:01 -0400 Subject: [PATCH 32/36] wip --- .../configs/debug/vicuna_no_chat_template.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 dev/experiments/audit-helm-reproduction/configs/debug/vicuna_no_chat_template.yaml diff --git a/dev/experiments/audit-helm-reproduction/configs/debug/vicuna_no_chat_template.yaml b/dev/experiments/audit-helm-reproduction/configs/debug/vicuna_no_chat_template.yaml new file mode 100644 index 0000000..bd67e0a --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/configs/debug/vicuna_no_chat_template.yaml @@ -0,0 +1,9 @@ +model_deployments: + - name: huggingface/vicuna-7b-v1.3 + model_name: lmsys/vicuna-7b-v1.3 + tokenizer_name: hf-internal-testing/llama-tokenizer + max_sequence_length: 2048 + client_spec: + class_name: "helm.clients.huggingface_client.HuggingFaceClient" + args: + apply_chat_template: false From 49cf97fc12bd1ee5c8cdb21fd8576a2e99c5225b Mon Sep 17 00:00:00 2001 From: joncrall Date: Mon, 30 Mar 2026 13:34:00 -0400 Subject: [PATCH 33/36] wip --- .../helm-reproduction-research-journal.md | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) diff --git a/dev/codex/helm-reproduction-research-journal.md b/dev/codex/helm-reproduction-research-journal.md index ebaa97b..1d1b56c 100644 --- a/dev/codex/helm-reproduction-research-journal.md +++ b/dev/codex/helm-reproduction-research-journal.md @@ -484,3 +484,86 @@ Interpretation: - the fixed Vicuna/HuggingFace path appears independently reproducible on the tested core metrics - at least for these experiments, the newer server run supports a positive cross-machine reproducibility result rather than revealing a hardware-specific failure mode + +## Updated Research Goal + +The project goal should now be stated more precisely: + +- produce a documented recipe for reproducing selected public HELM runs in a repeatable way +- quantify and bound the small remaining differences that are expected from local execution, hardware variation, and nondeterminism +- prefer benchmarks, models, and evaluation paths that are runnable with open weights and realistic "consumer-accessible" hardware constraints + +Interpretation: + +- this does not necessarily mean laptop-only reproduction +- but it should avoid closed-weight models and giant model families that are not meaningfully checkable by other researchers +- a practical target is open-weight models that can run on a single large consumer GPU or on a modest multi-GPU workstation + +Methodologically, this pushes us toward a reproducibility distribution rather than a single point estimate: + +- same-machine repeatability +- cross-machine repeatability +- official-vs-local drift +- benchmark-level and collection-level summaries + +## Large Grid Status: Runnable Recipe vs Unrunnable Cases + +The larger `audit-historic-grid` batch was informative, but many failures do **not** currently count as evidence against reproducibility. + +Observed: + +- total jobs: `286` +- completed jobs: `141` +- failed or incomplete jobs: `145` + +Representative failure buckets: + +- dataset loading / access / download issues + - RAFT requires `trust_remote_code=True` + - MATH failed to resolve `hendrycks/competition_math` + - bAbI and Natural Questions failed on direct data downloads via `wget` +- gated or external-service requirements + - `walledai/XSTest` is gated + - some annotation-heavy runs require OpenAI credentials +- model-specific packaging issues + - `aisingapore/sea-lion-7b-instruct` failed while resolving dynamic imports such as `flash_attn_triton.py` +- some broader model families remain insufficiently diagnosed from the raw logs alone and need targeted follow-up + +Interpretation: + +- the current historic candidate list is too broad to serve directly as a reproducibility benchmark suite +- we need a stricter "runnable recipe" filter: + - open-weight models + - locally runnable HF path + - datasets that are accessible without custom private credentials + - scenarios that do not require external proprietary annotators + - configurations that fit the hardware budget we expect others to have + +## Namek / Yardrat Subset Check + +The small cross-hardware Vicuna subset remains encouraging. + +Observed: + +- `audit-yardrat-subset` + - BoolQ: complete + - MMLU: complete + - NarrativeQA: complete +- `audit-namek-subset` + - BoolQ: complete + - MMLU: complete + - NarrativeQA: incomplete / still-running at time of inspection + +For the completed subset jobs: + +- no empty-completion pathology was observed +- Yardrat matched the previously successful Vicuna no-chat behavior: + - BoolQ core correctness metrics matched official exactly + - MMLU core metrics matched official exactly + - NarrativeQA remained very close on core metrics, with residual drift mainly in output length rather than correctness +- Namek BoolQ and MMLU also matched this same pattern + +Interpretation: + +- the corrected Vicuna recipe appears stable across multiple machines +- this strengthens the case that our main reproducibility result is positive when the configuration is correct and the task is runnable under the local open-weight recipe From 27cf0b4ce4fa19e450fec1fc2c98282da6d02bde Mon Sep 17 00:00:00 2001 From: joncrall Date: Mon, 30 Mar 2026 18:32:25 -0400 Subject: [PATCH 34/36] wip --- dev/codex/helm-reproduction-agent-brief.md | 332 +++++++++++ .../helm-reproduction-status-checkpoint.md | 514 ++++++++++++++++++ .../python/analyze_experiment_from_index.py | 195 ++++++- .../python/compare_pair.py | 11 + .../python/core_metric_report.py | 85 ++- .../python/paper_labels.py | 55 ++ .../rebuild_all_core_reports_from_index.py | 3 + 7 files changed, 1171 insertions(+), 24 deletions(-) create mode 100644 dev/codex/helm-reproduction-agent-brief.md create mode 100644 dev/codex/helm-reproduction-status-checkpoint.md create mode 100644 dev/experiments/audit-helm-reproduction/python/paper_labels.py diff --git a/dev/codex/helm-reproduction-agent-brief.md b/dev/codex/helm-reproduction-agent-brief.md new file mode 100644 index 0000000..b12c2f6 --- /dev/null +++ b/dev/codex/helm-reproduction-agent-brief.md @@ -0,0 +1,332 @@ +# HELM Reproduction Agent Brief + +Date: 2026-03-30 +Workspace: `/home/joncrall/code/aiq-magnet` + +## Mission + +Prepare a fresh agent to continue the HELM reproducibility project quickly and safely. + +The overarching goal is: + +- determine how much of public HELM is independently reproducible with local open-weight execution +- quantify the remaining differences rather than treating reproducibility as a binary +- build a paper-quality methodology around: + - same-machine repeatability + - cross-machine repeatability + - official-vs-local drift + - benchmark-level and collection-level summaries + +For publication framing, the likely paper story is: + +- a positive result for a carefully defined runnable subset of HELM +- with quantified bounds on unavoidable residual differences +- plus a clear failure taxonomy for cases that are not currently runnable or not reproducible under the open local recipe + +## Key Principle + +Do **not** interpret every failed large-grid job as a reproducibility failure. + +Many failures so far are recipe-scope failures: + +- gated datasets +- missing external credentials +- dataset download failures +- model packaging issues +- scenarios requiring proprietary annotators + +Those belong in the "not currently runnable under the recipe" bucket, not the "HELM is irreproducible" bucket. + +## Current Scientific Conclusion + +The strongest current positive signal is the corrected Vicuna no-chat recipe. + +For: + +- `boolq:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical` +- `mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical` +- `narrative_qa:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical` + +once `apply_chat_template: false` is used for the local HF deployment: + +- same-machine reruns are stable +- cross-machine reruns are stable +- official-vs-local agreement on core metrics is excellent +- remaining drift is mostly output length/style rather than correctness + +This is the current best evidence that a substantial subset of HELM is independently reproducible. + +## Root Cause Already Found + +There was a real false alarm earlier. + +Problem: + +- local HF Vicuna runs were auto-applying chat templating +- this caused severe empty-completion pathologies, especially on `narrative_qa` + +Fix: + +- use: + - `dev/experiments/audit-helm-reproduction/configs/debug/vicuna_no_chat_template.yaml` + +Effect: + +- empty completions disappeared +- core metrics became close or identical to official public HELM + +This is a critical lesson: + +- local execution-path bugs can masquerade as reproducibility failures +- any strong irreproducibility claim should be preceded by recipe validation diagnostics + +## Important Paths + +Public historic bundle: + +- `/data/crfm-helm-public` + +Local reproduced runs: + +- `/data/crfm-helm-audit` + +Main audit workflow: + +- `dev/experiments/audit-helm-reproduction` + +Research notes: + +- `dev/codex/helm-reproduction-research-journal.md` +- `dev/codex/reproduce-helm-session-v2.md` +- `dev/codex/kwdagger-notes.md` + +## Operational State Of Recent Experiments + +### 1. `audit-vicuna-nochat-overnight` + +This is the fixed-config baseline on the main machine. + +Status: + +- successful +- core reproducibility result is positive + +### 2. `audit-vicuna-nochat-server` + +Initial attempt: + +- failed due to GPU occupancy / OOM +- logs captured this clearly + +Later successful attempt: + +- matched the earlier fixed Vicuna result +- supports positive cross-machine reproducibility + +### 3. `audit-namek-subset` + +Subset intended for consumer-ish single-GPU cross-hardware checks. + +Observed: + +- BoolQ: complete +- MMLU: complete +- NarrativeQA: incomplete / hanging at time of inspection + +Completed jobs match the same positive Vicuna no-chat pattern. + +### 4. `audit-yardrat-subset` + +Observed: + +- all 3 target jobs completed +- terminal impression of failure was misleading +- copied artifacts show valid `DONE` files and healthy logs + +Scientific interpretation: + +- Yardrat supports the same positive cross-machine result as the other successful Vicuna no-chat runs + +### 5. `audit-historic-grid` + +Large broad candidate sweep built from `run_specs.yaml` / `run_details.yaml`. + +Observed: + +- total jobs: `286` +- completed jobs: `141` +- failed or incomplete: `145` + +This grid is useful mainly for failure taxonomy and runnable-subset refinement. + +## Large Grid Failure Taxonomy + +Representative buckets already observed: + +### Dataset / scenario access issues + +- RAFT: + - requires `trust_remote_code=True` +- MATH: + - dataset resolution failure for `hendrycks/competition_math` +- bAbI: + - failed on direct `wget` download +- Natural Questions: + - failed on direct `wget` download + +### Gated / credentialed cases + +- XSTest: + - gated dataset +- annotation-heavy runs: + - may require OpenAI credentials + +### Model packaging / environment issues + +- Sea Lion: + - missing dynamic import files such as `flash_attn_triton.py` + +### Important interpretation + +These are not currently evidence against reproducibility. + +They are evidence that the current candidate suite is too broad and must be filtered to a reproducibility-ready subset. + +## Runnable Recipe We Should Move Toward + +The reproducibility recipe should prioritize: + +- open-weight models +- local HF execution path +- models that fit on realistic consumer-accessible hardware + - single large consumer GPU + - or modest multi-GPU workstation +- datasets available without private credentials +- scenarios not requiring proprietary judge/annotator APIs +- configurations we can archive and rerun deterministically enough for audit purposes + +In practice this likely means: + +- do not center the paper on the entire raw HELM catalog +- instead define a reproducibility-ready subset with explicit inclusion criteria + +## Existing Tooling Worth Reusing + +### Historic candidate generation + +- `dev/poc/inspect_historic_helm_runs.py` +- outputs: + - `run_specs.yaml` + - `run_details.yaml` + +### Audit workflow + +- `dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh` +- `dev/experiments/audit-helm-reproduction/scripts/compare_batch.sh` +- `dev/experiments/audit-helm-reproduction/scripts/index_results.sh` +- `dev/experiments/audit-helm-reproduction/scripts/analyze_experiment_from_index.sh` +- `dev/experiments/audit-helm-reproduction/scripts/aggregate_core_reports.sh` + +### Manifest building + +- `dev/experiments/audit-helm-reproduction/scripts/make_historic_grid_manifest.sh` +- `dev/experiments/audit-helm-reproduction/scripts/make_machine_subset_manifest.sh` +- `dev/experiments/audit-helm-reproduction/scripts/make_machine_shard_manifest.sh` + +### Important note + +If running the analysis tooling on this machine, prefer: + +- `/home/joncrall/code/aiq-magnet/.venv/bin/python` + +because the default `python` here may not have `helm` installed. + +## Reporting / Analysis State + +The audit folder already contains: + +- per-run-spec core metric reports +- management summaries +- threshold-vs-agreement plots +- raw score distribution plots +- ECDF plots +- run-level metric tables +- instance sample reports +- experiment-level summaries +- overall reproducibility rollups + +Important conceptual improvement already adopted: + +- focus on reproducibility distributions / envelopes, not single exact-equality claims + +## What A Fresh Agent Should Do Next + +### 1. Curate a reproducibility-ready subset + +Turn `run_specs.yaml` into a filtered suite that excludes: + +- gated datasets +- credentialed annotator cases +- obviously broken download paths +- models with known packaging failures + +This should become the main benchmark set for the paper. + +### 2. Quantify the distribution of differences + +For the filtered subset: + +- same-machine repeatability +- cross-machine repeatability +- official-vs-local drift + +Summaries should include: + +- per-instance agreement +- run-level metric deltas +- benchmark-family rollups +- model-family rollups + +### 3. Keep correctness and style drift separate + +Important current pattern: + +- core correctness metrics often match exactly or nearly exactly +- output length can still differ substantially + +This distinction should be central in the paper. + +### 4. Make the runnable recipe explicit + +Document: + +- required configs +- model overrides +- environment assumptions +- hardware assumptions +- known excluded benchmark/model families and why they are excluded + +### 5. Continue failure taxonomy + +For the broad grid: + +- classify each failure into: + - unrunnable dataset + - missing credential + - model packaging/runtime issue + - true reproduction drift + - unknown + +This can become a useful table or appendix. + +## Current Bottom Line + +The project has moved from: + +- "maybe HELM is not reproducible" + +to something more precise: + +- a meaningful subset of HELM appears reproducible under a corrected local open-weight recipe +- many apparent failures are recipe or environment issues, not scientific counterexamples +- the next paper-quality step is to formalize the subset and quantify the reproducibility envelope diff --git a/dev/codex/helm-reproduction-status-checkpoint.md b/dev/codex/helm-reproduction-status-checkpoint.md new file mode 100644 index 0000000..f168f44 --- /dev/null +++ b/dev/codex/helm-reproduction-status-checkpoint.md @@ -0,0 +1,514 @@ +# HELM Reproduction Status Checkpoint + +Date: 2026-03-30 +Workspace: `/home/joncrall/code/aiq-magnet` + +## Purpose + +This document is a restart-safe checkpoint for the HELM reproduction project. + +It is meant to answer: + +- what we are trying to prove +- what we have already established +- what still needs to be done before we can make strong publication-quality claims +- what infrastructure work must be preserved as this project is separated from `magnet` + +This document is intentionally redundant with other notes, but should be the best +single place to restart from if prior chat context is lost. + +## Related Documents + +Read these first for deeper context and provenance: + +- `dev/codex/helm-reproduction-agent-brief.md` +- `dev/codex/helm-reproduction-research-journal.md` +- `dev/experiments/audit-helm-reproduction/README.md` +- `dev/codex/reproduce-helm-session-v2.md` +- `dev/codex/kwdagger-notes.md` + +Important code and experiment roots: + +- public historic HELM runs: `/data/crfm-helm-public` +- local reproduced runs: `/data/crfm-helm-audit` +- audit workflow: `dev/experiments/audit-helm-reproduction` + +## Core Research Goal + +The real paper question is not: + +- "can we reproduce every HELM run exactly?" + +The better question is: + +- "to what extent are public HELM results independently reproducible under an explicit open-weight local recipe, and how should residual differences be interpreted?" + +For a strong paper, we need a precise answer to four sub-questions: + +1. Same-machine repeatability: + how stable are local reruns under the same recipe? +2. Cross-machine repeatability: + how stable are local reruns across different GPU hardware and host environments? +3. Official-vs-local reproducibility: + when we compare to historic public HELM, how close are we on core benchmark outcomes? +4. Scope: + for what subset of HELM can these comparisons be run fairly under an open-weight local recipe? + +## Current Headline Conclusion + +The strongest current result is positive: + +- a corrected local Vicuna no-chat recipe reproduces a small but meaningful HELM subset very well +- this holds on multiple machines +- the remaining cross-machine differences are largely bookkeeping-style drift, not core-metric failure + +The best current positive evidence is for: + +- `boolq:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical` +- `mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical` +- `narrative_qa:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical` + +with: + +- same-machine repeatability +- cross-machine repeatability +- excellent official-vs-local agreement on core metrics + +This is the current basis for the paper framing: + +- open-weight HELM reproduction is possible for a clearly defined subset +- exact equality is too strong a target +- core-metric agreement and bounded drift are the right scientific objects + +## Critical Resolved Issue + +An earlier apparent failure was real in the sense that the outputs were bad, but +it was not evidence against reproducibility. + +Root cause: + +- the local HF Vicuna execution path was auto-applying a chat template +- this caused severe empty-completion pathologies, especially on `narrative_qa` + +Fix: + +- use `dev/experiments/audit-helm-reproduction/configs/debug/vicuna_no_chat_template.yaml` +- specifically, the corrected local deployment must use `apply_chat_template: false` + +Scientific lesson: + +- execution-path bugs can masquerade as irreproducibility +- any claim of irreproducibility must be preceded by recipe validation diagnostics + +## Questions We Have Answered Already + +### 1. Are exact mismatches always evidence of irreproducibility? + +No. + +We have already shown that: + +- some mismatches were caused by local recipe errors +- some mismatches are bookkeeping/runtime drift +- some large-grid failures are simply "not runnable under the current recipe" + +### 2. Is the corrected Vicuna local recipe repeatable on the same machine? + +Yes, for the current evaluated subset. + +The corrected Vicuna no-chat runs show strong same-machine repeatability on the +current core tasks. + +### 3. Is the corrected Vicuna local recipe repeatable across machines? + +Yes, for the tested subset. + +Current evidence supports positive cross-machine reproducibility across: + +- the internal main machine +- `namek` +- `yardrat` + +with the strongest evidence currently on: + +- BoolQ +- MMLU `us_foreign_policy` +- NarrativeQA + +### 4. Are cross-machine differences currently dominated by benchmark failure? + +No, not for the successful Vicuna no-chat subset. + +The current pairwise cross-machine reports are being diagnosed mainly as: + +- `bookkeeping_metric_drift` + +rather than core-metric collapse. + +### 5. Are many large-grid failures actually recipe-scope failures rather than negative reproducibility evidence? + +Yes. + +Observed buckets include: + +- gated datasets +- dataset download failures +- scenarios needing external credentials or proprietary annotators +- model packaging issues + +These belong in a runnable-scope taxonomy, not in a blanket "HELM is irreproducible" claim. + +### 6. Is apples-to-apples alignment on `max_eval_instances` enough by itself to explain the old official-vs-local drift? + +No. + +We already controlled this confounder, and substantial drift still persisted in +the older Pythia-style cases. + +### 7. Is local-vs-local noise generally much smaller than the big official-vs-local failures seen in the older cases? + +Yes. + +That earlier result remains important: + +- local repeat noise exists +- but the older structural official-vs-local failures were much larger than ordinary rerun variance + +This is part of the causal story for why "recipe-corrected subset succeeds" is a +real scientific result rather than noise. + +## Questions We Can Partially Answer But Not Yet Claim Broadly + +### 1. How broad is the reproducible open-weight subset? + +Partial answer: + +- at least the current corrected Vicuna subset is reproducible on key tasks + +Not yet established: + +- how far this extends across more benchmarks +- how far this extends across more model families +- whether we can support a broad claim beyond this subset + +### 2. Is reproducibility achievable on realistic consumer hardware? + +Partial answer: + +- yes for some tested runs on single- or modest-multi-GPU machines + +Not yet established: + +- the final size and diversity of the consumer-hardware reproducible subset +- a rigorous paper-ready inclusion criterion for "consumer-accessible" + +### 3. Are the remaining cross-machine differences scientifically negligible? + +Likely yes for the current successful subset, but we still need cleaner final summaries +and paper-ready language around: + +- what is core metric drift +- what is bookkeeping drift +- what is acceptable reproducibility tolerance + +## Questions We Still Need To Answer + +These are the main open scientific questions. + +### 1. What exact subset will the paper claim is reproducible? + +We need a final inclusion rule, not an ad hoc list. + +The paper subset should likely require: + +- open-weight model +- local HF execution +- no proprietary judge/annotator dependency +- dataset available without private credentials +- runs that fit on realistic hardware +- at least one official-vs-local comparison +- at least one same-machine repeat +- at least one cross-machine repeat where feasible + +### 2. How many benchmarks and scenarios in the candidate open-weight set are: + +- reproducible +- runnable but not yet reproducible +- not currently runnable under the recipe + +This needs a final table and failure taxonomy with counts, not just anecdotes. + +### 3. How much of the positive result is specific to Vicuna? + +This is still open. + +We need to decide whether the paper makes: + +- a model-specific claim +- a recipe-class claim +- or a broader open-weight HELM claim supported by more than one model family + +### 4. What is the best scientific notion of agreement? + +We already know strict equality is too brittle. + +We still need to settle the paper language for: + +- core-metric exact agreement +- agreement under tolerance sweeps +- benchmark-level reproducibility envelopes +- bookkeeping vs substantive drift + +### 5. What should count as a reproducibility failure versus a recipe-scope exclusion? + +This needs explicit policy language. + +Without this, readers can reasonably misinterpret the large-grid failures. + +### 6. What is the minimal cross-machine evidence needed for a strong claim? + +We likely need a clean answer to: + +- how many machines +- how different the machines need to be +- which benchmarks need cross-machine replication + +### 7. What should the final paper-facing machine labels and hardware descriptions be? + +We now have the infrastructure for this: + +- `dev/experiments/audit-helm-reproduction/configs/paper_label_mappings.yaml` + +But the final wording for figures, tables, and methods still needs to be chosen. + +## Loose Ends That Need To Be Tied Up + +### 1. Finish analysis on the latest external-machine results + +We need final refreshed summaries and top-level rollups that include: + +- `aiq-gpu` vs external-machine pairwise comparison lines +- paper-facing labels +- updated figure layouts + +### 2. Confirm and document missing or incomplete runs explicitly + +Example: + +- `namek` had incomplete coverage for `narrative_qa` at one point + +We need final status accounting so incomplete coverage is not confused with negative evidence. + +### 3. Turn the large-grid failure list into a formal taxonomy table + +Needed outputs: + +- failure categories +- counts +- representative examples +- which categories are exclusion criteria vs genuine reproducibility negatives + +### 4. Define the paper subset formally + +We need one checked-in manifest or selection file that becomes the canonical: + +- reproducibility-ready subset + +This should not live only in our heads or only in generated outputs. + +### 5. Decide whether to expand beyond Vicuna for the final claim + +If yes: + +- select one or more additional open-weight families +- run a smaller targeted grid +- compare whether the positive result generalizes + +If no: + +- narrow the paper claim clearly and honestly to the validated recipe scope + +### 6. Build paper-ready aggregate tables + +We still need tables that are easy to cite in a paper: + +- per-run benchmark table +- per-machine comparison table +- failure taxonomy table +- runnable subset summary table + +### 7. Finalize the methods story + +The paper methods section should clearly explain: + +- public historic bundle used +- local audit execution path +- local deployment override +- same-machine repeat protocol +- cross-machine repeat protocol +- analysis metrics and tolerance sweeps +- interpretation of bookkeeping vs core drift + +### 8. Make sure the latest results and code are restart-safe + +This means: + +- current docs up to date +- canonical paths documented +- commands reproducible +- key generated reports easy to rebuild + +## What Still Needs To Be Accomplished For Strong Claims + +To make strong and defensible claims, we should complete the following sequence. + +### Phase 1. Lock the subset and evidence base + +- finalize the reproducibility-ready benchmark subset +- finish the pending external-machine runs +- ensure each claimed benchmark has the needed comparison types + +Desired output: + +- a clear list of benchmark/model pairs that the paper will center + +### Phase 2. Quantify scope + +- produce counts for: + - candidate open-weight runs + - runnable runs + - successfully reproduced runs + - excluded runs by failure category + +Desired output: + +- a paper table and figure showing scope and exclusions + +### Phase 3. Quantify drift + +- summarize: + - same-machine repeatability + - cross-machine repeatability + - official-vs-local reproducibility + +Desired output: + +- benchmark-level and collection-level drift summaries +- agreement curves or summary statistics appropriate for a paper + +### Phase 4. Generalize or narrow honestly + +Choose one: + +- broaden support by validating more open-weight families +- or keep the claim narrow and precise around the validated Vicuna no-chat subset + +Desired output: + +- final claim wording that matches the evidence exactly + +### Phase 5. Write the paper narrative + +The likely narrative arc is: + +1. naive reproduction can fail for avoidable recipe reasons +2. once recipe confounders are fixed, a meaningful open-weight HELM subset becomes reproducible +3. reproducibility should be analyzed as bounded drift, not just exact equality +4. many apparent failures in broad sweeps are actually scope failures, not evidence against reproducibility + +## Proposed Research Plan From Here + +### Short-term plan + +1. Finish refreshing the latest experiment summaries and overall aggregates. +2. Produce a final large-grid failure taxonomy with counts. +3. Define the canonical paper subset manifest and commit it. +4. Create one paper-facing summary table for: + - same-machine + - cross-machine + - official-vs-local +5. Decide whether we need one additional model family for external validity. + +### Medium-term plan + +1. Run any last targeted experiments needed to close coverage gaps. +2. Freeze the methods and inclusion criteria. +3. Draft the core result section and the failure-taxonomy section in parallel. + +### Paper-writing plan + +The analysis should be organized around three claims: + +1. Positive claim: + a defined open-weight HELM subset is reproducible under a corrected local recipe. +2. Measurement claim: + reproducibility is better characterized by core-metric agreement and bounded drift than by exact equality. +3. Scope claim: + many failures in broad HELM sweeps are exclusions of the current recipe, not direct evidence of irreproducibility. + +## Infrastructure / Repository Split Note + +This experiment is likely to be separated into its own repository so it can be +independent from `magnet`. + +That split should preserve two categories of work: + +### 1. Audit-specific logic that belongs in the new repo + +Examples: + +- manifest generation +- run indexing +- pairwise comparison reports +- aggregate reporting +- failure taxonomy analysis +- paper-facing labeling and plotting utilities + +### 2. General-purpose changes that should be kept and pushed upstream from `magnet` + +Examples already relevant: + +- `magnet.backends.helm.cli.materialize_helm_run` + - support for `model_deployments_fpath` + - robust handling of optional CLI placeholder values + - provenance capture via `ProcessContext` + - preservation of useful materialization metadata/logging behavior + +These changes are not merely one-off experiment hacks. They improve the general +reproducibility and observability of HELM materialization and should not be lost +when the audit code is extracted. + +## Practical Restart Checklist + +If starting fresh: + +1. Read: + - `dev/codex/helm-reproduction-agent-brief.md` + - `dev/codex/helm-reproduction-status-checkpoint.md` + - `dev/codex/helm-reproduction-research-journal.md` +2. Inspect: + - `dev/experiments/audit-helm-reproduction/README.md` +3. Confirm data roots: + - `/data/crfm-helm-public` + - `/data/crfm-helm-audit` +4. Rebuild the latest reports from: + - `dev/experiments/audit-helm-reproduction/scripts/index_results.sh` + - `dev/experiments/audit-helm-reproduction/scripts/analyze_experiment_from_index.sh` + - `dev/experiments/audit-helm-reproduction/scripts/aggregate_core_reports.sh` +5. Review the latest experiment and aggregate summaries in: + - `dev/experiments/audit-helm-reproduction/reports/` +6. Continue from the short-term plan above. + +## Bottom Line + +The project is no longer at the stage of asking whether anything works at all. + +We already have a real positive reproducibility result for an open-weight HELM subset. + +The next stage is to: + +- formalize the subset +- quantify scope and exclusions +- tighten the drift analysis +- decide how broad the final claim should be +- separate the audit project cleanly from `magnet` without losing the upstream-worthy infrastructure improvements diff --git a/dev/experiments/audit-helm-reproduction/python/analyze_experiment_from_index.py b/dev/experiments/audit-helm-reproduction/python/analyze_experiment_from_index.py index 2b7e787..1731c27 100644 --- a/dev/experiments/audit-helm-reproduction/python/analyze_experiment_from_index.py +++ b/dev/experiments/audit-helm-reproduction/python/analyze_experiment_from_index.py @@ -1,6 +1,7 @@ from __future__ import annotations import argparse +import csv import datetime as datetime_mod import json import os @@ -12,6 +13,7 @@ from aggregate_core_reports import _find_curve_value, _find_pair, _write_latest_alias from common import audit_root, default_report_root, env_defaults +from paper_labels import load_paper_label_manager from rebuild_core_report_from_index import latest_index_csv, load_rows, slugify @@ -19,6 +21,73 @@ def _load_json(fpath: Path) -> dict[str, Any]: return json.loads(fpath.read_text()) +def _coerce_float(value: Any) -> float: + try: + return float(value) + except Exception: + return float("-inf") + + +def _is_truthy_text(value: Any) -> bool: + return str(value).strip().lower() in {"true", "1", "yes"} + + +def _latest_matching_aiq_gpu_row( + rows: list[dict[str, Any]], + *, + run_entry: str, + exclude_experiment_name: str, +) -> dict[str, Any] | None: + candidates = [] + for row in rows: + if row.get("run_entry") != run_entry: + continue + if row.get("experiment_name") == exclude_experiment_name: + continue + if row.get("machine_host") != "aiq-gpu": + continue + if row.get("status") not in {"computed", "reused", "unknown", ""}: + continue + if not _is_truthy_text(row.get("has_run_spec")): + continue + run_dir = row.get("run_dir") + if not run_dir: + continue + candidates.append(row) + if not candidates: + return None + candidates.sort( + key=lambda r: ( + _coerce_float(r.get("manifest_timestamp")), + str(r.get("experiment_name") or ""), + str(r.get("job_id") or ""), + ), + reverse=True, + ) + return candidates[0] + + +def _latest_pair_report(report_dpath: Path) -> tuple[Path | None, Path | None]: + json_cands = sorted(report_dpath.glob("pair_report_*.json"), reverse=True) + txt_cands = sorted(report_dpath.glob("pair_report_*.txt"), reverse=True) + return ( + json_cands[0] if json_cands else None, + txt_cands[0] if txt_cands else None, + ) + + +def _write_latest_pair_aliases(report_dpath: Path) -> dict[str, str]: + json_fpath, txt_fpath = _latest_pair_report(report_dpath) + created: dict[str, str] = {} + if json_fpath is not None: + _write_latest_alias(json_fpath, report_dpath, "pair_report.latest.json") + created["pair_report.latest.json"] = str(report_dpath / "pair_report.latest.json") + if txt_fpath is not None: + _write_latest_alias(txt_fpath, report_dpath, "pair_report.latest.txt") + created["pair_report.latest.txt"] = str(report_dpath / "pair_report.latest.txt") + return created + + def main() -> None: parser = argparse.ArgumentParser() parser.add_argument('--experiment-name', required=True) @@ -45,6 +114,7 @@ def main() -> None: rebuild_script = audit_root() / 'python' / 'rebuild_core_report_from_index.py' built_report_paths = [] + skipped_run_entries: list[dict[str, Any]] = [] for run_entry in run_entries: report_dpath = reports_dpath / f'core-metrics-{slugify(run_entry)}' cmd = [ @@ -57,7 +127,15 @@ def main() -> None: ] if args.allow_single_repeat: cmd.append('--allow-single-repeat') - subprocess.run(cmd, check=True) + try: + subprocess.run(cmd, check=True) + except subprocess.CalledProcessError as ex: + skipped_run_entries.append({ + 'run_entry': run_entry, + 'reason': 'rebuild_failed', + 'returncode': ex.returncode, + }) + continue built_report_paths.append(report_dpath / 'core_metric_report.latest.json') summary_rows = [] @@ -65,12 +143,16 @@ def main() -> None: if not report_json.exists(): continue report = _load_json(report_json) + report_dir = report_json.parent + selection_fpath = report_dir / 'report_selection.latest.json' + selection = _load_json(selection_fpath) if selection_fpath.exists() else {} repeat = _find_pair(report, 'kwdagger_repeat') official = _find_pair(report, 'official_vs_kwdagger') summary_rows.append({ 'experiment_name': args.experiment_name, 'run_spec_name': report.get('run_spec_name'), - 'report_dir': str(report_json.parent), + 'run_entry': selection.get('run_entry'), + 'report_dir': str(report_dir), 'generated_utc': report.get('generated_utc'), 'diagnostic_flags': report.get('diagnostic_flags', []), 'kwdagger_a_empty_completion_rate': (((report.get('run_diagnostics') or {}).get('kwdagger_a') or {}).get('empty_completion_rate')), @@ -86,6 +168,83 @@ def main() -> None: 'official_runlevel_max': (((official.get('run_level') or {}).get('overall_quantiles') or {}).get('abs_delta') or {}).get('max'), }) + compare_pair_script = audit_root() / 'python' / 'compare_pair.py' + paper_labels = load_paper_label_manager(style='paper_short') + summary_by_run_spec = { + row['run_spec_name']: row + for row in summary_rows + if row.get('run_spec_name') + } + cross_machine_rows: list[dict[str, Any]] = [] + for run_spec_name, summary_row in summary_by_run_spec.items(): + run_entry = summary_row.get('run_entry') + if not run_entry: + continue + experiment_match = next( + ( + row for row in experiment_rows + if row.get('run_entry') == run_entry and _is_truthy_text(row.get('has_run_spec')) and row.get('run_dir') + ), + None, + ) + if experiment_match is None: + continue + aiq_gpu_match = _latest_matching_aiq_gpu_row( + rows, + run_entry=run_entry, + exclude_experiment_name=args.experiment_name, + ) + if aiq_gpu_match is None: + continue + machine_a = 'aiq-gpu' + machine_b = str(experiment_match.get('machine_host') or args.experiment_name) + display_label_a = paper_labels.machine_label(machine_a) + display_label_b = paper_labels.machine_label(machine_b) + cross_report_dpath = Path(summary_row['report_dir']) / 'cross-machine-aiq-gpu' + cross_report_dpath.mkdir(parents=True, exist_ok=True) + compare_cmd = [ + env_defaults()['AIQ_PYTHON'], + str(compare_pair_script), + '--run-a', str(aiq_gpu_match['run_dir']), + '--run-b', str(experiment_match['run_dir']), + '--label-a', machine_a, + '--label-b', machine_b, + '--display-label-a', display_label_a, + '--display-label-b', display_label_b, + '--report-dpath', str(cross_report_dpath), + ] + subprocess.run(compare_cmd, check=True) + latest_links = _write_latest_pair_aliases(cross_report_dpath) + cross_json_fpath = cross_report_dpath / 'pair_report.latest.json' + cross_txt_fpath = cross_report_dpath / 'pair_report.latest.txt' + cross_payload = _load_json(cross_json_fpath) if cross_json_fpath.exists() else {} + strict = cross_payload.get('strict_summary', {}) or {} + cross_diag = (strict.get('diagnosis', {}) or {}) + cross_overall = ((strict.get('value_agreement', {}) or {}).get('overall', {}) or {}) + cross_means = ((strict.get('instance_value_agreement', {}) or {}).get('means', {}) or {}) + row = { + 'run_spec_name': run_spec_name, + 'run_entry': run_entry, + 'machine_a': machine_a, + 'machine_b': machine_b, + 'machine_a_display': display_label_a, + 'machine_b_display': display_label_b, + 'report_dir': str(cross_report_dpath), + 'report_json': str(cross_json_fpath) if cross_json_fpath.exists() else None, + 'report_txt': str(cross_txt_fpath) if cross_txt_fpath.exists() else None, + 'diagnosis_label': cross_diag.get('label'), + 'primary_reason_names': cross_diag.get('primary_reason_names'), + 'run_level_agree_ratio': cross_overall.get('agree_ratio'), + 'instance_level_agree_ratio': cross_means.get('agree_ratio'), + 'run_level_abs_p90': ((((cross_payload.get('distance_summary') or {}).get('run_level') or {}).get('overall') or {}).get('abs_delta') or {}).get('p90'), + 'run_level_abs_max': ((((cross_payload.get('distance_summary') or {}).get('run_level') or {}).get('overall') or {}).get('abs_delta') or {}).get('max'), + 'instance_level_abs_p90': ((((cross_payload.get('distance_summary') or {}).get('instance_level') or {}).get('overall') or {}).get('abs_delta') or {}).get('p90'), + 'instance_level_abs_max': ((((cross_payload.get('distance_summary') or {}).get('instance_level') or {}).get('overall') or {}).get('abs_delta') or {}).get('max'), + 'latest_links': latest_links, + } + summary_row['cross_machine_aiq_gpu'] = row + cross_machine_rows.append(row) + stamp = datetime_mod.datetime.now(datetime_mod.UTC).strftime('%Y%m%dT%H%M%SZ') history_dpath = out_dpath / '.history' / stamp[:8] history_dpath.mkdir(parents=True, exist_ok=True) @@ -100,7 +259,11 @@ def main() -> None: 'experiment_name': args.experiment_name, 'index_fpath': str(index_fpath), 'n_run_entries': len(run_entries), + 'n_built_reports': len(summary_rows), + 'n_skipped_run_entries': len(skipped_run_entries), 'run_entries': run_entries, + 'skipped_run_entries': skipped_run_entries, + 'cross_machine_rows': cross_machine_rows, 'rows': summary_rows, } json_fpath.write_text(json.dumps(payload, indent=2)) @@ -113,11 +276,39 @@ def main() -> None: lines.append(f'experiment_name: {args.experiment_name}') lines.append(f'index_fpath: {index_fpath}') lines.append(f'n_run_entries: {len(run_entries)}') + lines.append(f'n_built_reports: {len(summary_rows)}') + lines.append(f'n_skipped_run_entries: {len(skipped_run_entries)}') lines.append('') lines.append('run_entries:') for run_entry in run_entries: lines.append(f' - {run_entry}') + if skipped_run_entries: + lines.append('') + lines.append('skipped_run_entries:') + for item in skipped_run_entries: + lines.append(f" - run_entry: {item['run_entry']}") + lines.append(f" reason: {item['reason']}") + lines.append(f" returncode: {item['returncode']}") lines.append('') + if cross_machine_rows: + lines.append('cross_machine_aiq_gpu:') + for row in cross_machine_rows: + lines.append(f" - run_spec_name: {row['run_spec_name']}") + lines.append(f" machine_a: {row['machine_a']}") + lines.append(f" machine_a_display: {row['machine_a_display']}") + lines.append(f" machine_b: {row['machine_b']}") + lines.append(f" machine_b_display: {row['machine_b_display']}") + lines.append(f" report_dir: {row['report_dir']}") + lines.append(f" report_txt: {row['report_txt']}") + lines.append(f" diagnosis_label: {row['diagnosis_label']}") + lines.append(f" primary_reason_names: {row['primary_reason_names']}") + lines.append(f" run_level_agree_ratio: {row['run_level_agree_ratio']}") + lines.append(f" instance_level_agree_ratio: {row['instance_level_agree_ratio']}") + lines.append(f" run_level_abs_p90: {row['run_level_abs_p90']}") + lines.append(f" run_level_abs_max: {row['run_level_abs_max']}") + lines.append(f" instance_level_abs_p90: {row['instance_level_abs_p90']}") + lines.append(f" instance_level_abs_max: {row['instance_level_abs_max']}") + lines.append('') lines.append('per_run_spec:') for row in summary_rows: lines.append(f" - run_spec_name: {row['run_spec_name']}") diff --git a/dev/experiments/audit-helm-reproduction/python/compare_pair.py b/dev/experiments/audit-helm-reproduction/python/compare_pair.py index a1bc6a2..7a3c53b 100644 --- a/dev/experiments/audit-helm-reproduction/python/compare_pair.py +++ b/dev/experiments/audit-helm-reproduction/python/compare_pair.py @@ -74,6 +74,9 @@ def write_text_report(report: dict[str, Any], out_fpath: Path) -> None: run_dist = report.get('distance_summary', {}).get('run_level', {}) or {} inst_dist = report.get('distance_summary', {}).get('instance_level', {}) or {} sweep_hits = report.get('tolerance_highlights', {}) or {} + display = report.get('display_labels', {}) or {} + label_a = display.get('label_a') or report.get('inputs', {}).get('label_a') + label_b = display.get('label_b') or report.get('inputs', {}).get('label_b') lines = [] lines.append('Audit Pair Comparison') @@ -81,6 +84,8 @@ def write_text_report(report: dict[str, Any], out_fpath: Path) -> None: lines.append(f"generated_utc: {report.get('generated_utc')}") lines.append(f"run_a: {report.get('inputs', {}).get('run_a')}") lines.append(f"run_b: {report.get('inputs', {}).get('run_b')}") + lines.append(f'label_a: {label_a}') + lines.append(f'label_b: {label_b}') lines.append('') lines.append(f"diagnosis_label: {diag.get('label')}") lines.append(f"primary_reason_names: {diag.get('primary_reason_names')}") @@ -124,6 +129,8 @@ def main() -> None: parser.add_argument('--run-b', required=True) parser.add_argument('--label-a', default='A') parser.add_argument('--label-b', default='B') + parser.add_argument('--display-label-a', default=None) + parser.add_argument('--display-label-b', default=None) parser.add_argument('--report-dpath', required=True) parser.add_argument('--run-tolerances-yaml', default=None) parser.add_argument('--instance-tolerances-yaml', default=None) @@ -162,6 +169,10 @@ def main() -> None: 'label_a': args.label_a, 'label_b': args.label_b, }, + 'display_labels': { + 'label_a': args.display_label_a or args.label_a, + 'label_b': args.display_label_b or args.label_b, + }, 'strict_summary': strict_summary, 'distance_summary': distance_summary, 'tolerance_sweep': tolerance_sweep, diff --git a/dev/experiments/audit-helm-reproduction/python/core_metric_report.py b/dev/experiments/audit-helm-reproduction/python/core_metric_report.py index dd635b1..bf8824d 100644 --- a/dev/experiments/audit-helm-reproduction/python/core_metric_report.py +++ b/dev/experiments/audit-helm-reproduction/python/core_metric_report.py @@ -19,6 +19,7 @@ from magnet.backends.helm.helm_run_analysis import HelmRunAnalysis from magnet.backends.helm.helm_run_diff import HelmRunDiff from magnet.backends.helm.util import helm_metrics +from paper_labels import load_paper_label_manager def _quantile(values: list[float], q: float) -> float | None: @@ -77,6 +78,40 @@ def _load_json(fpath: Path) -> Any: return json.loads(fpath.read_text()) +def _load_optional_cross_machine_pair(report_dpath: Path) -> dict[str, Any] | None: + pair_fpath = report_dpath / 'cross-machine-aiq-gpu' / 'pair_report.latest.json' + if not pair_fpath.exists(): + return None + data = _load_json(pair_fpath) + display = data.get('display_labels', {}) or {} + label_a = ( + display.get('label_a') + or ((data.get('inputs') or {}).get('label_a')) + or 'aiq-gpu' + ) + label_b = ( + display.get('label_b') + or ((data.get('inputs') or {}).get('label_b')) + or 'other-machine' + ) + highlights = data.get('tolerance_highlights', {}) or {} + distance = data.get('distance_summary', {}) or {} + strict = data.get('strict_summary', {}) or {} + diagnosis = (strict.get('diagnosis') or {}) + return { + 'label': f'{label_a}_vs_{label_b}', + 'diagnosis': diagnosis, + 'run_level': { + 'agreement_vs_abs_tol': highlights.get('run_level', []) or [], + 'overall_quantiles': (distance.get('run_level') or {}).get('overall', {}) or {}, + }, + 'instance_level': { + 'agreement_vs_abs_tol': highlights.get('instance_level', []) or [], + 'overall_quantiles': (distance.get('instance_level') or {}).get('overall', {}) or {}, + }, + } + + def _collect_stat_means(stats: list[dict[str, Any]], metric_name: str) -> dict[str, float]: found = {} for row in stats: @@ -338,9 +373,11 @@ def _build_pair(run_a: str, run_b: str, label: str, thresholds: list[float]) -> } -def _agreement_curve_rows(pair_a: dict[str, Any], pair_b: dict[str, Any], level_key: str) -> list[dict[str, Any]]: +def _agreement_curve_rows(*pairs: dict[str, Any], level_key: str) -> list[dict[str, Any]]: rows = [] - for pair in [pair_a, pair_b]: + for pair in pairs: + if not pair: + continue for row in pair[level_key]['agreement_vs_abs_tol']: rows.append({ 'pair': pair['label'], @@ -350,8 +387,8 @@ def _agreement_curve_rows(pair_a: dict[str, Any], pair_b: dict[str, Any], level_ return rows -def _plot_distribution(ax, pair_a: dict[str, Any], pair_b: dict[str, Any], level_key: str) -> None: - rows = pd.DataFrame(_agreement_curve_rows(pair_a, pair_b, level_key)) +def _plot_distribution(ax, *pairs: dict[str, Any], level_key: str) -> None: + rows = pd.DataFrame(_agreement_curve_rows(*pairs, level_key=level_key)) sns.lineplot( ax=ax, data=rows, @@ -367,6 +404,7 @@ def _plot_distribution(ax, pair_a: dict[str, Any], pair_b: dict[str, Any], level ax.set_ylim(0, 1.02) ax.set_xlabel('Absolute Tolerance Threshold for Core Metric Difference') ax.set_ylabel('Fraction of Core Metric Comparisons in Agreement') + ax.tick_params(axis='x', rotation=28) ax.legend(title='') @@ -382,7 +420,7 @@ def _plot_quantiles(ax, pair_a: dict[str, Any], pair_b: dict[str, Any], level_ke ax.set_title(title) ax.set_xlabel('Quantile') ax.set_ylabel('Absolute Difference in Core Metric Value') - ax.legend() + ax.legend(title='') def _distribution_rows(pair: dict[str, Any]) -> pd.DataFrame: @@ -908,39 +946,42 @@ def main() -> None: _write_text(report, txt_fpath) _write_management_summary(report, mgmt_fpath) + extra_pair = _load_optional_cross_machine_pair(report_dpath) + paper_labels = load_paper_label_manager(style='paper_short') + extra_label = extra_pair['label'] if extra_pair is not None else None + pair_line = f'Pairs: {left["label"]} vs {right["label"]}' + if extra_label is not None: + pair_line += f' + {extra_label}' + pair_line = paper_labels.relabel_text(pair_line) sns.set_theme(style='whitegrid', context='talk') - fig, axes = plt.subplots(2, 2, figsize=(15, 10), constrained_layout=True) + fig, axes = plt.subplots(2, 2, figsize=(24, 14.5), constrained_layout=True) _plot_quantiles( axes[0, 0], left, right, 'run_level', - f"Run-Level Core Metric Difference Quantiles\nRun Spec: {run_spec_name}\nN={left['run_level']['n_rows']} vs N={right['run_level']['n_rows']}" + 'Run-Level Delta Quantiles' ) _plot_quantiles( axes[0, 1], left, right, 'instance_level', - f"Instance-Level Core Metric Difference Quantiles\nRun Spec: {run_spec_name}\nN={left['instance_level']['n_rows']} vs N={right['instance_level']['n_rows']}" - ) - _plot_distribution(axes[1, 0], left, right, 'run_level') - axes[1, 0].set_title( - f"Run-Level Core Metric Agreement vs Tolerance\n" - f"Run Spec: {run_spec_name}\n" - f"{left['label']} N={left['run_level']['n_rows']}, {right['label']} N={right['run_level']['n_rows']}" - ) - _plot_distribution(axes[1, 1], left, right, 'instance_level') - axes[1, 1].set_title( - f"Instance-Level Core Metric Agreement vs Tolerance\n" - f"Run Spec: {run_spec_name}\n" - f"{left['label']} N={left['instance_level']['n_rows']}, {right['label']} N={right['instance_level']['n_rows']}" + 'Instance-Level Delta Quantiles' ) + _plot_distribution(axes[1, 0], left, right, extra_pair, level_key='run_level') + axes[1, 0].set_title('Run-Level Agreement vs Tolerance', fontsize=11) + _plot_distribution(axes[1, 1], left, right, extra_pair, level_key='instance_level') + axes[1, 1].set_title('Instance-Level Agreement vs Tolerance', fontsize=11) + axes[0, 0].title.set_fontsize(11) + axes[0, 1].title.set_fontsize(11) fig.suptitle( 'Core Metric Agreement and Difference Summary\n' f'Run Spec: {run_spec_name}\n' - 'Comparing kwdagger repeatability against official-vs-kwdagger divergence.', - fontsize=18, + f'{pair_line}\n' + f'Run-level N: {left["run_level"]["n_rows"]} vs {right["run_level"]["n_rows"]} | ' + f'Instance-level N: {left["instance_level"]["n_rows"]} vs {right["instance_level"]["n_rows"]}', + fontsize=15, ) fig.savefig(fig_fpath, dpi=180) plt.close(fig) diff --git a/dev/experiments/audit-helm-reproduction/python/paper_labels.py b/dev/experiments/audit-helm-reproduction/python/paper_labels.py new file mode 100644 index 0000000..26566b5 --- /dev/null +++ b/dev/experiments/audit-helm-reproduction/python/paper_labels.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +import kwutil + +from common import audit_root + + +def paper_label_config_fpath() -> Path: + return audit_root() / 'configs' / 'paper_label_mappings.yaml' + + +def _load_config() -> dict[str, Any]: + fpath = paper_label_config_fpath() + if not fpath.exists(): + return {} + data = kwutil.Yaml.load(fpath) + if not isinstance(data, dict): + raise TypeError(f'Expected mapping in {fpath}') + return data + + +class PaperLabelManager: + """ + Lightweight relabel helper inspired by kwplot.managers.LabelManager. + + This keeps a single checked-in mapping file for paper-facing labels, while + still letting reports preserve the raw internal machine codes. + """ + + def __init__(self, style: str = 'paper_short'): + self.style = style + self.config = _load_config() + self.machine_map = self.config.get('machine_host', {}) or {} + + def machine_label(self, machine_host: str | None, *, fallback: str | None = None) -> str: + raw = str(machine_host or fallback or 'unknown') + info = self.machine_map.get(raw, {}) or {} + return str(info.get(self.style) or raw) + + def relabel_text(self, text: str | None) -> str | None: + if text is None: + return None + new_text = str(text) + keys = sorted(self.machine_map.keys(), key=len, reverse=True) + for key in keys: + repl = self.machine_label(key) + new_text = new_text.replace(str(key), repl) + return new_text + + +def load_paper_label_manager(style: str = 'paper_short') -> PaperLabelManager: + return PaperLabelManager(style=style) diff --git a/dev/experiments/audit-helm-reproduction/python/rebuild_all_core_reports_from_index.py b/dev/experiments/audit-helm-reproduction/python/rebuild_all_core_reports_from_index.py index 5ca28c1..f7634dd 100644 --- a/dev/experiments/audit-helm-reproduction/python/rebuild_all_core_reports_from_index.py +++ b/dev/experiments/audit-helm-reproduction/python/rebuild_all_core_reports_from_index.py @@ -29,6 +29,9 @@ def main() -> None: for run_entry in run_entries: matches = matching_rows(rows, run_entry) n = len(matches) + if n == 0: + skipped.append((run_entry, n, 'no_indexed_runs')) + continue if n < 2 and not args.allow_single_repeat: skipped.append((run_entry, n, 'not_enough_matching_runs')) continue From 24e5d0e87a48ef9a35ce28f80b516b2fb9e45a2c Mon Sep 17 00:00:00 2001 From: agent Date: Tue, 31 Mar 2026 21:54:29 +0000 Subject: [PATCH 35/36] Remove files ported to a new repo --- dev/codex/helm-reproduction-agent-brief.md | 332 ----- .../helm-reproduction-research-journal.md | 569 --------- .../helm-reproduction-status-checkpoint.md | 514 -------- dev/codex/kwdagger-notes.md | 30 - dev/codex/reproduce-helm-session-v2.md | 432 ------- .../audit-helm-reproduction/README.md | 786 ------------ .../configs/apples_manifest.yaml | 23 - .../debug/vicuna_no_chat_template.yaml | 9 - .../configs/manifest_template.yaml | 18 - .../configs/model_deployments_template.yaml | 11 - .../configs/smoke_manifest.yaml | 23 - .../examples/example_overnight_commands.sh | 24 - .../examples/example_smoke_commands.sh | 9 - .../python/aggregate_core_reports.py | 175 --- .../python/analyze_experiment_from_index.py | 340 ------ .../python/build_repro_manifest.py | 275 ----- .../audit-helm-reproduction/python/common.py | 68 -- .../python/compare_batch.py | 644 ---------- .../python/compare_pair.py | 192 --- .../python/core_metric_report.py | 1017 ---------------- .../python/index_results.py | 201 --- .../python/inspect_pair_samples.py | 94 -- .../python/make_manifest.py | 221 ---- .../python/metric_quantiles_report.py | 181 --- .../python/paper_labels.py | 55 - .../rebuild_all_core_reports_from_index.py | 68 -- .../python/rebuild_core_report_from_index.py | 242 ---- .../python/render_schedule_params.py | 79 -- .../python/resolve_run.py | 134 -- .../scripts/aggregate_core_reports.sh | 11 - .../scripts/analyze_experiment_from_index.sh | 9 - .../scripts/check_env.sh | 58 - .../audit-helm-reproduction/scripts/common.sh | 31 - .../scripts/compare_batch.sh | 13 - .../scripts/compare_entry.sh | 62 - .../scripts/compare_pair.sh | 16 - .../scripts/core_metric_report.sh | 9 - .../scripts/index_results.sh | 16 - .../scripts/inspect_pair_samples.sh | 9 - .../scripts/make_apples_manifest.sh | 22 - .../scripts/make_historic_grid_manifest.sh | 19 - .../scripts/make_machine_shard_manifest.sh | 29 - .../scripts/make_machine_subset_manifest.sh | 23 - .../scripts/make_repeat_pair_manifests.sh | 47 - .../scripts/make_smoke_manifest.sh | 20 - .../scripts/make_vicuna_nochat_manifest.sh | 19 - .../scripts/metric_quantiles_report.sh | 9 - .../rebuild_all_core_reports_from_index.sh | 11 - .../scripts/rebuild_core_report_from_index.sh | 12 - .../scripts/resolve_run.sh | 9 - .../scripts/run_from_manifest.sh | 54 - .../scripts/run_smoke.sh | 16 - dev/oneoff/check_results_overlap.py | 653 ---------- dev/oneoff/diagnose_reproducibility.py | 1080 ----------------- dev/poc/inspect_historic_helm_runs.py | 327 ----- 55 files changed, 9350 deletions(-) delete mode 100644 dev/codex/helm-reproduction-agent-brief.md delete mode 100644 dev/codex/helm-reproduction-research-journal.md delete mode 100644 dev/codex/helm-reproduction-status-checkpoint.md delete mode 100644 dev/codex/kwdagger-notes.md delete mode 100644 dev/codex/reproduce-helm-session-v2.md delete mode 100644 dev/experiments/audit-helm-reproduction/README.md delete mode 100644 dev/experiments/audit-helm-reproduction/configs/apples_manifest.yaml delete mode 100644 dev/experiments/audit-helm-reproduction/configs/debug/vicuna_no_chat_template.yaml delete mode 100644 dev/experiments/audit-helm-reproduction/configs/manifest_template.yaml delete mode 100644 dev/experiments/audit-helm-reproduction/configs/model_deployments_template.yaml delete mode 100644 dev/experiments/audit-helm-reproduction/configs/smoke_manifest.yaml delete mode 100755 dev/experiments/audit-helm-reproduction/examples/example_overnight_commands.sh delete mode 100755 dev/experiments/audit-helm-reproduction/examples/example_smoke_commands.sh delete mode 100644 dev/experiments/audit-helm-reproduction/python/aggregate_core_reports.py delete mode 100644 dev/experiments/audit-helm-reproduction/python/analyze_experiment_from_index.py delete mode 100644 dev/experiments/audit-helm-reproduction/python/build_repro_manifest.py delete mode 100644 dev/experiments/audit-helm-reproduction/python/common.py delete mode 100644 dev/experiments/audit-helm-reproduction/python/compare_batch.py delete mode 100644 dev/experiments/audit-helm-reproduction/python/compare_pair.py delete mode 100644 dev/experiments/audit-helm-reproduction/python/core_metric_report.py delete mode 100644 dev/experiments/audit-helm-reproduction/python/index_results.py delete mode 100644 dev/experiments/audit-helm-reproduction/python/inspect_pair_samples.py delete mode 100644 dev/experiments/audit-helm-reproduction/python/make_manifest.py delete mode 100644 dev/experiments/audit-helm-reproduction/python/metric_quantiles_report.py delete mode 100644 dev/experiments/audit-helm-reproduction/python/paper_labels.py delete mode 100644 dev/experiments/audit-helm-reproduction/python/rebuild_all_core_reports_from_index.py delete mode 100644 dev/experiments/audit-helm-reproduction/python/rebuild_core_report_from_index.py delete mode 100644 dev/experiments/audit-helm-reproduction/python/render_schedule_params.py delete mode 100644 dev/experiments/audit-helm-reproduction/python/resolve_run.py delete mode 100755 dev/experiments/audit-helm-reproduction/scripts/aggregate_core_reports.sh delete mode 100755 dev/experiments/audit-helm-reproduction/scripts/analyze_experiment_from_index.sh delete mode 100755 dev/experiments/audit-helm-reproduction/scripts/check_env.sh delete mode 100755 dev/experiments/audit-helm-reproduction/scripts/common.sh delete mode 100755 dev/experiments/audit-helm-reproduction/scripts/compare_batch.sh delete mode 100755 dev/experiments/audit-helm-reproduction/scripts/compare_entry.sh delete mode 100755 dev/experiments/audit-helm-reproduction/scripts/compare_pair.sh delete mode 100755 dev/experiments/audit-helm-reproduction/scripts/core_metric_report.sh delete mode 100755 dev/experiments/audit-helm-reproduction/scripts/index_results.sh delete mode 100755 dev/experiments/audit-helm-reproduction/scripts/inspect_pair_samples.sh delete mode 100755 dev/experiments/audit-helm-reproduction/scripts/make_apples_manifest.sh delete mode 100755 dev/experiments/audit-helm-reproduction/scripts/make_historic_grid_manifest.sh delete mode 100755 dev/experiments/audit-helm-reproduction/scripts/make_machine_shard_manifest.sh delete mode 100755 dev/experiments/audit-helm-reproduction/scripts/make_machine_subset_manifest.sh delete mode 100755 dev/experiments/audit-helm-reproduction/scripts/make_repeat_pair_manifests.sh delete mode 100755 dev/experiments/audit-helm-reproduction/scripts/make_smoke_manifest.sh delete mode 100755 dev/experiments/audit-helm-reproduction/scripts/make_vicuna_nochat_manifest.sh delete mode 100755 dev/experiments/audit-helm-reproduction/scripts/metric_quantiles_report.sh delete mode 100755 dev/experiments/audit-helm-reproduction/scripts/rebuild_all_core_reports_from_index.sh delete mode 100755 dev/experiments/audit-helm-reproduction/scripts/rebuild_core_report_from_index.sh delete mode 100755 dev/experiments/audit-helm-reproduction/scripts/resolve_run.sh delete mode 100755 dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh delete mode 100755 dev/experiments/audit-helm-reproduction/scripts/run_smoke.sh delete mode 100644 dev/oneoff/check_results_overlap.py delete mode 100644 dev/oneoff/diagnose_reproducibility.py delete mode 100644 dev/poc/inspect_historic_helm_runs.py diff --git a/dev/codex/helm-reproduction-agent-brief.md b/dev/codex/helm-reproduction-agent-brief.md deleted file mode 100644 index b12c2f6..0000000 --- a/dev/codex/helm-reproduction-agent-brief.md +++ /dev/null @@ -1,332 +0,0 @@ -# HELM Reproduction Agent Brief - -Date: 2026-03-30 -Workspace: `/home/joncrall/code/aiq-magnet` - -## Mission - -Prepare a fresh agent to continue the HELM reproducibility project quickly and safely. - -The overarching goal is: - -- determine how much of public HELM is independently reproducible with local open-weight execution -- quantify the remaining differences rather than treating reproducibility as a binary -- build a paper-quality methodology around: - - same-machine repeatability - - cross-machine repeatability - - official-vs-local drift - - benchmark-level and collection-level summaries - -For publication framing, the likely paper story is: - -- a positive result for a carefully defined runnable subset of HELM -- with quantified bounds on unavoidable residual differences -- plus a clear failure taxonomy for cases that are not currently runnable or not reproducible under the open local recipe - -## Key Principle - -Do **not** interpret every failed large-grid job as a reproducibility failure. - -Many failures so far are recipe-scope failures: - -- gated datasets -- missing external credentials -- dataset download failures -- model packaging issues -- scenarios requiring proprietary annotators - -Those belong in the "not currently runnable under the recipe" bucket, not the "HELM is irreproducible" bucket. - -## Current Scientific Conclusion - -The strongest current positive signal is the corrected Vicuna no-chat recipe. - -For: - -- `boolq:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical` -- `mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical` -- `narrative_qa:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical` - -once `apply_chat_template: false` is used for the local HF deployment: - -- same-machine reruns are stable -- cross-machine reruns are stable -- official-vs-local agreement on core metrics is excellent -- remaining drift is mostly output length/style rather than correctness - -This is the current best evidence that a substantial subset of HELM is independently reproducible. - -## Root Cause Already Found - -There was a real false alarm earlier. - -Problem: - -- local HF Vicuna runs were auto-applying chat templating -- this caused severe empty-completion pathologies, especially on `narrative_qa` - -Fix: - -- use: - - `dev/experiments/audit-helm-reproduction/configs/debug/vicuna_no_chat_template.yaml` - -Effect: - -- empty completions disappeared -- core metrics became close or identical to official public HELM - -This is a critical lesson: - -- local execution-path bugs can masquerade as reproducibility failures -- any strong irreproducibility claim should be preceded by recipe validation diagnostics - -## Important Paths - -Public historic bundle: - -- `/data/crfm-helm-public` - -Local reproduced runs: - -- `/data/crfm-helm-audit` - -Main audit workflow: - -- `dev/experiments/audit-helm-reproduction` - -Research notes: - -- `dev/codex/helm-reproduction-research-journal.md` -- `dev/codex/reproduce-helm-session-v2.md` -- `dev/codex/kwdagger-notes.md` - -## Operational State Of Recent Experiments - -### 1. `audit-vicuna-nochat-overnight` - -This is the fixed-config baseline on the main machine. - -Status: - -- successful -- core reproducibility result is positive - -### 2. `audit-vicuna-nochat-server` - -Initial attempt: - -- failed due to GPU occupancy / OOM -- logs captured this clearly - -Later successful attempt: - -- matched the earlier fixed Vicuna result -- supports positive cross-machine reproducibility - -### 3. `audit-namek-subset` - -Subset intended for consumer-ish single-GPU cross-hardware checks. - -Observed: - -- BoolQ: complete -- MMLU: complete -- NarrativeQA: incomplete / hanging at time of inspection - -Completed jobs match the same positive Vicuna no-chat pattern. - -### 4. `audit-yardrat-subset` - -Observed: - -- all 3 target jobs completed -- terminal impression of failure was misleading -- copied artifacts show valid `DONE` files and healthy logs - -Scientific interpretation: - -- Yardrat supports the same positive cross-machine result as the other successful Vicuna no-chat runs - -### 5. `audit-historic-grid` - -Large broad candidate sweep built from `run_specs.yaml` / `run_details.yaml`. - -Observed: - -- total jobs: `286` -- completed jobs: `141` -- failed or incomplete: `145` - -This grid is useful mainly for failure taxonomy and runnable-subset refinement. - -## Large Grid Failure Taxonomy - -Representative buckets already observed: - -### Dataset / scenario access issues - -- RAFT: - - requires `trust_remote_code=True` -- MATH: - - dataset resolution failure for `hendrycks/competition_math` -- bAbI: - - failed on direct `wget` download -- Natural Questions: - - failed on direct `wget` download - -### Gated / credentialed cases - -- XSTest: - - gated dataset -- annotation-heavy runs: - - may require OpenAI credentials - -### Model packaging / environment issues - -- Sea Lion: - - missing dynamic import files such as `flash_attn_triton.py` - -### Important interpretation - -These are not currently evidence against reproducibility. - -They are evidence that the current candidate suite is too broad and must be filtered to a reproducibility-ready subset. - -## Runnable Recipe We Should Move Toward - -The reproducibility recipe should prioritize: - -- open-weight models -- local HF execution path -- models that fit on realistic consumer-accessible hardware - - single large consumer GPU - - or modest multi-GPU workstation -- datasets available without private credentials -- scenarios not requiring proprietary judge/annotator APIs -- configurations we can archive and rerun deterministically enough for audit purposes - -In practice this likely means: - -- do not center the paper on the entire raw HELM catalog -- instead define a reproducibility-ready subset with explicit inclusion criteria - -## Existing Tooling Worth Reusing - -### Historic candidate generation - -- `dev/poc/inspect_historic_helm_runs.py` -- outputs: - - `run_specs.yaml` - - `run_details.yaml` - -### Audit workflow - -- `dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh` -- `dev/experiments/audit-helm-reproduction/scripts/compare_batch.sh` -- `dev/experiments/audit-helm-reproduction/scripts/index_results.sh` -- `dev/experiments/audit-helm-reproduction/scripts/analyze_experiment_from_index.sh` -- `dev/experiments/audit-helm-reproduction/scripts/aggregate_core_reports.sh` - -### Manifest building - -- `dev/experiments/audit-helm-reproduction/scripts/make_historic_grid_manifest.sh` -- `dev/experiments/audit-helm-reproduction/scripts/make_machine_subset_manifest.sh` -- `dev/experiments/audit-helm-reproduction/scripts/make_machine_shard_manifest.sh` - -### Important note - -If running the analysis tooling on this machine, prefer: - -- `/home/joncrall/code/aiq-magnet/.venv/bin/python` - -because the default `python` here may not have `helm` installed. - -## Reporting / Analysis State - -The audit folder already contains: - -- per-run-spec core metric reports -- management summaries -- threshold-vs-agreement plots -- raw score distribution plots -- ECDF plots -- run-level metric tables -- instance sample reports -- experiment-level summaries -- overall reproducibility rollups - -Important conceptual improvement already adopted: - -- focus on reproducibility distributions / envelopes, not single exact-equality claims - -## What A Fresh Agent Should Do Next - -### 1. Curate a reproducibility-ready subset - -Turn `run_specs.yaml` into a filtered suite that excludes: - -- gated datasets -- credentialed annotator cases -- obviously broken download paths -- models with known packaging failures - -This should become the main benchmark set for the paper. - -### 2. Quantify the distribution of differences - -For the filtered subset: - -- same-machine repeatability -- cross-machine repeatability -- official-vs-local drift - -Summaries should include: - -- per-instance agreement -- run-level metric deltas -- benchmark-family rollups -- model-family rollups - -### 3. Keep correctness and style drift separate - -Important current pattern: - -- core correctness metrics often match exactly or nearly exactly -- output length can still differ substantially - -This distinction should be central in the paper. - -### 4. Make the runnable recipe explicit - -Document: - -- required configs -- model overrides -- environment assumptions -- hardware assumptions -- known excluded benchmark/model families and why they are excluded - -### 5. Continue failure taxonomy - -For the broad grid: - -- classify each failure into: - - unrunnable dataset - - missing credential - - model packaging/runtime issue - - true reproduction drift - - unknown - -This can become a useful table or appendix. - -## Current Bottom Line - -The project has moved from: - -- "maybe HELM is not reproducible" - -to something more precise: - -- a meaningful subset of HELM appears reproducible under a corrected local open-weight recipe -- many apparent failures are recipe or environment issues, not scientific counterexamples -- the next paper-quality step is to formalize the subset and quantify the reproducibility envelope diff --git a/dev/codex/helm-reproduction-research-journal.md b/dev/codex/helm-reproduction-research-journal.md deleted file mode 100644 index 1d1b56c..0000000 --- a/dev/codex/helm-reproduction-research-journal.md +++ /dev/null @@ -1,569 +0,0 @@ -# HELM Reproduction Research Journal - -Date started: 2026-03-27 -Project: `/home/joncrall/code/aiq-magnet` -Scope: reproduce historic public HELM results with local `kwdagger` pipelines, explain observed drift, and build reporting that supports both management summaries and maintainer-grade technical diagnosis. - -## Research Goal - -Determine whether current locally reproduced HELM runs meaningfully differ from official public HELM results, and if so: - -- quantify how different they are -- separate ordinary rerun noise from structural drift -- identify likely causes such as deployment/provider changes, metric-spec changes, and execution-spec changes - -## Data Sources - -- Public historic bundle: - - `/data/crfm-helm-public` -- Local audit runs: - - `/data/crfm-helm-audit` -- Audit experiment workflow: - - `dev/experiments/audit-helm-reproduction` - -## Workflow Built During This Research - -- Created a reusable audit experiment folder under: - - `dev/experiments/audit-helm-reproduction` -- Added shell-first operator scripts for: - - environment validation - - manifest generation - - scheduling runs with `kwdagger` - - comparing completed batches to public HELM runs - - direct pairwise comparison of two concrete run directories -- Added pairwise reporting support that writes: - - JSON reports - - text summaries - - tolerance sweeps - -## Current Direction Shift - -We likely do not need more same-hardware repeat runs right now. - -- The current evidence is already strong that repeated `kwdagger` runs on the - same machine are very stable relative to the official-vs-local gap. -- The more interesting next source of variation is now cross-hardware / - cross-machine drift. -- That means future repeatability work should prioritize: - - different GPU architectures - - different VRAM tiers - - possibly different host environments - -This should reduce time spent waiting on redundant same-machine reruns while -improving the causal story for a paper-quality reproducibility analysis. - -## Important Audit / kwdagger Notes - -- `kwdagger` is the right execution mechanism for these experiments, but it has a few behaviors worth remembering: - - unset optional params may still be rendered into generated CLI commands - - list-valued params are best passed as YAML strings, not `nargs='*'` - - tmux queue name collisions can cause interactive prompts unless queue names are experiment-specific -- Pairwise and entry-based comparison tooling now validates finalized HELM run artifacts before attempting diffs. - - This avoids misleading Python tracebacks when a job exists but its run directory is incomplete. - -See also: -- `dev/codex/kwdagger-notes.md` -- `dev/codex/reproduce-helm-session-v2.md` - -## Provenance / Multi-Machine Goal - -Current status before this note: - -- historical audit results mostly recorded logical run config -- they did **not** explicitly record machine / GPU provenance in a structured, - analysis-friendly way - -This is a problem if we want to: - -- merge runs produced on different machines -- compare same-config runs across hardware -- avoid blocking on one machine just to obtain intermediate analyses - -Implementation direction: - -- record lightweight process provenance per materialized HELM job -- use `kwutil.ProcessContext` to capture: - - host - - user - - cwd - - OS / Python info - - memory / CPU metadata when available -- augment that with best-effort `nvidia-smi` GPU details when available - -Desired downstream behavior: - -- pairwise / aggregate analysis should work against whatever runs exist -- missing runs should reduce coverage, not block intermediate analysis -- when cross-machine drift appears, we should be able to attribute it to a - known machine / GPU context rather than reconstructing provenance manually - -Open goal: - -- teach the audit reporting layer to group / filter by machine provenance once - enough multi-machine runs exist - -## Deployment / Registry Findings - -- Some target models already have built-in HELM Hugging Face deployments: - - `eleutherai/pythia-6.9b` - - `lmsys/vicuna-7b-v1.3` - - `aisingapore/llama3-8b-cpt-sea-lionv2.1-instruct` -- Some newer or different families appear Together-backed in the built-in registry: - - example: `meta/llama-3-8b-chat` -- HELM’s experimental Hugging Face registration flags are not reliable enough by themselves for this workflow. - - robust reproduction work should prefer explicit deployment control, e.g. via custom `model_deployments.yaml` - -## Apples-To-Apples Finding - -The original smoke batch was useful as a workflow check, but not apples-to-apples because public matches were often at `max_eval_instances=1000` while the reproduced smoke runs used `100`. - -We then ran an apples-to-apples control batch aligned to `max_eval_instances=1000`. - -Key result: - -- the eval-size mismatch confounder was removed -- the remaining observed drift still persisted - -That means the disagreement is not explained solely by comparing `100` local examples to `1000` historic examples. - -## Representative Structural Drift - -For apples-to-apples cases, the main remaining execution-path difference was: - -- `adapter_spec.model_deployment` - -Representative examples showed: - -- historic public runs often record: - - `adapter_spec.model_deployment = null` -- local reproduced runs often record: - - `adapter_spec.model_deployment = huggingface/...` - -Metric spec drift was also observed: - -- historic often uses: - - `helm.benchmark.metrics.basic_metrics.BasicMetric` -- local reproduced runs often use: - - `BasicGenerationMetric` - - `BasicReferenceMetric` - - `InstancesPerSplitMetric` - -This is important because it means the disagreement is not just at the output level. The effective evaluation configuration is also changing. - -## Pairwise Comparison Baseline - -To separate structural drift from normal stochastic variation, repeated local runs were compared directly. - -Case studied in detail: - -- `boolq:model=eleutherai/pythia-6.9b,data_augmentation=canonical` - -Compared: - -- local repeat 1: - - `/data/crfm-helm-audit/audit-boolq-pythia-r1/helm/helm_id_13jkx9mm4k4n/benchmark_output/runs/audit-boolq-pythia-r1/boolq:model=eleutherai_pythia-6.9b,data_augmentation=canonical` -- local repeat 2: - - `/data/crfm-helm-audit/audit-boolq-pythia-r2/helm/helm_id_12jr5w48kge7/benchmark_output/runs/audit-boolq-pythia-r2/boolq:model=eleutherai_pythia-6.9b,data_augmentation=canonical` -- official public HELM: - - `/data/crfm-helm-public/classic/benchmark_output/runs/v0.3.0/boolq:model=eleutherai_pythia-6.9b,data_augmentation=canonical` - -### Local Repeatability Result - -`r1` vs `r2`: - -- diagnosis: - - `bookkeeping_metric_drift` -- strict run-level agree ratio: - - `0.9552238805970149` -- strict instance-level agree ratio: - - `0.9523809523809523` -- run-level max abs delta: - - `0.0015118565559387176` -- instance-level max abs delta: - - `0.4418189525604248` - -Interpretation: - -- local reruns are very close -- residual differences are mostly bookkeeping/runtime style noise, not large task-quality drift - -### Official vs Local Result - -Official `v0.3.0` vs local `r1`: - -- diagnosis: - - `multiple_primary_reasons` -- primary reason names: - - `deployment_drift` - - `execution_spec_drift` -- strict run-level agree ratio: - - `0.4626865671641791` -- strict instance-level agree ratio: - - `0.6577333333333333` -- run-level abs p90: - - `4.0` -- run-level abs max: - - `11.985` -- instance-level abs p90: - - `4.0` -- instance-level abs max: - - `75.73884344100952` - -Interpretation: - -- official-vs-local drift is much larger than local-vs-local drift -- that makes it very unlikely that ordinary nondeterminism is the main explanation - -## High-Level Verdict - -For the tested BoolQ/Pythia apples-to-apples case, we now have enough information to say: - -- the current locally reproduced HELM result is significantly different from the official public HELM result -- the difference is much larger than the observed local repeatability noise -- the most likely explanation is structural/configurational drift rather than ordinary stochastic rerun variance - -Important limitation: - -- this claim is solid for the tested case -- it should not automatically be generalized to all HELM benchmarks or model families without more cases - -## Tolerance Sweep Findings - -The pairwise tool supports tolerance sweeps across several preset thresholds. - -Current presets: - -- `strict`: `abs_tol=0.0`, `rel_tol=0.0` -- `tiny`: `abs_tol=1e-12`, `rel_tol=1e-6` -- `small`: `abs_tol=1e-9`, `rel_tol=1e-4` -- `medium`: `abs_tol=1e-6`, `rel_tol=1e-3` -- `loose`: `abs_tol=1e-3`, `rel_tol=1e-2` -- `xloose`: `abs_tol=1e-2`, `rel_tol=1e-1` -- `xxloose`: `abs_tol=1e-1`, `rel_tol=1.0` -- `extreme`: `abs_tol=1.0`, `rel_tol=10.0` - -For local BoolQ/Pythia repeats: - -- similarity becomes nearly perfect by `loose` / `xloose` - -For official vs local BoolQ/Pythia: - -- similarity stays much lower through `loose` and `xloose` -- it only collapses to `1.0` at the very permissive `xxloose` threshold - -Interpretation: - -- official-vs-local mismatch is robust to modest tolerance relaxation -- forcing them to look identical requires very permissive tolerances - -## Important Metric-Scale Caveat - -A single global absolute tolerance is not easy to interpret because the comparison spans different metric families with very different numeric scales. - -Observed for official vs local BoolQ/Pythia: - -- `core` metrics are approximately `[0, 1]` - - examples: - - `exact_match` - - `prefix_exact_match` - - `quasi_exact_match` - - run-level max abs delta in this class was only about: - - `0.021` -- `bookkeeping` metrics are not bounded to `[0, 1]` - - examples: - - `num_bytes` - - `num_output_tokens` - - `logprob` - - observed values included: - - `num_bytes`: `15.432` vs `3.448` - - per-instance `num_bytes`: `16.0` vs `3.0` - - per-instance `num_output_tokens`: `5.0` vs `1.0` - - per-instance `logprob`: values like `-3.64` vs `0.0` - -Implication: - -- `abs_tol=1.0` is extremely loose for bounded core metrics -- but it is not necessarily huge for bookkeeping metrics such as byte counts or token counts - -Therefore: - -- global tolerance sweeps are still useful as diagnostics -- but interpretation should move toward: - - per metric class tolerances - - and eventually per metric family tolerances - -## Current Reporting Files Worth Inspecting - -- Session journal: - - `dev/codex/reproduce-helm-session-v2.md` -- kwdagger notes: - - `dev/codex/kwdagger-notes.md` -- Pairwise report, local repeat: - - `dev/experiments/audit-helm-reproduction/reports/pairwise/boolq-pythia-repeat-wide/pair_report_20260327T011202Z.txt` -- Pairwise report, official vs local: - - `dev/experiments/audit-helm-reproduction/reports/pairwise/boolq-pythia-historic-wide/pair_report_20260327T011202Z.txt` - -## Recommended Next Steps - -- Add class-specific tolerance reporting: - - at least separate `core` from `bookkeeping` -- Add effect-size style summaries: - - compare official-vs-local distance to local-vs-local baseline distance -- Collect more repeat runs if formal significance estimates are desired - - current data is enough for a strong effect-size-style argument - - it is not enough for a stable p-value estimate -- Run a minimal Together-backed control on a representative case - - this should help separate provider/deployment effects from general HELM evolution - -## Direct NarrativeQA/Vicuna Debug Run - -We ran a focused local debug job outside kwdagger scheduling: - -- suite: `debug-narrative-vicuna-direct` -- run entry: `narrative_qa:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical` -- max_eval_instances: `20` - -Main result: - -- The direct run reproduces the same failure mode as the larger local NarrativeQA/Vicuna runs. -- This strongly argues against the issue being a kwdagger scheduling/orchestration bug. - -Observed from the raw run: - -- request count: `100` -- empty completions: `99` -- non-empty completions: `1` -- mean output token count: `0.13` -- token count histogram: - - `0`: `99` - - `13`: `1` - -Observed from `stats.json`: - -- `num_completion_tokens` mean on `test`: `0.0` -- `num_output_tokens` mean on `test`: `0.0` -- `finish_reason_unknown` mean on `test`: `1.0` -- `exact_match`, `quasi_exact_match`, `f1_score`, `rouge_l`: all `0.0` - -Relevant log clues: - -- HELM automatically set `apply_chat_template=True` -- HELM removed 4 in-context examples to fit the context window -- HELM logged stop/truncation warnings: - - `truncate_sequence needs to strip "\\n"` - - `truncate_sequence needs to strip ""` - -Current interpretation: - -- Most likely this is a local HELM Hugging Face Vicuna execution/configuration issue. -- The strongest current suspect is chat-template application on a non-chat-style NarrativeQA prompt. -- Secondary suspects are newline stop-sequence handling and/or immediate EOS/empty-generation behavior in the HF Vicuna path. - -Current follow-up experiment: - -- rerun the same benchmark with a custom `model_deployments.yaml` override that sets: - - `apply_chat_template: false` - -## NarrativeQA/Vicuna Root Cause Update - -The `apply_chat_template: false` rerun strongly supports a root-cause diagnosis. - -Run: - -- suite: `debug-narrative-vicuna-nochat` -- same benchmark/model family as the failing debug run -- same local Hugging Face deployment name -- overridden deployment config: - - `client_spec.args.apply_chat_template: false` - -Observed: - -- request count: `500` -- empty completions: `0` -- non-empty completions: `500` -- mean output token count: `12.894` - -Aggregate test metrics from the corrected local run: - -- `exact_match`: `0.2727` -- `quasi_exact_match`: `0.4026` -- `f1_score`: `0.6422` -- `rouge_l`: `0.6442` -- `bleu_1`: `0.5138` -- `bleu_4`: `0.0722` - -These are now close to the official public HELM run for the same benchmark/model pair. - -Conclusion: - -- The prior NarrativeQA/Vicuna failure was **not** good evidence of irreproducibility. -- It was caused by a local HELM/HuggingFace configuration issue. -- The main culprit appears to be automatic chat-template application for this run. - -Implications for the audit: - -- For local Hugging Face reproductions, `apply_chat_template` must be treated as an explicit controlled setting. -- Some earlier "failed reproductions" may need to be reinterpreted or rerun if they depended on HELM's automatic chat-template inference. -- The audit/reporting system should surface suspicious signals such as: - - high empty-completion rate - - near-zero `num_output_tokens` - - pervasive `finish_reason_unknown` - -## Server Batch Logging Validation - -The latest rsynced server-side experiment artifacts confirmed that the new per-job HELM logging capture is working as intended. - -Observed in: - -- `/data/crfm-helm-audit/audit-vicuna-nochat-server/helm/helm_id_dijo03bfux6g/` -- `/data/crfm-helm-audit/audit-vicuna-nochat-server/helm/helm_id_obr7gu9kxuql/` -- `/data/crfm-helm-audit/audit-vicuna-nochat-server/helm/helm_id_s2ez33ko97jb/` - -Each failed job now preserves: - -- `helm-run.log` -- `helm-run.debug.log` -- `process_context.json` -- `helm_log_config.yaml` -- `job_config.json` - -These logs show that the latest server failures were not new reproducibility failures. All three jobs failed during local Hugging Face model load with the same infrastructure error: - -- `torch.OutOfMemoryError` -- attempted allocation: about `172 MiB` -- GPU `0` free memory at failure: about `143.50 MiB` -- competing process `696468` already using about `89.43 GiB` - -Interpretation: - -- the `audit-vicuna-nochat-server` run failed due to GPU occupancy / infrastructure contention -- this does **not** currently count as evidence for or against HELM reproducibility -- the new logging path is useful and should remain part of all future materialized job outputs - -## Cross-Machine Vicuna No-Chat Confirmation - -After clearing the server-side GPU occupancy issue, the `audit-vicuna-nochat-server` batch completed successfully and matched the earlier `audit-vicuna-nochat-overnight` batch. - -High-level result: - -- for the three fixed-config Vicuna runs we tested, the server results were numerically identical to the earlier run on the other machine at the run-level metric summaries we inspected -- this is a meaningful positive signal for cross-machine reproducibility once the chat-template confounder is removed - -Benchmarks: - -- `boolq:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical` -- `mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical` -- `narrative_qa:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical` - -Observed diagnostics from the successful server run: - -- BoolQ: - - `N = 5000` - - empty completions: `0` - - mean output tokens: `1.0064` -- MMLU: - - `N = 327` - - empty completions: `0` - - mean output tokens: `1.0` -- NarrativeQA: - - `N = 2350` - - empty completions: `0` - - mean output tokens: `11.9498` - - output token quantiles: - - `p50 = 6` - - `p90 = 26` - - `max = 100` - -Comparison to official public HELM: - -- BoolQ correctness metrics match exactly; the remaining visible drift is shorter output length -- MMLU core metrics match exactly -- NarrativeQA remains very close to official on core metrics, with the main residual difference in answer length rather than answer quality - -Interpretation: - -- the fixed Vicuna/HuggingFace path appears independently reproducible on the tested core metrics -- at least for these experiments, the newer server run supports a positive cross-machine reproducibility result rather than revealing a hardware-specific failure mode - -## Updated Research Goal - -The project goal should now be stated more precisely: - -- produce a documented recipe for reproducing selected public HELM runs in a repeatable way -- quantify and bound the small remaining differences that are expected from local execution, hardware variation, and nondeterminism -- prefer benchmarks, models, and evaluation paths that are runnable with open weights and realistic "consumer-accessible" hardware constraints - -Interpretation: - -- this does not necessarily mean laptop-only reproduction -- but it should avoid closed-weight models and giant model families that are not meaningfully checkable by other researchers -- a practical target is open-weight models that can run on a single large consumer GPU or on a modest multi-GPU workstation - -Methodologically, this pushes us toward a reproducibility distribution rather than a single point estimate: - -- same-machine repeatability -- cross-machine repeatability -- official-vs-local drift -- benchmark-level and collection-level summaries - -## Large Grid Status: Runnable Recipe vs Unrunnable Cases - -The larger `audit-historic-grid` batch was informative, but many failures do **not** currently count as evidence against reproducibility. - -Observed: - -- total jobs: `286` -- completed jobs: `141` -- failed or incomplete jobs: `145` - -Representative failure buckets: - -- dataset loading / access / download issues - - RAFT requires `trust_remote_code=True` - - MATH failed to resolve `hendrycks/competition_math` - - bAbI and Natural Questions failed on direct data downloads via `wget` -- gated or external-service requirements - - `walledai/XSTest` is gated - - some annotation-heavy runs require OpenAI credentials -- model-specific packaging issues - - `aisingapore/sea-lion-7b-instruct` failed while resolving dynamic imports such as `flash_attn_triton.py` -- some broader model families remain insufficiently diagnosed from the raw logs alone and need targeted follow-up - -Interpretation: - -- the current historic candidate list is too broad to serve directly as a reproducibility benchmark suite -- we need a stricter "runnable recipe" filter: - - open-weight models - - locally runnable HF path - - datasets that are accessible without custom private credentials - - scenarios that do not require external proprietary annotators - - configurations that fit the hardware budget we expect others to have - -## Namek / Yardrat Subset Check - -The small cross-hardware Vicuna subset remains encouraging. - -Observed: - -- `audit-yardrat-subset` - - BoolQ: complete - - MMLU: complete - - NarrativeQA: complete -- `audit-namek-subset` - - BoolQ: complete - - MMLU: complete - - NarrativeQA: incomplete / still-running at time of inspection - -For the completed subset jobs: - -- no empty-completion pathology was observed -- Yardrat matched the previously successful Vicuna no-chat behavior: - - BoolQ core correctness metrics matched official exactly - - MMLU core metrics matched official exactly - - NarrativeQA remained very close on core metrics, with residual drift mainly in output length rather than correctness -- Namek BoolQ and MMLU also matched this same pattern - -Interpretation: - -- the corrected Vicuna recipe appears stable across multiple machines -- this strengthens the case that our main reproducibility result is positive when the configuration is correct and the task is runnable under the local open-weight recipe diff --git a/dev/codex/helm-reproduction-status-checkpoint.md b/dev/codex/helm-reproduction-status-checkpoint.md deleted file mode 100644 index f168f44..0000000 --- a/dev/codex/helm-reproduction-status-checkpoint.md +++ /dev/null @@ -1,514 +0,0 @@ -# HELM Reproduction Status Checkpoint - -Date: 2026-03-30 -Workspace: `/home/joncrall/code/aiq-magnet` - -## Purpose - -This document is a restart-safe checkpoint for the HELM reproduction project. - -It is meant to answer: - -- what we are trying to prove -- what we have already established -- what still needs to be done before we can make strong publication-quality claims -- what infrastructure work must be preserved as this project is separated from `magnet` - -This document is intentionally redundant with other notes, but should be the best -single place to restart from if prior chat context is lost. - -## Related Documents - -Read these first for deeper context and provenance: - -- `dev/codex/helm-reproduction-agent-brief.md` -- `dev/codex/helm-reproduction-research-journal.md` -- `dev/experiments/audit-helm-reproduction/README.md` -- `dev/codex/reproduce-helm-session-v2.md` -- `dev/codex/kwdagger-notes.md` - -Important code and experiment roots: - -- public historic HELM runs: `/data/crfm-helm-public` -- local reproduced runs: `/data/crfm-helm-audit` -- audit workflow: `dev/experiments/audit-helm-reproduction` - -## Core Research Goal - -The real paper question is not: - -- "can we reproduce every HELM run exactly?" - -The better question is: - -- "to what extent are public HELM results independently reproducible under an explicit open-weight local recipe, and how should residual differences be interpreted?" - -For a strong paper, we need a precise answer to four sub-questions: - -1. Same-machine repeatability: - how stable are local reruns under the same recipe? -2. Cross-machine repeatability: - how stable are local reruns across different GPU hardware and host environments? -3. Official-vs-local reproducibility: - when we compare to historic public HELM, how close are we on core benchmark outcomes? -4. Scope: - for what subset of HELM can these comparisons be run fairly under an open-weight local recipe? - -## Current Headline Conclusion - -The strongest current result is positive: - -- a corrected local Vicuna no-chat recipe reproduces a small but meaningful HELM subset very well -- this holds on multiple machines -- the remaining cross-machine differences are largely bookkeeping-style drift, not core-metric failure - -The best current positive evidence is for: - -- `boolq:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical` -- `mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical` -- `narrative_qa:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical` - -with: - -- same-machine repeatability -- cross-machine repeatability -- excellent official-vs-local agreement on core metrics - -This is the current basis for the paper framing: - -- open-weight HELM reproduction is possible for a clearly defined subset -- exact equality is too strong a target -- core-metric agreement and bounded drift are the right scientific objects - -## Critical Resolved Issue - -An earlier apparent failure was real in the sense that the outputs were bad, but -it was not evidence against reproducibility. - -Root cause: - -- the local HF Vicuna execution path was auto-applying a chat template -- this caused severe empty-completion pathologies, especially on `narrative_qa` - -Fix: - -- use `dev/experiments/audit-helm-reproduction/configs/debug/vicuna_no_chat_template.yaml` -- specifically, the corrected local deployment must use `apply_chat_template: false` - -Scientific lesson: - -- execution-path bugs can masquerade as irreproducibility -- any claim of irreproducibility must be preceded by recipe validation diagnostics - -## Questions We Have Answered Already - -### 1. Are exact mismatches always evidence of irreproducibility? - -No. - -We have already shown that: - -- some mismatches were caused by local recipe errors -- some mismatches are bookkeeping/runtime drift -- some large-grid failures are simply "not runnable under the current recipe" - -### 2. Is the corrected Vicuna local recipe repeatable on the same machine? - -Yes, for the current evaluated subset. - -The corrected Vicuna no-chat runs show strong same-machine repeatability on the -current core tasks. - -### 3. Is the corrected Vicuna local recipe repeatable across machines? - -Yes, for the tested subset. - -Current evidence supports positive cross-machine reproducibility across: - -- the internal main machine -- `namek` -- `yardrat` - -with the strongest evidence currently on: - -- BoolQ -- MMLU `us_foreign_policy` -- NarrativeQA - -### 4. Are cross-machine differences currently dominated by benchmark failure? - -No, not for the successful Vicuna no-chat subset. - -The current pairwise cross-machine reports are being diagnosed mainly as: - -- `bookkeeping_metric_drift` - -rather than core-metric collapse. - -### 5. Are many large-grid failures actually recipe-scope failures rather than negative reproducibility evidence? - -Yes. - -Observed buckets include: - -- gated datasets -- dataset download failures -- scenarios needing external credentials or proprietary annotators -- model packaging issues - -These belong in a runnable-scope taxonomy, not in a blanket "HELM is irreproducible" claim. - -### 6. Is apples-to-apples alignment on `max_eval_instances` enough by itself to explain the old official-vs-local drift? - -No. - -We already controlled this confounder, and substantial drift still persisted in -the older Pythia-style cases. - -### 7. Is local-vs-local noise generally much smaller than the big official-vs-local failures seen in the older cases? - -Yes. - -That earlier result remains important: - -- local repeat noise exists -- but the older structural official-vs-local failures were much larger than ordinary rerun variance - -This is part of the causal story for why "recipe-corrected subset succeeds" is a -real scientific result rather than noise. - -## Questions We Can Partially Answer But Not Yet Claim Broadly - -### 1. How broad is the reproducible open-weight subset? - -Partial answer: - -- at least the current corrected Vicuna subset is reproducible on key tasks - -Not yet established: - -- how far this extends across more benchmarks -- how far this extends across more model families -- whether we can support a broad claim beyond this subset - -### 2. Is reproducibility achievable on realistic consumer hardware? - -Partial answer: - -- yes for some tested runs on single- or modest-multi-GPU machines - -Not yet established: - -- the final size and diversity of the consumer-hardware reproducible subset -- a rigorous paper-ready inclusion criterion for "consumer-accessible" - -### 3. Are the remaining cross-machine differences scientifically negligible? - -Likely yes for the current successful subset, but we still need cleaner final summaries -and paper-ready language around: - -- what is core metric drift -- what is bookkeeping drift -- what is acceptable reproducibility tolerance - -## Questions We Still Need To Answer - -These are the main open scientific questions. - -### 1. What exact subset will the paper claim is reproducible? - -We need a final inclusion rule, not an ad hoc list. - -The paper subset should likely require: - -- open-weight model -- local HF execution -- no proprietary judge/annotator dependency -- dataset available without private credentials -- runs that fit on realistic hardware -- at least one official-vs-local comparison -- at least one same-machine repeat -- at least one cross-machine repeat where feasible - -### 2. How many benchmarks and scenarios in the candidate open-weight set are: - -- reproducible -- runnable but not yet reproducible -- not currently runnable under the recipe - -This needs a final table and failure taxonomy with counts, not just anecdotes. - -### 3. How much of the positive result is specific to Vicuna? - -This is still open. - -We need to decide whether the paper makes: - -- a model-specific claim -- a recipe-class claim -- or a broader open-weight HELM claim supported by more than one model family - -### 4. What is the best scientific notion of agreement? - -We already know strict equality is too brittle. - -We still need to settle the paper language for: - -- core-metric exact agreement -- agreement under tolerance sweeps -- benchmark-level reproducibility envelopes -- bookkeeping vs substantive drift - -### 5. What should count as a reproducibility failure versus a recipe-scope exclusion? - -This needs explicit policy language. - -Without this, readers can reasonably misinterpret the large-grid failures. - -### 6. What is the minimal cross-machine evidence needed for a strong claim? - -We likely need a clean answer to: - -- how many machines -- how different the machines need to be -- which benchmarks need cross-machine replication - -### 7. What should the final paper-facing machine labels and hardware descriptions be? - -We now have the infrastructure for this: - -- `dev/experiments/audit-helm-reproduction/configs/paper_label_mappings.yaml` - -But the final wording for figures, tables, and methods still needs to be chosen. - -## Loose Ends That Need To Be Tied Up - -### 1. Finish analysis on the latest external-machine results - -We need final refreshed summaries and top-level rollups that include: - -- `aiq-gpu` vs external-machine pairwise comparison lines -- paper-facing labels -- updated figure layouts - -### 2. Confirm and document missing or incomplete runs explicitly - -Example: - -- `namek` had incomplete coverage for `narrative_qa` at one point - -We need final status accounting so incomplete coverage is not confused with negative evidence. - -### 3. Turn the large-grid failure list into a formal taxonomy table - -Needed outputs: - -- failure categories -- counts -- representative examples -- which categories are exclusion criteria vs genuine reproducibility negatives - -### 4. Define the paper subset formally - -We need one checked-in manifest or selection file that becomes the canonical: - -- reproducibility-ready subset - -This should not live only in our heads or only in generated outputs. - -### 5. Decide whether to expand beyond Vicuna for the final claim - -If yes: - -- select one or more additional open-weight families -- run a smaller targeted grid -- compare whether the positive result generalizes - -If no: - -- narrow the paper claim clearly and honestly to the validated recipe scope - -### 6. Build paper-ready aggregate tables - -We still need tables that are easy to cite in a paper: - -- per-run benchmark table -- per-machine comparison table -- failure taxonomy table -- runnable subset summary table - -### 7. Finalize the methods story - -The paper methods section should clearly explain: - -- public historic bundle used -- local audit execution path -- local deployment override -- same-machine repeat protocol -- cross-machine repeat protocol -- analysis metrics and tolerance sweeps -- interpretation of bookkeeping vs core drift - -### 8. Make sure the latest results and code are restart-safe - -This means: - -- current docs up to date -- canonical paths documented -- commands reproducible -- key generated reports easy to rebuild - -## What Still Needs To Be Accomplished For Strong Claims - -To make strong and defensible claims, we should complete the following sequence. - -### Phase 1. Lock the subset and evidence base - -- finalize the reproducibility-ready benchmark subset -- finish the pending external-machine runs -- ensure each claimed benchmark has the needed comparison types - -Desired output: - -- a clear list of benchmark/model pairs that the paper will center - -### Phase 2. Quantify scope - -- produce counts for: - - candidate open-weight runs - - runnable runs - - successfully reproduced runs - - excluded runs by failure category - -Desired output: - -- a paper table and figure showing scope and exclusions - -### Phase 3. Quantify drift - -- summarize: - - same-machine repeatability - - cross-machine repeatability - - official-vs-local reproducibility - -Desired output: - -- benchmark-level and collection-level drift summaries -- agreement curves or summary statistics appropriate for a paper - -### Phase 4. Generalize or narrow honestly - -Choose one: - -- broaden support by validating more open-weight families -- or keep the claim narrow and precise around the validated Vicuna no-chat subset - -Desired output: - -- final claim wording that matches the evidence exactly - -### Phase 5. Write the paper narrative - -The likely narrative arc is: - -1. naive reproduction can fail for avoidable recipe reasons -2. once recipe confounders are fixed, a meaningful open-weight HELM subset becomes reproducible -3. reproducibility should be analyzed as bounded drift, not just exact equality -4. many apparent failures in broad sweeps are actually scope failures, not evidence against reproducibility - -## Proposed Research Plan From Here - -### Short-term plan - -1. Finish refreshing the latest experiment summaries and overall aggregates. -2. Produce a final large-grid failure taxonomy with counts. -3. Define the canonical paper subset manifest and commit it. -4. Create one paper-facing summary table for: - - same-machine - - cross-machine - - official-vs-local -5. Decide whether we need one additional model family for external validity. - -### Medium-term plan - -1. Run any last targeted experiments needed to close coverage gaps. -2. Freeze the methods and inclusion criteria. -3. Draft the core result section and the failure-taxonomy section in parallel. - -### Paper-writing plan - -The analysis should be organized around three claims: - -1. Positive claim: - a defined open-weight HELM subset is reproducible under a corrected local recipe. -2. Measurement claim: - reproducibility is better characterized by core-metric agreement and bounded drift than by exact equality. -3. Scope claim: - many failures in broad HELM sweeps are exclusions of the current recipe, not direct evidence of irreproducibility. - -## Infrastructure / Repository Split Note - -This experiment is likely to be separated into its own repository so it can be -independent from `magnet`. - -That split should preserve two categories of work: - -### 1. Audit-specific logic that belongs in the new repo - -Examples: - -- manifest generation -- run indexing -- pairwise comparison reports -- aggregate reporting -- failure taxonomy analysis -- paper-facing labeling and plotting utilities - -### 2. General-purpose changes that should be kept and pushed upstream from `magnet` - -Examples already relevant: - -- `magnet.backends.helm.cli.materialize_helm_run` - - support for `model_deployments_fpath` - - robust handling of optional CLI placeholder values - - provenance capture via `ProcessContext` - - preservation of useful materialization metadata/logging behavior - -These changes are not merely one-off experiment hacks. They improve the general -reproducibility and observability of HELM materialization and should not be lost -when the audit code is extracted. - -## Practical Restart Checklist - -If starting fresh: - -1. Read: - - `dev/codex/helm-reproduction-agent-brief.md` - - `dev/codex/helm-reproduction-status-checkpoint.md` - - `dev/codex/helm-reproduction-research-journal.md` -2. Inspect: - - `dev/experiments/audit-helm-reproduction/README.md` -3. Confirm data roots: - - `/data/crfm-helm-public` - - `/data/crfm-helm-audit` -4. Rebuild the latest reports from: - - `dev/experiments/audit-helm-reproduction/scripts/index_results.sh` - - `dev/experiments/audit-helm-reproduction/scripts/analyze_experiment_from_index.sh` - - `dev/experiments/audit-helm-reproduction/scripts/aggregate_core_reports.sh` -5. Review the latest experiment and aggregate summaries in: - - `dev/experiments/audit-helm-reproduction/reports/` -6. Continue from the short-term plan above. - -## Bottom Line - -The project is no longer at the stage of asking whether anything works at all. - -We already have a real positive reproducibility result for an open-weight HELM subset. - -The next stage is to: - -- formalize the subset -- quantify scope and exclusions -- tighten the drift analysis -- decide how broad the final claim should be -- separate the audit project cleanly from `magnet` without losing the upstream-worthy infrastructure improvements diff --git a/dev/codex/kwdagger-notes.md b/dev/codex/kwdagger-notes.md deleted file mode 100644 index fc326bb..0000000 --- a/dev/codex/kwdagger-notes.md +++ /dev/null @@ -1,30 +0,0 @@ -# kwdagger Notes - -Date started: 2026-03-24 -Project: `/home/joncrall/code/aiq-magnet` - -## Important Points - -- `kwdagger schedule` is correctly building the HELM smoke-test DAG as one node per `run_entry`; for the current smoke manifest that means 6 jobs total. -- With `backend=tmux`, `devices=0,1`, and `tmux_workers=2`, the queue is split into two tmux sessions, one per visible GPU, with jobs serialized within each session. -- `kwdagger` may render node defaults into the generated CLI even when the manifest omits them. - - Observed examples: - - `--precomputed_root=None` - - `--model_deployments_fpath=None` - - bare `--enable_huggingface_models` - - bare `--enable_local_huggingface_models` -- Because of that behavior, downstream CLIs should normalize null-like placeholders such as `None`, `null`, and empty strings instead of assuming omitted values stay omitted. -- For the HELM audit node, it is better to omit unset optional params when rendering the final command than to rely only on downstream normalization. - - The custom node `command` property now drops `None` and empty-list values before building the shell command. -- For list-valued params passed through `kwdagger`, prefer a single key/value YAML string over `nargs='*'`. - - Current audit/HELM example: - - `helm.enable_huggingface_models: '["repo-a", "repo-b"]'` - - `helm.enable_local_huggingface_models: '["/models/a"]'` - - The pipeline-facing script can decode that with `kwutil.Yaml.coerce`, then expand it into the downstream CLI format if needed. -- `scriptconfig` emitted a smartcast warning for comma-separated `devices`; this did not block scheduling, but it is a hint that list parsing behavior may change in a future version. -- Existing `cmd_queue` tmux sessions with the same queue name can trigger an interactive prompt asking whether older sessions should be killed. - - This is important for automation and unattended runs. - - Current mitigation in the audit runner: derive `--queue_name` from `experiment_name` instead of reusing the generic `schedule-eval`. -- The audit comparison step needs to discover completed jobs recursively under the kwdagger results root. - - Real layout is nested like: `///DONE` - - A shallow glob like `results/*/DONE` will miss all HELM jobs and incorrectly report `missing_kwdg_match`. diff --git a/dev/codex/reproduce-helm-session-v2.md b/dev/codex/reproduce-helm-session-v2.md deleted file mode 100644 index 43525a8..0000000 --- a/dev/codex/reproduce-helm-session-v2.md +++ /dev/null @@ -1,432 +0,0 @@ -# Reproduce HELM Session V2 - -Date: 2026-03-24 -Workspace: `/home/joncrall/code/aiq-magnet` -Python env: `/home/agent/.local/uv/envs/uvpy3.13.2/bin/python` -HELM installation: assumed available in the Python environment used by `magnet` -Precomputed HELM root: `/data/crfm-helm-public` - -## Task Summary - -Take a second pass at reproducing historic HELM evaluations with the local `kwdagger` pipelines, with a focus on cases where historic runs used Together-hosted model deployments but could plausibly be rerun locally with HELM's Hugging Face client on the available 4 x 96 GB GPUs. - -We also want to preserve useful discoveries here because rate limits are nearly exhausted for this session. - -## Findings So Far - -- The rough notes at the top of `dev/poc/inspect_historic_helm_runs.py` match the earlier workflow: - - generate candidate run specs from `/data/crfm-helm-public` - - filter them - - schedule `magnet.backends.helm.pipeline.helm_single_run_pipeline()` -- `dev/poc/inspect_historic_helm_runs.py` already contains an important filtering heuristic: - - it registers HELM built-in configs - - inspects model metadata and deployments from HELM - - keeps only models whose resolved deployment client is `helm.clients.huggingface_client.HuggingFaceClient` - - it also limits to open-access text models with `num_parameters <= 10e9` -- That means the previous workflow may have avoided explicit Together-to-HuggingFace rewrites in many cases by selecting only models that HELM already knew how to run through a Hugging Face deployment. -- Concrete registry examples from this session: - - `aisingapore/llama3-8b-cpt-sea-lionv2.1-instruct` - - default deployment available in HELM: `huggingface/llama3-8b-cpt-sea-lionv2.1-instruct` - - client: `helm.clients.huggingface_client.HuggingFaceClient` - - `lmsys/vicuna-7b-v1.3` - - default deployment available in HELM: `huggingface/vicuna-7b-v1.3` - - `eleutherai/pythia-6.9b` - - default deployment available in HELM: `huggingface/pythia-6.9b` - - `meta/llama-3-8b-chat` - - built-in default deployment appears to be `together/llama-3-8b-chat` - - no built-in Hugging Face deployment was present in the registry snapshot checked today - -## Historic Together Cases - -- `rg` over `/data/crfm-helm-public/**/run_spec.json` confirms many newer public bundles explicitly record Together deployments, e.g.: - - `together/deepseek-v3` - - `together/qwen2.5-7b-instruct-turbo` - - `together/qwen2.5-72b-instruct-turbo` - - `together/llama-3.2-11b-vision-instruct-turbo` - - `together/llama-3.2-90b-vision-instruct-turbo` - - `together/gpt-oss-20b` -- So the second-pass reproduction work really does need a principled local-deployment override path for some model families. - -## Pipeline / Code Notes - -- `magnet/backends/helm/pipeline.py` defines a thin kwdagger node around: - - `python -m magnet.backends.helm.cli.materialize_helm_run` -- `magnet/backends/helm/cli/materialize_helm_run.py` behavior today: - - tries to reuse a matching run from `precomputed_root` - - otherwise executes: - - `helm-run --run-entries --suite ...` - - no provider override or deployment rewrite option is exposed yet -- Matching of historic runs is based on run-entry token subset matching, with model normalization `/ -> _` -- Reuse search scans discovered `benchmark_output` trees and returns the best matching run directory - -## Immediate Hypothesis - -- If a historic run spec points at a Together-backed deployment, reproducing it locally likely requires one of: - - rewriting the `model=` token in the run-entry to a HELM deployment name that uses `HuggingFaceClient` - - or adding custom `prod_env/model_deployments.yaml` entries so the same logical model can resolve to a local Hugging Face deployment - - or both - -## Important HELM Behavior - -- `helm-run` calls: - - `register_builtin_configs_from_helm_package()` - - then `register_configs_from_directory(args.local_path)` -- Default `--local-path` is `prod_env` -- Because `materialize_helm_run.py` runs `helm-run` with `cwd=out_dpath`, a relative `local_path=prod_env` naturally maps to: - - `/prod_env` -- This gives us a clean way to inject per-job `model_deployments.yaml` overrides without modifying the shared HELM checkout. - -## New Code Support Added This Session - -- Updated `magnet/backends/helm/cli/materialize_helm_run.py` to support: - - `local_path` - - `model_deployments_fpath` - - `enable_huggingface_models` - - `enable_local_huggingface_models` -- Behavior: - - before computing a run, the wrapper now prepares the HELM local config directory - - if `model_deployments_fpath` is given, it copies that file to `/model_deployments.yaml` - - `helm-run` is now invoked with explicit `--local-path` - - optional HELM Hugging Face registration flags are passed through when specified -- Updated `magnet/backends/helm/pipeline.py` so these parameters are available from kwdagger. -- Updated `materialize_helm_run.py` to defensively normalize optional values that may be rendered by `kwdagger` as CLI placeholders. - - Real-world examples observed in generated commands: - - `--precomputed_root=None` - - `--model_deployments_fpath=None` - - bare `--enable_huggingface_models` - - bare `--enable_local_huggingface_models` - - The wrapper now treats null-like placeholders and empty optional list inputs as truly unset. -- For list-shaped params that are directly exposed to `kwdagger`, prefer a single YAML-encoded string value instead of `nargs='*'`. - - This fits kwdagger's key/value param model better. - - The HELM wrapper now decodes those values with `kwutil.Yaml.coerce` before calling `helm-run`. - -## Caveat About HELM's Experimental HuggingFace Registration - -- HELM's `--enable-huggingface-models` is not a universal solution. -- Observed limitations from quick checks: - - `meta/llama-3-8b-chat` - - failed because the repo id is not directly usable as a public HF model id in this environment - - likely also requires Hugging Face auth / correct repo id for gated models - - `eleutherai/pythia-6.9b` - - failed because HELM could not infer `model_max_length` - - `aisingapore/llama3-8b-cpt-sea-lionv2.1-instruct` - - failed for the same `model_max_length` inference reason - - `lmsys/vicuna-7b-v1.3` - - failed during tokenizer conversion in this environment -- Conclusion: - - custom `model_deployments.yaml` overrides are the more robust path for reproduction work - - pass-through support for HELM's flags is still useful for models where they happen to work - -## Likely Reproduction Strategy - -- Prefer models that already have built-in Hugging Face deployments when possible. -- For Together-only model families that are still locally runnable: - - create a custom HELM `model_deployments.yaml` - - point kwdagger jobs at it via `helm.model_deployments_fpath` - - keep the logical `adapter_spec.model` stable where possible, and change deployment resolution rather than renaming the model everywhere - -## Next Checks - -- Inspect a concrete historic run spec from `/data/crfm-helm-public` -- Inspect HELM registry entries for the corresponding model/deployment -- Determine whether a Hugging Face deployment already exists for those model names -- If not, decide whether to extend `materialize_helm_run.py` and/or add custom model deployment registration support - -## kwdagger Observations - -- The smoke-test audit manifest scheduled as expected: - - 6 total jobs - - 2 tmux workers - - `CUDA_VISIBLE_DEVICES` split across GPU 0 and GPU 1 -- For the HELM audit node, overriding the node `command` renderer is the cleanest way to suppress unset optional args. - - This avoids generated commands like: - - `--precomputed_root=None` - - `--model_deployments_fpath=None` - - bare `--enable_huggingface_models` -- `kwdagger` / `cmd_queue` may stop for an interactive prompt if older tmux queue sessions with the same queue name are still running. - - That behavior matters for unattended reproduction runs and should be accounted for in future script refinements. -- The smoke-test comparison is now working end to end for the 6-run control batch. - - Historic rows found: 6 - - kwdagger rows found: 6 - - current diagnosis label across all six cases: `multiple_primary_reasons` - - primary reason counts: - - `deployment_drift`: 6 - - `execution_spec_drift`: 6 - - full reason counts: - - `completion_content_drift`: 6 - - `core_metric_drift`: 6 - - `dataset_instance_drift`: 6 - - `dataset_variant_drift`: 6 - - `deployment_drift`: 6 - - `evaluation_spec_drift`: 6 - - `execution_spec_drift`: 6 - - `request_prompt_drift`: 1 - - Interpretation: - - this smoke batch validates the audit workflow plumbing - - it does not yet validate close behavioral reproduction of the historic runs - - the dominant differences are consistent with local HF execution diverging materially from the historic HELM deployment / execution configuration -- Representative case inspected: - - `mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=eleutherai/pythia-6.9b,data_augmentation=canonical` - - historic run selected by matcher: - - `/data/crfm-helm-public/classic/benchmark_output/runs/v0.2.4/...` - - primary drift: - - `deployment_drift` - - historic `adapter_spec.model_deployment`: `null` - - reproduced `adapter_spec.model_deployment`: `huggingface/pythia-6.9b` - - `execution_spec_drift` - - historic `adapter_spec.max_eval_instances`: `1000` - - reproduced `adapter_spec.max_eval_instances`: `100` - - additional evidence that this is not a close reproduction yet: - - dataset base coverage differs: historic 111 vs reproduced 100 - - variant coverage differs: historic 327 vs reproduced 294 - - metric spec set differs substantially: - - historic uses `BasicMetric` - - reproduced uses `BasicGenerationMetric`, `BasicReferenceMetric`, and `InstancesPerSplitMetric` - - core metric agreement is effectively zero in this case - - `core_agree_ratio: 0.0` - - completion agreement is only partial - - 294 comparable completions - - 28 mismatches - - equal ratio about `0.905` -- Reporting improvements added after inspecting the smoke batch: - - `compare_batch.py` now writes a management-facing text summary in addition to the technical summary and JSON outputs - - summaries now expose: - - `primary_reason_name_counts` - - `reason_counts` - - high-level findings suitable for executive reporting - - historic candidate selection in the compare step is now more explicit and records whether an exact historic `adapter_spec.max_eval_instances` match exists -- Current important finding: - - for the smoke manifest, the public HELM bundle does not appear to contain exact historic matches with requested `max_eval_instances=100` - - available matched public runs for the inspected `pythia-6.9b` MMLU case record `adapter_spec.max_eval_instances=1000` - - this means the current smoke comparison is useful for auditing drift, but it is not yet an apples-to-apples reproduction check on evaluation size -- Apples-to-apples smoke batch (`audit-smoke-apples`) result: - - transferred report bundle shows: - - historic rows found: 6 - - kwdagger rows found: 6 - - all 6 compared successfully - - all 6 cases report `historic_exact_requested_max_eval_match = True` - - the actual run-spec execution-path drift in the transferred case JSONL is now only: - - `adapter_spec.model_deployment` - - primary reasons across all 6 apples-to-apples cases: - - `deployment_drift` - - `execution_spec_drift` - - recurring secondary reasons across all 6 apples-to-apples cases: - - `evaluation_spec_drift` - - `core_metric_drift` - - `completion_content_drift` - - one case also shows: - - `request_prompt_drift` - - Important correction: - - the initial apples executive summary still claimed a requested `max_eval_instances` mismatch in all 6 cases - - inspection of the case JSONL indicates that was a reporting bug from how the reproduced-side value was extracted, not the real comparison outcome - - `compare_batch.py` was updated to read reproduced `run_spec.json` directly for `adapter_spec.max_eval_instances` - -## Apples-To-Apples Report - -Date: 2026-03-25 -Experiment: `audit-smoke-apples` - -### Executive Summary - -- The apples-to-apples smoke batch compared 6 reproduced runs against 6 matched historic public HELM runs. -- All 6 runs were paired and compared successfully. -- After aligning `max_eval_instances` to the historic public bundle (`1000`), the remaining dominant drift is not evaluation size. -- The main recurring technical differences are: - - explicit Hugging Face `model_deployment` in reproduced runs vs `null` deployment in historic runs - - evaluation-spec drift caused by metric class changes - - completion-content drift - - core-metric drift -- This means the audit workflow is functioning correctly, and the remaining mismatch is now focused on how HELM resolves and evaluates these runs, not on the basic scheduling / pairing machinery. - -### High-Level Conclusion - -- The earlier non-apples smoke batch was confounded by comparing reproduced `max_eval_instances=100` runs against historic `max_eval_instances=1000` runs. -- The apples-to-apples batch removes that confounder. -- In the apples batch, the real remaining execution drift appears to be: - - `adapter_spec.model_deployment` -- The real remaining evaluation drift appears to be: - - `BasicMetric` in historic runs vs newer split metric classes in reproduced runs -- The current evidence suggests that at least part of the reproduction difference is due to HELM config / registry / metric behavior differences across environments or versions, not merely operator error in kwdagger scheduling. - -### Apples Batch Artifacts - -- Report bundle transferred locally to: - - `/home/joncrall/code/aiq-magnet/audit-smoke-apples` -- Representative reproduced job directories transferred locally to: - - `/home/joncrall/code/aiq-magnet/helm_id_10qfn238081w` - - `/home/joncrall/code/aiq-magnet/helm_id_kydvdl7oawex` - -### Apples Batch Technical Findings - -- All 6 case rows in the transferred JSONL indicate: - - `historic_exact_requested_max_eval_match = True` -- The union of reported run-spec execution-path drift across the transferred apples case rows is: - - `adapter_spec.model_deployment` -- Recurring primary reasons in the transferred apples report: - - `deployment_drift` - - `execution_spec_drift` -- Recurring secondary reasons in the transferred apples report: - - `evaluation_spec_drift` - - `core_metric_drift` - - `completion_content_drift` -- One case additionally reported: - - `request_prompt_drift` - -### Important Correction To Earlier Reporting - -- The first apples executive summary incorrectly said all 6 cases had a requested `max_eval_instances` mismatch. -- Inspection of the transferred apples case JSONL and reproduced job directories shows: - - historic `max_eval_instances = 1000` - - reproduced `max_eval_instances = 1000` -- Therefore that headline was a reporting bug, not a real result. -- Root cause of the bug: - - compare-side extraction of reproduced `adapter_spec.max_eval_instances` used the wrong view of the run spec -- Fix: - - `compare_batch.py` now reads reproduced `run_spec.json` directly via `load_run_spec_json(...)` - -### Representative Evidence: MMLU / Pythia-6.9b - -Run entry: -- `mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=eleutherai/pythia-6.9b,data_augmentation=canonical` - -Historic matched run: -- `/data/crfm-helm-public/classic/benchmark_output/runs/v0.2.4/mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=eleutherai_pythia-6.9b,data_augmentation=canonical` - -Reproduced transferred job: -- `/home/joncrall/code/aiq-magnet/helm_id_10qfn238081w` - -Observed config differences: -- historic `adapter_spec.model_deployment`: `null` -- reproduced `adapter_spec.model_deployment`: `huggingface/pythia-6.9b` -- historic `adapter_spec.max_eval_instances`: `1000` -- reproduced `adapter_spec.max_eval_instances`: `1000` - -Observed metric class differences: -- historic: - - `helm.benchmark.metrics.basic_metrics.BasicMetric` -- reproduced: - - `helm.benchmark.metrics.basic_metrics.BasicGenerationMetric` - - `helm.benchmark.metrics.basic_metrics.BasicReferenceMetric` - - `helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric` - -Observed behavior-level consequences from the transferred case JSONL: -- completion equal ratio about `0.896` -- core metric agree ratio about `0.5` - -Interpretation: -- once eval size is aligned, this case still differs materially -- the remaining differences are consistent with deployment resolution and metric/evaluation config changes - -### Representative Evidence: BoolQ / Vicuna-7b-v1.3 - -Run entry: -- `boolq:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical` - -Historic matched run: -- `/data/crfm-helm-public/classic/benchmark_output/runs/v0.3.0/boolq:model=lmsys_vicuna-7b-v1.3,data_augmentation=canonical` - -Reproduced transferred job: -- `/home/joncrall/code/aiq-magnet/helm_id_kydvdl7oawex` - -Observed config differences: -- historic `adapter_spec.model_deployment`: `null` -- reproduced `adapter_spec.model_deployment`: `huggingface/vicuna-7b-v1.3` -- historic `adapter_spec.max_eval_instances`: `1000` -- reproduced `adapter_spec.max_eval_instances`: `1000` - -Observed metric class differences: -- historic: - - `helm.benchmark.metrics.basic_metrics.BasicMetric` - - multiple `helm.benchmark.metrics.bias_metrics.BiasMetric` -- reproduced: - - `helm.benchmark.metrics.basic_metrics.BasicGenerationMetric` - - `helm.benchmark.metrics.basic_metrics.BasicReferenceMetric` - - `helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric` - - the same family of `helm.benchmark.metrics.bias_metrics.BiasMetric` - -Observed behavior-level consequences from the transferred case JSONL: -- completion equal ratio about `0.083` -- core metric agree ratio `0.0` - -Interpretation: -- this is a very strong mismatch even under apples-to-apples eval size -- the local reproduction is not merely “slightly off”; it is behaviorally very different for this case - -### What We Can Say To Management - -- The reproduction audit workflow is working end to end. -- We now have a real apples-to-apples control batch, not just a workflow smoke test. -- Even under apples-to-apples eval size, reproduced results still differ materially from historic public HELM runs. -- The dominant remaining differences are concentrated in deployment resolution and evaluation/metric configuration, with downstream output and metric drift. -- This is enough evidence to justify deeper technical follow-up with HELM maintainers. - -### What We Can Say To HELM Maintainers - -- The reproduced runs use explicit Hugging Face deployments where the matched historic public runs record `adapter_spec.model_deployment = null`. -- The reproduced runs also use a different metric-spec layout: - - `BasicMetric` historically - - `BasicGenerationMetric` + `BasicReferenceMetric` + `InstancesPerSplitMetric` in reproduced runs -- These config-level differences correlate with substantial completion-content and core-metric drift even after aligning requested eval size. -- The next maintainer-facing step should focus on why the same logical run entry resolves to these different `adapter_spec` / `metric_specs` structures across environments or HELM versions. - -### Suggested Next Step - -- Re-run the apples comparison after the `compare_batch.py` bugfix so the management summary no longer falsely reports requested eval-size mismatch. -- Then produce: - - one management-facing memo from the executive summary - - one maintainer-facing technical note with the two representative cases above and direct `run_spec.json` evidence - -### Tooling Note: Pairwise Comparison Artifact Validation - -- On 2026-03-27, `resolve_run.py`, `compare_entry.sh`, and `compare_pair.py` were hardened to validate finalized HELM run artifacts before diffing. -- A kwdagger job can exist with: - - `job_config.json` - - local caches - - scenario assets - while still lacking the finalized run outputs needed for diffing. -- Required files checked now: - - `run_spec.json` - - `scenario_state.json` - - `stats.json` - - `per_instance_stats.json` -- When those are missing, the tooling now reports a structured status such as: - - `artifact_status: incomplete_run_dir` - - `missing_files: [...]` - instead of surfacing a `FileNotFoundError` from deep inside `HelmRunDiff`. - -## Pairwise BoolQ / Pythia Result Snapshot - -Compared: - -- official HELM public run: - - `/data/crfm-helm-public/classic/benchmark_output/runs/v0.3.0/boolq:model=eleutherai_pythia-6.9b,data_augmentation=canonical` -- local kwdagger repeats: - - `/data/crfm-helm-audit/audit-boolq-pythia-r1/helm/helm_id_13jkx9mm4k4n/benchmark_output/runs/audit-boolq-pythia-r1/boolq:model=eleutherai_pythia-6.9b,data_augmentation=canonical` - - `/data/crfm-helm-audit/audit-boolq-pythia-r2/helm/helm_id_12jr5w48kge7/benchmark_output/runs/audit-boolq-pythia-r2/boolq:model=eleutherai_pythia-6.9b,data_augmentation=canonical` - -Repeatability baseline (`r1` vs `r2`): - -- diagnosis: `bookkeeping_metric_drift` -- strict run-level agree ratio: `0.9552238805970149` -- strict instance-level agree ratio: `0.9523809523809523` -- run-level max abs delta: `0.0015118565559387176` -- instance-level max abs delta: `0.4418189525604248` -- interpretation: - - repeated local runs are very close - - residual drift is mostly bookkeeping/runtime noise - -Historic vs local (`v0.3.0` vs `r1`): - -- diagnosis: `multiple_primary_reasons` -- primary reason names: - - `deployment_drift` - - `execution_spec_drift` -- strict run-level agree ratio: `0.4626865671641791` -- strict instance-level agree ratio: `0.6577333333333333` -- run-level abs p90: `4.0` -- run-level abs max: `11.985` -- instance-level abs p90: `4.0` -- instance-level abs max: `75.73884344100952` -- interpretation: - - historic vs local drift is much larger than local repeatability drift - - this strongly suggests the remaining mismatch is structural/configurational rather than ordinary nondeterminism diff --git a/dev/experiments/audit-helm-reproduction/README.md b/dev/experiments/audit-helm-reproduction/README.md deleted file mode 100644 index 0a7458b..0000000 --- a/dev/experiments/audit-helm-reproduction/README.md +++ /dev/null @@ -1,786 +0,0 @@ -# Audit HELM Reproduction - -This folder contains a reusable, shell-first workflow for reproducing selected -historic HELM runs with the local `kwdagger` pipeline, then comparing the -reproduced outputs against the public HELM bundle. - -The design goals are: - -- work out of the box in the current environment, -- remain portable to a fresh operator via environment variables, -- keep heavyweight run artifacts outside the repo by default, -- start with a small smoke-test batch, -- scale later to larger reproduction attempts without redesign. - -Remote reproduction-only machines do not need `/data/crfm-helm-public` mounted. -The public HELM bundle is only required on the analysis machine that will do -historic comparisons later. If a manifest sets `precomputed_root: null`, the -runner now skips the `HELM_PRECOMPUTED_ROOT` existence check. - -## Defaults - -These defaults are chosen to work in the current environment: - -- `AIQ_MAGNET_ROOT=/home/joncrall/code/aiq-magnet` -- `AIQ_PYTHON=python` -- `HELM_PRECOMPUTED_ROOT=/data/crfm-helm-public` -- `AUDIT_RESULTS_ROOT=/data/crfm-helm-audit` - -Assumptions: - -- `magnet` is importable from `AIQ_PYTHON` -- `kwdagger` is on `PATH` -- `helm-run` is on `PATH` - -Optional environment variables: - -- `AUDIT_DEFAULT_MAX_EVAL_INSTANCES` -- `AUDIT_DEFAULT_TMUX_WORKERS` - -## Folder Layout - -- `configs/` - Checked-in manifests and templates. -- `scripts/` - Shell entrypoints for operators. -- `python/` - Python helpers used by the shell scripts. -- `reports/` - Lightweight comparison reports. -- `examples/` - Example command sequences. - -## Quick Start - -```bash -cd $HOME/code/aiq-magnet -``` - -1. Validate the environment: - -```bash -dev/experiments/audit-helm-reproduction/scripts/check_env.sh -``` - -2. Materialize a smoke-test manifest: - -```bash -dev/experiments/audit-helm-reproduction/scripts/make_smoke_manifest.sh -``` - -To materialize the first apples-to-apples control manifest instead: - -```bash -dev/experiments/audit-helm-reproduction/scripts/make_apples_manifest.sh -``` - -By default this writes: - -```text -dev/experiments/audit-helm-reproduction/configs/generated/smoke_manifest.generated.yaml -``` - -And the apples-to-apples variant writes: - -```text -dev/experiments/audit-helm-reproduction/configs/generated/apples_manifest.generated.yaml -``` - -3. Launch the smoke-test batch on the GPU machine: - -```bash -dev/experiments/audit-helm-reproduction/scripts/run_smoke.sh -``` - -To change which GPUs `kwdagger` schedules onto, set the manifest `devices` -field or regenerate the manifest with `--devices`, for example: - -```bash -dev/experiments/audit-helm-reproduction/scripts/make_smoke_manifest.sh \ - dev/experiments/audit-helm-reproduction/configs/generated/smoke_manifest.generated.yaml \ - --devices 2,3 -``` - -4. Compare the completed batch to the historic HELM bundle: - -```bash -dev/experiments/audit-helm-reproduction/scripts/compare_batch.sh -``` - -Reports are written by default to: - -```text -dev/experiments/audit-helm-reproduction/reports// -``` - -Heavy run outputs are written by default to: - -```text -/data/crfm-helm-audit// -``` - -## Refreshing Analysis From Latest Data - -When new raw audit runs have landed under `/data/crfm-helm-audit`, the easiest -way to refresh analysis is: - -1. rebuild the audit index -2. regenerate the specific report you care about from that index - -Rebuild the index: - -```bash -AUDIT_FALLBACK_HOST=aiq-gpu \ -dev/experiments/audit-helm-reproduction/scripts/index_results.sh -``` - -This writes: - -- `dev/experiments/audit-helm-reproduction/reports/indexes/audit_results_index_.jsonl` -- `dev/experiments/audit-helm-reproduction/reports/indexes/audit_results_index_.csv` -- `dev/experiments/audit-helm-reproduction/reports/indexes/audit_results_index_.txt` - -Use `AUDIT_FALLBACK_HOST` for older runs that predate explicit process -provenance capture. Newer runs will record machine context directly in each job -directory via `process_context.json`. - -Then rebuild a core metric report for a specific run entry: - -```bash -dev/experiments/audit-helm-reproduction/scripts/rebuild_core_report_from_index.sh \ - --run-entry 'narrative_qa:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical' -``` - -This will: - -- pick the newest two indexed kwdagger runs for that exact `run_entry` -- find the best matching historic HELM run from `/data/crfm-helm-public` -- regenerate the core metric report bundle automatically - -Each stable report directory now stores timestamped artifacts under: - -- `.history//` - -The top-level user-facing files are symlinks to the newest timestamped -artifacts, for example: - -- `core_metric_management_summary.latest.txt` -- `core_metric_report.latest.txt` -- `core_metric_report.latest.json` -- `core_metric_report.latest.png` -- `core_metric_overlay_distributions.latest.png` -- `core_metric_ecdfs.latest.png` -- `core_runlevel_table.latest.csv` -- `report_selection.latest.json` -- `instance_samples_kwdagger_repeat.latest.txt` -- `instance_samples_official_vs_kwdagger.latest.txt` -- `kwdagger_a.run` -- `kwdagger_b.run` -- `official.run` -- `kwdagger_a.job` -- `kwdagger_b.job` - -This is the preferred way to inspect the newest version of a report without -guessing which timestamp is current. - -The raw-run symlinks make it easy to jump directly from a report bundle back to -the exact local and historic run directories that were selected for analysis. -The instance sample reports are text summaries produced from -`HelmRunDiff.summarize_instances(...)` and are intended as the first stop when -you want to inspect prompts, completions, references, and the largest per-run -instance mismatches in a suspicious case. - -If only one local run exists and you still want a report, allow reuse of that -single run on both sides of the repeatability slot: - -```bash -dev/experiments/audit-helm-reproduction/scripts/rebuild_core_report_from_index.sh \ - --run-entry 'narrative_qa:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical' \ - --allow-single-repeat -``` - -This is the preferred workflow for fast iteration when we want to keep changing -what we measure or how we visualize it without manually reconstructing run -paths. - -To rebuild all currently available core reports from the latest index: - -```bash -bash dev/experiments/audit-helm-reproduction/scripts/rebuild_all_core_reports_from_index.sh -``` - -If you want the tool to emit reports even for entries with only one matching -kwdagger run, use: - -```bash -bash dev/experiments/audit-helm-reproduction/scripts/rebuild_all_core_reports_from_index.sh \ - --allow-single-repeat -``` - -After rebuilding the per-run-spec reports, generate an overall reproducibility -assessment across all currently available core reports: - -```bash -bash dev/experiments/audit-helm-reproduction/scripts/aggregate_core_reports.sh -``` - -This writes: - -- `reports/overall-reproducibility/.history//overall_reproducibility_summary_.*` -- stable symlinks: - - `overall_reproducibility_summary.latest.txt` - - `overall_reproducibility_summary.latest.json` - - `overall_reproducibility_summary.latest.csv` - - `overall_reproducibility_summary.latest.md` - -The runner also derives a distinct `kwdagger` queue name from the experiment -name, which helps avoid interactive tmux collision prompts when multiple audit -batches have been launched on the same machine. - -If you want to inspect a specific pair directly without rebuilding the full -core report, you can write an instance-sample inspection report with: - -```bash -bash dev/experiments/audit-helm-reproduction/scripts/inspect_pair_samples.sh \ - --run-a /path/to/run_a \ - --run-b /path/to/run_b \ - --label investigation_pair \ - --report-dpath dev/experiments/audit-helm-reproduction/reports/manual-inspection -``` - -For a newly synced kwdagger experiment, you can rebuild only the reports -relevant to that experiment and write a focused summary with: - -```bash -AUDIT_FALLBACK_HOST=aiq-gpu \ -bash dev/experiments/audit-helm-reproduction/scripts/index_results.sh - -bash dev/experiments/audit-helm-reproduction/scripts/analyze_experiment_from_index.sh \ - --experiment-name audit-vicuna-nochat-overnight \ - --allow-single-repeat -``` - -This writes a compact experiment-level summary under: - -- `reports/experiment-analysis-/` - -## Building Larger Historic Reproduction Batches - -The repo-root files: - -- `run_specs.yaml` -- `run_details.yaml` - -are the curated historic candidate list produced from: - -- `dev/poc/inspect_historic_helm_runs.py` - -You can regenerate them with: - -```bash -python dev/poc/inspect_historic_helm_runs.py \ - /data/crfm-helm-public \ - --out_fpath run_specs.yaml \ - --out_detail_fpath run_details.yaml -``` - -To build a larger refreshed kwdagger manifest from those historic candidates: - -```bash -bash dev/experiments/audit-helm-reproduction/scripts/make_historic_grid_manifest.sh \ - dev/experiments/audit-helm-reproduction/configs/generated/historic_grid.generated.yaml \ - --experiment-name audit-historic-grid \ - --suite audit-historic-grid \ - --devices 0,1 \ - --tmux-workers 2 \ - --max-eval-instances 1000 -``` - -This also writes a sidecar selection file: - -- `.../historic_grid.generated.yaml.selection.yaml` - -The selection sidecar records: - -- exactly which `run_entry` values were selected -- machine/shard settings -- any model override file that was applied -- matching metadata from `run_details.yaml` when available - -### Filtering The Historic Grid - -The larger builder supports shell-style pattern filtering and model/benchmark -selection. For example, a Vicuna-only slice over three benchmarks: - -```bash -bash dev/experiments/audit-helm-reproduction/scripts/make_historic_grid_manifest.sh \ - dev/experiments/audit-helm-reproduction/configs/generated/historic_vicuna_focus.generated.yaml \ - --experiment-name audit-historic-vicuna-focus \ - --suite audit-historic-vicuna-focus \ - --model lmsys/vicuna-7b-v1.3 \ - --include-pattern 'boolq:*' \ - --include-pattern 'mmlu:*' \ - --include-pattern 'narrative_qa:*' \ - --devices 0 \ - --tmux-workers 1 \ - --max-eval-instances 1000 -``` - -If any selected entries use `lmsys/vicuna-7b-v1.3`, the builder automatically -applies: - -- `dev/experiments/audit-helm-reproduction/configs/debug/vicuna_no_chat_template.yaml` - -so the fixed no-chat-template configuration is used. - -### Deterministic One-GPU Shards For Multiple Machines - -To split the same filtered candidate set deterministically across multiple -machines, use the shard builder. This is useful for `namek` and `yardrat`, -where only one GPU is available. - -Example: build 2 shards from the same filtered historic set. - -For `namek`: - -```bash -bash dev/experiments/audit-helm-reproduction/scripts/make_machine_shard_manifest.sh \ - namek \ - 0 \ - 2 \ - dev/experiments/audit-helm-reproduction/configs/generated/namek.generated.yaml \ - --model lmsys/vicuna-7b-v1.3 \ - --include-pattern 'boolq:*' \ - --include-pattern 'mmlu:*' \ - --include-pattern 'narrative_qa:*' \ - --max-eval-instances 1000 -``` - -For `yardrat`: - -```bash -bash dev/experiments/audit-helm-reproduction/scripts/make_machine_shard_manifest.sh \ - yardrat \ - 1 \ - 2 \ - dev/experiments/audit-helm-reproduction/configs/generated/yardrat.generated.yaml \ - --model lmsys/vicuna-7b-v1.3 \ - --include-pattern 'boolq:*' \ - --include-pattern 'mmlu:*' \ - --include-pattern 'narrative_qa:*' \ - --max-eval-instances 1000 -``` - -These manifests default to: - -- `devices: 0` -- `tmux_workers: 1` - -so they are safe for single-GPU machines unless explicitly overridden. - -### Same Subset On Multiple Machines For Hardware Comparison - -If the goal is to compare reproducibility across different hardware, both -machines should run the same subset rather than different shards. - -For `namek`: - -```bash -bash dev/experiments/audit-helm-reproduction/scripts/make_machine_subset_manifest.sh \ - namek \ - dev/experiments/audit-helm-reproduction/configs/generated/namek.subset.generated.yaml \ - --model lmsys/vicuna-7b-v1.3 \ - --include-pattern 'boolq:*' \ - --include-pattern 'mmlu:*us_foreign_policy*' \ - --include-pattern 'narrative_qa:*' \ - --max-eval-instances 1000 -``` - -For `yardrat`, use the same filters: - -```bash -bash dev/experiments/audit-helm-reproduction/scripts/make_machine_subset_manifest.sh \ - yardrat \ - dev/experiments/audit-helm-reproduction/configs/generated/yardrat.subset.generated.yaml \ - --model lmsys/vicuna-7b-v1.3 \ - --include-pattern 'boolq:*' \ - --include-pattern 'mmlu:*us_foreign_policy*' \ - --include-pattern 'narrative_qa:*' \ - --max-eval-instances 1000 -``` - -These produce two manifests with the same selected `run_entry` set but distinct -`experiment_name` / `suite` values, which makes later indexing and -cross-machine analysis easier. - -### Running The Larger Batch - -Once a manifest has been generated, launch it the same way as the smaller -batches: - -```bash -bash dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh \ - dev/experiments/audit-helm-reproduction/configs/generated/historic_grid.generated.yaml -``` - -After raw results are synced back, rebuild the index and analyze the specific -experiment: - -```bash -AUDIT_FALLBACK_HOST=aiq-gpu \ -bash dev/experiments/audit-helm-reproduction/scripts/index_results.sh - -bash dev/experiments/audit-helm-reproduction/scripts/analyze_experiment_from_index.sh \ - --experiment-name audit-historic-grid \ - --allow-single-repeat -``` - -Note: - -- this experiment analyzer uses the kwdagger results index -- it is intended for indexed kwdagger experiments -- direct one-off debug runs that were not scheduled as kwdagger jobs will not appear there - -## Reproducibility Checklist - -For any experiment you want to cite later, preserve all of the following: - -- the exact manifest YAML used to launch the run -- the exact results root under: - - `/data/crfm-helm-audit//` -- the generated comparison reports under: - - `dev/experiments/audit-helm-reproduction/reports//` -- the current git commit of `aiq-magnet` -- the Python executable used as `AIQ_PYTHON` -- the value of: - - `AIQ_MAGNET_ROOT` - - `HELM_PRECOMPUTED_ROOT` - - `AUDIT_RESULTS_ROOT` - -Recommended capture commands: - -```bash -git rev-parse HEAD -which "$AIQ_PYTHON" -dev/experiments/audit-helm-reproduction/scripts/check_env.sh -``` - -If you need to transfer the run to another machine for analysis, transfer: - -- the manifest YAML -- the report directory for the experiment -- the raw results directory for the experiment - -Minimum useful transfer set: - -- report directory only, if you only need summaries -- report directory plus 1-2 representative raw job directories, if you need direct run artifact inspection -- full raw experiment directory, if you may need to rerun local comparisons later - -For newer runs, also preserve: - -- `process_context.json` in each kwdagger HELM job directory - -That file records structured host/process provenance and is intended to support -future cross-machine and cross-hardware analysis. - -## Manifest Schema - -The workflow uses a small YAML manifest as the unit of experiment definition. - -Fields: - -- `schema_version` -- `experiment_name` -- `description` -- `run_entries` -- `max_eval_instances` -- `suite` -- `mode` -- `materialize` -- `backend` -- `devices` -- `tmux_workers` -- `local_path` -- `precomputed_root` -- `require_per_instance_stats` -- `model_deployments_fpath` -- `enable_huggingface_models` -- `enable_local_huggingface_models` - -See: - -- `configs/smoke_manifest.yaml` -- `configs/apples_manifest.yaml` -- `configs/manifest_template.yaml` - -## Smoke-Test Batch - -The checked-in smoke batch is intentionally small and uses only models that -already have built-in Hugging Face deployments in HELM: - -- `eleutherai/pythia-6.9b` -- `lmsys/vicuna-7b-v1.3` - -Tasks: - -- `mmlu:subject=us_foreign_policy,...` -- `boolq:...` -- `narrative_qa:...` - -Total: - -- 6 runs - -Defaults: - -- `max_eval_instances=100` -- low worker count -- no custom deployment override YAML - -## Apples-To-Apples Smoke Batch - -The first apples-to-apples control batch reuses the same 6 smoke entries, but -aligns `max_eval_instances` with the historic public bundle for those entries: - -- `max_eval_instances=1000` -- experiment name: `audit-smoke-apples` -- suite: `audit-smoke-apples` -- devices are still controlled by the manifest `devices` field or `--devices` - -This is the preferred first batch when the goal is reproduction fidelity rather -than just workflow validation. - -Suggested operator flow: - -```bash -dev/experiments/audit-helm-reproduction/scripts/make_apples_manifest.sh \ - dev/experiments/audit-helm-reproduction/configs/generated/apples_manifest.generated.yaml \ - --devices 0,1 - -dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh \ - dev/experiments/audit-helm-reproduction/configs/generated/apples_manifest.generated.yaml - -dev/experiments/audit-helm-reproduction/scripts/compare_batch.sh \ - dev/experiments/audit-helm-reproduction/configs/generated/apples_manifest.generated.yaml -``` - -Raw reproduced outputs are written to: - -```text -/data/crfm-helm-audit/audit-smoke-apples/ -``` - -Comparison reports are written to: - -```text -dev/experiments/audit-helm-reproduction/reports/audit-smoke-apples/ -``` - -Files to inspect first: - -- `management_summary_.txt` -- `compare_summary_.txt` -- `compare_cases_.jsonl` - -Files to transfer back for local analysis: - -- the entire report directory -- optionally the entire raw results directory if deeper run-by-run inspection is needed - -Example transfer commands depend on your setup, but a simple pattern is: - -```bash -ls -td dev/experiments/audit-helm-reproduction/reports/audit-smoke-apples/* -ls -td /data/crfm-helm-audit/audit-smoke-apples/* -``` - -Then transfer the newest report files and, if needed, the raw experiment root. - -## Exact Reproduction Cases Used In This Research - -The following cases were used as the first reproducibility controls in this -research thread. - -### Apples-To-Apples Control Batch - -Purpose: - -- compare current local kwdagger reproductions against official public HELM with matched `max_eval_instances` - -Manifest generation: - -```bash -dev/experiments/audit-helm-reproduction/scripts/make_apples_manifest.sh \ - dev/experiments/audit-helm-reproduction/configs/generated/apples_manifest.generated.yaml \ - --devices 0,1 -``` - -Run: - -```bash -dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh \ - dev/experiments/audit-helm-reproduction/configs/generated/apples_manifest.generated.yaml -``` - -Compare: - -```bash -dev/experiments/audit-helm-reproduction/scripts/compare_batch.sh \ - dev/experiments/audit-helm-reproduction/configs/generated/apples_manifest.generated.yaml -``` - -Outputs: - -- raw results: - - `/data/crfm-helm-audit/audit-smoke-apples/` -- reports: - - `dev/experiments/audit-helm-reproduction/reports/audit-smoke-apples/` - -### Pairwise Repeatability Control: BoolQ / Pythia - -Purpose: - -- measure ordinary local rerun drift on the same benchmark/model pair - -Manifest files used: - -- `dev/experiments/audit-helm-reproduction/configs/generated/boolq_pythia_r1.yaml` -- `dev/experiments/audit-helm-reproduction/configs/generated/boolq_pythia_r2.yaml` - -Run: - -```bash -dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh \ - dev/experiments/audit-helm-reproduction/configs/generated/boolq_pythia_r1.yaml - -dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh \ - dev/experiments/audit-helm-reproduction/configs/generated/boolq_pythia_r2.yaml -``` - -Direct pairwise compare of the two completed runs: - -```bash -dev/experiments/audit-helm-reproduction/scripts/compare_pair.sh \ - /data/crfm-helm-audit/audit-boolq-pythia-r1/helm/helm_id_13jkx9mm4k4n/benchmark_output/runs/audit-boolq-pythia-r1/boolq:model=eleutherai_pythia-6.9b,data_augmentation=canonical \ - /data/crfm-helm-audit/audit-boolq-pythia-r2/helm/helm_id_12jr5w48kge7/benchmark_output/runs/audit-boolq-pythia-r2/boolq:model=eleutherai_pythia-6.9b,data_augmentation=canonical \ - dev/experiments/audit-helm-reproduction/reports/pairwise/boolq-pythia-repeat -``` - -### Pairwise Official-vs-Local Control: BoolQ / Pythia - -Purpose: - -- compare one local reproduced run directly against the matched public HELM run - -Direct compare: - -```bash -dev/experiments/audit-helm-reproduction/scripts/compare_pair.sh \ - /data/crfm-helm-public/classic/benchmark_output/runs/v0.3.0/boolq:model=eleutherai_pythia-6.9b,data_augmentation=canonical \ - /data/crfm-helm-audit/audit-boolq-pythia-r1/helm/helm_id_13jkx9mm4k4n/benchmark_output/runs/audit-boolq-pythia-r1/boolq:model=eleutherai_pythia-6.9b,data_augmentation=canonical \ - dev/experiments/audit-helm-reproduction/reports/pairwise/boolq-pythia-historic -``` - -### Viewing The Key Reports - -```bash -cat dev/experiments/audit-helm-reproduction/reports/pairwise/boolq-pythia-repeat-wide/pair_report_20260327T011202Z.txt -cat dev/experiments/audit-helm-reproduction/reports/pairwise/boolq-pythia-historic-wide/pair_report_20260327T011202Z.txt -``` - -These two reports are the current best compact illustration of: - -- local repeatability drift being small -- official-vs-local drift being much larger -- global tolerance sweeps needing careful interpretation because metric scales differ across classes - -## Scaling Up - -For larger experiments: - -1. copy `configs/manifest_template.yaml`, -2. expand `run_entries`, -3. optionally introduce `model_deployments_fpath` for Together-only families, -4. run: - -```bash -dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh /path/to/manifest.yaml -dev/experiments/audit-helm-reproduction/scripts/compare_batch.sh /path/to/manifest.yaml -``` - -## Pairwise Run Reports - -To compare any two concrete HELM run directories directly, use: - -```bash -dev/experiments/audit-helm-reproduction/scripts/compare_pair.sh \ - /path/to/run_a \ - /path/to/run_b \ - dev/experiments/audit-helm-reproduction/reports/pairwise -``` - -This writes: - -- `pair_report_.json` -- `pair_report_.txt` - -The pairwise report includes: - -- strict diff diagnosis -- raw run-level distance distributions -- raw instance-level distance distributions -- tolerance sweeps across several preset thresholds - -Example using the local repeated `boolq + gpt2` runs: - -```bash -dev/experiments/audit-helm-reproduction/scripts/compare_pair.sh \ - /data/crfm-helm-audit/audit-boolq-gpt2-r1/helm/helm_id_lh2zobnkhuwi/benchmark_output/runs/audit-boolq-gpt2-r1/boolq:model=openai_gpt2,data_augmentation=canonical \ - /data/crfm-helm-audit/audit-boolq-gpt2-r2/helm/helm_id_lvb1vuf32m2g/benchmark_output/runs/audit-boolq-gpt2-r2/boolq:model=openai_gpt2,data_augmentation=canonical \ - dev/experiments/audit-helm-reproduction/reports/pairwise -``` - -Then inspect: - -```bash -ls -td dev/experiments/audit-helm-reproduction/reports/pairwise/* -cat dev/experiments/audit-helm-reproduction/reports/pairwise/pair_report_.txt -``` - -## Indexing Existing Audit Results - -The indexer scans all current audit outputs and builds a machine-readable -inventory of what exists. - -Use: - -```bash -AUDIT_FALLBACK_HOST=aiq-gpu \ -dev/experiments/audit-helm-reproduction/scripts/index_results.sh -``` - -The index currently records: - -- experiment name -- job id -- status -- run entry -- benchmark / model / method -- max eval instances -- resolved run directory -- machine host -- GPU fields when recorded -- provenance source (`recorded` vs `fallback`) - -This index is the preferred starting point for rebuilding reports against the -latest available runs without manually searching the raw results tree. - -## Notes - -- This workflow is intended for execution on an external GPU machine. -- The current development environment does not need a GPU to generate manifests - or comparison commands. -- Comparison relies on `HelmRunDiff` and will emit JSONL, JSON, text, and - optional sankey artifacts when the required plotting dependencies are - available. diff --git a/dev/experiments/audit-helm-reproduction/configs/apples_manifest.yaml b/dev/experiments/audit-helm-reproduction/configs/apples_manifest.yaml deleted file mode 100644 index e7f20cf..0000000 --- a/dev/experiments/audit-helm-reproduction/configs/apples_manifest.yaml +++ /dev/null @@ -1,23 +0,0 @@ -schema_version: 1 -experiment_name: audit-smoke-apples -description: Apples-to-apples smoke batch aligned to the historic public HELM requested max_eval_instances for the control entries. -run_entries: - - mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=eleutherai/pythia-6.9b,data_augmentation=canonical - - boolq:model=eleutherai/pythia-6.9b,data_augmentation=canonical - - narrative_qa:model=eleutherai/pythia-6.9b,data_augmentation=canonical - - mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical - - boolq:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical - - narrative_qa:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical -max_eval_instances: 1000 -suite: audit-smoke-apples -mode: compute_if_missing -materialize: symlink -backend: tmux -devices: 0,1 -tmux_workers: 2 -local_path: prod_env -precomputed_root: null -require_per_instance_stats: true -model_deployments_fpath: null -enable_huggingface_models: [] -enable_local_huggingface_models: [] diff --git a/dev/experiments/audit-helm-reproduction/configs/debug/vicuna_no_chat_template.yaml b/dev/experiments/audit-helm-reproduction/configs/debug/vicuna_no_chat_template.yaml deleted file mode 100644 index bd67e0a..0000000 --- a/dev/experiments/audit-helm-reproduction/configs/debug/vicuna_no_chat_template.yaml +++ /dev/null @@ -1,9 +0,0 @@ -model_deployments: - - name: huggingface/vicuna-7b-v1.3 - model_name: lmsys/vicuna-7b-v1.3 - tokenizer_name: hf-internal-testing/llama-tokenizer - max_sequence_length: 2048 - client_spec: - class_name: "helm.clients.huggingface_client.HuggingFaceClient" - args: - apply_chat_template: false diff --git a/dev/experiments/audit-helm-reproduction/configs/manifest_template.yaml b/dev/experiments/audit-helm-reproduction/configs/manifest_template.yaml deleted file mode 100644 index cdf3030..0000000 --- a/dev/experiments/audit-helm-reproduction/configs/manifest_template.yaml +++ /dev/null @@ -1,18 +0,0 @@ -schema_version: 1 -experiment_name: your-experiment-name -description: Describe the reproduction batch here. -run_entries: - - mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=eleutherai/pythia-6.9b,data_augmentation=canonical -max_eval_instances: 100 -suite: audit-custom -mode: compute_if_missing -materialize: symlink -backend: tmux -devices: 0,1 -tmux_workers: 2 -local_path: prod_env -precomputed_root: null -require_per_instance_stats: true -model_deployments_fpath: null -enable_huggingface_models: [] -enable_local_huggingface_models: [] diff --git a/dev/experiments/audit-helm-reproduction/configs/model_deployments_template.yaml b/dev/experiments/audit-helm-reproduction/configs/model_deployments_template.yaml deleted file mode 100644 index 1812858..0000000 --- a/dev/experiments/audit-helm-reproduction/configs/model_deployments_template.yaml +++ /dev/null @@ -1,11 +0,0 @@ -model_deployments: - - name: huggingface/example-local-deployment - model_name: creator/example-model - tokenizer_name: huggingface/example-local-deployment - max_sequence_length: 4096 - client_spec: - class_name: "helm.clients.huggingface_client.HuggingFaceClient" - args: - pretrained_model_name_or_path: creator/example-model - device_map: auto - torch_dtype: torch.bfloat16 diff --git a/dev/experiments/audit-helm-reproduction/configs/smoke_manifest.yaml b/dev/experiments/audit-helm-reproduction/configs/smoke_manifest.yaml deleted file mode 100644 index 775198b..0000000 --- a/dev/experiments/audit-helm-reproduction/configs/smoke_manifest.yaml +++ /dev/null @@ -1,23 +0,0 @@ -schema_version: 1 -experiment_name: audit-smoke -description: Small smoke-test batch for HELM reproduction auditing. -run_entries: - - mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=eleutherai/pythia-6.9b,data_augmentation=canonical - - boolq:model=eleutherai/pythia-6.9b,data_augmentation=canonical - - narrative_qa:model=eleutherai/pythia-6.9b,data_augmentation=canonical - - mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical - - boolq:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical - - narrative_qa:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical -max_eval_instances: 100 -suite: audit-smoke -mode: compute_if_missing -materialize: symlink -backend: tmux -devices: 0,1 -tmux_workers: 2 -local_path: prod_env -precomputed_root: null -require_per_instance_stats: true -model_deployments_fpath: null -enable_huggingface_models: [] -enable_local_huggingface_models: [] diff --git a/dev/experiments/audit-helm-reproduction/examples/example_overnight_commands.sh b/dev/experiments/audit-helm-reproduction/examples/example_overnight_commands.sh deleted file mode 100755 index a3aefad..0000000 --- a/dev/experiments/audit-helm-reproduction/examples/example_overnight_commands.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Example overnight control on a richer generation-style task. - -dev/experiments/audit-helm-reproduction/scripts/make_repeat_pair_manifests.sh \ - 'narrative_qa:model=eleutherai/pythia-6.9b,data_augmentation=canonical' \ - audit-narrative-pythia \ - dev/experiments/audit-helm-reproduction/configs/generated \ - --max-eval-instances 1000 \ - --devices 0 \ - --tmux-workers 1 - -dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh \ - dev/experiments/audit-helm-reproduction/configs/generated/audit-narrative-pythia_r1.yaml - -dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh \ - dev/experiments/audit-helm-reproduction/configs/generated/audit-narrative-pythia_r2.yaml - -dev/experiments/audit-helm-reproduction/scripts/compare_batch.sh \ - dev/experiments/audit-helm-reproduction/configs/generated/audit-narrative-pythia_r1.yaml - -dev/experiments/audit-helm-reproduction/scripts/compare_batch.sh \ - dev/experiments/audit-helm-reproduction/configs/generated/audit-narrative-pythia_r2.yaml diff --git a/dev/experiments/audit-helm-reproduction/examples/example_smoke_commands.sh b/dev/experiments/audit-helm-reproduction/examples/example_smoke_commands.sh deleted file mode 100755 index 56a50ee..0000000 --- a/dev/experiments/audit-helm-reproduction/examples/example_smoke_commands.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -AUDIT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" - -"$AUDIT_ROOT/scripts/check_env.sh" -"$AUDIT_ROOT/scripts/make_smoke_manifest.sh" -"$AUDIT_ROOT/scripts/run_smoke.sh" -"$AUDIT_ROOT/scripts/compare_batch.sh" diff --git a/dev/experiments/audit-helm-reproduction/python/aggregate_core_reports.py b/dev/experiments/audit-helm-reproduction/python/aggregate_core_reports.py deleted file mode 100644 index 8ef8666..0000000 --- a/dev/experiments/audit-helm-reproduction/python/aggregate_core_reports.py +++ /dev/null @@ -1,175 +0,0 @@ -from __future__ import annotations - -import argparse -import datetime as datetime_mod -import glob -import json -import os -from collections import Counter -from pathlib import Path -from typing import Any - -import pandas as pd - -from common import default_report_root - - -def _load_json(fpath: Path) -> dict[str, Any]: - return json.loads(fpath.read_text()) - - -def _find_pair(report: dict[str, Any], label: str) -> dict[str, Any]: - for pair in report.get('pairs', []): - if pair.get('label') == label: - return pair - return {} - - -def _find_curve_value(rows: list[dict[str, Any]], abs_tol: float) -> float | None: - for row in rows or []: - try: - if float(row.get('abs_tol')) == float(abs_tol): - return float(row.get('agree_ratio')) - except Exception: - pass - return None - - -def _assessment_label(repeat_agree_0: float | None, official_agree_01: float | None) -> str: - if repeat_agree_0 is None or official_agree_01 is None: - return 'unknown' - if repeat_agree_0 >= 0.99 and official_agree_01 >= 0.95: - return 'close_match' - if repeat_agree_0 >= 0.99 and official_agree_01 >= 0.75: - return 'moderate_drift' - if repeat_agree_0 >= 0.99: - return 'strong_drift' - return 'unstable_local_repeat' - - -def _slugify(text: str) -> str: - return ( - text.replace('/', '-') - .replace(':', '-') - .replace(',', '-') - .replace('=', '-') - .replace('@', '-') - .replace(' ', '-') - ) - - -def _write_latest_alias(src: Path, latest_root: Path, latest_name: str) -> None: - latest_fpath = latest_root / latest_name - if latest_fpath.exists() or latest_fpath.is_symlink(): - latest_fpath.unlink() - rel_src = os.path.relpath(src, start=latest_root) - os.symlink(rel_src, latest_fpath) - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument('--report-root', default=str(default_report_root())) - parser.add_argument('--out-dpath', default=None) - args = parser.parse_args() - - report_root = Path(args.report_root).expanduser().resolve() - out_dpath = Path(args.out_dpath).expanduser().resolve() if args.out_dpath else (report_root / 'overall-reproducibility') - out_dpath.mkdir(parents=True, exist_ok=True) - - report_paths = sorted(glob.glob(str(report_root / 'core-metrics-*' / 'core_metric_report.latest.json'))) - rows = [] - for p in report_paths: - fpath = Path(p) - report = _load_json(fpath) - repeat = _find_pair(report, 'kwdagger_repeat') - official = _find_pair(report, 'official_vs_kwdagger') - repeat_agree_0 = _find_curve_value(repeat.get('instance_level', {}).get('agreement_vs_abs_tol', []), 0.0) - official_agree_0 = _find_curve_value(official.get('instance_level', {}).get('agreement_vs_abs_tol', []), 0.0) - official_agree_01 = _find_curve_value(official.get('instance_level', {}).get('agreement_vs_abs_tol', []), 0.1) - official_agree_025 = _find_curve_value(official.get('instance_level', {}).get('agreement_vs_abs_tol', []), 0.25) - official_agree_05 = _find_curve_value(official.get('instance_level', {}).get('agreement_vs_abs_tol', []), 0.5) - rows.append({ - 'report_dir': str(fpath.parent), - 'report_json': str(fpath), - 'run_spec_name': report.get('run_spec_name'), - 'generated_utc': report.get('generated_utc'), - 'n_core_metrics': len((_find_pair(report, 'kwdagger_repeat').get('core_metrics') or [])), - 'diagnostic_flags': report.get('diagnostic_flags', []), - 'kwdagger_a_empty_completion_rate': (((report.get('run_diagnostics') or {}).get('kwdagger_a') or {}).get('empty_completion_rate')), - 'kwdagger_a_mean_output_tokens': ((((report.get('run_diagnostics') or {}).get('kwdagger_a') or {}).get('output_token_count') or {}).get('mean')), - 'official_empty_completion_rate': (((report.get('run_diagnostics') or {}).get('official') or {}).get('empty_completion_rate')), - 'official_mean_output_tokens': ((((report.get('run_diagnostics') or {}).get('official') or {}).get('output_token_count') or {}).get('mean')), - 'repeat_instance_agree_0': repeat_agree_0, - 'official_instance_agree_0': official_agree_0, - 'official_instance_agree_01': official_agree_01, - 'official_instance_agree_025': official_agree_025, - 'official_instance_agree_05': official_agree_05, - 'official_runlevel_p90': (((official.get('run_level') or {}).get('overall_quantiles') or {}).get('abs_delta') or {}).get('p90'), - 'official_runlevel_max': (((official.get('run_level') or {}).get('overall_quantiles') or {}).get('abs_delta') or {}).get('max'), - 'assessment_label': _assessment_label(repeat_agree_0, official_agree_01), - }) - - table = pd.DataFrame(rows).sort_values(['assessment_label', 'run_spec_name'], na_position='last') - stamp = datetime_mod.datetime.now(datetime_mod.UTC).strftime('%Y%m%dT%H%M%SZ') - history_dpath = out_dpath / '.history' / stamp[:8] - history_dpath.mkdir(parents=True, exist_ok=True) - - json_fpath = history_dpath / f'overall_reproducibility_summary_{stamp}.json' - csv_fpath = history_dpath / f'overall_reproducibility_summary_{stamp}.csv' - txt_fpath = history_dpath / f'overall_reproducibility_summary_{stamp}.txt' - md_fpath = history_dpath / f'overall_reproducibility_summary_{stamp}.md' - - summary = { - 'generated_utc': stamp, - 'n_reports': len(rows), - 'assessment_counts': dict(Counter(row['assessment_label'] for row in rows)), - 'run_specs': rows, - } - json_fpath.write_text(json.dumps(summary, indent=2)) - table.to_csv(csv_fpath, index=False) - md_fpath.write_text(table.to_markdown(index=False) + '\n') - - lines = [] - lines.append('Overall Reproducibility Assessment') - lines.append('') - lines.append(f'generated_utc: {stamp}') - lines.append(f'n_reports: {len(rows)}') - lines.append('') - lines.append('assessment_counts:') - for key, val in sorted(summary['assessment_counts'].items()): - lines.append(f' {key}: {val}') - lines.append('') - lines.append('per_run_spec:') - for row in rows: - lines.append(f" - run_spec_name: {row['run_spec_name']}") - lines.append(f" assessment_label: {row['assessment_label']}") - lines.append(f" diagnostic_flags: {row['diagnostic_flags']}") - lines.append(f" kwdagger_a_empty_completion_rate: {row['kwdagger_a_empty_completion_rate']}") - lines.append(f" kwdagger_a_mean_output_tokens: {row['kwdagger_a_mean_output_tokens']}") - lines.append(f" official_empty_completion_rate: {row['official_empty_completion_rate']}") - lines.append(f" official_mean_output_tokens: {row['official_mean_output_tokens']}") - lines.append(f" repeat_instance_agree_0: {row['repeat_instance_agree_0']}") - lines.append(f" official_instance_agree_0: {row['official_instance_agree_0']}") - lines.append(f" official_instance_agree_01: {row['official_instance_agree_01']}") - lines.append(f" official_instance_agree_025: {row['official_instance_agree_025']}") - lines.append(f" official_instance_agree_05: {row['official_instance_agree_05']}") - lines.append(f" official_runlevel_p90: {row['official_runlevel_p90']}") - lines.append(f" official_runlevel_max: {row['official_runlevel_max']}") - txt_fpath.write_text('\n'.join(lines) + '\n') - - for src, latest_name in [ - (json_fpath, 'overall_reproducibility_summary.latest.json'), - (csv_fpath, 'overall_reproducibility_summary.latest.csv'), - (txt_fpath, 'overall_reproducibility_summary.latest.txt'), - (md_fpath, 'overall_reproducibility_summary.latest.md'), - ]: - _write_latest_alias(src, out_dpath, latest_name) - - print(f'Wrote summary json: {json_fpath}') - print(f'Wrote summary csv: {csv_fpath}') - print(f'Wrote summary md: {md_fpath}') - print(f'Wrote summary txt: {txt_fpath}') - - -if __name__ == '__main__': - main() diff --git a/dev/experiments/audit-helm-reproduction/python/analyze_experiment_from_index.py b/dev/experiments/audit-helm-reproduction/python/analyze_experiment_from_index.py deleted file mode 100644 index 1731c27..0000000 --- a/dev/experiments/audit-helm-reproduction/python/analyze_experiment_from_index.py +++ /dev/null @@ -1,340 +0,0 @@ -from __future__ import annotations - -import argparse -import csv -import datetime as datetime_mod -import json -import os -import subprocess -from pathlib import Path -from typing import Any - -import pandas as pd - -from aggregate_core_reports import _find_curve_value, _find_pair, _write_latest_alias -from common import audit_root, default_report_root, env_defaults -from paper_labels import load_paper_label_manager -from rebuild_core_report_from_index import latest_index_csv, load_rows, slugify - - -def _load_json(fpath: Path) -> dict[str, Any]: - return json.loads(fpath.read_text()) - - -def _coerce_float(value: Any) -> float: - try: - return float(value) - except Exception: - return float("-inf") - - -def _is_truthy_text(value: Any) -> bool: - return str(value).strip().lower() in {"true", "1", "yes"} - - -def _latest_matching_aiq_gpu_row( - rows: list[dict[str, Any]], - *, - run_entry: str, - exclude_experiment_name: str, -) -> dict[str, Any] | None: - candidates = [] - for row in rows: - if row.get("run_entry") != run_entry: - continue - if row.get("experiment_name") == exclude_experiment_name: - continue - if row.get("machine_host") != "aiq-gpu": - continue - if row.get("status") not in {"computed", "reused", "unknown", ""}: - continue - if not _is_truthy_text(row.get("has_run_spec")): - continue - run_dir = row.get("run_dir") - if not run_dir: - continue - candidates.append(row) - if not candidates: - return None - candidates.sort( - key=lambda r: ( - _coerce_float(r.get("manifest_timestamp")), - str(r.get("experiment_name") or ""), - str(r.get("job_id") or ""), - ), - reverse=True, - ) - return candidates[0] - - -def _latest_pair_report(report_dpath: Path) -> tuple[Path | None, Path | None]: - json_cands = sorted(report_dpath.glob("pair_report_*.json"), reverse=True) - txt_cands = sorted(report_dpath.glob("pair_report_*.txt"), reverse=True) - return ( - json_cands[0] if json_cands else None, - txt_cands[0] if txt_cands else None, - ) - - -def _write_latest_pair_aliases(report_dpath: Path) -> dict[str, str]: - json_fpath, txt_fpath = _latest_pair_report(report_dpath) - created: dict[str, str] = {} - if json_fpath is not None: - _write_latest_alias(json_fpath, report_dpath, "pair_report.latest.json") - created["pair_report.latest.json"] = str(report_dpath / "pair_report.latest.json") - if txt_fpath is not None: - _write_latest_alias(txt_fpath, report_dpath, "pair_report.latest.txt") - created["pair_report.latest.txt"] = str(report_dpath / "pair_report.latest.txt") - return created - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument('--experiment-name', required=True) - parser.add_argument('--index-fpath', default=None) - parser.add_argument('--index-dpath', default=str(default_report_root() / 'indexes')) - parser.add_argument('--allow-single-repeat', action='store_true') - args = parser.parse_args() - - index_fpath = ( - Path(args.index_fpath).expanduser().resolve() - if args.index_fpath else - latest_index_csv(Path(args.index_dpath).expanduser().resolve()) - ) - rows = load_rows(index_fpath) - experiment_rows = [r for r in rows if r.get('experiment_name') == args.experiment_name] - if not experiment_rows: - raise SystemExit(f'No rows found for experiment_name={args.experiment_name!r}') - run_entries = sorted({r.get('run_entry') for r in experiment_rows if r.get('run_entry')}) - - out_dpath = default_report_root() / f'experiment-analysis-{slugify(args.experiment_name)}' - out_dpath.mkdir(parents=True, exist_ok=True) - reports_dpath = out_dpath / 'core-reports' - reports_dpath.mkdir(parents=True, exist_ok=True) - - rebuild_script = audit_root() / 'python' / 'rebuild_core_report_from_index.py' - built_report_paths = [] - skipped_run_entries: list[dict[str, Any]] = [] - for run_entry in run_entries: - report_dpath = reports_dpath / f'core-metrics-{slugify(run_entry)}' - cmd = [ - env_defaults()['AIQ_PYTHON'], - str(rebuild_script), - '--run-entry', str(run_entry), - '--index-fpath', str(index_fpath), - '--experiment-name', str(args.experiment_name), - '--report-dpath', str(report_dpath), - ] - if args.allow_single_repeat: - cmd.append('--allow-single-repeat') - try: - subprocess.run(cmd, check=True) - except subprocess.CalledProcessError as ex: - skipped_run_entries.append({ - 'run_entry': run_entry, - 'reason': 'rebuild_failed', - 'returncode': ex.returncode, - }) - continue - built_report_paths.append(report_dpath / 'core_metric_report.latest.json') - - summary_rows = [] - for report_json in built_report_paths: - if not report_json.exists(): - continue - report = _load_json(report_json) - report_dir = report_json.parent - selection_fpath = report_dir / 'report_selection.latest.json' - selection = _load_json(selection_fpath) if selection_fpath.exists() else {} - repeat = _find_pair(report, 'kwdagger_repeat') - official = _find_pair(report, 'official_vs_kwdagger') - summary_rows.append({ - 'experiment_name': args.experiment_name, - 'run_spec_name': report.get('run_spec_name'), - 'run_entry': selection.get('run_entry'), - 'report_dir': str(report_dir), - 'generated_utc': report.get('generated_utc'), - 'diagnostic_flags': report.get('diagnostic_flags', []), - 'kwdagger_a_empty_completion_rate': (((report.get('run_diagnostics') or {}).get('kwdagger_a') or {}).get('empty_completion_rate')), - 'kwdagger_a_mean_output_tokens': ((((report.get('run_diagnostics') or {}).get('kwdagger_a') or {}).get('output_token_count') or {}).get('mean')), - 'official_empty_completion_rate': (((report.get('run_diagnostics') or {}).get('official') or {}).get('empty_completion_rate')), - 'official_mean_output_tokens': ((((report.get('run_diagnostics') or {}).get('official') or {}).get('output_token_count') or {}).get('mean')), - 'repeat_instance_agree_0': _find_curve_value(repeat.get('instance_level', {}).get('agreement_vs_abs_tol', []), 0.0), - 'official_instance_agree_0': _find_curve_value(official.get('instance_level', {}).get('agreement_vs_abs_tol', []), 0.0), - 'official_instance_agree_01': _find_curve_value(official.get('instance_level', {}).get('agreement_vs_abs_tol', []), 0.1), - 'official_instance_agree_025': _find_curve_value(official.get('instance_level', {}).get('agreement_vs_abs_tol', []), 0.25), - 'official_instance_agree_05': _find_curve_value(official.get('instance_level', {}).get('agreement_vs_abs_tol', []), 0.5), - 'official_runlevel_p90': (((official.get('run_level') or {}).get('overall_quantiles') or {}).get('abs_delta') or {}).get('p90'), - 'official_runlevel_max': (((official.get('run_level') or {}).get('overall_quantiles') or {}).get('abs_delta') or {}).get('max'), - }) - - compare_pair_script = audit_root() / 'python' / 'compare_pair.py' - paper_labels = load_paper_label_manager(style='paper_short') - summary_by_run_spec = { - row['run_spec_name']: row - for row in summary_rows - if row.get('run_spec_name') - } - cross_machine_rows: list[dict[str, Any]] = [] - for run_spec_name, summary_row in summary_by_run_spec.items(): - run_entry = summary_row.get('run_entry') - if not run_entry: - continue - experiment_match = next( - ( - row for row in experiment_rows - if row.get('run_entry') == run_entry and _is_truthy_text(row.get('has_run_spec')) and row.get('run_dir') - ), - None, - ) - if experiment_match is None: - continue - aiq_gpu_match = _latest_matching_aiq_gpu_row( - rows, - run_entry=run_entry, - exclude_experiment_name=args.experiment_name, - ) - if aiq_gpu_match is None: - continue - machine_a = 'aiq-gpu' - machine_b = str(experiment_match.get('machine_host') or args.experiment_name) - display_label_a = paper_labels.machine_label(machine_a) - display_label_b = paper_labels.machine_label(machine_b) - cross_report_dpath = Path(summary_row['report_dir']) / 'cross-machine-aiq-gpu' - cross_report_dpath.mkdir(parents=True, exist_ok=True) - compare_cmd = [ - env_defaults()['AIQ_PYTHON'], - str(compare_pair_script), - '--run-a', str(aiq_gpu_match['run_dir']), - '--run-b', str(experiment_match['run_dir']), - '--label-a', machine_a, - '--label-b', machine_b, - '--display-label-a', display_label_a, - '--display-label-b', display_label_b, - '--report-dpath', str(cross_report_dpath), - ] - subprocess.run(compare_cmd, check=True) - latest_links = _write_latest_pair_aliases(cross_report_dpath) - cross_json_fpath = cross_report_dpath / 'pair_report.latest.json' - cross_txt_fpath = cross_report_dpath / 'pair_report.latest.txt' - cross_payload = _load_json(cross_json_fpath) if cross_json_fpath.exists() else {} - strict = cross_payload.get('strict_summary', {}) or {} - cross_diag = (strict.get('diagnosis', {}) or {}) - cross_overall = ((strict.get('value_agreement', {}) or {}).get('overall', {}) or {}) - cross_means = ((strict.get('instance_value_agreement', {}) or {}).get('means', {}) or {}) - row = { - 'run_spec_name': run_spec_name, - 'run_entry': run_entry, - 'machine_a': machine_a, - 'machine_b': machine_b, - 'machine_a_display': display_label_a, - 'machine_b_display': display_label_b, - 'report_dir': str(cross_report_dpath), - 'report_json': str(cross_json_fpath) if cross_json_fpath.exists() else None, - 'report_txt': str(cross_txt_fpath) if cross_txt_fpath.exists() else None, - 'diagnosis_label': cross_diag.get('label'), - 'primary_reason_names': cross_diag.get('primary_reason_names'), - 'run_level_agree_ratio': cross_overall.get('agree_ratio'), - 'instance_level_agree_ratio': cross_means.get('agree_ratio'), - 'run_level_abs_p90': ((((cross_payload.get('distance_summary') or {}).get('run_level') or {}).get('overall') or {}).get('abs_delta') or {}).get('p90'), - 'run_level_abs_max': ((((cross_payload.get('distance_summary') or {}).get('run_level') or {}).get('overall') or {}).get('abs_delta') or {}).get('max'), - 'instance_level_abs_p90': ((((cross_payload.get('distance_summary') or {}).get('instance_level') or {}).get('overall') or {}).get('abs_delta') or {}).get('p90'), - 'instance_level_abs_max': ((((cross_payload.get('distance_summary') or {}).get('instance_level') or {}).get('overall') or {}).get('abs_delta') or {}).get('max'), - 'latest_links': latest_links, - } - summary_row['cross_machine_aiq_gpu'] = row - cross_machine_rows.append(row) - - stamp = datetime_mod.datetime.now(datetime_mod.UTC).strftime('%Y%m%dT%H%M%SZ') - history_dpath = out_dpath / '.history' / stamp[:8] - history_dpath.mkdir(parents=True, exist_ok=True) - - table = pd.DataFrame(summary_rows).sort_values('run_spec_name') - json_fpath = history_dpath / f'experiment_summary_{stamp}.json' - csv_fpath = history_dpath / f'experiment_summary_{stamp}.csv' - txt_fpath = history_dpath / f'experiment_summary_{stamp}.txt' - - payload = { - 'generated_utc': stamp, - 'experiment_name': args.experiment_name, - 'index_fpath': str(index_fpath), - 'n_run_entries': len(run_entries), - 'n_built_reports': len(summary_rows), - 'n_skipped_run_entries': len(skipped_run_entries), - 'run_entries': run_entries, - 'skipped_run_entries': skipped_run_entries, - 'cross_machine_rows': cross_machine_rows, - 'rows': summary_rows, - } - json_fpath.write_text(json.dumps(payload, indent=2)) - table.to_csv(csv_fpath, index=False) - - lines = [] - lines.append('Experiment Analysis Summary') - lines.append('') - lines.append(f'generated_utc: {stamp}') - lines.append(f'experiment_name: {args.experiment_name}') - lines.append(f'index_fpath: {index_fpath}') - lines.append(f'n_run_entries: {len(run_entries)}') - lines.append(f'n_built_reports: {len(summary_rows)}') - lines.append(f'n_skipped_run_entries: {len(skipped_run_entries)}') - lines.append('') - lines.append('run_entries:') - for run_entry in run_entries: - lines.append(f' - {run_entry}') - if skipped_run_entries: - lines.append('') - lines.append('skipped_run_entries:') - for item in skipped_run_entries: - lines.append(f" - run_entry: {item['run_entry']}") - lines.append(f" reason: {item['reason']}") - lines.append(f" returncode: {item['returncode']}") - lines.append('') - if cross_machine_rows: - lines.append('cross_machine_aiq_gpu:') - for row in cross_machine_rows: - lines.append(f" - run_spec_name: {row['run_spec_name']}") - lines.append(f" machine_a: {row['machine_a']}") - lines.append(f" machine_a_display: {row['machine_a_display']}") - lines.append(f" machine_b: {row['machine_b']}") - lines.append(f" machine_b_display: {row['machine_b_display']}") - lines.append(f" report_dir: {row['report_dir']}") - lines.append(f" report_txt: {row['report_txt']}") - lines.append(f" diagnosis_label: {row['diagnosis_label']}") - lines.append(f" primary_reason_names: {row['primary_reason_names']}") - lines.append(f" run_level_agree_ratio: {row['run_level_agree_ratio']}") - lines.append(f" instance_level_agree_ratio: {row['instance_level_agree_ratio']}") - lines.append(f" run_level_abs_p90: {row['run_level_abs_p90']}") - lines.append(f" run_level_abs_max: {row['run_level_abs_max']}") - lines.append(f" instance_level_abs_p90: {row['instance_level_abs_p90']}") - lines.append(f" instance_level_abs_max: {row['instance_level_abs_max']}") - lines.append('') - lines.append('per_run_spec:') - for row in summary_rows: - lines.append(f" - run_spec_name: {row['run_spec_name']}") - lines.append(f" report_dir: {row['report_dir']}") - lines.append(f" diagnostic_flags: {row['diagnostic_flags']}") - lines.append(f" kwdagger_a_empty_completion_rate: {row['kwdagger_a_empty_completion_rate']}") - lines.append(f" kwdagger_a_mean_output_tokens: {row['kwdagger_a_mean_output_tokens']}") - lines.append(f" official_empty_completion_rate: {row['official_empty_completion_rate']}") - lines.append(f" official_mean_output_tokens: {row['official_mean_output_tokens']}") - lines.append(f" repeat_instance_agree_0: {row['repeat_instance_agree_0']}") - lines.append(f" official_instance_agree_0: {row['official_instance_agree_0']}") - lines.append(f" official_instance_agree_01: {row['official_instance_agree_01']}") - lines.append(f" official_instance_agree_025: {row['official_instance_agree_025']}") - lines.append(f" official_instance_agree_05: {row['official_instance_agree_05']}") - lines.append(f" official_runlevel_p90: {row['official_runlevel_p90']}") - lines.append(f" official_runlevel_max: {row['official_runlevel_max']}") - txt_fpath.write_text('\n'.join(lines) + '\n') - - _write_latest_alias(json_fpath, out_dpath, 'experiment_summary.latest.json') - _write_latest_alias(csv_fpath, out_dpath, 'experiment_summary.latest.csv') - _write_latest_alias(txt_fpath, out_dpath, 'experiment_summary.latest.txt') - - print(f'Wrote experiment summary json: {json_fpath}') - print(f'Wrote experiment summary csv: {csv_fpath}') - print(f'Wrote experiment summary txt: {txt_fpath}') - - -if __name__ == '__main__': - main() diff --git a/dev/experiments/audit-helm-reproduction/python/build_repro_manifest.py b/dev/experiments/audit-helm-reproduction/python/build_repro_manifest.py deleted file mode 100644 index a750d38..0000000 --- a/dev/experiments/audit-helm-reproduction/python/build_repro_manifest.py +++ /dev/null @@ -1,275 +0,0 @@ -from __future__ import annotations - -import argparse -import fnmatch -import math -from pathlib import Path -from typing import Any - -import kwutil - -from common import dump_yaml, env_defaults, repo_run_details_fpath, repo_run_specs_fpath - - -VICUNA_NOCHAT_OVERRIDE = ( - "dev/experiments/audit-helm-reproduction/configs/debug/" - "vicuna_no_chat_template.yaml" -) - - -def _load_run_specs(fpath: str | None) -> list[str]: - path = Path(fpath) if fpath else repo_run_specs_fpath() - data = kwutil.Yaml.load(path) - if not isinstance(data, list): - raise TypeError(f"run specs at {path} must decode to a list") - run_specs = [str(x) for x in data] - return list(dict.fromkeys(run_specs)) - - -def _load_run_details(fpath: str | None) -> list[dict[str, Any]]: - path = Path(fpath) if fpath else repo_run_details_fpath() - if not path.exists(): - return [] - data = kwutil.Yaml.load(path) - if not isinstance(data, list): - raise TypeError(f"run details at {path} must decode to a list") - rows = [row for row in data if isinstance(row, dict)] - return rows - - -def _matches_any(text: str, patterns: list[str]) -> bool: - return any(fnmatch.fnmatch(text, pat) for pat in patterns) - - -def _infer_benchmark(run_entry: str) -> str: - left = run_entry.split(":", 1)[0] - return left.split(",", 1)[0] - - -def _infer_model(run_entry: str) -> str | None: - for part in run_entry.replace(":", ",").split(","): - if part.startswith("model="): - return part.split("=", 1)[1] - return None - - -def _sort_key(run_entry: str) -> tuple[str, str, str]: - model = _infer_model(run_entry) or "" - benchmark = _infer_benchmark(run_entry) - return (model, benchmark, run_entry) - - -def _filter_run_entries( - run_entries: list[str], - *, - include_patterns: list[str], - exclude_patterns: list[str], - models: list[str], - benchmarks: list[str], -) -> list[str]: - filtered = [] - for run_entry in run_entries: - if include_patterns and not _matches_any(run_entry, include_patterns): - continue - if exclude_patterns and _matches_any(run_entry, exclude_patterns): - continue - model = _infer_model(run_entry) - benchmark = _infer_benchmark(run_entry) - if models and model not in set(models): - continue - if benchmarks and benchmark not in set(benchmarks): - continue - filtered.append(run_entry) - return filtered - - -def _shard_entries( - run_entries: list[str], - *, - num_shards: int | None, - shard_index: int | None, -) -> list[str]: - if num_shards is None and shard_index is None: - return run_entries - if num_shards is None or shard_index is None: - raise SystemExit("--num-shards and --shard-index must be provided together") - if num_shards <= 0: - raise SystemExit("--num-shards must be positive") - if shard_index < 0 or shard_index >= num_shards: - raise SystemExit("--shard-index must satisfy 0 <= shard-index < num-shards") - return [entry for idx, entry in enumerate(run_entries) if idx % num_shards == shard_index] - - -def _choose_model_override(run_entries: list[str], force_nochat: bool) -> str | None: - models = {_infer_model(entry) for entry in run_entries} - needs_vicuna = "lmsys/vicuna-7b-v1.3" in models - if force_nochat or needs_vicuna: - return VICUNA_NOCHAT_OVERRIDE - return None - - -def _detail_lut(rows: list[dict[str, Any]]) -> dict[str, dict[str, Any]]: - lut: dict[str, dict[str, Any]] = {} - for row in rows: - key = row.get("run_spec_name") - if isinstance(key, str) and key not in lut: - lut[key] = row - return lut - - -def _build_manifest( - *, - experiment_name: str, - description: str, - suite: str, - run_entries: list[str], - max_eval_instances: int, - tmux_workers: int, - devices: str, - model_deployments_fpath: str | None, -) -> dict[str, Any]: - return { - "schema_version": 1, - "experiment_name": experiment_name, - "description": description, - "run_entries": run_entries, - "max_eval_instances": max_eval_instances, - "suite": suite, - "mode": "compute_if_missing", - "materialize": "symlink", - "backend": "tmux", - "devices": devices, - "tmux_workers": tmux_workers, - "local_path": "prod_env", - "precomputed_root": None, - "require_per_instance_stats": True, - "model_deployments_fpath": model_deployments_fpath, - "enable_huggingface_models": [], - "enable_local_huggingface_models": [], - } - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--output", required=True) - parser.add_argument("--selection-output", default=None) - parser.add_argument("--run-specs-fpath", default=None) - parser.add_argument("--run-details-fpath", default=None) - parser.add_argument("--experiment-name", required=True) - parser.add_argument("--suite", required=True) - parser.add_argument("--description", default=None) - parser.add_argument("--devices", default=None) - parser.add_argument("--tmux-workers", type=int, default=None) - parser.add_argument("--max-eval-instances", type=int, default=None) - parser.add_argument("--limit", type=int, default=None) - parser.add_argument("--num-shards", type=int, default=None) - parser.add_argument("--shard-index", type=int, default=None) - parser.add_argument("--single-gpu", action="store_true") - parser.add_argument("--sort", default="model_benchmark", choices=["model_benchmark", "input"]) - parser.add_argument("--force-vicuna-nochat", action="store_true") - parser.add_argument("--include-pattern", action="append", default=[]) - parser.add_argument("--exclude-pattern", action="append", default=[]) - parser.add_argument("--model", action="append", default=[]) - parser.add_argument("--benchmark", action="append", default=[]) - args = parser.parse_args() - - defaults = env_defaults() - run_entries = _load_run_specs(args.run_specs_fpath) - run_details = _load_run_details(args.run_details_fpath) - detail_lut = _detail_lut(run_details) - - run_entries = _filter_run_entries( - run_entries, - include_patterns=args.include_pattern, - exclude_patterns=args.exclude_pattern, - models=args.model, - benchmarks=args.benchmark, - ) - if args.sort == "model_benchmark": - run_entries = sorted(run_entries, key=_sort_key) - run_entries = _shard_entries( - run_entries, - num_shards=args.num_shards, - shard_index=args.shard_index, - ) - if args.limit is not None: - run_entries = run_entries[: args.limit] - if not run_entries: - raise SystemExit("No run entries matched the requested filters") - - max_eval_instances = ( - args.max_eval_instances - if args.max_eval_instances is not None - else 1000 - ) - if args.single_gpu: - devices = args.devices if args.devices is not None else "0" - tmux_workers = args.tmux_workers if args.tmux_workers is not None else 1 - else: - devices = args.devices if args.devices is not None else "0,1" - tmux_workers = ( - args.tmux_workers - if args.tmux_workers is not None - else int(defaults["AUDIT_DEFAULT_TMUX_WORKERS"]) - ) - - model_override = _choose_model_override(run_entries, args.force_vicuna_nochat) - description = args.description or ( - f"Historic reproducibility batch with {len(run_entries)} run entries" - ) - manifest = _build_manifest( - experiment_name=args.experiment_name, - description=description, - suite=args.suite, - run_entries=run_entries, - max_eval_instances=max_eval_instances, - tmux_workers=tmux_workers, - devices=devices, - model_deployments_fpath=model_override, - ) - - out_fpath = Path(args.output) - out_fpath.parent.mkdir(parents=True, exist_ok=True) - out_fpath.write_text(dump_yaml(manifest)) - - selection_rows = [] - for idx, run_entry in enumerate(run_entries): - row = { - "index": idx, - "run_entry": run_entry, - "benchmark": _infer_benchmark(run_entry), - "model": _infer_model(run_entry), - "detail": detail_lut.get(run_entry), - } - selection_rows.append(row) - - selection = { - "experiment_name": args.experiment_name, - "suite": args.suite, - "manifest_fpath": str(out_fpath), - "selection_count": len(run_entries), - "num_shards": args.num_shards, - "shard_index": args.shard_index, - "limit": args.limit, - "devices": devices, - "tmux_workers": tmux_workers, - "max_eval_instances": max_eval_instances, - "model_deployments_fpath": model_override, - "include_patterns": args.include_pattern, - "exclude_patterns": args.exclude_pattern, - "models": args.model, - "benchmarks": args.benchmark, - "entries": selection_rows, - } - selection_fpath = ( - Path(args.selection_output) - if args.selection_output - else out_fpath.with_suffix(out_fpath.suffix + ".selection.yaml") - ) - selection_fpath.write_text(dump_yaml(selection)) - print(out_fpath) - print(selection_fpath) - - -if __name__ == "__main__": - main() diff --git a/dev/experiments/audit-helm-reproduction/python/common.py b/dev/experiments/audit-helm-reproduction/python/common.py deleted file mode 100644 index 7fdfd83..0000000 --- a/dev/experiments/audit-helm-reproduction/python/common.py +++ /dev/null @@ -1,68 +0,0 @@ -from __future__ import annotations - -import os -from pathlib import Path -from typing import Any - -import kwutil - - -def env_defaults() -> dict[str, str]: - return { - "AIQ_MAGNET_ROOT": os.environ.get( - "AIQ_MAGNET_ROOT", "/home/joncrall/code/aiq-magnet" - ), - "AIQ_PYTHON": os.environ.get("AIQ_PYTHON", "python"), - "HELM_PRECOMPUTED_ROOT": os.environ.get( - "HELM_PRECOMPUTED_ROOT", "/data/crfm-helm-public" - ), - "AUDIT_RESULTS_ROOT": os.environ.get( - "AUDIT_RESULTS_ROOT", "/data/crfm-helm-audit" - ), - "AUDIT_DEFAULT_MAX_EVAL_INSTANCES": os.environ.get( - "AUDIT_DEFAULT_MAX_EVAL_INSTANCES", "100" - ), - "AUDIT_DEFAULT_TMUX_WORKERS": os.environ.get( - "AUDIT_DEFAULT_TMUX_WORKERS", "2" - ), - } - - -def audit_root() -> Path: - return Path(__file__).resolve().parent.parent - - -def aiq_root() -> Path: - return Path(env_defaults()["AIQ_MAGNET_ROOT"]).expanduser().resolve() - - -def repo_run_specs_fpath() -> Path: - return aiq_root() / "run_specs.yaml" - - -def repo_run_details_fpath() -> Path: - return aiq_root() / "run_details.yaml" - - -def default_report_root() -> Path: - return audit_root() / "reports" - - -def load_manifest(manifest_fpath: str | os.PathLike[str]) -> dict[str, Any]: - data = kwutil.Yaml.load(Path(manifest_fpath)) - if not isinstance(data, dict): - raise TypeError("Manifest must decode to a dictionary") - return data - - -def dump_yaml(data: Any) -> str: - return kwutil.Yaml.dumps(data) - - -def experiment_result_dpath(manifest: dict[str, Any]) -> Path: - root = Path(env_defaults()["AUDIT_RESULTS_ROOT"]).expanduser().resolve() - return root / str(manifest["experiment_name"]) - - -def experiment_report_dpath(manifest: dict[str, Any]) -> Path: - return default_report_root() / str(manifest["experiment_name"]) diff --git a/dev/experiments/audit-helm-reproduction/python/compare_batch.py b/dev/experiments/audit-helm-reproduction/python/compare_batch.py deleted file mode 100644 index 557f216..0000000 --- a/dev/experiments/audit-helm-reproduction/python/compare_batch.py +++ /dev/null @@ -1,644 +0,0 @@ -from __future__ import annotations - -import argparse -import datetime as datetime_mod -import importlib.util -import json -from collections import Counter, defaultdict -from pathlib import Path -from typing import Any - -import kwutil -import ubelt as ub - -from common import ( - env_defaults, - experiment_report_dpath, - experiment_result_dpath, - load_manifest, -) -from magnet.backends.helm.cli.materialize_helm_run import discover_benchmark_output_dirs -from magnet.backends.helm.cli.materialize_helm_run import run_dir_matches_requested -from magnet.backends.helm.helm_outputs import HelmOutputs, HelmRun -from magnet.backends.helm.helm_run_diff import HelmRunDiff - - -def parse_helm_run_dir(run_dir: str) -> dict[str, str]: - p = ub.Path(run_dir) - parts = list(p.parts) - out = { - "helm_suite_name": "unknown", - "helm_version": "unknown", - "run_leaf": p.name, - } - try: - idx = parts.index("benchmark_output") - except ValueError: - idx = -1 - if idx >= 1: - out["helm_suite_name"] = str(parts[idx - 1]) - if idx >= 0 and (idx + 2) < len(parts): - out["helm_version"] = str(parts[idx + 2]) - else: - out["helm_version"] = str(p.parent.name) - return out - - -def load_run_spec_json(run_dir: str | Path) -> dict[str, Any]: - run_dir = Path(run_dir) - fpath = run_dir / "run_spec.json" - if not fpath.exists(): - return {} - return json.loads(fpath.read_text()) - - -def infer_benchmark_group(run_spec_name: str | None) -> str: - text = (run_spec_name or "").strip() - if not text: - return "unknown" - idxs = [i for i in [text.find(":"), text.find(",")] if i >= 0] - if idxs: - return text[: min(idxs)].strip() - return text - - -def collect_historic_candidates( - precomputed_root: str | Path, - run_entry: str, -) -> list[dict[str, Any]]: - candidates = [] - for bo in discover_benchmark_output_dirs([precomputed_root]): - try: - outputs = HelmOutputs.coerce(bo) - except Exception: - continue - for suite in outputs.suites(pattern="*"): - for run in suite.runs(pattern="*"): - run_dir = Path(run.path) - if not run_dir_matches_requested(run.name, run_entry): - continue - run_spec = load_run_spec_json(run_dir) - adapter_spec = run_spec.get("adapter_spec", {}) or {} - metric_specs = run_spec.get("metric_specs", []) or [] - candidates.append( - { - "run_dir": run_dir, - "run_name": run.name, - "source_root": bo, - "helm_version": run_dir.parent.name, - "requested_max_eval_instances": adapter_spec.get( - "max_eval_instances", None - ), - "model_deployment": adapter_spec.get( - "model_deployment", None - ), - "metric_class_names": [ - m.get("class_name", None) for m in metric_specs - ], - } - ) - return candidates - - -def choose_historic_candidate( - candidates: list[dict[str, Any]], - desired_max_eval_instances: int | None, -) -> tuple[dict[str, Any] | None, dict[str, Any]]: - if not candidates: - return None, { - "candidate_count": 0, - "exact_requested_max_eval_match": False, - "candidate_requested_max_eval_instances": [], - } - exact_matches = [] - if desired_max_eval_instances is not None: - exact_matches = [ - c - for c in candidates - if c.get("requested_max_eval_instances", None) - == desired_max_eval_instances - ] - ranked_pool = exact_matches if exact_matches else candidates - - def sort_key(c: dict[str, Any]): - req = c.get("requested_max_eval_instances", None) - req_dist = ( - abs(req - desired_max_eval_instances) - if req is not None and desired_max_eval_instances is not None - else float("inf") - ) - return (req_dist, str(c.get("helm_version", "")), str(c["run_dir"])) - - chosen = sorted(ranked_pool, key=sort_key)[0] - info = { - "candidate_count": len(candidates), - "exact_requested_max_eval_match": bool(exact_matches), - "candidate_requested_max_eval_instances": sorted( - {c.get("requested_max_eval_instances", None) for c in candidates}, - key=lambda x: (x is None, x), - ), - "chosen_requested_max_eval_instances": chosen.get( - "requested_max_eval_instances", None - ), - } - return chosen, info - - -def load_kwdg_rows(results_dpath: Path) -> tuple[list[dict[str, Any]], dict[str, dict[str, Any]]]: - finished_jobs = sorted( - fpath - for fpath in results_dpath.rglob("DONE") - if (fpath.parent / "job_config.json").exists() - ) - rows = [] - for fpath in ub.ProgIter(finished_jobs, desc="load kwdg runs"): - dpath = fpath.parent - try: - config = kwutil.Json.load(dpath / "job_config.json") - run_spec_name = config.get("helm.run_entry", None) - if run_spec_name is None: - continue - suites = HelmOutputs.coerce(dpath / "benchmark_output").suites() - runs = [] - for suite in suites: - runs.extend(list(suite.runs())) - if len(runs) != 1: - continue - run = HelmRun.coerce(runs[0]) - rows.append( - { - "dpath": str(dpath), - "run_spec_name": run_spec_name, - "run": run, - } - ) - except Exception: - continue - - lut = {} - for row in rows: - lut[row["run_spec_name"]] = row - return rows, lut - - -def aggregate_report(rows: list[dict[str, Any]]) -> dict[str, Any]: - status_counter = Counter() - diagnosis_counter = Counter() - reason_counter = Counter() - primary_reason_counter = Counter() - for row in rows: - status = row.get("status", "unknown") - status_counter[status] += 1 - if status != "compared": - continue - diag = row.get("diagnosis", {}) or {} - diagnosis_counter[diag.get("label", "unknown")] += 1 - for reason_name in diag.get("primary_reason_names", []) or []: - primary_reason_counter[reason_name] += 1 - for reason in diag.get("reasons", []) or []: - name = reason.get("name", "unknown") - reason_counter[name] += 1 - return { - "n_rows": len(rows), - "status_counts": dict(status_counter), - "diagnosis_label_counts": dict(diagnosis_counter), - "primary_reason_name_counts": dict(primary_reason_counter), - "reason_counts": dict(reason_counter), - } - - -def build_high_level_findings(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: - findings = [] - n_rows = len(rows) - if n_rows == 0: - return findings - - compared = sum(1 for row in rows if row.get("status") == "compared") - if compared == n_rows: - findings.append( - { - "label": "comparison_pipeline_working", - "severity": "info", - "summary": f"All {compared} requested runs were paired and compared successfully.", - } - ) - - requested_max_mismatch = sum( - 1 - for row in rows - if row.get("historic_requested_max_eval_instances", None) - != row.get("kwdg_requested_max_eval_instances", None) - ) - if requested_max_mismatch: - findings.append( - { - "label": "requested_max_eval_mismatch", - "severity": "high", - "summary": ( - f"{requested_max_mismatch}/{n_rows} cases use a historic run with " - "a different requested max_eval_instances value." - ), - } - ) - - no_exact_match = sum( - 1 - for row in rows - if not row.get("historic_exact_requested_max_eval_match", False) - ) - if no_exact_match: - findings.append( - { - "label": "no_exact_historic_eval_size_match", - "severity": "high", - "summary": ( - f"{no_exact_match}/{n_rows} cases did not have an exact historic match " - "for requested max_eval_instances in the public bundle." - ), - } - ) - - for reason_name, severity in [ - ("deployment_drift", "high"), - ("execution_spec_drift", "high"), - ("evaluation_spec_drift", "medium"), - ("dataset_variant_drift", "medium"), - ("dataset_instance_drift", "medium"), - ("core_metric_drift", "medium"), - ("completion_content_drift", "medium"), - ]: - count = sum( - 1 - for row in rows - if any( - reason.get("name") == reason_name - for reason in ((row.get("diagnosis", {}) or {}).get("reasons", []) or []) - ) - ) - if count: - findings.append( - { - "label": reason_name, - "severity": severity, - "summary": f"{reason_name} appears in {count}/{n_rows} compared cases.", - } - ) - return findings - - -def maybe_write_sankey_report( - case_rows: list[dict[str, Any]], report_dpath: Path, stamp: str -) -> dict[str, Any]: - source_fpath = ( - Path(__file__).resolve().parents[3] - / "oneoff" - / "diagnose_reproducibility.py" - ) - if not source_fpath.exists(): - return {"plotly_error": "sankey helper script not found"} - - spec = importlib.util.spec_from_file_location( - "audit_diag_helper", source_fpath - ) - if spec is None or spec.loader is None: - return {"plotly_error": "unable to import sankey helper"} - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module.write_sankey_report( - case_rows, report_dpath=ub.Path(report_dpath), stamp=stamp - ) - - -def build_historic_rows( - manifest: dict[str, Any], precomputed_root: str -) -> list[dict[str, Any]]: - rows = [] - for run_entry in manifest["run_entries"]: - desired_max_eval_instances = manifest.get("max_eval_instances", None) - candidates = collect_historic_candidates( - precomputed_root=precomputed_root, - run_entry=run_entry, - ) - match, match_info = choose_historic_candidate( - candidates, desired_max_eval_instances - ) - row = { - "run_spec_name": run_entry, - "run_dir": None, - "model": None, - "benchmark_group": infer_benchmark_group(run_entry), - "benchmark_name": "unknown", - "suite_name": "unknown", - "helm_version": "unknown", - "requested_max_eval_instances": None, - "model_deployment": None, - "metric_class_names": [], - "match_info": match_info, - } - if match is not None: - parsed = parse_helm_run_dir(str(match["run_dir"])) - row["run_dir"] = str(match["run_dir"]) - row["benchmark_name"] = parsed["helm_suite_name"] - row["suite_name"] = parsed["helm_suite_name"] - row["helm_version"] = str(match.get("helm_version", parsed["helm_version"])) - row["requested_max_eval_instances"] = match.get( - "requested_max_eval_instances", None - ) - row["model_deployment"] = match.get("model_deployment", None) - row["metric_class_names"] = match.get("metric_class_names", []) - if "model=" in run_entry: - model_text = run_entry.split("model=", 1)[1].split(",", 1)[0] - row["model"] = model_text - rows.append(row) - return rows - - -def write_summary_text( - summary_report: dict[str, Any], out_fpath: Path -) -> None: - inputs = summary_report.get("inputs", {}) or {} - lines = [] - lines.append(f"generated_utc: {summary_report['generated_utc']}") - lines.append(f"case_jsonl: {summary_report['report_case_jsonl']}") - lines.append(f"summary_json: {summary_report['report_summary_json']}") - if inputs: - lines.append("") - lines.append("inputs:") - for key, value in sorted(inputs.items()): - lines.append(f" {key}: {value}") - lines.append("") - lines.append("status_counts:") - for key, value in sorted( - summary_report["aggregate"]["status_counts"].items() - ): - lines.append(f" {key}: {value}") - lines.append("") - lines.append("diagnosis_label_counts:") - for key, value in sorted( - summary_report["aggregate"]["diagnosis_label_counts"].items() - ): - lines.append(f" {key}: {value}") - lines.append("") - lines.append("primary_reason_name_counts:") - for key, value in sorted( - summary_report["aggregate"].get("primary_reason_name_counts", {}).items() - ): - lines.append(f" {key}: {value}") - lines.append("") - lines.append("reason_counts:") - for key, value in sorted( - summary_report["aggregate"].get("reason_counts", {}).items() - ): - lines.append(f" {key}: {value}") - findings = summary_report.get("high_level_findings", []) or [] - if findings: - lines.append("") - lines.append("high_level_findings:") - for item in findings: - lines.append( - f" [{item.get('severity', 'info')}] {item.get('label')}: {item.get('summary')}" - ) - out_fpath.write_text("\n".join(lines) + "\n") - - -def write_management_summary( - summary_report: dict[str, Any], out_fpath: Path -) -> None: - inputs = summary_report.get("inputs", {}) or {} - findings = summary_report.get("high_level_findings", []) or [] - aggregate = summary_report.get("aggregate", {}) or {} - status_counts = aggregate.get("status_counts", {}) or {} - compared = status_counts.get("compared", 0) - total = sum(status_counts.values()) - lines = [] - lines.append("Audit HELM Reproduction: Executive Summary") - lines.append("") - lines.append( - f"Compared {inputs.get('n_manifest_run_entries', '?')} requested runs against " - f"{inputs.get('n_historic_rows', '?')} historic matches and " - f"{inputs.get('n_kwdg_rows', '?')} reproduced runs." - ) - lines.append(f"{compared}/{total} runs completed comparison successfully.") - lines.append("") - lines.append("Key findings:") - for item in findings: - lines.append(f"- [{item.get('severity', 'info').upper()}] {item.get('summary')}") - out_fpath.write_text("\n".join(lines) + "\n") - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--manifest", required=True) - parser.add_argument("--results-dpath", default=None) - parser.add_argument("--report-dpath", default=None) - parser.add_argument("--precomputed-root", default=None) - args = parser.parse_args() - - manifest = load_manifest(args.manifest) - defaults = env_defaults() - results_dpath = ( - Path(args.results_dpath).expanduser().resolve() - if args.results_dpath - else experiment_result_dpath(manifest) - ) - report_dpath = ( - Path(args.report_dpath).expanduser().resolve() - if args.report_dpath - else experiment_report_dpath(manifest) - ) - precomputed_root = args.precomputed_root or defaults["HELM_PRECOMPUTED_ROOT"] - report_dpath.mkdir(parents=True, exist_ok=True) - - historic_rows = build_historic_rows(manifest, precomputed_root) - kwdg_rows, kwdg_lut = load_kwdg_rows(results_dpath) - - stamp = datetime_mod.datetime.now(datetime_mod.UTC).strftime( - "%Y%m%dT%H%M%SZ" - ) - case_jsonl_fpath = report_dpath / f"compare_cases_{stamp}.jsonl" - summary_json_fpath = report_dpath / f"compare_summary_{stamp}.json" - summary_txt_fpath = report_dpath / f"compare_summary_{stamp}.txt" - management_txt_fpath = report_dpath / f"management_summary_{stamp}.txt" - - all_case_rows = [] - with case_jsonl_fpath.open("w", encoding="utf8") as file: - for idx, helm_row in enumerate(historic_rows, start=1): - run_spec_name = helm_row["run_spec_name"] - kwrow = kwdg_lut.get(run_spec_name, None) - case_row = { - "index": idx, - "run_spec_name": run_spec_name, - "benchmark_name": helm_row.get("benchmark_name", "unknown"), - "benchmark_group": helm_row.get("benchmark_group", "unknown"), - "suite_name": helm_row.get("suite_name", "unknown"), - "model_name": helm_row.get("model", None), - "helm_version": helm_row.get("helm_version", None), - "helm_run_dir": helm_row["run_dir"], - "kwdg_run_dir": None if kwrow is None else kwrow["dpath"], - "historic_requested_max_eval_instances": helm_row.get( - "requested_max_eval_instances", None - ), - "historic_model_deployment": helm_row.get( - "model_deployment", None - ), - "historic_metric_class_names": helm_row.get( - "metric_class_names", [] - ), - "historic_match_info": helm_row.get("match_info", {}), - "historic_exact_requested_max_eval_match": ( - helm_row.get("match_info", {}) or {} - ).get("exact_requested_max_eval_match", False), - "kwdg_requested_max_eval_instances": None, - } - - if helm_row["run_dir"] is None: - case_row.update( - { - "status": "missing_historic_match", - "diagnosis": { - "label": "missing_historic_match", - "primary_priority": 0, - "primary_reason_names": [ - "missing_historic_match" - ], - "reasons": [ - { - "name": "missing_historic_match", - "priority": 0, - "details": {}, - } - ], - }, - } - ) - elif kwrow is None: - case_row.update( - { - "status": "missing_kwdg_match", - "diagnosis": { - "label": "missing_kwdg_match", - "primary_priority": 0, - "primary_reason_names": ["missing_kwdg_match"], - "reasons": [ - { - "name": "missing_kwdg_match", - "priority": 0, - "details": {}, - } - ], - }, - } - ) - else: - try: - helm_run = HelmRun.coerce(helm_row["run_dir"]) - kwdg_run = kwrow["run"] - kwdg_run_spec = load_run_spec_json(kwdg_run.path) - case_row["kwdg_requested_max_eval_instances"] = ( - kwdg_run_spec.get("adapter_spec", {}) or {} - ).get( - "max_eval_instances", None - ) - rd = HelmRunDiff( - run_a=helm_run, - run_b=kwdg_run, - a_name="HELM", - b_name="KWDG", - ) - summary = rd.summary_dict(level=20) - diag = summary.get("diagnosis", {}) or {} - case_row.update( - { - "status": "compared", - "diagnosis": diag, - "run_spec_semantic": summary.get( - "run_spec_semantic", None - ), - "scenario_semantic": summary.get( - "scenario_semantic", None - ), - "dataset_overlap": summary.get( - "dataset_overlap", None - ), - "stats_coverage_by_name": summary.get( - "stats_coverage_by_name", None - ), - "stats_coverage_by_name_count": summary.get( - "stats_coverage_by_name_count", None - ), - "value_agreement": summary.get( - "value_agreement", None - ), - "instance_value_agreement": summary.get( - "instance_value_agreement", None - ), - } - ) - except Exception as ex: - case_row.update( - { - "status": "error", - "error": repr(ex), - "diagnosis": { - "label": "comparison_error", - "primary_priority": 0, - "primary_reason_names": ["comparison_error"], - "reasons": [ - { - "name": "comparison_error", - "priority": 0, - "details": {"error": repr(ex)}, - } - ], - }, - } - ) - - case_row = kwutil.Json.ensure_serializable(case_row) - file.write(json.dumps(case_row, ensure_ascii=False) + "\n") - file.flush() - all_case_rows.append(case_row) - - summary_report = { - "report_case_jsonl": str(case_jsonl_fpath), - "report_summary_json": str(summary_json_fpath), - "report_summary_txt": str(summary_txt_fpath), - "report_management_txt": str(management_txt_fpath), - "generated_utc": stamp, - "inputs": { - "manifest": str(Path(args.manifest).expanduser().resolve()), - "kwdg_results_dpath": str(results_dpath), - "precomputed_root": str(precomputed_root), - "n_manifest_run_entries": len(manifest["run_entries"]), - "n_kwdg_rows": len(kwdg_rows), - "n_historic_rows": len(historic_rows), - }, - "aggregate": aggregate_report(all_case_rows), - "high_level_findings": build_high_level_findings(all_case_rows), - } - try: - sankey_artifacts = maybe_write_sankey_report( - all_case_rows, report_dpath, stamp - ) - except Exception as ex: - sankey_artifacts = { - "plotly_error": f"failed to build sankey report: {ex!r}" - } - summary_report["artifacts"] = sankey_artifacts - summary_report = kwutil.Json.ensure_serializable(summary_report) - summary_json_fpath.write_text( - json.dumps(summary_report, indent=2, ensure_ascii=False) - ) - write_summary_text(summary_report, summary_txt_fpath) - write_management_summary(summary_report, management_txt_fpath) - - print(f"Wrote case report: {case_jsonl_fpath}") - print(f"Wrote summary report: {summary_json_fpath}") - print(f"Wrote summary text: {summary_txt_fpath}") - print(f"Wrote management summary: {management_txt_fpath}") - if sankey_artifacts.get("plotly_error", None): - print(f"Sankey note: {sankey_artifacts['plotly_error']}") - - -if __name__ == "__main__": - main() diff --git a/dev/experiments/audit-helm-reproduction/python/compare_pair.py b/dev/experiments/audit-helm-reproduction/python/compare_pair.py deleted file mode 100644 index 7a3c53b..0000000 --- a/dev/experiments/audit-helm-reproduction/python/compare_pair.py +++ /dev/null @@ -1,192 +0,0 @@ -from __future__ import annotations - -import argparse -import datetime as datetime_mod -import json -from pathlib import Path -from typing import Any - -import kwutil - -from magnet.backends.helm.helm_outputs import HelmRun -from magnet.backends.helm.helm_run_diff import HelmRunDiff - - -def load_yaml_or_default(text: str | None, default: list[dict[str, Any]]) -> list[dict[str, Any]]: - if text is None: - return default - data = kwutil.Yaml.coerce(text) - if not isinstance(data, list): - raise TypeError('Tolerance config must decode to a list of dictionaries') - return data - - -def default_tolerances() -> list[dict[str, Any]]: - return [ - {'name': 'strict', 'abs_tol': 0.0, 'rel_tol': 0.0}, - {'name': 'tiny', 'abs_tol': 1e-12, 'rel_tol': 1e-6}, - {'name': 'small', 'abs_tol': 1e-9, 'rel_tol': 1e-4}, - {'name': 'medium', 'abs_tol': 1e-6, 'rel_tol': 1e-3}, - {'name': 'loose', 'abs_tol': 1e-3, 'rel_tol': 1e-2}, - {'name': 'xloose', 'abs_tol': 1e-2, 'rel_tol': 1e-1}, - {'name': 'xxloose', 'abs_tol': 1e-1, 'rel_tol': 1.0}, - {'name': 'extreme', 'abs_tol': 1.0, 'rel_tol': 10.0}, - ] - - -def validate_run_dir(run_dpath: Path) -> None: - required_files = [ - 'run_spec.json', - 'scenario_state.json', - 'stats.json', - 'per_instance_stats.json', - ] - missing_files = [name for name in required_files if not (run_dpath / name).exists()] - if missing_files: - missing_text = ', '.join(missing_files) - raise SystemExit( - f'Run artifacts are incomplete for {run_dpath}. ' - f'Missing required files: {missing_text}' - ) - - -def summarize_tolerance_hits(sweep: dict[str, Any]) -> dict[str, Any]: - out = {'run_level': [], 'instance_level': []} - for level_key, target in [('run_level', 'overall'), ('instance_level', 'means')]: - for row in sweep.get(level_key, []): - summary = row.get('summary', {}) or {} - if level_key == 'run_level': - agree = ((summary.get('overall', {}) or {}).get('agree_ratio', None)) - else: - agree = ((summary.get('means', {}) or {}).get('agree_ratio', None)) - out[level_key].append({ - 'name': row.get('name'), - 'abs_tol': row.get('abs_tol'), - 'rel_tol': row.get('rel_tol'), - 'agree_ratio': agree, - }) - return out - - -def write_text_report(report: dict[str, Any], out_fpath: Path) -> None: - strict = report.get('strict_summary', {}) or {} - diag = strict.get('diagnosis', {}) or {} - run_dist = report.get('distance_summary', {}).get('run_level', {}) or {} - inst_dist = report.get('distance_summary', {}).get('instance_level', {}) or {} - sweep_hits = report.get('tolerance_highlights', {}) or {} - display = report.get('display_labels', {}) or {} - label_a = display.get('label_a') or report.get('inputs', {}).get('label_a') - label_b = display.get('label_b') or report.get('inputs', {}).get('label_b') - - lines = [] - lines.append('Audit Pair Comparison') - lines.append('') - lines.append(f"generated_utc: {report.get('generated_utc')}") - lines.append(f"run_a: {report.get('inputs', {}).get('run_a')}") - lines.append(f"run_b: {report.get('inputs', {}).get('run_b')}") - lines.append(f'label_a: {label_a}') - lines.append(f'label_b: {label_b}') - lines.append('') - lines.append(f"diagnosis_label: {diag.get('label')}") - lines.append(f"primary_reason_names: {diag.get('primary_reason_names')}") - lines.append('') - - lines.append('strict_agreement:') - overall = (strict.get('value_agreement', {}) or {}).get('overall', {}) or {} - lines.append(f" run_level_agree_ratio: {overall.get('agree_ratio')}") - means = (strict.get('instance_value_agreement', {}) or {}).get('means', {}) or {} - lines.append(f" instance_level_agree_ratio: {means.get('agree_ratio')}") - lines.append('') - - lines.append('distance_summary:') - lines.append(f" run_level_count: {(run_dist.get('overall', {}) or {}).get('count')}") - lines.append(f" run_level_abs_p50: {((run_dist.get('overall', {}) or {}).get('abs_delta', {}) or {}).get('p50')}") - lines.append(f" run_level_abs_p90: {((run_dist.get('overall', {}) or {}).get('abs_delta', {}) or {}).get('p90')}") - lines.append(f" run_level_abs_max: {((run_dist.get('overall', {}) or {}).get('abs_delta', {}) or {}).get('max')}") - lines.append(f" instance_level_count: {(inst_dist.get('overall', {}) or {}).get('count')}") - lines.append(f" instance_level_abs_p50: {((inst_dist.get('overall', {}) or {}).get('abs_delta', {}) or {}).get('p50')}") - lines.append(f" instance_level_abs_p90: {((inst_dist.get('overall', {}) or {}).get('abs_delta', {}) or {}).get('p90')}") - lines.append(f" instance_level_abs_max: {((inst_dist.get('overall', {}) or {}).get('abs_delta', {}) or {}).get('max')}") - lines.append('') - - lines.append('tolerance_sweep_run_level:') - for row in sweep_hits.get('run_level', []): - lines.append( - f" {row.get('name')}: abs_tol={row.get('abs_tol')} rel_tol={row.get('rel_tol')} agree_ratio={row.get('agree_ratio')}" - ) - lines.append('') - lines.append('tolerance_sweep_instance_level:') - for row in sweep_hits.get('instance_level', []): - lines.append( - f" {row.get('name')}: abs_tol={row.get('abs_tol')} rel_tol={row.get('rel_tol')} agree_ratio={row.get('agree_ratio')}" - ) - out_fpath.write_text('\n'.join(lines) + '\n') - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument('--run-a', required=True) - parser.add_argument('--run-b', required=True) - parser.add_argument('--label-a', default='A') - parser.add_argument('--label-b', default='B') - parser.add_argument('--display-label-a', default=None) - parser.add_argument('--display-label-b', default=None) - parser.add_argument('--report-dpath', required=True) - parser.add_argument('--run-tolerances-yaml', default=None) - parser.add_argument('--instance-tolerances-yaml', default=None) - args = parser.parse_args() - - report_dpath = Path(args.report_dpath).expanduser().resolve() - report_dpath.mkdir(parents=True, exist_ok=True) - stamp = datetime_mod.datetime.now(datetime_mod.UTC).strftime('%Y%m%dT%H%M%SZ') - - run_a_dpath = Path(args.run_a).expanduser().resolve() - run_b_dpath = Path(args.run_b).expanduser().resolve() - validate_run_dir(run_a_dpath) - validate_run_dir(run_b_dpath) - - run_a = HelmRun.coerce(run_a_dpath) - run_b = HelmRun.coerce(run_b_dpath) - diff = HelmRunDiff(run_a=run_a, run_b=run_b, a_name=args.label_a, b_name=args.label_b) - - run_tolerances = load_yaml_or_default(args.run_tolerances_yaml, default_tolerances()) - instance_tolerances = load_yaml_or_default(args.instance_tolerances_yaml, default_tolerances()) - - strict_summary = diff.summary_dict(level=20) - distance_summary = { - 'run_level': diff.value_distance_profile(), - 'instance_level': diff.instance_distance_profile(), - } - tolerance_sweep = diff.tolerance_sweep_summary( - run_tolerances=run_tolerances, - instance_tolerances=instance_tolerances, - ) - report = { - 'generated_utc': stamp, - 'inputs': { - 'run_a': str(run_a_dpath), - 'run_b': str(run_b_dpath), - 'label_a': args.label_a, - 'label_b': args.label_b, - }, - 'display_labels': { - 'label_a': args.display_label_a or args.label_a, - 'label_b': args.display_label_b or args.label_b, - }, - 'strict_summary': strict_summary, - 'distance_summary': distance_summary, - 'tolerance_sweep': tolerance_sweep, - 'tolerance_highlights': summarize_tolerance_hits(tolerance_sweep), - } - - json_fpath = report_dpath / f'pair_report_{stamp}.json' - txt_fpath = report_dpath / f'pair_report_{stamp}.txt' - report = kwutil.Json.ensure_serializable(report) - json_fpath.write_text(json.dumps(report, indent=2, ensure_ascii=False)) - write_text_report(report, txt_fpath) - print(f'Wrote pair report: {json_fpath}') - print(f'Wrote pair text: {txt_fpath}') - - -if __name__ == '__main__': - main() diff --git a/dev/experiments/audit-helm-reproduction/python/core_metric_report.py b/dev/experiments/audit-helm-reproduction/python/core_metric_report.py deleted file mode 100644 index bf8824d..0000000 --- a/dev/experiments/audit-helm-reproduction/python/core_metric_report.py +++ /dev/null @@ -1,1017 +0,0 @@ -from __future__ import annotations - -import argparse -import datetime as datetime_mod -import json -import math -import os -import shutil -import statistics -from pathlib import Path -from typing import Any - -import kwutil -import matplotlib.pyplot as plt -import pandas as pd -import seaborn as sns - -from magnet.backends.helm.helm_outputs import HelmRun -from magnet.backends.helm.helm_run_analysis import HelmRunAnalysis -from magnet.backends.helm.helm_run_diff import HelmRunDiff -from magnet.backends.helm.util import helm_metrics -from paper_labels import load_paper_label_manager - - -def _quantile(values: list[float], q: float) -> float | None: - if not values: - return None - values = sorted(values) - if q <= 0: - return values[0] - if q >= 1: - return values[-1] - pos = (len(values) - 1) * q - lo = math.floor(pos) - hi = math.ceil(pos) - if lo == hi: - return values[lo] - alpha = pos - lo - return values[lo] * (1 - alpha) + values[hi] * alpha - - -def _safe_float(x: Any) -> float | None: - try: - if x is None: - return None - return float(x) - except Exception: - return None - - -def _run_level_core_rows(diff: HelmRunDiff) -> list[dict[str, Any]]: - idx_a = diff.a.stat_index(drop_zero_count=True, require_mean=True, short_hash=diff.short_hash) - idx_b = diff.b.stat_index(drop_zero_count=True, require_mean=True, short_hash=diff.short_hash) - rows = [] - for k in set(idx_a) & set(idx_b): - a = idx_a[k] - b = idx_b[k] - if a.mean is None or b.mean is None: - continue - if a.metric_class != 'core': - continue - abs_delta = abs(a.mean - b.mean) - denom = max(abs(a.mean), abs(b.mean), 1e-12) - rel_delta = abs_delta / denom - rows.append({ - 'key': k, - 'metric': a.metric, - 'metric_class': a.metric_class, - 'a': float(a.mean), - 'b': float(b.mean), - 'abs_delta': abs_delta, - 'rel_delta': rel_delta, - }) - return rows - - -def _load_json(fpath: Path) -> Any: - return json.loads(fpath.read_text()) - - -def _load_optional_cross_machine_pair(report_dpath: Path) -> dict[str, Any] | None: - pair_fpath = report_dpath / 'cross-machine-aiq-gpu' / 'pair_report.latest.json' - if not pair_fpath.exists(): - return None - data = _load_json(pair_fpath) - display = data.get('display_labels', {}) or {} - label_a = ( - display.get('label_a') - or ((data.get('inputs') or {}).get('label_a')) - or 'aiq-gpu' - ) - label_b = ( - display.get('label_b') - or ((data.get('inputs') or {}).get('label_b')) - or 'other-machine' - ) - highlights = data.get('tolerance_highlights', {}) or {} - distance = data.get('distance_summary', {}) or {} - strict = data.get('strict_summary', {}) or {} - diagnosis = (strict.get('diagnosis') or {}) - return { - 'label': f'{label_a}_vs_{label_b}', - 'diagnosis': diagnosis, - 'run_level': { - 'agreement_vs_abs_tol': highlights.get('run_level', []) or [], - 'overall_quantiles': (distance.get('run_level') or {}).get('overall', {}) or {}, - }, - 'instance_level': { - 'agreement_vs_abs_tol': highlights.get('instance_level', []) or [], - 'overall_quantiles': (distance.get('instance_level') or {}).get('overall', {}) or {}, - }, - } - - -def _collect_stat_means(stats: list[dict[str, Any]], metric_name: str) -> dict[str, float]: - found = {} - for row in stats: - name = row.get('name') - if not isinstance(name, dict): - continue - if name.get('name') != metric_name: - continue - split = name.get('split') - found[str(split)] = row.get('mean') - return found - - -def _run_diagnostics(run_path: str) -> dict[str, Any]: - run_path = str(Path(run_path).expanduser().resolve()) - run_dpath = Path(run_path) - scenario_state = _load_json(run_dpath / 'scenario_state.json') - stats = _load_json(run_dpath / 'stats.json') - reqs = scenario_state.get('request_states', []) - - token_counts = [] - empty_completion_count = 0 - nonempty_completion_count = 0 - completion_count = 0 - for rs in reqs: - comps = (rs.get('result') or {}).get('completions') or [] - if not comps: - continue - completion_count += 1 - c0 = comps[0] or {} - text = c0.get('text', '') - toklist = c0.get('tokens') or [] - token_counts.append(len(toklist)) - if text == '': - empty_completion_count += 1 - else: - nonempty_completion_count += 1 - - mean_tokens = statistics.mean(token_counts) if token_counts else None - return { - 'run_path': run_path, - 'run_name': run_dpath.name, - 'n_request_states': len(reqs), - 'n_with_completions': completion_count, - 'empty_completion_count': empty_completion_count, - 'nonempty_completion_count': nonempty_completion_count, - 'empty_completion_rate': ( - empty_completion_count / completion_count if completion_count else None - ), - 'output_token_count': { - 'mean': mean_tokens, - 'p50': _quantile(token_counts, 0.5), - 'p90': _quantile(token_counts, 0.9), - 'max': _quantile(token_counts, 1.0), - }, - 'stats_means': { - 'num_output_tokens': _collect_stat_means(stats, 'num_output_tokens'), - 'num_completion_tokens': _collect_stat_means(stats, 'num_completion_tokens'), - 'finish_reason_unknown': _collect_stat_means(stats, 'finish_reason_unknown'), - }, - } - - -def _diagnostic_flags(run_diagnostics: dict[str, dict[str, Any]]) -> list[str]: - flags = [] - for label, diag in run_diagnostics.items(): - rate = diag.get('empty_completion_rate') - mean_tokens = (diag.get('output_token_count') or {}).get('mean') - if rate is not None and rate > 0.1: - flags.append(f'{label}:high_empty_completion_rate') - if mean_tokens is not None and mean_tokens < 1.0: - flags.append(f'{label}:near_zero_mean_output_tokens') - official = run_diagnostics.get('official', {}) - kwdagger_a = run_diagnostics.get('kwdagger_a', {}) - off_rate = official.get('empty_completion_rate') - kwa_rate = kwdagger_a.get('empty_completion_rate') - if off_rate is not None and kwa_rate is not None and off_rate < 0.01 and kwa_rate > 0.1: - flags.append('official_vs_kwdagger_a:empty_completion_pathology') - return flags - - -def _iter_joined_rows(joined, row_by_key): - if row_by_key is not None: - return row_by_key.values() - if isinstance(joined, dict): - return joined.values() - if hasattr(joined, '__iter__'): - return joined - return [] - - -def _row_key(row: Any) -> Any: - return ( - getattr(row, 'key', None) - or getattr(row, 'stat_key', None) - or getattr(row, 'row_key', None) - or row - ) - - -def _instance_level_core_rows(diff: HelmRunDiff) -> list[dict[str, Any]]: - joined_a = diff.a.joined_instance_stat_table(assert_assumptions=False, short_hash=diff.short_hash) - joined_b = diff.b.joined_instance_stat_table(assert_assumptions=False, short_hash=diff.short_hash) - map_a = getattr(joined_a, 'row_by_key', None) - map_b = getattr(joined_b, 'row_by_key', None) - if map_a is None: - map_a = {_row_key(r): r for r in _iter_joined_rows(joined_a, map_a)} - if map_b is None: - map_b = {_row_key(r): r for r in _iter_joined_rows(joined_b, map_b)} - - rows = [] - for k in set(map_a) & set(map_b): - ra = map_a[k] - rb = map_b[k] - sa = getattr(ra, 'stat', None) if hasattr(ra, 'stat') else (ra.get('stat') if isinstance(ra, dict) else None) - sb = getattr(rb, 'stat', None) if hasattr(rb, 'stat') else (rb.get('stat') if isinstance(rb, dict) else None) - ma = _safe_float((sa or {}).get('mean') if isinstance(sa, dict) else getattr(sa, 'mean', None)) - mb = _safe_float((sb or {}).get('mean') if isinstance(sb, dict) else getattr(sb, 'mean', None)) - ca = int((sa or {}).get('count', 0) or 0) if isinstance(sa, dict) else int(getattr(sa, 'count', 0) or 0) - cb = int((sb or {}).get('count', 0) or 0) if isinstance(sb, dict) else int(getattr(sb, 'count', 0) or 0) - if ma is None or mb is None or ca == 0 or cb == 0: - continue - name_obj = (sa or {}).get('name') if isinstance(sa, dict) else getattr(sa, 'name_obj', None) - metric = name_obj.get('name') if isinstance(name_obj, dict) else getattr(sa, 'metric', None) - metric_class, _ = helm_metrics.classify_metric(metric) - if metric_class != 'core': - continue - abs_delta = abs(ma - mb) - denom = max(abs(ma), abs(mb), 1e-12) - rel_delta = abs_delta / denom - rows.append({ - 'key': k, - 'metric': metric, - 'metric_class': metric_class, - 'a': ma, - 'b': mb, - 'abs_delta': abs_delta, - 'rel_delta': rel_delta, - }) - return rows - - -def _group_quantiles(rows: list[dict[str, Any]]) -> dict[str, Any]: - values = sorted(float(r['abs_delta']) for r in rows) - return { - 'count': len(values), - 'abs_delta': { - 'min': _quantile(values, 0.0), - 'p50': _quantile(values, 0.5), - 'p90': _quantile(values, 0.9), - 'p95': _quantile(values, 0.95), - 'p99': _quantile(values, 0.99), - 'max': _quantile(values, 1.0), - }, - } - - -def _metric_quantiles(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: - by_metric: dict[str, list[dict[str, Any]]] = {} - for row in rows: - by_metric.setdefault(str(row['metric']), []).append(row) - out = [] - for metric, items in sorted(by_metric.items()): - info = _group_quantiles(items) - info['metric'] = metric - out.append(info) - return out - - -def _metric_descriptor(metric: str) -> dict[str, str]: - if metric in { - 'exact_match', - 'prefix_exact_match', - 'quasi_exact_match', - 'quasi_prefix_exact_match', - 'exact_match@5', - 'prefix_exact_match@5', - 'quasi_exact_match@5', - 'quasi_prefix_exact_match@5', - }: - return { - 'kind': 'binary', - 'range': '0 to 1', - 'direction': 'higher is better', - } - if metric in {'bleu_1', 'bleu_4', 'f1_score', 'rouge_l'}: - return { - 'kind': 'bounded overlap score', - 'range': '0 to 1', - 'direction': 'higher is better', - } - return { - 'kind': 'score', - 'range': 'metric-dependent', - 'direction': 'higher is better unless documented otherwise', - } - - -def _should_treat_as_discrete(values) -> bool: - values = [float(v) for v in values if v is not None] - unique_values = sorted(set(values)) - if not unique_values: - return False - return len(unique_values) <= 8 and all(v in {0.0, 1.0} for v in unique_values) - - -def _agreement_curve(rows: list[dict[str, Any]], thresholds: list[float]) -> list[dict[str, Any]]: - if not rows: - return [] - vals = [float(r['abs_delta']) for r in rows] - out = [] - for t in thresholds: - matched = sum(v <= t for v in vals) - out.append({ - 'abs_tol': t, - 'agree_ratio': matched / len(vals), - 'matched': matched, - 'count': len(vals), - }) - return out - - -def _infer_run_spec_name(*run_paths: str) -> str: - names = [Path(p).name for p in run_paths if p] - names = [n for n in names if n] - if not names: - return 'unknown_run_spec' - unique = sorted(set(names)) - if len(unique) == 1: - return unique[0] - return unique[0] - - -def _build_pair(run_a: str, run_b: str, label: str, thresholds: list[float]) -> dict[str, Any]: - diff = HelmRunDiff(HelmRun.coerce(run_a), HelmRun.coerce(run_b), a_name=f'{label}:A', b_name=f'{label}:B') - run_rows = _run_level_core_rows(diff) - inst_rows = _instance_level_core_rows(diff) - return { - 'label': label, - 'inputs': { - 'run_a': str(Path(run_a).expanduser().resolve()), - 'run_b': str(Path(run_b).expanduser().resolve()), - }, - 'diagnosis': diff.summary_dict(level=20).get('diagnosis', {}), - 'core_metrics': sorted({str(r['metric']) for r in inst_rows}), - 'run_level': { - 'n_rows': len(run_rows), - 'overall_quantiles': _group_quantiles(run_rows), - 'by_metric': _metric_quantiles(run_rows), - 'agreement_vs_abs_tol': _agreement_curve(run_rows, thresholds), - }, - 'instance_level': { - 'n_rows': len(inst_rows), - 'overall_quantiles': _group_quantiles(inst_rows), - 'by_metric': _metric_quantiles(inst_rows), - 'agreement_vs_abs_tol': _agreement_curve(inst_rows, thresholds), - }, - '_instance_rows': inst_rows, - } - - -def _agreement_curve_rows(*pairs: dict[str, Any], level_key: str) -> list[dict[str, Any]]: - rows = [] - for pair in pairs: - if not pair: - continue - for row in pair[level_key]['agreement_vs_abs_tol']: - rows.append({ - 'pair': pair['label'], - 'abs_tol': float(row['abs_tol']), - 'agree_ratio': float(row['agree_ratio']), - }) - return rows - - -def _plot_distribution(ax, *pairs: dict[str, Any], level_key: str) -> None: - rows = pd.DataFrame(_agreement_curve_rows(*pairs, level_key=level_key)) - sns.lineplot( - ax=ax, - data=rows, - x='abs_tol', - y='agree_ratio', - hue='pair', - style='pair', - markers=True, - dashes=False, - linewidth=2, - ) - ax.set_xscale('symlog', linthresh=1e-12) - ax.set_ylim(0, 1.02) - ax.set_xlabel('Absolute Tolerance Threshold for Core Metric Difference') - ax.set_ylabel('Fraction of Core Metric Comparisons in Agreement') - ax.tick_params(axis='x', rotation=28) - ax.legend(title='') - - -def _plot_quantiles(ax, pair_a: dict[str, Any], pair_b: dict[str, Any], level_key: str, title: str) -> None: - labels = ['p50', 'p90', 'p95', 'p99', 'max'] - x = list(range(len(labels))) - a_vals = [pair_a[level_key]['overall_quantiles']['abs_delta'][k] for k in labels] - b_vals = [pair_b[level_key]['overall_quantiles']['abs_delta'][k] for k in labels] - ax.plot(x, a_vals, marker='o', label=pair_a['label']) - ax.plot(x, b_vals, marker='o', label=pair_b['label']) - ax.set_xticks(x, labels) - ax.set_yscale('symlog', linthresh=1e-12) - ax.set_title(title) - ax.set_xlabel('Quantile') - ax.set_ylabel('Absolute Difference in Core Metric Value') - ax.legend(title='') - - -def _distribution_rows(pair: dict[str, Any]) -> pd.DataFrame: - rows = [] - for row in pair.get('_instance_rows', []): - rows.append({ - 'pair': pair['label'], - 'metric': row['metric'], - 'side': 'A', - 'value': float(row['a']), - }) - rows.append({ - 'pair': pair['label'], - 'metric': row['metric'], - 'side': 'B', - 'value': float(row['b']), - }) - return pd.DataFrame(rows) - - -def _plot_metric_distributions(fig_dpath: Path, stamp: str, left: dict[str, Any], right: dict[str, Any], run_spec_name: str) -> Path: - df = pd.concat([ - _distribution_rows(left), - _distribution_rows(right), - ], ignore_index=True) - metrics = sorted(df['metric'].dropna().unique().tolist()) - pair_order = [left['label'], right['label']] - fig, axes = plt.subplots( - len(pair_order), - len(metrics), - figsize=(5.2 * len(metrics), 4.2 * len(pair_order)), - constrained_layout=True, - ) - if len(pair_order) == 1 and len(metrics) == 1: - axes = [[axes]] - elif len(pair_order) == 1: - axes = [axes] - elif len(metrics) == 1: - axes = [[ax] for ax in axes] - for row_idx, pair_label in enumerate(pair_order): - for col_idx, metric in enumerate(metrics): - ax = axes[row_idx][col_idx] - sub = df[(df['pair'] == pair_label) & (df['metric'] == metric)] - discrete = _should_treat_as_discrete(sub['value'].tolist()) - sns.histplot( - data=sub, - x='value', - hue='side', - stat='probability', - common_norm=False, - discrete=discrete, - multiple='dodge', - shrink=0.8, - bins=None if discrete else 20, - ax=ax, - ) - ax.set_title(f'{pair_label}\n{metric}') - ax.set_xlabel('Core metric value') - ax.set_ylabel('Probability') - legend = ax.get_legend() - if legend is not None: - legend.set_title('') - fig.suptitle( - 'Core Metric Score Distributions Within Each Comparison Pair\n' - f'Run Spec: {run_spec_name}\n' - 'Each panel shows the per-instance score distribution for side A vs side B.', - fontsize=16, - ) - out_fpath = fig_dpath / f'core_metric_distributions_{stamp}.png' - fig.savefig(out_fpath, dpi=180) - plt.close(fig) - return out_fpath - - -def _single_run_instance_core_rows(run_path: str, label: str) -> pd.DataFrame: - ana = HelmRunAnalysis(HelmRun.coerce(run_path), name=label) - joined = ana.joined_instance_stat_table(assert_assumptions=False) - row_by_key = getattr(joined, 'row_by_key', None) or {} - rows = [] - for row in row_by_key.values(): - stat = row.stat - mean = _safe_float(stat.get('mean')) - count = int(stat.get('count', 0) or 0) - if mean is None or count == 0: - continue - name_obj = stat.get('name', {}) - metric = name_obj.get('name') - metric_class, _ = helm_metrics.classify_metric(metric) - if metric_class != 'core': - continue - rows.append({ - 'run': label, - 'metric': metric, - 'value': float(mean), - }) - return pd.DataFrame(rows) - - -def _plot_three_run_metric_distributions( - fig_dpath: Path, - stamp: str, - kwdagger_a_run: str, - kwdagger_b_run: str, - official_run: str, - run_spec_name: str, -) -> Path: - df = pd.concat([ - _single_run_instance_core_rows(kwdagger_a_run, 'kwdagger A'), - _single_run_instance_core_rows(kwdagger_b_run, 'kwdagger B'), - _single_run_instance_core_rows(official_run, 'official'), - ], ignore_index=True) - metrics = sorted(df['metric'].dropna().unique().tolist()) - run_order = ['kwdagger A', 'kwdagger B', 'official'] - fig, axes = plt.subplots( - len(metrics), - len(run_order), - figsize=(5.0 * len(run_order), 3.2 * len(metrics)), - constrained_layout=True, - ) - if len(metrics) == 1 and len(run_order) == 1: - axes = [[axes]] - elif len(metrics) == 1: - axes = [axes] - elif len(run_order) == 1: - axes = [[ax] for ax in axes] - for row_idx, metric in enumerate(metrics): - for col_idx, run_label in enumerate(run_order): - ax = axes[row_idx][col_idx] - sub = df[(df['metric'] == metric) & (df['run'] == run_label)] - discrete = _should_treat_as_discrete(sub['value'].tolist()) - sns.histplot( - data=sub, - x='value', - stat='probability', - discrete=discrete, - shrink=0.8, - bins=None if discrete else 20, - ax=ax, - color='#4C72B0', - ) - if row_idx == 0: - ax.set_title(run_label) - ax.set_xlabel('Core metric value') - ax.set_ylabel(metric if col_idx == 0 else '') - fig.suptitle( - 'Per-Run Instance-Level Core Metric Score Distributions\n' - f'Run Spec: {run_spec_name}\n' - 'Columns are kwdagger repeat A, kwdagger repeat B, and the official HELM run.', - fontsize=16, - ) - out_fpath = fig_dpath / f'core_metric_three_run_distributions_{stamp}.png' - fig.savefig(out_fpath, dpi=180) - plt.close(fig) - return out_fpath - - -def _plot_overlay_metric_distributions( - fig_dpath: Path, - stamp: str, - kwdagger_a_run: str, - kwdagger_b_run: str, - official_run: str, - run_spec_name: str, -) -> Path: - df = pd.concat([ - _single_run_instance_core_rows(kwdagger_a_run, 'kwdagger A'), - _single_run_instance_core_rows(kwdagger_b_run, 'kwdagger B'), - _single_run_instance_core_rows(official_run, 'official'), - ], ignore_index=True) - metrics = sorted(df['metric'].dropna().unique().tolist()) - fig, axes = plt.subplots( - len(metrics), - 1, - figsize=(10, 3.2 * len(metrics)), - constrained_layout=True, - ) - if len(metrics) == 1: - axes = [axes] - for ax, metric in zip(axes, metrics): - sub = df[df['metric'] == metric].copy() - discrete = _should_treat_as_discrete(sub['value'].tolist()) - sns.histplot( - data=sub, - x='value', - hue='run', - stat='probability', - common_norm=False, - element='step', - fill=False, - multiple='layer', - discrete=discrete, - bins=None if discrete else 20, - ax=ax, - ) - desc = _metric_descriptor(metric) - ax.set_title( - f"{metric} ({desc['kind']}, {desc['range']}, {desc['direction']})" - ) - ax.set_xlabel('Instance-level metric value') - ax.set_ylabel('Probability') - legend = ax.get_legend() - if legend is not None: - legend.set_title('') - fig.suptitle( - 'Overlay of Per-Instance Core Metric Score Distributions by Run\n' - f'Run Spec: {run_spec_name}\n' - 'This shows the raw score distributions for each core metric across kwdagger repeats and the official HELM run.', - fontsize=16, - ) - out_fpath = fig_dpath / f'core_metric_overlay_distributions_{stamp}.png' - fig.savefig(out_fpath, dpi=180) - plt.close(fig) - return out_fpath - - -def _plot_overlay_metric_ecdfs( - fig_dpath: Path, - stamp: str, - kwdagger_a_run: str, - kwdagger_b_run: str, - official_run: str, - run_spec_name: str, -) -> Path: - df = pd.concat([ - _single_run_instance_core_rows(kwdagger_a_run, 'kwdagger A'), - _single_run_instance_core_rows(kwdagger_b_run, 'kwdagger B'), - _single_run_instance_core_rows(official_run, 'official'), - ], ignore_index=True) - metrics = sorted(df['metric'].dropna().unique().tolist()) - fig, axes = plt.subplots( - len(metrics), - 1, - figsize=(10, 3.2 * len(metrics)), - constrained_layout=True, - ) - if len(metrics) == 1: - axes = [axes] - for ax, metric in zip(axes, metrics): - sub = df[df['metric'] == metric].copy() - sns.ecdfplot( - data=sub, - x='value', - hue='run', - ax=ax, - ) - desc = _metric_descriptor(metric) - ax.set_title( - f"{metric} ECDF ({desc['kind']}, {desc['range']}, {desc['direction']})" - ) - ax.set_xlabel('Instance-level metric value') - ax.set_ylabel('Cumulative fraction of instances') - legend = ax.get_legend() - if legend is not None: - legend.set_title('') - fig.suptitle( - 'ECDF of Per-Instance Core Metric Scores by Run\n' - f'Run Spec: {run_spec_name}\n' - 'This often communicates sparse or zero-heavy metric distributions more clearly than histograms.', - fontsize=16, - ) - out_fpath = fig_dpath / f'core_metric_ecdfs_{stamp}.png' - fig.savefig(out_fpath, dpi=180) - plt.close(fig) - return out_fpath - - -def _single_run_core_stat_index(run_path: str) -> dict[str, Any]: - ana = HelmRunAnalysis(HelmRun.coerce(run_path)) - idx = ana.stat_index(drop_zero_count=True, require_mean=True) - return {k: v for k, v in idx.items() if v.metric_class == 'core'} - - -def _write_three_run_runlevel_table( - out_dpath: Path, - stamp: str, - kwdagger_a_run: str, - kwdagger_b_run: str, - official_run: str, -) -> tuple[Path, Path]: - idx_a = _single_run_core_stat_index(kwdagger_a_run) - idx_b = _single_run_core_stat_index(kwdagger_b_run) - idx_o = _single_run_core_stat_index(official_run) - keys = sorted(set(idx_a) & set(idx_b) & set(idx_o)) - rows = [] - for key in keys: - a = idx_a[key] - b = idx_b[key] - o = idx_o[key] - rows.append({ - 'stat_key': key, - 'metric': a.metric, - 'kwdagger_a': a.mean, - 'kwdagger_b': b.mean, - 'official': o.mean, - 'delta_official_vs_kwdagger_a': None if a.mean is None or o.mean is None else abs(o.mean - a.mean), - 'delta_kwdagger_a_vs_kwdagger_b': None if a.mean is None or b.mean is None else abs(a.mean - b.mean), - }) - table = pd.DataFrame(rows) - csv_fpath = out_dpath / f'core_runlevel_table_{stamp}.csv' - md_fpath = out_dpath / f'core_runlevel_table_{stamp}.md' - table.to_csv(csv_fpath, index=False) - md_fpath.write_text(table.to_markdown(index=False) + '\n') - return csv_fpath, md_fpath - - -def _strip_private(obj: Any) -> Any: - if isinstance(obj, dict): - return { - k: _strip_private(v) - for k, v in obj.items() - if not str(k).startswith('_') - } - if isinstance(obj, list): - return [_strip_private(v) for v in obj] - return obj - - -def _write_text(report: dict[str, Any], out_fpath: Path) -> None: - left, right = report['pairs'] - lines = [] - lines.append('Core Metric Report') - lines.append('') - lines.append(f"generated_utc: {report['generated_utc']}") - lines.append(f"run_spec_name: {report['run_spec_name']}") - lines.append(f"left_label: {left['label']}") - lines.append(f"right_label: {right['label']}") - lines.append(f"diagnostic_flags: {report.get('diagnostic_flags', [])}") - lines.append('') - lines.append('core_metrics:') - for metric in left['core_metrics']: - lines.append(f' - {metric}') - lines.append('') - lines.append('run_diagnostics:') - for label, diag in report.get('run_diagnostics', {}).items(): - lines.append(f' {label}:') - lines.append(f" n_request_states: {diag.get('n_request_states')}") - lines.append(f" n_with_completions: {diag.get('n_with_completions')}") - lines.append(f" empty_completion_count: {diag.get('empty_completion_count')}") - lines.append(f" empty_completion_rate: {diag.get('empty_completion_rate')}") - lines.append(f" output_token_count: {json.dumps(diag.get('output_token_count'))}") - lines.append(f" stats_means: {json.dumps(diag.get('stats_means'))}") - lines.append('') - for pair in report['pairs']: - lines.append(f"pair: {pair['label']}") - lines.append(f" diagnosis: {pair['diagnosis'].get('label')}") - lines.append(f" primary_reason_names: {pair['diagnosis'].get('primary_reason_names')}") - lines.append(f" run_level_n: {pair['run_level']['n_rows']}") - lines.append(f" instance_level_n: {pair['instance_level']['n_rows']}") - lines.append(f" run_level_quantiles: {json.dumps(pair['run_level']['overall_quantiles']['abs_delta'])}") - lines.append(f" instance_level_quantiles: {json.dumps(pair['instance_level']['overall_quantiles']['abs_delta'])}") - lines.append(' by_metric:') - for row in pair['instance_level']['by_metric']: - lines.append( - f" - metric={row['metric']} count={row['count']} " - f"p50={row['abs_delta']['p50']} p90={row['abs_delta']['p90']} " - f"p95={row['abs_delta']['p95']} p99={row['abs_delta']['p99']} " - f"max={row['abs_delta']['max']}" - ) - lines.append(' agreement_vs_abs_tol:') - for row in pair['instance_level']['agreement_vs_abs_tol']: - lines.append( - f" - abs_tol={row['abs_tol']} agree_ratio={row['agree_ratio']}" - ) - lines.append('') - out_fpath.write_text('\n'.join(lines) + '\n') - - -def _find_curve_value(rows: list[dict[str, Any]], abs_tol: float) -> float | None: - for row in rows: - if float(row.get('abs_tol', float('nan'))) == float(abs_tol): - return row.get('agree_ratio') - return None - - -def _write_management_summary(report: dict[str, Any], out_fpath: Path) -> None: - left, right = report['pairs'] - lines = [] - lines.append('Core Metric Executive Summary') - lines.append('') - lines.append(f"generated_utc: {report['generated_utc']}") - lines.append(f"run_spec_name: {report['run_spec_name']}") - lines.append(f"core_metrics: {', '.join(left.get('core_metrics', []))}") - lines.append(f"diagnostic_flags: {report.get('diagnostic_flags', [])}") - lines.append('') - lines.append('metric_descriptions:') - for metric in left.get('core_metrics', []): - desc = _metric_descriptor(metric) - lines.append( - f" - {metric}: {desc['kind']}; {desc['range']}; {desc['direction']}" - ) - lines.append('') - lines.append('run_diagnostics:') - for label, diag in report.get('run_diagnostics', {}).items(): - lines.append(f' {label}:') - lines.append(f" n_request_states: {diag.get('n_request_states')}") - lines.append(f" n_with_completions: {diag.get('n_with_completions')}") - lines.append(f" empty_completion_count: {diag.get('empty_completion_count')}") - lines.append(f" empty_completion_rate: {diag.get('empty_completion_rate')}") - lines.append(f" mean_output_tokens_from_state: {(diag.get('output_token_count') or {}).get('mean')}") - lines.append(f" p90_output_tokens_from_state: {(diag.get('output_token_count') or {}).get('p90')}") - lines.append(f" num_output_tokens_from_stats: {(diag.get('stats_means') or {}).get('num_output_tokens')}") - lines.append(f" finish_reason_unknown_from_stats: {(diag.get('stats_means') or {}).get('finish_reason_unknown')}") - lines.append('') - lines.append(f"{left['label']}:") - lines.append(f" diagnosis: {left['diagnosis'].get('label')}") - lines.append(f" run-level N: {left['run_level']['n_rows']}") - lines.append(f" instance-level N: {left['instance_level']['n_rows']}") - lines.append( - f" instance agreement at abs_tol=0.0: {_find_curve_value(left['instance_level']['agreement_vs_abs_tol'], 0.0)}" - ) - lines.append( - f" run-level abs delta max: {left['run_level']['overall_quantiles']['abs_delta']['max']}" - ) - lines.append( - f" instance-level abs delta max: {left['instance_level']['overall_quantiles']['abs_delta']['max']}" - ) - lines.append('') - lines.append(f"{right['label']}:") - lines.append(f" diagnosis: {right['diagnosis'].get('label')}") - lines.append(f" run-level N: {right['run_level']['n_rows']}") - lines.append(f" instance-level N: {right['instance_level']['n_rows']}") - for tol in [0.0, 1e-3, 1e-2, 1e-1, 2.5e-1, 5e-1, 1.0]: - lines.append( - f" instance agreement at abs_tol={tol}: " - f"{_find_curve_value(right['instance_level']['agreement_vs_abs_tol'], tol)}" - ) - lines.append( - f" run-level abs delta p90/max: " - f"{right['run_level']['overall_quantiles']['abs_delta']['p90']} / " - f"{right['run_level']['overall_quantiles']['abs_delta']['max']}" - ) - lines.append( - f" instance-level abs delta p99/max: " - f"{right['instance_level']['overall_quantiles']['abs_delta']['p99']} / " - f"{right['instance_level']['overall_quantiles']['abs_delta']['max']}" - ) - out_fpath.write_text('\n'.join(lines) + '\n') - - -def _write_latest_alias(src: Path, latest_root: Path, latest_name: str) -> Path: - latest_fpath = latest_root / latest_name - if latest_fpath.exists() or latest_fpath.is_symlink(): - latest_fpath.unlink() - rel_src = os.path.relpath(src, start=latest_fpath.parent) - os.symlink(rel_src, latest_fpath) - return latest_fpath - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument('--left-run-a', required=True) - parser.add_argument('--left-run-b', required=True) - parser.add_argument('--left-label', required=True) - parser.add_argument('--right-run-a', required=True) - parser.add_argument('--right-run-b', required=True) - parser.add_argument('--right-label', required=True) - parser.add_argument('--report-dpath', required=True) - args = parser.parse_args() - - thresholds = [0.0, 1e-12, 1e-9, 1e-6, 1e-4, 1e-3, 1e-2, 2e-2, 5e-2, 1e-1, 2.5e-1, 5e-1, 1.0] - report_dpath = Path(args.report_dpath).expanduser().resolve() - report_dpath.mkdir(parents=True, exist_ok=True) - stamp = datetime_mod.datetime.now(datetime_mod.UTC).strftime('%Y%m%dT%H%M%SZ') - history_dpath = report_dpath / '.history' / stamp[:8] - history_dpath.mkdir(parents=True, exist_ok=True) - run_spec_name = _infer_run_spec_name(args.left_run_a, args.left_run_b, args.right_run_a) - - left = _build_pair(args.left_run_a, args.left_run_b, args.left_label, thresholds) - right = _build_pair(args.right_run_a, args.right_run_b, args.right_label, thresholds) - run_diagnostics = { - 'kwdagger_a': _run_diagnostics(args.left_run_a), - 'kwdagger_b': _run_diagnostics(args.left_run_b), - 'official': _run_diagnostics(args.right_run_a), - } - report = { - 'generated_utc': stamp, - 'run_spec_name': run_spec_name, - 'thresholds': thresholds, - 'pairs': [left, right], - 'run_diagnostics': run_diagnostics, - 'diagnostic_flags': _diagnostic_flags(run_diagnostics), - } - - json_fpath = history_dpath / f'core_metric_report_{stamp}.json' - txt_fpath = history_dpath / f'core_metric_report_{stamp}.txt' - mgmt_fpath = history_dpath / f'core_metric_management_summary_{stamp}.txt' - fig_fpath = history_dpath / f'core_metric_report_{stamp}.png' - dist_fig_fpath = _plot_metric_distributions(history_dpath, stamp, left, right, run_spec_name) - three_run_dist_fpath = _plot_three_run_metric_distributions( - history_dpath, - stamp, - args.left_run_a, - args.left_run_b, - args.right_run_a, - run_spec_name, - ) - overlay_dist_fpath = _plot_overlay_metric_distributions( - history_dpath, - stamp, - args.left_run_a, - args.left_run_b, - args.right_run_a, - run_spec_name, - ) - ecdf_fig_fpath = _plot_overlay_metric_ecdfs( - history_dpath, - stamp, - args.left_run_a, - args.left_run_b, - args.right_run_a, - run_spec_name, - ) - runlevel_csv_fpath, runlevel_md_fpath = _write_three_run_runlevel_table( - history_dpath, - stamp, - args.left_run_a, - args.left_run_b, - args.right_run_a, - ) - - report = kwutil.Json.ensure_serializable(_strip_private(report)) - json_fpath.write_text(json.dumps(report, indent=2)) - _write_text(report, txt_fpath) - _write_management_summary(report, mgmt_fpath) - - extra_pair = _load_optional_cross_machine_pair(report_dpath) - paper_labels = load_paper_label_manager(style='paper_short') - extra_label = extra_pair['label'] if extra_pair is not None else None - pair_line = f'Pairs: {left["label"]} vs {right["label"]}' - if extra_label is not None: - pair_line += f' + {extra_label}' - pair_line = paper_labels.relabel_text(pair_line) - sns.set_theme(style='whitegrid', context='talk') - fig, axes = plt.subplots(2, 2, figsize=(24, 14.5), constrained_layout=True) - _plot_quantiles( - axes[0, 0], - left, - right, - 'run_level', - 'Run-Level Delta Quantiles' - ) - _plot_quantiles( - axes[0, 1], - left, - right, - 'instance_level', - 'Instance-Level Delta Quantiles' - ) - _plot_distribution(axes[1, 0], left, right, extra_pair, level_key='run_level') - axes[1, 0].set_title('Run-Level Agreement vs Tolerance', fontsize=11) - _plot_distribution(axes[1, 1], left, right, extra_pair, level_key='instance_level') - axes[1, 1].set_title('Instance-Level Agreement vs Tolerance', fontsize=11) - axes[0, 0].title.set_fontsize(11) - axes[0, 1].title.set_fontsize(11) - fig.suptitle( - 'Core Metric Agreement and Difference Summary\n' - f'Run Spec: {run_spec_name}\n' - f'{pair_line}\n' - f'Run-level N: {left["run_level"]["n_rows"]} vs {right["run_level"]["n_rows"]} | ' - f'Instance-level N: {left["instance_level"]["n_rows"]} vs {right["instance_level"]["n_rows"]}', - fontsize=15, - ) - fig.savefig(fig_fpath, dpi=180) - plt.close(fig) - - latest_map = { - json_fpath: 'core_metric_report.latest.json', - txt_fpath: 'core_metric_report.latest.txt', - mgmt_fpath: 'core_metric_management_summary.latest.txt', - fig_fpath: 'core_metric_report.latest.png', - dist_fig_fpath: 'core_metric_distributions.latest.png', - three_run_dist_fpath: 'core_metric_three_run_distributions.latest.png', - overlay_dist_fpath: 'core_metric_overlay_distributions.latest.png', - ecdf_fig_fpath: 'core_metric_ecdfs.latest.png', - runlevel_csv_fpath: 'core_runlevel_table.latest.csv', - runlevel_md_fpath: 'core_runlevel_table.latest.md', - } - for src, latest_name in latest_map.items(): - _write_latest_alias(src, report_dpath, latest_name) - - print(f'Wrote core metric report: {json_fpath}') - print(f'Wrote core metric text: {txt_fpath}') - print(f'Wrote core metric management summary: {mgmt_fpath}') - print(f'Wrote core metric plot: {fig_fpath}') - print(f'Wrote core metric distributions: {dist_fig_fpath}') - print(f'Wrote core metric three-run distributions: {three_run_dist_fpath}') - print(f'Wrote core metric overlay distributions: {overlay_dist_fpath}') - print(f'Wrote core metric ecdfs: {ecdf_fig_fpath}') - print(f'Wrote core run-level table csv: {runlevel_csv_fpath}') - print(f'Wrote core run-level table md: {runlevel_md_fpath}') - - -if __name__ == '__main__': - main() diff --git a/dev/experiments/audit-helm-reproduction/python/index_results.py b/dev/experiments/audit-helm-reproduction/python/index_results.py deleted file mode 100644 index 7cbe7b9..0000000 --- a/dev/experiments/audit-helm-reproduction/python/index_results.py +++ /dev/null @@ -1,201 +0,0 @@ -from __future__ import annotations - -import argparse -import datetime as datetime_mod -import json -from collections import Counter -from pathlib import Path -from typing import Any - -import kwutil -import pandas as pd - -from common import default_report_root, env_defaults -from magnet.backends.helm.cli.materialize_helm_run import parse_run_entry_description -from magnet.backends.helm.helm_outputs import HelmOutputs - - -def _safe_json_load(fpath: Path) -> dict[str, Any]: - if not fpath.exists(): - return {} - try: - return json.loads(fpath.read_text()) - except Exception: - return {} - - -def _first_run_dir(job_dpath: Path) -> Path | None: - bo = job_dpath / 'benchmark_output' - if not bo.exists(): - return None - try: - outputs = HelmOutputs.coerce(bo) - except Exception: - return None - runs = [] - for suite in outputs.suites(pattern='*'): - runs.extend(list(suite.runs())) - if not runs: - return None - return Path(runs[0].path) - - -def _process_context_info(process_context: dict[str, Any], fallback_host: str | None) -> dict[str, Any]: - props = process_context.get('properties', {}) if isinstance(process_context, dict) else {} - machine = props.get('machine', {}) if isinstance(props.get('machine', {}), dict) else {} - extra = props.get('extra', {}) if isinstance(props.get('extra', {}), dict) else {} - env = extra.get('env', {}) if isinstance(extra.get('env', {}), dict) else {} - nvidia_smi = extra.get('nvidia_smi', {}) if isinstance(extra.get('nvidia_smi', {}), dict) else {} - gpus = nvidia_smi.get('gpus', []) if isinstance(nvidia_smi.get('gpus', []), list) else [] - - host = machine.get('host') - provenance = 'recorded' - if not host: - host = fallback_host - provenance = 'fallback' if fallback_host else 'unknown' - - return { - 'machine_host': host, - 'machine_user': machine.get('user'), - 'machine_os': machine.get('os_name'), - 'machine_arch': machine.get('arch'), - 'python_version': machine.get('py_version'), - 'cuda_visible_devices': env.get('CUDA_VISIBLE_DEVICES'), - 'gpu_count': len(gpus), - 'gpu_names': [g.get('name') for g in gpus if isinstance(g, dict)], - 'gpu_memory_total_mb': [g.get('memory_total_mb') for g in gpus if isinstance(g, dict)], - 'provenance_source': provenance, - } - - -def _row_for_job(job_config_fpath: Path, fallback_host: str | None) -> dict[str, Any]: - job_dpath = job_config_fpath.parent - adapter_manifest = _safe_json_load(job_dpath / 'adapter_manifest.json') - process_context = _safe_json_load(job_dpath / 'process_context.json') - if not process_context: - process_context = adapter_manifest.get('process_context', {}) if isinstance(adapter_manifest, dict) else {} - run_dir = _first_run_dir(job_dpath) - run_spec = _safe_json_load(run_dir / 'run_spec.json') if run_dir else {} - - job_config = _safe_json_load(job_config_fpath) - run_entry = job_config.get('helm.run_entry') - benchmark = None - model = None - method = None - if run_entry: - try: - benchmark, tokens = parse_run_entry_description(run_entry) - model = tokens.get('model') - method = tokens.get('method') - except Exception: - benchmark = None - - context_info = _process_context_info(process_context, fallback_host) - adapter_spec = run_spec.get('adapter_spec', {}) if isinstance(run_spec, dict) else {} - metric_specs = run_spec.get('metric_specs', []) if isinstance(run_spec, dict) else [] - - row = { - 'experiment_name': job_dpath.parent.parent.name if job_dpath.parent.name == 'helm' else job_dpath.parent.name, - 'job_id': job_dpath.name, - 'job_dpath': str(job_dpath), - 'status': adapter_manifest.get('status'), - 'manifest_timestamp': adapter_manifest.get('timestamp'), - 'run_entry': run_entry, - 'benchmark': benchmark, - 'model': model, - 'method': method, - 'suite': job_config.get('helm.suite'), - 'max_eval_instances': job_config.get('helm.max_eval_instances'), - 'run_dir': str(run_dir) if run_dir else None, - 'has_run_dir': bool(run_dir and run_dir.exists()), - 'has_run_spec': bool(run_dir and (run_dir / 'run_spec.json').exists()), - 'has_stats': bool(run_dir and (run_dir / 'stats.json').exists()), - 'has_per_instance_stats': bool(run_dir and (run_dir / 'per_instance_stats.json').exists()), - 'model_deployment': adapter_spec.get('model_deployment'), - 'metric_class_names': [m.get('class_name') for m in metric_specs if isinstance(m, dict)], - } - row.update(context_info) - return row - - -def _write_summary(rows: list[dict[str, Any]], out_fpath: Path) -> None: - benchmark_counts = Counter(row.get('benchmark') or 'unknown' for row in rows) - model_counts = Counter(row.get('model') or 'unknown' for row in rows) - host_counts = Counter(row.get('machine_host') or 'unknown' for row in rows) - status_counts = Counter(row.get('status') or 'unknown' for row in rows) - - lines = [] - lines.append('Audit Results Index Summary') - lines.append('') - lines.append(f'n_rows: {len(rows)}') - lines.append('') - lines.append('status_counts:') - for key, val in sorted(status_counts.items()): - lines.append(f' {key}: {val}') - lines.append('') - lines.append('machine_host_counts:') - for key, val in sorted(host_counts.items()): - lines.append(f' {key}: {val}') - lines.append('') - lines.append('benchmark_counts:') - for key, val in sorted(benchmark_counts.items()): - lines.append(f' {key}: {val}') - lines.append('') - lines.append('model_counts:') - for key, val in sorted(model_counts.items()): - lines.append(f' {key}: {val}') - out_fpath.write_text('\n'.join(lines) + '\n') - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument('--results-root', default=env_defaults()['AUDIT_RESULTS_ROOT']) - parser.add_argument('--report-dpath', default=str(default_report_root() / 'indexes')) - parser.add_argument('--fallback-host', default=None) - args = parser.parse_args() - - results_root = Path(args.results_root).expanduser().resolve() - report_dpath = Path(args.report_dpath).expanduser().resolve() - report_dpath.mkdir(parents=True, exist_ok=True) - stamp = datetime_mod.datetime.now(datetime_mod.UTC).strftime('%Y%m%dT%H%M%SZ') - - rows = [] - for job_config_fpath in sorted(results_root.rglob('job_config.json')): - try: - rows.append(_row_for_job(job_config_fpath, args.fallback_host)) - except Exception as ex: - rows.append({ - 'job_dpath': str(job_config_fpath.parent), - 'status': 'index_error', - 'error': repr(ex), - 'machine_host': args.fallback_host, - 'provenance_source': 'fallback' if args.fallback_host else 'unknown', - }) - - jsonl_fpath = report_dpath / f'audit_results_index_{stamp}.jsonl' - csv_fpath = report_dpath / f'audit_results_index_{stamp}.csv' - summary_fpath = report_dpath / f'audit_results_index_{stamp}.txt' - - with jsonl_fpath.open('w') as file: - for row in rows: - file.write(json.dumps(kwutil.Json.ensure_serializable(row)) + '\n') - - table = pd.DataFrame(rows) - if not table.empty: - preferred = [ - 'experiment_name', 'job_id', 'status', 'benchmark', 'model', 'method', - 'max_eval_instances', 'machine_host', 'gpu_count', 'gpu_names', - 'cuda_visible_devices', 'provenance_source', 'run_dir', - ] - cols = [c for c in preferred if c in table.columns] + [c for c in table.columns if c not in preferred] - table = table[cols] - table.to_csv(csv_fpath, index=False) - _write_summary(rows, summary_fpath) - - print(f'Wrote jsonl index: {jsonl_fpath}') - print(f'Wrote csv index: {csv_fpath}') - print(f'Wrote summary: {summary_fpath}') - - -if __name__ == '__main__': - main() diff --git a/dev/experiments/audit-helm-reproduction/python/inspect_pair_samples.py b/dev/experiments/audit-helm-reproduction/python/inspect_pair_samples.py deleted file mode 100644 index 2336b23..0000000 --- a/dev/experiments/audit-helm-reproduction/python/inspect_pair_samples.py +++ /dev/null @@ -1,94 +0,0 @@ -from __future__ import annotations - -import argparse -import datetime as datetime_mod -import os -from pathlib import Path - -from magnet.backends.helm.helm_outputs import HelmRun -from magnet.backends.helm.helm_run_diff import HelmRunDiff - - -def _safe_unlink(path: Path) -> None: - if path.exists() or path.is_symlink(): - path.unlink() - - -def _write_latest_alias(src: Path, latest_root: Path, latest_name: str) -> Path: - latest_fpath = latest_root / latest_name - _safe_unlink(latest_fpath) - rel_src = os.path.relpath(src, start=latest_fpath.parent) - os.symlink(rel_src, latest_fpath) - return latest_fpath - - -def _slugify(text: str) -> str: - return ( - str(text) - .replace('/', '-') - .replace(':', '-') - .replace(',', '-') - .replace('=', '-') - .replace('@', '-') - .replace(' ', '-') - ) - - -def _infer_run_spec_name(*run_paths: str) -> str: - names = [Path(p).name for p in run_paths if p] - names = [n for n in names if n] - if not names: - return 'unknown_run_spec' - unique = sorted(set(names)) - if len(unique) == 1: - return unique[0] - return unique[0] - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument('--run-a', required=True) - parser.add_argument('--run-b', required=True) - parser.add_argument('--label', required=True) - parser.add_argument('--report-dpath', required=True) - parser.add_argument('--top-n', type=int, default=8) - parser.add_argument('--show-details', type=int, default=6) - parser.add_argument('--level', type=int, default=30) - args = parser.parse_args() - - report_dpath = Path(args.report_dpath).expanduser().resolve() - report_dpath.mkdir(parents=True, exist_ok=True) - stamp = datetime_mod.datetime.now(datetime_mod.UTC).strftime('%Y%m%dT%H%M%SZ') - history_dpath = report_dpath / '.history' / stamp[:8] - history_dpath.mkdir(parents=True, exist_ok=True) - - diff = HelmRunDiff( - HelmRun.coerce(args.run_a), - HelmRun.coerce(args.run_b), - a_name=f'{args.label}:A', - b_name=f'{args.label}:B', - ) - run_spec_name = _infer_run_spec_name(args.run_a, args.run_b) - lines: list[str] = [] - lines.append(f'Instance Sample Inspection') - lines.append(f'label: {args.label}') - lines.append(f'run_spec_name: {run_spec_name}') - lines.append(f'run_a: {Path(args.run_a).expanduser().resolve()}') - lines.append(f'run_b: {Path(args.run_b).expanduser().resolve()}') - lines.append('') - diff.summarize_instances( - level=args.level, - top_n=args.top_n, - show_details=args.show_details, - writer=lines.append, - ) - out_fpath = history_dpath / f'instance_samples_{_slugify(args.label)}_{stamp}.txt' - out_fpath.write_text('\n'.join(lines) + '\n') - latest_name = f'instance_samples_{_slugify(args.label)}.latest.txt' - latest_fpath = _write_latest_alias(out_fpath, report_dpath, latest_name) - print(f'Wrote instance sample report: {out_fpath}') - print(f'Updated latest link: {latest_fpath}') - - -if __name__ == '__main__': - main() diff --git a/dev/experiments/audit-helm-reproduction/python/make_manifest.py b/dev/experiments/audit-helm-reproduction/python/make_manifest.py deleted file mode 100644 index 64fee0f..0000000 --- a/dev/experiments/audit-helm-reproduction/python/make_manifest.py +++ /dev/null @@ -1,221 +0,0 @@ -from __future__ import annotations - -import argparse -import os -from pathlib import Path - -import kwutil - -from common import dump_yaml, env_defaults, repo_run_specs_fpath - - -SMOKE_RUN_ENTRIES = [ - "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=eleutherai/pythia-6.9b,data_augmentation=canonical", - "boolq:model=eleutherai/pythia-6.9b,data_augmentation=canonical", - "narrative_qa:model=eleutherai/pythia-6.9b,data_augmentation=canonical", - "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical", - "boolq:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical", - "narrative_qa:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical", -] - -VICUNA_NOCHAT_RUN_ENTRIES = [ - "mmlu:subject=us_foreign_policy,method=multiple_choice_joint,model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical", - "boolq:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical", - "narrative_qa:model=lmsys/vicuna-7b-v1.3,data_augmentation=canonical", -] - - -def _validate_entries_exist(run_entries: list[str]) -> list[str]: - fpath = repo_run_specs_fpath() - all_run_specs = set(kwutil.Yaml.load(fpath)) - missing = [entry for entry in run_entries if entry not in all_run_specs] - return missing - - -def _build_manifest( - *, - experiment_name: str, - description: str, - run_entries: list[str], - max_eval_instances: int, - suite: str, - tmux_workers: int, - devices: str, -) -> dict: - missing = _validate_entries_exist(run_entries) - if missing: - raise RuntimeError( - "Manifest entries were not found in run_specs.yaml: " - + kwutil.Json.dumps(missing) - ) - return { - "schema_version": 1, - "experiment_name": experiment_name, - "description": description, - "run_entries": run_entries, - "max_eval_instances": max_eval_instances, - "suite": suite, - "mode": "compute_if_missing", - "materialize": "symlink", - "backend": "tmux", - "devices": devices, - "tmux_workers": tmux_workers, - "local_path": "prod_env", - "precomputed_root": None, - "require_per_instance_stats": True, - "model_deployments_fpath": None, - "enable_huggingface_models": [], - "enable_local_huggingface_models": [], - } - - -def build_smoke_manifest(args: argparse.Namespace) -> dict: - defaults = env_defaults() - max_eval_instances = ( - args.max_eval_instances - if args.max_eval_instances is not None - else int(defaults["AUDIT_DEFAULT_MAX_EVAL_INSTANCES"]) - ) - tmux_workers = ( - args.tmux_workers - if args.tmux_workers is not None - else int(defaults["AUDIT_DEFAULT_TMUX_WORKERS"]) - ) - devices = args.devices if args.devices is not None else "0,1" - return _build_manifest( - experiment_name=args.experiment_name, - description="Small smoke-test batch for HELM reproduction auditing.", - run_entries=SMOKE_RUN_ENTRIES, - max_eval_instances=max_eval_instances, - suite=args.suite, - tmux_workers=tmux_workers, - devices=devices, - ) - - -def build_apples_manifest(args: argparse.Namespace) -> dict: - defaults = env_defaults() - # Historic public matches for the current smoke-control entries all use 1000. - max_eval_instances = ( - args.max_eval_instances - if args.max_eval_instances is not None - else 1000 - ) - tmux_workers = ( - args.tmux_workers - if args.tmux_workers is not None - else int(defaults["AUDIT_DEFAULT_TMUX_WORKERS"]) - ) - devices = args.devices if args.devices is not None else "0,1" - return _build_manifest( - experiment_name=args.experiment_name, - description=( - "Apples-to-apples smoke batch aligned to the historic public HELM " - "requested max_eval_instances for the control entries." - ), - run_entries=SMOKE_RUN_ENTRIES, - max_eval_instances=max_eval_instances, - suite=args.suite, - tmux_workers=tmux_workers, - devices=devices, - ) - - -def build_single_manifest(args: argparse.Namespace) -> dict: - defaults = env_defaults() - if not args.run_entry: - raise SystemExit('--run-entry is required for --manifest-type single') - max_eval_instances = ( - args.max_eval_instances - if args.max_eval_instances is not None - else int(defaults["AUDIT_DEFAULT_MAX_EVAL_INSTANCES"]) - ) - tmux_workers = ( - args.tmux_workers - if args.tmux_workers is not None - else int(defaults["AUDIT_DEFAULT_TMUX_WORKERS"]) - ) - devices = args.devices if args.devices is not None else "0" - description = ( - args.description - if args.description is not None - else f"Single-run audit manifest for {args.run_entry}" - ) - return _build_manifest( - experiment_name=args.experiment_name, - description=description, - run_entries=[args.run_entry], - max_eval_instances=max_eval_instances, - suite=args.suite, - tmux_workers=tmux_workers, - devices=devices, - ) - - -def build_vicuna_nochat_manifest(args: argparse.Namespace) -> dict: - defaults = env_defaults() - max_eval_instances = ( - args.max_eval_instances - if args.max_eval_instances is not None - else 1000 - ) - tmux_workers = ( - args.tmux_workers - if args.tmux_workers is not None - else 1 - ) - devices = args.devices if args.devices is not None else "0" - manifest = _build_manifest( - experiment_name=args.experiment_name, - description=( - "Overnight Vicuna batch with chat templating explicitly disabled " - "via a model_deployments override." - ), - run_entries=VICUNA_NOCHAT_RUN_ENTRIES, - max_eval_instances=max_eval_instances, - suite=args.suite, - tmux_workers=tmux_workers, - devices=devices, - ) - manifest["model_deployments_fpath"] = ( - "dev/experiments/audit-helm-reproduction/configs/debug/" - "vicuna_no_chat_template.yaml" - ) - return manifest - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument( - "--manifest-type", - default="smoke", - choices=["smoke", "apples", "single", "vicuna_nochat"], - ) - parser.add_argument("--output", required=True) - parser.add_argument("--experiment-name", default="audit-smoke") - parser.add_argument("--suite", default="audit-smoke") - parser.add_argument("--max-eval-instances", type=int, default=None) - parser.add_argument("--tmux-workers", type=int, default=None) - parser.add_argument("--devices", default=None) - parser.add_argument("--run-entry", default=None) - parser.add_argument("--description", default=None) - args = parser.parse_args() - - if args.manifest_type == "smoke": - manifest = build_smoke_manifest(args) - elif args.manifest_type == "apples": - manifest = build_apples_manifest(args) - elif args.manifest_type == "single": - manifest = build_single_manifest(args) - elif args.manifest_type == "vicuna_nochat": - manifest = build_vicuna_nochat_manifest(args) - else: - raise NotImplementedError(args.manifest_type) - out_fpath = Path(args.output) - out_fpath.parent.mkdir(parents=True, exist_ok=True) - out_fpath.write_text(dump_yaml(manifest)) - print(out_fpath) - - -if __name__ == "__main__": - main() diff --git a/dev/experiments/audit-helm-reproduction/python/metric_quantiles_report.py b/dev/experiments/audit-helm-reproduction/python/metric_quantiles_report.py deleted file mode 100644 index 79ae83a..0000000 --- a/dev/experiments/audit-helm-reproduction/python/metric_quantiles_report.py +++ /dev/null @@ -1,181 +0,0 @@ -from __future__ import annotations - -import argparse -import datetime as datetime_mod -import json -from pathlib import Path -from typing import Any - -import kwutil - -from magnet.backends.helm.helm_outputs import HelmRun -from magnet.backends.helm.helm_run_diff import HelmRunDiff - - -def _pair_report(run_a: str, run_b: str, label: str) -> dict[str, Any]: - diff = HelmRunDiff( - run_a=HelmRun.coerce(run_a), - run_b=HelmRun.coerce(run_b), - a_name=f'{label}:A', - b_name=f'{label}:B', - ) - return { - 'label': label, - 'inputs': { - 'run_a': str(Path(run_a).expanduser().resolve()), - 'run_b': str(Path(run_b).expanduser().resolve()), - }, - 'run_level': { - 'agreement': diff._value_agreement_summary(), - 'distance': diff.value_distance_profile(), - }, - 'instance_level': { - 'agreement': diff.instance_agreement_profile(), - 'distance': diff.instance_distance_profile(), - }, - 'diagnosis': diff.summary_dict(level=20).get('diagnosis', {}), - } - - -def _instance_metric_rows(pair: dict[str, Any]) -> dict[tuple[str, str | None], dict[str, Any]]: - distance_rows = pair['instance_level']['distance'].get('by_metric', []) - agreement_rows = pair['instance_level']['agreement'].get('by_metric', []) - agree_lut = { - (row.get('metric_class'), row.get('metric')): row - for row in agreement_rows - } - rows = {} - for row in distance_rows: - key = (row.get('metric_class'), row.get('metric')) - rows[key] = { - 'metric_class': row.get('metric_class'), - 'metric': row.get('metric'), - 'count': row.get('summary', {}).get('count'), - 'agree_ratio': agree_lut.get(key, {}).get('agree_ratio'), - 'mismatched': agree_lut.get(key, {}).get('mismatched'), - 'abs_delta': row.get('summary', {}).get('abs_delta', {}), - 'rel_delta': row.get('summary', {}).get('rel_delta', {}), - } - return rows - - -def _run_class_rows(pair: dict[str, Any]) -> dict[str, dict[str, Any]]: - agree = pair['run_level']['agreement'].get('by_class', {}) - dist = pair['run_level']['distance'].get('by_class', {}) - rows = {} - for cls in sorted(set(agree) | set(dist)): - rows[cls] = { - 'metric_class': cls, - 'comparable': (agree.get(cls) or {}).get('comparable'), - 'agree_ratio': (agree.get(cls) or {}).get('agree_ratio'), - 'mismatched': (agree.get(cls) or {}).get('mismatched'), - 'abs_delta': (dist.get(cls) or {}).get('abs_delta', {}), - 'rel_delta': (dist.get(cls) or {}).get('rel_delta', {}), - 'count': (dist.get(cls) or {}).get('count'), - } - return rows - - -def _fmt(x: Any) -> str: - if x is None: - return 'NA' - if isinstance(x, float): - return f'{x:.6g}' - return str(x) - - -def _build_text(report: dict[str, Any]) -> str: - left = report['pairs'][0] - right = report['pairs'][1] - left_run = _run_class_rows(left) - right_run = _run_class_rows(right) - left_inst = _instance_metric_rows(left) - right_inst = _instance_metric_rows(right) - - lines = [] - lines.append('Metric Quantiles Comparison') - lines.append('') - lines.append(f"generated_utc: {report['generated_utc']}") - lines.append(f"left_label: {left['label']}") - lines.append(f"right_label: {right['label']}") - lines.append('') - lines.append('pair_diagnoses:') - lines.append(f" {left['label']}: {left['diagnosis'].get('label')}") - lines.append(f" {right['label']}: {right['diagnosis'].get('label')}") - lines.append('') - lines.append('run_level_by_class:') - for cls in sorted(set(left_run) | set(right_run)): - lrow = left_run.get(cls, {}) - rrow = right_run.get(cls, {}) - lines.append(f' class={cls}') - lines.append( - f" {left['label']}: agree={_fmt(lrow.get('agree_ratio'))} count={_fmt(lrow.get('count'))} " - f"abs_p50={_fmt((lrow.get('abs_delta') or {}).get('p50'))} " - f"abs_p90={_fmt((lrow.get('abs_delta') or {}).get('p90'))} " - f"abs_p99={_fmt((lrow.get('abs_delta') or {}).get('p99'))} " - f"abs_max={_fmt((lrow.get('abs_delta') or {}).get('max'))}" - ) - lines.append( - f" {right['label']}: agree={_fmt(rrow.get('agree_ratio'))} count={_fmt(rrow.get('count'))} " - f"abs_p50={_fmt((rrow.get('abs_delta') or {}).get('p50'))} " - f"abs_p90={_fmt((rrow.get('abs_delta') or {}).get('p90'))} " - f"abs_p99={_fmt((rrow.get('abs_delta') or {}).get('p99'))} " - f"abs_max={_fmt((rrow.get('abs_delta') or {}).get('max'))}" - ) - lines.append('') - lines.append('instance_level_by_metric:') - for key in sorted(set(left_inst) | set(right_inst), key=lambda k: (str(k[0]), str(k[1]))): - lrow = left_inst.get(key, {}) - rrow = right_inst.get(key, {}) - lines.append(f' metric_class={key[0]} metric={key[1]}') - lines.append( - f" {left['label']}: agree={_fmt(lrow.get('agree_ratio'))} count={_fmt(lrow.get('count'))} " - f"abs_p50={_fmt((lrow.get('abs_delta') or {}).get('p50'))} " - f"abs_p90={_fmt((lrow.get('abs_delta') or {}).get('p90'))} " - f"abs_p99={_fmt((lrow.get('abs_delta') or {}).get('p99'))} " - f"abs_max={_fmt((lrow.get('abs_delta') or {}).get('max'))}" - ) - lines.append( - f" {right['label']}: agree={_fmt(rrow.get('agree_ratio'))} count={_fmt(rrow.get('count'))} " - f"abs_p50={_fmt((rrow.get('abs_delta') or {}).get('p50'))} " - f"abs_p90={_fmt((rrow.get('abs_delta') or {}).get('p90'))} " - f"abs_p99={_fmt((rrow.get('abs_delta') or {}).get('p99'))} " - f"abs_max={_fmt((rrow.get('abs_delta') or {}).get('max'))}" - ) - return '\n'.join(lines) + '\n' - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument('--left-run-a', required=True) - parser.add_argument('--left-run-b', required=True) - parser.add_argument('--left-label', required=True) - parser.add_argument('--right-run-a', required=True) - parser.add_argument('--right-run-b', required=True) - parser.add_argument('--right-label', required=True) - parser.add_argument('--report-dpath', required=True) - args = parser.parse_args() - - report_dpath = Path(args.report_dpath).expanduser().resolve() - report_dpath.mkdir(parents=True, exist_ok=True) - stamp = datetime_mod.datetime.now(datetime_mod.UTC).strftime('%Y%m%dT%H%M%SZ') - - report = { - 'generated_utc': stamp, - 'pairs': [ - _pair_report(args.left_run_a, args.left_run_b, args.left_label), - _pair_report(args.right_run_a, args.right_run_b, args.right_label), - ], - } - report = kwutil.Json.ensure_serializable(report) - - json_fpath = report_dpath / f'metric_quantiles_{stamp}.json' - txt_fpath = report_dpath / f'metric_quantiles_{stamp}.txt' - json_fpath.write_text(json.dumps(report, indent=2)) - txt_fpath.write_text(_build_text(report)) - print(f'Wrote metric quantiles report: {json_fpath}') - print(f'Wrote metric quantiles text: {txt_fpath}') - - -if __name__ == '__main__': - main() diff --git a/dev/experiments/audit-helm-reproduction/python/paper_labels.py b/dev/experiments/audit-helm-reproduction/python/paper_labels.py deleted file mode 100644 index 26566b5..0000000 --- a/dev/experiments/audit-helm-reproduction/python/paper_labels.py +++ /dev/null @@ -1,55 +0,0 @@ -from __future__ import annotations - -from pathlib import Path -from typing import Any - -import kwutil - -from common import audit_root - - -def paper_label_config_fpath() -> Path: - return audit_root() / 'configs' / 'paper_label_mappings.yaml' - - -def _load_config() -> dict[str, Any]: - fpath = paper_label_config_fpath() - if not fpath.exists(): - return {} - data = kwutil.Yaml.load(fpath) - if not isinstance(data, dict): - raise TypeError(f'Expected mapping in {fpath}') - return data - - -class PaperLabelManager: - """ - Lightweight relabel helper inspired by kwplot.managers.LabelManager. - - This keeps a single checked-in mapping file for paper-facing labels, while - still letting reports preserve the raw internal machine codes. - """ - - def __init__(self, style: str = 'paper_short'): - self.style = style - self.config = _load_config() - self.machine_map = self.config.get('machine_host', {}) or {} - - def machine_label(self, machine_host: str | None, *, fallback: str | None = None) -> str: - raw = str(machine_host or fallback or 'unknown') - info = self.machine_map.get(raw, {}) or {} - return str(info.get(self.style) or raw) - - def relabel_text(self, text: str | None) -> str | None: - if text is None: - return None - new_text = str(text) - keys = sorted(self.machine_map.keys(), key=len, reverse=True) - for key in keys: - repl = self.machine_label(key) - new_text = new_text.replace(str(key), repl) - return new_text - - -def load_paper_label_manager(style: str = 'paper_short') -> PaperLabelManager: - return PaperLabelManager(style=style) diff --git a/dev/experiments/audit-helm-reproduction/python/rebuild_all_core_reports_from_index.py b/dev/experiments/audit-helm-reproduction/python/rebuild_all_core_reports_from_index.py deleted file mode 100644 index f7634dd..0000000 --- a/dev/experiments/audit-helm-reproduction/python/rebuild_all_core_reports_from_index.py +++ /dev/null @@ -1,68 +0,0 @@ -from __future__ import annotations - -import argparse -import subprocess -from collections import Counter -from pathlib import Path - -from common import audit_root, default_report_root, env_defaults -from rebuild_core_report_from_index import latest_index_csv, load_rows, matching_rows -from compare_batch import collect_historic_candidates, choose_historic_candidate - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument('--index-fpath', default=None) - parser.add_argument('--index-dpath', default=str(default_report_root() / 'indexes')) - parser.add_argument('--report-root', default=str(default_report_root())) - parser.add_argument('--allow-single-repeat', action='store_true') - args = parser.parse_args() - - index_fpath = Path(args.index_fpath) if args.index_fpath else latest_index_csv(Path(args.index_dpath)) - rows = load_rows(index_fpath) - run_entries = sorted({row.get('run_entry') for row in rows if row.get('run_entry')}) - counts = Counter(row.get('run_entry') for row in rows if row.get('run_entry')) - - script = audit_root() / 'python' / 'rebuild_core_report_from_index.py' - built = 0 - skipped = [] - for run_entry in run_entries: - matches = matching_rows(rows, run_entry) - n = len(matches) - if n == 0: - skipped.append((run_entry, n, 'no_indexed_runs')) - continue - if n < 2 and not args.allow_single_repeat: - skipped.append((run_entry, n, 'not_enough_matching_runs')) - continue - desired_max = None - try: - desired_max = int(matches[0].get('max_eval_instances')) if matches and matches[0].get('max_eval_instances') else None - except Exception: - desired_max = None - historic_candidates = collect_historic_candidates(env_defaults()['HELM_PRECOMPUTED_ROOT'], run_entry) - chosen_historic, _info = choose_historic_candidate(historic_candidates, desired_max) - if chosen_historic is None: - skipped.append((run_entry, n, 'no_historic_match')) - continue - cmd = [ - env_defaults()['AIQ_PYTHON'], - str(script), - '--run-entry', str(run_entry), - '--index-fpath', str(index_fpath), - ] - if args.allow_single_repeat: - cmd.append('--allow-single-repeat') - subprocess.run(cmd, check=True) - built += 1 - - print(f'index_fpath={index_fpath}') - print(f'n_unique_run_entries={len(run_entries)}') - print(f'n_reports_built={built}') - print(f'n_skipped={len(skipped)}') - for run_entry, n, reason in skipped: - print(f'skipped run_entry={run_entry!r} matching_runs={n} reason={reason}') - - -if __name__ == '__main__': - main() diff --git a/dev/experiments/audit-helm-reproduction/python/rebuild_core_report_from_index.py b/dev/experiments/audit-helm-reproduction/python/rebuild_core_report_from_index.py deleted file mode 100644 index 0274d21..0000000 --- a/dev/experiments/audit-helm-reproduction/python/rebuild_core_report_from_index.py +++ /dev/null @@ -1,242 +0,0 @@ -from __future__ import annotations - -import argparse -import csv -import datetime as datetime_mod -import json -import os -import subprocess -from pathlib import Path -from typing import Any - -from common import audit_root, default_report_root, env_defaults -from compare_batch import collect_historic_candidates, choose_historic_candidate - - -def latest_index_csv(index_dpath: Path) -> Path: - cands = sorted(index_dpath.glob('audit_results_index_*.csv'), reverse=True) - if not cands: - raise FileNotFoundError(f'No index csv files found in {index_dpath}') - return cands[0] - - -def load_rows(index_fpath: Path) -> list[dict[str, Any]]: - with index_fpath.open() as file: - return list(csv.DictReader(file)) - - -def _coerce_float(x): - try: - return float(x) - except Exception: - return float('-inf') - - -def matching_rows( - rows: list[dict[str, Any]], - run_entry: str, - experiment_name: str | None = None, -) -> list[dict[str, Any]]: - out = [] - for row in rows: - if row.get('run_entry') != run_entry: - continue - if experiment_name is not None and row.get('experiment_name') != experiment_name: - continue - if row.get('status') not in {'computed', 'reused', 'unknown', ''}: - continue - if row.get('has_run_spec', '').lower() not in {'true', '1'}: - continue - run_dir = row.get('run_dir') - if not run_dir: - continue - out.append(row) - out.sort(key=lambda r: (_coerce_float(r.get('manifest_timestamp')), r.get('experiment_name', '')), reverse=True) - return out - - -def slugify(text: str) -> str: - return ( - text.replace('/', '-') - .replace(':', '-') - .replace(',', '-') - .replace('=', '-') - .replace('@', '-') - ) - - -def _write_latest_selection(report_dpath: Path, selection: dict[str, Any]) -> Path: - stamp = datetime_mod.datetime.now(datetime_mod.UTC).strftime('%Y%m%dT%H%M%SZ') - history_dpath = report_dpath / '.history' / stamp[:8] - history_dpath.mkdir(parents=True, exist_ok=True) - history_fpath = history_dpath / f'report_selection_{stamp}.json' - history_fpath.write_text(json.dumps(selection, indent=2)) - latest_fpath = report_dpath / 'report_selection.latest.json' - if latest_fpath.exists() or latest_fpath.is_symlink(): - latest_fpath.unlink() - rel_src = os.path.relpath(history_fpath, start=report_dpath) - os.symlink(rel_src, latest_fpath) - return latest_fpath - - -def _safe_unlink(path: Path) -> None: - if path.exists() or path.is_symlink(): - path.unlink() - - -def _symlink(target: str | os.PathLike[str], link_path: Path) -> Path: - target = Path(target).expanduser().resolve() - link_path.parent.mkdir(parents=True, exist_ok=True) - _safe_unlink(link_path) - rel_src = os.path.relpath(target, start=link_path.parent) - os.symlink(rel_src, link_path) - return link_path - - -def _find_kwdagger_job_dpath(run_dpath: str | os.PathLike[str]) -> Path | None: - current = Path(run_dpath).expanduser().resolve() - for cand in [current, *current.parents]: - if (cand / 'job_config.json').exists(): - return cand - return None - - -def _write_selected_run_symlinks(report_dpath: Path, selection: dict[str, Any]) -> dict[str, str]: - created = {} - run_targets = { - 'kwdagger_a.run': selection['left_run_a'], - 'kwdagger_b.run': selection['left_run_b'], - 'official.run': selection['right_run_a'], - } - for name, target in run_targets.items(): - created[name] = str(_symlink(target, report_dpath / name)) - - job_targets = { - 'kwdagger_a.job': _find_kwdagger_job_dpath(selection['left_run_a']), - 'kwdagger_b.job': _find_kwdagger_job_dpath(selection['left_run_b']), - } - for name, target in job_targets.items(): - if target is not None: - created[name] = str(_symlink(target, report_dpath / name)) - return created - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument('--run-entry', required=True) - parser.add_argument('--index-fpath', default=None) - parser.add_argument('--index-dpath', default=str(default_report_root() / 'indexes')) - parser.add_argument('--precomputed-root', default=env_defaults()['HELM_PRECOMPUTED_ROOT']) - parser.add_argument('--report-dpath', default=None) - parser.add_argument('--left-label', default='kwdagger_repeat') - parser.add_argument('--right-label', default='official_vs_kwdagger') - parser.add_argument('--allow-single-repeat', action='store_true') - parser.add_argument('--experiment-name', default=None) - args = parser.parse_args() - - index_fpath = ( - Path(args.index_fpath).expanduser().resolve() - if args.index_fpath else - latest_index_csv(Path(args.index_dpath).expanduser().resolve()) - ) - rows = load_rows(index_fpath) - matches = matching_rows(rows, args.run_entry, experiment_name=args.experiment_name) - if not matches: - if args.experiment_name is not None: - raise SystemExit( - f'No indexed kwdagger runs found for run_entry={args.run_entry!r} ' - f'within experiment_name={args.experiment_name!r}' - ) - raise SystemExit(f'No indexed kwdagger runs found for run_entry={args.run_entry!r}') - - if len(matches) >= 2: - left_a = matches[0]['run_dir'] - left_b = matches[1]['run_dir'] - elif args.allow_single_repeat: - left_a = matches[0]['run_dir'] - left_b = matches[0]['run_dir'] - else: - raise SystemExit( - f'Need at least 2 matching kwdagger runs for run_entry={args.run_entry!r}; ' - f'found {len(matches)}. Use --allow-single-repeat to duplicate the latest run.' - ) - - desired_max = None - try: - desired_max = int(matches[0].get('max_eval_instances')) if matches[0].get('max_eval_instances') else None - except Exception: - desired_max = None - - historic_candidates = collect_historic_candidates(args.precomputed_root, args.run_entry) - chosen_historic, info = choose_historic_candidate(historic_candidates, desired_max) - if chosen_historic is None: - raise SystemExit(f'No historic HELM candidate found for run_entry={args.run_entry!r}') - - report_dpath = Path(args.report_dpath) if args.report_dpath else ( - default_report_root() / f'core-metrics-{slugify(args.run_entry)}' - ) - report_dpath = report_dpath.expanduser().resolve() - report_dpath.mkdir(parents=True, exist_ok=True) - - script = audit_root() / 'python' / 'core_metric_report.py' - cmd = [ - env_defaults()['AIQ_PYTHON'], - str(script), - '--left-run-a', str(left_a), - '--left-run-b', str(left_b), - '--left-label', args.left_label, - '--right-run-a', str(chosen_historic['run_dir']), - '--right-run-b', str(left_a), - '--right-label', args.right_label, - '--report-dpath', str(report_dpath), - ] - print(f'index_fpath={index_fpath}') - print(f'left_run_a={left_a}') - print(f'left_run_b={left_b}') - print(f'right_run_a={chosen_historic["run_dir"]}') - print(f'report_dpath={report_dpath}') - print(f'historic_info={info}') - selection = { - 'index_fpath': str(index_fpath), - 'run_entry': args.run_entry, - 'left_run_a': str(left_a), - 'left_run_b': str(left_b), - 'right_run_a': str(chosen_historic['run_dir']), - 'left_label': args.left_label, - 'right_label': args.right_label, - 'report_dpath': str(report_dpath), - 'experiment_name': args.experiment_name, - 'historic_info': info, - } - selection_fpath = _write_latest_selection(report_dpath, selection) - link_info = _write_selected_run_symlinks(report_dpath, selection) - selection['selected_run_links'] = link_info - selection_fpath = _write_latest_selection(report_dpath, selection) - print(f'selection_fpath={selection_fpath}') - subprocess.run(cmd, check=True) - - inspect_script = audit_root() / 'python' / 'inspect_pair_samples.py' - sample_jobs = [ - [ - env_defaults()['AIQ_PYTHON'], - str(inspect_script), - '--run-a', str(left_a), - '--run-b', str(left_b), - '--label', args.left_label, - '--report-dpath', str(report_dpath), - ], - [ - env_defaults()['AIQ_PYTHON'], - str(inspect_script), - '--run-a', str(chosen_historic['run_dir']), - '--run-b', str(left_a), - '--label', args.right_label, - '--report-dpath', str(report_dpath), - ], - ] - for inspect_cmd in sample_jobs: - subprocess.run(inspect_cmd, check=True) - - -if __name__ == '__main__': - main() diff --git a/dev/experiments/audit-helm-reproduction/python/render_schedule_params.py b/dev/experiments/audit-helm-reproduction/python/render_schedule_params.py deleted file mode 100644 index f7bdcec..0000000 --- a/dev/experiments/audit-helm-reproduction/python/render_schedule_params.py +++ /dev/null @@ -1,79 +0,0 @@ -from __future__ import annotations - -import argparse -import json - -from common import dump_yaml, experiment_result_dpath, load_manifest - - -def build_schedule_params(manifest: dict) -> dict: - matrix = { - "helm.run_entry": list(manifest["run_entries"]), - "helm.max_eval_instances": [manifest["max_eval_instances"]], - "helm.precomputed_root": manifest.get("precomputed_root", None), - "helm.suite": [manifest.get("suite", "audit-smoke")], - "helm.require_per_instance_stats": [ - manifest.get("require_per_instance_stats", True) - ], - "helm.mode": [manifest.get("mode", "compute_if_missing")], - "helm.materialize": [manifest.get("materialize", "symlink")], - "helm.local_path": [manifest.get("local_path", "prod_env")], - } - model_deployments_fpath = manifest.get("model_deployments_fpath", None) - if model_deployments_fpath is not None: - matrix["helm.model_deployments_fpath"] = [model_deployments_fpath] - enable_hf = manifest.get("enable_huggingface_models", []) - if enable_hf: - matrix["helm.enable_huggingface_models"] = [json.dumps(enable_hf)] - enable_local_hf = manifest.get("enable_local_huggingface_models", []) - if enable_local_hf: - matrix["helm.enable_local_huggingface_models"] = [json.dumps(enable_local_hf)] - return { - "pipeline": "magnet.backends.helm.pipeline.helm_single_run_pipeline()", - "matrix": matrix, - } - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument("--manifest", required=True) - parser.add_argument( - "--mode", - required=True, - choices=[ - "params", - "experiment_name", - "result_dpath", - "backend", - "tmux_workers", - "devices", - "precomputed_root", - ], - ) - args = parser.parse_args() - manifest = load_manifest(args.manifest) - - if args.mode == "params": - print(dump_yaml(build_schedule_params(manifest)), end="") - elif args.mode == "experiment_name": - print(manifest["experiment_name"]) - elif args.mode == "result_dpath": - print(experiment_result_dpath(manifest)) - elif args.mode == "backend": - print(manifest.get("backend", "tmux")) - elif args.mode == "tmux_workers": - print(manifest.get("tmux_workers", 2)) - elif args.mode == "devices": - print(manifest.get("devices", "0,1")) - elif args.mode == "precomputed_root": - value = manifest.get("precomputed_root", None) - if value is None: - print("") - else: - print(value) - else: - raise AssertionError(args.mode) - - -if __name__ == "__main__": - main() diff --git a/dev/experiments/audit-helm-reproduction/python/resolve_run.py b/dev/experiments/audit-helm-reproduction/python/resolve_run.py deleted file mode 100644 index 35e9693..0000000 --- a/dev/experiments/audit-helm-reproduction/python/resolve_run.py +++ /dev/null @@ -1,134 +0,0 @@ -from __future__ import annotations - -import argparse -import json -from pathlib import Path -from typing import Any - -import kwutil - -from magnet.backends.helm.cli.materialize_helm_run import ( - discover_benchmark_output_dirs, - run_dir_matches_requested, -) -from magnet.backends.helm.helm_outputs import HelmOutputs - - -def load_json(fpath: Path) -> dict[str, Any]: - return json.loads(fpath.read_text()) - - -def summarize_run_artifacts(run_dir: Path | None) -> dict[str, Any]: - required_files = [ - 'run_spec.json', - 'scenario_state.json', - 'stats.json', - 'per_instance_stats.json', - ] - if run_dir is None: - return { - 'artifact_status': 'missing_run_dir', - 'missing_files': required_files, - } - missing_files = [name for name in required_files if not (run_dir / name).exists()] - return { - 'artifact_status': 'ready' if not missing_files else 'incomplete_run_dir', - 'missing_files': missing_files, - } - - -def resolve_kwdagger_run(results_dpath: Path, run_entry: str) -> dict[str, Any] | None: - helm_root = results_dpath / 'helm' - if not helm_root.exists(): - return None - matches = [] - for job_cfg in helm_root.glob('*/job_config.json'): - data = load_json(job_cfg) - if data.get('helm.run_entry') != run_entry: - continue - job_dpath = job_cfg.parent - run_dirs = [] - try: - suites = HelmOutputs.coerce(job_dpath / 'benchmark_output').suites() - for suite in suites: - for run in suite.runs(): - run_dirs.append(Path(run.path)) - except Exception: - run_dirs = [] - if len(run_dirs) == 1: - run_dir = run_dirs[0] - else: - run_dir = None - for candidate in run_dirs: - if run_dir_matches_requested(candidate.name, run_entry): - run_dir = candidate - break - match = { - 'job_id': job_dpath.name, - 'job_dpath': str(job_dpath), - 'run_dir': None if run_dir is None else str(run_dir), - 'run_entry': run_entry, - } - match.update(summarize_run_artifacts(run_dir)) - matches.append(match) - if not matches: - return None - if len(matches) > 1: - raise RuntimeError( - f'Found multiple kwdagger matches for {run_entry}: {kwutil.Json.dumps(matches, indent=2)}' - ) - return matches[0] - - -def resolve_historic_run(precomputed_root: Path, run_entry: str) -> dict[str, Any] | None: - matches = [] - for bo in discover_benchmark_output_dirs([precomputed_root]): - try: - outputs = HelmOutputs.coerce(bo) - except Exception: - continue - for suite in outputs.suites(): - for run in suite.runs(): - run_dir = Path(run.path) - if not run_dir_matches_requested(run.name, run_entry): - continue - match = { - 'run_dir': str(run_dir), - 'suite': suite.path.name if hasattr(suite, 'path') else None, - 'helm_version': run_dir.parent.name, - 'run_entry': run_entry, - } - match.update(summarize_run_artifacts(run_dir)) - matches.append(match) - if not matches: - return None - return { - 'matches': matches, - } - - -def main() -> None: - parser = argparse.ArgumentParser() - parser.add_argument('--mode', required=True, choices=['kwdg', 'historic']) - parser.add_argument('--run-entry', required=True) - parser.add_argument('--results-dpath', default=None) - parser.add_argument('--precomputed-root', default='/data/crfm-helm-public') - args = parser.parse_args() - - if args.mode == 'kwdg': - if args.results_dpath is None: - raise SystemExit('--results-dpath is required in kwdg mode') - info = resolve_kwdagger_run( - results_dpath=Path(args.results_dpath).expanduser().resolve(), - run_entry=args.run_entry, - ) - else: - info = resolve_historic_run( - precomputed_root=Path(args.precomputed_root).expanduser().resolve(), - run_entry=args.run_entry, - ) - print(kwutil.Json.dumps(info, indent=2)) - - -if __name__ == '__main__': - main() diff --git a/dev/experiments/audit-helm-reproduction/scripts/aggregate_core_reports.sh b/dev/experiments/audit-helm-reproduction/scripts/aggregate_core_reports.sh deleted file mode 100755 index 9511a7f..0000000 --- a/dev/experiments/audit-helm-reproduction/scripts/aggregate_core_reports.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set +x - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/common.sh" -audit::set_defaults - -AUDIT_PYTHON_DIR="${AUDIT_ROOT}/python" - -"$AIQ_PYTHON" "$AUDIT_PYTHON_DIR/aggregate_core_reports.py" "$@" diff --git a/dev/experiments/audit-helm-reproduction/scripts/analyze_experiment_from_index.sh b/dev/experiments/audit-helm-reproduction/scripts/analyze_experiment_from_index.sh deleted file mode 100755 index 0175e2a..0000000 --- a/dev/experiments/audit-helm-reproduction/scripts/analyze_experiment_from_index.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set +x - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/common.sh" -audit::set_defaults - -"$AIQ_PYTHON" "${AUDIT_ROOT}/python/analyze_experiment_from_index.py" "$@" diff --git a/dev/experiments/audit-helm-reproduction/scripts/check_env.sh b/dev/experiments/audit-helm-reproduction/scripts/check_env.sh deleted file mode 100755 index bad151f..0000000 --- a/dev/experiments/audit-helm-reproduction/scripts/check_env.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "${SCRIPT_DIR}/common.sh" - -audit::set_defaults - -printf 'Audit environment\n' -printf '=================\n' -audit::print_env -printf '\n' - -for path_var in AIQ_MAGNET_ROOT; do - path="${!path_var}" - if [[ ! -e "$path" ]]; then - printf '%s does not exist: %s\n' "$path_var" "$path" >&2 - exit 1 - fi -done - -REQUIRE_PRECOMPUTED_ROOT="${AUDIT_REQUIRE_PRECOMPUTED_ROOT:-1}" -if [[ "$REQUIRE_PRECOMPUTED_ROOT" == "1" ]]; then - if [[ ! -e "$HELM_PRECOMPUTED_ROOT" ]]; then - printf '%s does not exist: %s\n' "HELM_PRECOMPUTED_ROOT" "$HELM_PRECOMPUTED_ROOT" >&2 - exit 1 - fi -fi - -if [[ ! -d "$AUDIT_RESULTS_ROOT" ]]; then - if mkdir -p "$AUDIT_RESULTS_ROOT" 2>/dev/null; then - : - else - printf 'Warning: unable to create AUDIT_RESULTS_ROOT: %s\n' "$AUDIT_RESULTS_ROOT" >&2 - printf 'The external runner should create this path or override AUDIT_RESULTS_ROOT.\n' >&2 - fi -fi -if ! command -v kwdagger >/dev/null 2>&1; then - printf 'kwdagger not found on PATH\n' >&2 - exit 1 -fi - -if ! command -v helm-run >/dev/null 2>&1; then - printf 'helm-run not found on PATH\n' >&2 - exit 1 -fi - -if ! command -v "$AIQ_PYTHON" >/dev/null 2>&1; then - printf 'Configured AIQ_PYTHON not found on PATH: %s\n' "$AIQ_PYTHON" >&2 - exit 1 -fi - -if ! "$AIQ_PYTHON" -c "import magnet" >/dev/null 2>&1; then - printf 'Unable to import magnet from %s\n' "$AIQ_PYTHON" >&2 - exit 1 -fi - -printf 'Environment looks good.\n' diff --git a/dev/experiments/audit-helm-reproduction/scripts/common.sh b/dev/experiments/audit-helm-reproduction/scripts/common.sh deleted file mode 100755 index 8fd51cb..0000000 --- a/dev/experiments/audit-helm-reproduction/scripts/common.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -AUDIT_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -AUDIT_ROOT="$(cd "${AUDIT_SCRIPT_DIR}/.." && pwd)" - -audit::set_defaults() { - export AIQ_MAGNET_ROOT="${AIQ_MAGNET_ROOT:-$HOME/code/aiq-magnet}" - export AIQ_PYTHON="${AIQ_PYTHON:-python}" - export HELM_PRECOMPUTED_ROOT="${HELM_PRECOMPUTED_ROOT:-/data/crfm-helm-public}" - export AUDIT_RESULTS_ROOT="${AUDIT_RESULTS_ROOT:-/data/crfm-helm-audit}" - export AUDIT_DEFAULT_MAX_EVAL_INSTANCES="${AUDIT_DEFAULT_MAX_EVAL_INSTANCES:-100}" - export AUDIT_DEFAULT_TMUX_WORKERS="${AUDIT_DEFAULT_TMUX_WORKERS:-2}" -} - -audit::print_env() { - printf 'AIQ_MAGNET_ROOT=%s\n' "$AIQ_MAGNET_ROOT" - printf 'AIQ_PYTHON=%s\n' "$AIQ_PYTHON" - printf 'HELM_PRECOMPUTED_ROOT=%s\n' "$HELM_PRECOMPUTED_ROOT" - printf 'AUDIT_RESULTS_ROOT=%s\n' "$AUDIT_RESULTS_ROOT" - printf 'AUDIT_DEFAULT_MAX_EVAL_INSTANCES=%s\n' "$AUDIT_DEFAULT_MAX_EVAL_INSTANCES" - printf 'AUDIT_DEFAULT_TMUX_WORKERS=%s\n' "$AUDIT_DEFAULT_TMUX_WORKERS" -} - -audit::require_file() { - local path="$1" - if [[ ! -f "$path" ]]; then - printf 'Missing required file: %s\n' "$path" >&2 - exit 1 - fi -} diff --git a/dev/experiments/audit-helm-reproduction/scripts/compare_batch.sh b/dev/experiments/audit-helm-reproduction/scripts/compare_batch.sh deleted file mode 100755 index fe9209d..0000000 --- a/dev/experiments/audit-helm-reproduction/scripts/compare_batch.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "${SCRIPT_DIR}/common.sh" - -audit::set_defaults - -MANIFEST="${1:-${AUDIT_ROOT}/configs/generated/smoke_manifest.generated.yaml}" -audit::require_file "$MANIFEST" - -"$AIQ_PYTHON" "${AUDIT_ROOT}/python/compare_batch.py" \ - --manifest "$MANIFEST" diff --git a/dev/experiments/audit-helm-reproduction/scripts/compare_entry.sh b/dev/experiments/audit-helm-reproduction/scripts/compare_entry.sh deleted file mode 100755 index f3f0c91..0000000 --- a/dev/experiments/audit-helm-reproduction/scripts/compare_entry.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "${SCRIPT_DIR}/common.sh" - -audit::set_defaults - -RESULTS_DPATH="${1:?need kwdagger results root}" -RUN_ENTRY="${2:?need run entry}" -REPORT_DPATH="${3:-${AUDIT_ROOT}/reports/pairwise}" - -kwdg_json="$("$AIQ_PYTHON" "${AUDIT_ROOT}/python/resolve_run.py" \ - --mode kwdg \ - --results-dpath "$RESULTS_DPATH" \ - --run-entry "$RUN_ENTRY")" - -historic_json="$("$AIQ_PYTHON" "${AUDIT_ROOT}/python/resolve_run.py" \ - --mode historic \ - --precomputed-root "$HELM_PRECOMPUTED_ROOT" \ - --run-entry "$RUN_ENTRY")" - -kwdg_status="$(python - <<'PY' "$kwdg_json" -import json, sys -obj = json.loads(sys.argv[1]) -print(obj.get('artifact_status', 'unknown')) -PY -)" - -kwdg_run_dir="$(python - <<'PY' "$kwdg_json" -import json, sys -obj = json.loads(sys.argv[1]) -print(obj.get('run_dir') or '') -PY -)" - -if [[ "$kwdg_status" != "ready" ]]; then - echo "Resolved kwdagger job, but HELM run artifacts are not ready." >&2 - python - <<'PY' "$kwdg_json" -import json, sys -obj = json.loads(sys.argv[1]) -print(json.dumps(obj, indent=2)) -PY - exit 1 -fi - -historic_run_dir="$(python - <<'PY' "$historic_json" -import json, sys -obj = json.loads(sys.argv[1]) -matches = obj.get('matches', []) -if not matches: - raise SystemExit('No historic matches found') -ready = [m for m in matches if m.get('artifact_status') == 'ready'] -target = ready[-1] if ready else matches[-1] -print(target['run_dir']) -PY -)" - -"${AUDIT_ROOT}/scripts/compare_pair.sh" \ - "$historic_run_dir" \ - "$kwdg_run_dir" \ - "$REPORT_DPATH" diff --git a/dev/experiments/audit-helm-reproduction/scripts/compare_pair.sh b/dev/experiments/audit-helm-reproduction/scripts/compare_pair.sh deleted file mode 100755 index 710306a..0000000 --- a/dev/experiments/audit-helm-reproduction/scripts/compare_pair.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "${SCRIPT_DIR}/common.sh" - -audit::set_defaults - -RUN_A="${1:?need run A path}" -RUN_B="${2:?need run B path}" -REPORT_DPATH="${3:-${AUDIT_ROOT}/reports/pairwise}" - -"$AIQ_PYTHON" "${AUDIT_ROOT}/python/compare_pair.py" \ - --run-a "$RUN_A" \ - --run-b "$RUN_B" \ - --report-dpath "$REPORT_DPATH" diff --git a/dev/experiments/audit-helm-reproduction/scripts/core_metric_report.sh b/dev/experiments/audit-helm-reproduction/scripts/core_metric_report.sh deleted file mode 100755 index 43e15ee..0000000 --- a/dev/experiments/audit-helm-reproduction/scripts/core_metric_report.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "${SCRIPT_DIR}/common.sh" - -audit::set_defaults - -"$AIQ_PYTHON" "${AUDIT_ROOT}/python/core_metric_report.py" "$@" diff --git a/dev/experiments/audit-helm-reproduction/scripts/index_results.sh b/dev/experiments/audit-helm-reproduction/scripts/index_results.sh deleted file mode 100755 index 5527963..0000000 --- a/dev/experiments/audit-helm-reproduction/scripts/index_results.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set +x - -SCRIPT_DIR="$(dirname "$(readlink -f "$0")")" -source "$SCRIPT_DIR/common.sh" -audit::set_defaults - -fallback_host="${AUDIT_FALLBACK_HOST:-}" -AUDIT_PYTHON_DIR="${AUDIT_ROOT}/python" -AUDIT_REPORT_ROOT="${AUDIT_ROOT}/reports" - -"$AIQ_PYTHON" "$AUDIT_PYTHON_DIR/index_results.py" \ - --results-root "${1:-$AUDIT_RESULTS_ROOT}" \ - --report-dpath "${2:-$AUDIT_REPORT_ROOT/indexes}" \ - ${fallback_host:+--fallback-host "$fallback_host"} diff --git a/dev/experiments/audit-helm-reproduction/scripts/inspect_pair_samples.sh b/dev/experiments/audit-helm-reproduction/scripts/inspect_pair_samples.sh deleted file mode 100755 index 8f18224..0000000 --- a/dev/experiments/audit-helm-reproduction/scripts/inspect_pair_samples.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "${SCRIPT_DIR}/common.sh" - -audit::set_defaults - -"$AIQ_PYTHON" "${AUDIT_ROOT}/python/inspect_pair_samples.py" "$@" diff --git a/dev/experiments/audit-helm-reproduction/scripts/make_apples_manifest.sh b/dev/experiments/audit-helm-reproduction/scripts/make_apples_manifest.sh deleted file mode 100755 index b037d22..0000000 --- a/dev/experiments/audit-helm-reproduction/scripts/make_apples_manifest.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "${SCRIPT_DIR}/common.sh" - -audit::set_defaults - -OUTPUT="${1:-${AUDIT_ROOT}/configs/generated/apples_manifest.generated.yaml}" -if [[ $# -gt 0 ]]; then - shift -fi -mkdir -p "$(dirname "$OUTPUT")" - -"$AIQ_PYTHON" "${AUDIT_ROOT}/python/make_manifest.py" \ - --manifest-type apples \ - --experiment-name audit-smoke-apples \ - --suite audit-smoke-apples \ - --output "$OUTPUT" \ - "$@" - -printf 'Wrote apples-to-apples manifest: %s\n' "$OUTPUT" diff --git a/dev/experiments/audit-helm-reproduction/scripts/make_historic_grid_manifest.sh b/dev/experiments/audit-helm-reproduction/scripts/make_historic_grid_manifest.sh deleted file mode 100755 index 750a2f2..0000000 --- a/dev/experiments/audit-helm-reproduction/scripts/make_historic_grid_manifest.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "${SCRIPT_DIR}/common.sh" - -audit::set_defaults - -OUTPATH="${1:-${AUDIT_ROOT}/configs/generated/historic_grid.generated.yaml}" -shift || true - -"$AIQ_PYTHON" "${AUDIT_ROOT}/python/build_repro_manifest.py" \ - --output "$OUTPATH" \ - --experiment-name audit-historic-grid \ - --suite audit-historic-grid \ - "$@" - -printf 'Wrote historic grid manifest: %s\n' "$OUTPATH" -printf 'Wrote selection sidecar: %s.selection.yaml\n' "$OUTPATH" diff --git a/dev/experiments/audit-helm-reproduction/scripts/make_machine_shard_manifest.sh b/dev/experiments/audit-helm-reproduction/scripts/make_machine_shard_manifest.sh deleted file mode 100755 index 323746e..0000000 --- a/dev/experiments/audit-helm-reproduction/scripts/make_machine_shard_manifest.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "${SCRIPT_DIR}/common.sh" - -audit::set_defaults - -MACHINE_NAME="${1:?machine name required, e.g. namek or yardrat}" -SHARD_INDEX="${2:?shard index required}" -NUM_SHARDS="${3:?num shards required}" -OUTPATH="${4:-${AUDIT_ROOT}/configs/generated/${MACHINE_NAME}.generated.yaml}" -if [[ $# -gt 0 ]]; then shift; fi -if [[ $# -gt 0 ]]; then shift; fi -if [[ $# -gt 0 ]]; then shift; fi -if [[ $# -gt 0 ]]; then shift; fi - -"$AIQ_PYTHON" "${AUDIT_ROOT}/python/build_repro_manifest.py" \ - --output "$OUTPATH" \ - --experiment-name "audit-${MACHINE_NAME}" \ - --suite "audit-${MACHINE_NAME}" \ - --single-gpu \ - --devices 0 \ - --num-shards "$NUM_SHARDS" \ - --shard-index "$SHARD_INDEX" \ - "$@" - -printf 'Wrote machine shard manifest: %s\n' "$OUTPATH" -printf 'Wrote selection sidecar: %s.selection.yaml\n' "$OUTPATH" diff --git a/dev/experiments/audit-helm-reproduction/scripts/make_machine_subset_manifest.sh b/dev/experiments/audit-helm-reproduction/scripts/make_machine_subset_manifest.sh deleted file mode 100755 index 2c77eba..0000000 --- a/dev/experiments/audit-helm-reproduction/scripts/make_machine_subset_manifest.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "${SCRIPT_DIR}/common.sh" - -audit::set_defaults - -MACHINE_NAME="${1:?machine name required, e.g. namek or yardrat}" -OUTPATH="${2:-${AUDIT_ROOT}/configs/generated/${MACHINE_NAME}.subset.generated.yaml}" -if [[ $# -gt 0 ]]; then shift; fi -if [[ $# -gt 0 ]]; then shift; fi - -"$AIQ_PYTHON" "${AUDIT_ROOT}/python/build_repro_manifest.py" \ - --output "$OUTPATH" \ - --experiment-name "audit-${MACHINE_NAME}-subset" \ - --suite "audit-${MACHINE_NAME}-subset" \ - --single-gpu \ - --devices 0 \ - "$@" - -printf 'Wrote machine subset manifest: %s\n' "$OUTPATH" -printf 'Wrote selection sidecar: %s.selection.yaml\n' "$OUTPATH" diff --git a/dev/experiments/audit-helm-reproduction/scripts/make_repeat_pair_manifests.sh b/dev/experiments/audit-helm-reproduction/scripts/make_repeat_pair_manifests.sh deleted file mode 100755 index fea3767..0000000 --- a/dev/experiments/audit-helm-reproduction/scripts/make_repeat_pair_manifests.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "${SCRIPT_DIR}/common.sh" - -audit::set_defaults - -RUN_ENTRY="${1:-narrative_qa:model=eleutherai/pythia-6.9b,data_augmentation=canonical}" -BASE_NAME="${2:-audit-narrative-pythia}" -OUTPUT_DIR="${3:-${AUDIT_ROOT}/configs/generated}" -if [[ $# -gt 0 ]]; then - shift -fi -if [[ $# -gt 0 ]]; then - shift -fi -if [[ $# -gt 0 ]]; then - shift -fi - -mkdir -p "$OUTPUT_DIR" - -R1_OUTPUT="${OUTPUT_DIR}/${BASE_NAME}_r1.yaml" -R2_OUTPUT="${OUTPUT_DIR}/${BASE_NAME}_r2.yaml" - -"$AIQ_PYTHON" "${AUDIT_ROOT}/python/make_manifest.py" \ - --manifest-type single \ - --run-entry "$RUN_ENTRY" \ - --experiment-name "${BASE_NAME}-r1" \ - --suite "${BASE_NAME}-r1" \ - --description "Repeat run 1 for ${RUN_ENTRY}" \ - --output "$R1_OUTPUT" \ - "$@" - -"$AIQ_PYTHON" "${AUDIT_ROOT}/python/make_manifest.py" \ - --manifest-type single \ - --run-entry "$RUN_ENTRY" \ - --experiment-name "${BASE_NAME}-r2" \ - --suite "${BASE_NAME}-r2" \ - --description "Repeat run 2 for ${RUN_ENTRY}" \ - --output "$R2_OUTPUT" \ - "$@" - -printf 'Wrote repeat manifests:\n' -printf ' %s\n' "$R1_OUTPUT" -printf ' %s\n' "$R2_OUTPUT" diff --git a/dev/experiments/audit-helm-reproduction/scripts/make_smoke_manifest.sh b/dev/experiments/audit-helm-reproduction/scripts/make_smoke_manifest.sh deleted file mode 100755 index f959ee0..0000000 --- a/dev/experiments/audit-helm-reproduction/scripts/make_smoke_manifest.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "${SCRIPT_DIR}/common.sh" - -audit::set_defaults - -OUTPUT="${1:-${AUDIT_ROOT}/configs/generated/smoke_manifest.generated.yaml}" -if [[ $# -gt 0 ]]; then - shift -fi -mkdir -p "$(dirname "$OUTPUT")" - -"$AIQ_PYTHON" "${AUDIT_ROOT}/python/make_manifest.py" \ - --manifest-type smoke \ - --output "$OUTPUT" \ - "$@" - -printf 'Wrote smoke manifest: %s\n' "$OUTPUT" diff --git a/dev/experiments/audit-helm-reproduction/scripts/make_vicuna_nochat_manifest.sh b/dev/experiments/audit-helm-reproduction/scripts/make_vicuna_nochat_manifest.sh deleted file mode 100755 index f84643e..0000000 --- a/dev/experiments/audit-helm-reproduction/scripts/make_vicuna_nochat_manifest.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "${SCRIPT_DIR}/common.sh" - -audit::set_defaults - -OUTPATH="${1:-${AUDIT_ROOT}/configs/generated/vicuna_nochat_overnight.generated.yaml}" -shift || true - -"$AIQ_PYTHON" "${AUDIT_ROOT}/python/make_manifest.py" \ - --manifest-type vicuna_nochat \ - --output "$OUTPATH" \ - --experiment-name audit-vicuna-nochat-overnight \ - --suite audit-vicuna-nochat-overnight \ - "$@" - -printf 'Wrote Vicuna no-chat manifest: %s\n' "$OUTPATH" diff --git a/dev/experiments/audit-helm-reproduction/scripts/metric_quantiles_report.sh b/dev/experiments/audit-helm-reproduction/scripts/metric_quantiles_report.sh deleted file mode 100755 index cf12a7b..0000000 --- a/dev/experiments/audit-helm-reproduction/scripts/metric_quantiles_report.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "${SCRIPT_DIR}/common.sh" - -audit::set_defaults - -"$AIQ_PYTHON" "${AUDIT_ROOT}/python/metric_quantiles_report.py" "$@" diff --git a/dev/experiments/audit-helm-reproduction/scripts/rebuild_all_core_reports_from_index.sh b/dev/experiments/audit-helm-reproduction/scripts/rebuild_all_core_reports_from_index.sh deleted file mode 100755 index d1a5a36..0000000 --- a/dev/experiments/audit-helm-reproduction/scripts/rebuild_all_core_reports_from_index.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set +x - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/common.sh" -audit::set_defaults - -AUDIT_PYTHON_DIR="${AUDIT_ROOT}/python" - -"$AIQ_PYTHON" "$AUDIT_PYTHON_DIR/rebuild_all_core_reports_from_index.py" "$@" diff --git a/dev/experiments/audit-helm-reproduction/scripts/rebuild_core_report_from_index.sh b/dev/experiments/audit-helm-reproduction/scripts/rebuild_core_report_from_index.sh deleted file mode 100755 index d9d7604..0000000 --- a/dev/experiments/audit-helm-reproduction/scripts/rebuild_core_report_from_index.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set +x - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "$SCRIPT_DIR/common.sh" -audit::set_defaults - -AUDIT_PYTHON_DIR="${AUDIT_ROOT}/python" -AUDIT_REPORT_ROOT="${AUDIT_ROOT}/reports" - -"$AIQ_PYTHON" "$AUDIT_PYTHON_DIR/rebuild_core_report_from_index.py" "$@" diff --git a/dev/experiments/audit-helm-reproduction/scripts/resolve_run.sh b/dev/experiments/audit-helm-reproduction/scripts/resolve_run.sh deleted file mode 100755 index d0ea3d9..0000000 --- a/dev/experiments/audit-helm-reproduction/scripts/resolve_run.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "${SCRIPT_DIR}/common.sh" - -audit::set_defaults - -"$AIQ_PYTHON" "${AUDIT_ROOT}/python/resolve_run.py" "$@" diff --git a/dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh b/dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh deleted file mode 100755 index 9f50491..0000000 --- a/dev/experiments/audit-helm-reproduction/scripts/run_from_manifest.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set +x - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "${SCRIPT_DIR}/common.sh" - -audit::set_defaults - -MANIFEST="${1:-${AUDIT_ROOT}/configs/smoke_manifest.yaml}" -audit::require_file "$MANIFEST" - -EXPERIMENT_NAME="$("$AIQ_PYTHON" "${AUDIT_ROOT}/python/render_schedule_params.py" \ - --manifest "$MANIFEST" --mode experiment_name)" -RESULT_DPATH="$("$AIQ_PYTHON" "${AUDIT_ROOT}/python/render_schedule_params.py" \ - --manifest "$MANIFEST" --mode result_dpath)" -PARAMS="$("$AIQ_PYTHON" "${AUDIT_ROOT}/python/render_schedule_params.py" \ - --manifest "$MANIFEST" --mode params)" -BACKEND="$("$AIQ_PYTHON" "${AUDIT_ROOT}/python/render_schedule_params.py" \ - --manifest "$MANIFEST" --mode backend)" -TMUX_WORKERS="$("$AIQ_PYTHON" "${AUDIT_ROOT}/python/render_schedule_params.py" \ - --manifest "$MANIFEST" --mode tmux_workers)" -DEVICES="$("$AIQ_PYTHON" "${AUDIT_ROOT}/python/render_schedule_params.py" \ - --manifest "$MANIFEST" --mode devices)" -PRECOMPUTED_ROOT="$("$AIQ_PYTHON" "${AUDIT_ROOT}/python/render_schedule_params.py" \ - --manifest "$MANIFEST" --mode precomputed_root)" - -if [[ -n "$PRECOMPUTED_ROOT" ]]; then - AUDIT_REQUIRE_PRECOMPUTED_ROOT=1 "${AUDIT_ROOT}/scripts/check_env.sh" >/dev/null -else - AUDIT_REQUIRE_PRECOMPUTED_ROOT=0 "${AUDIT_ROOT}/scripts/check_env.sh" >/dev/null -fi - -mkdir -p "$RESULT_DPATH" - -printf 'Launching experiment: %s\n' "$EXPERIMENT_NAME" -printf 'Results root: %s\n' "$RESULT_DPATH" -printf 'Backend: %s\n' "$BACKEND" -printf 'Devices: %s\n' "$DEVICES" -printf 'tmux_workers: %s\n' "$TMUX_WORKERS" - -QUEUE_NAME="$(printf 'audit-%s' "$EXPERIMENT_NAME" | tr -c 'A-Za-z0-9._-' '-')" -printf 'queue_name: %s\n' "$QUEUE_NAME" -printf 'Schedule params:\n%s\n' "$PARAMS" - -kwdagger schedule \ - --queue_name="$QUEUE_NAME" \ - --params="$PARAMS" \ - --devices="$DEVICES" \ - --tmux_workers="$TMUX_WORKERS" \ - --root_dpath="$RESULT_DPATH" \ - --backend="$BACKEND" \ - --skip_existing=1 \ - --run=1 diff --git a/dev/experiments/audit-helm-reproduction/scripts/run_smoke.sh b/dev/experiments/audit-helm-reproduction/scripts/run_smoke.sh deleted file mode 100755 index fd37374..0000000 --- a/dev/experiments/audit-helm-reproduction/scripts/run_smoke.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -set +x - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -source "${SCRIPT_DIR}/common.sh" - -audit::set_defaults - -MANIFEST="${1:-${AUDIT_ROOT}/configs/generated/smoke_manifest.generated.yaml}" - -if [[ ! -f "$MANIFEST" ]]; then - "${AUDIT_ROOT}/scripts/make_smoke_manifest.sh" "$MANIFEST" -fi - -"${AUDIT_ROOT}/scripts/run_from_manifest.sh" "$MANIFEST" diff --git a/dev/oneoff/check_results_overlap.py b/dev/oneoff/check_results_overlap.py deleted file mode 100644 index 553ce26..0000000 --- a/dev/oneoff/check_results_overlap.py +++ /dev/null @@ -1,653 +0,0 @@ -""" -Notebook style code the developer is working with interactively by copy/pasting -blocks into IPython. - -!uv pip install kaleido plotly -""" - -import pandas as pd -import ubelt as ub -import kwutil -from magnet.backends.helm.helm_outputs import HelmRun -from magnet.backends.helm.helm_outputs import HelmOutputs -from magnet.backends.helm.helm_run_analysis import HelmRunAnalysis -from magnet.backends.helm.helm_run_diff import HelmRunDiff -from magnet.utils import sankey - -""" -!python ~/code/aiq-magnet/dev/poc/inspect_historic_helm_runs.py /data/crfm-helm-public --out_fpath run_specs.yaml --out_detail_fpath run_details.yaml -""" -helm_rows = kwutil.Yaml.load('run_details.yaml') - - -# duplicates = dict(ub.find_duplicates([r['run_spec_name'] for r in helm_rows])) -# for dupname, dupx in duplicates: -# for idx in dupx: -# row = helm_rows[idx] -# print(f'row = {ub.urepr(row, nl=1)}') - - -if 0: - # Debug HelmRunAnalysis - run_dir = '/data/crfm-helm-public/classic/benchmark_output/runs/v0.3.0/wikifact:k=5,subject=symptoms_and_signs,model=lmsys_vicuna-7b-v1.3' - helm_run = HelmRun.coerce(run_dir) - self = HelmRunAnalysis(helm_run) - self.summary(level=10) - run_dir = '/data/crfm-helm-public/classic/benchmark_output/runs/v0.2.4/boolq:model=eleutherai_pythia-2.8b-v0,data_augmentation=canonical/' - helm_run = HelmRun.coerce(run_dir) - self = HelmRunAnalysis(helm_run) - self.summary(level=10) - run_dir = '/data/crfm-helm-public/capabilities/benchmark_output/runs/v1.12.0/ifeval:model=openai_gpt-oss-20b/' - helm_run = HelmRun.coerce(run_dir) - self = HelmRunAnalysis(helm_run) - self.summary(level=10) - - for helm_row in helm_rows: - run_dir = ub.Path(helm_row['run_dir']) - helm_run = HelmRun.coerce(run_dir) - self = HelmRunAnalysis(helm_run) - -finished_jobs = list( - ub.Path('~/code/aiq-magnet/results/helm').expand().glob( - '*/DONE' - ) -) -kwdagger_rows = [] -for fpath in finished_jobs: - config = kwutil.Json.coerce(fpath.parent / 'job_config.json') - run_spec_name = config['helm.run_entry'] - dpath = fpath.parent - runs = HelmOutputs.coerce(dpath / 'benchmark_output').suites()[0].runs() - assert len(runs) == 1 - run = runs[0] - kwdagger_rows.append( - { - 'dpath': dpath, - 'run_spec_name': run_spec_name, - 'run': run, - } - ) -kwdagger_lut = {r['run_spec_name']: r for r in kwdagger_rows} - -kwd_duplicates = dict(ub.find_duplicates([r['run_spec_name'] for r in kwdagger_rows])) -assert not len(kwd_duplicates) - -print(f'len(helm_rows)={len(helm_rows)}') -print(f'len(kwdagger_rows)={len(kwdagger_rows)}') - - -def make_bucket_fn( - edges_desc, - *, - nan_label="unknown (no comparable core)", - endpoints_as_strict_buckets=False, -): - """ - edges_desc: descending bin edges, e.g. [1.0, 0.90, 0.75, 0.50, 0.0] - - endpoints_as_strict_buckets: - - False (default): bins are intervals like "0.9–<1.0" plus a top open bin ">=1.0" - - True: create exact buckets for the endpoints (e.g. "1.0" and "0.0") - """ - edges = list(edges_desc) - assert len(edges) >= 2, "need at least two edges" - assert all(edges[i] >= edges[i + 1] for i in range(len(edges) - 1)), "edges must be descending" - - def _fmt_edge(v: float) -> str: - s = f"{v:.2f}".rstrip("0").rstrip(".") - return s if s else "0" - - top = edges[0] - bot = edges[-1] - - def bucket(x): - import math - if x is None or (isinstance(x, float) and math.isnan(x)): - return nan_label - - # Strict endpoint buckets - if endpoints_as_strict_buckets: - if x == top: - return _fmt_edge(top) - if x == bot: - return _fmt_edge(bot) - - # Top open bucket (captures >top, and also ==top when not strict) - if x >= top: - return f">={_fmt_edge(top)}" - - # Interior interval buckets: [lo, hi) - for i in range(1, len(edges)): - hi = edges[i - 1] - lo = edges[i] - if x >= lo: - return f"{_fmt_edge(lo)}–<{_fmt_edge(hi)}" - - # Below bottom edge - return f"<{_fmt_edge(bot)}" - - return bucket - - -def sankey_stats(rd: HelmRunDiff) -> dict: - """ - Return a small, stable set of fields intended for building Sankey tables. - """ - s = rd.summary_dict(level=1) - va = s.get('value_agreement') or {} - by_class = va.get('by_class') or {} - overall = va.get('overall') or {} - - core = by_class.get('core') or {} - book = by_class.get('bookkeeping') or {} - - core_ratio = core.get('agree_ratio', None) - book_ratio = book.get('agree_ratio', None) - overall_ratio = overall.get('agree_ratio', None) - - spec_ok = bool(s.get('run_spec_dict_ok', False)) - scen_ok = s.get('scenario_ok', None) # True/False/None - - spec_status = 'spec match' if spec_ok else 'spec mismatch' - if scen_ok is None: - scenario_status = 'scenario unknown' - else: - scenario_status = 'scenario match' if scen_ok else 'scenario mismatch' - - # stats_name_status is cheap and very useful as “schema drift” indicator - cov = s.get('stats_coverage_by_name') or {} - stats_name_status = ( - 'stats names match' - if (cov.get('only_a', 0) == 0 and cov.get('only_b', 0) == 0) - else 'stats names mismatch' - ) - - def _bucket_ratio(x: float | None, *, good=0.995, ok=0.95) -> str: - if x is None: - return 'unknown' - if x >= good: - return 'high' - if x >= ok: - return 'medium' - return 'low' - - core_b = _bucket_ratio(core_ratio) - book_b = _bucket_ratio(book_ratio) - - if core_b == 'high' and (book_b in {'high', 'unknown'}): - agreement_quality = 'match' - elif core_b == 'high' and book_b in {'medium', 'low'}: - agreement_quality = 'core match, bookkeeping differs' - elif core_b == 'medium': - agreement_quality = 'core partial' - elif core_b == 'low': - agreement_quality = 'core mismatch' - else: - agreement_quality = 'unknown' - - return { - # orthogonal statuses - 'spec_status': spec_status, - 'scenario_status': scenario_status, - 'stats_name_status': stats_name_status, - 'agreement_quality': agreement_quality, - # numeric signals - 'run_agree_ratio_core': core_ratio, - 'run_agree_ratio_bookkeeping': book_ratio, - 'run_agree_ratio_overall': overall_ratio, - 'comparable_core': core.get('comparable', None), - 'mismatched_core': core.get('mismatched', None), - 'comparable_bookkeeping': book.get('comparable', None), - 'mismatched_bookkeeping': book.get('mismatched', None), - 'comparable_overall': overall.get('comparable', None), - 'mismatched_overall': overall.get('mismatched', None), - } - return row - - -def modified_wormhole_send(fpath): - import subprocess - import selectors - - class NonBlockingPopenIO: - def __init__(self, p: subprocess.Popen, max_bytes=65536): - if p.stdout is None or p.stderr is None: - raise ValueError("Start Popen with stdout=PIPE and stderr=PIPE") - self.p = p - self.max_bytes = max_bytes - self.sel = selectors.DefaultSelector() - self.sel.register(p.stdout, selectors.EVENT_READ, data="stdout") - self.sel.register(p.stderr, selectors.EVENT_READ, data="stderr") - - def drain(self, timeout=0.0): - """ - Read whatever is available right now (bounded), without blocking. - Returns (stdout_bytes, stderr_bytes). - """ - out = b"" - err = b"" - for key, _ in self.sel.select(timeout): - stream = key.fileobj - name = key.data - # read1() avoids blocking for "more"; fallback to read() - reader = getattr(stream, "read1", stream.read) - data = reader(self.max_bytes) - if not data: - continue - if name == "stdout": - out += data - else: - err += data - return out, err - - cmd = ['wormhole', 'send', '--no-qr', str(fpath)] - - p = subprocess.Popen( - cmd, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - bufsize=0, - ) - - import time - time.sleep(1) - - io = NonBlockingPopenIO(p) - - # call this whenever you want (e.g., in your main loop) - out, err = io.drain(timeout=0.0) - if out: - out_text = out.decode("utf-8", errors="replace") - print("STDOUT:", out_text, end="") - if err: - err_text = err.decode("utf-8", errors="replace") - print("STDERR:", err_text, end="") - else: - err_text = '' - - code = [p for p in err_text.split('\n') if p][-1].split(' ')[-1] - - print(ub.codeblock( - f""" - # On the host run: - rm -rf {fpath} - # Run the wormhole command - wormhole recieve {code} --accept-file - eog {fpath} - """ - )) - p.communicate() - - -sankey_rows = [] -rundiffs = [] - -for helm_row in ub.ProgIter(helm_rows, desc='compare runs'): - run_dir = ub.Path(helm_row['run_dir']) - suite_name = run_dir.parent.name - benchmark_name = run_dir.parent.parent.parent.parent.name - assert run_dir.parent.parent.parent.name == 'benchmark_output' - assert run_dir.parent.parent.name == 'runs' - helm_row['suite_name'] = suite_name - helm_row['benchmark_name'] = benchmark_name - run_dir = ub.Path(helm_row['run_dir']) - run_spec_name = helm_row['run_spec_name'] - - kwrow = kwdagger_lut.get(run_spec_name) - helm_row['reproduced_step1'] = kwrow is not None - - # Base row (always emitted, even if not attempted) - row = { - 'run_spec_name': run_spec_name, - 'run_dir': str(run_dir), - 'suite_name': suite_name, - 'benchmark_name': benchmark_name, - 'model_name': helm_row['model'], - # default pipeline fields - 'reproduced_step1': False, - 'attempt_status': None, - 'attempt_error': None, - # default agreement fields - 'agreement_bucket': None, - 'agreement_bucket_instances': None, - 'run_agree_ratio_core': None, - 'run_agree_ratio_bookkeeping': None, - 'run_agree_ratio_overall': None, - 'inst_agree_ratio_unperturbed': None, - 'inst_agree_ratio_perturbed': None, - 'n_instance_mismatched': None, - # signatures (fill opportunistically) - 'sig_run_spec': None, - 'sig_scenario': None, - 'sig_stats_name': None, - } - row['reproduced_step1'] = kwrow is not None - sankey_rows.append(row) - - if kwrow is None: - row['attempt_status'] = 'not attempted' - row['agreement_bucket'] = 'not attempted' - - helm_row['agreement_bucket_base_task'] = 'not attempted' - continue - - # raise Exception - - # Attempt exists: try to compare - row['attempt_status'] = 'compared' - try: - helm_run = HelmRun.coerce(run_dir) - kwdg_run = kwrow['run'] - - a = HelmRunAnalysis(helm_run, name='HELM') - b = HelmRunAnalysis(kwdg_run, name='KWDG') - - # Light signatures: cheap to compute and very useful for debugging. - sa = a.summary_dict(level=0) - sb = b.summary_dict(level=0) - row['sig_run_spec'] = ( - f'{sa["signatures"].get("run_spec_sig")}|{sb["signatures"].get("run_spec_sig")}' - ) - row['sig_scenario'] = ( - f'{sa["signatures"].get("scenario_sig")}|{sb["signatures"].get("scenario_sig")}' - ) - row['sig_stats_name'] = ( - f'{sa["signatures"].get("stats_name_sig")}|{sb["signatures"].get("stats_name_sig")}' - ) - - rd = HelmRunDiff( - run_a=helm_run, run_b=kwdg_run, a_name='HELM', b_name='KWDG' - ) - rundiffs.append(rd) # save for later drilldown - row.update(sankey_stats(rd)) - # Keep the row attached to the rundiff for easy interactive use - rd.row = row - - except Exception as ex: - raise - row['attempt_status'] = 'error' - row['attempt_error'] = repr(ex) - row['agreement_bucket'] = 'error' - - # # raise Exception - - # helm_run = HelmRun.coerce(run_dir) - # kwdg_run = kwrow['run'] - - # a = HelmRunAnalysis(helm_run) - # b = HelmRunAnalysis(kwdg_run) - - # if 0: - # a.summary(level=10) - # b.summary(level=10) - - # rd = HelmRunDiff( - # run_a=helm_run, run_b=kwdg_run, a_name='HELM', b_name='KWDG' - # ) - # self = rd # NOQA - # rd.summary(level=1) - # rd.summarize_instances() - - # if 0: - # table1 = rd.a.joined_instance_stat_table() - # table2 = rd.a.joined_instance_stat_table() - - # instance_id = 'id1237' - # keys1 = table1.variant_keys_for_instance(instance_id) - # keys2 = table2.variant_keys_for_instance(instance_id) - # print(f'keys1 = {ub.urepr(keys1, nl=1)}') - # print(f'keys2 = {ub.urepr(keys2, nl=1)}') - # assert set(keys1) == set(keys2) - # for k1 in keys1: - # table1.rows_by_variant[k1] - # table2.rows_by_variant[k1] - - # print(rd.drilldown_core_metric_instances()) - # rd.lookup_instance(('instance_id', 'id14045'), which='a') - # rd.lookup_instance(('instance_id', 'id14045'), which='b') - - # raise Exception - - # helm_row.update(rd.summary_base_task()) - # helm_row.update(rd.summary_core()) - - # helm_stats = helm_run.json.stats() - # kwdg_stats = kwdg_run.json.stats() - - # # row = compare.compare_run_pair(helm_stats, kwdg_stats, rel_tol=1e-4, abs_tol=1e-8) - # helm_row.update(row) - - -DEVELOPER_DETAILED_DIFF_ANALYSIS = True -if DEVELOPER_DETAILED_DIFF_ANALYSIS: - # import json - # with open('rundiff.jsonl', mode='a', encoding='utf8') as file: - for rd in ub.ProgIter(rundiffs, desc='drill down', verbose=3): - # rd.summary(level=0) - # rd.summary() - a = rd.a - b = rd.b - rd = HelmRunDiff(run_a=a, run_b=b, a_name='HELM', b_name='KWDG') - - summary = rd.summary_dict(level=100) - # json.dumps(summary, ensure_ascii=False) - raise Exception - - # list(kwutil.Json.find_unserializable(summary)) - # summary = kwutil.Json.ensure_serializable(summary) - # file.write(json.dumps(summary, ensure_ascii=False) + "\n") - # file.flush() # optional; good if you want progress written even if interrupted - - core_agreement = summary['value_agreement']['by_class']['core'] - - if 0: - idx = a.stat_index(drop_zero_count=True, require_mean=True) - core_a = pd.DataFrame( - {k: m for k, m in idx.items() if m.metric_class == 'core'}.values() - ) - idx = b.stat_index(drop_zero_count=True, require_mean=True) - core_b = pd.DataFrame( - {k: m for k, m in idx.items() if m.metric_class == 'core'}.values() - ) - spec_a = rd.a.run_spec() - spec_b = rd.b.run_spec() - print(f'spec_a = {ub.urepr(spec_a, nl=3)}') - print(f'spec_b = {ub.urepr(spec_b, nl=3)}') - rd.summarize_instances() - -df = pd.DataFrame(sankey_rows) -df_comp = df[df['attempt_status'] == 'compared'] -print(df.value_counts(['benchmark_name', 'reproduced_step1'])) -print( - df.value_counts( - [ - 'benchmark_name', - 'reproduced_step1', - 'spec_status', - 'agreement_quality', - ] - ) -) -print(df.value_counts(['attempt_status', 'agreement_bucket']).sort_index()) - - -def diagnose_status(row): - if row['attempt_status'] == 'not attempted': - return None - mismatch = set() - print(f'row={row}') - if row['spec_status'].split(' ')[-1] == 'mismatch': - mismatch |= {'run_spec'} - if row['scenario_status'].split(' ')[-1] == 'mismatch': - mismatch |= {'scenario_spec'} - # if row['scenario_status'].split(' ')[-1] == 'mismatch': - # mismatch |= {'stats_name'} - if not mismatch: - return 'specs match' - else: - return 'mismatch: ' + ', '.join(sorted(mismatch)) - -df['spec_diagnostic'] = [diagnose_status(row) for _, row in df.iterrows()] -df['spec_diagnostic'].value_counts() - - -def attempt_status(row: dict[str, object]) -> str: - return ( - 'attempted' if row.get('reproduced_step1', False) else 'not_attempted' - ) - - -def attempt_label(row: dict[str, object]) -> str: - # We already computed this in the table builder - return str(row.get('attempt_status', 'unknown')) - - -def agreement_label(row: dict[str, object]) -> str: - # Used in the sankey plan; keep it stable. - return row.get('agreement_bucket_base_task', 'unknown') - - -CORE_IOU_BINS = [ - 1.0, - # 0.90, - 0.75, - 0.50, - 0.25, - 0.00 -] -df['core_iou'] = (df['comparable_core'] - df['mismatched_core']) / df[ - 'comparable_core' -] -core_iou_bucket = make_bucket_fn(CORE_IOU_BINS, endpoints_as_strict_buckets=True, nan_label='no comparable metrics') -df['core_iou_bucket'] = df['core_iou'].map(core_iou_bucket) -print(df['core_iou_bucket'].value_counts()) - -df['spec_status'].value_counts() -df['scenario_status'].value_counts() -df['stats_name_status'].value_counts() - -# Sankey plan: same skeleton, but add a core IoU bucket stage - -# root = sankey.Root('All Attempts') -# # When we get the data if one of the names isn't available, we handle it by -# # dynamically adding it, but here we can use the result object to specify -# # cases. -# splits = root.split(by='attempt_status') -# compared = splits.add_case(value='compared') -# failcase = splits.add_case(value='not attempted') -# failcase.set_label('Failed') - -# bench_group = compared.group(by='benchmark_name', name='benchmark') -# bench_group.group(by='core_iou_bucket') - - -# # Note it should alway be possible to put "benchmarks" before attempt status like: -# root = sankey.Root('All Attempts') -# bench_group = root.group(by='benchmark_name', name='benchmark') -# splits = bench_group.split(by='attempt_status') -# splits['not attempted'].set_label('Failed') -# compared = splits.cases['compared'] # behaves like a defaultdict - -# compared.group(by='spec_diagnostic') -# compared.group(by='core_iou_bucket') - - -# --- -# Does this make sense? - - -from magnet.utils import sankey_builder -root = sankey_builder.Root() -bench_groups = root.group(by='benchmark_name') - -rungroup = bench_groups.group(by='attempt_status') -compared_node = rungroup['compared'] -compared_node.label = 'Run' -unrun_node = rungroup['not attempted'] -unrun_node.label = 'Failed' - -compared_node \ - .group(by='core_iou_bucket') - # .group(by='spec_diagnostic') \ - -# plan = sankey.Plan( -# sankey.Root(f'Attempted Runs n={len(df)}'), -# sankey.Split('Runs', 'attempt_status', branches={ -# 'compared': sankey.Plan( -# sankey.Group('benchmark', by='benchmark_name'), -# sankey.Bucket('core_iou', by='core_iou_bucket'), -# ), -# 'not attempted': sankey.Node('Failed') # I want any that meet this condition to be sent to a node called failed. -# }) -# ) - -print(root.to_text()) - -G = root.build_sankey(df.to_dict('records'), label_fmt='{value}') -print(G.summarize(max_edges=150)) - -G.nodes['CONST']['label'] = f'Attempted Runs n={len(df)}' -# fig = G.to_plotly(title='HELM Reproduction Funnel') - - -import plotly.graph_objects as go -node_labels, source, target, value = G._to_sankey_data() -sankey = go.Sankey( - # arrangement='freeform', - node=dict(label=node_labels, pad=15, thickness=18), - link=dict(source=source, target=target, value=value), -) -fig = go.Figure(sankey) -title = 'Title' -fig.update_layout(title_text=title, font_size=14) - - -fpath = 'helm_repro_sankey.jpg' -fig.write_image(fpath, scale=4.0) -import kwplot -kwplot.cropwhite_ondisk(fpath) -print(f'Wrote helm_repro_sankey: {fpath}') -modified_wormhole_send(fpath) - -# --- Per-benchmark drilldown sankeys (deeper, but still run-level) --- -# Here we drill by model within each benchmark (only if model_name exists). -# If model_name is missing, you can swap this to group by run_spec_name prefix, etc. -bench_groups = ub.group_items( - sankey_rows, key=lambda r: r.get('benchmark_name', 'unknown') -) - -out_dpath = ub.Path('benchmark_sankeys').ensuredir() -for bench, rows in bench_groups.items(): - if bench in {None, 'unknown'}: - continue - - # Skip tiny groups if you want: - # if len(rows) < 5: continue - - plan_bench = sankey.Plan( - sankey.Root(f'{bench}'), - sankey.Group('model', by='model_name'), - sankey.Bucket('spec', by='spec_status'), - sankey.Bucket('agreement', by='agreement_quality'), - ) - - Gb = plan_bench.build_sankey(rows, label_fmt='{name}: {value}') - figb = Gb.to_plotly(title=f'HELM Repro Funnel: {bench}') - - # sanitize bench for filename - bench_slug = ub.Path(str(bench)).name.replace('/', '_') - fpath_b = out_dpath / f'{bench_slug}.jpg' - figb.write_image(str(fpath_b)) - print(f'Wrote benchmark sankey: {fpath_b}') - -if 0: - print( - ub.codeblock( - f""" - # On Host - rm -rf {out_dpath} - # Run the wormhole command - """ - ) - ) - ub.cmd(f'wormhole send {out_dpath}', verbose=3) diff --git a/dev/oneoff/diagnose_reproducibility.py b/dev/oneoff/diagnose_reproducibility.py deleted file mode 100644 index aea7af2..0000000 --- a/dev/oneoff/diagnose_reproducibility.py +++ /dev/null @@ -1,1080 +0,0 @@ -""" -Script-style reproducibility diagnostics for HELM vs KWDG runs. - -This is intentionally notebook-like and hard-coded for local iteration. -It writes a case-by-case JSONL stream as it runs, plus a summary JSON report. -""" - -from __future__ import annotations - -import datetime as datetime_mod -import json -from collections import Counter, defaultdict -from typing import Any - -import kwutil -import ubelt as ub - -from magnet.backends.helm.helm_outputs import HelmOutputs -from magnet.backends.helm.helm_outputs import HelmRun -from magnet.backends.helm.helm_run_diff import HelmRunDiff -from magnet.utils import sankey_builder - - -HELM_DETAILS_FPATH = ub.Path('run_details.yaml') -# Use repository-local results by default to avoid HOME-dependent ambiguity. -KWDG_RESULTS_DPATH = (ub.Path.cwd() / 'results/helm').resolve() -REPORT_DPATH = ub.Path('dev/oneoff/repro_reports').ensuredir() - - -def parse_helm_version(version_text: str) -> tuple[int, ...]: - """ - Parse version strings like "v0.3.0" into sortable tuples. - """ - text = str(version_text).strip() - if text.startswith('v'): - text = text[1:] - parts = [] - for tok in text.split('.'): - if tok.isdigit(): - parts.append(int(tok)) - else: - parts.append(0) - return tuple(parts) - - -def parse_helm_run_dir(run_dir: str) -> dict[str, str]: - """Parse HELM public run_dir path components.""" - p = ub.Path(run_dir) - parts = list(p.parts) - out = { - 'helm_suite_name': 'unknown', - 'helm_version': 'unknown', - 'run_leaf': p.name, - } - try: - idx = parts.index('benchmark_output') - except ValueError: - idx = -1 - if idx >= 1: - out['helm_suite_name'] = str(parts[idx - 1]) - if idx >= 0 and (idx + 2) < len(parts): - # .../benchmark_output/runs// - out['helm_version'] = str(parts[idx + 2]) - else: - out['helm_version'] = str(p.parent.name) - return out - - -def infer_benchmark_group( - run_spec_name: str | None, - scenario_class: str | None = None, -) -> str: - """Infer benchmark/scenario family key (e.g. babi_qa, mmlu, raft).""" - text = (run_spec_name or '').strip() - if text: - idxs = [i for i in [text.find(':'), text.find(',')] if i >= 0] - if idxs: - head = text[: min(idxs)].strip() - else: - head = text - if head: - return head - if scenario_class: - base = str(scenario_class).split('.')[-1] - if base.endswith('Scenario'): - base = base[:-8] - if base: - return base - return 'unknown' - - -def select_latest_helm_rows(helm_rows: list[dict[str, Any]]) -> list[dict[str, Any]]: - by_name = defaultdict(list) - for row in helm_rows: - run_dir = ub.Path(row['run_dir']) - parsed = parse_helm_run_dir(str(run_dir)) - version = parsed['helm_version'] - row = dict(row) - row['helm_version'] = version - row['helm_version_tuple'] = parse_helm_version(version) - row['suite_name'] = parsed['helm_suite_name'] - row['benchmark_name'] = parsed['helm_suite_name'] - row['benchmark_group'] = infer_benchmark_group( - row.get('run_spec_name', None), - row.get('scenario_class', None), - ) - by_name[row['run_spec_name']].append(row) - - latest_rows = [] - for _, items in by_name.items(): - best = max( - items, - key=lambda r: ( - r['helm_version_tuple'], - str(r['run_dir']), - ), - ) - latest_rows.append(best) - - latest_rows = sorted( - latest_rows, - key=lambda r: (r.get('benchmark_name', ''), r.get('run_spec_name', '')), - ) - return latest_rows - - -def load_kwdg_rows(results_dpath: ub.Path) -> tuple[list[dict[str, Any]], dict[str, dict[str, Any]]]: - finished_jobs = sorted(results_dpath.glob('*/DONE')) - rows = [] - for fpath in ub.ProgIter(finished_jobs, desc='load kwdg runs'): - dpath = fpath.parent - try: - config = kwutil.Json.coerce(dpath / 'job_config.json') - run_spec_name = config['helm.run_entry'] - suites = HelmOutputs.coerce(dpath / 'benchmark_output').suites() - if not suites: - continue - runs = suites[0].runs() - if len(runs) != 1: - continue - run = runs[0] - rows.append( - { - 'dpath': str(dpath), - 'run_spec_name': run_spec_name, - 'run': run, - } - ) - except Exception: - continue - - lut = {} - dups = defaultdict(list) - for row in rows: - name = row['run_spec_name'] - if name in lut: - dups[name].append(row['dpath']) - lut[name] = row - if dups: - print(f'WARNING: found {len(dups)} duplicate KWDG run_spec_name entries') - return rows, lut - - -def aggregate_report(rows: list[dict[str, Any]]) -> dict[str, Any]: - status_counter = Counter() - diagnosis_counter = Counter() - reason_counter = Counter() - reason_by_priority = defaultdict(Counter) - reason_detail_counter = defaultdict(Counter) - inferred_causal_counter = Counter() - deployment_transition_counter = Counter() - inferred_causal_examples: list[dict[str, Any]] = [] - - def _reason_lut(diag: dict[str, Any]) -> dict[str, dict[str, Any]]: - out = {} - for reason in diag.get('reasons', []) or []: - if not isinstance(reason, dict): - continue - name = reason.get('name', None) - if name is None: - continue - out[str(name)] = reason - return out - - def _summarize_eval_detail(details: dict[str, Any]) -> str: - delta = details.get('metric_specs_multiset_delta', {}) or {} - if not isinstance(delta, dict): - return 'metric_specs_multiset_delta=unknown' - n_added = delta.get('n_added', None) - n_removed = delta.get('n_removed', None) - added_structured = delta.get('added_structured', []) or [] - removed_structured = delta.get('removed_structured', []) or [] - added_classes = sorted( - { - str(x.get('class_name', '?')) - for x in added_structured - if isinstance(x, dict) - } - ) - removed_classes = sorted( - { - str(x.get('class_name', '?')) - for x in removed_structured - if isinstance(x, dict) - } - ) - added_s = ','.join(added_classes[:3]) if added_classes else '-' - removed_s = ','.join(removed_classes[:3]) if removed_classes else '-' - return ( - f'n_added={n_added},n_removed={n_removed},' - f'added_classes={added_s},removed_classes={removed_s}' - ) - - for row in rows: - status = row.get('status', 'unknown') - status_counter[status] += 1 - if status != 'compared': - continue - - diag = row.get('diagnosis', {}) or {} - diagnosis_counter[diag.get('label', 'unknown')] += 1 - - for reason in diag.get('reasons', []) or []: - name = reason.get('name', 'unknown') - priority = reason.get('priority', None) - reason_counter[name] += 1 - reason_by_priority[str(priority)][name] += 1 - - reason_lut = _reason_lut(diag) - dep = reason_lut.get('deployment_drift', None) - eval_spec = reason_lut.get('evaluation_spec_drift', None) - dep_transition = None - - if dep is not None: - dep_details = dep.get('details', {}) or {} - a_val = dep_details.get('a_value', None) - b_val = dep_details.get('b_value', None) - dep_transition = f'{a_val!r} -> {b_val!r}' - deployment_transition_counter[dep_transition] += 1 - reason_detail_counter['deployment_drift'][dep_transition] += 1 - - if eval_spec is not None: - eval_details = eval_spec.get('details', {}) or {} - eval_summary = _summarize_eval_detail(eval_details) - reason_detail_counter['evaluation_spec_drift'][eval_summary] += 1 - - if dep is not None and eval_spec is not None: - dep_p = dep.get('priority', None) - eval_p = eval_spec.get('priority', None) - if ( - isinstance(dep_p, int) - and isinstance(eval_p, int) - and dep_p <= eval_p - ): - inferred_causal_counter['deployment_precedes_eval_spec_drift'] += 1 - if dep_transition is not None: - inferred_causal_counter[ - f'deployment_transition::{dep_transition}' - ] += 1 - if len(inferred_causal_examples) < 20: - inferred_causal_examples.append( - { - 'run_spec_name': row.get('run_spec_name', None), - 'deployment_transition': dep_transition, - 'deployment_priority': dep_p, - 'eval_priority': eval_p, - } - ) - - out = { - 'n_rows': len(rows), - 'status_counts': dict(status_counter), - 'diagnosis_label_counts': dict(diagnosis_counter), - 'reason_counts': dict(reason_counter), - 'reason_counts_by_priority': { - p: dict(c) for p, c in reason_by_priority.items() - }, - 'reason_detail_counts': { - name: dict(counter.most_common(20)) - for name, counter in reason_detail_counter.items() - }, - 'deployment_transition_counts': dict( - deployment_transition_counter.most_common(20) - ), - 'inferred_causal_counts': dict(inferred_causal_counter), - 'inferred_causal_examples': inferred_causal_examples, - } - return out - - -def _normalize_primary_reasons(diag: dict[str, Any]) -> list[str]: - raw = diag.get('primary_reason_names', []) or [] - if isinstance(raw, str): - return [raw] - out = [str(x) for x in raw if x is not None] - return sorted(out) - - -def _all_reason_names(diag: dict[str, Any]) -> set[str]: - reasons = diag.get('reasons', []) or [] - out = set() - for r in reasons: - if not isinstance(r, dict): - continue - name = r.get('name', None) - if name is not None: - out.add(str(name)) - # Also include primary reasons for compatibility with abbreviated records. - for name in _normalize_primary_reasons(diag): - out.add(str(name)) - return out - - -def _bucket_execution_state(reason_names: set[str]) -> str: - has_dep = 'deployment_drift' in reason_names - has_exec = 'execution_spec_drift' in reason_names - if has_dep and has_exec: - return 'deployment+execution' - if has_dep: - return 'deployment_only' - if has_exec: - return 'execution_only' - return 'none' - - -def _bucket_dataset_state(reason_names: set[str]) -> str: - has_error = 'dataset_overlap_error' in reason_names - has_membership = bool( - {'dataset_instance_drift', 'dataset_variant_drift'} & reason_names - ) - has_input_prompt = bool( - {'dataset_input_drift', 'request_prompt_drift'} & reason_names - ) - if has_error: - return 'error' - if has_membership and has_input_prompt: - return 'membership+input_prompt' - if has_membership: - return 'membership_only' - if has_input_prompt: - return 'input_prompt_only' - return 'none' - - -def _bucket_eval_state(reason_names: set[str]) -> str: - has_eval = 'evaluation_spec_drift' in reason_names - has_completion = 'completion_content_drift' in reason_names - if has_eval and has_completion: - return 'eval_spec+completion' - if has_eval: - return 'eval_spec_only' - if has_completion: - return 'completion_only' - return 'none' - - -def _bucket_core_state(reason_names: set[str]) -> str: - if 'core_metric_drift' in reason_names: - return 'core_drift' - if 'no_comparable_core_metrics' in reason_names: - return 'no_comparable_core' - if 'bookkeeping_metric_drift' in reason_names: - return 'bookkeeping_only' - return 'core_match' - - -def _benchmark_or_suite(row: dict[str, Any]) -> str: - group = row.get('benchmark_group', None) - bench = row.get('benchmark_name', None) - suite = row.get('suite_name', None) - for cand in [group, bench, suite]: - if cand is None: - continue - text = str(cand).strip() - if text and text.lower() != 'unknown': - return text - return 'unknown' - - -def _performance_bucket(row: dict[str, Any]) -> str: - """Bucket core metric agreement for plot readability.""" - va = row.get('value_agreement', {}) or {} - core = ((va.get('by_class') or {}).get('core') or {}) - agree = core.get('agree_ratio', None) - if agree is None: - return 'n/a' - try: - x = float(agree) - except Exception: - return 'n/a' - if x <= 0.0: - return '0%' - if x < 0.50: - return '0-50%' - if x < 0.75: - return '50-75%' - if x < 1.0: - # Not requested explicitly, but needed to avoid hiding these cases. - return '75-100%' - return '100%' - - -def _varying_keys(rows: list[dict[str, Any]], keys: list[str]) -> list[str]: - """Return keys that vary across rows (ignoring stages with 1 unique value).""" - out = [] - for key in keys: - vals = {r.get(key, None) for r in rows} - if len(vals) > 1: - out.append(key) - return out - - -def build_sankey_rows(case_rows: list[dict[str, Any]]) -> list[dict[str, Any]]: - """ - Build normalized row records for diagnosis Sankey construction. - - Example: - >>> rows = [ - ... {'status': 'compared', 'diagnosis': {'label': 'reproduced', 'primary_priority': 0, 'primary_reason_names': ['no_detected_drift']}}, - ... {'status': 'compared', 'diagnosis': {'label': 'multiple_primary_reasons', 'primary_priority': 0, 'primary_reason_names': ['deployment_drift', 'execution_spec_drift']}}, - ... {'status': 'missing_kwdg_match', 'diagnosis': {'label': 'missing_kwdg_match'}}, - ... ] - >>> sk = build_sankey_rows(rows) - >>> assert len(sk) == 3 - >>> assert sk[0]['repro_outcome'] == 'reproduced_core' - >>> assert sk[1]['execution_state'] == 'deployment+execution' - >>> assert sk[2]['repro_outcome'] == 'not_compared' - >>> assert sk[0]['benchmark_or_suite'] == 'unknown' - >>> assert sk[0]['performance_bucket'] == 'n/a' - """ - sink_rows = [] - for row in case_rows: - status = str(row.get('status', 'unknown')) - diag = row.get('diagnosis', {}) or {} - label = str(diag.get('label', 'unknown')) - reason_names = _all_reason_names(diag) - reasons = _normalize_primary_reasons(diag) - if reasons: - reason_bucket = ' + '.join(reasons) - else: - reason_bucket = label - - p = diag.get('primary_priority', None) - if isinstance(p, int): - priority_bucket = f'P{p}' - else: - priority_bucket = 'P?' - - core_state = _bucket_core_state(reason_names) - execution_state = _bucket_execution_state(reason_names) - dataset_state = _bucket_dataset_state(reason_names) - eval_state = _bucket_eval_state(reason_names) - perf_bucket = _performance_bucket(row) - benchmark_or_suite = _benchmark_or_suite(row) - - if status != 'compared': - repro_outcome = 'not_compared' - # keep non-compared flow compact in the sankey. - priority_bucket = 'n/a' - reason_bucket = label - core_state = 'n/a' - execution_state = 'n/a' - dataset_state = 'n/a' - eval_state = 'n/a' - perf_bucket = 'n/a' - else: - if core_state in {'core_match', 'bookkeeping_only'}: - repro_outcome = 'reproduced_core' - elif core_state == 'core_drift': - repro_outcome = 'non_reproduced_core' - else: - repro_outcome = 'unknown_core' - - sink_rows.append( - { - 'status': status, - 'benchmark_or_suite': benchmark_or_suite, - 'repro_outcome': repro_outcome, - 'performance_bucket': perf_bucket, - 'core_state': core_state, - 'execution_state': execution_state, - 'dataset_state': dataset_state, - 'eval_state': eval_state, - 'diagnosis_label': label, - 'primary_priority': priority_bucket, - 'primary_reasons': reason_bucket, - } - ) - return sink_rows - - -def write_sankey_report( - case_rows: list[dict[str, Any]], - *, - report_dpath: ub.Path, - stamp: str, -) -> dict[str, Any]: - """Build and write diagnosis Sankey artifacts.""" - sankey_rows = build_sankey_rows(case_rows) - plotly_errors: list[str] = [] - stage_defs: dict[str, list[str]] = { - 'status': [ - 'compared: HELM and KWDG pair was compared.', - 'missing_kwdg_match: no matching KWDG run for this HELM run_spec.', - 'error: comparison failed.', - ], - 'bench': [ - 'inferred scenario family from HELM run_spec_name (e.g. math, mmlu, raft).', - ], - 'core%': [ - 'core metric agreement bucket (value_agreement.by_class.core.agree_ratio):', - '0% -> agree_ratio == 0.0', - '0-50% -> 0.0 < agree_ratio < 0.5', - '50-75% -> 0.5 <= agree_ratio < 0.75', - '75-100% -> 0.75 <= agree_ratio < 1.0', - '100% -> agree_ratio == 1.0', - 'n/a -> no comparable core metrics', - ], - 'exec': [ - 'execution-level run-spec drift summary.', - 'deployment+execution: deployment_drift + execution_spec_drift', - 'deployment_only: deployment_drift only', - 'execution_only: execution_spec_drift only', - 'none: no execution/deployment drift detected', - ], - 'data': [ - 'dataset/request drift summary.', - 'none: no detected dataset membership or input/prompt drift', - 'input_prompt_only: dataset_input_drift and/or request_prompt_drift', - 'membership_only: dataset_instance_drift and/or dataset_variant_drift', - 'membership+input_prompt: both membership and input/prompt drift', - 'error: dataset overlap computation failed', - ], - 'eval': [ - 'evaluation/content drift summary.', - 'none: no evaluation schema/content drift detected', - 'eval_spec_only: evaluation_spec_drift only', - 'completion_only: completion_content_drift only', - 'eval_spec+completion: both evaluation_spec_drift and completion_content_drift', - ], - 'primary': [ - 'concatenated primary_reason_names from diagnosis.', - 'Primary means lowest priority value (most upstream stage).', - ], - 'core_outcome': [ - 'high-level core reproducibility outcome.', - 'reproduced_core / non_reproduced_core / unknown_core', - ], - 'core_state': [ - 'core-metric reason bucket from diagnosis reasons.', - 'core_match / core_drift / bookkeeping_only / no_comparable_core', - ], - 'diag': [ - 'diagnosis.label (top-level diagnosis label).', - ], - 'prio': [ - 'diagnosis.primary_priority (0 is most upstream/significant).', - ], - } - - def _graph_key_text(title: str, stage_names: list[str]) -> str: - lines: list[str] = [] - lines.append('Sankey Key') - lines.append('----------') - lines.append(f'Graph: {title}') - lines.append('Stage order: ' + ' -> '.join(stage_names)) - lines.append('') - for stage in stage_names: - lines.append(f'{stage}:') - defs = stage_defs.get(stage, ['(no definition available)']) - for d in defs: - lines.append(f' {d}') - lines.append('') - return '\n'.join(lines).rstrip() + '\n' - - def _emit_graph( - *, - kind: str, - title: str, - rows: list[dict[str, Any]], - root, - stage_names: list[str], - ) -> dict[str, Any]: - graph = root.build_sankey(rows, label_fmt='{name}: {value}') - graph_summary = graph.summarize(max_edges=300) - plan_text = root.to_text() - - stem = report_dpath / f'diagnose_repro_sankey_{stamp}_{kind}' - json_fpath = stem.augment(ext='.json') - txt_fpath = stem.augment(ext='.txt') - key_fpath = stem.augment(stemsuffix='_key', ext='.txt') - html_fpath = stem.augment(ext='.html') - png_fpath = stem.augment(ext='.png') - jpg_fpath = stem.augment(ext='.jpg') - - node_labels, source, target, value = graph._to_sankey_data() - payload = kwutil.Json.ensure_serializable( - { - 'kind': kind, - 'title': title, - 'n_rows': len(rows), - 'rows': rows, - 'node_labels': node_labels, - 'source': source, - 'target': target, - 'value': value, - } - ) - json_fpath.write_text(json.dumps(payload, indent=2, ensure_ascii=False)) - txt_fpath.write_text(plan_text + '\n\n' + graph_summary + '\n') - key_fpath.write_text(_graph_key_text(title, stage_names)) - - out = { - 'json': str(json_fpath), - 'txt': str(txt_fpath), - 'key_txt': str(key_fpath), - 'html': None, - 'png': None, - 'jpg': None, - } - try: - fig = graph.to_plotly(title=title) - fig.write_html(str(html_fpath), include_plotlyjs='cdn') - out['html'] = str(html_fpath) - try: - # Keep interactive HTML defaults; apply readability tuning only - # to static JPG exports. - import plotly.graph_objects as go - - fig_static = go.Figure(fig.to_dict()) - node_labels = payload.get('node_labels', []) - max_label_len = max((len(str(x)) for x in node_labels), default=20) - export_width = min(5200, max(2800, 1700 + max_label_len * 18)) - export_height = min(5200, max(2200, 1000 + len(node_labels) * 50)) - export_scale = 2.25 - fig_static.update_traces( - node=dict(pad=20, thickness=20), - ) - fig_static.update_layout( - font_size=13, - width=export_width, - height=export_height, - margin=dict(l=30, r=30, t=70, b=30), - paper_bgcolor='white', - plot_bgcolor='white', - ) - fig_static.write_image(str(jpg_fpath), scale=export_scale) - out['jpg'] = str(jpg_fpath) - except Exception as ex: - plotly_errors.append( - f'[{kind}] unable to write sankey JPG: {ex!r}' - ) - except Exception as ex: - plotly_errors.append( - f'[{kind}] unable to write sankey HTML/images: {ex!r}' - ) - return out - - # Level 1: all attempts - root_all = sankey_builder.Root(label=f'Run Pairs n={len(sankey_rows)}') - status_group = root_all.group(by='status', name='status') - compared = status_group['compared'] - compared.label = 'status: compared' - status_group['missing_kwdg_match'].label = 'status: missing_kwdg_match' - status_group['error'].label = 'status: error' - status_group['unknown'].label = 'status: unknown' - - compared_rows = [r for r in sankey_rows if r.get('status') == 'compared'] - node = compared - key_to_name = { - 'benchmark_or_suite': 'bench', - 'performance_bucket': 'core%', - 'execution_state': 'exec', - 'dataset_state': 'data', - 'eval_state': 'eval', - 'primary_reasons': 'primary', - } - stage_order = [ - 'benchmark_or_suite', - 'performance_bucket', - 'execution_state', - 'dataset_state', - 'eval_state', - 'primary_reasons', - ] - selected_stage_keys = _varying_keys(compared_rows, stage_order) - for key in selected_stage_keys: - node = node.group(by=key, name=key_to_name[key]) - all_stage_names = ['status'] + [key_to_name[k] for k in selected_stage_keys] - - all_art = _emit_graph( - kind='all_attempts', - title='HELM/KWDG Reproducibility Diagnosis (All Attempts)', - rows=sankey_rows, - root=root_all, - stage_names=all_stage_names, - ) - - # Level 2: compared-only detailed - if compared_rows: - root_comp = sankey_builder.Root( - label=f'Compared Pairs n={len(compared_rows)}' - ) - node2 = root_comp - stage_order2 = [ - 'benchmark_or_suite', - 'performance_bucket', - 'execution_state', - 'dataset_state', - 'eval_state', - 'primary_reasons', - ] - selected_stage_keys2 = _varying_keys(compared_rows, stage_order2) - for key in selected_stage_keys2: - node2 = node2.group(by=key, name=key_to_name[key]) - compared_stage_names = [key_to_name[k] for k in selected_stage_keys2] - compared_art = _emit_graph( - kind='compared_detail', - title='HELM/KWDG Reproducibility Diagnosis (Compared Only)', - rows=compared_rows, - root=root_comp, - stage_names=compared_stage_names, - ) - else: - compared_art = { - 'json': None, - 'txt': None, - 'key_txt': None, - 'html': None, - 'png': None, - 'jpg': None, - } - - artifacts: dict[str, Any] = { - # Backward-compatible keys point to the all-attempts sankey. - 'sankey_json': all_art['json'], - 'sankey_txt': all_art['txt'], - 'sankey_key_txt': all_art['key_txt'], - 'sankey_html': all_art['html'], - 'sankey_png': all_art['png'], - 'sankey_jpg': all_art['jpg'], - # Additional detailed level. - 'sankey_compared_json': compared_art['json'], - 'sankey_compared_txt': compared_art['txt'], - 'sankey_compared_key_txt': compared_art['key_txt'], - 'sankey_compared_html': compared_art['html'], - 'sankey_compared_png': compared_art['png'], - 'sankey_compared_jpg': compared_art['jpg'], - 'sankey_compared_full_json': None, - 'sankey_compared_full_txt': None, - 'sankey_compared_full_key_txt': None, - 'sankey_compared_full_html': None, - 'sankey_compared_full_png': None, - 'sankey_compared_full_jpg': None, - 'plotly_error': (' | '.join(plotly_errors) if plotly_errors else None), - } - - # Level 3: compared-only, full (unpruned) diagnostic pipeline. - if compared_rows: - root_comp_full = sankey_builder.Root( - label=f'Compared Pairs (Full) n={len(compared_rows)}' - ) - node3 = root_comp_full - key_to_name_full = { - 'benchmark_or_suite': 'bench', - 'repro_outcome': 'core_outcome', - 'performance_bucket': 'core%', - 'core_state': 'core_state', - 'execution_state': 'exec', - 'dataset_state': 'data', - 'diagnosis_label': 'diag', - 'eval_state': 'eval', - 'primary_priority': 'prio', - 'primary_reasons': 'primary', - } - stage_order3 = [ - 'benchmark_or_suite', - 'repro_outcome', - 'performance_bucket', - 'core_state', - 'execution_state', - 'dataset_state', - 'diagnosis_label', - 'eval_state', - 'primary_priority', - 'primary_reasons', - ] - for key in stage_order3: - node3 = node3.group(by=key, name=key_to_name_full[key]) - compared_full_stage_names = [key_to_name_full[k] for k in stage_order3] - compared_full_art = _emit_graph( - kind='compared_full', - title='HELM/KWDG Reproducibility Diagnosis (Compared Full Pipeline)', - rows=compared_rows, - root=root_comp_full, - stage_names=compared_full_stage_names, - ) - artifacts['sankey_compared_full_json'] = compared_full_art['json'] - artifacts['sankey_compared_full_txt'] = compared_full_art['txt'] - artifacts['sankey_compared_full_key_txt'] = compared_full_art['key_txt'] - artifacts['sankey_compared_full_html'] = compared_full_art['html'] - artifacts['sankey_compared_full_png'] = compared_full_art['png'] - artifacts['sankey_compared_full_jpg'] = compared_full_art['jpg'] - - return kwutil.Json.ensure_serializable(artifacts) - - -def main(): - if not HELM_DETAILS_FPATH.exists(): - raise FileNotFoundError( - f'Expected HELM detail file at {HELM_DETAILS_FPATH}' - ) - - helm_rows = kwutil.Yaml.load(HELM_DETAILS_FPATH) - latest_helm_rows = select_latest_helm_rows(helm_rows) - - if not latest_helm_rows: - raise RuntimeError('No HELM rows found') - - kwdg_rows, kwdg_lut = load_kwdg_rows(KWDG_RESULTS_DPATH) - - print(f'Loaded HELM rows: all={len(helm_rows)} latest_only={len(latest_helm_rows)}') - print(f'Loaded KWDG rows: {len(kwdg_rows)}') - - stamp = datetime_mod.datetime.now(datetime_mod.UTC).strftime('%Y%m%dT%H%M%SZ') - case_jsonl_fpath = REPORT_DPATH / f'diagnose_repro_cases_{stamp}.jsonl' - summary_json_fpath = REPORT_DPATH / f'diagnose_repro_summary_{stamp}.json' - - all_case_rows = [] - with case_jsonl_fpath.open('w', encoding='utf8') as file: - for idx, helm_row in enumerate( - ub.ProgIter(latest_helm_rows, desc='compare latest helm vs kwdg'), start=1 - ): - run_spec_name = helm_row['run_spec_name'] - kwrow = kwdg_lut.get(run_spec_name, None) - case_row = { - 'index': idx, - 'run_spec_name': run_spec_name, - 'benchmark_name': helm_row.get('benchmark_name', 'unknown'), - 'benchmark_group': helm_row.get('benchmark_group', 'unknown'), - 'suite_name': helm_row.get('suite_name', 'unknown'), - 'model_name': helm_row.get('model', None), - 'helm_version': helm_row.get('helm_version', None), - 'helm_run_dir': str(helm_row['run_dir']), - 'kwdg_run_dir': None if kwrow is None else kwrow['dpath'], - } - - if kwrow is None: - case_row.update( - { - 'status': 'missing_kwdg_match', - 'diagnosis': { - 'label': 'missing_kwdg_match', - 'primary_priority': 0, - 'primary_reason_names': ['missing_kwdg_match'], - 'reasons': [ - { - 'name': 'missing_kwdg_match', - 'priority': 0, - 'details': {}, - } - ], - }, - } - ) - print( - f'[{idx:03d}] {run_spec_name} -> missing_kwdg_match' - ) - else: - try: - helm_run = HelmRun.coerce(helm_row['run_dir']) - kwdg_run = kwrow['run'] - rd = HelmRunDiff( - run_a=helm_run, - run_b=kwdg_run, - a_name='HELM', - b_name='KWDG', - ) - summary = rd.summary_dict(level=20) - diag = summary.get('diagnosis', {}) or {} - - case_row.update( - { - 'status': 'compared', - 'diagnosis': diag, - 'run_spec_semantic': summary.get( - 'run_spec_semantic', None - ), - 'scenario_semantic': summary.get( - 'scenario_semantic', None - ), - 'dataset_overlap': summary.get( - 'dataset_overlap', None - ), - 'stats_coverage_by_name': summary.get( - 'stats_coverage_by_name', None - ), - 'stats_coverage_by_name_count': summary.get( - 'stats_coverage_by_name_count', None - ), - 'value_agreement': summary.get( - 'value_agreement', None - ), - 'instance_value_agreement': summary.get( - 'instance_value_agreement', None - ), - } - ) - primary = diag.get('label', 'unknown') - p = diag.get('primary_priority', None) - primary_names = diag.get('primary_reason_names', []) or [] - print( - f'[{idx:03d}] {run_spec_name} -> {primary} ' - f'(p={p}, primary_reasons={primary_names})' - ) - reasons = diag.get('reasons', []) or [] - for reason in reasons: - if reason.get('name') == 'deployment_drift': - det = reason.get('details', {}) or {} - print( - ' deployment: ' - f'{det.get("a_value", None)!r} -> {det.get("b_value", None)!r}' - ) - except Exception as ex: - case_row.update( - { - 'status': 'error', - 'error': repr(ex), - 'diagnosis': { - 'label': 'comparison_error', - 'primary_priority': 0, - 'primary_reason_names': ['comparison_error'], - 'reasons': [ - { - 'name': 'comparison_error', - 'priority': 0, - 'details': {'error': repr(ex)}, - } - ], - }, - } - ) - print(f'[{idx:03d}] {run_spec_name} -> ERROR: {ex!r}') - - case_row = kwutil.Json.ensure_serializable(case_row) - file.write(json.dumps(case_row, ensure_ascii=False) + '\n') - file.flush() - all_case_rows.append(case_row) - - summary_report = { - 'report_case_jsonl': str(case_jsonl_fpath), - 'report_summary_json': str(summary_json_fpath), - 'generated_utc': stamp, - 'inputs': { - 'helm_detail_fpath': str(HELM_DETAILS_FPATH), - 'kwdg_results_dpath': str(KWDG_RESULTS_DPATH), - 'n_helm_rows_all': len(helm_rows), - 'n_helm_rows_latest': len(latest_helm_rows), - 'n_kwdg_rows': len(kwdg_rows), - }, - 'aggregate': aggregate_report(all_case_rows), - } - try: - sankey_artifacts = write_sankey_report( - all_case_rows, report_dpath=REPORT_DPATH, stamp=stamp - ) - except Exception as ex: - sankey_artifacts = { - 'sankey_json': None, - 'sankey_txt': None, - 'sankey_key_txt': None, - 'sankey_html': None, - 'sankey_png': None, - 'sankey_jpg': None, - 'sankey_compared_json': None, - 'sankey_compared_txt': None, - 'sankey_compared_key_txt': None, - 'sankey_compared_html': None, - 'sankey_compared_png': None, - 'sankey_compared_jpg': None, - 'sankey_compared_full_json': None, - 'sankey_compared_full_txt': None, - 'sankey_compared_full_key_txt': None, - 'sankey_compared_full_html': None, - 'sankey_compared_full_png': None, - 'sankey_compared_full_jpg': None, - 'plotly_error': f'failed to build sankey report: {ex!r}', - } - summary_report['artifacts'] = sankey_artifacts - - summary_report = kwutil.Json.ensure_serializable(summary_report) - summary_json_fpath.write_text( - json.dumps(summary_report, indent=2, ensure_ascii=False) - ) - - print('---') - print(f'Wrote case report: {case_jsonl_fpath}') - print(f'Wrote summary report: {summary_json_fpath}') - if sankey_artifacts.get('sankey_json', None): - print(f'Wrote sankey JSON: {sankey_artifacts["sankey_json"]}') - if sankey_artifacts.get('sankey_txt', None): - print(f'Wrote sankey TXT: {sankey_artifacts["sankey_txt"]}') - if sankey_artifacts.get('sankey_html', None): - print(f'Wrote sankey HTML: {sankey_artifacts["sankey_html"]}') - if sankey_artifacts.get('sankey_png', None): - print(f'Wrote sankey PNG: {sankey_artifacts["sankey_png"]}') - if sankey_artifacts.get('sankey_jpg', None): - print(f'Wrote sankey JPG: {sankey_artifacts["sankey_jpg"]}') - if sankey_artifacts.get('sankey_key_txt', None): - print(f'Wrote sankey key: {sankey_artifacts["sankey_key_txt"]}') - if sankey_artifacts.get('sankey_compared_json', None): - print( - f'Wrote compared-detail sankey JSON: ' - f'{sankey_artifacts["sankey_compared_json"]}' - ) - if sankey_artifacts.get('sankey_compared_txt', None): - print( - f'Wrote compared-detail sankey TXT: ' - f'{sankey_artifacts["sankey_compared_txt"]}' - ) - if sankey_artifacts.get('sankey_compared_html', None): - print( - f'Wrote compared-detail sankey HTML: ' - f'{sankey_artifacts["sankey_compared_html"]}' - ) - if sankey_artifacts.get('sankey_compared_png', None): - print( - f'Wrote compared-detail sankey PNG: ' - f'{sankey_artifacts["sankey_compared_png"]}' - ) - if sankey_artifacts.get('sankey_compared_jpg', None): - print( - f'Wrote compared-detail sankey JPG: ' - f'{sankey_artifacts["sankey_compared_jpg"]}' - ) - if sankey_artifacts.get('sankey_compared_key_txt', None): - print( - f'Wrote compared-detail sankey key: ' - f'{sankey_artifacts["sankey_compared_key_txt"]}' - ) - if sankey_artifacts.get('sankey_compared_full_json', None): - print( - f'Wrote compared-full sankey JSON: ' - f'{sankey_artifacts["sankey_compared_full_json"]}' - ) - if sankey_artifacts.get('sankey_compared_full_txt', None): - print( - f'Wrote compared-full sankey TXT: ' - f'{sankey_artifacts["sankey_compared_full_txt"]}' - ) - if sankey_artifacts.get('sankey_compared_full_html', None): - print( - f'Wrote compared-full sankey HTML: ' - f'{sankey_artifacts["sankey_compared_full_html"]}' - ) - if sankey_artifacts.get('sankey_compared_full_png', None): - print( - f'Wrote compared-full sankey PNG: ' - f'{sankey_artifacts["sankey_compared_full_png"]}' - ) - if sankey_artifacts.get('sankey_compared_full_jpg', None): - print( - f'Wrote compared-full sankey JPG: ' - f'{sankey_artifacts["sankey_compared_full_jpg"]}' - ) - if sankey_artifacts.get('sankey_compared_full_key_txt', None): - print( - f'Wrote compared-full sankey key: ' - f'{sankey_artifacts["sankey_compared_full_key_txt"]}' - ) - if sankey_artifacts.get('plotly_error', None): - print(f'Sankey note: {sankey_artifacts["plotly_error"]}') - print( - 'Diagnosis label counts: ' - + ub.urepr(summary_report['aggregate']['diagnosis_label_counts'], nl=0) - ) - - -if __name__ == '__main__': - main() diff --git a/dev/poc/inspect_historic_helm_runs.py b/dev/poc/inspect_historic_helm_runs.py deleted file mode 100644 index 30e0840..0000000 --- a/dev/poc/inspect_historic_helm_runs.py +++ /dev/null @@ -1,327 +0,0 @@ -r""" -Compile a reproduction list from existing HELM outputs on disk. - -Given one or more roots that contain HELM outputs, discover all run directories -and emit a list of run specs you can feed into kwdagger / helm-run. - -Outputs are structured so you can: -- reproduce exact run directories (by using run_entry == run directory name) -- optionally include max_eval_instances inferred from per_instance_stats.json - -Ignore: - - ls /data/crfm-helm-public/thaiexam/benchmark_output/runs/v1.1.0/thai_exam:exam=tpat1,method=multiple_choice_joint,model=aisingapore_llama3-8b-cpt-sea-lionv2.1-instruct - - python ~/code/aiq-magnet/dev/poc/inspect_historic_helm_runs.py /data/crfm-helm-public --out_fpath run_specs.yaml --out_detail_fpath run_details.yaml - - cat run_specs.yaml | grep -v together > run_specs2.yaml - - python ~/code/aiq-magnet/dev/poc/inspect_historic_helm_runs.py /data/Public/AIQ/crfm-helm-public/ - - # we need fully featured helm installed - uv pip install crfm-helm[all] -U - - # Need to login to huggingface can pass token via --token - hf auth login - - # Need TogetherAPI credentials - - kwdagger schedule \ - --params=" - pipeline: 'magnet.backends.helm.pipeline.helm_single_run_pipeline()' - matrix: - helm.run_entry: - - __include__: run_specs2.yaml - helm.max_eval_instances: - - 1000 - helm.precomputed_root: null - " \ - --devices="0,1,2,3" \ - --tmux_workers=4 \ - --root_dpath=$PWD/results \ - --backend=tmux \ - --skip_existing=1 \ - --run=1 -""" - -from __future__ import annotations -from pathlib import Path -from typing import Iterable, Any - -import ubelt as ub -import kwutil -import scriptconfig as scfg -from loguru import logger - -from magnet.backends.helm.helm_outputs import HelmOutputs, HelmRun - -# Reuse your existing discovery + inference logic -from magnet.backends.helm.cli.materialize_helm_run import ( - discover_benchmark_output_dirs, - infer_num_instances, - is_complete_run_dir, -) - - -class CompileHelmReproListConfig(scfg.DataConfig): - roots = scfg.Value( - ['/data/crfm-helm-public'], - nargs="+", - help=( - "One or more roots that either ARE a benchmark_output dir, contain " - "benchmark_output dirs, or contain suite/benchmark_output dirs." - ), - position=1, - ) - - suite_pattern = scfg.Value( - "*", - help="Glob applied to benchmark_output/runs/ directories.", - ) - - run_pattern = scfg.Value( - "*:*", - help="Glob applied within each suite to select runs (default selects HELM run dirs).", - ) - - require_per_instance_stats = scfg.Value( - False, - help="If True, only include runs that have per_instance_stats.json.", - ) - - include_max_eval_instances = scfg.Value( - False, - help="If True, infer max_eval_instances from per_instance_stats.json when possible. CAN BE VERY SLOW", - ) - - out_fpath = scfg.Value( - None, - help="Where to write output. If omitted, prints to stdout.", - ) - - out_detail_fpath = scfg.Value( - None, - help="Where to write detailed output.", - ) - - dedupe = scfg.Value( - True, - help="If True, dedupe identical (suite, run_entry, max_eval_instances) rows.", - ) - - @classmethod - def main(cls, argv=None, **kwargs): - """ - Example: - >>> # It's a good idea to setup a doctest. - >>> import sys, ubelt - >>> sys.path.append(ubelt.expandpath('~/code/aiq-magnet/dev/poc')) - >>> from inspect_historic_helm_runs import * # NOQA - >>> argv = False - >>> kwargs = dict() - >>> cls = CompileHelmReproListConfig - >>> config = cls(**kwargs) - >>> cls.main(argv=argv, **config) - """ - config = cls.cli(argv=argv, data=kwargs, verbose="auto") - roots = [Path(r).expanduser() for r in config.roots] - if not roots: - raise SystemExit("Must provide at least one root") - - suite_pattern = config.suite_pattern - run_pattern = config.run_pattern - require_per_instance_stats = config.require_per_instance_stats - include_max_eval_instances = config.include_max_eval_instances - - runs = gather_runs( - roots=roots, - suite_pattern=suite_pattern, - run_pattern=run_pattern, - require_per_instance_stats=require_per_instance_stats, - include_max_eval_instances=include_max_eval_instances, - ) - rows = build_run_table(runs) - - scenario_histo = ub.dict_hist([r['scenario_class'] for r in rows]) - model_histo = ub.dict_hist([r['model'] for r in rows]) - scenario_histo = ub.udict.sorted_values(scenario_histo) - model_histo = ub.udict.sorted_values(model_histo) - print(f'scenario_histo = {ub.urepr(scenario_histo, nl=1)}') - print(f'model_histo = {ub.urepr(model_histo, nl=1)}') - - from helm.benchmark import config_registry - from helm.benchmark import model_deployment_registry - config_registry.register_builtin_configs_from_helm_package() - model_rows = [] - for model_name, count in model_histo.items(): - try: - model_meta = model_deployment_registry.get_model_metadata(model_name) - model_row = model_meta.__dict__ | {'count': count} - if model_meta.deployment_names: - for deploy_name in model_meta.deployment_names: - deploy_info = model_deployment_registry.get_model_deployment(deploy_name) - model_row['client'] = deploy_info.client_spec.class_name - model_rows.append(model_row) - except (TypeError, ValueError) as ex: - logger.warning(f'missing: model_name = {ub.urepr(model_name, nl=1)} {ex}') - - if 0: - ub.dict_hist([r.get('client') for r in model_rows]) - - require_tags = { - 'FULL_FUNCTIONALITY_TEXT_MODEL_TAG' - } - MAX_PARAMS = 10e9 - # MAX_PARAMS = 200e9 - - # Filter to text models that will fit in memory - chosen_model_rows = [ - r for r in model_rows if ( - set(r['tags']).issuperset(require_tags) and - (r['num_parameters'] is not None and r['num_parameters'] <= MAX_PARAMS) and - (r['access'] == 'open') and - (r.get('client') == 'helm.clients.huggingface_client.HuggingFaceClient') - ) - ] - chosen_model_names = {r['name'] for r in chosen_model_rows} - logger.info('Filter to {} / {} models', len(chosen_model_rows), len(model_rows)) - - chosen_rows = [r for r in rows if r['model'] in chosen_model_names] - logger.info('Filter to {} / {} runs', len(chosen_rows), len(rows)) - # logger.info(f'chosen_rows = {ub.urepr(chosen_rows, nl=1)}') - - if 1: - # Show filtered histograms - scenario_histo = ub.dict_hist([r['scenario_class'] for r in chosen_rows]) - model_histo = ub.dict_hist([r['model'] for r in chosen_rows]) - scenario_histo = ub.udict.sorted_values(scenario_histo) - model_histo = ub.udict.sorted_values(model_histo) - logger.info(f'scenario_histo = {ub.urepr(scenario_histo, nl=1)}') - logger.info(f'model_histo = {ub.urepr(model_histo, nl=1)}') - - if config.out_detail_fpath: - text = kwutil.Yaml.dumps(chosen_rows) - Path(config.out_detail_fpath).write_text(text) - logger.success("Wrote {}", config.out_detail_fpath) - - run_spec_names = [r["run_spec_name"] for r in chosen_rows] - text = kwutil.Yaml.dumps(run_spec_names) - if config.out_fpath: - Path(config.out_fpath).write_text(text) - logger.success("Wrote {}", config.out_fpath) - else: - print(text, end="") - - -def gather_runs( - roots: Iterable[Path], - suite_pattern: str = "*", - run_pattern: str = "*:*", - require_per_instance_stats: bool = False, - include_max_eval_instances: bool = True, -) -> list[HelmRun]: - - # Discover all benchmark_output dirs under provided roots - logger.info('Discover benchmarks') - bo_dirs = list(ub.ProgIter(discover_benchmark_output_dirs(roots), desc='discovering benchmarks', verbose=3, homogeneous=False)) - logger.info('Finished Discover benchmarks') - if not bo_dirs: - logger.warning("No benchmark_output dirs found under roots={}", roots) - - runs: list[HelmRun] = [] - for bo in ub.ProgIter(bo_dirs, desc='Check dirs'): - try: - outputs = HelmOutputs.coerce(bo) - except Exception: - continue - - for suite in outputs.suites(pattern=suite_pattern): - for run in suite.runs(pattern=run_pattern): - run_dir = Path(run.path) - - run = HelmRun(run_dir) - - # TODO: if not run.exists(): - # ... - # Only include if it looks “complete enough” - if not is_complete_run_dir(run_dir, require_per_instance_stats=require_per_instance_stats): - continue - - runs.append(run) - - # Stable order - logger.info('Found {} run directories', len(runs)) - return runs - - -def build_run_table(runs: list[HelmRun]) -> list[dict]: - rows = [] - - include_max_eval_instances = False - mismatches = [] - for run in ub.ProgIter(runs, desc='Extract run spec info'): - max_eval_instances = None - if include_max_eval_instances: - max_eval_instances = infer_num_instances(run.path) - - # Not sure if there is an advantage to msgspec or json here - # ZFS is likely messing up my timings. - if 1: - run_spec = run.json.run_spec() - scenario_class = run_spec['scenario_spec']['class_name'] - model = run_spec['adapter_spec']['model'] - run_spec_name = run_spec['name'] - else: - run_spec = run.msgspec.run_spec() - scenario_class = run_spec.scenario_spec.class_name - model = run_spec.adapter_spec.model - run_spec_name = run_spec.name - - if run.path.name != run_spec_name.replace('/', '_'): - mismatches.append({ - 'run.path.parent': run.path.parent, - 'run.path.name': run.path.name, - 'run_spec_name': run_spec_name, - }) - - # Hack: run spec names sometimes don't correctly encode the model - FIX_RUN_SPEC_NAME = True - if FIX_RUN_SPEC_NAME: - normalized_model = model.replace('/', '_') - run_spec_name = run_spec_name.replace(normalized_model, model) - - rows.append({ - # "benchmark_output_dir": str(Path(outputs.root_dir)), - # "suite": suite.name, - # # Use run directory name as the canonical "run_entry" to reproduce. - # # This is faithful even if HELM normalized defaults into the name. - - # Use run directory name as the canonical "run_entry" to reproduce. - # This is faithful even if HELM normalized defaults into the name. - "run_spec_name": run_spec_name, - "run_dir": str(run.path), - "max_eval_instances": max_eval_instances, - 'model': model, - 'scenario_class': scenario_class, - }) - logger.warning(f'mismatches = {ub.urepr(mismatches, nl=2, align=":")}') - rows.sort(key=lambda r: (r["run_dir"])) - return rows - - -def dedupe_rows(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: - seen = set() - out = [] - for r in rows: - key = (r["suite"], r["run_entry"], r.get("max_eval_instances", None)) - if key in seen: - continue - seen.add(key) - out.append(r) - return out - - -__cli__ = CompileHelmReproListConfig - -if __name__ == "__main__": - __cli__.main() From c842443a46583a374450d0a39991a1b2bab47b8b Mon Sep 17 00:00:00 2001 From: joncrall Date: Wed, 1 Apr 2026 13:29:13 -0400 Subject: [PATCH 36/36] Fix test --- magnet/backends/helm/helm_run_diff.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/magnet/backends/helm/helm_run_diff.py b/magnet/backends/helm/helm_run_diff.py index 3a8144e..81e462e 100644 --- a/magnet/backends/helm/helm_run_diff.py +++ b/magnet/backends/helm/helm_run_diff.py @@ -1432,14 +1432,19 @@ def add_reason(name: str, priority: int, details: dict[str, Any]) -> None: ) execution_ok = bool(run_spec_semantic.get('execution_ok', False)) - if not execution_ok: + execution_paths = run_spec_semantic.get('execution_paths', []) or [] + deployment_paths = run_spec_semantic.get('deployment_paths', []) or [] + non_deployment_execution_paths = [ + p + for p in execution_paths + if not str(p).startswith('adapter_spec.model_deployment') + ] + if not execution_ok and non_deployment_execution_paths: add_reason( 'execution_spec_drift', 0, { - 'execution_paths': run_spec_semantic.get( - 'execution_paths', [] - ), + 'execution_paths': execution_paths, 'execution_value_examples': run_spec_semantic.get( 'execution_value_examples', [] ), @@ -1458,11 +1463,11 @@ def add_reason(name: str, priority: int, details: dict[str, Any]) -> None: 'execution_paths': [ p for p in ( - run_spec_semantic.get('execution_paths', []) or [] + execution_paths ) if str(p).startswith('adapter_spec.model_deployment') ] - or (run_spec_semantic.get('deployment_paths', []) or []), + or deployment_paths, }, )