fix(aether-evals): capture agent diff from start commit

crates/aether-evals/src/evals/workspace.rs

@@ -158,7 +158,8 @@ impl Workspace {
         };
 
         let repo = GitRepo::from_path(self.path());
-        let agent_diff = repo.diff_unstaged().ok().map(|diff| GitDiff { stats: DiffStats::from_diff(&diff), diff });
+        let agent_diff =
+            repo.diff_range(start_commit, None).ok().map(|diff| GitDiff { stats: DiffStats::from_diff(&diff), diff });
         let reference_diff =
             repo.diff(start_commit, gold_commit).ok().map(|diff| GitDiff { stats: DiffStats::from_diff(&diff), diff });
 
@@ -278,6 +279,66 @@ mod tests {
         assert!(agent_diff.unwrap().diff.contains("root edited"), "agent diff should capture the edit");
     }
 
+    #[test]
+    fn capture_git_diffs_includes_committed_agent_changes() {
+        let (repo, start, gold) = init_repo();
+        let workspace = Workspace::from_git_repo(GitRepoSpec {
+            url: format!("file://{}", repo.path().display()),
+            start_commit: start,
+            gold_commit: gold,
+            subdir: None,
+        })
+        .unwrap();
+
+        write(workspace.join("root.txt"), "agent committed\n").unwrap();
+        git(workspace.root_path(), &["add", "."]);
+        git(
+            workspace.root_path(),
+            &["-c", "user.email=agent@example.com", "-c", "user.name=Agent", "commit", "-m", "agent change"],
+        );
+
+        let (agent_diff, _) = workspace.capture_git_diffs();
+        let agent_diff = agent_diff.unwrap();
+
+        assert!(agent_diff.diff.contains("+agent committed"));
+        assert_eq!(agent_diff.stats.files_changed, 1);
+        assert_eq!(agent_diff.stats.lines_added, 1);
+        assert_eq!(agent_diff.stats.lines_removed, 1);
+    }
+
+    #[test]
+    fn capture_git_diffs_includes_staged_unstaged_added_and_deleted_changes() {
+        let (repo, start, gold) = init_repo();
+        let workspace = Workspace::from_git_repo(GitRepoSpec {
+            url: format!("file://{}", repo.path().display()),
+            start_commit: start,
+            gold_commit: gold,
+            subdir: None,
+        })
+        .unwrap();
+
+        write(workspace.join("root.txt"), "staged change\n").unwrap();
+        write(workspace.join("added.txt"), "added\n").unwrap();
+        git(workspace.root_path(), &["add", "root.txt", "added.txt"]);
+        write(workspace.join("pkg/inner.txt"), "unstaged change\n").unwrap();
+        std::fs::remove_file(workspace.join("deleted.txt")).unwrap();
+
+        let (agent_diff, _) = workspace.capture_git_diffs();
+        let agent_diff = agent_diff.unwrap();
+
+        assert!(agent_diff.diff.contains("root.txt"));
+        assert!(agent_diff.diff.contains("+staged change"));
+        assert!(agent_diff.diff.contains("pkg/inner.txt"));
+        assert!(agent_diff.diff.contains("+unstaged change"));
+        assert!(agent_diff.diff.contains("added.txt"));
+        assert!(agent_diff.diff.contains("+added"));
+        assert!(agent_diff.diff.contains("deleted.txt"));
+        assert!(agent_diff.diff.contains("-before deleted"));
+        assert_eq!(agent_diff.stats.files_changed, 4);
+        assert_eq!(agent_diff.stats.lines_added, 3);
+        assert_eq!(agent_diff.stats.lines_removed, 3);
+    }
+
     #[test]
     fn from_git_bundle_missing_file_errors() {
         let result = Workspace::from_git_bundle(GitBundleSpec {
@@ -298,6 +359,7 @@ mod tests {
         git(path, &["config", "user.name", "Eval"]);
 
         write(path.join("root.txt"), "root v1\n").unwrap();
+        write(path.join("deleted.txt"), "before deleted\n").unwrap();
         create_dir_all(path.join("pkg")).unwrap();
         write(path.join("pkg").join("inner.txt"), "inner v1\n").unwrap();
         git(path, &["add", "."]);

crates/aether-evals/src/git_repo.rs

@@ -116,13 +116,6 @@ impl GitRepo {
     pub fn diff(&self, from_commit: &str, to_commit: &str) -> Result<String, GitRepoError> {
         self.diff_range(from_commit, Some(to_commit))
     }
-
-    /// Get the diff of unstaged changes in the working directory
-    ///
-    /// Returns the output of `git diff` which shows all unstaged changes
-    pub fn diff_unstaged(&self) -> Result<String, GitRepoError> {
-        self.diff_range("HEAD", None)
-    }
 }
 
 /// Run `git` (optionally inside `cwd`) and return its stdout.

packages/aether-evals/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aether-agent/evals",
-  "version": "0.3.0",
+  "version": "0.3.1",
   "description": "Evaluation harness for the Aether agent SDK",
   "repository": {
     "type": "git",

packages/aether-evals/src/git.ts

@@ -76,10 +76,6 @@ export class GitRepo {
     const result = await git(["-C", this.path, "diff", range], signal);
     return result.stdout;
   }
-
-  async diffUnstaged(signal?: AbortSignal): Promise<string> {
-    return this.diff("HEAD", undefined, signal);
-  }
 }
 
 async function git(args: string[], signal?: AbortSignal) {

packages/aether-evals/src/workspace.ts

@@ -180,7 +180,7 @@ export class Workspace implements AsyncDisposable {
     const repo = GitRepo.fromPath(this.path);
     const { startCommit, goldCommit } = commits;
     const [agentDiff, referenceDiff] = await Promise.all([
-      captureDiff(() => repo.diffUnstaged()),
+      captureDiff(() => repo.diff(startCommit)),
       captureDiff(() => repo.diff(startCommit, goldCommit)),
     ]);
     return { agentDiff, referenceDiff };

packages/aether-evals/test/workspace.test.ts

@@ -1,6 +1,13 @@
 import { execFileSync } from "node:child_process";
 import { existsSync } from "node:fs";
-import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import {
+  mkdir,
+  mkdtemp,
+  readFile,
+  rm,
+  unlink,
+  writeFile,
+} from "node:fs/promises";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { afterEach, describe, expect, it } from "vitest";
@@ -129,6 +136,75 @@ describe("createWorkspace", () => {
     expect(referenceDiff?.stats.filesChanged).toBe(2);
   });
 
+  it("captures committed agent changes from the start commit", async () => {
+    const ws = await createGitWorkspace({ "app.txt": "before\n" });
+    const workspaceGit = (...args: string[]) =>
+      execFileSync("git", args, { cwd: ws.rootPath, stdio: "pipe" });
+    await writeFile(ws.join("app.txt"), "agent committed\n");
+    workspaceGit("add", ".");
+    workspaceGit(
+      "-c",
+      "user.email=agent@example.com",
+      "-c",
+      "user.name=Agent",
+      "commit",
+      "-m",
+      "agent change",
+    );
+
+    const { agentDiff } = await ws.captureGitDiffs();
+
+    expect(agentDiff?.diff).toContain("+agent committed");
+    expect(agentDiff?.stats).toMatchObject({
+      filesChanged: 1,
+      linesAdded: 1,
+      linesRemoved: 1,
+    });
+  });
+
+  it("captures staged, unstaged, added, and deleted tracked changes", async () => {
+    const ws = await createGitWorkspace({
+      "staged.txt": "before staged\n",
+      "unstaged.txt": "before unstaged\n",
+      "deleted.txt": "before deleted\n",
+    });
+    const workspaceGit = (...args: string[]) =>
+      execFileSync("git", args, { cwd: ws.rootPath, stdio: "pipe" });
+
+    await writeFile(ws.join("staged.txt"), "after staged\n");
+    await writeFile(ws.join("added.txt"), "added\n");
+    workspaceGit("add", "staged.txt", "added.txt");
+    await writeFile(ws.join("unstaged.txt"), "after unstaged\n");
+    await unlink(ws.join("deleted.txt"));
+
+    const { agentDiff } = await ws.captureGitDiffs();
+
+    expect(agentDiff?.diff).toContain("staged.txt");
+    expect(agentDiff?.diff).toContain("+after staged");
+    expect(agentDiff?.diff).toContain("unstaged.txt");
+    expect(agentDiff?.diff).toContain("+after unstaged");
+    expect(agentDiff?.diff).toContain("added.txt");
+    expect(agentDiff?.diff).toContain("+added");
+    expect(agentDiff?.diff).toContain("deleted.txt");
+    expect(agentDiff?.diff).toContain("-before deleted");
+    expect(agentDiff?.stats).toEqual({
+      filesChanged: 4,
+      linesAdded: 3,
+      linesRemoved: 3,
+    });
+  });
+
+  it("captures an unchanged workspace as an empty agent diff", async () => {
+    const ws = await createGitWorkspace({ "app.txt": "unchanged\n" });
+
+    const { agentDiff } = await ws.captureGitDiffs();
+
+    expect(agentDiff).toEqual({
+      diff: "",
+      stats: { filesChanged: 0, linesAdded: 0, linesRemoved: 0 },
+    });
+  });
+
   it("creates a bundle and instantiates a workspace from it, honoring subdir and diffs", async () => {
     const repo = track(await mkdtemp(join(tmpdir(), "ws-repo-")));
     const git = (...args: string[]) =>
@@ -199,3 +275,27 @@ function track(path: string): string {
   cleanups.push(path);
   return path;
 }
+
+async function createGitWorkspace(
+  files: Record<string, string>,
+): Promise<Workspace> {
+  const repo = track(await mkdtemp(join(tmpdir(), "ws-repo-")));
+  const git = (...args: string[]) =>
+    execFileSync("git", args, { cwd: repo, stdio: "pipe" });
+  git("init", "--initial-branch", "main");
+  git("config", "user.email", "eval@example.com");
+  git("config", "user.name", "Eval");
+  for (const [path, content] of Object.entries(files)) {
+    await writeFile(join(repo, path), content);
+  }
+  git("add", ".");
+  git("commit", "-m", "start");
+  const startCommit = git("rev-parse", "HEAD").toString().trim();
+  const workspace = await Workspace.fromGitRepo({
+    url: `file://${repo}`,
+    startCommit,
+    goldCommit: startCommit,
+  });
+  cleanups.push(workspace.rootPath);
+  return workspace;
+}