diff --git a/crates/aether-evals/src/evals/workspace.rs b/crates/aether-evals/src/evals/workspace.rs index db24c1934..25e32ea23 100644 --- a/crates/aether-evals/src/evals/workspace.rs +++ b/crates/aether-evals/src/evals/workspace.rs @@ -158,7 +158,8 @@ impl Workspace { }; let repo = GitRepo::from_path(self.path()); - let agent_diff = repo.diff_unstaged().ok().map(|diff| GitDiff { stats: DiffStats::from_diff(&diff), diff }); + let agent_diff = + repo.diff_range(start_commit, None).ok().map(|diff| GitDiff { stats: DiffStats::from_diff(&diff), diff }); let reference_diff = repo.diff(start_commit, gold_commit).ok().map(|diff| GitDiff { stats: DiffStats::from_diff(&diff), diff }); @@ -278,6 +279,66 @@ mod tests { assert!(agent_diff.unwrap().diff.contains("root edited"), "agent diff should capture the edit"); } + #[test] + fn capture_git_diffs_includes_committed_agent_changes() { + let (repo, start, gold) = init_repo(); + let workspace = Workspace::from_git_repo(GitRepoSpec { + url: format!("file://{}", repo.path().display()), + start_commit: start, + gold_commit: gold, + subdir: None, + }) + .unwrap(); + + write(workspace.join("root.txt"), "agent committed\n").unwrap(); + git(workspace.root_path(), &["add", "."]); + git( + workspace.root_path(), + &["-c", "user.email=agent@example.com", "-c", "user.name=Agent", "commit", "-m", "agent change"], + ); + + let (agent_diff, _) = workspace.capture_git_diffs(); + let agent_diff = agent_diff.unwrap(); + + assert!(agent_diff.diff.contains("+agent committed")); + assert_eq!(agent_diff.stats.files_changed, 1); + assert_eq!(agent_diff.stats.lines_added, 1); + assert_eq!(agent_diff.stats.lines_removed, 1); + } + + #[test] + fn capture_git_diffs_includes_staged_unstaged_added_and_deleted_changes() { + let (repo, start, gold) = init_repo(); + let workspace = Workspace::from_git_repo(GitRepoSpec { + url: format!("file://{}", repo.path().display()), + start_commit: start, + gold_commit: gold, + subdir: None, + }) + .unwrap(); + + write(workspace.join("root.txt"), "staged change\n").unwrap(); + write(workspace.join("added.txt"), "added\n").unwrap(); + git(workspace.root_path(), &["add", "root.txt", "added.txt"]); + write(workspace.join("pkg/inner.txt"), "unstaged change\n").unwrap(); + std::fs::remove_file(workspace.join("deleted.txt")).unwrap(); + + let (agent_diff, _) = workspace.capture_git_diffs(); + let agent_diff = agent_diff.unwrap(); + + assert!(agent_diff.diff.contains("root.txt")); + assert!(agent_diff.diff.contains("+staged change")); + assert!(agent_diff.diff.contains("pkg/inner.txt")); + assert!(agent_diff.diff.contains("+unstaged change")); + assert!(agent_diff.diff.contains("added.txt")); + assert!(agent_diff.diff.contains("+added")); + assert!(agent_diff.diff.contains("deleted.txt")); + assert!(agent_diff.diff.contains("-before deleted")); + assert_eq!(agent_diff.stats.files_changed, 4); + assert_eq!(agent_diff.stats.lines_added, 3); + assert_eq!(agent_diff.stats.lines_removed, 3); + } + #[test] fn from_git_bundle_missing_file_errors() { let result = Workspace::from_git_bundle(GitBundleSpec { @@ -298,6 +359,7 @@ mod tests { git(path, &["config", "user.name", "Eval"]); write(path.join("root.txt"), "root v1\n").unwrap(); + write(path.join("deleted.txt"), "before deleted\n").unwrap(); create_dir_all(path.join("pkg")).unwrap(); write(path.join("pkg").join("inner.txt"), "inner v1\n").unwrap(); git(path, &["add", "."]); diff --git a/crates/aether-evals/src/git_repo.rs b/crates/aether-evals/src/git_repo.rs index 12dbde64a..51fc44c8b 100644 --- a/crates/aether-evals/src/git_repo.rs +++ b/crates/aether-evals/src/git_repo.rs @@ -116,13 +116,6 @@ impl GitRepo { pub fn diff(&self, from_commit: &str, to_commit: &str) -> Result { self.diff_range(from_commit, Some(to_commit)) } - - /// Get the diff of unstaged changes in the working directory - /// - /// Returns the output of `git diff` which shows all unstaged changes - pub fn diff_unstaged(&self) -> Result { - self.diff_range("HEAD", None) - } } /// Run `git` (optionally inside `cwd`) and return its stdout. diff --git a/packages/aether-evals/package.json b/packages/aether-evals/package.json index 185c2ec8e..e6d0b6c61 100644 --- a/packages/aether-evals/package.json +++ b/packages/aether-evals/package.json @@ -1,6 +1,6 @@ { "name": "@aether-agent/evals", - "version": "0.3.0", + "version": "0.3.1", "description": "Evaluation harness for the Aether agent SDK", "repository": { "type": "git", diff --git a/packages/aether-evals/src/git.ts b/packages/aether-evals/src/git.ts index 0df96ebd9..1e6a5cb5d 100644 --- a/packages/aether-evals/src/git.ts +++ b/packages/aether-evals/src/git.ts @@ -76,10 +76,6 @@ export class GitRepo { const result = await git(["-C", this.path, "diff", range], signal); return result.stdout; } - - async diffUnstaged(signal?: AbortSignal): Promise { - return this.diff("HEAD", undefined, signal); - } } async function git(args: string[], signal?: AbortSignal) { diff --git a/packages/aether-evals/src/workspace.ts b/packages/aether-evals/src/workspace.ts index 9dc5b295b..d71d46ef3 100644 --- a/packages/aether-evals/src/workspace.ts +++ b/packages/aether-evals/src/workspace.ts @@ -180,7 +180,7 @@ export class Workspace implements AsyncDisposable { const repo = GitRepo.fromPath(this.path); const { startCommit, goldCommit } = commits; const [agentDiff, referenceDiff] = await Promise.all([ - captureDiff(() => repo.diffUnstaged()), + captureDiff(() => repo.diff(startCommit)), captureDiff(() => repo.diff(startCommit, goldCommit)), ]); return { agentDiff, referenceDiff }; diff --git a/packages/aether-evals/test/workspace.test.ts b/packages/aether-evals/test/workspace.test.ts index f2eb53503..46f336bda 100644 --- a/packages/aether-evals/test/workspace.test.ts +++ b/packages/aether-evals/test/workspace.test.ts @@ -1,6 +1,13 @@ import { execFileSync } from "node:child_process"; import { existsSync } from "node:fs"; -import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { + mkdir, + mkdtemp, + readFile, + rm, + unlink, + writeFile, +} from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; @@ -129,6 +136,75 @@ describe("createWorkspace", () => { expect(referenceDiff?.stats.filesChanged).toBe(2); }); + it("captures committed agent changes from the start commit", async () => { + const ws = await createGitWorkspace({ "app.txt": "before\n" }); + const workspaceGit = (...args: string[]) => + execFileSync("git", args, { cwd: ws.rootPath, stdio: "pipe" }); + await writeFile(ws.join("app.txt"), "agent committed\n"); + workspaceGit("add", "."); + workspaceGit( + "-c", + "user.email=agent@example.com", + "-c", + "user.name=Agent", + "commit", + "-m", + "agent change", + ); + + const { agentDiff } = await ws.captureGitDiffs(); + + expect(agentDiff?.diff).toContain("+agent committed"); + expect(agentDiff?.stats).toMatchObject({ + filesChanged: 1, + linesAdded: 1, + linesRemoved: 1, + }); + }); + + it("captures staged, unstaged, added, and deleted tracked changes", async () => { + const ws = await createGitWorkspace({ + "staged.txt": "before staged\n", + "unstaged.txt": "before unstaged\n", + "deleted.txt": "before deleted\n", + }); + const workspaceGit = (...args: string[]) => + execFileSync("git", args, { cwd: ws.rootPath, stdio: "pipe" }); + + await writeFile(ws.join("staged.txt"), "after staged\n"); + await writeFile(ws.join("added.txt"), "added\n"); + workspaceGit("add", "staged.txt", "added.txt"); + await writeFile(ws.join("unstaged.txt"), "after unstaged\n"); + await unlink(ws.join("deleted.txt")); + + const { agentDiff } = await ws.captureGitDiffs(); + + expect(agentDiff?.diff).toContain("staged.txt"); + expect(agentDiff?.diff).toContain("+after staged"); + expect(agentDiff?.diff).toContain("unstaged.txt"); + expect(agentDiff?.diff).toContain("+after unstaged"); + expect(agentDiff?.diff).toContain("added.txt"); + expect(agentDiff?.diff).toContain("+added"); + expect(agentDiff?.diff).toContain("deleted.txt"); + expect(agentDiff?.diff).toContain("-before deleted"); + expect(agentDiff?.stats).toEqual({ + filesChanged: 4, + linesAdded: 3, + linesRemoved: 3, + }); + }); + + it("captures an unchanged workspace as an empty agent diff", async () => { + const ws = await createGitWorkspace({ "app.txt": "unchanged\n" }); + + const { agentDiff } = await ws.captureGitDiffs(); + + expect(agentDiff).toEqual({ + diff: "", + stats: { filesChanged: 0, linesAdded: 0, linesRemoved: 0 }, + }); + }); + it("creates a bundle and instantiates a workspace from it, honoring subdir and diffs", async () => { const repo = track(await mkdtemp(join(tmpdir(), "ws-repo-"))); const git = (...args: string[]) => @@ -199,3 +275,27 @@ function track(path: string): string { cleanups.push(path); return path; } + +async function createGitWorkspace( + files: Record, +): Promise { + const repo = track(await mkdtemp(join(tmpdir(), "ws-repo-"))); + const git = (...args: string[]) => + execFileSync("git", args, { cwd: repo, stdio: "pipe" }); + git("init", "--initial-branch", "main"); + git("config", "user.email", "eval@example.com"); + git("config", "user.name", "Eval"); + for (const [path, content] of Object.entries(files)) { + await writeFile(join(repo, path), content); + } + git("add", "."); + git("commit", "-m", "start"); + const startCommit = git("rev-parse", "HEAD").toString().trim(); + const workspace = await Workspace.fromGitRepo({ + url: `file://${repo}`, + startCommit, + goldCommit: startCommit, + }); + cleanups.push(workspace.rootPath); + return workspace; +}