diff --git a/.changeset/scoped-eval-target.md b/.changeset/scoped-eval-target.md new file mode 100644 index 000000000..51546be0e --- /dev/null +++ b/.changeset/scoped-eval-target.md @@ -0,0 +1,5 @@ +--- +"eve": patch +--- + +Make `eve eval` match scoped package targets against the same unscoped agent id used at runtime. diff --git a/packages/eve/src/evals/cli/eval.ts b/packages/eve/src/evals/cli/eval.ts index e84a13045..5d1e14249 100644 --- a/packages/eve/src/evals/cli/eval.ts +++ b/packages/eve/src/evals/cli/eval.ts @@ -230,10 +230,15 @@ async function readExpectedAgentName(appRoot: string): Promise { const packageJson = JSON.parse(await readFile(join(appRoot, "package.json"), "utf8")) as { readonly name?: unknown; }; - return typeof packageJson.name === "string" && packageJson.name.length > 0 - ? packageJson.name - : basename(appRoot); - } catch { - return basename(appRoot); - } + if (typeof packageJson.name === "string" && packageJson.name.length > 0) { + return unscopedPackageName(packageJson.name); + } + } catch {} + + return basename(appRoot); +} + +function unscopedPackageName(packageName: string): string { + const slashIndex = packageName.lastIndexOf("/"); + return slashIndex === -1 ? packageName : packageName.slice(slashIndex + 1); } diff --git a/packages/eve/test/scenarios/eval-command-environment.scenario.test.ts b/packages/eve/test/scenarios/eval-command-environment.scenario.test.ts index d4db77e9f..459d46d97 100644 --- a/packages/eve/test/scenarios/eval-command-environment.scenario.test.ts +++ b/packages/eve/test/scenarios/eval-command-environment.scenario.test.ts @@ -86,6 +86,44 @@ const TEST_CONFIG = { }; describe("eve eval environment loading", () => { + it("matches scoped package names against the runtime agent id", async () => { + const fixtureRoot = await createEnvironmentFixture(); + const previousCwd = process.cwd(); + const logger = { + error: vi.fn(), + log: vi.fn(), + }; + const exit = vi.spyOn(process, "exit").mockImplementation(() => undefined as never); + const evaluation = makeEvaluation("demo-eval"); + + await writeFile(join(fixtureRoot, "package.json"), '{ "name": "@acme/agent" }\n'); + process.chdir(fixtureRoot); + mockedEvalDependencies.discoverAndImportEvals.mockResolvedValue([evaluation]); + mockedEvalDependencies.discoverEvalConfig.mockResolvedValue(TEST_CONFIG); + mockedEvalDependencies.resolveEvalTargetHandle.mockResolvedValue({ + attachSession: vi.fn(), + capabilities: { devRoutes: false }, + dispatchSchedule: vi.fn(), + fetch: vi.fn(), + kind: "remote", + url: "https://example.com", + }); + mockedEvalDependencies.executeEval.mockResolvedValue(makeEvalResult(evaluation.id)); + + try { + await runCli(["eval", "--url", "https://example.com"], logger); + } finally { + process.chdir(previousCwd); + } + + expect(mockedEvalDependencies.resolveEvalTargetHandle).toHaveBeenCalledWith( + expect.objectContaining({ + expectedAgentName: "agent", + }), + ); + expect(exit).toHaveBeenCalledWith(0); + }); + it("loads local env files before resolving a remote target", async () => { const fixtureRoot = await createEnvironmentFixture(); const resolvedFixtureRoot = await realpath(fixtureRoot);