[WIP] Add support for eval-level default expect checks

docs/guide/writing-evals.md

@@ -98,6 +98,11 @@ cases:
     collect_artifacts:            # Glob patterns selecting workspace files to download (see below)
       - "**/*.json"
       - "report/**"
+    expect:                       # Default expect checks applied to all cases (see Expect section)
+      exit_code: 0
+      must_not_contain:
+        - "TODO"
+        - "I cannot"
   parallelism: 2                  # Case parallelism, default 1
   retry_policy:
     max_retries: 1
@@ -487,8 +492,46 @@ expect:
     - "output.json"
   files_not_exist:              # Files that must not exist
     - "temp.log"
+  file_contains:                # Files must contain specific text
+    - path: "review.md"
+      content: "security"
+  golden_file: "expected.txt"   # Compare output to golden file
 ```
 
+#### Default expect checks
+
+Define common expect checks once in `cases.defaults.expect` instead of repeating them in every case:
+
+```yaml
+# In eval.yaml
+cases:
+  defaults:
+    expect:
+      exit_code: 0
+      must_not_contain:
+        - "TODO"
+        - "I cannot"
+      files_exist:
+        - "result.json"
+```
+
+Each case can add to or override these defaults:
+
+```yaml
+# In case.yaml
+expect:
+  must_contain:
+    - "SPECIAL_OK"              # Added to defaults
+  exit_code: 1                  # Overrides default exit_code
+```
+
+**Merge semantics:**
+
+- **Slice fields** (`must_contain`, `must_not_contain`, `files_exist`, `files_not_exist`, `file_contains`) are **appended** and de-duplicated (defaults first).
+- **Scalar fields** (`exit_code`, `golden_file`) are **overridden** by the case when set.
+- If no case-level `expect` is defined, defaults are used as-is.
+- The merged expect runs before judge; a failing merged expect skips judge execution.
+
 ### judge: rule_based — deterministic rules
 
 Decide pass/fail by declarative rules — fully deterministic and reproducible:

docs/zh/guide/writing-evals.md

@@ -97,6 +97,11 @@ cases:
     collect_artifacts:            # 用 glob 指定要下载的 workspace 产物文件（见下文）
       - "**/*.json"
       - "report/**"
+    expect:                       # 默认 expect 检查，应用于所有用例（见 Expect 章节）
+      exit_code: 0
+      must_not_contain:
+        - "TODO"
+        - "I cannot"
   parallelism: 2                  # 用例并行数，默认 1
   retry_policy:                   # 重试策略
     max_retries: 1
@@ -469,8 +474,46 @@ expect:
     - "output.json"
   files_not_exist:              # 期望这些文件不存在
     - "temp.log"
+  file_contains:                # 文件必须包含特定文本
+    - path: "review.md"
+      content: "security"
+  golden_file: "expected.txt"   # 与黄金文件比对输出
 ```
 
+#### 默认 expect 检查
+
+在 `cases.defaults.expect` 中一次性定义通用的 expect 检查，而不是在每个用例中重复：
+
+```yaml
+# 在 eval.yaml 中
+cases:
+  defaults:
+    expect:
+      exit_code: 0
+      must_not_contain:
+        - "TODO"
+        - "I cannot"
+      files_exist:
+        - "result.json"
+```
+
+每个用例可以添加或覆盖这些默认值：
+
+```yaml
+# 在 case.yaml 中
+expect:
+  must_contain:
+    - "SPECIAL_OK"              # 添加到默认检查
+  exit_code: 1                  # 覆盖默认的 exit_code
+```
+
+**合并语义：**
+
+- **切片字段**（`must_contain`、`must_not_contain`、`files_exist`、`files_not_exist`、`file_contains`）会被**追加**并去重（默认值在前）。
+- **标量字段**（`exit_code`、`golden_file`）在用例层面设置时会**覆盖**默认值。
+- 如果用例未定义 `expect`，则直接使用默认值。
+- 合并后的 expect 在 judge 之前运行；如果合并后的 expect 失败，会跳过 judge 执行。
+
 ### judge: rule_based — 确定性规则评估
 
 基于声明式规则判定用例是否通过，结果完全确定、可重复：

internal/config/schema.go

@@ -176,6 +176,11 @@ type CaseDefaults struct {
 	// path, regardless of whether the agent succeeded, failed, or timed out.
 	// Per-case CaseConfig.CollectArtifacts is merged (union) on top of this.
 	CollectArtifacts []string `yaml:"collect_artifacts,omitempty"`
+	// Expect provides default expect checks applied to all cases. Per-case
+	// Expect is merged on top: slice fields (must_contain, must_not_contain,
+	// files_exist, files_not_exist, file_contains) are appended; scalar fields
+	// (exit_code, golden_file) are overridden when set at the case level.
+	Expect Expect `yaml:"expect,omitempty"`
 }
 
 // RetryPolicy describes retry behavior.

internal/evaluator/evaluator.go

@@ -509,7 +509,11 @@ func (e *defaultEvaluator) runExpectPreCheck(
 	turnsTotal int,
 	result *EvalResult,
 ) bool {
-	expectCfg := resolveExpectConfig(&caseCfg.Expect)
+	// Merge default expect with case-level expect
+	defaultExpect := e.evalCfg.Cases.Defaults.Expect
+	mergedExpect := mergeExpectConfig(&defaultExpect, &caseCfg.Expect)
+
+	expectCfg := resolveExpectConfig(&mergedExpect)
 	if expectCfg == nil {
 		return false
 	}
@@ -809,6 +813,106 @@ func resolveExpectConfig(expectCfg *config.Expect) *config.Expect {
 	return expectCfg
 }
 
+// mergeExpectConfig merges default expect checks with case-level expect.
+// Slice fields (must_contain, must_not_contain, files_exist, files_not_exist,
+// file_contains) are appended and de-duplicated. Scalar fields (exit_code,
+// golden_file) are overridden by the case when set.
+func mergeExpectConfig(defaults, caseExpect *config.Expect) config.Expect {
+	if defaults == nil && caseExpect == nil {
+		return config.Expect{}
+	}
+	if defaults == nil {
+		return *caseExpect
+	}
+	if caseExpect == nil {
+		return *defaults
+	}
+
+	merged := config.Expect{}
+
+	// Merge slice fields with de-duplication
+	merged.MustContain = mergeAndDeduplicate(defaults.MustContain, caseExpect.MustContain)
+	merged.MustNotContain = mergeAndDeduplicate(defaults.MustNotContain, caseExpect.MustNotContain)
+	merged.FilesExist = mergeAndDeduplicate(defaults.FilesExist, caseExpect.FilesExist)
+	merged.FilesNotExist = mergeAndDeduplicate(defaults.FilesNotExist, caseExpect.FilesNotExist)
+	merged.FileContains = mergeFileContains(defaults.FileContains, caseExpect.FileContains)
+
+	// Scalar fields: case overrides default when set
+	if caseExpect.ExitCode != nil {
+		merged.ExitCode = caseExpect.ExitCode
+	} else {
+		merged.ExitCode = defaults.ExitCode
+	}
+
+	if caseExpect.GoldenFile != "" {
+		merged.GoldenFile = caseExpect.GoldenFile
+	} else {
+		merged.GoldenFile = defaults.GoldenFile
+	}
+
+	return merged
+}
+
+// mergeAndDeduplicate merges two string slices and removes duplicates.
+func mergeAndDeduplicate(a, b []string) []string {
+	if len(a) == 0 && len(b) == 0 {
+		return nil
+	}
+
+	seen := make(map[string]bool)
+	var result []string
+
+	for _, s := range a {
+		if !seen[s] {
+			seen[s] = true
+			result = append(result, s)
+		}
+	}
+
+	for _, s := range b {
+		if !seen[s] {
+			seen[s] = true
+			result = append(result, s)
+		}
+	}
+
+	return result
+}
+
+// mergeFileContains merges two FileContainsCheck slices.
+// Duplicates (same path+content) are removed.
+func mergeFileContains(a, b []config.FileContainsCheck) []config.FileContainsCheck {
+	if len(a) == 0 && len(b) == 0 {
+		return nil
+	}
+
+	type key struct {
+		path    string
+		content string
+	}
+
+	seen := make(map[key]bool)
+	var result []config.FileContainsCheck
+
+	for _, check := range a {
+		k := key{path: check.Path, content: check.Content}
+		if !seen[k] {
+			seen[k] = true
+			result = append(result, check)
+		}
+	}
+
+	for _, check := range b {
+		k := key{path: check.Path, content: check.Content}
+		if !seen[k] {
+			seen[k] = true
+			result = append(result, check)
+		}
+	}
+
+	return result
+}
+
 func (e *defaultEvaluator) prepareRuntimeForCase(ctx context.Context, caseCfg *config.CaseConfig, configName string, ag agent.Agent) (runtime.Runtime, error) {
 	rtCfg := e.evalCfg.Environment.ToRuntimeConfig()
 	rtCfg.Delete = e.deleteWorkspace