diff --git a/docs/guide/writing-evals.md b/docs/guide/writing-evals.md index 232c358a..af0b08be 100644 --- a/docs/guide/writing-evals.md +++ b/docs/guide/writing-evals.md @@ -98,6 +98,11 @@ cases: collect_artifacts: # Glob patterns selecting workspace files to download (see below) - "**/*.json" - "report/**" + expect: # Default expect checks applied to all cases (see Expect section) + exit_code: 0 + must_not_contain: + - "TODO" + - "I cannot" parallelism: 2 # Case parallelism, default 1 retry_policy: max_retries: 1 @@ -487,8 +492,46 @@ expect: - "output.json" files_not_exist: # Files that must not exist - "temp.log" + file_contains: # Files must contain specific text + - path: "review.md" + content: "security" + golden_file: "expected.txt" # Compare output to golden file ``` +#### Default expect checks + +Define common expect checks once in `cases.defaults.expect` instead of repeating them in every case: + +```yaml +# In eval.yaml +cases: + defaults: + expect: + exit_code: 0 + must_not_contain: + - "TODO" + - "I cannot" + files_exist: + - "result.json" +``` + +Each case can add to or override these defaults: + +```yaml +# In case.yaml +expect: + must_contain: + - "SPECIAL_OK" # Added to defaults + exit_code: 1 # Overrides default exit_code +``` + +**Merge semantics:** + +- **Slice fields** (`must_contain`, `must_not_contain`, `files_exist`, `files_not_exist`, `file_contains`) are **appended** and de-duplicated (defaults first). +- **Scalar fields** (`exit_code`, `golden_file`) are **overridden** by the case when set. +- If no case-level `expect` is defined, defaults are used as-is. +- The merged expect runs before judge; a failing merged expect skips judge execution. + ### judge: rule_based — deterministic rules Decide pass/fail by declarative rules — fully deterministic and reproducible: diff --git a/docs/zh/guide/writing-evals.md b/docs/zh/guide/writing-evals.md index 25663ea6..63b4b94b 100644 --- a/docs/zh/guide/writing-evals.md +++ b/docs/zh/guide/writing-evals.md @@ -97,6 +97,11 @@ cases: collect_artifacts: # 用 glob 指定要下载的 workspace 产物文件(见下文) - "**/*.json" - "report/**" + expect: # 默认 expect 检查,应用于所有用例(见 Expect 章节) + exit_code: 0 + must_not_contain: + - "TODO" + - "I cannot" parallelism: 2 # 用例并行数,默认 1 retry_policy: # 重试策略 max_retries: 1 @@ -469,8 +474,46 @@ expect: - "output.json" files_not_exist: # 期望这些文件不存在 - "temp.log" + file_contains: # 文件必须包含特定文本 + - path: "review.md" + content: "security" + golden_file: "expected.txt" # 与黄金文件比对输出 ``` +#### 默认 expect 检查 + +在 `cases.defaults.expect` 中一次性定义通用的 expect 检查,而不是在每个用例中重复: + +```yaml +# 在 eval.yaml 中 +cases: + defaults: + expect: + exit_code: 0 + must_not_contain: + - "TODO" + - "I cannot" + files_exist: + - "result.json" +``` + +每个用例可以添加或覆盖这些默认值: + +```yaml +# 在 case.yaml 中 +expect: + must_contain: + - "SPECIAL_OK" # 添加到默认检查 + exit_code: 1 # 覆盖默认的 exit_code +``` + +**合并语义:** + +- **切片字段**(`must_contain`、`must_not_contain`、`files_exist`、`files_not_exist`、`file_contains`)会被**追加**并去重(默认值在前)。 +- **标量字段**(`exit_code`、`golden_file`)在用例层面设置时会**覆盖**默认值。 +- 如果用例未定义 `expect`,则直接使用默认值。 +- 合并后的 expect 在 judge 之前运行;如果合并后的 expect 失败,会跳过 judge 执行。 + ### judge: rule_based — 确定性规则评估 基于声明式规则判定用例是否通过,结果完全确定、可重复: diff --git a/internal/config/schema.go b/internal/config/schema.go index f59169ec..9b62713e 100644 --- a/internal/config/schema.go +++ b/internal/config/schema.go @@ -176,6 +176,11 @@ type CaseDefaults struct { // path, regardless of whether the agent succeeded, failed, or timed out. // Per-case CaseConfig.CollectArtifacts is merged (union) on top of this. CollectArtifacts []string `yaml:"collect_artifacts,omitempty"` + // Expect provides default expect checks applied to all cases. Per-case + // Expect is merged on top: slice fields (must_contain, must_not_contain, + // files_exist, files_not_exist, file_contains) are appended; scalar fields + // (exit_code, golden_file) are overridden when set at the case level. + Expect Expect `yaml:"expect,omitempty"` } // RetryPolicy describes retry behavior. diff --git a/internal/evaluator/evaluator.go b/internal/evaluator/evaluator.go index bd790537..a013badb 100644 --- a/internal/evaluator/evaluator.go +++ b/internal/evaluator/evaluator.go @@ -509,7 +509,11 @@ func (e *defaultEvaluator) runExpectPreCheck( turnsTotal int, result *EvalResult, ) bool { - expectCfg := resolveExpectConfig(&caseCfg.Expect) + // Merge default expect with case-level expect + defaultExpect := e.evalCfg.Cases.Defaults.Expect + mergedExpect := mergeExpectConfig(&defaultExpect, &caseCfg.Expect) + + expectCfg := resolveExpectConfig(&mergedExpect) if expectCfg == nil { return false } @@ -809,6 +813,106 @@ func resolveExpectConfig(expectCfg *config.Expect) *config.Expect { return expectCfg } +// mergeExpectConfig merges default expect checks with case-level expect. +// Slice fields (must_contain, must_not_contain, files_exist, files_not_exist, +// file_contains) are appended and de-duplicated. Scalar fields (exit_code, +// golden_file) are overridden by the case when set. +func mergeExpectConfig(defaults, caseExpect *config.Expect) config.Expect { + if defaults == nil && caseExpect == nil { + return config.Expect{} + } + if defaults == nil { + return *caseExpect + } + if caseExpect == nil { + return *defaults + } + + merged := config.Expect{} + + // Merge slice fields with de-duplication + merged.MustContain = mergeAndDeduplicate(defaults.MustContain, caseExpect.MustContain) + merged.MustNotContain = mergeAndDeduplicate(defaults.MustNotContain, caseExpect.MustNotContain) + merged.FilesExist = mergeAndDeduplicate(defaults.FilesExist, caseExpect.FilesExist) + merged.FilesNotExist = mergeAndDeduplicate(defaults.FilesNotExist, caseExpect.FilesNotExist) + merged.FileContains = mergeFileContains(defaults.FileContains, caseExpect.FileContains) + + // Scalar fields: case overrides default when set + if caseExpect.ExitCode != nil { + merged.ExitCode = caseExpect.ExitCode + } else { + merged.ExitCode = defaults.ExitCode + } + + if caseExpect.GoldenFile != "" { + merged.GoldenFile = caseExpect.GoldenFile + } else { + merged.GoldenFile = defaults.GoldenFile + } + + return merged +} + +// mergeAndDeduplicate merges two string slices and removes duplicates. +func mergeAndDeduplicate(a, b []string) []string { + if len(a) == 0 && len(b) == 0 { + return nil + } + + seen := make(map[string]bool) + var result []string + + for _, s := range a { + if !seen[s] { + seen[s] = true + result = append(result, s) + } + } + + for _, s := range b { + if !seen[s] { + seen[s] = true + result = append(result, s) + } + } + + return result +} + +// mergeFileContains merges two FileContainsCheck slices. +// Duplicates (same path+content) are removed. +func mergeFileContains(a, b []config.FileContainsCheck) []config.FileContainsCheck { + if len(a) == 0 && len(b) == 0 { + return nil + } + + type key struct { + path string + content string + } + + seen := make(map[key]bool) + var result []config.FileContainsCheck + + for _, check := range a { + k := key{path: check.Path, content: check.Content} + if !seen[k] { + seen[k] = true + result = append(result, check) + } + } + + for _, check := range b { + k := key{path: check.Path, content: check.Content} + if !seen[k] { + seen[k] = true + result = append(result, check) + } + } + + return result +} + func (e *defaultEvaluator) prepareRuntimeForCase(ctx context.Context, caseCfg *config.CaseConfig, configName string, ag agent.Agent) (runtime.Runtime, error) { rtCfg := e.evalCfg.Environment.ToRuntimeConfig() rtCfg.Delete = e.deleteWorkspace diff --git a/internal/evaluator/evaluator_test.go b/internal/evaluator/evaluator_test.go index 6dfe377c..c4124a45 100644 --- a/internal/evaluator/evaluator_test.go +++ b/internal/evaluator/evaluator_test.go @@ -1175,6 +1175,125 @@ func TestExecuteCase_ExpectPass_ThenJudge(t *testing.T) { } } +func TestExecuteCase_DefaultExpect_AppliedToCase(t *testing.T) { + // Test that default expect checks from CaseDefaults are applied + e := newTestEvaluator(EvalOptions{ + Agent: &mockAgent{name: "test", output: "no default keyword"}, + EvalCfg: &config.EvalConfig{ + Cases: config.CasesConfig{ + Defaults: config.CaseDefaults{ + Expect: config.Expect{ + MustContain: []string{"default-keyword"}, + }, + }, + }, + }, + }) + + caseCfg := &config.CaseConfig{ + ID: "case-default-expect", + Title: "Default Expect Test", + Input: config.Input{Prompt: "test"}, + } + + result := e.executeCase(context.Background(), caseCfg, "with_skill", &mockRuntime{workspace: "/tmp/test"}, nil) + + if result.Status != judge.StatusFail { + t.Errorf("expected FAIL status, got %s", result.Status) + } + if result.ExpectResult == nil { + t.Fatal("expected non-nil ExpectResult") + } + if result.ExpectResult.Passed { + t.Error("expected expect check to fail due to default expect") + } +} + +func TestExecuteCase_DefaultExpect_MergedWithCaseExpect(t *testing.T) { + // Test that default and case-level expects are merged + e := newTestEvaluator(EvalOptions{ + Agent: &mockAgent{name: "test", output: "has default-keyword and case-keyword"}, + EvalCfg: &config.EvalConfig{ + Cases: config.CasesConfig{ + Defaults: config.CaseDefaults{ + Expect: config.Expect{ + MustContain: []string{"default-keyword"}, + }, + }, + }, + }, + }) + + caseCfg := &config.CaseConfig{ + ID: "case-merged-expect", + Title: "Merged Expect Test", + Input: config.Input{Prompt: "test"}, + Expect: config.Expect{ + MustContain: []string{"case-keyword"}, + }, + } + + result := e.executeCase(context.Background(), caseCfg, "with_skill", &mockRuntime{workspace: "/tmp/test"}, nil) + + if result.Status != judge.StatusPass { + t.Errorf("expected PASS status, got %s", result.Status) + } + if result.ExpectResult == nil { + t.Fatal("expected non-nil ExpectResult") + } + if !result.ExpectResult.Passed { + t.Error("expected both default and case expect checks to pass") + } +} + +func TestExecuteCase_DefaultExpect_CaseOverridesExitCode(t *testing.T) { + // Test that case-level exit_code overrides default + exitCode0 := 0 + exitCode1 := 1 + + e := newTestEvaluator(EvalOptions{ + Agent: &mockAgent{ + name: "test", + runFunc: func(ctx context.Context, rt runtime.Runtime, opts agent.ExecOptions, messages []transcript.Message) (*agent.SessionResult, error) { + return &agent.SessionResult{ + FinalMessage: "result", + ExitCode: 1, + }, nil + }, + }, + EvalCfg: &config.EvalConfig{ + Cases: config.CasesConfig{ + Defaults: config.CaseDefaults{ + Expect: config.Expect{ + ExitCode: &exitCode0, + }, + }, + }, + }, + }) + + caseCfg := &config.CaseConfig{ + ID: "case-override-exitcode", + Title: "Override Exit Code Test", + Input: config.Input{Prompt: "test"}, + Expect: config.Expect{ + ExitCode: &exitCode1, // Case expects exit 1, should pass + }, + } + + result := e.executeCase(context.Background(), caseCfg, "with_skill", &mockRuntime{workspace: "/tmp/test"}, nil) + + if result.Status != judge.StatusPass { + t.Errorf("expected PASS status (case exit_code overrides default), got %s", result.Status) + } + if result.ExpectResult == nil { + t.Fatal("expected non-nil ExpectResult") + } + if !result.ExpectResult.Passed { + t.Error("expected expect check to pass with case-level exit_code override") + } +} + func TestExecuteCase_RetryPolicyRetriesErrors(t *testing.T) { t.Parallel() @@ -2100,6 +2219,376 @@ func TestResolveExpectConfig(t *testing.T) { } } +//nolint:funlen // Comprehensive table-driven test with multiple test cases +func TestMergeExpectConfig(t *testing.T) { + t.Parallel() + + exitCode0 := 0 + exitCode1 := 1 + + tests := []struct { + name string + defaults *config.Expect + caseExp *config.Expect + expected config.Expect + }{ + { + name: "nil defaults returns case expect", + defaults: nil, + caseExp: &config.Expect{ + MustContain: []string{"foo"}, + ExitCode: &exitCode0, + }, + expected: config.Expect{ + MustContain: []string{"foo"}, + ExitCode: &exitCode0, + }, + }, + { + name: "empty case returns defaults", + defaults: &config.Expect{ + MustContain: []string{"default"}, + ExitCode: &exitCode0, + }, + caseExp: &config.Expect{}, + expected: config.Expect{ + MustContain: []string{"default"}, + ExitCode: &exitCode0, + }, + }, + { + name: "slice fields are appended", + defaults: &config.Expect{ + MustContain: []string{"default1"}, + MustNotContain: []string{"bad1"}, + FilesExist: []string{"file1.txt"}, + }, + caseExp: &config.Expect{ + MustContain: []string{"case1"}, + MustNotContain: []string{"bad2"}, + FilesExist: []string{"file2.txt"}, + }, + expected: config.Expect{ + MustContain: []string{"default1", "case1"}, + MustNotContain: []string{"bad1", "bad2"}, + FilesExist: []string{"file1.txt", "file2.txt"}, + }, + }, + { + name: "duplicate strings are removed", + defaults: &config.Expect{ + MustContain: []string{"foo", "bar"}, + FilesExist: []string{"a.txt", "b.txt"}, + }, + caseExp: &config.Expect{ + MustContain: []string{"foo", "baz"}, + FilesExist: []string{"b.txt", "c.txt"}, + }, + expected: config.Expect{ + MustContain: []string{"foo", "bar", "baz"}, + FilesExist: []string{"a.txt", "b.txt", "c.txt"}, + }, + }, + { + name: "scalar fields: case overrides default exit_code", + defaults: &config.Expect{ + ExitCode: &exitCode0, + }, + caseExp: &config.Expect{ + ExitCode: &exitCode1, + }, + expected: config.Expect{ + ExitCode: &exitCode1, + }, + }, + { + name: "scalar fields: default exit_code when case not set", + defaults: &config.Expect{ + ExitCode: &exitCode0, + }, + caseExp: &config.Expect{}, + expected: config.Expect{ + ExitCode: &exitCode0, + }, + }, + { + name: "scalar fields: case golden_file overrides default", + defaults: &config.Expect{ + GoldenFile: "default.txt", + }, + caseExp: &config.Expect{ + GoldenFile: "case.txt", + }, + expected: config.Expect{ + GoldenFile: "case.txt", + }, + }, + { + name: "scalar fields: default golden_file when case not set", + defaults: &config.Expect{ + GoldenFile: "default.txt", + }, + caseExp: &config.Expect{}, + expected: config.Expect{ + GoldenFile: "default.txt", + }, + }, + { + name: "file_contains merges and deduplicates", + defaults: &config.Expect{ + FileContains: []config.FileContainsCheck{ + {Path: "a.txt", Content: "foo"}, + {Path: "b.txt", Content: "bar"}, + }, + }, + caseExp: &config.Expect{ + FileContains: []config.FileContainsCheck{ + {Path: "a.txt", Content: "foo"}, // duplicate + {Path: "c.txt", Content: "baz"}, + }, + }, + expected: config.Expect{ + FileContains: []config.FileContainsCheck{ + {Path: "a.txt", Content: "foo"}, + {Path: "b.txt", Content: "bar"}, + {Path: "c.txt", Content: "baz"}, + }, + }, + }, + { + name: "complex merge: all fields", + defaults: &config.Expect{ + MustContain: []string{"default"}, + MustNotContain: []string{"bad"}, + ExitCode: &exitCode0, + FilesExist: []string{"default.txt"}, + FilesNotExist: []string{"bad.txt"}, + GoldenFile: "default_golden.txt", + FileContains: []config.FileContainsCheck{{Path: "x.txt", Content: "y"}}, + }, + caseExp: &config.Expect{ + MustContain: []string{"case"}, + ExitCode: &exitCode1, + FilesExist: []string{"case.txt"}, + FilesNotExist: []string{"unwanted.txt"}, + FileContains: []config.FileContainsCheck{{Path: "z.txt", Content: "w"}}, + }, + expected: config.Expect{ + MustContain: []string{"default", "case"}, + MustNotContain: []string{"bad"}, + ExitCode: &exitCode1, // case overrides + FilesExist: []string{"default.txt", "case.txt"}, + FilesNotExist: []string{"bad.txt", "unwanted.txt"}, + GoldenFile: "default_golden.txt", // case didn't set, default remains + FileContains: []config.FileContainsCheck{ + {Path: "x.txt", Content: "y"}, + {Path: "z.txt", Content: "w"}, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + result := mergeExpectConfig(tt.defaults, tt.caseExp) + + // Compare slices + if !stringSlicesEqual(result.MustContain, tt.expected.MustContain) { + t.Errorf("MustContain: got %v, want %v", result.MustContain, tt.expected.MustContain) + } + if !stringSlicesEqual(result.MustNotContain, tt.expected.MustNotContain) { + t.Errorf("MustNotContain: got %v, want %v", result.MustNotContain, tt.expected.MustNotContain) + } + if !stringSlicesEqual(result.FilesExist, tt.expected.FilesExist) { + t.Errorf("FilesExist: got %v, want %v", result.FilesExist, tt.expected.FilesExist) + } + if !stringSlicesEqual(result.FilesNotExist, tt.expected.FilesNotExist) { + t.Errorf("FilesNotExist: got %v, want %v", result.FilesNotExist, tt.expected.FilesNotExist) + } + + // Compare scalars + if (result.ExitCode == nil) != (tt.expected.ExitCode == nil) { + t.Errorf("ExitCode nil mismatch: got %v, want %v", result.ExitCode, tt.expected.ExitCode) + } else if result.ExitCode != nil && *result.ExitCode != *tt.expected.ExitCode { + t.Errorf("ExitCode: got %d, want %d", *result.ExitCode, *tt.expected.ExitCode) + } + + if result.GoldenFile != tt.expected.GoldenFile { + t.Errorf("GoldenFile: got %s, want %s", result.GoldenFile, tt.expected.GoldenFile) + } + + // Compare FileContains + if !fileContainsSlicesEqual(result.FileContains, tt.expected.FileContains) { + t.Errorf("FileContains: got %v, want %v", result.FileContains, tt.expected.FileContains) + } + }) + } +} + +func TestMergeAndDeduplicate(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + a []string + b []string + expected []string + }{ + { + name: "both nil", + a: nil, + b: nil, + expected: nil, + }, + { + name: "both empty", + a: []string{}, + b: []string{}, + expected: nil, + }, + { + name: "a nil, b has items", + a: nil, + b: []string{"x", "y"}, + expected: []string{"x", "y"}, + }, + { + name: "a has items, b nil", + a: []string{"a", "b"}, + b: nil, + expected: []string{"a", "b"}, + }, + { + name: "no duplicates", + a: []string{"a", "b"}, + b: []string{"c", "d"}, + expected: []string{"a", "b", "c", "d"}, + }, + { + name: "with duplicates", + a: []string{"a", "b", "c"}, + b: []string{"b", "c", "d"}, + expected: []string{"a", "b", "c", "d"}, + }, + { + name: "all duplicates", + a: []string{"x", "y"}, + b: []string{"x", "y"}, + expected: []string{"x", "y"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + result := mergeAndDeduplicate(tt.a, tt.b) + if !stringSlicesEqual(result, tt.expected) { + t.Errorf("got %v, want %v", result, tt.expected) + } + }) + } +} + +func TestMergeFileContains(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + a []config.FileContainsCheck + b []config.FileContainsCheck + expected []config.FileContainsCheck + }{ + { + name: "both nil", + a: nil, + b: nil, + expected: nil, + }, + { + name: "both empty", + a: []config.FileContainsCheck{}, + b: []config.FileContainsCheck{}, + expected: nil, + }, + { + name: "no duplicates", + a: []config.FileContainsCheck{ + {Path: "a.txt", Content: "foo"}, + }, + b: []config.FileContainsCheck{ + {Path: "b.txt", Content: "bar"}, + }, + expected: []config.FileContainsCheck{ + {Path: "a.txt", Content: "foo"}, + {Path: "b.txt", Content: "bar"}, + }, + }, + { + name: "exact duplicates", + a: []config.FileContainsCheck{ + {Path: "a.txt", Content: "foo"}, + {Path: "b.txt", Content: "bar"}, + }, + b: []config.FileContainsCheck{ + {Path: "a.txt", Content: "foo"}, + }, + expected: []config.FileContainsCheck{ + {Path: "a.txt", Content: "foo"}, + {Path: "b.txt", Content: "bar"}, + }, + }, + { + name: "same path different content", + a: []config.FileContainsCheck{ + {Path: "a.txt", Content: "foo"}, + }, + b: []config.FileContainsCheck{ + {Path: "a.txt", Content: "bar"}, + }, + expected: []config.FileContainsCheck{ + {Path: "a.txt", Content: "foo"}, + {Path: "a.txt", Content: "bar"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + result := mergeFileContains(tt.a, tt.b) + if !fileContainsSlicesEqual(result, tt.expected) { + t.Errorf("got %v, want %v", result, tt.expected) + } + }) + } +} + +// Helper functions for test comparisons. +func stringSlicesEqual(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func fileContainsSlicesEqual(a, b []config.FileContainsCheck) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i].Path != b[i].Path || a[i].Content != b[i].Content { + return false + } + } + return true +} + func TestEvalResult_Fields(t *testing.T) { r := EvalResult{ CaseID: "test-case", diff --git a/skills/skill-upper/references/case-yaml.md b/skills/skill-upper/references/case-yaml.md index cc6164e0..1eeeffff 100644 --- a/skills/skill-upper/references/case-yaml.md +++ b/skills/skill-upper/references/case-yaml.md @@ -88,10 +88,20 @@ expect: - "output.json" files_not_exist: - "temp.log" + file_contains: + - path: "review.md" + content: "security" + golden_file: "expected.txt" ``` expect 不通过时,judge 会被跳过。用它来快速过滤明显不合格的输出,节省 token。 +**与 `cases.defaults.expect` 合并:** + +- 如果 `eval.yaml` 的 `cases.defaults.expect` 定义了默认检查(如 `exit_code: 0`, `must_not_contain: ["TODO"]`),每个用例的 expect 会与之合并 +- 切片字段(`must_contain`, `must_not_contain`, `files_exist`, `files_not_exist`, `file_contains`)追加去重;标量字段(`exit_code`, `golden_file`)由用例覆盖 +- 用例未设 expect 时,直接使用默认值 + ## 常见写法 - **纯文本路由 Skill**:`expect.must_contain` + `judge.rule_based.output_contains` diff --git a/skills/skill-upper/references/eval-yaml.md b/skills/skill-upper/references/eval-yaml.md index efc3034b..2187bd96 100644 --- a/skills/skill-upper/references/eval-yaml.md +++ b/skills/skill-upper/references/eval-yaml.md @@ -37,6 +37,11 @@ cases: collect_artifacts: # 可选:用 glob 采集 workspace 产物文件 - "**/*.json" - "report/**" + expect: # 可选:默认 expect 检查,应用于所有用例 + exit_code: 0 + must_not_contain: + - "TODO" + - "I cannot" parallelism: 2 retry_policy: max_retries: 1