From a5fbf068c82431d5ce388071b16eaa2d694a33cd Mon Sep 17 00:00:00 2001 From: gziv Date: Mon, 1 Jun 2026 13:18:38 +0300 Subject: [PATCH 1/6] feat: convert hello-world to ASE eval (full - with evals.json) Co-authored-by: Cursor --- submissions/hello-world/evals/evals.json | 16 ++++++++++++++++ submissions/hello-world/metadata.yaml | 8 ++------ 2 files changed, 18 insertions(+), 6 deletions(-) create mode 100644 submissions/hello-world/evals/evals.json diff --git a/submissions/hello-world/evals/evals.json b/submissions/hello-world/evals/evals.json new file mode 100644 index 0000000..1e9f1c3 --- /dev/null +++ b/submissions/hello-world/evals/evals.json @@ -0,0 +1,16 @@ +{ + "skill_name": "hello-world", + "evals": [ + { + "id": "hello-world-eval", + "name": "File Creation Knowledge", + "prompt": "A user asks you to create a file called hello.txt with 'Hello, world!' as the content.\n\nExplain how you would accomplish this task. Describe the file path, the exact content, and the method you would use to create the file.", + "expected_output": "A response that describes creating a file named hello.txt with the exact content 'Hello, world!' (including the comma and exclamation mark).", + "assertions": [ + "The output specifies creating a file named 'hello.txt' in the current directory.", + "The output includes the exact content 'Hello, world!' with proper punctuation (comma after Hello, exclamation mark at end).", + "The output describes a method to create the file (e.g., using a text editor, echo command, write function, or file creation API)." + ] + } + ] +} diff --git a/submissions/hello-world/metadata.yaml b/submissions/hello-world/metadata.yaml index 238c9fe..3c593b2 100644 --- a/submissions/hello-world/metadata.yaml +++ b/submissions/hello-world/metadata.yaml @@ -1,12 +1,8 @@ name: hello-world description: "Create a file with specific content -- minimal smoke-test submission" persona: general -version: "0.1.3" +version: "0.2.0" tags: - smoke-test - trivial -cpus: 1 -memory_mb: 512 -storage_mb: 1024 -experiment: - n_trials: 1 +eval_engine: ase From b1f47cd3eb91dee919d4615291aeaa173d6fed67 Mon Sep 17 00:00:00 2001 From: gziv Date: Mon, 1 Jun 2026 13:24:20 +0300 Subject: [PATCH 2/6] trigger: re-run pipeline Co-authored-by: Cursor From a49650e1d50016c09bf397d99f02fe2b2bd3d0c5 Mon Sep 17 00:00:00 2001 From: gziv Date: Mon, 1 Jun 2026 13:27:02 +0300 Subject: [PATCH 3/6] feat: add hello-world-full ASE submission (with evals.json) Co-authored-by: Cursor --- submissions/hello-world-full/evals/evals.json | 16 ++++++++++++++++ submissions/hello-world-full/metadata.yaml | 8 ++++++++ submissions/hello-world-full/skills/SKILL.md | 13 +++++++++++++ 3 files changed, 37 insertions(+) create mode 100644 submissions/hello-world-full/evals/evals.json create mode 100644 submissions/hello-world-full/metadata.yaml create mode 100644 submissions/hello-world-full/skills/SKILL.md diff --git a/submissions/hello-world-full/evals/evals.json b/submissions/hello-world-full/evals/evals.json new file mode 100644 index 0000000..25d776e --- /dev/null +++ b/submissions/hello-world-full/evals/evals.json @@ -0,0 +1,16 @@ +{ + "skill_name": "hello-world", + "evals": [ + { + "id": "hello-world-eval", + "name": "File Creation Knowledge", + "prompt": "A user asks you to create a file called hello.txt with 'Hello, world!' as the content.\n\nExplain how you would accomplish this task. Describe the file path, the exact content, and the method you would use to create the file.", + "expected_output": "A response that describes creating a file named hello.txt with the exact content 'Hello, world!' (including the comma and exclamation mark).", + "assertions": [ + "The output specifies creating a file named 'hello.txt' in the current directory.", + "The output includes the exact content 'Hello, world!' with proper punctuation (comma after Hello, exclamation mark at end).", + "The output describes a method to create the file (e.g., using a text editor, echo command, write function, or file creation API)." + ] + } + ] +} diff --git a/submissions/hello-world-full/metadata.yaml b/submissions/hello-world-full/metadata.yaml new file mode 100644 index 0000000..b5656c3 --- /dev/null +++ b/submissions/hello-world-full/metadata.yaml @@ -0,0 +1,8 @@ +name: hello-world-full +description: "Create a file with specific content -- ASE smoke-test (with evals.json)" +persona: general +version: "0.1.0" +tags: + - smoke-test + - ase +eval_engine: ase diff --git a/submissions/hello-world-full/skills/SKILL.md b/submissions/hello-world-full/skills/SKILL.md new file mode 100644 index 0000000..4836731 --- /dev/null +++ b/submissions/hello-world-full/skills/SKILL.md @@ -0,0 +1,13 @@ +# Hello World Skill + +A trivial file-creation skill used as a smoke test for the ABEvalFlow pipeline. + +## What the agent must do + +Given an instruction to create a file with specific content, the agent should +produce the correct file at the correct path with the exact expected content. + +## Evaluation criteria + +- File exists at the specified path +- File content matches the expected string exactly (including punctuation) From 6229dcdce53b3ac318b02a99752168b2ce1f2fda Mon Sep 17 00:00:00 2001 From: gziv Date: Mon, 1 Jun 2026 14:06:11 +0300 Subject: [PATCH 4/6] trigger: re-run with ASE evals generation support Co-authored-by: Cursor From 861a85f78e258164968ac0ff5e7f40179da17d0d Mon Sep 17 00:00:00 2001 From: gziv Date: Mon, 1 Jun 2026 14:10:33 +0300 Subject: [PATCH 5/6] trigger: use feature branch for pipeline code Co-authored-by: Cursor From a59fdf8723bf8a2cf0a5014daed50a6e43abecd4 Mon Sep 17 00:00:00 2001 From: gziv Date: Mon, 1 Jun 2026 14:15:33 +0300 Subject: [PATCH 6/6] fix: add YAML frontmatter to SKILL.md Co-authored-by: Cursor --- submissions/hello-world-full/skills/SKILL.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/submissions/hello-world-full/skills/SKILL.md b/submissions/hello-world-full/skills/SKILL.md index 4836731..eed9419 100644 --- a/submissions/hello-world-full/skills/SKILL.md +++ b/submissions/hello-world-full/skills/SKILL.md @@ -1,3 +1,8 @@ +--- +name: hello-world +description: A trivial file-creation skill for smoke testing +--- + # Hello World Skill A trivial file-creation skill used as a smoke test for the ABEvalFlow pipeline.