diff --git a/submissions/hello-world-full/evals/evals.json b/submissions/hello-world-full/evals/evals.json new file mode 100644 index 0000000..25d776e --- /dev/null +++ b/submissions/hello-world-full/evals/evals.json @@ -0,0 +1,16 @@ +{ + "skill_name": "hello-world", + "evals": [ + { + "id": "hello-world-eval", + "name": "File Creation Knowledge", + "prompt": "A user asks you to create a file called hello.txt with 'Hello, world!' as the content.\n\nExplain how you would accomplish this task. Describe the file path, the exact content, and the method you would use to create the file.", + "expected_output": "A response that describes creating a file named hello.txt with the exact content 'Hello, world!' (including the comma and exclamation mark).", + "assertions": [ + "The output specifies creating a file named 'hello.txt' in the current directory.", + "The output includes the exact content 'Hello, world!' with proper punctuation (comma after Hello, exclamation mark at end).", + "The output describes a method to create the file (e.g., using a text editor, echo command, write function, or file creation API)." + ] + } + ] +} diff --git a/submissions/hello-world-full/metadata.yaml b/submissions/hello-world-full/metadata.yaml new file mode 100644 index 0000000..b5656c3 --- /dev/null +++ b/submissions/hello-world-full/metadata.yaml @@ -0,0 +1,8 @@ +name: hello-world-full +description: "Create a file with specific content -- ASE smoke-test (with evals.json)" +persona: general +version: "0.1.0" +tags: + - smoke-test + - ase +eval_engine: ase diff --git a/submissions/hello-world-full/skills/SKILL.md b/submissions/hello-world-full/skills/SKILL.md new file mode 100644 index 0000000..eed9419 --- /dev/null +++ b/submissions/hello-world-full/skills/SKILL.md @@ -0,0 +1,18 @@ +--- +name: hello-world +description: A trivial file-creation skill for smoke testing +--- + +# Hello World Skill + +A trivial file-creation skill used as a smoke test for the ABEvalFlow pipeline. + +## What the agent must do + +Given an instruction to create a file with specific content, the agent should +produce the correct file at the correct path with the exact expected content. + +## Evaluation criteria + +- File exists at the specified path +- File content matches the expected string exactly (including punctuation) diff --git a/submissions/hello-world/evals/evals.json b/submissions/hello-world/evals/evals.json new file mode 100644 index 0000000..1e9f1c3 --- /dev/null +++ b/submissions/hello-world/evals/evals.json @@ -0,0 +1,16 @@ +{ + "skill_name": "hello-world", + "evals": [ + { + "id": "hello-world-eval", + "name": "File Creation Knowledge", + "prompt": "A user asks you to create a file called hello.txt with 'Hello, world!' as the content.\n\nExplain how you would accomplish this task. Describe the file path, the exact content, and the method you would use to create the file.", + "expected_output": "A response that describes creating a file named hello.txt with the exact content 'Hello, world!' (including the comma and exclamation mark).", + "assertions": [ + "The output specifies creating a file named 'hello.txt' in the current directory.", + "The output includes the exact content 'Hello, world!' with proper punctuation (comma after Hello, exclamation mark at end).", + "The output describes a method to create the file (e.g., using a text editor, echo command, write function, or file creation API)." + ] + } + ] +} diff --git a/submissions/hello-world/metadata.yaml b/submissions/hello-world/metadata.yaml index 238c9fe..3c593b2 100644 --- a/submissions/hello-world/metadata.yaml +++ b/submissions/hello-world/metadata.yaml @@ -1,12 +1,8 @@ name: hello-world description: "Create a file with specific content -- minimal smoke-test submission" persona: general -version: "0.1.3" +version: "0.2.0" tags: - smoke-test - trivial -cpus: 1 -memory_mb: 512 -storage_mb: 1024 -experiment: - n_trials: 1 +eval_engine: ase