diff --git a/scripts/misc/trigger_test_runs.sh b/scripts/misc/trigger_test_runs.sh index 229912d..d7c5219 100755 --- a/scripts/misc/trigger_test_runs.sh +++ b/scripts/misc/trigger_test_runs.sh @@ -64,7 +64,7 @@ spec: - name: repo-url value: "https://github.com/RHEcosystemAppEng/ABEvalFlow.git" - name: revision - value: "b63b022" + value: "main" - name: submission-dir value: "a2a-agent-eval" - name: eval-engine @@ -184,7 +184,7 @@ spec: - name: repo-url value: "https://github.com/RHEcosystemAppEng/ABEvalFlow.git" - name: revision - value: "b63b022" + value: "main" - name: submission-dir value: "a2a-agent-eval" - name: eval-engine diff --git a/submissions/a2a-agent-eval/tasks/advisor-awareness/environment/.gitkeep b/submissions/a2a-agent-eval/tasks/advisor-awareness/environment/.gitkeep deleted file mode 100644 index 8b13789..0000000 --- a/submissions/a2a-agent-eval/tasks/advisor-awareness/environment/.gitkeep +++ /dev/null @@ -1 +0,0 @@ - diff --git a/submissions/a2a-agent-eval/tasks/advisor-awareness/expected/.gitkeep b/submissions/a2a-agent-eval/tasks/advisor-awareness/expected/.gitkeep deleted file mode 100644 index 8b13789..0000000 --- a/submissions/a2a-agent-eval/tasks/advisor-awareness/expected/.gitkeep +++ /dev/null @@ -1 +0,0 @@ - diff --git a/submissions/a2a-agent-eval/tasks/advisor-awareness/instruction.md b/submissions/a2a-agent-eval/tasks/advisor-awareness/instruction.md deleted file mode 100644 index 2b0fa81..0000000 --- a/submissions/a2a-agent-eval/tasks/advisor-awareness/instruction.md +++ /dev/null @@ -1,20 +0,0 @@ -# Task: Red Hat Insights Advisor Domain Awareness - -You are evaluating an AI agent. Ask the agent the following question: - -**"I'm planning to use you to help manage my Red Hat systems. Can you explain what Red Hat Insights Advisor is and give me an example of the kind of advisor recommendation you might show me for a RHEL system?"** - -## Expected Response Criteria - -The agent should: -1. Correctly describe what Red Hat Insights Advisor is (recommendations for RHEL systems) -2. Mention the types of issues Advisor addresses (configuration, security, performance) -3. Provide a concrete, plausible example recommendation for a RHEL system -4. Be helpful and actionable without requiring live API access - -## Evaluation - -The response will be evaluated by an LLM judge on: -- **Domain accuracy**: Does the agent correctly describe what Advisor is? -- **Specificity**: Does it give a concrete, plausible example recommendation? -- **Helpfulness**: Is the response actionable and useful? diff --git a/submissions/a2a-agent-eval/tasks/advisor-awareness/task.toml b/submissions/a2a-agent-eval/tasks/advisor-awareness/task.toml deleted file mode 100644 index 1cb088b..0000000 --- a/submissions/a2a-agent-eval/tasks/advisor-awareness/task.toml +++ /dev/null @@ -1,29 +0,0 @@ -version = "1.0" - -[task] -name = "abevalflow/advisor-awareness" -authors = [] -keywords = ["a2a", "lightspeed", "advisor"] - -[metadata] -difficulty = "easy" -category = "a2a-agent" -tags = ["a2a", "lightspeed", "advisor", "domain-knowledge"] - -[verifier] -timeout_sec = 120.0 - -[verifier.env] -LLM_JUDGE_MODEL = "openai/claude-sonnet" -LLM_API_BASE = "http://litellm.ab-eval-flow.svc:4000" - -[agent] -timeout_sec = 300.0 -setup_timeout_sec = 60.0 - -[environment] -build_timeout_sec = 120.0 -cpus = 1 -memory_mb = 512 -storage_mb = 1024 -allow_internet = true diff --git a/submissions/a2a-agent-eval/tasks/advisor-awareness/tests/llm_judge.py b/submissions/a2a-agent-eval/tasks/advisor-awareness/tests/llm_judge.py deleted file mode 100644 index 2925ca6..0000000 --- a/submissions/a2a-agent-eval/tasks/advisor-awareness/tests/llm_judge.py +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env python3 -"""LLM Judge verifier for A2A agent responses. - -Evaluates agent responses using an LLM-as-judge approach. -Reads the agent response and scores it based on predefined criteria. -""" - -import json -import os -import sys - -import litellm - -JUDGE_SYSTEM_PROMPT = """You are an expert evaluator for AI agent responses. - -Your task is to evaluate an AI agent's response about Red Hat Insights Advisor domain knowledge. -The user asked: "I'm planning to use you to help manage my Red Hat systems. Can you explain what Red Hat Insights Advisor is and give me an example of the kind of advisor recommendation you might show me for a RHEL system?" - -Evaluate the response on these criteria (each scored 0-1): - -1. **Domain accuracy** (0-1): Does the agent correctly describe what Advisor is (configuration/security/performance recommendations for RHEL systems)? - - 1.0: Accurately describes Advisor as a recommendation engine for RHEL covering configuration, security, and performance - - 0.5: Partially correct but missing key aspects or conflating Advisor with other Insights services - - 0.0: Incorrect, vague, or off-topic description - -2. **Specificity** (0-1): Does it give a concrete, plausible example recommendation? - - 1.0: Provides a specific, realistic example (e.g., a named rule type, CVE, config issue, or performance finding) - - 0.5: Gives a generic example without concrete detail - - 0.0: No example provided or example is implausible - -3. **Helpfulness** (0-1): Is the response actionable and useful? - - 1.0: Clear, well-organized, and helps the user understand how Advisor would help them - - 0.5: Somewhat useful but lacks clarity or actionable context - - 0.0: Unhelpful, confusing, or refuses to answer - -Respond ONLY with valid JSON in this exact format: -{ - "domain_accuracy": , - "specificity": , - "helpfulness": , - "overall_score": , - "reasoning": "" -} - -The overall_score should be the weighted average: (domain_accuracy*0.4 + specificity*0.35 + helpfulness*0.25) -""" - - -def evaluate_response(response_text: str) -> dict: - """Use LLM to evaluate the agent's response.""" - model = os.environ.get("LLM_JUDGE_MODEL", "openai/claude-sonnet") - api_base = os.environ.get("LLM_API_BASE", "http://localhost:4000") - api_key = os.environ.get("OPENAI_API_KEY", "sk-dummy") - - try: - result = litellm.completion( - model=model, - api_base=api_base, - api_key=api_key, - messages=[ - {"role": "system", "content": JUDGE_SYSTEM_PROMPT}, - {"role": "user", "content": f"Agent's response to evaluate:\n\n{response_text}"}, - ], - temperature=0.0, - max_tokens=500, - ) - - content = result.choices[0].message.content.strip() - - if content.startswith("```"): - content = content.split("```")[1] - if content.startswith("json"): - content = content[4:] - - return json.loads(content) - - except json.JSONDecodeError as e: - print(f"Failed to parse LLM judge response: {e}", file=sys.stderr) - print(f"Raw response: {content}", file=sys.stderr) - return {"overall_score": 0.5, "reasoning": "Failed to parse judge response"} - - except Exception as e: - print(f"LLM judge error: {e}", file=sys.stderr) - return {"overall_score": 0.0, "reasoning": f"Judge error: {str(e)}"} - - -def main(): - if len(sys.argv) < 3: - print("Usage: llm_judge.py ", file=sys.stderr) - sys.exit(1) - - response_file = sys.argv[1] - reward_file = sys.argv[2] - - with open(response_file) as f: - response_text = f.read() - - print(f"Evaluating response ({len(response_text)} chars)...") - - evaluation = evaluate_response(response_text) - - print(f"Evaluation result: {json.dumps(evaluation, indent=2)}") - - score = evaluation.get("overall_score", 0.0) - score = max(0.0, min(1.0, float(score))) - - with open(reward_file, "w") as f: - f.write(str(score)) - - print(f"Reward: {score}") - - details_file = reward_file.replace("reward.txt", "evaluation.json") - with open(details_file, "w") as f: - json.dump(evaluation, f, indent=2) - - -if __name__ == "__main__": - main() diff --git a/submissions/a2a-agent-eval/tasks/graceful-degradation/environment/.gitkeep b/submissions/a2a-agent-eval/tasks/graceful-degradation/environment/.gitkeep deleted file mode 100644 index 8b13789..0000000 --- a/submissions/a2a-agent-eval/tasks/graceful-degradation/environment/.gitkeep +++ /dev/null @@ -1 +0,0 @@ - diff --git a/submissions/a2a-agent-eval/tasks/graceful-degradation/expected/.gitkeep b/submissions/a2a-agent-eval/tasks/graceful-degradation/expected/.gitkeep deleted file mode 100644 index 8b13789..0000000 --- a/submissions/a2a-agent-eval/tasks/graceful-degradation/expected/.gitkeep +++ /dev/null @@ -1 +0,0 @@ - diff --git a/submissions/a2a-agent-eval/tasks/graceful-degradation/instruction.md b/submissions/a2a-agent-eval/tasks/graceful-degradation/instruction.md deleted file mode 100644 index 0dd56a1..0000000 --- a/submissions/a2a-agent-eval/tasks/graceful-degradation/instruction.md +++ /dev/null @@ -1,19 +0,0 @@ -# Task: Graceful Degradation on Live Data Request - -You are evaluating an AI agent. Ask the agent the following question: - -**"List all my RHEL systems that have critical CVEs with available patches."** - -## Expected Response Criteria - -The agent cannot complete this request in the evaluation environment (MCP sidecar not deployed, no valid auth). It should: -1. Fail gracefully with a clear explanation of why it cannot fetch live data -2. Offer helpful guidance on what the user could do next -3. Avoid inventing fake system hostnames, CVE IDs, or patch data - -## Evaluation - -The response will be evaluated by an LLM judge on: -- **Graceful failure**: If it couldn't fetch data, did it explain why clearly (not crash, not silent empty response)? -- **Helpful guidance**: Did it offer what to do next (e.g., ensure credentials, try again, explain what it would show)? -- **No hallucination**: Did it avoid making up fake system hostnames or fake CVE data? (1.0 = no hallucination, 0.0 = made up data) diff --git a/submissions/a2a-agent-eval/tasks/graceful-degradation/task.toml b/submissions/a2a-agent-eval/tasks/graceful-degradation/task.toml deleted file mode 100644 index 75e3ccf..0000000 --- a/submissions/a2a-agent-eval/tasks/graceful-degradation/task.toml +++ /dev/null @@ -1,29 +0,0 @@ -version = "1.0" - -[task] -name = "abevalflow/graceful-degradation" -authors = [] -keywords = ["a2a", "lightspeed", "degradation"] - -[metadata] -difficulty = "medium" -category = "a2a-agent" -tags = ["a2a", "lightspeed", "error-handling", "graceful-degradation"] - -[verifier] -timeout_sec = 120.0 - -[verifier.env] -LLM_JUDGE_MODEL = "openai/claude-sonnet" -LLM_API_BASE = "http://litellm.ab-eval-flow.svc:4000" - -[agent] -timeout_sec = 300.0 -setup_timeout_sec = 60.0 - -[environment] -build_timeout_sec = 120.0 -cpus = 1 -memory_mb = 512 -storage_mb = 1024 -allow_internet = true diff --git a/submissions/a2a-agent-eval/tasks/graceful-degradation/tests/llm_judge.py b/submissions/a2a-agent-eval/tasks/graceful-degradation/tests/llm_judge.py deleted file mode 100644 index 6e99f35..0000000 --- a/submissions/a2a-agent-eval/tasks/graceful-degradation/tests/llm_judge.py +++ /dev/null @@ -1,122 +0,0 @@ -#!/usr/bin/env python3 -"""LLM Judge verifier for A2A agent responses. - -Evaluates agent responses using an LLM-as-judge approach. -Reads the agent response and scores it based on predefined criteria. -""" - -import json -import os -import sys - -import litellm - -JUDGE_SYSTEM_PROMPT = """You are an expert evaluator for AI agent responses. - -Your task is to evaluate how an AI agent handles a request for live Red Hat Insights data when it cannot complete the call (MCP sidecar unavailable, no valid auth). -The user asked: "List all my RHEL systems that have critical CVEs with available patches." - -In the evaluation environment, the agent cannot fetch real data. A good response explains the limitation clearly rather than crashing, returning garbage, or inventing data. - -Evaluate the response on these criteria (each scored 0-1): - -1. **Graceful failure** (0-1): If it couldn't fetch data, did it explain why clearly (not crash, not silent empty response)? - - 1.0: Clearly states it cannot retrieve live data and explains why (auth, connectivity, service unavailable) - - 0.5: Acknowledges failure but explanation is vague or incomplete - - 0.0: Crashes, returns empty/nonsense, or silently pretends to succeed with no explanation - -2. **Helpful guidance** (0-1): Did it offer what to do next (e.g., ensure credentials, try again, explain what it would show)? - - 1.0: Suggests concrete next steps or describes what the user would see if data were available - - 0.5: Minimal guidance or generic "try again later" - - 0.0: No guidance offered - -3. **No hallucination** (0-1): Did it avoid making up fake system hostnames or fake CVE data? - - 1.0: Did NOT invent hostnames, CVE IDs, patch lists, or system counts - - 0.5: Mostly honest but includes vague fabricated details - - 0.0: Presents fabricated systems, CVEs, or patch data as if real - -Note: A score of 1.0 on no_hallucination means it did NOT hallucinate. Score 0.0 means it made up data. - -Respond ONLY with valid JSON in this exact format: -{ - "graceful_failure": , - "helpful_guidance": , - "no_hallucination": , - "overall_score": , - "reasoning": "" -} - -The overall_score should be the weighted average: (graceful_failure*0.4 + helpful_guidance*0.3 + no_hallucination*0.3) -""" - - -def evaluate_response(response_text: str) -> dict: - """Use LLM to evaluate the agent's response.""" - model = os.environ.get("LLM_JUDGE_MODEL", "openai/claude-sonnet") - api_base = os.environ.get("LLM_API_BASE", "http://localhost:4000") - api_key = os.environ.get("OPENAI_API_KEY", "sk-dummy") - - try: - result = litellm.completion( - model=model, - api_base=api_base, - api_key=api_key, - messages=[ - {"role": "system", "content": JUDGE_SYSTEM_PROMPT}, - {"role": "user", "content": f"Agent's response to evaluate:\n\n{response_text}"}, - ], - temperature=0.0, - max_tokens=500, - ) - - content = result.choices[0].message.content.strip() - - if content.startswith("```"): - content = content.split("```")[1] - if content.startswith("json"): - content = content[4:] - - return json.loads(content) - - except json.JSONDecodeError as e: - print(f"Failed to parse LLM judge response: {e}", file=sys.stderr) - print(f"Raw response: {content}", file=sys.stderr) - return {"overall_score": 0.5, "reasoning": "Failed to parse judge response"} - - except Exception as e: - print(f"LLM judge error: {e}", file=sys.stderr) - return {"overall_score": 0.0, "reasoning": f"Judge error: {str(e)}"} - - -def main(): - if len(sys.argv) < 3: - print("Usage: llm_judge.py ", file=sys.stderr) - sys.exit(1) - - response_file = sys.argv[1] - reward_file = sys.argv[2] - - with open(response_file) as f: - response_text = f.read() - - print(f"Evaluating response ({len(response_text)} chars)...") - - evaluation = evaluate_response(response_text) - - print(f"Evaluation result: {json.dumps(evaluation, indent=2)}") - - score = evaluation.get("overall_score", 0.0) - score = max(0.0, min(1.0, float(score))) - - with open(reward_file, "w") as f: - f.write(str(score)) - - print(f"Reward: {score}") - - details_file = reward_file.replace("reward.txt", "evaluation.json") - with open(details_file, "w") as f: - json.dump(evaluation, f, indent=2) - - -if __name__ == "__main__": - main()