Novel LLM evaluation using semantic extraction + deterministic scoring
Problem: LLM-as-judge is inconsistent. Traditional parsing misses semantic content.
Solution: Use LLM to extract factual, observable properties from responses, then apply deterministic scoring rules.
# ❌ Traditional approaches
rating = gpt4("Rate this response 1-10") # Inconsistent LLM scoring
mentions = "photosynthesis" in response # Dumb keyword matching
# ✅ Our approach: LLM extraction + deterministic scoring
extracted = llm.extract(response, {
"biological_processes_mentioned": ["photosynthesis", "respiration"],
"examples_count": 3,
"question_marks_count": 2,
"scientific_terms_mentioned": ["ATP", "chlorophyll"],
"first_person_pronouns_count": 1
})
score = calculate_score(extracted, weights={
"process_coverage": 0.3,
"example_density": 0.2,
"question_factor": -0.1,
"terminology_score": 0.3,
"pronoun_factor": 0.1
}) # Deterministic math on LLM-extracted data- Simple Context: LLM analyzes response content for factual properties
- Leverages: LLM's semantic understanding capabilities
- Avoids: LLM's inconsistent scoring behavior
- Output: Structured data that can be scored consistently
- Input: Structured data from LLM extraction
- Process: Mathematical rules applied consistently
- Output: Transparent, reproducible scores
- Semantic Understanding: LLM extracts meaningful content insights
- Neutral Analysis: LLM analyzes content for factual, observable properties
- Reliable Scoring: Mathematical rules ensure consistency
- Separated Logic: Extraction and scoring are completely separate processes
from evaluat3 import Evaluator
# Define what to extract and how to score it
config = {
"extraction_schema": {
"scientific_concepts_mentioned": "list[str]",
"examples_provided_count": "int",
"unclear_statements_count": "int",
"technical_vocabulary_used": "list[str]",
"informal_language_count": "int",
"questions_asked_count": "int"
},
"scoring_rules": {
"concept_coverage": {
"calculation": "min(1.0, len(scientific_concepts_mentioned) / 5)",
"weight": 0.25
},
"example_factor": {
"calculation": "min(1.0, examples_provided_count / 3)",
"weight": 0.2
},
"clarity_factor": {
"calculation": "max(0.0, 1.0 - unclear_statements_count / 5)",
"weight": 0.15
},
"vocabulary_score": {
"calculation": "min(1.0, len(technical_vocabulary_used) / 6)",
"weight": 0.25
},
"engagement_bonus": {
"calculation": "min(1.0, questions_asked_count / 2)",
"weight": 0.1
},
"formality_adjustment": {
"calculation": "max(0.0, 1.0 - informal_language_count / 4)",
"weight": 0.05
}
}
}
evaluator = Evaluator(config)
result = evaluator.evaluate("Explain photosynthesis", llm_response)# Simple content analysis request
extraction_prompt = f"""
Analyze the following response and extract factual information:
Response about photosynthesis:
"{llm_response}"
Extract the following data:
- scientific_concepts_mentioned: [list the biological concepts mentioned by name]
- examples_provided_count: count of specific examples given
- unclear_statements_count: count of statements that lack precision
- technical_vocabulary_used: [list of scientific/technical terms used]
- informal_language_count: count of casual or conversational expressions
- questions_asked_count: count of questions posed in the response
Return as JSON.
"""
extracted_data = llm.extract(extraction_prompt)
# Returns: {
# "scientific_concepts_mentioned": ["photosynthesis", "chloroplasts", "glucose production"],
# "examples_provided_count": 2,
# "unclear_statements_count": 1,
# "technical_vocabulary_used": ["ATP", "chlorophyll", "carbon dioxide"],
# "informal_language_count": 2,
# "questions_asked_count": 0
# }# Apply mathematical rules to extracted data (separate from extraction)
scores = {}
# Concept coverage: 3 concepts mentioned, normalized to max of 5
scores["concept_coverage"] = min(1.0, len(extracted_data["scientific_concepts_mentioned"]) / 5) * 0.25
# Example factor: 2 examples given, normalized to max of 3
scores["example_factor"] = min(1.0, extracted_data["examples_provided_count"] / 3) * 0.2
# Clarity factor: fewer unclear statements = higher score
scores["clarity_factor"] = max(0.0, 1.0 - extracted_data["unclear_statements_count"] / 5) * 0.15
# Vocabulary score: 3 technical terms, normalized to max of 6
scores["vocabulary_score"] = min(1.0, len(extracted_data["technical_vocabulary_used"]) / 6) * 0.25
# Engagement bonus: questions show engagement (could be good or bad)
scores["engagement_bonus"] = min(1.0, extracted_data["questions_asked_count"] / 2) * 0.1
# Formality adjustment: informal language impact (direction unclear)
scores["formality_adjustment"] = max(0.0, 1.0 - extracted_data["informal_language_count"] / 4) * 0.05
final_score = sum(scores.values()) # Pure mathematics, separate processresult = {
"extracted_data": {
"scientific_concepts_mentioned": ["photosynthesis", "chloroplasts", "glucose production"],
"examples_provided_count": 2,
"unclear_statements_count": 1,
"technical_vocabulary_used": ["ATP", "chlorophyll", "carbon dioxide"],
"informal_language_count": 2,
"questions_asked_count": 0
},
"scoring_breakdown": {
"concept_coverage": 0.15, # (3/5) * 0.25
"example_factor": 0.13, # (2/3) * 0.2
"clarity_factor": 0.12, # (1.0 - 1/5) * 0.15
"vocabulary_score": 0.125, # (3/6) * 0.25
"engagement_bonus": 0.0, # (0/2) * 0.1
"formality_adjustment": 0.025 # (1.0 - 2/4) * 0.05
},
"final_score": 0.56,
"explanation": "Content analysis: 3 concepts, 2 examples, moderate clarity, technical vocabulary used"
}- LLM extracts meaningful content insights through natural content analysis
- Understands context, relationships, and nuanced content patterns
- Same extracted data always produces the same score
- Mathematical rules eliminate evaluator bias and variance
- Metrics are genuinely factual and observable
- No obvious "good" or "bad" direction for most properties
- Works for any type of response content (writing, code, math, etc.)
- Factual extraction request with no evaluation context
- Metrics have ambiguous optimization direction
- Scoring weights and formulas are separate from extraction
- Works across all content types (not just text analysis)
- Leverages LLM capabilities without LLM scoring inconsistency
- Fast deterministic scoring once data is extracted
| Approach | Semantic Understanding | Scoring Consistency | Transparency | Gaming Resistance |
|---|---|---|---|---|
| LLM-as-Judge | ✅ High | ❌ Very Low | ❌ Black Box | ❌ Low |
| Traditional Parsing | ❌ None | ✅ Perfect | ✅ Full | |
| Human Evaluation | ✅ High | ✅ High | ||
| Evaluat3 | ✅ High | ✅ Perfect | ✅ Full | ✅ High |
Define what semantic information to extract:
extraction_schemas = {
"technical_content": {
"technical_concepts_mentioned": "list[str]",
"code_blocks_count": "int",
"word_count": "int",
"programming_languages_mentioned": "list[str]",
"questions_posed_count": "int"
},
"support_response": {
"emotions_referenced": "list[str]",
"numbered_steps_count": "int",
"action_words_used": "list[str]",
"contact_information_provided": "list[str]",
"time_estimates_given": "list[str]"
},
"mathematical_solution": {
"formulas_presented": "list[str]",
"calculation_steps_count": "int",
"variables_defined": "list[str]",
"units_specified": "list[str]",
"verification_methods_count": "int"
}
}Deterministic mathematical rules applied to extracted data:
scoring_rules = {
"concept_density": {
"calculation": "min(1.0, len(technical_concepts_mentioned) / 5)",
"weight": 0.4
},
"word_count_score": {
"calculation": "min(1.0, explanation_word_count / 200)",
"weight": 0.3
},
"code_presence_score": {
"calculation": "min(1.0, code_blocks_count / 2)",
"weight": 0.3
}
}tests/
├── gitignored_tests/ # Real test data (not in git)
│ ├── customer_service/
│ ├── technical_docs/
│ └── content_creation/
├── extraction_schemas/ # What to extract (in git)
│ ├── customer_service.json
│ └── technical_docs.json
└── scoring_rules/ # How to score (in git)
├── customer_service.json
└── technical_docs.json
Original Response:
"I understand you're frustrated with the billing error. Here's how to resolve it: 1) Log into your account, 2) Go to billing disputes, 3) Upload your receipt. This should be processed within 2-3 business days. If you need immediate help, call our support line at 1-800-HELP."
LLM Extraction:
{
"customer_emotions_acknowledged": ["frustrated"],
"solution_steps_count": 3,
"action_verbs_used": ["log", "go", "upload", "call"],
"time_references_given": ["2-3 business days"],
"contact_methods_mentioned": ["phone", "support line"],
"numbers_mentioned": ["1-800-HELP"]
}Deterministic Scoring:
scores = {
"emotion_recognition": len(customer_emotions_acknowledged) * 0.2, # 1 emotion acknowledged
"action_density": min(1.0, len(action_verbs_used) / 4) * 0.3, # 4 action verbs used
"reference_completeness": len(time_references_given) * 0.2, # 1 time reference given
"contact_information": len(contact_methods_mentioned) * 0.3 # 2 contact methods mentioned
}
final_score = sum(scores.values()) # Mathematical sum of extracted features🔬 Research Scientist? → Psychology Study Example
💼 Product Manager? → Customer Support Example
🤖 AI Developer? → MCP Integration Example
⚙️ DevOps Engineer? → CI/CD Integration Example
from evaluat3 import Evaluator
# Use a preset for common use cases
evaluator = Evaluator.from_preset("customer_support")
# Or define your own schema
config = {
"extraction_schema": {
"technical_concepts": "list[str]",
"examples_count": "int",
"questions_asked": "int"
},
"scoring_rules": {
"concept_coverage": {
"calculation": "min(1.0, len(technical_concepts) / 5)",
"weight": 0.4
},
"example_richness": {
"calculation": "min(1.0, examples_count / 3)",
"weight": 0.3
},
"engagement_factor": {
"calculation": "min(1.0, questions_asked / 2)",
"weight": 0.3
}
}
}
evaluator = Evaluator(config)
result = evaluator.evaluate(prompt, llm_response)
print(f"Score: {result.final_score}")
print(f"Breakdown: {result.scoring_breakdown}")All examples are living tests - they prove the system works:
- 📁 Browse Examples - See it working for different use cases
- 🌐 Try Web Demo - Interactive playground
- 📚 Read Documentation - User-specific guides
We welcome contributions! The key areas:
- Extraction Schema Patterns: Common patterns for different domains
- Scoring Rule Templates: Reusable mathematical scoring approaches
- Domain-Specific Configs: Pre-built configs for common use cases
MIT License - see LICENSE for details.
Evaluat3: Semantic understanding meets consistent scoring.