From aab7157f4ed0e98a24a7a4e09238ff98e4bea140 Mon Sep 17 00:00:00 2001 From: alfekka Date: Mon, 9 Jun 2025 00:39:57 +0300 Subject: [PATCH 01/39] refactoring metrics --- demo_all_metrics.py | 163 ++++++++++++++ demo_new_metrics.py | 97 +++++++++ src/atgen/metrics/bart_score.py | 118 ----------- src/atgen/metrics/base/__init__.py | 8 + src/atgen/metrics/base/base_metric.py | 70 ++++++ src/atgen/metrics/base/config.py | 0 src/atgen/metrics/lexical/__init__.py | 9 + src/atgen/metrics/lexical/bleu_metric.py | 95 +++++++++ src/atgen/metrics/lexical/rouge_metric.py | 72 +++++++ src/atgen/metrics/linguistic/__init__.py | 7 + src/atgen/metrics/linguistic/cola_metric.py | 118 +++++++++++ src/atgen/metrics/llm_based/__init__.py | 19 ++ .../llm_based/answer_relevancy_metric.py | 46 ++++ .../metrics/llm_based/base_deepeval_metric.py | 97 +++++++++ .../metrics/llm_based/bigbench_hard_metric.py | 200 ++++++++++++++++++ src/atgen/metrics/llm_based/evaluation_llm.py | 94 ++++++++ .../metrics/llm_based/faithfulness_metric.py | 51 +++++ .../llm_based/prompt_alignment_metric.py | 57 +++++ .../metrics/llm_based/summarization_metric.py | 46 ++++ src/atgen/metrics/semantic/__init__.py | 11 + .../metrics/semantic/alignscore_metric.py | 101 +++++++++ .../metrics/semantic/bart_score_metric.py | 153 ++++++++++++++ src/atgen/metrics/semantic/sentbert_metric.py | 142 +++++++++++++ 23 files changed, 1656 insertions(+), 118 deletions(-) create mode 100644 demo_all_metrics.py create mode 100644 demo_new_metrics.py delete mode 100644 src/atgen/metrics/bart_score.py create mode 100644 src/atgen/metrics/base/__init__.py create mode 100644 src/atgen/metrics/base/base_metric.py create mode 100644 src/atgen/metrics/base/config.py create mode 100644 src/atgen/metrics/lexical/__init__.py create mode 100644 src/atgen/metrics/lexical/bleu_metric.py create mode 100644 src/atgen/metrics/lexical/rouge_metric.py create mode 100644 src/atgen/metrics/linguistic/__init__.py create mode 100644 src/atgen/metrics/linguistic/cola_metric.py create mode 100644 src/atgen/metrics/llm_based/__init__.py create mode 100644 src/atgen/metrics/llm_based/answer_relevancy_metric.py create mode 100644 src/atgen/metrics/llm_based/base_deepeval_metric.py create mode 100644 src/atgen/metrics/llm_based/bigbench_hard_metric.py create mode 100644 src/atgen/metrics/llm_based/evaluation_llm.py create mode 100644 src/atgen/metrics/llm_based/faithfulness_metric.py create mode 100644 src/atgen/metrics/llm_based/prompt_alignment_metric.py create mode 100644 src/atgen/metrics/llm_based/summarization_metric.py create mode 100644 src/atgen/metrics/semantic/__init__.py create mode 100644 src/atgen/metrics/semantic/alignscore_metric.py create mode 100644 src/atgen/metrics/semantic/bart_score_metric.py create mode 100644 src/atgen/metrics/semantic/sentbert_metric.py diff --git a/demo_all_metrics.py b/demo_all_metrics.py new file mode 100644 index 0000000..bf0f057 --- /dev/null +++ b/demo_all_metrics.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +""" +Comprehensive demonstration of the refactored metrics system including DeepEval metrics. +""" + +import sys +import os +sys.path.append('src') + +from atgen.metrics.base import BaseMetric, MetricConfig +from atgen.metrics.lexical import BleuMetric, RougeMetric +from atgen.metrics.semantic import BartScoreMetric, AlignScoreMetric, SentBertMetric +from atgen.metrics.linguistic import ColaMetric +# Note: llm-based directory needs to be renamed to llm_based for Python imports +# from atgen.metrics.llm_based import ( +# DeepEvalAnswerRelevancyMetric, +# DeepEvalFaithfulnessMetric, +# DeepEvalSummarizationMetric, +# DeepEvalPromptAlignmentMetric, +# EvaluationLLM +# ) + + +def demo_all_metrics(): + """Demonstrate all metrics in the new system.""" + print("šŸŽÆ Comprehensive Metrics System Demonstration") + print("=" * 80) + + # Sample data + predictions = [ + "The cat is sitting on the mat peacefully.", + "Python is an excellent programming language for machine learning.", + "Deep learning has revolutionized artificial intelligence applications." + ] + + references = [ + "A cat is resting on the mat.", + "Python is great for ML development.", + "Deep learning has transformed AI." + ] + + original_texts = [ + "There is a cat on a mat", + "What makes Python good for ML?", + "How has deep learning impacted AI?" + ] + + # Create different configurations for different metric types + base_config = MetricConfig( + batch_size=2, + device="cpu", # Use CPU for demo + cache_dir="cache", + aggregate=True + ) + + # DeepEval config (requires API key) + deepeval_config = MetricConfig( + batch_size=2, + device="cpu", + cache_dir="cache", + aggregate=True, + api_key=os.getenv("OPENROUTER_API_KEY"), # Set this in environment + base_url="https://openrouter.ai/api/v1", + model="openai/gpt-4o-mini", # Cheaper model for demo + threshold=0.5, + include_reason=False, + strict_mode=False, + async_mode=False, # Synchronous for simpler demo + verbose_mode=False + ) + + # Organize metrics by category + metrics_by_category = { + "šŸ“ Lexical Metrics": [ + ("BLEU", BleuMetric(base_config)), + ("ROUGE", RougeMetric(base_config)), + ], + "šŸ§  Semantic Metrics": [ + ("SentenceBERT", SentBertMetric(base_config)), + # ("BARTScore", BartScoreMetric(base_config)), # Uncomment if you have dependencies + # ("AlignScore", AlignScoreMetric(base_config)), # Uncomment if you have dependencies + ], + "šŸ—£ļø Linguistic Quality": [ + ("CoLA (Grammaticality)", ColaMetric(base_config)), + ], + # "šŸ¤– LLM-based (DeepEval)": [ + # ("Answer Relevancy", DeepEvalAnswerRelevancyMetric(deepeval_config)), + # ("Faithfulness", DeepEvalFaithfulnessMetric(deepeval_config)), + # ("Summarization", DeepEvalSummarizationMetric(deepeval_config)), + # ("Prompt Alignment", DeepEvalPromptAlignmentMetric(deepeval_config)), + # ] + } + + print(f"šŸ“Š Testing with {len(predictions)} samples\n") + + # Test each category + for category, metrics in metrics_by_category.items(): + print(f"{category}") + print("-" * 60) + + for metric_name, metric in metrics: + print(f" šŸ” {metric_name} ({metric.__class__.__name__})") + print(f" Available: {metric.is_available()}") + + if metric.is_available(): + try: + # Configure test based on metric requirements + if "DeepEval" in metric.__class__.__name__: + if deepeval_config.api_key is None: + print(f" āš ļø API key required (set OPENROUTER_API_KEY)") + print() + continue + + # Different DeepEval metrics need different inputs + if "PromptAlignment" in metric.__class__.__name__: + results = metric.calculate(predictions, references, original_texts) + else: + results = metric.calculate(predictions, None, original_texts) + + elif metric_name == "CoLA (Grammaticality)": + # CoLA only needs predictions + results = metric.calculate(predictions) + + elif metric_name == "SentenceBERT": + # SentBERT can use both references and original texts + results = metric.calculate(predictions, references, original_texts) + + else: + # Standard metrics (BLEU, ROUGE) need references + results = metric.calculate(predictions, references) + + print(f" Results: {results}") + + except Exception as e: + print(f" āŒ Error: {e}") + else: + print(f" āš ļø Dependencies not available") + + print() + + print() + + print("šŸŒŸ System Architecture Highlights:") + print(" āœ… BaseMetric abstract class with consistent interface") + print(" āœ… MetricConfig for centralized configuration") + print(" āœ… Category-based organization (lexical, semantic, linguistic, llm-based)") + print(" āœ… Automatic dependency checking") + print(" āœ… Lazy model initialization") + print(" āœ… Proper error handling and logging") + print(" āœ… Support for multiple references") + print(" āœ… API-based metrics with custom LLM implementation") + print(" āœ… Configurable aggregation") + print(" āœ… Type safety with full type hints") + + print("\nšŸ”§ Next Steps:") + print(" ā€¢ Strategy factory pattern for metric selection") + print(" ā€¢ Configuration-based metric instantiation") + print(" ā€¢ Integration with existing compute_metrics.py") + print(" ā€¢ Performance optimization and caching") + + +if __name__ == "__main__": + demo_all_metrics() \ No newline at end of file diff --git a/demo_new_metrics.py b/demo_new_metrics.py new file mode 100644 index 0000000..e2b47d6 --- /dev/null +++ b/demo_new_metrics.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +""" +Demonstration of the new refactored metrics system. +""" + +import sys +import os +sys.path.append('src') + +from atgen.metrics.base import BaseMetric, MetricConfig +from atgen.metrics.lexical import BleuMetric, RougeMetric +from atgen.metrics.semantic import BartScoreMetric, AlignScoreMetric, SentBertMetric +from atgen.metrics.linguistic import ColaMetric + + +def demo_metrics(): + """Demonstrate the new metrics system.""" + print("šŸŽÆ New Metrics System Demonstration") + print("=" * 60) + + # Sample data + predictions = [ + "The cat is on the mat", + "I love programming in Python", + "Machine learning is fascinating" + ] + + references = [ + "A cat is sitting on the mat", + "I enjoy coding with Python", + "Machine learning is very interesting" + ] + + original_texts = [ + "There is a cat on a mat", + "Programming with Python is great", + "ML technology is amazing" + ] + + # Create metrics with custom config + config = MetricConfig( + batch_size=8, + device="cpu", # Use CPU for demo + cache_dir="cache", + aggregate=True + ) + + metrics = [ + ("BLEU", BleuMetric(config)), + ("ROUGE", RougeMetric(config)), + ("CoLA (Grammaticality)", ColaMetric(config)), + ("SentenceBERT", SentBertMetric(config)), + # ("BARTScore", BartScoreMetric(config)), # Uncomment if you have the dependencies + # ("AlignScore", AlignScoreMetric(config)), # Uncomment if you have the dependencies + ] + + print(f"šŸ“Š Testing with {len(predictions)} samples\n") + + for metric_name, metric in metrics: + print(f"šŸ” Testing {metric_name} ({metric.__class__.__name__})") + print(f" Available: {metric.is_available()}") + + if metric.is_available(): + try: + # Test different scenarios based on metric requirements + if metric_name == "CoLA (Grammaticality)": + # CoLA only needs predictions + results = metric.calculate(predictions) + elif metric_name == "SentenceBERT": + # SentBERT can use both references and original texts + results = metric.calculate(predictions, references, original_texts) + else: + # Standard metrics (BLEU, ROUGE) need references + results = metric.calculate(predictions, references) + + print(f" Results: {results}") + + except Exception as e: + print(f" āŒ Error: {e}") + else: + print(f" āš ļø Dependencies not available") + + print() + + print("šŸŒŸ Key Features of the New System:") + print(" āœ… Clean inheritance from BaseMetric") + print(" āœ… Consistent configuration via MetricConfig") + print(" āœ… Automatic dependency checking") + print(" āœ… Lazy model initialization") + print(" āœ… Proper error handling") + print(" āœ… Organized by category (lexical, semantic, linguistic)") + print(" āœ… Type hints throughout") + print(" āœ… Configurable aggregation") + + +if __name__ == "__main__": + demo_metrics() \ No newline at end of file diff --git a/src/atgen/metrics/bart_score.py b/src/atgen/metrics/bart_score.py deleted file mode 100644 index 1e8f808..0000000 --- a/src/atgen/metrics/bart_score.py +++ /dev/null @@ -1,118 +0,0 @@ -# %% -import traceback -from typing import List -from tqdm import tqdm - -import numpy as np -import torch -import torch.nn as nn -from transformers import BartTokenizer, BartForConditionalGeneration - - -class BARTScorer: - def __init__( - self, - device="cuda", # device="cuda:0", - max_length=1024, - checkpoint="facebook/bart-large-cnn", - cache_dir: str = "cache", - ): - # Set up model - self.device = device - self.max_length = max_length - self.tokenizer = BartTokenizer.from_pretrained(checkpoint, cache_dir=cache_dir) - self.model = BartForConditionalGeneration.from_pretrained( - checkpoint, cache_dir=cache_dir - ) - self.model.eval() - self.model.to(device) - - # Set up loss - self.loss_fct = nn.NLLLoss( - reduction="none", ignore_index=self.model.config.pad_token_id - ) - self.lsm = nn.LogSoftmax(dim=1) - - def load(self, path=None): - """Load model from paraphrase finetuning""" - if path is None: - path = "models/bart.pth" - self.model.load_state_dict(torch.load(path, map_location=self.device)) - - def score(self, srcs, tgts, batch_size=4): - """Score a batch of examples""" - score_list = [] - for i in tqdm(range(0, len(srcs), batch_size), desc="Calculating BARTScore..."): - src_list = srcs[i : i + batch_size] - tgt_list = tgts[i : i + batch_size] - try: - with torch.no_grad(): - encoded_src = self.tokenizer( - src_list, - max_length=self.max_length, - truncation=True, - padding=True, - return_tensors="pt", - ) - encoded_tgt = self.tokenizer( - tgt_list, - max_length=self.max_length, - truncation=True, - padding=True, - return_tensors="pt", - ) - src_tokens = encoded_src["input_ids"].to(self.device) - src_mask = encoded_src["attention_mask"].to(self.device) - - tgt_tokens = encoded_tgt["input_ids"].to(self.device) - tgt_mask = encoded_tgt["attention_mask"] - tgt_len = tgt_mask.sum(dim=1).to(self.device) - - output = self.model( - input_ids=src_tokens, attention_mask=src_mask, labels=tgt_tokens - ) - logits = output.logits.view(-1, self.model.config.vocab_size) - loss = self.loss_fct(self.lsm(logits), tgt_tokens.view(-1)) - loss = loss.view(tgt_tokens.shape[0], -1) - loss = loss.sum(dim=1) / tgt_len - curr_score_list = [-x.item() for x in loss] - score_list += curr_score_list - - except RuntimeError: - traceback.print_exc() - print(f"source: {src_list}") - print(f"target: {tgt_list}") - exit(0) - return score_list - - def multi_ref_score(self, srcs, tgts: List[List[str]], agg="mean", batch_size=4): - # Assert we have the same number of references - ref_nums = [len(x) for x in tgts] - if len(set(ref_nums)) > 1: - raise Exception("You have different number of references per test sample.") - - ref_num = len(tgts[0]) - score_matrix = [] - for i in range(ref_num): - curr_tgts = [x[i] for x in tgts] - scores = self.score(srcs, curr_tgts, batch_size) - score_matrix.append(scores) - if agg == "mean": - score_list = np.mean(score_matrix, axis=0) - elif agg == "max": - score_list = np.max(score_matrix, axis=0) - else: - raise NotImplementedError - return list(score_list) - - def test(self, batch_size=3): - """Test""" - src_list = [ - "This is a very good idea. Although simple, but very insightful.", - "Can I take a look?", - "Do not trust him, he is a liar.", - ] - - tgt_list = ["That's stupid.", "What's the problem?", "He is trustworthy."] - - print(self.score(src_list, tgt_list, batch_size)) diff --git a/src/atgen/metrics/base/__init__.py b/src/atgen/metrics/base/__init__.py new file mode 100644 index 0000000..a0f1510 --- /dev/null +++ b/src/atgen/metrics/base/__init__.py @@ -0,0 +1,8 @@ +"""Base classes and configuration for metrics.""" + +from .base_metric import BaseMetric, MetricConfig + +__all__ = [ + "BaseMetric", + "MetricConfig", +] \ No newline at end of file diff --git a/src/atgen/metrics/base/base_metric.py b/src/atgen/metrics/base/base_metric.py new file mode 100644 index 0000000..6d5802c --- /dev/null +++ b/src/atgen/metrics/base/base_metric.py @@ -0,0 +1,70 @@ +from abc import ABC, abstractmethod +from typing import Optional +import logging +from dataclasses import dataclass + + +@dataclass +class MetricConfig: + """Base configuration for metrics.""" + # General parameters + batch_size: int = 32 + device: str = "cuda" + cache_dir: str = "cache" + aggregate: bool = True + + # Model-specific parameters (for local models) + checkpoint: Optional[str] = None + + # API-based parameters (for LLM-based metrics) + api_key: Optional[str] = None + base_url: Optional[str] = None + model: Optional[str] = None + + # DeepEval specific parameters + threshold: float = 0.5 + include_reason: bool = False + strict_mode: bool = False + async_mode: bool = True + verbose_mode: bool = False + truths_extraction_limit: Optional[int] = None + + +class BaseMetric(ABC): + def __init__(self, config: Optional[MetricConfig] = None): + self.config = config or MetricConfig() + self.logger = logging.getLogger(self.__class__.__name__) + self._is_available = None + self._model = None + + # Validate configuration + self._validate_config() + + # Check dependencies + if not self.is_available(): + self.logger.warning(f"{self.__class__.__name__} dependencies not available") + + @property + def name(self) -> str: + """Return the metric name (defaults to class name).""" + return self.__class__.__name__.replace("Metric", "").lower() + + def _validate_config(self): + """Validate configuration - can be overridden by subclasses.""" + pass + + def is_available(self) -> bool: + """Check if dependencies are available.""" + if self._is_available is None: + self._is_available = self._check_dependencies() + return self._is_available + + @abstractmethod + def _check_dependencies(self) -> bool: + """Check if required dependencies are available.""" + pass + + @abstractmethod + def calculate(self, predictions, references, original_texts): + pass + \ No newline at end of file diff --git a/src/atgen/metrics/base/config.py b/src/atgen/metrics/base/config.py new file mode 100644 index 0000000..e69de29 diff --git a/src/atgen/metrics/lexical/__init__.py b/src/atgen/metrics/lexical/__init__.py new file mode 100644 index 0000000..cab0fff --- /dev/null +++ b/src/atgen/metrics/lexical/__init__.py @@ -0,0 +1,9 @@ +"""Lexical metrics for text evaluation.""" + +from .bleu_metric import BleuMetric +from .rouge_metric import RougeMetric + +__all__ = [ + "BleuMetric", + "RougeMetric", +] \ No newline at end of file diff --git a/src/atgen/metrics/lexical/bleu_metric.py b/src/atgen/metrics/lexical/bleu_metric.py new file mode 100644 index 0000000..458b6fd --- /dev/null +++ b/src/atgen/metrics/lexical/bleu_metric.py @@ -0,0 +1,95 @@ +from typing import Dict, List, Optional, Union +import numpy as np +from nltk import download +from nltk.tokenize import word_tokenize +from nltk.translate.bleu_score import corpus_bleu +import logging + +from ..base.base_metric import BaseMetric, MetricConfig + +# Ensure nltk data is downloaded +try: + word_tokenize("test") +except LookupError: + download('punkt') + + +def smoothing_function(p_n, references, hypothesis, hyp_len): + """ + Smooth-BLEU (BLEUS) as proposed in the paper: + Chin-Yew Lin, Franz Josef Och. ORANGE: a method for evaluating automatic + evaluation metrics for machine translation. COLING 2004. + """ + smoothed_p_n = [] + for i, p_i in enumerate(p_n, start=1): + # Smoothing is not applied for unigrams + if i > 1: + # If hypothesis length is lower than the current order, its value equals (0 + 1) / (0 + 1) = 0 + if hyp_len < i: + assert p_i.denominator == 1 + smoothed_p_n.append(1) + # Otherwise apply smoothing + else: + smoothed_p_i = (p_i.numerator + 1) / (p_i.denominator + 1) + smoothed_p_n.append(smoothed_p_i) + else: + smoothed_p_n.append(p_i) + return smoothed_p_n + + +class BleuMetric(BaseMetric): + """BLEU (Bilingual Evaluation Understudy) metric implementation.""" + + @property + def category(self) -> str: + return "lexical" + + @property + def supports_multiple_references(self) -> bool: + return True + + def _check_dependencies(self) -> bool: + """Check if NLTK is available.""" + try: + import nltk + # Try to use word_tokenize to ensure punkt is available + word_tokenize("test") + return True + except (ImportError, LookupError): + return False + + def _compute( + self, + predictions: List[str], + references: Optional[List[Union[str, List[str]]]] = None, + original_texts: Optional[List[str]] = None + ) -> Dict[str, Union[float, np.ndarray]]: + """ + Compute BLEU scores. + + Args: + predictions: List of predicted texts + references: List of reference texts (can be list of lists for multiple references) + original_texts: Not used for BLEU computation + + Returns: + Dictionary with BLEU scores + """ + scores = [] + + for pred, ref in zip(predictions, references): + # Ensure references is always a list of lists + if isinstance(ref, str): + ref = [ref] + + # Tokenize + tok_ref = [word_tokenize(r) for r in ref] + tok_pred = word_tokenize(pred) + + try: + score = corpus_bleu(tok_ref, [tok_pred], smoothing_function=smoothing_function) + scores.append(score) + except (KeyError, ZeroDivisionError): + scores.append(0.0) + + return {"bleu": np.array(scores)} \ No newline at end of file diff --git a/src/atgen/metrics/lexical/rouge_metric.py b/src/atgen/metrics/lexical/rouge_metric.py new file mode 100644 index 0000000..aa47246 --- /dev/null +++ b/src/atgen/metrics/lexical/rouge_metric.py @@ -0,0 +1,72 @@ +from typing import List, Dict, Union, Optional +import numpy as np +from evaluate import load + +from ..base.base_metric import BaseMetric, MetricConfig + + +class RougeMetric(BaseMetric): + """ROUGE (Recall-Oriented Understudy for Gisting Evaluation) metric.""" + + def __init__(self, config: Optional[MetricConfig] = None): + super().__init__(config) + self.rouge = None + + def _check_dependencies(self) -> bool: + """Check if ROUGE is available.""" + try: + from evaluate import load + return True + except ImportError: + return False + + def _initialize_rouge(self): + """Initialize ROUGE if not already initialized.""" + if self.rouge is None: + self.rouge = load("rouge", cache_dir=self.config.cache_dir) + + def calculate(self, predictions: List[str], references: Optional[List[Union[str, List[str]]]] = None, original_texts: Optional[List[str]] = None) -> Dict[str, float]: + """ + Calculate ROUGE scores. + + Args: + predictions: List of predicted texts + references: List of reference texts (can be list of lists for multiple references) + original_texts: Not used for ROUGE computation + + Returns: + Dictionary with ROUGE scores + """ + if not self.is_available(): + raise RuntimeError("ROUGE dependencies not available") + + if references is None: + raise ValueError("ROUGE requires reference texts") + + self._initialize_rouge() + + # ROUGE expects different format for multiple references + if isinstance(references[0], list): + # Multiple references - convert to format expected by ROUGE + rouge_references = references + else: + # Single references + rouge_references = references + + results = self.rouge.compute( + predictions=predictions, + references=rouge_references, + use_stemmer=True, + ) + + # Convert to float values + scores = {} + for key, value in results.items(): + if isinstance(value, (int, float)): + scores[key] = float(value) + elif hasattr(value, 'item'): # For numpy scalars + scores[key] = float(value.item()) + else: + scores[key] = float(value) + + return scores \ No newline at end of file diff --git a/src/atgen/metrics/linguistic/__init__.py b/src/atgen/metrics/linguistic/__init__.py new file mode 100644 index 0000000..b119e6d --- /dev/null +++ b/src/atgen/metrics/linguistic/__init__.py @@ -0,0 +1,7 @@ +"""Linguistic quality metrics for text evaluation.""" + +from .cola_metric import ColaMetric + +__all__ = [ + "ColaMetric", +] \ No newline at end of file diff --git a/src/atgen/metrics/linguistic/cola_metric.py b/src/atgen/metrics/linguistic/cola_metric.py new file mode 100644 index 0000000..3410aeb --- /dev/null +++ b/src/atgen/metrics/linguistic/cola_metric.py @@ -0,0 +1,118 @@ +from typing import List, Dict, Union, Optional +import numpy as np +import torch +from datasets import Dataset +from torch.utils.data import DataLoader +from transformers import ( + AutoModelForSequenceClassification, + AutoTokenizer, + DataCollatorWithPadding, +) +import nltk + +from ..base.base_metric import BaseMetric, MetricConfig + +# Ensure NLTK data is downloaded +try: + nltk.data.find('tokenizers/punkt') +except LookupError: + nltk.download('punkt') + + +class ColaMetric(BaseMetric): + """CoLA (Corpus of Linguistic Acceptability) metric for evaluating grammaticality.""" + + def __init__(self, config: Optional[MetricConfig] = None): + super().__init__(config) + self.model = None + self.tokenizer = None + self.checkpoint = getattr(config, 'checkpoint', 'Aktsvigun/electra-large-cola') if config else 'Aktsvigun/electra-large-cola' + + def _check_dependencies(self) -> bool: + """Check if required dependencies are available.""" + try: + import torch + from transformers import AutoModelForSequenceClassification, AutoTokenizer + from datasets import Dataset + import nltk + return True + except ImportError: + return False + + def _initialize_model(self): + """Initialize the CoLA model if not already initialized.""" + if self.model is None: + self.model = AutoModelForSequenceClassification.from_pretrained( + self.checkpoint, + cache_dir=self.config.cache_dir + ).to(self.config.device) + self.tokenizer = AutoTokenizer.from_pretrained( + self.checkpoint, + cache_dir=self.config.cache_dir + ) + + def calculate(self, predictions: List[str], references: Optional[List[Union[str, List[str]]]] = None, original_texts: Optional[List[str]] = None) -> Dict[str, float]: + """ + Calculate CoLA grammaticality scores. + + Args: + predictions: List of predicted texts to evaluate for grammaticality + references: Not used for CoLA + original_texts: Not used for CoLA + + Returns: + Dictionary with CoLA scores + """ + if not self.is_available(): + raise RuntimeError("CoLA dependencies not available") + + self._initialize_model() + + # Split texts into sentences + text_sentences = [nltk.sent_tokenize(text) for text in predictions] + len_maps = np.cumsum([len(x) for x in text_sentences]) + sentences = [sent for text in text_sentences for sent in text] + + # Tokenize sentences + def tokenize_fn(instance): + return self.tokenizer(instance["text"], truncation=True) + + tokenized_data = Dataset.from_dict({"text": sentences}).map( + tokenize_fn, remove_columns=["text"], batched=True + ) + data_collator = DataCollatorWithPadding(tokenizer=self.tokenizer) + dataloader = DataLoader( + tokenized_data, + batch_size=self.config.batch_size, + shuffle=False, + collate_fn=data_collator + ) + + # Calculate probabilities + sent_probas = torch.empty(len(sentences), dtype=torch.float32, device=self.config.device) + probas = torch.empty(len(predictions), dtype=torch.float32, device=self.config.device) + + start = 0 + end = self.config.batch_size + + with torch.no_grad(): + for i, batch in enumerate(dataloader): + batch = {k: v.to(self.config.device) for k, v in batch.items()} + batch_pred = self.model(**batch) + batch_probas = 1 / (1 + (-batch_pred.logits[:, 1]).exp()) + sent_probas[start:end].copy_(batch_probas) + start = end + end += self.config.batch_size + + # Aggregate sentence scores to text scores + for i, end_idx in enumerate(len_maps): + start_idx = len_maps[i - 1] if i != 0 else 0 + probas[i].copy_(sent_probas[start_idx:end_idx].mean()) + + scores = {"cola": probas.cpu().detach().numpy()} + + # Aggregate if requested + if self.config.aggregate: + scores = {key: float(np.mean(value)) for key, value in scores.items()} + + return scores \ No newline at end of file diff --git a/src/atgen/metrics/llm_based/__init__.py b/src/atgen/metrics/llm_based/__init__.py new file mode 100644 index 0000000..45993a2 --- /dev/null +++ b/src/atgen/metrics/llm_based/__init__.py @@ -0,0 +1,19 @@ +"""LLM-based metrics for text evaluation using DeepEval.""" + +from .evaluation_llm import EvaluationLLM +from .base_deepeval_metric import BaseDeepEvalMetric +from .answer_relevancy_metric import DeepEvalAnswerRelevancyMetric +from .faithfulness_metric import DeepEvalFaithfulnessMetric +from .summarization_metric import DeepEvalSummarizationMetric +from .prompt_alignment_metric import DeepEvalPromptAlignmentMetric +from .bigbench_hard_metric import BigBenchHardMetric + +__all__ = [ + "EvaluationLLM", + "BaseDeepEvalMetric", + "DeepEvalAnswerRelevancyMetric", + "DeepEvalFaithfulnessMetric", + "DeepEvalSummarizationMetric", + "DeepEvalPromptAlignmentMetric", + "BigBenchHardMetric", +] \ No newline at end of file diff --git a/src/atgen/metrics/llm_based/answer_relevancy_metric.py b/src/atgen/metrics/llm_based/answer_relevancy_metric.py new file mode 100644 index 0000000..c605375 --- /dev/null +++ b/src/atgen/metrics/llm_based/answer_relevancy_metric.py @@ -0,0 +1,46 @@ +""" +DeepEval Answer Relevancy metric for evaluating how well outputs answer inputs. +""" + +from typing import List, Dict, Union, Optional +from deepeval.test_case import LLMTestCase +from deepeval.metrics import AnswerRelevancyMetric + +from .base_deepeval_metric import BaseDeepEvalMetric + + +class DeepEvalAnswerRelevancyMetric(BaseDeepEvalMetric): + """DeepEval Answer Relevancy metric for evaluating how well the output answers the input.""" + + def calculate(self, predictions: List[str], references: Optional[List[Union[str, List[str]]]] = None, original_texts: Optional[List[str]] = None) -> Dict[str, float]: + """ + Calculate Answer Relevancy scores. + + Args: + predictions: List of predicted texts + references: Not used for Answer Relevancy + original_texts: List of original/source texts (required) + + Returns: + Dictionary with Answer Relevancy scores + """ + if not self.is_available(): + raise RuntimeError("DeepEval dependencies not available") + + if original_texts is None: + raise ValueError("Answer Relevancy metric requires original texts") + + # Create DeepEval metric + metric = self._create_deepeval_metric(AnswerRelevancyMetric) + + # Create test cases + test_cases = [] + for pred, src in zip(predictions, original_texts): + test_case = LLMTestCase( + input=src, + actual_output=pred, + ) + test_cases.append(test_case) + + # Run evaluation + return self._run_evaluation(test_cases, metric, "deepeval_answer_relevance") \ No newline at end of file diff --git a/src/atgen/metrics/llm_based/base_deepeval_metric.py b/src/atgen/metrics/llm_based/base_deepeval_metric.py new file mode 100644 index 0000000..c881a00 --- /dev/null +++ b/src/atgen/metrics/llm_based/base_deepeval_metric.py @@ -0,0 +1,97 @@ +""" +Base class for DeepEval metrics. +""" + +import os +import sys +from typing import List, Dict, Union, Optional +import numpy as np +from deepeval import evaluate +from deepeval.test_case import LLMTestCase + +from ..base.base_metric import BaseMetric, MetricConfig +from .evaluation_llm import EvaluationLLM + + +class BaseDeepEvalMetric(BaseMetric): + """Base class for all DeepEval metrics.""" + + def __init__(self, config: Optional[MetricConfig] = None): + super().__init__(config) + self.llm = None + + def _check_dependencies(self) -> bool: + """Check if DeepEval dependencies are available.""" + try: + from deepeval import evaluate + from deepeval.test_case import LLMTestCase + from openai import OpenAI, AsyncOpenAI + return True + except ImportError: + return False + + def _validate_config(self): + """Validate DeepEval configuration.""" + super()._validate_config() + if self.config.api_key is None: + self.logger.warning("API key is recommended for DeepEval metrics") + + def _initialize_llm(self): + """Initialize the LLM for evaluation if not already initialized.""" + if self.llm is None: + self.llm = EvaluationLLM( + api_key=self.config.api_key, + model=self.config.model or "openai/gpt-4o-2024-11-20", + base_url=self.config.base_url or "https://openrouter.ai/api/v1", + ) + + def _create_deepeval_metric(self, metric_class, **kwargs): + """Create a DeepEval metric instance with common parameters.""" + self._initialize_llm() + + return metric_class( + threshold=self.config.threshold, + model=self.llm, + include_reason=self.config.include_reason, + strict_mode=self.config.strict_mode, + async_mode=self.config.async_mode, + **kwargs + ) + + def _run_evaluation(self, test_cases: List[LLMTestCase], metric, metric_name: str) -> Dict[str, float]: + """Run evaluation with proper output handling.""" + if not test_cases: + return {} + + # Disable printing to console during evaluation if not verbose + original_stdout = sys.stdout + if not self.config.verbose_mode: + sys.stdout = open(os.devnull, "w") + + try: + # Run evaluation + evaluation_results = evaluate( + test_cases=test_cases, + metrics=[metric], + run_async=self.config.async_mode, + ) + + # Process results + scores = [] + for result in evaluation_results.test_results: + scores.append(1 if result.success else 0) + + # Calculate average score + if scores: + if self.config.aggregate: + return {metric_name: float(np.mean(scores))} + else: + return {metric_name: np.array(scores)} + else: + return {metric_name: 0.0} + + finally: + # Restore stdout + if not self.config.verbose_mode: + sys.stdout.close() + sys.stdout = original_stdout \ No newline at end of file diff --git a/src/atgen/metrics/llm_based/bigbench_hard_metric.py b/src/atgen/metrics/llm_based/bigbench_hard_metric.py new file mode 100644 index 0000000..236d5e2 --- /dev/null +++ b/src/atgen/metrics/llm_based/bigbench_hard_metric.py @@ -0,0 +1,200 @@ +""" +BigBenchHard metric for evaluating model performance on challenging reasoning tasks. +""" + +from typing import List, Dict, Union, Optional +from transformers import GenerationMixin, PreTrainedTokenizerBase +from deepeval.benchmarks import BigBenchHard +from deepeval.benchmarks.big_bench_hard.template import BigBenchHardTemplate +from deepeval.models.base_model import DeepEvalBaseLLM + +from ..base.base_metric import BaseMetric, MetricConfig + + +class BigBenchHardModel(DeepEvalBaseLLM): + """DeepEval model wrapper for BigBenchHard evaluation.""" + + def __init__( + self, + model: GenerationMixin, + tokenizer: PreTrainedTokenizerBase, + device: str, + model_name: str = "Model", + **generation_kwargs, + ) -> None: + self.model = model + self.tokenizer = tokenizer + self.device = device + self.model_name = model_name + self.generation_kwargs = generation_kwargs + + def load_model(self) -> GenerationMixin: + return self.model + + def generate(self, prompt: str, **kwargs) -> str: + model = self.load_model() + model.to(self.device) + model_inputs = self.tokenizer([prompt], return_tensors="pt").to(self.device) + generated_ids = model.generate(**model_inputs, **self.generation_kwargs) + return self.tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0] + + async def a_generate(self, prompt: str, **kwargs) -> str: + return self.generate(prompt) + + def batch_generate(self, prompts: list[str], **kwargs) -> list[str]: + model = self.load_model() + model.to(self.device) + model_inputs = self.tokenizer(prompts, return_tensors="pt").to(self.device) + generated_ids = model.generate(**model_inputs, **self.generation_kwargs) + return self.tokenizer.batch_decode(generated_ids, skip_special_tokens=True) + + def get_model_name(self): + return self.model_name + + +class BigBenchHardFixed(BigBenchHard): + """ + Fixed version of BigBenchHard that addresses batch prediction issues. + + The main branch for deepeval contains broken code, as "NumberModel" is not defined: + https://github.com/confident-ai/deepeval/blob/main/deepeval/benchmarks/big_bench_hard/big_bench_hard.py#L153 + """ + + def batch_predict(self, model, task, goldens): + prompts = [] + for golden in goldens: + prompt: dict = BigBenchHardTemplate.generate_output( + input=golden.input, + task=task, + n_shots=self.n_shots, + enable_cot=self.enable_cot, + ) + prompts.append(prompt) + + # Enforced model generation + prompts = [ + prompt + "Make sure to output only the numerical answer." + for prompt in prompts + ] + predictions = model.batch_generate(prompts) + predictions = [str(pred) for pred in predictions] + + if len(predictions) != len(goldens): + raise ValueError( + "Custom `batch_generate` method did not return the same " + "number of generations as the number of prompts." + ) + + res = [] + for i in range(len(predictions)): + prediction = predictions[i] + prediction = prediction.split()[-1] + prediction = prediction[:-1] if self.enable_cot else prediction + golden = goldens[i] + + # Define Metric + score = self.scorer.exact_match_score(golden.expected_output, prediction) + res.append({"prediction": prediction, "score": score}) + + return res + + +class BigBenchHardMetric(BaseMetric): + """BigBenchHard benchmark metric for evaluating reasoning performance.""" + + def __init__(self, config: Optional[MetricConfig] = None): + super().__init__(config) + self.model = None + self.tokenizer = None + self.benchmark = None + self.benchmark_model = None + + # Generation parameters + self.generation_params = getattr(config, 'generation_params', {}) if config else {} + self.benchmark_params = getattr(config, 'benchmark_params', {}) if config else {} + self.model_name = getattr(config, 'model_name', 'Model') if config else 'Model' + + def _check_dependencies(self) -> bool: + """Check if BigBenchHard dependencies are available.""" + try: + from deepeval.benchmarks import BigBenchHard + from transformers import GenerationMixin, PreTrainedTokenizerBase + return True + except ImportError: + return False + + def _validate_config(self): + """Validate BigBenchHard configuration.""" + super()._validate_config() + # BigBenchHard doesn't use the standard prediction/reference format + # so we don't need the usual validation + + def set_model_and_tokenizer(self, model: GenerationMixin, tokenizer: PreTrainedTokenizerBase): + """ + Set the model and tokenizer for evaluation. + + Args: + model: The model to evaluate + tokenizer: The tokenizer for the model + """ + self.model = model + self.tokenizer = tokenizer + + # Initialize the benchmark model and benchmark + self.benchmark_model = BigBenchHardModel( + model=self.model, + tokenizer=self.tokenizer, + device=self.config.device, + model_name=self.model_name, + **self.generation_params + ) + self.benchmark = BigBenchHardFixed(**self.benchmark_params) + + def calculate(self, predictions: List[str], references: Optional[List[Union[str, List[str]]]] = None, original_texts: Optional[List[str]] = None) -> Dict[str, float]: + """ + Calculate BigBenchHard score. + + Note: This method signature is maintained for consistency with BaseMetric, + but BigBenchHard doesn't use predictions/references in the traditional way. + The model and tokenizer must be set using set_model_and_tokenizer() before calling this. + + Args: + predictions: Not used for BigBenchHard + references: Not used for BigBenchHard + original_texts: Not used for BigBenchHard + + Returns: + Dictionary with BigBenchHard score + """ + if not self.is_available(): + raise RuntimeError("BigBenchHard dependencies not available") + + if self.model is None or self.tokenizer is None: + raise ValueError("Model and tokenizer must be set using set_model_and_tokenizer() before evaluation") + + if self.benchmark_model is None or self.benchmark is None: + self.set_model_and_tokenizer(self.model, self.tokenizer) + + # Run the benchmark evaluation + self.benchmark.evaluate(model=self.benchmark_model, batch_size=self.config.batch_size) + + return {"bigbench_hard_score": float(self.benchmark.overall_score)} + + def evaluate_benchmark(self, model: GenerationMixin, tokenizer: PreTrainedTokenizerBase, batch_size: Optional[int] = None) -> float: + """ + Convenience method for direct benchmark evaluation. + + Args: + model: The model to evaluate + tokenizer: The tokenizer for the model + batch_size: Batch size for evaluation (uses config default if None) + + Returns: + Overall benchmark score + """ + self.set_model_and_tokenizer(model, tokenizer) + + eval_batch_size = batch_size if batch_size is not None else self.config.batch_size + self.benchmark.evaluate(model=self.benchmark_model, batch_size=eval_batch_size) + + return self.benchmark.overall_score \ No newline at end of file diff --git a/src/atgen/metrics/llm_based/evaluation_llm.py b/src/atgen/metrics/llm_based/evaluation_llm.py new file mode 100644 index 0000000..556ca79 --- /dev/null +++ b/src/atgen/metrics/llm_based/evaluation_llm.py @@ -0,0 +1,94 @@ +""" +Custom Evaluation LLM implementation for DeepEval metrics. +""" + +from openai import OpenAI, AsyncOpenAI +from deepeval.models.base_model import DeepEvalBaseLLM + + +class EvaluationLLM(DeepEvalBaseLLM): + """ + Custom Evaluation LLM implementation for DeepEval. + + This class implements the DeepEvalBaseLLM interface to allow using + custom models with DeepEval metrics. + """ + + def __init__( + self, + api_key=None, + model="openai/gpt-4o-2024-11-20", + base_url="https://openrouter.ai/api/v1", + ): + """ + Initialize the Evaluation LLM. + + Args: + api_key: Evaluation API key + model: Model identifier (e.g., "openai/gpt-4o-2024-11-20") + base_url: Evaluation API base URL + """ + self.api_key = api_key + self.model_name = model + self.base_url = base_url + self.client = None + self.async_client = None + self.OpenAI = OpenAI + self.AsyncOpenAI = AsyncOpenAI + + def load_model(self): + """Load and return the client.""" + if self.client is None: + self.client = self.OpenAI( + base_url=self.base_url, + api_key=self.api_key, + ) + return self.client + + def load_async_model(self): + """Load and return the async client.""" + if self.async_client is None: + self.async_client = self.AsyncOpenAI( + base_url=self.base_url, + api_key=self.api_key, + ) + return self.async_client + + def generate(self, prompt: str) -> str: + """ + Generate a response from the evaluation model. + + Args: + prompt: The prompt to send to the model + + Returns: + The model's response as a string + """ + client = self.load_model() + response = client.chat.completions.create( + model=self.model_name, + messages=[{"role": "user", "content": prompt}], + ) + return response.choices[0].message.content + + async def a_generate(self, prompt: str) -> str: + """ + Asynchronously generate a response from the evaluation model. + + Args: + prompt: The prompt to send to the model + + Returns: + The model's response as a string + """ + # Use the async client for async operations + client = self.load_async_model() + response = await client.chat.completions.create( + model=self.model_name, + messages=[{"role": "user", "content": prompt}], + ) + return response.choices[0].message.content + + def get_model_name(self): + """Return the name of the model.""" + return f"EvaluationLLM: {self.model_name}" \ No newline at end of file diff --git a/src/atgen/metrics/llm_based/faithfulness_metric.py b/src/atgen/metrics/llm_based/faithfulness_metric.py new file mode 100644 index 0000000..9f42e07 --- /dev/null +++ b/src/atgen/metrics/llm_based/faithfulness_metric.py @@ -0,0 +1,51 @@ +""" +DeepEval Faithfulness metric for evaluating factual consistency with input. +""" + +from typing import List, Dict, Union, Optional +from deepeval.test_case import LLMTestCase +from deepeval.metrics import FaithfulnessMetric + +from .base_deepeval_metric import BaseDeepEvalMetric + + +class DeepEvalFaithfulnessMetric(BaseDeepEvalMetric): + """DeepEval Faithfulness metric for evaluating factual consistency with the input.""" + + def calculate(self, predictions: List[str], references: Optional[List[Union[str, List[str]]]] = None, original_texts: Optional[List[str]] = None) -> Dict[str, float]: + """ + Calculate Faithfulness scores. + + Args: + predictions: List of predicted texts + references: Not used for Faithfulness + original_texts: List of original/source texts (required) + + Returns: + Dictionary with Faithfulness scores + """ + if not self.is_available(): + raise RuntimeError("DeepEval dependencies not available") + + if original_texts is None: + raise ValueError("Faithfulness metric requires original texts") + + # Create DeepEval metric with truths extraction limit if specified + kwargs = {} + if self.config.truths_extraction_limit is not None: + kwargs['truths_extraction_limit'] = self.config.truths_extraction_limit + + metric = self._create_deepeval_metric(FaithfulnessMetric, **kwargs) + + # Create test cases + test_cases = [] + for pred, src in zip(predictions, original_texts): + test_case = LLMTestCase( + input=src, + actual_output=pred, + retrieval_context=[src], # Use source as retrieval context + ) + test_cases.append(test_case) + + # Run evaluation + return self._run_evaluation(test_cases, metric, "deepeval_faithfulness") \ No newline at end of file diff --git a/src/atgen/metrics/llm_based/prompt_alignment_metric.py b/src/atgen/metrics/llm_based/prompt_alignment_metric.py new file mode 100644 index 0000000..0805159 --- /dev/null +++ b/src/atgen/metrics/llm_based/prompt_alignment_metric.py @@ -0,0 +1,57 @@ +""" +DeepEval Prompt Alignment metric for evaluating alignment with expected output. +""" + +from typing import List, Dict, Union, Optional +from deepeval.test_case import LLMTestCase +from deepeval.metrics import PromptAlignmentMetric + +from .base_deepeval_metric import BaseDeepEvalMetric + + +class DeepEvalPromptAlignmentMetric(BaseDeepEvalMetric): + """DeepEval Prompt Alignment metric for evaluating alignment with the expected output.""" + + def calculate(self, predictions: List[str], references: Optional[List[Union[str, List[str]]]] = None, original_texts: Optional[List[str]] = None) -> Dict[str, float]: + """ + Calculate Prompt Alignment scores. + + Args: + predictions: List of predicted texts + references: List of reference texts (required) + original_texts: List of original/source texts (required) + + Returns: + Dictionary with Prompt Alignment scores + """ + if not self.is_available(): + raise RuntimeError("DeepEval dependencies not available") + + if original_texts is None: + raise ValueError("Prompt Alignment metric requires original texts") + + if references is None: + raise ValueError("Prompt Alignment metric requires reference texts") + + if isinstance(references[0], list): + self.logger.error("Prompt Alignment does not support multiple references. Skipping...") + return {} + + # Create DeepEval metric with default prompt instructions + metric = self._create_deepeval_metric( + PromptAlignmentMetric, + prompt_instructions=["Do what you are told to do in the prompt"] + ) + + # Create test cases + test_cases = [] + for pred, ref, src in zip(predictions, references, original_texts): + test_case = LLMTestCase( + input=src, + actual_output=pred, + expected_output=ref, + ) + test_cases.append(test_case) + + # Run evaluation + return self._run_evaluation(test_cases, metric, "deepeval_prompt_alignment") \ No newline at end of file diff --git a/src/atgen/metrics/llm_based/summarization_metric.py b/src/atgen/metrics/llm_based/summarization_metric.py new file mode 100644 index 0000000..08d0624 --- /dev/null +++ b/src/atgen/metrics/llm_based/summarization_metric.py @@ -0,0 +1,46 @@ +""" +DeepEval Summarization metric for evaluating summarization quality. +""" + +from typing import List, Dict, Union, Optional +from deepeval.test_case import LLMTestCase +from deepeval.metrics import SummarizationMetric + +from .base_deepeval_metric import BaseDeepEvalMetric + + +class DeepEvalSummarizationMetric(BaseDeepEvalMetric): + """DeepEval Summarization metric for evaluating summarization quality.""" + + def calculate(self, predictions: List[str], references: Optional[List[Union[str, List[str]]]] = None, original_texts: Optional[List[str]] = None) -> Dict[str, float]: + """ + Calculate Summarization scores. + + Args: + predictions: List of predicted texts + references: Not used for Summarization + original_texts: List of original/source texts (required) + + Returns: + Dictionary with Summarization scores + """ + if not self.is_available(): + raise RuntimeError("DeepEval dependencies not available") + + if original_texts is None: + raise ValueError("Summarization metric requires original texts") + + # Create DeepEval metric + metric = self._create_deepeval_metric(SummarizationMetric) + + # Create test cases + test_cases = [] + for pred, src in zip(predictions, original_texts): + test_case = LLMTestCase( + input=src, + actual_output=pred, + ) + test_cases.append(test_case) + + # Run evaluation + return self._run_evaluation(test_cases, metric, "deepeval_summarization") \ No newline at end of file diff --git a/src/atgen/metrics/semantic/__init__.py b/src/atgen/metrics/semantic/__init__.py new file mode 100644 index 0000000..8d848a1 --- /dev/null +++ b/src/atgen/metrics/semantic/__init__.py @@ -0,0 +1,11 @@ +"""Semantic metrics for text evaluation.""" + +from .bart_score_metric import BartScoreMetric +from .alignscore_metric import AlignScoreMetric +from .sentbert_metric import SentBertMetric + +__all__ = [ + "BartScoreMetric", + "AlignScoreMetric", + "SentBertMetric", +] \ No newline at end of file diff --git a/src/atgen/metrics/semantic/alignscore_metric.py b/src/atgen/metrics/semantic/alignscore_metric.py new file mode 100644 index 0000000..efc0bd2 --- /dev/null +++ b/src/atgen/metrics/semantic/alignscore_metric.py @@ -0,0 +1,101 @@ +import os +from typing import List, Dict, Union, Optional +from urllib.request import urlretrieve +from pathlib import Path +import numpy as np + +from ..base.base_metric import BaseMetric, MetricConfig + +# Default AlignScore checkpoint path +ALIGNSCORE_CHECKPOINT_PATH = os.getenv( + "ALIGNSCORE_CHECKPOINT_PATH", + os.path.join( + Path(__file__).parents[4], # Going up to repository root + "external_metrics/AlignScore/model/AlignScore-base.ckpt", + ), +) + + +class AlignScoreMetric(BaseMetric): + """AlignScore metric for evaluating factual consistency.""" + + def __init__(self, config: Optional[MetricConfig] = None): + super().__init__(config) + self.scorer = None + self.checkpoint_path = getattr(config, 'checkpoint_path', ALIGNSCORE_CHECKPOINT_PATH) if config else ALIGNSCORE_CHECKPOINT_PATH + + def _check_dependencies(self) -> bool: + """Check if AlignScore is available.""" + try: + from alignscore import AlignScore + return True + except ImportError: + return False + + def _initialize_scorer(self): + """Initialize the AlignScore scorer if not already initialized.""" + if self.scorer is None: + if not os.path.exists(self.checkpoint_path): + # Download the checkpoint if it doesn't exist + os.makedirs(os.path.dirname(self.checkpoint_path), exist_ok=True) + urlretrieve( + "https://huggingface.co/yzha/AlignScore/resolve/main/AlignScore-base.ckpt", + self.checkpoint_path, + ) + + from alignscore import AlignScore + self.scorer = AlignScore( + model="roberta-base", + batch_size=self.config.batch_size, + device=self.config.device, + ckpt_path=self.checkpoint_path, + evaluation_mode="nli_sp", + ) + + def calculate(self, predictions: List[str], references: Optional[List[Union[str, List[str]]]] = None, original_texts: Optional[List[str]] = None) -> Dict[str, float]: + """ + Calculate AlignScore metrics. + + Args: + predictions: List of predicted texts + references: List of reference texts (single references only) + original_texts: List of original/source texts (required) + + Returns: + Dictionary with AlignScore results + """ + if not self.is_available(): + raise RuntimeError("AlignScore dependencies not available") + + if original_texts is None: + raise ValueError("AlignScore requires original texts") + + if references is not None and isinstance(references[0], list): + self.logger.error("AlignScore does not support multiple references. Skipping...") + return {} + + self._initialize_scorer() + + # Fix: AlignScore outputs an error if a text is empty, so we need to add some content + original_texts = [text if text else " " for text in original_texts] + predictions = [text if text else " " for text in predictions] + + scores = {} + + # Calculate AlignScore between predictions and original texts + scores_pred = self.scorer.score(contexts=original_texts, claims=predictions) + scores["alignscore"] = scores_pred + + if references is not None: + references = [text if text else " " for text in references] + # Calculate baseline AlignScore between references and original texts + scores_baseline = self.scorer.score(contexts=original_texts, claims=references) + # Calculate relative score + scores_rel = np.array(scores_pred) / np.array(scores_baseline) + scores["alignscore_rel"] = scores_rel + + # Aggregate if requested + if self.config.aggregate: + scores = {key: float(np.mean(value)) for key, value in scores.items()} + + return scores \ No newline at end of file diff --git a/src/atgen/metrics/semantic/bart_score_metric.py b/src/atgen/metrics/semantic/bart_score_metric.py new file mode 100644 index 0000000..c970a1f --- /dev/null +++ b/src/atgen/metrics/semantic/bart_score_metric.py @@ -0,0 +1,153 @@ +import traceback +from typing import List, Dict, Union, Optional +import numpy as np +import torch +import torch.nn as nn +from transformers import BartTokenizer, BartForConditionalGeneration +from tqdm import tqdm + +from ..base.base_metric import BaseMetric, MetricConfig + + +class BARTScorer: + """Internal BARTScorer implementation.""" + + def __init__( + self, + device="cuda", + max_length=1024, + checkpoint="facebook/bart-large-cnn", + cache_dir: str = "cache", + ): + # Set up model + self.device = device + self.max_length = max_length + self.tokenizer = BartTokenizer.from_pretrained(checkpoint, cache_dir=cache_dir) + self.model = BartForConditionalGeneration.from_pretrained( + checkpoint, cache_dir=cache_dir + ) + self.model.eval() + self.model.to(device) + + # Set up loss + self.loss_fct = nn.NLLLoss( + reduction="none", ignore_index=self.model.config.pad_token_id + ) + self.lsm = nn.LogSoftmax(dim=1) + + def score(self, srcs, tgts, batch_size=4): + """Score a batch of examples""" + score_list = [] + for i in tqdm(range(0, len(srcs), batch_size), desc="Calculating BARTScore..."): + src_list = srcs[i : i + batch_size] + tgt_list = tgts[i : i + batch_size] + try: + with torch.no_grad(): + encoded_src = self.tokenizer( + src_list, + max_length=self.max_length, + truncation=True, + padding=True, + return_tensors="pt", + ) + encoded_tgt = self.tokenizer( + tgt_list, + max_length=self.max_length, + truncation=True, + padding=True, + return_tensors="pt", + ) + src_tokens = encoded_src["input_ids"].to(self.device) + src_mask = encoded_src["attention_mask"].to(self.device) + + tgt_tokens = encoded_tgt["input_ids"].to(self.device) + tgt_mask = encoded_tgt["attention_mask"] + tgt_len = tgt_mask.sum(dim=1).to(self.device) + + output = self.model( + input_ids=src_tokens, attention_mask=src_mask, labels=tgt_tokens + ) + logits = output.logits.view(-1, self.model.config.vocab_size) + loss = self.loss_fct(self.lsm(logits), tgt_tokens.view(-1)) + loss = loss.view(tgt_tokens.shape[0], -1) + loss = loss.sum(dim=1) / tgt_len + curr_score_list = [-x.item() for x in loss] + score_list += curr_score_list + + except RuntimeError: + traceback.print_exc() + print(f"source: {src_list}") + print(f"target: {tgt_list}") + exit(0) + return score_list + + +class BartScoreMetric(BaseMetric): + """BARTScore metric for evaluating text generation quality.""" + + def __init__(self, config: Optional[MetricConfig] = None): + super().__init__(config) + self.scorer = None + self.checkpoint = getattr(config, 'checkpoint', 'facebook/bart-large-cnn') if config else 'facebook/bart-large-cnn' + + def _check_dependencies(self) -> bool: + """Check if required dependencies are available.""" + try: + import torch + from transformers import BartTokenizer, BartForConditionalGeneration + return True + except ImportError: + return False + + def _initialize_scorer(self): + """Initialize the BARTScorer if not already initialized.""" + if self.scorer is None: + self.scorer = BARTScorer( + device=self.config.device, + checkpoint=self.checkpoint, + cache_dir=self.config.cache_dir, + ) + + def calculate(self, predictions: List[str], references: Optional[List[Union[str, List[str]]]] = None, original_texts: Optional[List[str]] = None) -> Dict[str, float]: + """ + Calculate BARTScore metrics. + + Args: + predictions: List of predicted texts + references: List of reference texts (can be list of lists for multiple references) + original_texts: List of original/source texts + + Returns: + Dictionary with BARTScore results + """ + if not self.is_available(): + raise RuntimeError("BARTScore dependencies not available") + + self._initialize_scorer() + + scores = {} + + # Calculate source-hypothesis score (BARTScore-sh) + if original_texts is not None: + sh_scores = self.scorer.score(original_texts, predictions, batch_size=self.config.batch_size) + scores["BARTScore-sh"] = np.array(sh_scores) + + # Calculate hypothesis-reference score (BARTScore-hr) + if references is not None: + if isinstance(references[0], list): + # Handle multiple references - take maximum score + hr_scores = [] + for ref, pred in zip(references, predictions): + inst_pred = [pred for _ in range(len(ref))] + inst_scores = self.scorer.score(inst_pred, ref, batch_size=self.config.batch_size) + hr_scores.append(max(inst_scores)) + scores["BARTScore-hr"] = np.array(hr_scores) + else: + hr_scores = self.scorer.score(predictions, references, batch_size=self.config.batch_size) + scores["BARTScore-hr"] = np.array(hr_scores) + + # Aggregate if requested + if self.config.aggregate: + scores = {key: float(np.mean(value)) for key, value in scores.items()} + + return scores \ No newline at end of file diff --git a/src/atgen/metrics/semantic/sentbert_metric.py b/src/atgen/metrics/semantic/sentbert_metric.py new file mode 100644 index 0000000..43d5fb0 --- /dev/null +++ b/src/atgen/metrics/semantic/sentbert_metric.py @@ -0,0 +1,142 @@ +from typing import List, Dict, Union, Optional +import numpy as np +import torch +import torch.nn.functional as F +from transformers import AutoTokenizer, AutoModel + +from ..base.base_metric import BaseMetric, MetricConfig + + +class SentBertMetric(BaseMetric): + """SentenceBERT semantic similarity metric.""" + + def __init__(self, config: Optional[MetricConfig] = None): + super().__init__(config) + self.model = None + self.tokenizer = None + self.checkpoint = getattr(config, 'checkpoint', 'sentence-transformers/all-mpnet-base-v2') if config else 'sentence-transformers/all-mpnet-base-v2' + + def _check_dependencies(self) -> bool: + """Check if required dependencies are available.""" + try: + import torch + from transformers import AutoTokenizer, AutoModel + return True + except ImportError: + return False + + def _initialize_model(self): + """Initialize the SentenceBERT model if not already initialized.""" + if self.model is None: + self.tokenizer = AutoTokenizer.from_pretrained( + self.checkpoint, + cache_dir=self.config.cache_dir + ) + self.model = AutoModel.from_pretrained( + self.checkpoint, + cache_dir=self.config.cache_dir + ).to(self.config.device) + + @staticmethod + def mean_pooling(model_output, attention_mask): + """Mean Pooling - Take attention mask into account for correct averaging.""" + token_embeddings = model_output[0] # First element contains all token embeddings + input_mask_expanded = ( + attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float() + ) + return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp( + input_mask_expanded.sum(1), min=1e-9 + ) + + def _compute_similarity(self, source_texts: List[str], ref_texts: List[str]) -> np.ndarray: + """Compute semantic similarity between source and reference texts.""" + assert len(source_texts) == len(ref_texts) + + # Make batch_size an odd number for better batch processing + batch_size = self.config.batch_size + if batch_size % 2 == 0: + batch_size -= 1 + half_batch_size = batch_size // 2 + n_texts = len(source_texts) + scores = np.empty(n_texts, dtype=np.float32) + start = 0 + end = 0 + + while end < n_texts: + end += half_batch_size + batch_idx = slice(start, end) + + # Tokenize sentences + encoded_input = self.tokenizer( + source_texts[batch_idx] + ref_texts[batch_idx], + padding=True, + truncation=True, + return_tensors="pt", + ) + encoded_input = { + key: value.to(self.config.device) for key, value in encoded_input.items() + } + + # Calculate embeddings + with torch.no_grad(): + model_output = self.model(**encoded_input) + + # Perform pooling + sent_embs = self.mean_pooling(model_output, encoded_input["attention_mask"]) + + # Normalize embeddings + sent_embs = F.normalize(sent_embs, p=2, dim=1) + + n_source_embs = len(sent_embs) // 2 + scores[batch_idx] = ( + (sent_embs[:n_source_embs] * sent_embs[n_source_embs:]) + .sum(-1) + .cpu() + .detach() + .numpy() + ) + start = end + + return scores + + def calculate(self, predictions: List[str], references: Optional[List[Union[str, List[str]]]] = None, original_texts: Optional[List[str]] = None) -> Dict[str, float]: + """ + Calculate SentenceBERT semantic similarity. + + Args: + predictions: List of predicted texts + references: List of reference texts + original_texts: List of original/source texts + + Returns: + Dictionary with semantic similarity scores + """ + if not self.is_available(): + raise RuntimeError("SentenceBERT dependencies not available") + + self._initialize_model() + + scores = {} + + # Similarity between predictions and references + if references is not None: + if isinstance(references[0], list): + # Handle multiple references - compute average similarity + ref_scores = [] + for pred, ref_list in zip(predictions, references): + pred_list = [pred] * len(ref_list) + sim_scores = self._compute_similarity(pred_list, ref_list) + ref_scores.append(np.mean(sim_scores)) + scores["sentbert_pred_ref"] = np.array(ref_scores) + else: + scores["sentbert_pred_ref"] = self._compute_similarity(predictions, references) + + # Similarity between predictions and original texts + if original_texts is not None: + scores["sentbert_pred_src"] = self._compute_similarity(predictions, original_texts) + + # Aggregate if requested + if self.config.aggregate: + scores = {key: float(np.mean(value)) for key, value in scores.items()} + + return scores \ No newline at end of file From 3f0946337883adaae23702d2b9b96311e7fbaa16 Mon Sep 17 00:00:00 2001 From: alfekka Date: Mon, 9 Jun 2025 00:55:59 +0300 Subject: [PATCH 02/39] removed dependencies chek --- demo_all_metrics.py | 40 ++++-- example_bigbench_usage.py | 97 ++++++++++++++ src/atgen/metrics/__init__.py | 12 -- src/atgen/metrics/base/base_metric.py | 20 +-- src/atgen/metrics/big_bench_hard_metric.py | 120 ------------------ src/atgen/metrics/lexical/bleu_metric.py | 9 -- src/atgen/metrics/lexical/rouge_metric.py | 10 -- src/atgen/metrics/linguistic/cola_metric.py | 12 -- .../metrics/llm_based/base_deepeval_metric.py | 10 -- .../metrics/semantic/alignscore_metric.py | 15 +-- .../metrics/semantic/bart_score_metric.py | 10 +- src/atgen/metrics/semantic/sentbert_metric.py | 13 +- 12 files changed, 134 insertions(+), 234 deletions(-) create mode 100644 example_bigbench_usage.py delete mode 100644 src/atgen/metrics/__init__.py delete mode 100644 src/atgen/metrics/big_bench_hard_metric.py diff --git a/demo_all_metrics.py b/demo_all_metrics.py index bf0f057..f32c864 100644 --- a/demo_all_metrics.py +++ b/demo_all_metrics.py @@ -11,14 +11,14 @@ from atgen.metrics.lexical import BleuMetric, RougeMetric from atgen.metrics.semantic import BartScoreMetric, AlignScoreMetric, SentBertMetric from atgen.metrics.linguistic import ColaMetric -# Note: llm-based directory needs to be renamed to llm_based for Python imports -# from atgen.metrics.llm_based import ( -# DeepEvalAnswerRelevancyMetric, -# DeepEvalFaithfulnessMetric, -# DeepEvalSummarizationMetric, -# DeepEvalPromptAlignmentMetric, -# EvaluationLLM -# ) +from atgen.metrics.llm_based import ( + DeepEvalAnswerRelevancyMetric, + DeepEvalFaithfulnessMetric, + DeepEvalSummarizationMetric, + DeepEvalPromptAlignmentMetric, + BigBenchHardMetric, + EvaluationLLM +) def demo_all_metrics(): @@ -83,12 +83,15 @@ def demo_all_metrics(): "šŸ—£ļø Linguistic Quality": [ ("CoLA (Grammaticality)", ColaMetric(base_config)), ], - # "šŸ¤– LLM-based (DeepEval)": [ - # ("Answer Relevancy", DeepEvalAnswerRelevancyMetric(deepeval_config)), - # ("Faithfulness", DeepEvalFaithfulnessMetric(deepeval_config)), - # ("Summarization", DeepEvalSummarizationMetric(deepeval_config)), - # ("Prompt Alignment", DeepEvalPromptAlignmentMetric(deepeval_config)), - # ] + "šŸ¤– LLM-based (DeepEval)": [ + ("Answer Relevancy", DeepEvalAnswerRelevancyMetric(deepeval_config)), + ("Faithfulness", DeepEvalFaithfulnessMetric(deepeval_config)), + ("Summarization", DeepEvalSummarizationMetric(deepeval_config)), + ("Prompt Alignment", DeepEvalPromptAlignmentMetric(deepeval_config)), + ], + "šŸ§® Benchmark Metrics": [ + ("BigBenchHard", BigBenchHardMetric(deepeval_config)), + ] } print(f"šŸ“Š Testing with {len(predictions)} samples\n") @@ -117,6 +120,14 @@ def demo_all_metrics(): else: results = metric.calculate(predictions, None, original_texts) + elif metric_name == "BigBenchHard": + # BigBenchHard needs a model and tokenizer, skip for demo + print(f" āš ļø Requires model and tokenizer (use set_model_and_tokenizer())") + print(f" šŸ’” Example: metric.set_model_and_tokenizer(model, tokenizer)") + print(f" score = metric.evaluate_benchmark(model, tokenizer)") + print() + continue + elif metric_name == "CoLA (Grammaticality)": # CoLA only needs predictions results = metric.calculate(predictions) @@ -149,6 +160,7 @@ def demo_all_metrics(): print(" āœ… Proper error handling and logging") print(" āœ… Support for multiple references") print(" āœ… API-based metrics with custom LLM implementation") + print(" āœ… Benchmark metrics (BigBenchHard)") print(" āœ… Configurable aggregation") print(" āœ… Type safety with full type hints") diff --git a/example_bigbench_usage.py b/example_bigbench_usage.py new file mode 100644 index 0000000..aebd473 --- /dev/null +++ b/example_bigbench_usage.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +""" +Example usage of BigBenchHard metric with the new architecture. +""" + +import sys +sys.path.append('src') + +from atgen.metrics.base import MetricConfig +from atgen.metrics.llm_based import BigBenchHardMetric + + +def example_bigbench_usage(): + """Example of how to use BigBenchHard metric.""" + print("šŸ§® BigBenchHard Metric Usage Example") + print("=" * 60) + + # Create configuration for BigBenchHard + config = MetricConfig( + batch_size=8, + device="cuda", # or "cpu" + cache_dir="cache", + model_name="MyAwesomeModel", + # Benchmark-specific parameters + benchmark_params={ + "n_shots": 3, # Number of few-shot examples + "enable_cot": True, # Enable chain-of-thought + }, + # Generation parameters for the model + generation_params={ + "max_new_tokens": 100, + "temperature": 0.1, + "do_sample": False, + } + ) + + # Create the metric + metric = BigBenchHardMetric(config) + + print(f"āœ… Metric created: {metric.name}") + print(f"šŸ“¦ Dependencies available: {metric.is_available()}") + + if not metric.is_available(): + print("āŒ BigBenchHard dependencies not available") + print(" Install with: pip install deepeval") + return + + print("\nšŸ”§ Usage Options:") + print("\n1. Option 1: Using the standard calculate() interface:") + print(" ```python") + print(" # First set the model and tokenizer") + print(" metric.set_model_and_tokenizer(model, tokenizer)") + print(" ") + print(" # Then calculate (predictions/references are ignored)") + print(" results = metric.calculate([], [], [])") + print(" print(results) # {'bigbench_hard_score': 0.85}") + print(" ```") + + print("\n2. Option 2: Using the convenience method:") + print(" ```python") + print(" # Direct evaluation") + print(" score = metric.evaluate_benchmark(model, tokenizer, batch_size=16)") + print(" print(f'BigBenchHard score: {score}')") + print(" ```") + + print("\n3. Option 3: Using the original interface style:") + print(" ```python") + print(" # Set model first") + print(" metric.set_model_and_tokenizer(model, tokenizer)") + print(" ") + print(" # Get the overall score") + print(" score = metric.benchmark.overall_score") + print(" ```") + + print("\nšŸ“ Configuration Details:") + print(f" ā€¢ Batch size: {config.batch_size}") + print(f" ā€¢ Device: {config.device}") + print(f" ā€¢ Model name: {config.model_name}") + print(f" ā€¢ Benchmark params: {config.benchmark_params}") + print(f" ā€¢ Generation params: {config.generation_params}") + + print("\nšŸŽÆ What BigBenchHard Evaluates:") + print(" ā€¢ Challenging reasoning tasks from BIG-bench") + print(" ā€¢ Mathematical reasoning") + print(" ā€¢ Logical reasoning") + print(" ā€¢ Multi-step problem solving") + print(" ā€¢ Chain-of-thought reasoning (if enabled)") + + print("\nšŸ’” Integration Notes:") + print(" ā€¢ Follows the same BaseMetric interface as other metrics") + print(" ā€¢ Supports the standard MetricConfig system") + print(" ā€¢ Can be used in metric factories and pipelines") + print(" ā€¢ Provides both direct and standard interfaces") + + +if __name__ == "__main__": + example_bigbench_usage() \ No newline at end of file diff --git a/src/atgen/metrics/__init__.py b/src/atgen/metrics/__init__.py deleted file mode 100644 index 302149a..0000000 --- a/src/atgen/metrics/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -AVAILABLE_METRICS = [ - "rouge1", - "rouge2", - "rougeL", - "rougeLsum", - "bleu", - "sacrebleu", - "BARTScore-hr", - "BARTScore-sh", - "alignscore", - "BigBenchHardScore", -] diff --git a/src/atgen/metrics/base/base_metric.py b/src/atgen/metrics/base/base_metric.py index 6d5802c..6940bb6 100644 --- a/src/atgen/metrics/base/base_metric.py +++ b/src/atgen/metrics/base/base_metric.py @@ -1,7 +1,7 @@ from abc import ABC, abstractmethod -from typing import Optional +from typing import Optional, Dict, Any import logging -from dataclasses import dataclass +from dataclasses import dataclass, field @dataclass @@ -15,6 +15,7 @@ class MetricConfig: # Model-specific parameters (for local models) checkpoint: Optional[str] = None + model_name: Optional[str] = None # API-based parameters (for LLM-based metrics) api_key: Optional[str] = None @@ -28,6 +29,10 @@ class MetricConfig: async_mode: bool = True verbose_mode: bool = False truths_extraction_limit: Optional[int] = None + + # BigBenchHard specific parameters + benchmark_params: Dict[str, Any] = field(default_factory=dict) + generation_params: Dict[str, Any] = field(default_factory=dict) class BaseMetric(ABC): @@ -53,17 +58,6 @@ def _validate_config(self): """Validate configuration - can be overridden by subclasses.""" pass - def is_available(self) -> bool: - """Check if dependencies are available.""" - if self._is_available is None: - self._is_available = self._check_dependencies() - return self._is_available - - @abstractmethod - def _check_dependencies(self) -> bool: - """Check if required dependencies are available.""" - pass - @abstractmethod def calculate(self, predictions, references, original_texts): pass diff --git a/src/atgen/metrics/big_bench_hard_metric.py b/src/atgen/metrics/big_bench_hard_metric.py deleted file mode 100644 index 1ec5227..0000000 --- a/src/atgen/metrics/big_bench_hard_metric.py +++ /dev/null @@ -1,120 +0,0 @@ -from deepeval.benchmarks import BigBenchHard -from deepeval.benchmarks.big_bench_hard.template import BigBenchHardTemplate -from deepeval.models.base_model import DeepEvalBaseLLM -from transformers import ( - GenerationMixin, - PreTrainedTokenizerBase, -) - -""" -https://arxiv.org/abs/2210.09261v1 - -For benchmark_params, refer to https://docs.confident-ai.com/docs/benchmarks-big-bench-hard -""" - - -class BigBenchHardModel(DeepEvalBaseLLM): - def __init__( - self, - model: GenerationMixin, - tokenizer: PreTrainedTokenizerBase, - device: str, - model_name: str = "Model", - **generation_kwargs, - ) -> None: - self.model = model - self.tokenizer = tokenizer - self.device = device - self.model_name = model_name - self.generation_kwargs = generation_kwargs - - def load_model(self) -> GenerationMixin: - return self.model - - def generate(self, prompt: str, **kwargs) -> str: - model = self.load_model() - model.to(self.device) - model_inputs = self.tokenizer([prompt], return_tensors="pt").to(self.device) - generated_ids = model.generate(**model_inputs, **self.generation_kwargs) - return self.tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0] - - async def a_generate(self, prompt: str, **kwargs) -> str: - return self.generate(prompt) - - def batch_generate(self, prompts: list[str], **kwargs) -> list[str]: - model = self.load_model() - model.to(self.device) - model_inputs = self.tokenizer(prompts, return_tensors="pt").to(self.device) - generated_ids = model.generate(**model_inputs, **self.generation_kwargs) - return self.tokenizer.batch_decode(generated_ids, skip_special_tokens=True) - - def get_model_name(self): - return self.model_name - - -""" -The main branch for deepeval seems to contain broken code, as "NumberModel" is not defined: -https://github.com/confident-ai/deepeval/blob/main/deepeval/benchmarks/big_bench_hard/big_bench_hard.py#L153 - -This child class fixes batch prediction. -""" - - -class BigBenchHardFixed(BigBenchHard): - def batch_predict(self, model, task, goldens): - prompts = [] - for golden in goldens: - prompt: dict = BigBenchHardTemplate.generate_output( - input=golden.input, - task=task, - n_shots=self.n_shots, - enable_cot=self.enable_cot, - ) - prompts.append(prompt) - - # Enforced model generation - prompts = [ - prompt + "Make sure to output only the numerical answer." - for prompt in prompts - ] - predictions = model.batch_generate(prompts) - predictions = [str(pred) for pred in predictions] - - if len(predictions) is not len(goldens): - raise ValueError( - "Custom `batch_generate` method did not return the same " - "number of generations as the number of prompts." - ) - - res = [] - for i in range(len(predictions)): - prediction = predictions[i] - prediction = prediction.split()[-1] - prediction = prediction[:-1] if self.enable_cot else prediction - golden = goldens[i] - - # Define Metric - score = self.scorer.exact_match_score(golden.expected_output, prediction) - res.append({"prediction": prediction, "score": score}) - - return res - - -class BigBenchHardMetric: - def __init__( - self, - model: GenerationMixin, - tokenizer: PreTrainedTokenizerBase, - device: str = "cuda", - model_name: str = "Model", - generation_params: dict = {}, - benchmark_params: dict = {}, - ) -> None: - self.model = BigBenchHardModel( - model, tokenizer, device, model_name, **generation_params - ) - self.benchmark = BigBenchHardFixed(**benchmark_params) - - def score(self, batch_size: int | None = None) -> float: - self.benchmark.evaluate(model=self.model, batch_size=batch_size) - return self.benchmark.overall_score diff --git a/src/atgen/metrics/lexical/bleu_metric.py b/src/atgen/metrics/lexical/bleu_metric.py index 458b6fd..260cc25 100644 --- a/src/atgen/metrics/lexical/bleu_metric.py +++ b/src/atgen/metrics/lexical/bleu_metric.py @@ -48,15 +48,6 @@ def category(self) -> str: def supports_multiple_references(self) -> bool: return True - def _check_dependencies(self) -> bool: - """Check if NLTK is available.""" - try: - import nltk - # Try to use word_tokenize to ensure punkt is available - word_tokenize("test") - return True - except (ImportError, LookupError): - return False def _compute( self, diff --git a/src/atgen/metrics/lexical/rouge_metric.py b/src/atgen/metrics/lexical/rouge_metric.py index aa47246..e63a362 100644 --- a/src/atgen/metrics/lexical/rouge_metric.py +++ b/src/atgen/metrics/lexical/rouge_metric.py @@ -12,14 +12,6 @@ def __init__(self, config: Optional[MetricConfig] = None): super().__init__(config) self.rouge = None - def _check_dependencies(self) -> bool: - """Check if ROUGE is available.""" - try: - from evaluate import load - return True - except ImportError: - return False - def _initialize_rouge(self): """Initialize ROUGE if not already initialized.""" if self.rouge is None: @@ -37,8 +29,6 @@ def calculate(self, predictions: List[str], references: Optional[List[Union[str, Returns: Dictionary with ROUGE scores """ - if not self.is_available(): - raise RuntimeError("ROUGE dependencies not available") if references is None: raise ValueError("ROUGE requires reference texts") diff --git a/src/atgen/metrics/linguistic/cola_metric.py b/src/atgen/metrics/linguistic/cola_metric.py index 3410aeb..9f7edb5 100644 --- a/src/atgen/metrics/linguistic/cola_metric.py +++ b/src/atgen/metrics/linguistic/cola_metric.py @@ -28,16 +28,6 @@ def __init__(self, config: Optional[MetricConfig] = None): self.tokenizer = None self.checkpoint = getattr(config, 'checkpoint', 'Aktsvigun/electra-large-cola') if config else 'Aktsvigun/electra-large-cola' - def _check_dependencies(self) -> bool: - """Check if required dependencies are available.""" - try: - import torch - from transformers import AutoModelForSequenceClassification, AutoTokenizer - from datasets import Dataset - import nltk - return True - except ImportError: - return False def _initialize_model(self): """Initialize the CoLA model if not already initialized.""" @@ -63,8 +53,6 @@ def calculate(self, predictions: List[str], references: Optional[List[Union[str, Returns: Dictionary with CoLA scores """ - if not self.is_available(): - raise RuntimeError("CoLA dependencies not available") self._initialize_model() diff --git a/src/atgen/metrics/llm_based/base_deepeval_metric.py b/src/atgen/metrics/llm_based/base_deepeval_metric.py index c881a00..ccdbef3 100644 --- a/src/atgen/metrics/llm_based/base_deepeval_metric.py +++ b/src/atgen/metrics/llm_based/base_deepeval_metric.py @@ -20,16 +20,6 @@ def __init__(self, config: Optional[MetricConfig] = None): super().__init__(config) self.llm = None - def _check_dependencies(self) -> bool: - """Check if DeepEval dependencies are available.""" - try: - from deepeval import evaluate - from deepeval.test_case import LLMTestCase - from openai import OpenAI, AsyncOpenAI - return True - except ImportError: - return False - def _validate_config(self): """Validate DeepEval configuration.""" super()._validate_config() diff --git a/src/atgen/metrics/semantic/alignscore_metric.py b/src/atgen/metrics/semantic/alignscore_metric.py index efc0bd2..2fffda2 100644 --- a/src/atgen/metrics/semantic/alignscore_metric.py +++ b/src/atgen/metrics/semantic/alignscore_metric.py @@ -3,7 +3,7 @@ from urllib.request import urlretrieve from pathlib import Path import numpy as np - +from alignscore import AlignScore from ..base.base_metric import BaseMetric, MetricConfig # Default AlignScore checkpoint path @@ -24,13 +24,6 @@ def __init__(self, config: Optional[MetricConfig] = None): self.scorer = None self.checkpoint_path = getattr(config, 'checkpoint_path', ALIGNSCORE_CHECKPOINT_PATH) if config else ALIGNSCORE_CHECKPOINT_PATH - def _check_dependencies(self) -> bool: - """Check if AlignScore is available.""" - try: - from alignscore import AlignScore - return True - except ImportError: - return False def _initialize_scorer(self): """Initialize the AlignScore scorer if not already initialized.""" @@ -43,7 +36,6 @@ def _initialize_scorer(self): self.checkpoint_path, ) - from alignscore import AlignScore self.scorer = AlignScore( model="roberta-base", batch_size=self.config.batch_size, @@ -63,10 +55,7 @@ def calculate(self, predictions: List[str], references: Optional[List[Union[str, Returns: Dictionary with AlignScore results - """ - if not self.is_available(): - raise RuntimeError("AlignScore dependencies not available") - + """ if original_texts is None: raise ValueError("AlignScore requires original texts") diff --git a/src/atgen/metrics/semantic/bart_score_metric.py b/src/atgen/metrics/semantic/bart_score_metric.py index c970a1f..f03ee7e 100644 --- a/src/atgen/metrics/semantic/bart_score_metric.py +++ b/src/atgen/metrics/semantic/bart_score_metric.py @@ -90,15 +90,7 @@ def __init__(self, config: Optional[MetricConfig] = None): self.scorer = None self.checkpoint = getattr(config, 'checkpoint', 'facebook/bart-large-cnn') if config else 'facebook/bart-large-cnn' - def _check_dependencies(self) -> bool: - """Check if required dependencies are available.""" - try: - import torch - from transformers import BartTokenizer, BartForConditionalGeneration - return True - except ImportError: - return False - + def _initialize_scorer(self): """Initialize the BARTScorer if not already initialized.""" if self.scorer is None: diff --git a/src/atgen/metrics/semantic/sentbert_metric.py b/src/atgen/metrics/semantic/sentbert_metric.py index 43d5fb0..4d0adca 100644 --- a/src/atgen/metrics/semantic/sentbert_metric.py +++ b/src/atgen/metrics/semantic/sentbert_metric.py @@ -16,14 +16,6 @@ def __init__(self, config: Optional[MetricConfig] = None): self.tokenizer = None self.checkpoint = getattr(config, 'checkpoint', 'sentence-transformers/all-mpnet-base-v2') if config else 'sentence-transformers/all-mpnet-base-v2' - def _check_dependencies(self) -> bool: - """Check if required dependencies are available.""" - try: - import torch - from transformers import AutoTokenizer, AutoModel - return True - except ImportError: - return False def _initialize_model(self): """Initialize the SentenceBERT model if not already initialized.""" @@ -110,10 +102,7 @@ def calculate(self, predictions: List[str], references: Optional[List[Union[str, Returns: Dictionary with semantic similarity scores - """ - if not self.is_available(): - raise RuntimeError("SentenceBERT dependencies not available") - + """ self._initialize_model() scores = {} From ba5551190dcf4832794e0b7f2ea59bbe0f4ab57d Mon Sep 17 00:00:00 2001 From: alfekka Date: Mon, 9 Jun 2025 00:56:17 +0300 Subject: [PATCH 03/39] added default factory --- src/atgen/metrics/__init__.py | 134 ++++ src/atgen/metrics/base/base_metric.py | 18 +- src/atgen/metrics/compute_metrics.py | 403 +++++----- src/atgen/metrics/factory.py | 206 +++++ src/atgen/metrics/lexical/bleu_metric.py | 2 - src/atgen/metrics/linguistic/cola_metric.py | 1 - src/atgen/metrics/metrics.py | 731 ------------------ src/atgen/metrics/semantic/sentbert_metric.py | 10 - 8 files changed, 537 insertions(+), 968 deletions(-) create mode 100644 src/atgen/metrics/__init__.py create mode 100644 src/atgen/metrics/factory.py delete mode 100644 src/atgen/metrics/metrics.py diff --git a/src/atgen/metrics/__init__.py b/src/atgen/metrics/__init__.py new file mode 100644 index 0000000..f4ef337 --- /dev/null +++ b/src/atgen/metrics/__init__.py @@ -0,0 +1,134 @@ +""" +Metrics module for evaluating text generation tasks. + +This module provides a comprehensive collection of metrics organized by category: +- Lexical: BLEU, ROUGE +- Semantic: BARTScore, AlignScore, SentBERT +- Linguistic: CoLA +- LLM-based: DeepEval metrics, BigBenchHard + +The module uses a factory pattern for creating metrics and supports +configuration-based instantiation. +""" + +# Base classes +from .base import BaseMetric, MetricConfig + +# Lexical metrics +from .lexical import BleuMetric, RougeMetric + +# Semantic metrics +from .semantic import BartScoreMetric, AlignScoreMetric, SentBertMetric + +# Linguistic metrics +from .linguistic import ColaMetric + +# LLM-based metrics +from .llm_based import ( + EvaluationLLM, + BaseDeepEvalMetric, + DeepEvalAnswerRelevancyMetric, + DeepEvalFaithfulnessMetric, + DeepEvalSummarizationMetric, + DeepEvalPromptAlignmentMetric, + BigBenchHardMetric, +) + +# Factory system +from .factory import ( + MetricsFactory, + MetricsConfig, + get_metric_categories, + get_metric_requirements, +) + +# Compute system +from .compute_metrics_v2 import ( + compute_metrics_v2, + compute_metrics_from_config, + get_default_config, + get_comprehensive_config, + get_deepeval_config, +) + +# Legacy compute system (for backward compatibility) +from .compute_metrics import compute_metrics + +# Legacy metrics functions (for backward compatibility) +from .metrics import ( + compute_bleu, + compute_rouge, + compute_bartscore, + compute_alignscore, + compute_sentbert, + compute_cola, +) + + +# Version +__version__ = "2.0.0" + +# Main exports +__all__ = [ + # Base classes + "BaseMetric", + "MetricConfig", + + # Lexical metrics + "BleuMetric", + "RougeMetric", + + # Semantic metrics + "BartScoreMetric", + "AlignScoreMetric", + "SentBertMetric", + + # Linguistic metrics + "ColaMetric", + + # LLM-based metrics + "EvaluationLLM", + "BaseDeepEvalMetric", + "DeepEvalAnswerRelevancyMetric", + "DeepEvalFaithfulnessMetric", + "DeepEvalSummarizationMetric", + "DeepEvalPromptAlignmentMetric", + "BigBenchHardMetric", + + # Factory system + "MetricsFactory", + "MetricsConfig", + "get_metric_categories", + "get_metric_requirements", + + # New compute system + "compute_metrics_v2", + "compute_metrics_from_config", + "get_default_config", + "get_comprehensive_config", + "get_deepeval_config", + + # Legacy compatibility + "compute_metrics", + "compute_bleu", + "compute_rouge", + "compute_bartscore", + "compute_alignscore", + "compute_sentbert", + "compute_cola", +] + + +def get_available_metrics(): + """Get all available metrics.""" + return MetricsFactory.get_available_metrics() + + +def create_metric(name: str, config: MetricConfig = None): + """Create a metric by name.""" + return MetricsFactory.create_metric(name, config) + + +def create_metrics_from_config(config): + """Create metrics from configuration.""" + return MetricsFactory.create_from_config(config) \ No newline at end of file diff --git a/src/atgen/metrics/base/base_metric.py b/src/atgen/metrics/base/base_metric.py index 6940bb6..048b1ec 100644 --- a/src/atgen/metrics/base/base_metric.py +++ b/src/atgen/metrics/base/base_metric.py @@ -6,31 +6,24 @@ @dataclass class MetricConfig: - """Base configuration for metrics.""" - # General parameters batch_size: int = 32 device: str = "cuda" cache_dir: str = "cache" aggregate: bool = True - # Model-specific parameters (for local models) checkpoint: Optional[str] = None model_name: Optional[str] = None - # API-based parameters (for LLM-based metrics) api_key: Optional[str] = None base_url: Optional[str] = None model: Optional[str] = None - # DeepEval specific parameters threshold: float = 0.5 include_reason: bool = False strict_mode: bool = False async_mode: bool = True verbose_mode: bool = False truths_extraction_limit: Optional[int] = None - - # BigBenchHard specific parameters benchmark_params: Dict[str, Any] = field(default_factory=dict) generation_params: Dict[str, Any] = field(default_factory=dict) @@ -39,23 +32,14 @@ class BaseMetric(ABC): def __init__(self, config: Optional[MetricConfig] = None): self.config = config or MetricConfig() self.logger = logging.getLogger(self.__class__.__name__) - self._is_available = None - self._model = None - # Validate configuration self._validate_config() - - # Check dependencies - if not self.is_available(): - self.logger.warning(f"{self.__class__.__name__} dependencies not available") - + @property def name(self) -> str: - """Return the metric name (defaults to class name).""" return self.__class__.__name__.replace("Metric", "").lower() def _validate_config(self): - """Validate configuration - can be overridden by subclasses.""" pass @abstractmethod diff --git a/src/atgen/metrics/compute_metrics.py b/src/atgen/metrics/compute_metrics.py index 87e57f3..4632b1a 100644 --- a/src/atgen/metrics/compute_metrics.py +++ b/src/atgen/metrics/compute_metrics.py @@ -1,231 +1,220 @@ from time import time -from typing import List, Dict +from typing import List, Dict, Union, Optional import logging -from omegaconf import DictConfig - import numpy as np -from evaluate import load -from tqdm import tqdm - -from .metrics import ( - pair_bleu, - calculate_bart_score, - calculate_alignscore, - calculate_deepeval_metrics, - is_bart_score_available, - is_alignscore_available, -) -from .supported_models_and_metrics import API_MODELS, DEEPEVAL_METRICS +from omegaconf import DictConfiga +from .factory import MetricsFactory, MetricsConfig, get_metric_requirements +from .base import MetricConfig -log = logging.getLogger() +logger = logging.getLogger(__name__) def compute_metrics( - generated_texts, - reference_texts, - original_texts, - config: DictConfig, - cache_dir: str = "cache", + predictions: List[str], + references: Optional[List[Union[str, List[str]]]] = None, + original_texts: Optional[List[str]] = None, + metrics_config: Optional[Union[Dict, DictConfig, MetricsConfig]] = None, + model=None, + tokenizer=None, ) -> Dict[str, float]: """ - Compute various metrics for generated texts. - + Compute various metrics for generated texts using the new architecture. + Args: - generated_texts: List of generated texts to evaluate - reference_texts: List of reference texts (ground truth) or list of lists of reference texts + predictions: List of generated texts to evaluate + references: List of reference texts (ground truth) or list of lists for multiple references original_texts: List of source texts - config: Configuration for evaluation - - additional_metrics: List of additional metrics to use. Options include: - - "bartscore": BARTScore metrics - - "alignscore": AlignScore metrics - - DeepEval metrics (requires API key): - - "deepeval_answer_relevance": Evaluates how well the output answers the input - - "deepeval_faithfulness": Evaluates factual consistency with the input - - "deepeval_summarization": Evaluates summarization quality - - "deepeval_prompt_alignment": Evaluates alignment with the expected output - - provider: API key for the provider - - api_key: Model identifier to use - - model: Provider name (openai, anthropic, openrouter, or custom) - - base_url: API base URL (if None, uses default for the provider) - - deepeval_threshold: Threshold for DeepEval metrics (default: 0.5) - - deepeval_include_reason: Include reason for evaluation score (default: False) - - deepeval_strict_mode: Enforce binary metric score (default: False) - - deepeval_async_mode: Enable concurrent execution (default: True) - - deepeval_verbose_mode: Print intermediate steps (default: False) - - deepeval_truths_extraction_limit: Maximum number of factual truths to extract (default: None) + metrics_config: Configuration for metrics (dict, DictConfig, or MetricsConfig) + model: Model instance (required for BigBenchHard) + tokenizer: Tokenizer instance (required for BigBenchHard) + Returns: - Dictionary with metric scores - - Note: - The OpenRouterLLM class is also available for direct use with DeepEval metrics: - - ```python - from atgen.metrics import OpenRouterLLM - from deepeval.metrics import AnswerRelevanceMetric - - llm = OpenRouterLLM( - api_key="your_openrouter_api_key", - model="openai/gpt-4o-2024-11-20" - ) - - metric = AnswerRelevanceMetric(model=llm) - ``` + Dictionary with metric scores and timing information """ - # Load metrics that are always used - sacrebleu = load("sacrebleu", cache_dir=cache_dir) - rouge = load("rouge", cache_dir=cache_dir) - - result = {} - result["word_length_gen"] = np.array( - [len(text.split()) for text in generated_texts] - ) - - time_dict = {} - - # Metrics that use both the generated texts and the original texts and - # those that do not require reference texts - src_word_lengths = np.array([len(text.split()) for text in original_texts]) - - # Avoid division by zero - src_word_lengths_safe = np.where(src_word_lengths > 0, src_word_lengths, 1) - result["word_length_src_rel"] = result["word_length_gen"] / src_word_lengths_safe - if "bartscore" in config.additional_metrics: - log.info("Calculating BARTScore scores...") - start_time = time() - result.update( - calculate_bart_score( - preds=generated_texts, - texts=original_texts, - refs=reference_texts, - batch_size=4, - cache_dir=cache_dir, - ) + start_total = time() + + # Handle different config types + if metrics_config is None: + # Default configuration + config = MetricsConfig( + metrics=["bleu", "rouge"], + batch_size=32, + device="cuda", + aggregate=True ) - time_dict["time_bartscore"] = time() - start_time - # Metrics that use both the generated texts and the reference texts - if reference_texts is not None: - # Exact match - if isinstance(reference_texts[0], list): - result["exact_match"] = np.array( - [ - any(pred == one_ref for one_ref in ref) - for pred, ref in zip(generated_texts, reference_texts) - ] - ) + elif isinstance(metrics_config, dict): + config = MetricsConfig(**metrics_config) + elif isinstance(metrics_config, DictConfig): + # Convert OmegaConf to dict then to MetricsConfig + config_dict = dict(metrics_config) + config = MetricsConfig(**config_dict) + else: + config = metrics_config + + # Validate inputs + if not predictions: + raise ValueError("predictions cannot be empty") + + # Get metric requirements + requirements = get_metric_requirements() + + # Create metrics using factory + metrics = MetricsFactory.create_metrics(config) + + if not metrics: + logger.warning("No metrics were successfully created") + return {} + + results = {} + timing_info = {} + + # Basic statistics + results["word_length_gen"] = float(np.mean([len(text.split()) for text in predictions])) + + if original_texts: + src_lengths = np.array([len(text.split()) for text in original_texts]) + gen_lengths = np.array([len(text.split()) for text in predictions]) + # Avoid division by zero + src_lengths_safe = np.where(src_lengths > 0, src_lengths, 1) + results["word_length_src_rel"] = float(np.mean(gen_lengths / src_lengths_safe)) + + if references: + if isinstance(references[0], list): + ref_lengths = np.array([len(refs[0].split()) for refs in references]) else: - result["exact_match"] = np.array( - [pred == ref for pred, ref in zip(generated_texts, reference_texts)] - ) - # BLEU - start_time = time() - result["bleu"] = np.array( - [ - pair_bleu(references=ref, prediction=pred) - for pred, ref in tqdm(zip(generated_texts, reference_texts)) + ref_lengths = np.array([len(text.split()) for text in references]) + + gen_lengths = np.array([len(text.split()) for text in predictions]) + if isinstance(references[0], list): + exact_matches = [ + any(pred == ref for ref in ref_list) + for pred, ref_list in zip(predictions, references) ] - ) - time_dict["time_bleu"] = time() - start_time - # ROUGE - start_time = time() - result.update( - rouge.compute( - predictions=generated_texts, - references=reference_texts, - use_stemmer=True, - ) - ) - time_dict["time_rouge"] = time() - start_time - # Sacrebleu + else: + exact_matches = [pred == ref for pred, ref in zip(predictions, references)] + results["exact_match"] = float(np.mean(exact_matches)) + + ref_lengths_safe = np.where(ref_lengths > 0, ref_lengths, 1) + results["word_length_rel"] = float(np.mean(gen_lengths / ref_lengths_safe)) + + for metric_name, metric in metrics.items(): + logger.info(f"Computing {metric_name}...") start_time = time() - sacrebleu_references = ( - [[ref] for ref in reference_texts] - if not isinstance(reference_texts[0], list) - else reference_texts - ) - sacrebleu_result = sacrebleu.compute( - predictions=generated_texts, references=sacrebleu_references - ) - result["sacrebleu"] = sacrebleu_result.pop("score") - time_dict["time_sacrebleu"] = time() - start_time - # Lengths - ref_word_lengths = np.array([len(text.split()) for text in reference_texts]) - # Avoid division by zero - ref_word_lengths_safe = np.where(ref_word_lengths > 0, ref_word_lengths, 1) - result["word_length_rel"] = result["word_length_gen"] / ref_word_lengths_safe + + try: + req = requirements.get(metric_name, {}) + + if req.get("requires_references", False) and references is None: + logger.warning(f"Skipping {metric_name}: requires references but none provided") + continue + + if req.get("requires_original_texts", False) and original_texts is None: + logger.warning(f"Skipping {metric_name}: requires original texts but none provided") + continue + + if metric_name in ["bigbench_hard", "big_bench_hard"]: + if model is None or tokenizer is None: + logger.warning(f"Skipping {metric_name}: requires model and tokenizer") + continue + + metric.set_model_and_tokenizer(model, tokenizer) + metric_results = metric.calculate([], [], []) + else: + metric_results = metric.calculate(predictions, references, original_texts) + + for key, value in metric_results.items(): + results[key] = value + + timing_info[f"time_{metric_name}"] = time() - start_time + logger.info(f"Completed {metric_name} in {timing_info[f'time_{metric_name}']:.2f}s") + + except Exception as e: + logger.error(f"Error computing {metric_name}: {e}") + timing_info[f"time_{metric_name}"] = time() - start_time + + # Add total timing + timing_info["time_total"] = time() - start_total + results.update(timing_info) + + logger.info(f"Computed {len(metrics)} metrics in {timing_info['time_total']:.2f}s") + return results + + +def compute_metrics_from_config( + predictions: List[str], + references: Optional[List[Union[str, List[str]]]] = None, + original_texts: Optional[List[str]] = None, + config_dict: Dict = None, + model=None, + tokenizer=None, +) -> Dict[str, float]: + """ + Convenience function to compute metrics from a configuration dictionary. + + Args: + predictions: List of generated texts + references: List of reference texts + original_texts: List of source texts + config_dict: Configuration dictionary + model: Model instance (for BigBenchHard) + tokenizer: Tokenizer instance (for BigBenchHard) + + Returns: + Dictionary with metric scores + """ + return compute_metrics( + predictions=predictions, + references=references, + original_texts=original_texts, + metrics_config=config_dict, + model=model, + tokenizer=tokenizer + ) - # AlignScore - if "alignscore" in config.additional_metrics: - log.info("Calculating AlignScore scores...") - start_time = time() - alignscores = calculate_alignscore( - generated_texts, reference_texts, original_texts - ) - if alignscores is not None: - result.update(alignscores) - time_dict["time_alignscore"] = time() - start_time - # DeepEval metrics - deepeval_metrics_to_calculate = [ - metric for metric in DEEPEVAL_METRICS if metric in config.additional_metrics - ] +def get_default_config() -> MetricsConfig: + """Get a default metrics configuration.""" + return MetricsConfig( + metrics=["bleu", "rouge"], + batch_size=32, + device="cuda", + cache_dir="cache", + aggregate=True + ) - if deepeval_metrics_to_calculate: - if isinstance(reference_texts[0], list): - log.error("DeepEval does not support multiple references. Skipping...") - else: - # Validate OpenRouter model - only warn if not in predefined list, but still use it - provider = config["provider"] - if config.model not in API_MODELS.get(provider): - log.warning( - f"Using custom model: {config.model}. " - + ( - f"Available models: {API_MODELS[provider]}" - if provider in API_MODELS - else "" - ) - ) - log.info( - f"Calculating DeepEval metrics: {', '.join(deepeval_metrics_to_calculate)}..." - ) - start_time = time() - result.update( - calculate_deepeval_metrics( - predictions=generated_texts, - references=reference_texts, - original_texts=original_texts, - metrics_to_calculate=deepeval_metrics_to_calculate, - base_url=config.base_url, - api_key=config.api_key, - model=config.model, - threshold=config.deepeval_threshold, - include_reason=config.deepeval_include_reason, - strict_mode=config.deepeval_strict_mode, - async_mode=config.deepeval_async_mode, - verbose_mode=config.deepeval_verbose_mode, - truths_extraction_limit=config.deepeval_truths_extraction_limit, - ) - ) - time_dict["time_deepeval"] = time() - start_time - for key, value in result.items(): - if isinstance(value, np.ndarray): - result[key] = float(np.mean(value)) - elif isinstance(value, (int, float)): - # Ensure numerical values are converted to float - result[key] = float(value) - # Make sure non-numerical values that aren't reasons are preserved - elif not key.endswith("_reasons") and not "_reason" in key.lower(): - continue +def get_comprehensive_config() -> MetricsConfig: + """Get a comprehensive metrics configuration with all metrics.""" + return MetricsConfig( + metrics=[ + "bleu", "rouge", + "sentbert", + "cola", + ], + batch_size=16, + device="cuda", + cache_dir="cache", + aggregate=True + ) - # Filter out reason fields from the final aggregated results - more robust filtering - result = { - key: value - for key, value in sorted(result.items()) - if not key.endswith("_reasons") - and not "_reason" in key.lower() - and isinstance(value, (int, float)) # Ensure we only keep numerical metrics - } - result.update(time_dict) - return result +def get_deepeval_config(api_key: str, model: str = "openai/gpt-4o-mini") -> MetricsConfig: + return MetricsConfig( + metrics=[ + "deepeval_answer_relevance", + "deepeval_faithfulness", + "deepeval_summarization", + "deepeval_prompt_alignment" + ], + batch_size=8, + device="cpu", + cache_dir="cache", + aggregate=True, + api_key=api_key, + base_url="https://openrouter.ai/api/v1", + model=model, + threshold=0.5, + async_mode=True, + verbose_mode=False + ) \ No newline at end of file diff --git a/src/atgen/metrics/factory.py b/src/atgen/metrics/factory.py new file mode 100644 index 0000000..f777676 --- /dev/null +++ b/src/atgen/metrics/factory.py @@ -0,0 +1,206 @@ +from typing import Dict, List, Optional, Union, Any +from dataclasses import dataclass, field +import logging + +from .base import BaseMetric, MetricConfig +from .lexical import BleuMetric, RougeMetric +from .semantic import BartScoreMetric, AlignScoreMetric, SentBertMetric +from .linguistic import ColaMetric +from .llm_based import ( + DeepEvalAnswerRelevancyMetric, + DeepEvalFaithfulnessMetric, + DeepEvalSummarizationMetric, + DeepEvalPromptAlignmentMetric, + BigBenchHardMetric, +) + +logger = logging.getLogger(__name__) + + +@dataclass +class MetricsConfig: + """Configuration for the metrics system.""" + + batch_size: int = 32 + device: str = "cuda" + cache_dir: str = "cache" + aggregate: bool = True + + metrics: List[str] = field(default_factory=list) + + checkpoint: Optional[str] = None + model_name: Optional[str] = None + + api_key: Optional[str] = None + base_url: Optional[str] = None + model: Optional[str] = None + + threshold: float = 0.5 + include_reason: bool = False + strict_mode: bool = False + async_mode: bool = True + verbose_mode: bool = False + truths_extraction_limit: Optional[int] = None + + benchmark_params: Dict[str, Any] = field(default_factory=dict) + generation_params: Dict[str, Any] = field(default_factory=dict) + custom_params: Dict[str, Any] = field(default_factory=dict) + + +class MetricsFactory: + """Factory for creating metric instances.""" + + # Registry mapping metric names to classes + _metric_registry = { + # Lexical metrics + "bleu": BleuMetric, + "rouge": RougeMetric, + + # Semantic metrics + "bartscore": BartScoreMetric, + "alignscore": AlignScoreMetric, + "sentbert": SentBertMetric, + "sentence_bert": SentBertMetric, # Alias + + # Linguistic metrics + "cola": ColaMetric, + "grammaticality": ColaMetric, # Alias + + # LLM-based metrics + "deepeval_answer_relevance": DeepEvalAnswerRelevancyMetric, + "deepeval_faithfulness": DeepEvalFaithfulnessMetric, + "deepeval_summarization": DeepEvalSummarizationMetric, + "deepeval_prompt_alignment": DeepEvalPromptAlignmentMetric, + + # Benchmark metrics + "bigbench_hard": BigBenchHardMetric, + "big_bench_hard": BigBenchHardMetric, # Alias + } + + @classmethod + def register_metric(cls, name: str, metric_class: type): + """Register a new metric class.""" + cls._metric_registry[name] = metric_class + logger.info(f"Registered metric: {name} -> {metric_class.__name__}") + + @classmethod + def get_available_metrics(cls) -> List[str]: + """Get list of all available metric names.""" + return list(cls._metric_registry.keys()) + + @classmethod + def create_metric(cls, name: str, config: Optional[MetricConfig] = None) -> BaseMetric: + """ + Create a single metric instance. + + Args: + name: Name of the metric to create + config: Configuration for the metric + + Returns: + Metric instance + + Raises: + ValueError: If metric name is not recognized + """ + name = name.lower().strip() + + if name not in cls._metric_registry: + available = ", ".join(cls.get_available_metrics()) + raise ValueError(f"Unknown metric '{name}'. Available metrics: {available}") + + metric_class = cls._metric_registry[name] + return metric_class(config) + + @classmethod + def create_metrics(cls, metrics_config: MetricsConfig) -> Dict[str, BaseMetric]: + """ + Create multiple metric instances from configuration. + + Args: + metrics_config: Configuration containing list of metrics to create + + Returns: + Dictionary mapping metric names to instances + """ + base_config = MetricConfig( + batch_size=metrics_config.batch_size, + device=metrics_config.device, + cache_dir=metrics_config.cache_dir, + aggregate=metrics_config.aggregate, + checkpoint=metrics_config.checkpoint, + model_name=metrics_config.model_name, + api_key=metrics_config.api_key, + base_url=metrics_config.base_url, + model=metrics_config.model, + threshold=metrics_config.threshold, + include_reason=metrics_config.include_reason, + strict_mode=metrics_config.strict_mode, + async_mode=metrics_config.async_mode, + verbose_mode=metrics_config.verbose_mode, + truths_extraction_limit=metrics_config.truths_extraction_limit, + benchmark_params=metrics_config.benchmark_params, + generation_params=metrics_config.generation_params, + ) + + metrics = {} + for metric_name in metrics_config.metrics: + try: + metric = cls.create_metric(metric_name, base_config) + metrics[metric_name] = metric + logger.info(f"Created metric: {metric_name}") + except Exception as e: + logger.error(f"Failed to create metric '{metric_name}': {e}") + + return metrics + + @classmethod + def create_from_config(cls, config: Union[Dict[str, Any], MetricsConfig]) -> Dict[str, BaseMetric]: + """ + Create metrics from dictionary or MetricsConfig. + + Args: + config: Configuration dictionary or MetricsConfig instance + + Returns: + Dictionary mapping metric names to instances + """ + if isinstance(config, dict): + metrics_config = MetricsConfig(**config) + else: + metrics_config = config + + return cls.create_metrics(metrics_config) + + +def get_metric_categories() -> Dict[str, List[str]]: + """Get metrics organized by category.""" + return { + "lexical": ["bleu", "rouge"], + "semantic": ["bartscore", "alignscore", "sentbert"], + "linguistic": ["cola"], + "llm_based": [ + "deepeval_answer_relevance", + "deepeval_faithfulness", + "deepeval_summarization", + "deepeval_prompt_alignment" + ], + "benchmark": ["bigbench_hard"] + } + + +def get_metric_requirements() -> Dict[str, Dict[str, bool]]: + """Get input requirements for each metric.""" + return { + "bleu": {"requires_references": True, "requires_original_texts": False}, + "rouge": {"requires_references": True, "requires_original_texts": False}, + "bartscore": {"requires_references": True, "requires_original_texts": True}, + "alignscore": {"requires_references": True, "requires_original_texts": True}, + "sentbert": {"requires_references": False, "requires_original_texts": False}, + "cola": {"requires_references": False, "requires_original_texts": False}, + "deepeval_answer_relevance": {"requires_references": False, "requires_original_texts": True}, + "deepeval_faithfulness": {"requires_references": False, "requires_original_texts": True}, + "deepeval_summarization": {"requires_references": False, "requires_original_texts": True}, + "deepeval_prompt_alignment": {"requires_references": True, "requires_original_texts": True}, + "bigbench_hard": {"requires_references": False, "requires_original_texts": False}, + } \ No newline at end of file diff --git a/src/atgen/metrics/lexical/bleu_metric.py b/src/atgen/metrics/lexical/bleu_metric.py index 260cc25..10d985e 100644 --- a/src/atgen/metrics/lexical/bleu_metric.py +++ b/src/atgen/metrics/lexical/bleu_metric.py @@ -7,7 +7,6 @@ from ..base.base_metric import BaseMetric, MetricConfig -# Ensure nltk data is downloaded try: word_tokenize("test") except LookupError: @@ -38,7 +37,6 @@ def smoothing_function(p_n, references, hypothesis, hyp_len): class BleuMetric(BaseMetric): - """BLEU (Bilingual Evaluation Understudy) metric implementation.""" @property def category(self) -> str: diff --git a/src/atgen/metrics/linguistic/cola_metric.py b/src/atgen/metrics/linguistic/cola_metric.py index 9f7edb5..3158eb0 100644 --- a/src/atgen/metrics/linguistic/cola_metric.py +++ b/src/atgen/metrics/linguistic/cola_metric.py @@ -12,7 +12,6 @@ from ..base.base_metric import BaseMetric, MetricConfig -# Ensure NLTK data is downloaded try: nltk.data.find('tokenizers/punkt') except LookupError: diff --git a/src/atgen/metrics/metrics.py b/src/atgen/metrics/metrics.py deleted file mode 100644 index 551b4f5..0000000 --- a/src/atgen/metrics/metrics.py +++ /dev/null @@ -1,731 +0,0 @@ -from math import ceil -import os -import sys -from openai import OpenAI, AsyncOpenAI -from typing import List, Dict -from urllib.request import urlretrieve -import logging -from pathlib import Path -import nltk -import numpy as np -import torch -import torch.nn.functional as F -from datasets import Dataset -from nltk import ngrams -from nltk.stem import porter -from nltk.tokenize import word_tokenize, sent_tokenize -from nltk.translate.bleu_score import corpus_bleu -from rouge_score import tokenize -from torch.utils.data import DataLoader -from transformers import ( - AutoModelForSequenceClassification, - AutoTokenizer, - AutoModel, - DataCollatorWithPadding, -) - -log = logging.getLogger(__name__) - -try: - from .bart_score import BARTScorer - - is_bart_score_available = True -except ImportError: - log.warning( - "BARTScorer not found, please install it (see `install.sh`). Skipping the BARTScore metric." - ) - is_bart_score_available = False - -try: - from alignscore import AlignScore - - is_alignscore_available = True -except ImportError: - log.warning( - "AlignScore not found, please install it (see `install.sh`). Skipping the AlignScore metric." - ) - is_alignscore_available = False - -from deepeval import evaluate -from deepeval.models.base_model import DeepEvalBaseLLM -from deepeval.test_case import LLMTestCase -from deepeval.metrics import ( - AnswerRelevancyMetric, - FaithfulnessMetric, - SummarizationMetric, - PromptAlignmentMetric, -) - - -ALIGNSCORE_CHECKPOINT_PATH = os.getenv( - "ALIGNSCORE_CHECKPOINT_PATH", - # Going up 3 levels from metrics.py: src/atgen/metrics -> repository root - os.path.join( - Path(__file__).parents[3], - "external_metrics/AlignScore/model/AlignScore-base.ckpt", - ), -) - - -def decode(eval_preds, tokenizer): - predictions, labels, *inputs = eval_preds - predictions = np.where(predictions != -100, predictions, tokenizer.pad_token_id) - decoded_preds = tokenizer.batch_decode(predictions, skip_special_tokens=True) - # Replace -100 in the labels as we can't decode them. - labels = np.where(labels != -100, labels, tokenizer.pad_token_id) - decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True) - - decoded_preds = [pred.strip() for pred in decoded_preds] - decoded_labels = [label.strip() for label in decoded_labels] - - if len(inputs) > 0: - input_ids = inputs[0] - input_ids = np.where(input_ids != -100, input_ids, tokenizer.pad_token_id) - decoded_texts = tokenizer.batch_decode(input_ids, skip_special_tokens=True) - decoded_texts = [text.strip() for text in decoded_texts] - return decoded_preds, decoded_labels, decoded_texts - - return decoded_preds, decoded_labels - - -def smoothing_function(p_n, references, hypothesis, hyp_len): - """ - Smooth-BLEU (BLEUS) as proposed in the paper: - Chin-Yew Lin, Franz Josef Och. ORANGE: a method for evaluating automatic - evaluation metrics for machine translation. COLING 2004. - """ - smoothed_p_n = [] - for i, p_i in enumerate(p_n, start=1): - # Smoothing is not applied for unigrams - if i > 1: - # If hypothesis length is lower than the current order, its value equals (0 + 1) / (0 + 1) = 0 - if hyp_len < i: - assert p_i.denominator == 1 - smoothed_p_n.append(1) - # Otherwise apply smoothing - else: - smoothed_p_i = (p_i.numerator + 1) / (p_i.denominator + 1) - smoothed_p_n.append(smoothed_p_i) - else: - smoothed_p_n.append(p_i) - return smoothed_p_n - - -def pair_bleu(references, prediction): - """ - Compute the bleu score between two given texts. - A smoothing function is used to avoid zero scores when - there are no common higher order n-grams between the - texts. - """ - if not isinstance(references, list): - references = [references] - tok_ref = [word_tokenize(ref) for ref in references] - tok_pred = word_tokenize(prediction) - try: - return corpus_bleu(tok_ref, [tok_pred], smoothing_function=smoothing_function) - except (KeyError, ZeroDivisionError): - return 0.0 - - -def calculate_bart_score( - preds, - refs=None, - texts=None, - scorer=None, - batch_size=4, - aggregate=True, - cache_dir: str = "cache", -): - if not is_bart_score_available: - return None - if scorer is None: - scorer = BARTScorer(cache_dir=cache_dir) - scores = {} - if texts is not None: - scores["BARTScore-sh"] = np.array( - scorer.score(texts, preds, batch_size=batch_size) - ) - if refs is not None: - # scores["BARTScore-rh"] = np.array(scorer.score(refs, preds, batch_size=batch_size)) - if isinstance(refs[0], list): - scores_hr = [] - for ref, pred in zip(refs, preds): - inst_pred = [pred for _ in range(len(ref))] - # Take a maximum within the observation similar to ROUGE - inst_score_hr = max(scorer.score(inst_pred, ref, batch_size=batch_size)) - scores_hr.append(inst_score_hr) - scores["BARTScore-hr"] = np.array(scores_hr) - else: - scores["BARTScore-hr"] = np.array( - scorer.score(preds, refs, batch_size=batch_size) - ) - # scores["BARTScore-fa"] = (scores["BARTScore-rh"] + scores["BARTScore-hr"]) / 2 - - if aggregate: - scores = {key: np.mean(value) for key, value in scores.items()} - return scores - - -def calculate_summac_score( - predictions: List[str], - texts: List[str], - labels: List[str] = None, - aggregate: bool = True, -) -> Dict[str, np.ndarray]: - scorer = SummaCZS(granularity="sentence", model_name="vitc") - preds_score = scorer.score(texts, predictions)["scores"] - if labels is not None: - labels_score = scorer.score(texts, labels)["scores"] - rel_score = np.array(preds_score) / np.array(labels_score) - if aggregate: - preds_score = np.mean(preds_score) - if labels is not None: - rel_score = np.mean(rel_score) - if labels is not None: - return {"SummaC-tp": preds_score, "SummaC-rel": rel_score} - return {"SummaC-tp": preds_score} - - -def calculate_cola_model_predictions( - texts, - checkpoint="Aktsvigun/electra-large-cola", - batch_size=64, - device="cuda", - return_sent_data: bool = False, - aggregate: bool = True, - cache_dir: str = "cache", -): - model = AutoModelForSequenceClassification.from_pretrained( - "Aktsvigun/electra-large-cola" - ).to(device) - tokenizer = AutoTokenizer.from_pretrained(checkpoint) - - text_sentences = [nltk.sent_tokenize(text) for text in texts] - len_maps = np.cumsum([len(x) for x in text_sentences]) - sentences = [sent for text in text_sentences for sent in text] - - def tokenize_fn(instance): - return tokenizer(instance["text"], truncation=True) - - tokenized_data = Dataset.from_dict({"text": sentences}).map( - tokenize_fn, remove_columns=["text"], batched=True - ) - data_collator = DataCollatorWithPadding(tokenizer=tokenizer) - dataloader = DataLoader( - tokenized_data, batch_size=batch_size, shuffle=False, collate_fn=data_collator - ) - - sent_probas = torch.empty(len(sentences), dtype=torch.float32, device=device) - probas = torch.empty(len(texts), dtype=torch.float32, device=device) - start = 0 - end = batch_size - with torch.no_grad(): - for i, batch in enumerate(dataloader): - batch_pred = model(**{k: v.cuda() for k, v in batch.items()}) - batch_probas = 1 / (1 + (-batch_pred.logits[:, 1]).exp()) - sent_probas[start:end].copy_(batch_probas) - start = end - end += batch_size - - for i, end_idx in enumerate(len_maps): - start_idx = len_maps[i - 1] if i != 0 else 0 - probas[i].copy_(sent_probas[start_idx:end_idx].mean()) - - if aggregate: - return probas.mean().item() - if return_sent_data: - return ( - probas.cpu().detach().numpy(), - sent_probas.cpu().detach().numpy(), - sentences, - ) - return probas.cpu().detach().numpy() - - -def calculate_infolm_score(predictions, references, batch_size=4): - from torchmetrics.text.infolm import InfoLM - - assert len(predictions) == len(references), "Lengths must coincide!" - infolm_fr = InfoLM(measure_to_use="fisher_rao") - infolm_ab = InfoLM(measure_to_use="ab", alpha=1.0, beta=1.0) - - infolm_fr_scores, infolm_ab_scores = [], [] - idf_ref, idf_hyps = infolm_fr.prepare_idfs(references, predictions) - - num_batches = ceil(len(predictions) / batch_size) - for i in range(num_batches): - batch_preds = predictions[i * batch_size : (i + 1) * batch_size] - batch_refs = references[i * batch_size : (i + 1) * batch_size] - - infolm_fr_scores += infolm_fr.evaluate_batch( - batch_preds, batch_refs, idf_ref=idf_ref, idf_hyps=idf_hyps - )["fisher_rao"] - infolm_ab_scores = infolm_ab.evaluate_batch( - batch_preds, batch_refs, idf_ref=idf_ref, idf_hyps=idf_hyps - )["ab"] - - return infolm_fr_scores, infolm_ab_scores - - -def calculate_abstractiveness_scores( - predictions, texts, references=None, aggregate: bool = True -): - stemmer = porter.PorterStemmer() - tokenized_preds = [tokenize.tokenize(x, stemmer) for x in predictions] - tokenized_texts = [tokenize.tokenize(x, stemmer) for x in texts] - if references is not None: - tokenized_refs = [tokenize.tokenize(x, stemmer) for x in references] - else: - tokenized_refs = tokenized_preds - - result = {} - for use_modified in [False, True]: - for n in range(1, 5): - pred_ngram_overlaps = [] - label_ngram_overlaps = [] - for pred, label, text in zip( - tokenized_preds, tokenized_refs, tokenized_texts - ): - pred_pair_ngram_overlap = calculate_ngram_overlap( - pred, text, n, use_modified - ) - pred_ngram_overlaps.append(pred_pair_ngram_overlap) - if references is not None: - label_pair_ngram_overlap = calculate_ngram_overlap( - label, text, n, use_modified - ) - label_ngram_overlaps.append(label_pair_ngram_overlap) - key = f"ngram_overlap_{n}" if use_modified else f"novel_ngrams_{n}" - - pred_ngram_overlaps = np.array(pred_ngram_overlaps) - cond_abs = ~np.isnan(pred_ngram_overlaps) - result[key + "_abs"] = pred_ngram_overlaps[cond_abs] - - if references is not None: - label_ngram_overlaps = np.array(label_ngram_overlaps) - cond_rel = cond_abs & ~np.isnan(label_ngram_overlaps) - result[key + "_rel"] = ( - pred_ngram_overlaps[cond_rel] / label_ngram_overlaps[cond_rel] - ) - - if aggregate: - for key, value in result.items(): - result[key] = np.mean(value) - - return result - - -def calculate_ngram_overlap(summary, text, n=1, use_modified=True): - summary_ngrams = list(ngrams(summary, n)) - text_ngrams = list(ngrams(text, n)) - - if len(summary_ngrams) > 0: - ngrams_intersection = set(summary_ngrams).intersection(set(text_ngrams)) - if use_modified: - word_is_part_of_ngram_copied = [ - any((x in ngram for ngram in ngrams_intersection)) for x in summary - ] - return 1 - sum(word_is_part_of_ngram_copied) / len( - word_is_part_of_ngram_copied - ) - else: - return sum([x not in ngrams_intersection for x in summary_ngrams]) / len( - summary_ngrams - ) - return np.nan - - -class SentBert: - def __init__( - self, - checkpoint: str = "sentence-transformers/all-mpnet-base-v2", - device: str = "cuda", - cache_dir: str = "cache", - ): - self.tokenizer = AutoTokenizer.from_pretrained(checkpoint, cache_dir=cache_dir) - self.model = AutoModel.from_pretrained(checkpoint, cache_dir=cache_dir).to( - device - ) - self.device = device - - def __call__( - self, source_texts: List[str], ref_texts: List[str], batch_size: int = 32 - ) -> np.ndarray: - assert len(source_texts) == len(ref_texts) - # Make batch_size an even number - if batch_size % 2 == 0: - batch_size -= 1 - half_batch_size = batch_size // 2 - n_texts = len(source_texts) - scores = np.empty(n_texts, dtype=np.float32) - start = 0 - end = 0 - - while end < n_texts: - end += half_batch_size - batch_idx = slice(start, end) - # Tokenize sentences - encoded_input = self.tokenizer( - source_texts[batch_idx] + ref_texts[batch_idx], - padding=True, - truncation=True, - return_tensors="pt", - ) - encoded_input = { - key: value.to(self.device) for key, value in encoded_input.items() - } - # Calculate the probability of belonging to the positive class - model_output = self.model(**encoded_input) - # Perform pooling - sent_embs = self.mean_pooling(model_output, encoded_input["attention_mask"]) - # Normalize embeddings - sent_embs = F.normalize(sent_embs, p=2, dim=1) - n_source_embs = len(sent_embs) // 2 - scores[batch_idx] = ( - (sent_embs[:n_source_embs] * sent_embs[n_source_embs:]) - .sum(-1) - .cpu() - .detach() - .numpy() - ) - start = end - - return scores - - @staticmethod - def mean_pooling(model_output, attention_mask): - """ - Mean Pooling - Take attention mask into account for correct averaging - """ - token_embeddings = model_output[ - 0 - ] # First element of model_output contains all token embeddings - input_mask_expanded = ( - attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float() - ) - return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp( - input_mask_expanded.sum(1), min=1e-9 - ) - - -def calculate_alignscore( - predictions, - references, - original_texts, - batch_size: int = 32, - device: str = "cuda", - cache_dir: str = "cache", -): - if not is_alignscore_available: - return None - if isinstance(references[0], list): - log.error("AlignScore does not support multiple references. Skipping...") - return None - if not os.path.exists(ALIGNSCORE_CHECKPOINT_PATH): - urlretrieve( - "https://huggingface.co/yzha/AlignScore/resolve/main/AlignScore-base.ckpt", - ALIGNSCORE_CHECKPOINT_PATH, - ) - - scorer = AlignScore( - model="roberta-base", - batch_size=batch_size, - device=device, - ckpt_path=ALIGNSCORE_CHECKPOINT_PATH, - evaluation_mode="nli_sp", - ) - # Fix: alignscore outputs an error if a text is empty, so we need to add some content to such texts - original_texts = [text if text else " " for text in original_texts] - predictions = [text if text else " " for text in predictions] - references = [text if text else " " for text in references] - - scores_ref = scorer.score(contexts=original_texts, claims=predictions) - scores_baseline = scorer.score(contexts=original_texts, claims=references) - scores_rel = np.array(scores_ref) / np.array(scores_baseline) - return {"alignscore": scores_ref, "alignscore_rel": scores_rel} - - -class EvaluationLLM(DeepEvalBaseLLM): - """ - Custom Evaluation LLM implementation for DeepEval. - - This class implements the DeepEvalBaseLLM interface to allow using - custom models with DeepEval metrics. - """ - - def __init__( - self, - api_key=None, - model="openai/gpt-4o-2024-11-20", - base_url="https://openrouter.ai/api/v1", - ): - """ - Initialize the Evaluation LLM. - - Args: - api_key: Evaluation API key - model: Model identifier (e.g., "openai/gpt-4o-2024-11-20") - base_url: Evaluation API base URL - """ - self.api_key = api_key - - self.model_name = model - self.base_url = base_url - self.client = None - self.async_client = None - self.OpenAI = OpenAI - self.AsyncOpenAI = AsyncOpenAI - - def load_model(self): - """Load and return the client.""" - if self.client is None: - self.client = self.OpenAI( - base_url=self.base_url, - api_key=self.api_key, - ) - return self.client - - def load_async_model(self): - """Load and return the async client.""" - if self.async_client is None: - self.async_client = self.AsyncOpenAI( - base_url=self.base_url, - api_key=self.api_key, - ) - return self.async_client - - def generate(self, prompt: str) -> str: - """ - Generate a response from the evaluation model. - - Args: - prompt: The prompt to send to the model - - Returns: - The model's response as a string - """ - client = self.load_model() - response = client.chat.completions.create( - model=self.model_name, - messages=[{"role": "user", "content": prompt}], - ) - return response.choices[0].message.content - - async def a_generate(self, prompt: str) -> str: - """ - Asynchronously generate a response from the evaluation model. - - Args: - prompt: The prompt to send to the model - - Returns: - The model's response as a string - """ - # Use the async client for async operations - client = self.load_async_model() - response = await client.chat.completions.create( - model=self.model_name, - messages=[{"role": "user", "content": prompt}], - ) - return response.choices[0].message.content - - def get_model_name(self): - """Return the name of the model.""" - return f"EvaluationLLM: {self.model_name}" - - -def calculate_deepeval_metrics( - predictions, - references, - original_texts, - metrics_to_calculate=None, - base_url: str = "https://openrouter.ai/api/v1", - api_key: str = None, - model="openai/gpt-4o-2024-11-20", - threshold=0.5, - include_reason=False, - strict_mode=False, - async_mode=True, - verbose_mode=False, - truths_extraction_limit=None, -): - """ - Calculate DeepEval metrics using EvaluationLLM. - - Args: - predictions: List of generated texts - references: List of reference texts - original_texts: List of source texts - metrics_to_calculate: List of metrics to calculate. Options: - ["deepeval_answer_relevance", "deepeval_faithfulness", "deepeval_summarization", "deepeval_prompt_alignment"] - api_key: Evaluation API key - base_url: Evaluation API base URL - model: Evaluation model to use - threshold: Threshold for metrics (default: 0.5) - include_reason: Include reason for evaluation score (default: False) - strict_mode: Enforce binary metric score (1 for perfection, 0 otherwise) (default: False) - async_mode: Enable concurrent execution (default: True) - verbose_mode: Print intermediate steps (default: False) - truths_extraction_limit: Maximum number of factual truths to extract (default: None) - - Returns: - Dictionary with metric scores - """ - - if not metrics_to_calculate: - metrics_to_calculate = [ - "deepeval_answer_relevance", - "deepeval_faithfulness", - "deepeval_summarization", - "deepeval_prompt_alignment", - ] - - # Create EvaluationLLM instance - llm = EvaluationLLM( - base_url=base_url, - api_key=api_key, - model=model, - ) - - results = {} - - # Create metrics based on selected options - metrics = [] - metric_name_mapping = {} # Maps metric class name to the deepeval metric name - - # Dictionary to store test cases for each metric - metric_test_cases = {} - - if "deepeval_answer_relevance" in metrics_to_calculate: - metric = AnswerRelevancyMetric( - threshold=threshold, - model=llm, - include_reason=include_reason, - strict_mode=strict_mode, - async_mode=async_mode, - ) - metrics.append(metric) - metric_name_mapping[metric.__class__.__name__] = "deepeval_answer_relevance" - - # Create specific test cases for AnswerRelevancy metric - answer_relevance_test_cases = [] - for i, (pred, src) in enumerate(zip(predictions, original_texts)): - test_case = LLMTestCase( - input=src, - actual_output=pred, - ) - answer_relevance_test_cases.append(test_case) - metric_test_cases[metric.__class__.__name__] = answer_relevance_test_cases - - if "deepeval_faithfulness" in metrics_to_calculate: - metric = FaithfulnessMetric( - threshold=threshold, - model=llm, - include_reason=include_reason, - strict_mode=strict_mode, - async_mode=async_mode, - truths_extraction_limit=truths_extraction_limit, - ) - metrics.append(metric) - metric_name_mapping[metric.__class__.__name__] = "deepeval_faithfulness" - - # Create specific test cases for Faithfulness metric - faithfulness_test_cases = [] - for i, (pred, src) in enumerate(zip(predictions, original_texts)): - test_case = LLMTestCase( - input=src, - actual_output=pred, - retrieval_context=[src], - ) - faithfulness_test_cases.append(test_case) - metric_test_cases[metric.__class__.__name__] = faithfulness_test_cases - - if "deepeval_summarization" in metrics_to_calculate: - metric = SummarizationMetric( - threshold=threshold, - model=llm, - include_reason=include_reason, - strict_mode=strict_mode, - async_mode=async_mode, - ) - metrics.append(metric) - metric_name_mapping[metric.__class__.__name__] = "deepeval_summarization" - - # Create specific test cases for Summarization metric - summarization_test_cases = [] - for i, (pred, src) in enumerate(zip(predictions, original_texts)): - test_case = LLMTestCase( - input=src, - actual_output=pred, - ) - summarization_test_cases.append(test_case) - metric_test_cases[metric.__class__.__name__] = summarization_test_cases - - if "deepeval_prompt_alignment" in metrics_to_calculate: - metric = PromptAlignmentMetric( - threshold=threshold, - model=llm, - prompt_instructions=["Do what you are told to do in the prompt"], - include_reason=include_reason, - strict_mode=strict_mode, - async_mode=async_mode, - ) - metrics.append(metric) - metric_name_mapping[metric.__class__.__name__] = "deepeval_prompt_alignment" - - # Create specific test cases for PromptAlignment metric - prompt_alignment_test_cases = [] - for i, (pred, ref, src) in enumerate( - zip(predictions, references, original_texts) - ): - test_case = LLMTestCase( - input=src, - actual_output=pred, - expected_output=ref, - ) - prompt_alignment_test_cases.append(test_case) - metric_test_cases[metric.__class__.__name__] = prompt_alignment_test_cases - - # Run evaluation for each metric separately - for metric in metrics: - metric_class_name = metric.__class__.__name__ - test_cases = metric_test_cases.get(metric_class_name, []) - - if test_cases: - # Disable printing to console during evaluation if not verbose - original_stdout = sys.stdout - if not verbose_mode: - sys.stdout = open(os.devnull, "w") - - try: - # Run evaluation for this specific metric - evaluation_results = evaluate( - test_cases=test_cases, - metrics=[metric], - run_async=async_mode, - ) - - deepeval_metric_name = metric_name_mapping.get(metric_class_name) - scores = [] - reasons = [] - - # Process results for this metric - for result in evaluation_results.test_results: - scores.append(1 if result.success else 0) - - # Calculate average score - if scores: - results[deepeval_metric_name] = np.mean(scores) - - finally: - # Restore stdout - if not verbose_mode: - sys.stdout.close() - sys.stdout = original_stdout - print("================================================") - print("Results:") - print(results) - print("================================================") - - return results diff --git a/src/atgen/metrics/semantic/sentbert_metric.py b/src/atgen/metrics/semantic/sentbert_metric.py index 4d0adca..756e389 100644 --- a/src/atgen/metrics/semantic/sentbert_metric.py +++ b/src/atgen/metrics/semantic/sentbert_metric.py @@ -8,7 +8,6 @@ class SentBertMetric(BaseMetric): - """SentenceBERT semantic similarity metric.""" def __init__(self, config: Optional[MetricConfig] = None): super().__init__(config) @@ -18,7 +17,6 @@ def __init__(self, config: Optional[MetricConfig] = None): def _initialize_model(self): - """Initialize the SentenceBERT model if not already initialized.""" if self.model is None: self.tokenizer = AutoTokenizer.from_pretrained( self.checkpoint, @@ -58,7 +56,6 @@ def _compute_similarity(self, source_texts: List[str], ref_texts: List[str]) -> end += half_batch_size batch_idx = slice(start, end) - # Tokenize sentences encoded_input = self.tokenizer( source_texts[batch_idx] + ref_texts[batch_idx], padding=True, @@ -69,14 +66,11 @@ def _compute_similarity(self, source_texts: List[str], ref_texts: List[str]) -> key: value.to(self.config.device) for key, value in encoded_input.items() } - # Calculate embeddings with torch.no_grad(): model_output = self.model(**encoded_input) - # Perform pooling sent_embs = self.mean_pooling(model_output, encoded_input["attention_mask"]) - # Normalize embeddings sent_embs = F.normalize(sent_embs, p=2, dim=1) n_source_embs = len(sent_embs) // 2 @@ -107,10 +101,8 @@ def calculate(self, predictions: List[str], references: Optional[List[Union[str, scores = {} - # Similarity between predictions and references if references is not None: if isinstance(references[0], list): - # Handle multiple references - compute average similarity ref_scores = [] for pred, ref_list in zip(predictions, references): pred_list = [pred] * len(ref_list) @@ -120,11 +112,9 @@ def calculate(self, predictions: List[str], references: Optional[List[Union[str, else: scores["sentbert_pred_ref"] = self._compute_similarity(predictions, references) - # Similarity between predictions and original texts if original_texts is not None: scores["sentbert_pred_src"] = self._compute_similarity(predictions, original_texts) - # Aggregate if requested if self.config.aggregate: scores = {key: float(np.mean(value)) for key, value in scores.items()} From 6d61e0e22661519c2a90d3ea8ec3c98f05efa46b Mon Sep 17 00:00:00 2001 From: alfekka Date: Mon, 9 Jun 2025 01:38:30 +0300 Subject: [PATCH 04/39] bug fixes --- demo_all_metrics.py | 175 ------------------ demo_new_metrics.py | 97 ---------- example_bigbench_usage.py | 97 ---------- src/atgen/metrics/__init__.py | 45 +---- src/atgen/metrics/base/base_metric.py | 2 + src/atgen/metrics/compute_metrics.py | 42 +++-- src/atgen/metrics/factory.py | 79 ++++---- .../metrics/llm_based/base_deepeval_metric.py | 35 +++- 8 files changed, 108 insertions(+), 464 deletions(-) delete mode 100644 demo_all_metrics.py delete mode 100644 demo_new_metrics.py delete mode 100644 example_bigbench_usage.py diff --git a/demo_all_metrics.py b/demo_all_metrics.py deleted file mode 100644 index f32c864..0000000 --- a/demo_all_metrics.py +++ /dev/null @@ -1,175 +0,0 @@ -#!/usr/bin/env python3 -""" -Comprehensive demonstration of the refactored metrics system including DeepEval metrics. -""" - -import sys -import os -sys.path.append('src') - -from atgen.metrics.base import BaseMetric, MetricConfig -from atgen.metrics.lexical import BleuMetric, RougeMetric -from atgen.metrics.semantic import BartScoreMetric, AlignScoreMetric, SentBertMetric -from atgen.metrics.linguistic import ColaMetric -from atgen.metrics.llm_based import ( - DeepEvalAnswerRelevancyMetric, - DeepEvalFaithfulnessMetric, - DeepEvalSummarizationMetric, - DeepEvalPromptAlignmentMetric, - BigBenchHardMetric, - EvaluationLLM -) - - -def demo_all_metrics(): - """Demonstrate all metrics in the new system.""" - print("šŸŽÆ Comprehensive Metrics System Demonstration") - print("=" * 80) - - # Sample data - predictions = [ - "The cat is sitting on the mat peacefully.", - "Python is an excellent programming language for machine learning.", - "Deep learning has revolutionized artificial intelligence applications." - ] - - references = [ - "A cat is resting on the mat.", - "Python is great for ML development.", - "Deep learning has transformed AI." - ] - - original_texts = [ - "There is a cat on a mat", - "What makes Python good for ML?", - "How has deep learning impacted AI?" - ] - - # Create different configurations for different metric types - base_config = MetricConfig( - batch_size=2, - device="cpu", # Use CPU for demo - cache_dir="cache", - aggregate=True - ) - - # DeepEval config (requires API key) - deepeval_config = MetricConfig( - batch_size=2, - device="cpu", - cache_dir="cache", - aggregate=True, - api_key=os.getenv("OPENROUTER_API_KEY"), # Set this in environment - base_url="https://openrouter.ai/api/v1", - model="openai/gpt-4o-mini", # Cheaper model for demo - threshold=0.5, - include_reason=False, - strict_mode=False, - async_mode=False, # Synchronous for simpler demo - verbose_mode=False - ) - - # Organize metrics by category - metrics_by_category = { - "šŸ“ Lexical Metrics": [ - ("BLEU", BleuMetric(base_config)), - ("ROUGE", RougeMetric(base_config)), - ], - "šŸ§  Semantic Metrics": [ - ("SentenceBERT", SentBertMetric(base_config)), - # ("BARTScore", BartScoreMetric(base_config)), # Uncomment if you have dependencies - # ("AlignScore", AlignScoreMetric(base_config)), # Uncomment if you have dependencies - ], - "šŸ—£ļø Linguistic Quality": [ - ("CoLA (Grammaticality)", ColaMetric(base_config)), - ], - "šŸ¤– LLM-based (DeepEval)": [ - ("Answer Relevancy", DeepEvalAnswerRelevancyMetric(deepeval_config)), - ("Faithfulness", DeepEvalFaithfulnessMetric(deepeval_config)), - ("Summarization", DeepEvalSummarizationMetric(deepeval_config)), - ("Prompt Alignment", DeepEvalPromptAlignmentMetric(deepeval_config)), - ], - "šŸ§® Benchmark Metrics": [ - ("BigBenchHard", BigBenchHardMetric(deepeval_config)), - ] - } - - print(f"šŸ“Š Testing with {len(predictions)} samples\n") - - # Test each category - for category, metrics in metrics_by_category.items(): - print(f"{category}") - print("-" * 60) - - for metric_name, metric in metrics: - print(f" šŸ” {metric_name} ({metric.__class__.__name__})") - print(f" Available: {metric.is_available()}") - - if metric.is_available(): - try: - # Configure test based on metric requirements - if "DeepEval" in metric.__class__.__name__: - if deepeval_config.api_key is None: - print(f" āš ļø API key required (set OPENROUTER_API_KEY)") - print() - continue - - # Different DeepEval metrics need different inputs - if "PromptAlignment" in metric.__class__.__name__: - results = metric.calculate(predictions, references, original_texts) - else: - results = metric.calculate(predictions, None, original_texts) - - elif metric_name == "BigBenchHard": - # BigBenchHard needs a model and tokenizer, skip for demo - print(f" āš ļø Requires model and tokenizer (use set_model_and_tokenizer())") - print(f" šŸ’” Example: metric.set_model_and_tokenizer(model, tokenizer)") - print(f" score = metric.evaluate_benchmark(model, tokenizer)") - print() - continue - - elif metric_name == "CoLA (Grammaticality)": - # CoLA only needs predictions - results = metric.calculate(predictions) - - elif metric_name == "SentenceBERT": - # SentBERT can use both references and original texts - results = metric.calculate(predictions, references, original_texts) - - else: - # Standard metrics (BLEU, ROUGE) need references - results = metric.calculate(predictions, references) - - print(f" Results: {results}") - - except Exception as e: - print(f" āŒ Error: {e}") - else: - print(f" āš ļø Dependencies not available") - - print() - - print() - - print("šŸŒŸ System Architecture Highlights:") - print(" āœ… BaseMetric abstract class with consistent interface") - print(" āœ… MetricConfig for centralized configuration") - print(" āœ… Category-based organization (lexical, semantic, linguistic, llm-based)") - print(" āœ… Automatic dependency checking") - print(" āœ… Lazy model initialization") - print(" āœ… Proper error handling and logging") - print(" āœ… Support for multiple references") - print(" āœ… API-based metrics with custom LLM implementation") - print(" āœ… Benchmark metrics (BigBenchHard)") - print(" āœ… Configurable aggregation") - print(" āœ… Type safety with full type hints") - - print("\nšŸ”§ Next Steps:") - print(" ā€¢ Strategy factory pattern for metric selection") - print(" ā€¢ Configuration-based metric instantiation") - print(" ā€¢ Integration with existing compute_metrics.py") - print(" ā€¢ Performance optimization and caching") - - -if __name__ == "__main__": - demo_all_metrics() \ No newline at end of file diff --git a/demo_new_metrics.py b/demo_new_metrics.py deleted file mode 100644 index e2b47d6..0000000 --- a/demo_new_metrics.py +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env python3 -""" -Demonstration of the new refactored metrics system. -""" - -import sys -import os -sys.path.append('src') - -from atgen.metrics.base import BaseMetric, MetricConfig -from atgen.metrics.lexical import BleuMetric, RougeMetric -from atgen.metrics.semantic import BartScoreMetric, AlignScoreMetric, SentBertMetric -from atgen.metrics.linguistic import ColaMetric - - -def demo_metrics(): - """Demonstrate the new metrics system.""" - print("šŸŽÆ New Metrics System Demonstration") - print("=" * 60) - - # Sample data - predictions = [ - "The cat is on the mat", - "I love programming in Python", - "Machine learning is fascinating" - ] - - references = [ - "A cat is sitting on the mat", - "I enjoy coding with Python", - "Machine learning is very interesting" - ] - - original_texts = [ - "There is a cat on a mat", - "Programming with Python is great", - "ML technology is amazing" - ] - - # Create metrics with custom config - config = MetricConfig( - batch_size=8, - device="cpu", # Use CPU for demo - cache_dir="cache", - aggregate=True - ) - - metrics = [ - ("BLEU", BleuMetric(config)), - ("ROUGE", RougeMetric(config)), - ("CoLA (Grammaticality)", ColaMetric(config)), - ("SentenceBERT", SentBertMetric(config)), - # ("BARTScore", BartScoreMetric(config)), # Uncomment if you have the dependencies - # ("AlignScore", AlignScoreMetric(config)), # Uncomment if you have the dependencies - ] - - print(f"šŸ“Š Testing with {len(predictions)} samples\n") - - for metric_name, metric in metrics: - print(f"šŸ” Testing {metric_name} ({metric.__class__.__name__})") - print(f" Available: {metric.is_available()}") - - if metric.is_available(): - try: - # Test different scenarios based on metric requirements - if metric_name == "CoLA (Grammaticality)": - # CoLA only needs predictions - results = metric.calculate(predictions) - elif metric_name == "SentenceBERT": - # SentBERT can use both references and original texts - results = metric.calculate(predictions, references, original_texts) - else: - # Standard metrics (BLEU, ROUGE) need references - results = metric.calculate(predictions, references) - - print(f" Results: {results}") - - except Exception as e: - print(f" āŒ Error: {e}") - else: - print(f" āš ļø Dependencies not available") - - print() - - print("šŸŒŸ Key Features of the New System:") - print(" āœ… Clean inheritance from BaseMetric") - print(" āœ… Consistent configuration via MetricConfig") - print(" āœ… Automatic dependency checking") - print(" āœ… Lazy model initialization") - print(" āœ… Proper error handling") - print(" āœ… Organized by category (lexical, semantic, linguistic)") - print(" āœ… Type hints throughout") - print(" āœ… Configurable aggregation") - - -if __name__ == "__main__": - demo_metrics() \ No newline at end of file diff --git a/example_bigbench_usage.py b/example_bigbench_usage.py deleted file mode 100644 index aebd473..0000000 --- a/example_bigbench_usage.py +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env python3 -""" -Example usage of BigBenchHard metric with the new architecture. -""" - -import sys -sys.path.append('src') - -from atgen.metrics.base import MetricConfig -from atgen.metrics.llm_based import BigBenchHardMetric - - -def example_bigbench_usage(): - """Example of how to use BigBenchHard metric.""" - print("šŸ§® BigBenchHard Metric Usage Example") - print("=" * 60) - - # Create configuration for BigBenchHard - config = MetricConfig( - batch_size=8, - device="cuda", # or "cpu" - cache_dir="cache", - model_name="MyAwesomeModel", - # Benchmark-specific parameters - benchmark_params={ - "n_shots": 3, # Number of few-shot examples - "enable_cot": True, # Enable chain-of-thought - }, - # Generation parameters for the model - generation_params={ - "max_new_tokens": 100, - "temperature": 0.1, - "do_sample": False, - } - ) - - # Create the metric - metric = BigBenchHardMetric(config) - - print(f"āœ… Metric created: {metric.name}") - print(f"šŸ“¦ Dependencies available: {metric.is_available()}") - - if not metric.is_available(): - print("āŒ BigBenchHard dependencies not available") - print(" Install with: pip install deepeval") - return - - print("\nšŸ”§ Usage Options:") - print("\n1. Option 1: Using the standard calculate() interface:") - print(" ```python") - print(" # First set the model and tokenizer") - print(" metric.set_model_and_tokenizer(model, tokenizer)") - print(" ") - print(" # Then calculate (predictions/references are ignored)") - print(" results = metric.calculate([], [], [])") - print(" print(results) # {'bigbench_hard_score': 0.85}") - print(" ```") - - print("\n2. Option 2: Using the convenience method:") - print(" ```python") - print(" # Direct evaluation") - print(" score = metric.evaluate_benchmark(model, tokenizer, batch_size=16)") - print(" print(f'BigBenchHard score: {score}')") - print(" ```") - - print("\n3. Option 3: Using the original interface style:") - print(" ```python") - print(" # Set model first") - print(" metric.set_model_and_tokenizer(model, tokenizer)") - print(" ") - print(" # Get the overall score") - print(" score = metric.benchmark.overall_score") - print(" ```") - - print("\nšŸ“ Configuration Details:") - print(f" ā€¢ Batch size: {config.batch_size}") - print(f" ā€¢ Device: {config.device}") - print(f" ā€¢ Model name: {config.model_name}") - print(f" ā€¢ Benchmark params: {config.benchmark_params}") - print(f" ā€¢ Generation params: {config.generation_params}") - - print("\nšŸŽÆ What BigBenchHard Evaluates:") - print(" ā€¢ Challenging reasoning tasks from BIG-bench") - print(" ā€¢ Mathematical reasoning") - print(" ā€¢ Logical reasoning") - print(" ā€¢ Multi-step problem solving") - print(" ā€¢ Chain-of-thought reasoning (if enabled)") - - print("\nšŸ’” Integration Notes:") - print(" ā€¢ Follows the same BaseMetric interface as other metrics") - print(" ā€¢ Supports the standard MetricConfig system") - print(" ā€¢ Can be used in metric factories and pipelines") - print(" ā€¢ Provides both direct and standard interfaces") - - -if __name__ == "__main__": - example_bigbench_usage() \ No newline at end of file diff --git a/src/atgen/metrics/__init__.py b/src/atgen/metrics/__init__.py index f4ef337..7142cfa 100644 --- a/src/atgen/metrics/__init__.py +++ b/src/atgen/metrics/__init__.py @@ -1,15 +1,3 @@ -""" -Metrics module for evaluating text generation tasks. - -This module provides a comprehensive collection of metrics organized by category: -- Lexical: BLEU, ROUGE -- Semantic: BARTScore, AlignScore, SentBERT -- Linguistic: CoLA -- LLM-based: DeepEval metrics, BigBenchHard - -The module uses a factory pattern for creating metrics and supports -configuration-based instantiation. -""" # Base classes from .base import BaseMetric, MetricConfig @@ -43,26 +31,16 @@ ) # Compute system -from .compute_metrics_v2 import ( - compute_metrics_v2, +from .compute_metrics import ( + compute_metrics, compute_metrics_from_config, get_default_config, get_comprehensive_config, get_deepeval_config, ) -# Legacy compute system (for backward compatibility) -from .compute_metrics import compute_metrics - -# Legacy metrics functions (for backward compatibility) -from .metrics import ( - compute_bleu, - compute_rouge, - compute_bartscore, - compute_alignscore, - compute_sentbert, - compute_cola, -) + + # Version @@ -102,20 +80,13 @@ "get_metric_requirements", # New compute system - "compute_metrics_v2", + "compute_metrics", "compute_metrics_from_config", "get_default_config", "get_comprehensive_config", "get_deepeval_config", - # Legacy compatibility - "compute_metrics", - "compute_bleu", - "compute_rouge", - "compute_bartscore", - "compute_alignscore", - "compute_sentbert", - "compute_cola", + "AVAILABLE_METRICS", ] @@ -131,4 +102,6 @@ def create_metric(name: str, config: MetricConfig = None): def create_metrics_from_config(config): """Create metrics from configuration.""" - return MetricsFactory.create_from_config(config) \ No newline at end of file + return MetricsFactory.create_from_config(config) + +AVAILABLE_METRICS = MetricsFactory.get_available_metrics() \ No newline at end of file diff --git a/src/atgen/metrics/base/base_metric.py b/src/atgen/metrics/base/base_metric.py index 048b1ec..69e6595 100644 --- a/src/atgen/metrics/base/base_metric.py +++ b/src/atgen/metrics/base/base_metric.py @@ -14,6 +14,8 @@ class MetricConfig: checkpoint: Optional[str] = None model_name: Optional[str] = None + # API configuration parameters + provider: Optional[str] = None api_key: Optional[str] = None base_url: Optional[str] = None model: Optional[str] = None diff --git a/src/atgen/metrics/compute_metrics.py b/src/atgen/metrics/compute_metrics.py index 4632b1a..2b0cb36 100644 --- a/src/atgen/metrics/compute_metrics.py +++ b/src/atgen/metrics/compute_metrics.py @@ -2,7 +2,7 @@ from typing import List, Dict, Union, Optional import logging import numpy as np -from omegaconf import DictConfiga +from omegaconf import DictConfig from .factory import MetricsFactory, MetricsConfig, get_metric_requirements from .base import MetricConfig @@ -11,46 +11,56 @@ def compute_metrics( - predictions: List[str], - references: Optional[List[Union[str, List[str]]]] = None, + generated_texts: List[str], + reference_texts: Optional[List[Union[str, List[str]]]] = None, original_texts: Optional[List[str]] = None, - metrics_config: Optional[Union[Dict, DictConfig, MetricsConfig]] = None, + config: Optional[Union[Dict, DictConfig, MetricsConfig]] = None, model=None, tokenizer=None, + cache_dir: Optional[str] = None, ) -> Dict[str, float]: """ Compute various metrics for generated texts using the new architecture. Args: - predictions: List of generated texts to evaluate - references: List of reference texts (ground truth) or list of lists for multiple references + generated_texts: List of generated texts to evaluate + reference_texts: List of reference texts (ground truth) or list of lists for multiple references original_texts: List of source texts - metrics_config: Configuration for metrics (dict, DictConfig, or MetricsConfig) + config: Configuration for metrics (dict, DictConfig, or MetricsConfig) model: Model instance (required for BigBenchHard) tokenizer: Tokenizer instance (required for BigBenchHard) + cache_dir: Cache directory for storing intermediate results Returns: Dictionary with metric scores and timing information """ start_total = time() + predictions = generated_texts + references = reference_texts + + # Handle cache_dir in the config + if cache_dir is not None and isinstance(config, dict): + config = dict(config) # Make a copy + config["cache_dir"] = cache_dir + # Handle different config types - if metrics_config is None: + if config is None: # Default configuration - config = MetricsConfig( + metrics_config = MetricsConfig( metrics=["bleu", "rouge"], batch_size=32, device="cuda", aggregate=True ) - elif isinstance(metrics_config, dict): - config = MetricsConfig(**metrics_config) - elif isinstance(metrics_config, DictConfig): + elif isinstance(config, dict): + metrics_config = MetricsConfig(**config) + elif isinstance(config, DictConfig): # Convert OmegaConf to dict then to MetricsConfig - config_dict = dict(metrics_config) - config = MetricsConfig(**config_dict) + config_dict = dict(config) + metrics_config = MetricsConfig(**config_dict) else: - config = metrics_config + metrics_config = config # Validate inputs if not predictions: @@ -60,7 +70,7 @@ def compute_metrics( requirements = get_metric_requirements() # Create metrics using factory - metrics = MetricsFactory.create_metrics(config) + metrics = MetricsFactory.create_metrics(metrics_config) if not metrics: logger.warning("No metrics were successfully created") diff --git a/src/atgen/metrics/factory.py b/src/atgen/metrics/factory.py index f777676..501bf4c 100644 --- a/src/atgen/metrics/factory.py +++ b/src/atgen/metrics/factory.py @@ -19,7 +19,6 @@ @dataclass class MetricsConfig: - """Configuration for the metrics system.""" batch_size: int = 32 device: str = "cuda" @@ -28,9 +27,12 @@ class MetricsConfig: metrics: List[str] = field(default_factory=list) + additional_metrics: List[str] = field(default_factory=list) + checkpoint: Optional[str] = None model_name: Optional[str] = None + provider: Optional[str] = None api_key: Optional[str] = None base_url: Optional[str] = None model: Optional[str] = None @@ -42,37 +44,61 @@ class MetricsConfig: verbose_mode: bool = False truths_extraction_limit: Optional[int] = None + deepeval_threshold: Optional[float] = None + deepeval_include_reason: Optional[bool] = None + deepeval_strict_mode: Optional[bool] = None + deepeval_async_mode: Optional[bool] = None + deepeval_verbose_mode: Optional[bool] = None + deepeval_truths_extraction_limit: Optional[int] = None + benchmark_params: Dict[str, Any] = field(default_factory=dict) generation_params: Dict[str, Any] = field(default_factory=dict) + custom_params: Dict[str, Any] = field(default_factory=dict) + + def __post_init__(self): + """Post-initialization to handle legacy parameter mappings.""" + # Merge additional_metrics into metrics for backward compatibility + if self.additional_metrics: + self.metrics.extend(self.additional_metrics) + # Remove duplicates while preserving order + seen = set() + self.metrics = [x for x in self.metrics if not (x in seen or seen.add(x))] + + # Handle DeepEval legacy parameter mappings + if self.deepeval_threshold is not None: + self.threshold = self.deepeval_threshold + if self.deepeval_include_reason is not None: + self.include_reason = self.deepeval_include_reason + if self.deepeval_strict_mode is not None: + self.strict_mode = self.deepeval_strict_mode + if self.deepeval_async_mode is not None: + self.async_mode = self.deepeval_async_mode + if self.deepeval_verbose_mode is not None: + self.verbose_mode = self.deepeval_verbose_mode + if self.deepeval_truths_extraction_limit is not None: + self.truths_extraction_limit = self.deepeval_truths_extraction_limit class MetricsFactory: """Factory for creating metric instances.""" - # Registry mapping metric names to classes _metric_registry = { - # Lexical metrics "bleu": BleuMetric, "rouge": RougeMetric, - # Semantic metrics "bartscore": BartScoreMetric, "alignscore": AlignScoreMetric, "sentbert": SentBertMetric, "sentence_bert": SentBertMetric, # Alias - # Linguistic metrics "cola": ColaMetric, "grammaticality": ColaMetric, # Alias - # LLM-based metrics "deepeval_answer_relevance": DeepEvalAnswerRelevancyMetric, "deepeval_faithfulness": DeepEvalFaithfulnessMetric, "deepeval_summarization": DeepEvalSummarizationMetric, "deepeval_prompt_alignment": DeepEvalPromptAlignmentMetric, - - # Benchmark metrics "bigbench_hard": BigBenchHardMetric, "big_bench_hard": BigBenchHardMetric, # Alias } @@ -90,19 +116,7 @@ def get_available_metrics(cls) -> List[str]: @classmethod def create_metric(cls, name: str, config: Optional[MetricConfig] = None) -> BaseMetric: - """ - Create a single metric instance. - - Args: - name: Name of the metric to create - config: Configuration for the metric - - Returns: - Metric instance - - Raises: - ValueError: If metric name is not recognized - """ + name = name.lower().strip() if name not in cls._metric_registry: @@ -114,15 +128,6 @@ def create_metric(cls, name: str, config: Optional[MetricConfig] = None) -> Base @classmethod def create_metrics(cls, metrics_config: MetricsConfig) -> Dict[str, BaseMetric]: - """ - Create multiple metric instances from configuration. - - Args: - metrics_config: Configuration containing list of metrics to create - - Returns: - Dictionary mapping metric names to instances - """ base_config = MetricConfig( batch_size=metrics_config.batch_size, device=metrics_config.device, @@ -130,6 +135,7 @@ def create_metrics(cls, metrics_config: MetricsConfig) -> Dict[str, BaseMetric]: aggregate=metrics_config.aggregate, checkpoint=metrics_config.checkpoint, model_name=metrics_config.model_name, + provider=metrics_config.provider, api_key=metrics_config.api_key, base_url=metrics_config.base_url, model=metrics_config.model, @@ -156,15 +162,7 @@ def create_metrics(cls, metrics_config: MetricsConfig) -> Dict[str, BaseMetric]: @classmethod def create_from_config(cls, config: Union[Dict[str, Any], MetricsConfig]) -> Dict[str, BaseMetric]: - """ - Create metrics from dictionary or MetricsConfig. - - Args: - config: Configuration dictionary or MetricsConfig instance - - Returns: - Dictionary mapping metric names to instances - """ + if isinstance(config, dict): metrics_config = MetricsConfig(**config) else: @@ -174,7 +172,6 @@ def create_from_config(cls, config: Union[Dict[str, Any], MetricsConfig]) -> Dic def get_metric_categories() -> Dict[str, List[str]]: - """Get metrics organized by category.""" return { "lexical": ["bleu", "rouge"], "semantic": ["bartscore", "alignscore", "sentbert"], @@ -190,7 +187,7 @@ def get_metric_categories() -> Dict[str, List[str]]: def get_metric_requirements() -> Dict[str, Dict[str, bool]]: - """Get input requirements for each metric.""" + return { "bleu": {"requires_references": True, "requires_original_texts": False}, "rouge": {"requires_references": True, "requires_original_texts": False}, diff --git a/src/atgen/metrics/llm_based/base_deepeval_metric.py b/src/atgen/metrics/llm_based/base_deepeval_metric.py index ccdbef3..c7e1f3d 100644 --- a/src/atgen/metrics/llm_based/base_deepeval_metric.py +++ b/src/atgen/metrics/llm_based/base_deepeval_metric.py @@ -29,10 +29,41 @@ def _validate_config(self): def _initialize_llm(self): """Initialize the LLM for evaluation if not already initialized.""" if self.llm is None: + # Determine base_url based on provider if not explicitly set + base_url = self.config.base_url + if base_url is None and self.config.provider: + provider = self.config.provider.lower() + if provider == "openai": + base_url = "https://api.openai.com/v1" + elif provider == "anthropic": + base_url = "https://api.anthropic.com/v1" + elif provider == "openrouter": + base_url = "https://openrouter.ai/api/v1" + else: + self.logger.warning(f"Unknown provider '{provider}', using default base_url") + base_url = "https://openrouter.ai/api/v1" + else: + base_url = base_url or "https://openrouter.ai/api/v1" + + # Determine default model based on provider if not explicitly set + model = self.config.model + if model is None and self.config.provider: + provider = self.config.provider.lower() + if provider == "openai": + model = "gpt-4o-mini" + elif provider == "anthropic": + model = "claude-3-5-sonnet" + elif provider == "openrouter": + model = "openai/gpt-4o-mini" + else: + model = "openai/gpt-4o-mini" + else: + model = model or "openai/gpt-4o-mini" + self.llm = EvaluationLLM( api_key=self.config.api_key, - model=self.config.model or "openai/gpt-4o-2024-11-20", - base_url=self.config.base_url or "https://openrouter.ai/api/v1", + model=model, + base_url=base_url, ) def _create_deepeval_metric(self, metric_class, **kwargs): From ea38b00e6d10a6e72e6a1b74268db35a91a0259f Mon Sep 17 00:00:00 2001 From: alfekka Date: Tue, 10 Jun 2025 01:44:49 +0300 Subject: [PATCH 05/39] bug fixes + add tests --- src/atgen/metrics/compute_metrics.py | 6 +- src/atgen/metrics/lexical/bleu_metric.py | 12 +- src/atgen/metrics/lexical/rouge_metric.py | 7 + src/atgen/metrics/linguistic/cola_metric.py | 2 +- src/atgen/metrics/semantic/sentbert_metric.py | 2 +- tests/test_metrics.py | 708 ++++++++++++++++++ 6 files changed, 729 insertions(+), 8 deletions(-) create mode 100644 tests/test_metrics.py diff --git a/src/atgen/metrics/compute_metrics.py b/src/atgen/metrics/compute_metrics.py index 2b0cb36..52cead5 100644 --- a/src/atgen/metrics/compute_metrics.py +++ b/src/atgen/metrics/compute_metrics.py @@ -174,10 +174,10 @@ def compute_metrics_from_config( Dictionary with metric scores """ return compute_metrics( - predictions=predictions, - references=references, + generated_texts=predictions, + reference_texts=references, original_texts=original_texts, - metrics_config=config_dict, + config=config_dict, model=model, tokenizer=tokenizer ) diff --git a/src/atgen/metrics/lexical/bleu_metric.py b/src/atgen/metrics/lexical/bleu_metric.py index 10d985e..d6af063 100644 --- a/src/atgen/metrics/lexical/bleu_metric.py +++ b/src/atgen/metrics/lexical/bleu_metric.py @@ -47,7 +47,7 @@ def supports_multiple_references(self) -> bool: return True - def _compute( + def calculate( self, predictions: List[str], references: Optional[List[Union[str, List[str]]]] = None, @@ -76,9 +76,15 @@ def _compute( tok_pred = word_tokenize(pred) try: - score = corpus_bleu(tok_ref, [tok_pred], smoothing_function=smoothing_function) + score = corpus_bleu([tok_ref], [tok_pred], smoothing_function=smoothing_function) scores.append(score) except (KeyError, ZeroDivisionError): scores.append(0.0) - return {"bleu": np.array(scores)} \ No newline at end of file + scores_dict = {"bleu": np.array(scores)} + + # Apply aggregation if requested + if self.config.aggregate: + scores_dict = {key: float(np.mean(value)) for key, value in scores_dict.items()} + + return scores_dict \ No newline at end of file diff --git a/src/atgen/metrics/lexical/rouge_metric.py b/src/atgen/metrics/lexical/rouge_metric.py index e63a362..2a11a4c 100644 --- a/src/atgen/metrics/lexical/rouge_metric.py +++ b/src/atgen/metrics/lexical/rouge_metric.py @@ -59,4 +59,11 @@ def calculate(self, predictions: List[str], references: Optional[List[Union[str, else: scores[key] = float(value) + # ROUGE already aggregates by default, but we need to handle the case + # where it returns arrays (though this is typically rare) + if self.config.aggregate: + for key, value in scores.items(): + if isinstance(value, np.ndarray): + scores[key] = float(np.mean(value)) + return scores \ No newline at end of file diff --git a/src/atgen/metrics/linguistic/cola_metric.py b/src/atgen/metrics/linguistic/cola_metric.py index 3158eb0..b333dbb 100644 --- a/src/atgen/metrics/linguistic/cola_metric.py +++ b/src/atgen/metrics/linguistic/cola_metric.py @@ -25,7 +25,7 @@ def __init__(self, config: Optional[MetricConfig] = None): super().__init__(config) self.model = None self.tokenizer = None - self.checkpoint = getattr(config, 'checkpoint', 'Aktsvigun/electra-large-cola') if config else 'Aktsvigun/electra-large-cola' + self.checkpoint = (config.checkpoint if config and config.checkpoint else 'Aktsvigun/electra-large-cola') def _initialize_model(self): diff --git a/src/atgen/metrics/semantic/sentbert_metric.py b/src/atgen/metrics/semantic/sentbert_metric.py index 756e389..d872405 100644 --- a/src/atgen/metrics/semantic/sentbert_metric.py +++ b/src/atgen/metrics/semantic/sentbert_metric.py @@ -13,7 +13,7 @@ def __init__(self, config: Optional[MetricConfig] = None): super().__init__(config) self.model = None self.tokenizer = None - self.checkpoint = getattr(config, 'checkpoint', 'sentence-transformers/all-mpnet-base-v2') if config else 'sentence-transformers/all-mpnet-base-v2' + self.checkpoint = (config.checkpoint if config and config.checkpoint else 'sentence-transformers/all-mpnet-base-v2') def _initialize_model(self): diff --git a/tests/test_metrics.py b/tests/test_metrics.py new file mode 100644 index 0000000..42382c3 --- /dev/null +++ b/tests/test_metrics.py @@ -0,0 +1,708 @@ +import pytest +import numpy as np +from unittest.mock import Mock, patch +import os +import tempfile +import shutil + +from atgen.metrics import ( + BaseMetric, + MetricConfig, + MetricsConfig, + MetricsFactory, + BleuMetric, + RougeMetric, + SentBertMetric, + ColaMetric, + BartScoreMetric, + AlignScoreMetric, + compute_metrics, + compute_metrics_from_config, + get_default_config, + get_comprehensive_config, + get_metric_categories, + get_metric_requirements, +) + + +class TestMetricConfig: + """Test MetricConfig dataclass.""" + + def test_default_config(self): + """Test default configuration values.""" + config = MetricConfig() + assert config.batch_size == 32 + assert config.device == "cuda" + assert config.cache_dir == "cache" + assert config.aggregate is True + assert config.threshold == 0.5 + + def test_custom_config(self): + """Test custom configuration values.""" + config = MetricConfig( + batch_size=16, + device="cpu", + cache_dir="custom_cache", + aggregate=False, + threshold=0.7 + ) + assert config.batch_size == 16 + assert config.device == "cpu" + assert config.cache_dir == "custom_cache" + assert config.aggregate is False + assert config.threshold == 0.7 + + +class TestMetricsConfig: + """Test MetricsConfig dataclass.""" + + def test_default_config(self): + """Test default configuration values.""" + config = MetricsConfig() + assert config.batch_size == 32 + assert config.device == "cuda" + assert config.cache_dir == "cache" + assert config.aggregate is True + assert config.metrics == [] + + def test_additional_metrics_merge(self): + """Test that additional_metrics are merged into metrics.""" + config = MetricsConfig( + metrics=["bleu", "rouge"], + additional_metrics=["sentbert", "cola"] + ) + expected_metrics = ["bleu", "rouge", "sentbert", "cola"] + assert config.metrics == expected_metrics + + def test_duplicate_metrics_removed(self): + """Test that duplicate metrics are removed while preserving order.""" + config = MetricsConfig( + metrics=["bleu", "rouge", "bleu"], + additional_metrics=["sentbert", "rouge"] + ) + expected_metrics = ["bleu", "rouge", "sentbert"] + assert config.metrics == expected_metrics + + def test_deepeval_legacy_params(self): + """Test that legacy DeepEval parameters are mapped correctly.""" + config = MetricsConfig( + deepeval_threshold=0.8, + deepeval_include_reason=True, + deepeval_strict_mode=True, + deepeval_async_mode=False, + deepeval_verbose_mode=True, + deepeval_truths_extraction_limit=10 + ) + assert config.threshold == 0.8 + assert config.include_reason is True + assert config.strict_mode is True + assert config.async_mode is False + assert config.verbose_mode is True + assert config.truths_extraction_limit == 10 + + +class TestMetricsFactory: + """Test MetricsFactory class.""" + + def test_get_available_metrics(self): + """Test getting available metrics.""" + metrics = MetricsFactory.get_available_metrics() + assert isinstance(metrics, list) + assert len(metrics) > 0 + assert "bleu" in metrics + assert "rouge" in metrics + assert "sentbert" in metrics + assert "cola" in metrics + + def test_create_metric_success(self): + """Test successful metric creation.""" + config = MetricConfig(device="cpu") + metric = MetricsFactory.create_metric("bleu", config) + assert isinstance(metric, BleuMetric) + assert metric.config.device == "cpu" + + def test_create_metric_case_insensitive(self): + """Test that metric creation is case insensitive.""" + metric_lower = MetricsFactory.create_metric("bleu") + metric_upper = MetricsFactory.create_metric("BLEU") + metric_mixed = MetricsFactory.create_metric("Bleu") + + assert type(metric_lower) == type(metric_upper) == type(metric_mixed) + + def test_create_metric_unknown(self): + """Test error handling for unknown metrics.""" + with pytest.raises(ValueError, match="Unknown metric"): + MetricsFactory.create_metric("unknown_metric") + + def test_create_metrics_from_config(self): + """Test creating multiple metrics from config.""" + config = MetricsConfig( + metrics=["bleu", "rouge", "sentbert"], + device="cpu" + ) + metrics = MetricsFactory.create_metrics(config) + + assert len(metrics) == 3 + assert "bleu" in metrics + assert "rouge" in metrics + assert "sentbert" in metrics + assert all(isinstance(m, BaseMetric) for m in metrics.values()) + + def test_create_from_config_dict(self): + """Test creating metrics from dictionary config.""" + config_dict = { + "metrics": ["bleu", "rouge"], + "device": "cpu", + "batch_size": 16 + } + metrics = MetricsFactory.create_from_config(config_dict) + + assert len(metrics) == 2 + assert "bleu" in metrics + assert "rouge" in metrics + + def test_register_custom_metric(self): + """Test registering a custom metric.""" + class CustomMetric(BaseMetric): + def calculate(self, predictions, references, original_texts): + return {"custom": 0.5} + + MetricsFactory.register_metric("custom", CustomMetric) + + try: + available = MetricsFactory.get_available_metrics() + assert "custom" in available + + metric = MetricsFactory.create_metric("custom") + assert isinstance(metric, CustomMetric) + finally: + # Clean up + if "custom" in MetricsFactory._metric_registry: + del MetricsFactory._metric_registry["custom"] + + +class TestIdenticalStringsBasic: + """Test all metrics with identical strings to ensure basic functionality.""" + + @pytest.fixture + def identical_data(self): + """Sample data with identical predictions and references.""" + return { + "predictions": ["This is a test sentence.", "Another test sentence here."], + "references": ["This is a test sentence.", "Another test sentence here."], + "original_texts": ["Original text one.", "Original text two."] + } + + @pytest.fixture + def temp_cache_dir(self): + """Create temporary cache directory for tests.""" + temp_dir = tempfile.mkdtemp() + yield temp_dir + shutil.rmtree(temp_dir, ignore_errors=True) + + def test_bleu_identical_strings(self, identical_data, temp_cache_dir): + """Test BLEU metric with identical strings.""" + config = MetricConfig(device="cpu", cache_dir=temp_cache_dir) + metric = BleuMetric(config) + + results = metric.calculate( + identical_data["predictions"], + identical_data["references"], + identical_data["original_texts"] + ) + + assert "bleu" in results + if isinstance(results["bleu"], np.ndarray): + # Should be perfect scores for identical strings + assert all(score == 1.0 for score in results["bleu"]) + else: + assert results["bleu"] == 1.0 + + def test_rouge_identical_strings(self, identical_data, temp_cache_dir): + """Test ROUGE metric with identical strings.""" + config = MetricConfig(device="cpu", cache_dir=temp_cache_dir) + metric = RougeMetric(config) + + results = metric.calculate( + identical_data["predictions"], + identical_data["references"], + identical_data["original_texts"] + ) + + # ROUGE should return perfect scores for identical strings + rouge_keys = ["rouge1", "rouge2", "rougeL"] + for key in rouge_keys: + if key in results: + if isinstance(results[key], np.ndarray): + assert all(score == 1.0 for score in results[key]) + else: + assert results[key] == 1.0 + + @patch('torch.cuda.is_available', return_value=False) + def test_sentbert_identical_strings(self, mock_cuda, identical_data, temp_cache_dir): + """Test SentBERT metric with identical strings.""" + config = MetricConfig(device="cpu", cache_dir=temp_cache_dir) + metric = SentBertMetric(config) + + results = metric.calculate( + identical_data["predictions"], + identical_data["references"], + identical_data["original_texts"] + ) + + # SentBERT should return high similarity scores for identical strings + if "sentbert_pred_ref" in results: + if isinstance(results["sentbert_pred_ref"], np.ndarray): + assert all(score > 0.99 for score in results["sentbert_pred_ref"]) + else: + assert results["sentbert_pred_ref"] > 0.99 + + @patch('torch.cuda.is_available', return_value=False) + def test_cola_identical_strings(self, mock_cuda, identical_data, temp_cache_dir): + """Test CoLA metric with identical strings.""" + config = MetricConfig(device="cpu", cache_dir=temp_cache_dir) + metric = ColaMetric(config) + + results = metric.calculate( + identical_data["predictions"], + identical_data["references"], + identical_data["original_texts"] + ) + + # CoLA should return grammaticality scores + assert "cola" in results + if isinstance(results["cola"], np.ndarray): + assert all(0 <= score <= 1 for score in results["cola"]) + else: + assert 0 <= results["cola"] <= 1 + + @patch('torch.cuda.is_available', return_value=False) + def test_bartscore_identical_strings(self, mock_cuda, identical_data, temp_cache_dir): + """Test BARTScore metric with identical strings.""" + config = MetricConfig(device="cpu", cache_dir=temp_cache_dir, batch_size=2) + metric = BartScoreMetric(config) + + try: + results = metric.calculate( + identical_data["predictions"], + identical_data["references"], + identical_data["original_texts"] + ) + + # BARTScore should return high scores for identical strings + assert isinstance(results, dict) + assert len(results) > 0 + + # Check that scores are reasonable (BARTScore can vary but should be positive for identical strings) + for key, value in results.items(): + if isinstance(value, np.ndarray): + assert all(isinstance(score, (int, float)) for score in value) + else: + assert isinstance(value, (int, float)) + + except Exception as e: + # BARTScore might not be available or have dependency issues + pytest.skip(f"BARTScore test skipped due to: {e}") + + @patch('torch.cuda.is_available', return_value=False) + def test_alignscore_identical_strings(self, mock_cuda, identical_data, temp_cache_dir): + """Test AlignScore metric with identical strings.""" + config = MetricConfig(device="cpu", cache_dir=temp_cache_dir, batch_size=2) + metric = AlignScoreMetric(config) + + try: + results = metric.calculate( + identical_data["predictions"], + identical_data["references"], + identical_data["original_texts"] + ) + + # AlignScore should return high scores for identical strings + assert isinstance(results, dict) + assert len(results) > 0 + + # Check that scores are reasonable + for key, value in results.items(): + if isinstance(value, np.ndarray): + assert all(isinstance(score, (int, float)) for score in value) + else: + assert isinstance(value, (int, float)) + + except Exception as e: + # AlignScore might not be available or have dependency issues + pytest.skip(f"AlignScore test skipped due to: {e}") + + +class TestComputeMetrics: + """Test the main compute_metrics function.""" + + @pytest.fixture + def sample_data(self): + """Sample data for testing.""" + return { + "predictions": ["This is a test.", "Another test."], + "references": ["This is a test.", "Another test."], + "original_texts": ["Source text one.", "Source text two."] + } + + @pytest.fixture + def temp_cache_dir(self): + """Create temporary cache directory for tests.""" + temp_dir = tempfile.mkdtemp() + yield temp_dir + shutil.rmtree(temp_dir, ignore_errors=True) + + def test_compute_metrics_default_config(self, sample_data, temp_cache_dir): + """Test compute_metrics with default configuration.""" + results = compute_metrics( + generated_texts=sample_data["predictions"], + reference_texts=sample_data["references"], + original_texts=sample_data["original_texts"], + cache_dir=temp_cache_dir + ) + + # Should include basic metrics + assert isinstance(results, dict) + assert len(results) > 0 + + # Should include timing information + assert any(key.startswith("time_") for key in results.keys()) + assert "time_total" in results + + # Should include basic statistics + assert "word_length_gen" in results + assert "exact_match" in results + + def test_compute_metrics_custom_config(self, sample_data, temp_cache_dir): + """Test compute_metrics with custom configuration.""" + config = MetricsConfig( + metrics=["bleu", "rouge"], + device="cpu", + cache_dir=temp_cache_dir, + batch_size=16 + ) + + results = compute_metrics( + generated_texts=sample_data["predictions"], + reference_texts=sample_data["references"], + original_texts=sample_data["original_texts"], + config=config + ) + + assert isinstance(results, dict) + assert "bleu" in results + assert any(key.startswith("rouge") for key in results.keys()) + + def test_compute_metrics_no_references(self, sample_data, temp_cache_dir): + """Test compute_metrics without references.""" + config = MetricsConfig( + metrics=["sentbert", "cola"], + device="cpu", + cache_dir=temp_cache_dir + ) + + results = compute_metrics( + generated_texts=sample_data["predictions"], + reference_texts=None, + original_texts=sample_data["original_texts"], + config=config + ) + + assert isinstance(results, dict) + # Should still include basic statistics + assert "word_length_gen" in results + + def test_compute_metrics_empty_predictions(self): + """Test compute_metrics with empty predictions.""" + with pytest.raises(ValueError, match="predictions cannot be empty"): + compute_metrics(generated_texts=[]) + + def test_compute_metrics_from_config(self, sample_data, temp_cache_dir): + """Test compute_metrics_from_config function.""" + config_dict = { + "metrics": ["bleu", "rouge"], + "device": "cpu", + "cache_dir": temp_cache_dir + } + + results = compute_metrics_from_config( + predictions=sample_data["predictions"], + references=sample_data["references"], + original_texts=sample_data["original_texts"], + config_dict=config_dict + ) + + assert isinstance(results, dict) + assert len(results) > 0 + + +class TestMetricRequirements: + """Test metric requirements and categories.""" + + def test_get_metric_categories(self): + """Test getting metric categories.""" + categories = get_metric_categories() + + assert isinstance(categories, dict) + assert "lexical" in categories + assert "semantic" in categories + assert "linguistic" in categories + assert "llm_based" in categories + + # Check specific metrics in categories + assert "bleu" in categories["lexical"] + assert "rouge" in categories["lexical"] + assert "sentbert" in categories["semantic"] + assert "cola" in categories["linguistic"] + + def test_get_metric_requirements(self): + """Test getting metric requirements.""" + requirements = get_metric_requirements() + + assert isinstance(requirements, dict) + + # Test specific requirements + assert requirements["bleu"]["requires_references"] is True + assert requirements["bleu"]["requires_original_texts"] is False + + assert requirements["sentbert"]["requires_references"] is False + assert requirements["sentbert"]["requires_original_texts"] is False + + assert requirements["cola"]["requires_references"] is False + assert requirements["cola"]["requires_original_texts"] is False + + +class TestConfigurationFunctions: + """Test configuration helper functions.""" + + def test_get_default_config(self): + """Test get_default_config function.""" + config = get_default_config() + + assert isinstance(config, MetricsConfig) + assert "bleu" in config.metrics + assert "rouge" in config.metrics + assert config.batch_size == 32 + assert config.device == "cuda" + + def test_get_comprehensive_config(self): + """Test get_comprehensive_config function.""" + config = get_comprehensive_config() + + assert isinstance(config, MetricsConfig) + assert len(config.metrics) > 2 # Should include more metrics + assert "bleu" in config.metrics + assert "rouge" in config.metrics + assert "sentbert" in config.metrics + assert "cola" in config.metrics + + +class TestEdgeCases: + """Test edge cases and error handling.""" + + @pytest.fixture + def temp_cache_dir(self): + """Create temporary cache directory for tests.""" + temp_dir = tempfile.mkdtemp() + yield temp_dir + shutil.rmtree(temp_dir, ignore_errors=True) + + def test_empty_strings(self, temp_cache_dir): + """Test metrics with empty strings.""" + config = MetricsConfig( + metrics=["bleu", "rouge"], + device="cpu", + cache_dir=temp_cache_dir + ) + + results = compute_metrics( + generated_texts=["", ""], + reference_texts=["", ""], + original_texts=["", ""], + config=config + ) + + assert isinstance(results, dict) + # Should handle empty strings gracefully + + def test_single_item_lists(self, temp_cache_dir): + """Test metrics with single item lists.""" + config = MetricsConfig( + metrics=["bleu", "rouge"], + device="cpu", + cache_dir=temp_cache_dir + ) + + results = compute_metrics( + generated_texts=["Single test sentence."], + reference_texts=["Single test sentence."], + original_texts=["Single source text."], + config=config + ) + + assert isinstance(results, dict) + assert len(results) > 0 + + def test_multiple_references(self, temp_cache_dir): + """Test metrics with multiple references.""" + config = MetricsConfig( + metrics=["bleu", "rouge"], + device="cpu", + cache_dir=temp_cache_dir + ) + + results = compute_metrics( + generated_texts=["Test sentence."], + reference_texts=[["Test sentence.", "Another reference."]], + original_texts=["Source text."], + config=config + ) + + assert isinstance(results, dict) + assert len(results) > 0 + + def test_mismatched_lengths(self, temp_cache_dir): + """Test metrics with mismatched input lengths.""" + config = MetricsConfig( + metrics=["bleu"], + device="cpu", + cache_dir=temp_cache_dir + ) + + # This should handle gracefully or raise appropriate error + try: + results = compute_metrics( + generated_texts=["Test sentence.", "Another sentence."], + reference_texts=["Test sentence."], # Shorter list + original_texts=["Source text."], + config=config + ) + # If it doesn't raise an error, check that results are reasonable + assert isinstance(results, dict) + except (ValueError, IndexError, AssertionError): + # These are acceptable errors for mismatched lengths + pass + + +class TestMetricIntegration: + """Integration tests for the complete metrics system.""" + + @pytest.fixture + def temp_cache_dir(self): + """Create temporary cache directory for tests.""" + temp_dir = tempfile.mkdtemp() + yield temp_dir + shutil.rmtree(temp_dir, ignore_errors=True) + + def test_all_lexical_metrics(self, temp_cache_dir): + """Test all lexical metrics together.""" + config = MetricsConfig( + metrics=["bleu", "rouge"], + device="cpu", + cache_dir=temp_cache_dir, + aggregate=True + ) + + results = compute_metrics( + generated_texts=["This is a comprehensive test sentence.", "Another test for metrics."], + reference_texts=["This is a comprehensive test sentence.", "Another test for metrics."], + original_texts=["Source text one.", "Source text two."], + config=config + ) + + assert "bleu" in results + assert any(key.startswith("rouge") for key in results.keys()) + + # All scores should be 1.0 for identical strings + # BLEU is aggregated so should be a single value + assert isinstance(results["bleu"], (int, float)) + assert results["bleu"] == 1.0 + + @patch('torch.cuda.is_available', return_value=False) + def test_comprehensive_metrics_suite(self, mock_cuda, temp_cache_dir): + """Test a comprehensive suite of metrics.""" + config = MetricsConfig( + metrics=["bleu", "rouge", "sentbert", "cola"], + device="cpu", + cache_dir=temp_cache_dir, + batch_size=8, + aggregate=True + ) + + results = compute_metrics( + generated_texts=[ + "This is a well-formed grammatical sentence.", + "Another properly structured sentence here." + ], + reference_texts=[ + "This is a well-formed grammatical sentence.", + "Another properly structured sentence here." + ], + original_texts=[ + "Source text for the first sentence.", + "Source text for the second sentence." + ], + config=config + ) + + # Should include results from all metric types + assert "bleu" in results + assert any(key.startswith("rouge") for key in results.keys()) + assert any(key.startswith("sentbert") for key in results.keys()) + assert "cola" in results + + # Should include timing and basic stats + assert "time_total" in results + assert "word_length_gen" in results + assert "exact_match" in results + + # Exact match should be 1.0 for identical strings + assert results["exact_match"] == 1.0 + + @patch('torch.cuda.is_available', return_value=False) + def test_semantic_metrics_suite(self, mock_cuda, temp_cache_dir): + """Test semantic metrics including BARTScore and AlignScore.""" + config = MetricsConfig( + metrics=["sentbert", "bartscore", "alignscore"], + device="cpu", + cache_dir=temp_cache_dir, + batch_size=4, + aggregate=True + ) + + try: + results = compute_metrics( + generated_texts=[ + "This is a semantic similarity test.", + "Another sentence for testing." + ], + reference_texts=[ + "This is a semantic similarity test.", + "Another sentence for testing." + ], + original_texts=[ + "Source text for semantic testing.", + "Another source text here." + ], + config=config + ) + + # Should include results from semantic metrics + assert isinstance(results, dict) + assert len(results) > 0 + + # Should include timing and basic stats + assert "time_total" in results + assert "word_length_gen" in results + assert "exact_match" in results + + # Exact match should be 1.0 for identical strings + assert results["exact_match"] == 1.0 + + except Exception as e: + # Some semantic metrics might not be available + pytest.skip(f"Semantic metrics test skipped due to: {e}") + + +if __name__ == "__main__": + pytest.main([__file__]) \ No newline at end of file From bb2490acd8169fadab35451c158b33b0e27f6143 Mon Sep 17 00:00:00 2001 From: alfekka Date: Fri, 13 Jun 2025 16:05:37 +0300 Subject: [PATCH 06/39] move tests into test folder --- {tests => test}/test_metrics.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {tests => test}/test_metrics.py (100%) diff --git a/tests/test_metrics.py b/test/test_metrics.py similarity index 100% rename from tests/test_metrics.py rename to test/test_metrics.py From 2e82f4db274496b89d14127c3767748162c37a96 Mon Sep 17 00:00:00 2001 From: Akim Tsvigun Date: Sat, 7 Jun 2025 10:07:24 +0000 Subject: [PATCH 07/39] preparing camera-ready version --- .../api_labellers/openai_labeller.py | 6 +- src/atgen/run_scripts/run_active_learning.py | 6 +- src/atgen/utils/constants.py | 2 +- src/atgen/utils/data/get_output_field.py | 11 ++++ .../utils/data/get_preprocess_function.py | 59 +++++++++++++++---- src/atgen/utils/data/load_data.py | 17 +++--- .../utils/data/prepare_conversational_data.py | 6 +- src/atgen/utils/generate.py | 26 ++++++-- src/atgen/utils/resolvers.py | 2 +- src/atgen/utils/training_utils.py | 20 ++++--- 10 files changed, 113 insertions(+), 42 deletions(-) create mode 100644 src/atgen/utils/data/get_output_field.py diff --git a/src/atgen/labellers/api_labellers/openai_labeller.py b/src/atgen/labellers/api_labellers/openai_labeller.py index 529cfbc..0c0530e 100644 --- a/src/atgen/labellers/api_labellers/openai_labeller.py +++ b/src/atgen/labellers/api_labellers/openai_labeller.py @@ -10,7 +10,7 @@ from shutil import rmtree from ..base_labeller import BaseLabeler -from ...utils.constants import DEEPSEEK_R1_END_REASONING_TOKEN, MESSAGES_COLUMN_NAME +from ...utils.constants import REASONING_END_TOKEN, MESSAGES_COLUMN_NAME log = logging.getLogger() @@ -126,8 +126,8 @@ def _sync_call(self, dataset: Dataset) -> Dataset: # Remove thinking tokens from DeepSeek-R1 if "deepseek-r1" in self.config.parameters.model: for i, annotation in enumerate(annotations): - annotations[i] = DEEPSEEK_R1_END_REASONING_TOKEN.join( - annotation.split(DEEPSEEK_R1_END_REASONING_TOKEN)[1:] + annotations[i] = REASONING_END_TOKEN.join( + annotation.split(REASONING_END_TOKEN)[1:] ).strip() # Add the annotations as a new column in the dataset diff --git a/src/atgen/run_scripts/run_active_learning.py b/src/atgen/run_scripts/run_active_learning.py index b22824a..b009409 100644 --- a/src/atgen/run_scripts/run_active_learning.py +++ b/src/atgen/run_scripts/run_active_learning.py @@ -9,7 +9,7 @@ from typing import Union import logging from atgen.utils.main_decorator import main_decorator -from atgen.utils.constants import DEFAULT_CONFIG_NAME +from atgen.utils.constants import DEFAULT_CONFIG_NAME, UNLABELED_DATA_SPLIT_DEFAULT_NAME, TEST_DATA_SPLIT_DEFAULT_NAME log = logging.getLogger() @@ -89,14 +89,14 @@ def run_active_learning(config, workdir: Union[str, Path]): log.info("Loading data.") unlabeled_data = load_data( data_config=config.data, - split=config.data.unlabeled_data_split_name, + split=UNLABELED_DATA_SPLIT_DEFAULT_NAME, cache_dir=config.cache_dir, seed=seed, ) if has_test: test_data = load_data( data_config=config.data, - split=config.data.test_split_name, + split=TEST_DATA_SPLIT_DEFAULT_NAME, cache_dir=config.cache_dir, seed=seed, ) diff --git a/src/atgen/utils/constants.py b/src/atgen/utils/constants.py index 2bfff31..a10fadc 100644 --- a/src/atgen/utils/constants.py +++ b/src/atgen/utils/constants.py @@ -6,7 +6,7 @@ DEFAULT_TEMPERATURE = 0.6 DEFAULT_TOP_P = 1 -DEEPSEEK_R1_END_REASONING_TOKEN = "" +REASONING_END_TOKEN = "" APPROX_NUM_SYSTEM_TOKENS_PER_MESSAGE = 10 DEFAULT_CONFIG_NAME = "base" diff --git a/src/atgen/utils/data/get_output_field.py b/src/atgen/utils/data/get_output_field.py new file mode 100644 index 0000000..ecb06eb --- /dev/null +++ b/src/atgen/utils/data/get_output_field.py @@ -0,0 +1,11 @@ +from typing import Union + +from datasets import Dataset + + +def _get_output_field(data: Union[Dataset, dict], output_field: list[str]) -> str: + """Access nested dictionary using a tuple path.""" + result = data + for key in output_field: + result = result[key] + return result \ No newline at end of file diff --git a/src/atgen/utils/data/get_preprocess_function.py b/src/atgen/utils/data/get_preprocess_function.py index ba31b8b..5955336 100644 --- a/src/atgen/utils/data/get_preprocess_function.py +++ b/src/atgen/utils/data/get_preprocess_function.py @@ -1,3 +1,5 @@ +from typing import Optional + from omegaconf import DictConfig from ..constants import MESSAGES_COLUMN_NAME @@ -11,6 +13,7 @@ def get_preprocess_function( is_in_conversational_format: bool = False, input_column_name: str = "input", output_column_name: str = "output", + assistant_response_start: Optional[str] = None, ): """ Creates and returns an appropriate preprocessing function based on: @@ -36,9 +39,7 @@ def preprocess_fn(instance): # Copy the first few-shot message and prepend system prompt first_fs = few_shot_messages[0].copy() first_fs["content"] = system_prompt + "\n\n" + first_fs["content"] - messages.append(first_fs) - # Add remaining few-shot messages - messages.extend(few_shot_messages[1:]) + messages = [first_fs] + few_shot_messages[1:] else: # No few-shot messages if is_in_conversational_format: @@ -47,6 +48,9 @@ def preprocess_fn(instance): conv_messages[0]["content"] = ( system_prompt + "\n\n" + conv_messages[0]["content"] ) + conv_messages = _maybe_add_assistant_response_start( + conv_messages, assistant_response_start + ) return {MESSAGES_COLUMN_NAME: conv_messages} else: # Add user message with system prompt @@ -66,10 +70,11 @@ def preprocess_fn(instance): if is_in_conversational_format: # Return the messages directly if in conversational format - return { - MESSAGES_COLUMN_NAME: few_shot_messages - + instance[input_column_name] - } + messages = few_shot_messages + instance[input_column_name] + messages = _maybe_add_assistant_response_start( + messages, assistant_response_start + ) + return {MESSAGES_COLUMN_NAME: messages} else: # Add user message messages.append( @@ -79,11 +84,23 @@ def preprocess_fn(instance): # Handle different split types if is_in_conversational_format: # For conversational format, we've already handled this above - return {MESSAGES_COLUMN_NAME: messages + instance[input_column_name]} + messages += instance[input_column_name] + messages = _maybe_add_assistant_response_start( + messages, assistant_response_start + ) + return {MESSAGES_COLUMN_NAME: messages} elif split == "train": # For training, add the assistant's response + if assistant_response_start: + assistant_message = ( + assistant_response_start + instance[output_column_name] + ) + else: + assistant_message = instance[output_column_name] + messages.append({"role": "assistant", "content": assistant_message}) + elif split == "test" and assistant_response_start: messages.append( - {"role": "assistant", "content": instance[output_column_name]} + {"role": "assistant", "content": assistant_response_start} ) return {MESSAGES_COLUMN_NAME: messages} @@ -104,7 +121,11 @@ def preprocess_fn(instance): # Handle different formats if is_in_conversational_format: # For conversational format, append input messages to existing ones - return {MESSAGES_COLUMN_NAME: messages + instance[input_column_name]} + messages += instance[input_column_name] + messages = _maybe_add_assistant_response_start( + messages, assistant_response_start + ) + return {MESSAGES_COLUMN_NAME: messages} else: # Add user message messages.append( @@ -113,10 +134,26 @@ def preprocess_fn(instance): # For training, add the assistant's response if split == "train": + if assistant_response_start: + assistant_message = ( + assistant_response_start + instance[output_column_name] + ) + else: + assistant_message = instance[output_column_name] + messages.append({"role": "assistant", "content": assistant_message}) + elif split == "test" and assistant_response_start: messages.append( - {"role": "assistant", "content": instance[output_column_name]} + {"role": "assistant", "content": assistant_response_start} ) return {MESSAGES_COLUMN_NAME: messages} return preprocess_fn + + +def _maybe_add_assistant_response_start( + messages: list[dict[str, str]], assistant_response_start: Optional[str] +) -> list[dict[str, str]]: + if assistant_response_start: + messages[-1]["content"] = assistant_response_start + messages[-1]["content"] + return messages diff --git a/src/atgen/utils/data/load_data.py b/src/atgen/utils/data/load_data.py index 0351b4b..b3201be 100644 --- a/src/atgen/utils/data/load_data.py +++ b/src/atgen/utils/data/load_data.py @@ -2,7 +2,7 @@ from typing import Union from datasets import load_dataset, load_from_disk, Dataset, DatasetDict -from omegaconf import DictConfig +from omegaconf import DictConfig, ListConfig def _fetch_dataset( @@ -10,8 +10,11 @@ def _fetch_dataset( subset_name: str, fetch_kwargs: dict | DictConfig, ) -> Dataset: + # Load a subset of a dataset from HuggingFace + if isinstance(dataset_name_or_path, (list, ListConfig)): + dataset = load_dataset(*dataset_name_or_path, **fetch_kwargs) # Load local dataset - if os.path.exists(dataset_name_or_path): + elif os.path.exists(dataset_name_or_path): # Load a saved on disk dataset if os.path.isdir(dataset_name_or_path): # Remove `cache_dir` from fetch_kwargs @@ -28,8 +31,6 @@ def _fetch_dataset( f"Unexpected format {dataset_name_or_path.split('.')[-1]} of the dataset. Supported formats: csv, json." ) # Load dataset from HuggingFace - elif isinstance(dataset_name_or_path, list): - dataset = load_dataset(*dataset_name_or_path, **fetch_kwargs) else: dataset = load_dataset(dataset_name_or_path, **fetch_kwargs) @@ -51,7 +52,9 @@ def _take_subset(dataset_subset: Dataset, size: int, seed: int) -> Dataset: return dataset_subset dataset_subset = dataset_subset.shuffle(seed=seed) dataset_subset = dataset_subset.select(range(size)) - dataset_subset = dataset_subset.remove_columns(["id"]).add_column("id", list(range(len(dataset_subset)))) + dataset_subset = dataset_subset.remove_columns(["id"]).add_column( + "id", list(range(len(dataset_subset))) + ) return dataset_subset @@ -62,10 +65,10 @@ def load_data( seed: int, ) -> Dataset: if split == "train": - subset_name = data_config.get("train_subset_name", split) + subset_name = data_config.get("train_split_name", split) subset_size = data_config.get("train_subset_size") elif split == "test": - subset_name = data_config.get("test_subset_name", split) + subset_name = data_config.get("test_split_name", split) subset_size = data_config.get("test_subset_size") else: raise NotImplementedError( diff --git a/src/atgen/utils/data/prepare_conversational_data.py b/src/atgen/utils/data/prepare_conversational_data.py index 86f1f10..eb82713 100644 --- a/src/atgen/utils/data/prepare_conversational_data.py +++ b/src/atgen/utils/data/prepare_conversational_data.py @@ -28,7 +28,9 @@ def prepare_conversational_data( ] for (fs_input, fs_output) in zip( few_shot_examples[input_column_name], - few_shot_examples[output_column_name], + few_shot_examples[output_column_name] + if isinstance(output_column_name, str) + else few_shot_examples[output_column_name][0], ) ] ) @@ -42,8 +44,8 @@ def prepare_conversational_data( is_in_conversational_format=data_config.is_in_conversational_format, input_column_name=input_column_name, output_column_name=output_column_name, + assistant_response_start=data_config.assistant_response_start, ) - dataset = dataset.map( preprocess_fn, batched=False, diff --git a/src/atgen/utils/generate.py b/src/atgen/utils/generate.py index 95a86c0..922ffc7 100644 --- a/src/atgen/utils/generate.py +++ b/src/atgen/utils/generate.py @@ -21,7 +21,7 @@ DEFAULT_TEMPERATURE, DEFAULT_TOP_P, MESSAGES_COLUMN_NAME, - DEEPSEEK_R1_END_REASONING_TOKEN, + REASONING_END_TOKEN, ) from .training_utils import _get_data_collator @@ -72,12 +72,22 @@ def generate_vllm( gc.collect() cuda.empty_cache() - params = SamplingParams( + sampling_params = SamplingParams( temperature=inference_config.get("temperature", DEFAULT_TEMPERATURE), seed=42, # TODO: make arbitrary max_tokens=inference_config.max_new_tokens, top_p=inference_config.get("top_p", DEFAULT_TOP_P), ) + if data_config.assistant_response_start: + generation_params = { + "add_generation_prompt": False, + "continue_final_message": True, + } + else: + generation_params = { + "add_generation_prompt": True, + "continue_final_message": False, + } generations = [] num_batches = ceil(len(data) / inference_config.batch_size) @@ -85,16 +95,16 @@ def generate_vllm( batch = data[ i * inference_config.batch_size : (i + 1) * inference_config.batch_size ][MESSAGES_COLUMN_NAME] - out = llm_runner.chat(batch, params, use_tqdm=False) + out = llm_runner.chat( + batch, sampling_params, use_tqdm=False, **generation_params + ) batch_generations = [x.outputs[0].text for x in out] generations += batch_generations # Remove reasoning tokens from DeepSeek-R1 if "deepseek-r1" in llm_runner.llm_engine.model_config.model: for i, generation in enumerate(generations): - generations[i] = DEEPSEEK_R1_END_REASONING_TOKEN.join( - generation.split(DEEPSEEK_R1_END_REASONING_TOKEN)[1:] - ).strip() + generations[i] = _remove_thinking_part(generation) if delete_vllm_after_inference: del llm_runner @@ -318,3 +328,7 @@ def tokenize_conversational_example( input_ids = tokenizer.apply_chat_template(example["messages"]) attention_mask = [1 for _ in range(len(input_ids))] return {"input_ids": input_ids, "attention_mask": attention_mask} + + +def _remove_thinking_part(text: str) -> str: + return REASONING_END_TOKEN.join(text.split(REASONING_END_TOKEN)[1:]).strip() diff --git a/src/atgen/utils/resolvers.py b/src/atgen/utils/resolvers.py index 629b453..891c7e1 100644 --- a/src/atgen/utils/resolvers.py +++ b/src/atgen/utils/resolvers.py @@ -36,7 +36,7 @@ def register_resolvers() -> None: "multiply_with_few_shot": multiply_with_few_shot, "to_string": to_string, } - + for name, resolver_fn in resolvers_to_register.items(): try: OmegaConf.register_new_resolver(name, resolver_fn) diff --git a/src/atgen/utils/training_utils.py b/src/atgen/utils/training_utils.py index 88c2d55..1646d3f 100644 --- a/src/atgen/utils/training_utils.py +++ b/src/atgen/utils/training_utils.py @@ -16,9 +16,9 @@ ) from trl import SFTTrainer, SFTConfig, DataCollatorForCompletionOnlyLM from trl.data_utils import ( - is_conversational, - maybe_apply_chat_template, - maybe_convert_to_chatml + is_conversational, + maybe_apply_chat_template, + maybe_convert_to_chatml, ) from trl.trainer.utils import ConstantLengthDataset from accelerate import PartialState @@ -87,7 +87,7 @@ class DataCollatorForLastCompletionOnlyLM(DataCollatorForCompletionOnlyLM): Data collator that extends DataCollatorForCompletionOnlyLM to only train on the last assistant message. It ensures that only the final assistant response will contribute to the loss, while all previous messages (including earlier assistant responses) are masked with the ignore_index. - + """ def torch_call(self, examples): @@ -241,8 +241,12 @@ def _get_train_eval_datasets( batched=True, fn_kwargs={"tokenizer": tokenizer, "data_collator": data_collator}, ).filter(lambda x: x[TEXT_FIELD] != "") - logger.warning(f"Truncated {orig_train_data_len - len(train_data)} examples from train set.") - logger.warning(f"Truncated {orig_eval_data_len - len(eval_data)} examples from eval set.") + logger.warning( + f"Truncated {orig_train_data_len - len(train_data)} examples from train set." + ) + logger.warning( + f"Truncated {orig_eval_data_len - len(eval_data)} examples from eval set." + ) return train_data, eval_data @@ -269,6 +273,7 @@ def _formatting_fn(examples, tokenizer: PreTrainedTokenizerFast): add_generation_prompt=False, ) + def get_trainer( config: DictConfig, model: PreTrainedModel | PeftModel, @@ -298,7 +303,7 @@ def get_trainer( tokenizer=tokenizer, data_collator=data_collator, ) - + return SFTTrainer( model=model, processing_class=tokenizer, @@ -309,4 +314,3 @@ def get_trainer( callbacks=callbacks, formatting_func=partial(_formatting_fn, tokenizer=tokenizer), ) - From 55eaffc15cff0693fa39f2b9425d03d8cf7a1d98 Mon Sep 17 00:00:00 2001 From: Akim Tsvigun Date: Sat, 7 Jun 2025 10:08:49 +0000 Subject: [PATCH 08/39] Readme & License fixed --- LICENSE.md | 2 +- README.md | 120 ++++++++++++++++++++++++++++++-- configs/base.yaml | 5 +- configs/custom.yaml | 81 --------------------- configs/data/aeslc.yaml | 3 +- configs/data/base.yaml | 25 +++---- configs/data/test.yaml | 1 + configs/data/trivia_qa.yaml | 13 ++++ configs/test.yaml | 3 +- configs/test2.yaml | 79 --------------------- pages/0_Configure_Experiment.py | 53 +++++++++----- 11 files changed, 185 insertions(+), 200 deletions(-) delete mode 100644 configs/custom.yaml create mode 100644 configs/data/trivia_qa.yaml delete mode 100644 configs/test2.yaml diff --git a/LICENSE.md b/LICENSE.md index c8acd71..33243b5 100644 --- a/LICENSE.md +++ b/LICENSE.md @@ -1,4 +1,4 @@ -Copyright (c) 2023 MBZUAI +Copyright (c) 2025 MBZUAI Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/README.md b/README.md index 74a4be0..e387c5f 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,118 @@ -# ATGen: Active Text Generation +# ATGen: Active Learning for Natural Language Generation -## How to launch without config +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) + +A comprehensive toolkit for applying active learning techniques to natural language generation tasks. This repository contains implementations of various active learning strategies specifically designed for text generation models, helping to reduce annotation costs while maximizing model performance. + +## šŸŒŸ Features + +- **Multiple Active Learning Strategies**: Implementation of strategies like HUDS, HADAS, FAC-LOC, IDDS, and more +- **Flexible Model Support**: Compatible with various language models (Qwen, Llama, etc.) +- **Comprehensive Evaluation**: Supports multiple evaluation metrics including ROUGE, BLEU, BERTScore, AlignScore, etc. +- **Interactive Visualization**: Streamlit dashboard for exploring results and comparing strategies +- **Hydra Configuration**: Easily configurable experiments through Hydra's YAML-based configuration system +- **PEFT Integration**: Efficient fine-tuning using Parameter-Efficient Fine-Tuning methods + +## šŸ“‹ Requirements + +- Python 3.10+ +- CUDA-compatible GPU (for model training) +- Dependencies listed in `requirements.txt` + +## šŸ”§ Installation + +1. Clone the repository: + ```bash + git clone https://github.com/Aktsvigun/atgen.git + cd atgen + ``` + +2. Run the installation script: + ```bash + bash install.sh + ``` + + This will install: + - All required Python packages + - External metrics (submodlib, AlignScore) + - Required NLP resources + +## šŸš€ Usage + +### Running Active Learning Experiments + +Experiments can be launched using the `run-al` command: + +```bash +CUDA_VISIBLE_DEVICES=0 HYDRA_CONFIG_NAME=base run-al +``` + +Parameters: +- `CUDA_VISIBLE_DEVICES`: Specify which GPU to use +- `HYDRA_CONFIG_NAME`: Configuration file (e.g., `base`, `custom`, `test`) + +Additional parameters can be overridden via the command line following Hydra's syntax: + +```bash +CUDA_VISIBLE_DEVICES=0 HYDRA_CONFIG_NAME=base run-al al.strategy=huds model.checkpoint=Qwen/Qwen2.5-7B +``` + +### Interactive Dashboard + +Launch the Streamlit application to explore and visualize your experiments: ```bash -HYDRA_CONFIG_NAME=base HYDRA_CONFIG_PATH=./../../../configs python3 src/atgen/run_scripts/run_active_learning.py al.query_size=10 al.num_iterations=5 a -l.query_size=10 data.dataset=Harvard/gigaword data.input_column_name=document data.output_column_name=summary labeler.type=golden al.strategy=random -``` \ No newline at end of file +streamlit run Welcome.py +``` + +Navigate to `http://localhost:8501` in your web browser to access the dashboard. + +## šŸ“ Project Structure + +- `configs/`: Configuration files for experiments + - `al/`: Active learning strategy configurations + - `data/`: Dataset configurations + - `labeller/`: Labeller configurations +- `src/atgen/`: Main package + - `strategies/`: Implementation of active learning strategies + - `metrics/`: Code for evaluation metrics + - `utils/`: Utility functions + - `run_scripts/`: Scripts for running experiments + - `labellers/`: Labelling mechanisms + - `visualize/`: Visualization tools +- `pages/`: Streamlit application pages +- `outputs/`: Experimental results storage +- `cache/`: Cached computations to speed up repeated runs + +## šŸ“š Supported Active Learning Strategies + +- `huds`: Hypothetical Document Scoring +- `hadas`: Harmonic Diversity Scoring +- `random`: Random sampling baseline +- `fac-loc`: Facility Location strategy +- `idds`: Improved Diverse Density Scoring +- And more... + +## šŸ“Š Supported Datasets + +The toolkit comes pre-configured for several datasets including summarization, question answering, and other generative tasks. Custom datasets can be added by creating new configuration files. + +## šŸ¤ Contributing + +Contributions are welcome! Please feel free to submit a Pull Request. + +## šŸ“œ License + +This project is licensed under the MIT License - see the [LICENSE.md](LICENSE.md) file for details. + +## šŸ”— Citation + +If you use this toolkit in your research, please cite: + +``` +@software{atgen, + title = {ATGen: Active Learning for Natural Language Generation}, + url = {https://github.com/Aktsvigun/atgen}, + year = {2025}, +} +``` \ No newline at end of file diff --git a/configs/base.yaml b/configs/base.yaml index 38cd28b..fae2e4c 100644 --- a/configs/base.yaml +++ b/configs/base.yaml @@ -30,11 +30,12 @@ al: budget: model: - checkpoint: Qwen/Qwen2.5-1.5B-Instruct + checkpoint: Qwen/Qwen3-1.7B quantize: False model_max_length: ${multiply_with_few_shot:${data.input_max_length},${data.few_shot.count}} dtype: bfloat16 save_in_fp_32: false + assistant_response_start: "\n\n\n\n" peft: use: True r: 16 @@ -80,7 +81,7 @@ evaluation: provider: openrouter base_url: https://openrouter.ai/api/v1 api_key: - model: anthropic/claude-3-7-sonnet + model: nebius/Qwen/Qwen3-235B-A22B deepeval_threshold: 0.5 deepeval_include_reason: False deepeval_strict_mode: False diff --git a/configs/custom.yaml b/configs/custom.yaml deleted file mode 100644 index 33ad53e..0000000 --- a/configs/custom.yaml +++ /dev/null @@ -1,81 +0,0 @@ -defaults: - - labeller: custom_llm - - data: aeslc - - al: random - -experiment_name: base -name: base -output_dir: outputs -offline_mode: False -cache_dir: cache -seed: 42 -al: - # Strategy configuration now loaded from configs/al/.yaml - # Other AL settings that aren't strategy-specific - init_query_size: 10 - query_size: 10 - num_iterations: 5 - evaluate_zero_iteration: True - subsample_size: -1 - required_performance: - rouge1: 0.5 - budget: - -model: - checkpoint: Qwen/Qwen2.5-1.5B-Instruct - quantize: False - model_max_length: ${multiply_with_few_shot:${data.input_max_length},${data.few_shot.count}} - dtype: bfloat16 - save_in_fp_32: false - peft: - use: True - r: 64 - lora_alpha: 64 - lora_dropout: 0.1 - bias: 'none' - seed: ${seed} - use_gradient_checkpointing: 'unsloth' - modules: - -training: - dev_split_size: 0.2 - hyperparameters: - num_epochs: 5 - train_batch_size: 6 - eval_batch_size: 4 - gradient_accumulation_steps: 2 - lr: 0.00003 - warmup_ratio: 0.03 - weight_decay: 0.01 - max_grad_norm: 1. - early_stopping_patience: 5 - model_max_length: ${model.model_max_length} - dataset_num_proc: ${data.num_proc} - packing: False - lr_scheduler_type: cosine - gradient_checkpointing: True - -inference: - framework: 'vllm' - max_new_tokens: ${data.output_max_length} - batch_size: 8 - gpu_memory_utilization: 0.5 - temperature: 0.6 - top_p: 0.1 - model: ${model.checkpoint} - -evaluation: - additional_metrics: - - alignscore - - bartscore - provider: openrouter - base_url: https://openrouter.ai/api/v1 - api_key: - model: anthropic/claude-3-7-sonnet - deepeval_threshold: 0.5 - deepeval_include_reason: False - deepeval_strict_mode: False - deepeval_async_mode: True - deepeval_verbose_mode: False - deepeval_truths_extraction_limit: 10 - \ No newline at end of file diff --git a/configs/data/aeslc.yaml b/configs/data/aeslc.yaml index 00cffeb..87f3a27 100644 --- a/configs/data/aeslc.yaml +++ b/configs/data/aeslc.yaml @@ -9,4 +9,5 @@ input_max_length: 1024 output_max_length: 20 fetch_kwargs: {} is_in_conversational_format: false -system_prompt: "Write a subject text for the email.\nEmail:" +system_prompt: "Write a subject text for the email.\n\n" +assistant_response_start: ${model.assistant_response_start} diff --git a/configs/data/base.yaml b/configs/data/base.yaml index def309f..9b55985 100644 --- a/configs/data/base.yaml +++ b/configs/data/base.yaml @@ -1,15 +1,16 @@ -dataset: SpeedOfMagic/gigaword_tiny -few_shot_examples: +dataset: str or list[str] +few_shot_examples: list[dict[str, str]] fetch_kwargs: {} few_shot: count: 0 -unlabeled_data_split_name: train -test_split_name: test -input_column_name: document -input_max_length: 100 -output_column_name: summary -output_max_length: 20 -system_prompt: "Write a headline for the article in lowercase." -test_subset_size: 3 -train_subset_size: 100 -is_in_conversational_format: false +unlabeled_data_split_name: str +test_split_name: str +input_column_name: str +input_max_length: int +output_column_name: str or list[str] +output_max_length: int +test_subset_size: int +train_subset_size: int +system_prompt: str +assistant_response_start: str +is_in_conversational_format: bool diff --git a/configs/data/test.yaml b/configs/data/test.yaml index d267886..faa4787 100644 --- a/configs/data/test.yaml +++ b/configs/data/test.yaml @@ -12,3 +12,4 @@ system_prompt: "Write a headline for the article in lowercase." test_subset_size: 3 train_subset_size: 100 is_in_conversational_format: false +assistant_response_start: ${model.assistant_response_start} diff --git a/configs/data/trivia_qa.yaml b/configs/data/trivia_qa.yaml new file mode 100644 index 0000000..ad85e98 --- /dev/null +++ b/configs/data/trivia_qa.yaml @@ -0,0 +1,13 @@ +dataset: ['mandarjoshi/trivia_qa', 'rc.wikipedia'] +input_column_name: 'question' +output_column_name: ['answer', 'value'] +unlabeled_data_split_name: train +test_split_name: validation +train_subset_size: null +test_subset_size: 1_000 +input_max_length: 1024 +output_max_length: 20 +fetch_kwargs: {} +is_in_conversational_format: false +system_prompt: "Please answer the question:\n\n" +assistant_response_start: ${model.assistant_response_start} diff --git a/configs/test.yaml b/configs/test.yaml index 8528ba8..e762094 100644 --- a/configs/test.yaml +++ b/configs/test.yaml @@ -20,11 +20,12 @@ al: budget: model: - checkpoint: Qwen/Qwen2.5-0.5B-Instruct + checkpoint: Qwen/Qwen3-0.6B quantize: False model_max_length: ${multiply_with_few_shot:${data.input_max_length},${data.few_shot.count}} dtype: bfloat16 save_in_fp_32: false + assistant_response_start: "\n\n\n\n" peft: use: True r: 64 diff --git a/configs/test2.yaml b/configs/test2.yaml deleted file mode 100644 index a6ec2f2..0000000 --- a/configs/test2.yaml +++ /dev/null @@ -1,79 +0,0 @@ -defaults: - - labeller: golden - - data: test - - al: random - -experiment_name: test2 -name: test2 -output_dir: outputs -offline_mode: False -cache_dir: cache -seed: 42 -al: - init_query_size: 42 - query_size: 20 - num_iterations: 3 - evaluate_zero_iteration: True - subsample_size: -1 - required_performance: - rouge1: 0.5 - budget: - -model: - checkpoint: Qwen/Qwen2.5-1.5B-Instruct - quantize: False - model_max_length: ${multiply_with_few_shot:${data.input_max_length},${data.few_shot.count}} - dtype: bfloat16 - save_in_fp_32: false - peft: - use: True - r: 16 - lora_alpha: 16 - lora_dropout: 0.1 - bias: 'none' - seed: ${seed} - use_gradient_checkpointing: 'unsloth' - modules: - -training: - dev_split_size: 0.2 - hyperparameters: - num_epochs: 15 - train_batch_size: 64 - eval_batch_size: 64 - gradient_accumulation_steps: 1 - lr: 0.00003 - warmup_ratio: 0.03 - weight_decay: 0.01 - max_grad_norm: 1. - early_stopping_patience: 10 - model_max_length: ${model.model_max_length} - dataset_num_proc: ${data.num_proc} - packing: False - lr_scheduler_type: cosine - gradient_checkpointing: True - -inference: - framework: 'vllm' - max_new_tokens: ${data.output_max_length} - batch_size: 8 - gpu_memory_utilization: 0.5 - temperature: 0.6 - top_p: 0.1 - model: ${model.checkpoint} - -evaluation: - additional_metrics: - - alignscore - - bartscore - provider: openrouter - base_url: https://openrouter.ai/api/v1 - api_key: - model: anthropic/claude-3-7-sonnet - deepeval_threshold: 0.5 - deepeval_include_reason: False - deepeval_strict_mode: False - deepeval_async_mode: True - deepeval_verbose_mode: False - deepeval_truths_extraction_limit: 10 - \ No newline at end of file diff --git a/pages/0_Configure_Experiment.py b/pages/0_Configure_Experiment.py index 18626ca..b1e761e 100644 --- a/pages/0_Configure_Experiment.py +++ b/pages/0_Configure_Experiment.py @@ -429,13 +429,13 @@ def main(): col1, col2, col3 = st.columns(3) with col1: if st.button("šŸ“Š View Metrics", use_container_width=True): - st.switch_page("./pages/1_Metrics.py") + st.switch_page("1_Metrics") with col2: if st.button("šŸ·ļø View Labeled Examples", use_container_width=True): - st.switch_page("./pages/2_Labeled_examples.py") + st.switch_page("2_Labeled_examples") with col3: if st.button("šŸ‘©ā€šŸŽØ Annotate Examples", use_container_width=True): - st.switch_page("./pages/3_Annotation.py") + st.switch_page("3_Annotation") st.stop() st.markdown( @@ -573,10 +573,10 @@ def main(): labeller = st.radio( "šŸ‘Øā€šŸ’¼ Labeller", [ + "Golden (only for benchmarking)", "Open-source / Custom LLM", "API LLM", "Human", - "Golden (only for benchmarking)", ], help="Select the type of labeller to use for data annotation", ) @@ -592,12 +592,29 @@ def main(): help="Cost paid to human annotators per example", ) elif labeller == "custom_llm": - model_checkpoint = st.text_input( + labeller_checkpoint = st.text_input( "šŸ¤– Model checkpoint from HuggingFace", - value="Qwen/QwQ-32B", + value="Qwen/Qwen3-32B", help="HuggingFace model ID for the custom LLM", ) elif labeller == "api_llm": + # Add data privacy disclaimer + st.markdown( + """ +
+

+ āš ļø Data Privacy Notice +

+

+ Important: When using API-based labellers (OpenAI, Anthropic, etc.), your dataset will be sent to external services for processing. + Please ensure you have the necessary permissions and that your data complies with the respective service providers' terms of use and privacy policies. + Consider using local/custom models if your data contains sensitive or proprietary information. +

+
+ """, + unsafe_allow_html=True, + ) + col1, col2 = st.columns(2) with col1: provider = st.radio( @@ -664,7 +681,7 @@ def main(): # Dataset input dataset = st.text_input( "šŸ“š Dataset or path to data", - value="SpeedOfMagic/gigaword_tiny", + value="Yale-LILY/aeslc", help="HuggingFace dataset ID or local path to dataset", ) @@ -788,13 +805,13 @@ def main(): with col1: input_field = st.text_input( "šŸ“„ Input field name", - value="document", + value="email_body", help="Name of the field containing input text in the dataset", ) with col2: reference_field = st.text_input( "šŸ“¤ Reference field name", - value="summary", + value="subject_line", help="Name of the field containing reference output in the dataset", ) @@ -822,7 +839,7 @@ def main(): with col1: model_checkpoint = st.text_input( "šŸ¤– Model checkpoint", - value="Qwen/Qwen2.5-1.5B-Instruct", + value="Qwen/Qwen3-1.7B", help="HuggingFace model ID for generation", ) @@ -1266,7 +1283,7 @@ def main(): ] = price_input_per_example elif labeller == "custom_llm": if "model_checkpoint" in locals(): - config["labeller"]["model"]["checkpoint"] = model_checkpoint + config["labeller"]["model"]["checkpoint"] = labeller_checkpoint elif labeller == "api_llm": config["labeller"]["api_key"] = api_key config["labeller"]["provider"] = provider @@ -1401,24 +1418,24 @@ def main(): nav_col1, nav_col2, nav_col3 = st.columns(3) with nav_col1: if st.button("šŸ“Š View Metrics", use_container_width=True): - st.switch_page("1_Metrics.py") + st.switch_page("1_Metrics") with nav_col2: if st.button( "šŸ·ļø View Labeled Examples", use_container_width=True ): - st.switch_page("2_Labeled_examples.py") + st.switch_page("2_Labeled_examples") with nav_col3: if st.button( "šŸ‘©ā€šŸŽØ Annotate Examples", use_container_width=True ): - st.switch_page("3_Annotation.py") + st.switch_page("3_Annotation") except Exception as e: # Update status to failed update_experiment_status(STATUS_FAILED) - st.error(f"An error occurred: {str(e)}") - import traceback - - st.code(traceback.format_exc(), language="python") + # st.error(f"An error occurred: {str(e)}") + # import sys, pdb + # exc_type, exc_value, exc_traceback = sys.exc_info() + # pdb.post_mortem(exc_traceback) elif is_valid_required_performance: st.error("āŒ You didn't fill one of the required arguments.") From a716a54331d7d3941498111e94d415f17fea5741 Mon Sep 17 00:00:00 2001 From: Akim Tsvigun Date: Mon, 9 Jun 2025 14:37:46 +0000 Subject: [PATCH 09/39] fixed version for multi-ref datasets --- configs/data/trivia_qa.yaml | 2 +- pages/0_Configure_Experiment.py | 2 +- src/atgen/labellers/golden_labeller.py | 27 +- src/atgen/metrics/compute_metrics.py | 171 ++++- ... deepeval_supported_models_and_metrics.py} | 0 src/atgen/metrics/metrics.py | 637 ++++++++++++++++++ src/atgen/run_scripts/run_active_learning.py | 30 +- src/atgen/utils/constants.py | 3 + src/atgen/utils/data/__init__.py | 2 + .../utils/data/get_output_column_name.py | 19 + src/atgen/utils/data/get_output_field.py | 11 - .../utils/data/get_preprocess_function.py | 12 +- src/atgen/utils/data/load_data.py | 30 +- .../utils/data/prepare_conversational_data.py | 17 +- 14 files changed, 919 insertions(+), 44 deletions(-) rename src/atgen/metrics/{supported_models_and_metrics.py => deepeval_supported_models_and_metrics.py} (100%) create mode 100644 src/atgen/metrics/metrics.py create mode 100644 src/atgen/utils/data/get_output_column_name.py delete mode 100644 src/atgen/utils/data/get_output_field.py diff --git a/configs/data/trivia_qa.yaml b/configs/data/trivia_qa.yaml index ad85e98..4ee7533 100644 --- a/configs/data/trivia_qa.yaml +++ b/configs/data/trivia_qa.yaml @@ -1,6 +1,6 @@ dataset: ['mandarjoshi/trivia_qa', 'rc.wikipedia'] input_column_name: 'question' -output_column_name: ['answer', 'value'] +output_column_name: {'train': ['answer', 'value'], 'test': ['answer', 'aliases']} unlabeled_data_split_name: train test_split_name: validation train_subset_size: null diff --git a/pages/0_Configure_Experiment.py b/pages/0_Configure_Experiment.py index b1e761e..3a65e4d 100644 --- a/pages/0_Configure_Experiment.py +++ b/pages/0_Configure_Experiment.py @@ -16,7 +16,7 @@ from pathlib import Path from atgen.run_scripts import run_active_learning -from atgen.metrics.supported_models_and_metrics import ( +from atgen.metrics.deepeval_supported_models_and_metrics import ( get_available_metrics, get_available_models, ) diff --git a/src/atgen/labellers/golden_labeller.py b/src/atgen/labellers/golden_labeller.py index 6d2fd45..5e20589 100644 --- a/src/atgen/labellers/golden_labeller.py +++ b/src/atgen/labellers/golden_labeller.py @@ -1,3 +1,5 @@ +from typing import Union +from omegaconf import DictConfig, ListConfig from datasets import Dataset from .base_labeller import BaseLabeler @@ -6,7 +8,26 @@ # Labels rows by using their label (should be used when evaluating strategies) class GoldenLabeler(BaseLabeler): def __call__(self, dataset: Dataset) -> Dataset: - assert ( - self.output_column_name in dataset.column_names - ), "Column with labels was not found in dataset" + _check_output_column_in_dataset(column_names=self.output_column_name, dataset=dataset) return dataset + + +def _check_output_column_in_dataset( + column_names: Union[DictConfig, ListConfig, dict[str, Union[str, list[str]]], list[str], str], + dataset: Union[Dataset, dict[str, Union[str, dict[str, str]]]], +) -> None: + if isinstance(column_names, str): + assert ( + column_names in dataset.column_names + ), f"Column {column_names} with labels was not found in dataset" + elif isinstance(column_names, (ListConfig, list)): + if isinstance(dataset, Dataset): + assert column_names[0] in dataset.column_names, f"Column {column_names[0]} with labels was not found in dataset" + dataset = dataset[0] + else: + assert column_names[0] in dataset.keys(), f"Column {column_names[0]} with labels was not found in dataset" + if len(column_names) > 1: + _check_output_column_in_dataset(column_names=column_names[1:], dataset=dataset[column_names[0]]) + elif isinstance(column_names, (DictConfig, dict)): + for purpose, column_name in column_names.items(): + _check_output_column_in_dataset(column_names=column_name, dataset=dataset) diff --git a/src/atgen/metrics/compute_metrics.py b/src/atgen/metrics/compute_metrics.py index 52cead5..cd81a62 100644 --- a/src/atgen/metrics/compute_metrics.py +++ b/src/atgen/metrics/compute_metrics.py @@ -1,5 +1,9 @@ from time import time +<<<<<<< HEAD from typing import List, Dict, Union, Optional +======= +from typing import Dict, Union +>>>>>>> eaad08f (fixed version for multi-ref datasets) import logging import numpy as np from omegaconf import DictConfig @@ -7,7 +11,22 @@ from .factory import MetricsFactory, MetricsConfig, get_metric_requirements from .base import MetricConfig +<<<<<<< HEAD logger = logging.getLogger(__name__) +======= +from .metrics import ( + pair_bleu, + calculate_bart_score, + calculate_alignscore, + calculate_deepeval_metrics, + is_bart_score_available, + is_alignscore_available, +) +from .deepeval_supported_models_and_metrics import API_MODELS, DEEPEVAL_METRICS + + +log = logging.getLogger() +>>>>>>> eaad08f (fixed version for multi-ref datasets) def compute_metrics( @@ -193,6 +212,7 @@ def get_default_config() -> MetricsConfig: aggregate=True ) +<<<<<<< HEAD def get_comprehensive_config() -> MetricsConfig: """Get a comprehensive metrics configuration with all metrics.""" @@ -207,8 +227,97 @@ def get_comprehensive_config() -> MetricsConfig: cache_dir="cache", aggregate=True ) +======= + # Avoid division by zero + src_word_lengths_safe = np.where(src_word_lengths > 0, src_word_lengths, 1) + result["word_length_src_rel"] = result["word_length_gen"] / src_word_lengths_safe + if "bartscore" in config.additional_metrics and is_bart_score_available: + log.info("Calculating BARTScore scores...") + start_time = time() + result.update( + calculate_bart_score( + preds=generated_texts, + texts=original_texts, + refs=reference_texts, + batch_size=4, + cache_dir=cache_dir, + ) + ) + time_dict["time_bartscore"] = time() - start_time + # Metrics that use both the generated texts and the reference texts + if reference_texts is not None: + # Exact match + if isinstance(reference_texts[0], list): + result["exact_match"] = np.array( + [ + any(pred == one_ref for one_ref in ref) + for pred, ref in zip(generated_texts, reference_texts) + ] + ) + else: + result["exact_match"] = np.array( + [pred == ref for pred, ref in zip(generated_texts, reference_texts)] + ) + # BLEU + start_time = time() + result["bleu"] = np.array( + [ + pair_bleu(references=ref, prediction=pred) + for pred, ref in tqdm(zip(generated_texts, reference_texts)) + ] + ) + time_dict["time_bleu"] = time() - start_time + # ROUGE + start_time = time() + result.update( + rouge.compute( + predictions=generated_texts, + references=reference_texts, + use_stemmer=True, + ) + ) + time_dict["time_rouge"] = time() - start_time + # Sacrebleu + start_time = time() + if not isinstance(reference_texts[0], list): + sacrebleu_references = [[ref] for ref in reference_texts] + sacrebleu_result = sacrebleu.compute( + predictions=generated_texts, references=sacrebleu_references + ) + result["sacrebleu"] = sacrebleu_result.pop("score") + else: + sacrebleu_scores = [] + for pred, ref in zip(generated_texts, reference_texts): + sacrebleu_result = sacrebleu.compute( + predictions=[pred], references=[ref] + ) + sacrebleu_scores.append(sacrebleu_result.pop("score")) + result["sacrebleu"] = sacrebleu_scores + + time_dict["time_sacrebleu"] = time() - start_time + # Lengths + if isinstance(reference_texts[0], list): + ref_word_lengths = np.array([np.mean([len(text.split()) for text in ref]) for ref in reference_texts]) + else: + ref_word_lengths = np.array([len(ref.split()) for ref in reference_texts]) + # Avoid division by zero + ref_word_lengths_safe = np.where(ref_word_lengths > 0, ref_word_lengths, 1) + result["word_length_rel"] = result["word_length_gen"] / ref_word_lengths_safe + # AlignScore + if "alignscore" in config.additional_metrics and is_alignscore_available: + log.info("Calculating AlignScore scores...") + start_time = time() + alignscores = calculate_alignscore( + generated_texts, reference_texts, original_texts + ) + if alignscores is not None: + result.update(alignscores) + time_dict["time_alignscore"] = time() - start_time +>>>>>>> eaad08f (fixed version for multi-ref datasets) + +<<<<<<< HEAD def get_deepeval_config(api_key: str, model: str = "openai/gpt-4o-mini") -> MetricsConfig: return MetricsConfig( metrics=[ @@ -227,4 +336,64 @@ def get_deepeval_config(api_key: str, model: str = "openai/gpt-4o-mini") -> Metr threshold=0.5, async_mode=True, verbose_mode=False - ) \ No newline at end of file + ) +======= + if deepeval_metrics_to_calculate: + if isinstance(reference_texts[0], list): + log.error("DeepEval does not support multiple references. Skipping...") + else: + # Validate OpenRouter model - only warn if not in predefined list, but still use it + provider = config["provider"] + if config.model not in API_MODELS.get(provider): + log.warning( + f"Using custom model: {config.model}. " + + ( + f"Available models: {API_MODELS[provider]}" + if provider in API_MODELS + else "" + ) + ) + log.info( + f"Calculating DeepEval metrics: {', '.join(deepeval_metrics_to_calculate)}..." + ) + start_time = time() + result.update( + calculate_deepeval_metrics( + predictions=generated_texts, + references=reference_texts, + original_texts=original_texts, + metrics_to_calculate=deepeval_metrics_to_calculate, + base_url=config.base_url, + api_key=config.api_key, + model=config.model, + threshold=config.deepeval_threshold, + include_reason=config.deepeval_include_reason, + strict_mode=config.deepeval_strict_mode, + async_mode=config.deepeval_async_mode, + verbose_mode=config.deepeval_verbose_mode, + truths_extraction_limit=config.deepeval_truths_extraction_limit, + ) + ) + time_dict["time_deepeval"] = time() - start_time + + for key, value in result.items(): + if isinstance(value, np.ndarray): + result[key] = float(np.mean(value)) + elif isinstance(value, (int, float)): + # Ensure numerical values are converted to float + result[key] = float(value) + # Make sure non-numerical values that aren't reasons are preserved + elif not key.endswith("_reasons") and not "_reason" in key.lower(): + continue + + # Filter out reason fields from the final aggregated results - more robust filtering + result = { + key: value + for key, value in sorted(result.items()) + if not key.endswith("_reasons") + and not "_reason" in key.lower() + and isinstance(value, (int, float)) # Ensure we only keep numerical metrics + } + + return result +>>>>>>> eaad08f (fixed version for multi-ref datasets) diff --git a/src/atgen/metrics/supported_models_and_metrics.py b/src/atgen/metrics/deepeval_supported_models_and_metrics.py similarity index 100% rename from src/atgen/metrics/supported_models_and_metrics.py rename to src/atgen/metrics/deepeval_supported_models_and_metrics.py diff --git a/src/atgen/metrics/metrics.py b/src/atgen/metrics/metrics.py new file mode 100644 index 0000000..9659eee --- /dev/null +++ b/src/atgen/metrics/metrics.py @@ -0,0 +1,637 @@ +from math import ceil +import os +import sys +from openai import OpenAI, AsyncOpenAI +from typing import Union +from urllib.request import urlretrieve +import logging +from pathlib import Path +import nltk +import numpy as np +import torch +import torch.nn.functional as F +from datasets import Dataset +from nltk import ngrams +from nltk.stem import porter +from nltk.tokenize import word_tokenize, sent_tokenize +from nltk.translate.bleu_score import corpus_bleu +from rouge_score import tokenize +from torch.utils.data import DataLoader +from transformers import ( + AutoModelForSequenceClassification, + AutoTokenizer, + AutoModel, + DataCollatorWithPadding, +) + +log = logging.getLogger(__name__) + +try: + from .bart_score import BARTScorer + + is_bart_score_available = True +except ImportError: + log.warning( + "BARTScorer not found, please install it (see `install.sh`). Skipping the BARTScore metric." + ) + is_bart_score_available = False + +try: + from alignscore import AlignScore + + is_alignscore_available = True +except ImportError: + log.warning( + "AlignScore not found, please install it (see `install.sh`). Skipping the AlignScore metric." + ) + is_alignscore_available = False + +from deepeval import evaluate +from deepeval.models.base_model import DeepEvalBaseLLM +from deepeval.test_case import LLMTestCase +from deepeval.metrics import ( + AnswerRelevancyMetric, + FaithfulnessMetric, + SummarizationMetric, + PromptAlignmentMetric, +) + + +ALIGNSCORE_CHECKPOINT_PATH = os.getenv( + "ALIGNSCORE_CHECKPOINT_PATH", + # Going up 3 levels from metrics.py: src/atgen/metrics -> repository root + os.path.join( + Path(__file__).parents[3], + "external_metrics/AlignScore/model/AlignScore-base.ckpt", + ), +) + + +def decode(eval_preds, tokenizer): + predictions, labels, *inputs = eval_preds + predictions = np.where(predictions != -100, predictions, tokenizer.pad_token_id) + decoded_preds = tokenizer.batch_decode(predictions, skip_special_tokens=True) + # Replace -100 in the labels as we can't decode them. + labels = np.where(labels != -100, labels, tokenizer.pad_token_id) + decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True) + + decoded_preds = [pred.strip() for pred in decoded_preds] + decoded_labels = [label.strip() for label in decoded_labels] + + if len(inputs) > 0: + input_ids = inputs[0] + input_ids = np.where(input_ids != -100, input_ids, tokenizer.pad_token_id) + decoded_texts = tokenizer.batch_decode(input_ids, skip_special_tokens=True) + decoded_texts = [text.strip() for text in decoded_texts] + return decoded_preds, decoded_labels, decoded_texts + + return decoded_preds, decoded_labels + + +def smoothing_function(p_n, references, hypothesis, hyp_len): + """ + Smooth-BLEU (BLEUS) as proposed in the paper: + Chin-Yew Lin, Franz Josef Och. ORANGE: a method for evaluating automatic + evaluation metrics for machine translation. COLING 2004. + """ + smoothed_p_n = [] + for i, p_i in enumerate(p_n, start=1): + # Smoothing is not applied for unigrams + if i > 1: + # If hypothesis length is lower than the current order, its value equals (0 + 1) / (0 + 1) = 0 + if hyp_len < i: + assert p_i.denominator == 1 + smoothed_p_n.append(1) + # Otherwise apply smoothing + else: + smoothed_p_i = (p_i.numerator + 1) / (p_i.denominator + 1) + smoothed_p_n.append(smoothed_p_i) + else: + smoothed_p_n.append(p_i) + return smoothed_p_n + + +def pair_bleu(references: list[str] | str, prediction: str): + """ + Compute the bleu score between two given texts. + A smoothing function is used to avoid zero scores when + there are no common higher order n-grams between the + texts. + """ + if isinstance(references, str): + tok_ref = [[word_tokenize(references)]] + else: + tok_ref = [[word_tokenize(ref) for ref in references]] + tok_pred = [word_tokenize(prediction)] + try: + return corpus_bleu(tok_ref, tok_pred, smoothing_function=smoothing_function) + except (KeyError, ZeroDivisionError): + return 0.0 + + +def calculate_bart_score( + preds, + refs=None, + texts=None, + scorer=None, + batch_size=4, + aggregate=True, + cache_dir: str = "cache", +): + if not is_bart_score_available: + return None + if scorer is None: + scorer = BARTScorer(cache_dir=cache_dir) + scores = {} + if texts is not None: + scores["BARTScore-sh"] = np.array( + scorer.score(texts, preds, batch_size=batch_size) + ) + if refs is not None: + # scores["BARTScore-rh"] = np.array(scorer.score(refs, preds, batch_size=batch_size)) + if isinstance(refs[0], list): + scores_hr = [] + for ref, pred in zip(refs, preds): + inst_pred = [pred for _ in range(len(ref))] + # Take a maximum within the observation similar to ROUGE + inst_score_hr = max(scorer.score(inst_pred, ref, batch_size=batch_size)) + scores_hr.append(inst_score_hr) + scores["BARTScore-hr"] = np.array(scores_hr) + else: + scores["BARTScore-hr"] = np.array( + scorer.score(preds, refs, batch_size=batch_size) + ) + # scores["BARTScore-fa"] = (scores["BARTScore-rh"] + scores["BARTScore-hr"]) / 2 + + if aggregate: + scores = {key: np.mean(value) for key, value in scores.items()} + return scores + + +def calculate_abstractiveness_scores( + predictions, texts, references=None, aggregate: bool = True +): + stemmer = porter.PorterStemmer() + tokenized_preds = [tokenize.tokenize(x, stemmer) for x in predictions] + tokenized_texts = [tokenize.tokenize(x, stemmer) for x in texts] + if references is not None: + tokenized_refs = [tokenize.tokenize(x, stemmer) for x in references] + else: + tokenized_refs = tokenized_preds + + result = {} + for use_modified in [False, True]: + for n in range(1, 5): + pred_ngram_overlaps = [] + label_ngram_overlaps = [] + for pred, label, text in zip( + tokenized_preds, tokenized_refs, tokenized_texts + ): + pred_pair_ngram_overlap = calculate_ngram_overlap( + pred, text, n, use_modified + ) + pred_ngram_overlaps.append(pred_pair_ngram_overlap) + if references is not None: + label_pair_ngram_overlap = calculate_ngram_overlap( + label, text, n, use_modified + ) + label_ngram_overlaps.append(label_pair_ngram_overlap) + key = f"ngram_overlap_{n}" if use_modified else f"novel_ngrams_{n}" + + pred_ngram_overlaps = np.array(pred_ngram_overlaps) + cond_abs = ~np.isnan(pred_ngram_overlaps) + result[key + "_abs"] = pred_ngram_overlaps[cond_abs] + + if references is not None: + label_ngram_overlaps = np.array(label_ngram_overlaps) + cond_rel = cond_abs & ~np.isnan(label_ngram_overlaps) + result[key + "_rel"] = ( + pred_ngram_overlaps[cond_rel] / label_ngram_overlaps[cond_rel] + ) + + if aggregate: + for key, value in result.items(): + result[key] = np.mean(value) + + return result + + +def calculate_ngram_overlap(summary, text, n=1, use_modified=True): + summary_ngrams = list(ngrams(summary, n)) + text_ngrams = list(ngrams(text, n)) + + if len(summary_ngrams) > 0: + ngrams_intersection = set(summary_ngrams).intersection(set(text_ngrams)) + if use_modified: + word_is_part_of_ngram_copied = [ + any((x in ngram for ngram in ngrams_intersection)) for x in summary + ] + return 1 - sum(word_is_part_of_ngram_copied) / len( + word_is_part_of_ngram_copied + ) + else: + return sum([x not in ngrams_intersection for x in summary_ngrams]) / len( + summary_ngrams + ) + return np.nan + + +class SentBert: + def __init__( + self, + checkpoint: str = "sentence-transformers/all-mpnet-base-v2", + device: str = "cuda", + cache_dir: str = "cache", + ): + self.tokenizer = AutoTokenizer.from_pretrained(checkpoint, cache_dir=cache_dir) + self.model = AutoModel.from_pretrained(checkpoint, cache_dir=cache_dir).to( + device + ) + self.device = device + + def __call__( + self, source_texts: list[str], ref_texts: list[str], batch_size: int = 32 + ) -> np.ndarray: + assert len(source_texts) == len(ref_texts) + # Make batch_size an even number + if batch_size % 2 == 0: + batch_size -= 1 + half_batch_size = batch_size // 2 + n_texts = len(source_texts) + scores = np.empty(n_texts, dtype=np.float32) + start = 0 + end = 0 + + while end < n_texts: + end += half_batch_size + batch_idx = slice(start, end) + # Tokenize sentences + encoded_input = self.tokenizer( + source_texts[batch_idx] + ref_texts[batch_idx], + padding=True, + truncation=True, + return_tensors="pt", + ) + encoded_input = { + key: value.to(self.device) for key, value in encoded_input.items() + } + # Calculate the probability of belonging to the positive class + model_output = self.model(**encoded_input) + # Perform pooling + sent_embs = self.mean_pooling(model_output, encoded_input["attention_mask"]) + # Normalize embeddings + sent_embs = F.normalize(sent_embs, p=2, dim=1) + n_source_embs = len(sent_embs) // 2 + scores[batch_idx] = ( + (sent_embs[:n_source_embs] * sent_embs[n_source_embs:]) + .sum(-1) + .cpu() + .detach() + .numpy() + ) + start = end + + return scores + + @staticmethod + def mean_pooling(model_output, attention_mask): + """ + Mean Pooling - Take attention mask into account for correct averaging + """ + token_embeddings = model_output[ + 0 + ] # First element of model_output contains all token embeddings + input_mask_expanded = ( + attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float() + ) + return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp( + input_mask_expanded.sum(1), min=1e-9 + ) + + +def calculate_alignscore( + predictions: list[str], + references: Union[list[str], list[list[str]]], + original_texts: list[str], + batch_size: int = 32, + device: str = "cuda", + cache_dir: str = "cache", +): + if not is_alignscore_available: + return None + if isinstance(references[0], list): + log.error("AlignScore does not support multiple references. Skipping...") + return None + if not os.path.exists(ALIGNSCORE_CHECKPOINT_PATH): + urlretrieve( + "https://huggingface.co/yzha/AlignScore/resolve/main/AlignScore-base.ckpt", + ALIGNSCORE_CHECKPOINT_PATH, + ) + + scorer = AlignScore( + model="roberta-base", + batch_size=batch_size, + device=device, + ckpt_path=ALIGNSCORE_CHECKPOINT_PATH, + evaluation_mode="nli_sp", + ) + # Fix: alignscore outputs an error if a text is empty, so we need to add some content to such texts + original_texts = [text if text else " " for text in original_texts] + predictions = [text if text else " " for text in predictions] + references = [text if text else " " for text in references] + + scores_ref = scorer.score(contexts=original_texts, claims=predictions) + if isinstance(references[0], list): + scores_baseline = [] + for orig_text, refs in zip(original_texts, references): + inst_baseline_scores = scorer.score(contexts=[orig_text] * len(refs), claims=refs) + scores_baseline.append(max(inst_baseline_scores)) + else: + scores_baseline = scorer.score(contexts=original_texts, claims=references) + scores_rel = np.array(scores_ref) / np.array(scores_baseline) + return {"alignscore": scores_ref, "alignscore_rel": scores_rel} + + +class EvaluationLLM(DeepEvalBaseLLM): + """ + Custom Evaluation LLM implementation for DeepEval. + + This class implements the DeepEvalBaseLLM interface to allow using + custom models with DeepEval metrics. + """ + + def __init__( + self, + api_key=None, + model="openai/gpt-4o-2024-11-20", + base_url="https://openrouter.ai/api/v1", + ): + """ + Initialize the Evaluation LLM. + + Args: + api_key: Evaluation API key + model: Model identifier (e.g., "openai/gpt-4o-2024-11-20") + base_url: Evaluation API base URL + """ + self.api_key = api_key + + self.model_name = model + self.base_url = base_url + self.client = None + self.async_client = None + self.OpenAI = OpenAI + self.AsyncOpenAI = AsyncOpenAI + + def load_model(self): + """Load and return the client.""" + if self.client is None: + self.client = self.OpenAI( + base_url=self.base_url, + api_key=self.api_key, + ) + return self.client + + def load_async_model(self): + """Load and return the async client.""" + if self.async_client is None: + self.async_client = self.AsyncOpenAI( + base_url=self.base_url, + api_key=self.api_key, + ) + return self.async_client + + def generate(self, prompt: str) -> str: + """ + Generate a response from the evaluation model. + + Args: + prompt: The prompt to send to the model + + Returns: + The model's response as a string + """ + client = self.load_model() + response = client.chat.completions.create( + model=self.model_name, + messages=[{"role": "user", "content": prompt}], + ) + return response.choices[0].message.content + + async def a_generate(self, prompt: str) -> str: + """ + Asynchronously generate a response from the evaluation model. + + Args: + prompt: The prompt to send to the model + + Returns: + The model's response as a string + """ + # Use the async client for async operations + client = self.load_async_model() + response = await client.chat.completions.create( + model=self.model_name, + messages=[{"role": "user", "content": prompt}], + ) + return response.choices[0].message.content + + def get_model_name(self): + """Return the name of the model.""" + return f"EvaluationLLM: {self.model_name}" + + +def calculate_deepeval_metrics( + predictions, + references, + original_texts, + metrics_to_calculate=None, + base_url: str = "https://openrouter.ai/api/v1", + api_key: str = None, + model="openai/gpt-4o-2024-11-20", + threshold=0.5, + include_reason=False, + strict_mode=False, + async_mode=True, + verbose_mode=False, + truths_extraction_limit=None, +): + """ + Calculate DeepEval metrics using EvaluationLLM. + + Args: + predictions: list of generated texts + references: list of reference texts + original_texts: list of source texts + metrics_to_calculate: list of metrics to calculate. Options: + ["deepeval_answer_relevance", "deepeval_faithfulness", "deepeval_summarization", "deepeval_prompt_alignment"] + api_key: Evaluation API key + base_url: Evaluation API base URL + model: Evaluation model to use + threshold: Threshold for metrics (default: 0.5) + include_reason: Include reason for evaluation score (default: False) + strict_mode: Enforce binary metric score (1 for perfection, 0 otherwise) (default: False) + async_mode: Enable concurrent execution (default: True) + verbose_mode: Print intermediate steps (default: False) + truths_extraction_limit: Maximum number of factual truths to extract (default: None) + + Returns: + dictionary with metric scores + """ + + if not metrics_to_calculate: + metrics_to_calculate = [ + "deepeval_answer_relevance", + "deepeval_faithfulness", + "deepeval_summarization", + "deepeval_prompt_alignment", + ] + + # Create EvaluationLLM instance + llm = EvaluationLLM( + base_url=base_url, + api_key=api_key, + model=model, + ) + + results = {} + + # Create metrics based on selected options + metrics = [] + metric_name_mapping = {} # Maps metric class name to the deepeval metric name + + # Dictionary to store test cases for each metric + metric_test_cases = {} + + if "deepeval_answer_relevance" in metrics_to_calculate: + metric = AnswerRelevancyMetric( + threshold=threshold, + model=llm, + include_reason=include_reason, + strict_mode=strict_mode, + async_mode=async_mode, + ) + metrics.append(metric) + metric_name_mapping[metric.__class__.__name__] = "deepeval_answer_relevance" + + # Create specific test cases for AnswerRelevancy metric + answer_relevance_test_cases = [] + for i, (pred, src) in enumerate(zip(predictions, original_texts)): + test_case = LLMTestCase( + input=src, + actual_output=pred, + ) + answer_relevance_test_cases.append(test_case) + metric_test_cases[metric.__class__.__name__] = answer_relevance_test_cases + + if "deepeval_faithfulness" in metrics_to_calculate: + metric = FaithfulnessMetric( + threshold=threshold, + model=llm, + include_reason=include_reason, + strict_mode=strict_mode, + async_mode=async_mode, + truths_extraction_limit=truths_extraction_limit, + ) + metrics.append(metric) + metric_name_mapping[metric.__class__.__name__] = "deepeval_faithfulness" + + # Create specific test cases for Faithfulness metric + faithfulness_test_cases = [] + for i, (pred, src) in enumerate(zip(predictions, original_texts)): + test_case = LLMTestCase( + input=src, + actual_output=pred, + retrieval_context=[src], + ) + faithfulness_test_cases.append(test_case) + metric_test_cases[metric.__class__.__name__] = faithfulness_test_cases + + if "deepeval_summarization" in metrics_to_calculate: + metric = SummarizationMetric( + threshold=threshold, + model=llm, + include_reason=include_reason, + strict_mode=strict_mode, + async_mode=async_mode, + ) + metrics.append(metric) + metric_name_mapping[metric.__class__.__name__] = "deepeval_summarization" + + # Create specific test cases for Summarization metric + summarization_test_cases = [] + for i, (pred, src) in enumerate(zip(predictions, original_texts)): + test_case = LLMTestCase( + input=src, + actual_output=pred, + ) + summarization_test_cases.append(test_case) + metric_test_cases[metric.__class__.__name__] = summarization_test_cases + + if "deepeval_prompt_alignment" in metrics_to_calculate: + metric = PromptAlignmentMetric( + threshold=threshold, + model=llm, + prompt_instructions=["Do what you are told to do in the prompt"], + include_reason=include_reason, + strict_mode=strict_mode, + async_mode=async_mode, + ) + metrics.append(metric) + metric_name_mapping[metric.__class__.__name__] = "deepeval_prompt_alignment" + + # Create specific test cases for PromptAlignment metric + prompt_alignment_test_cases = [] + for i, (pred, ref, src) in enumerate( + zip(predictions, references, original_texts) + ): + test_case = LLMTestCase( + input=src, + actual_output=pred, + expected_output=ref, + ) + prompt_alignment_test_cases.append(test_case) + metric_test_cases[metric.__class__.__name__] = prompt_alignment_test_cases + + # Run evaluation for each metric separately + for metric in metrics: + metric_class_name = metric.__class__.__name__ + test_cases = metric_test_cases.get(metric_class_name, []) + + if test_cases: + # Disable printing to console during evaluation if not verbose + original_stdout = sys.stdout + if not verbose_mode: + sys.stdout = open(os.devnull, "w") + + try: + # Run evaluation for this specific metric + evaluation_results = evaluate( + test_cases=test_cases, + metrics=[metric], + run_async=async_mode, + ) + + deepeval_metric_name = metric_name_mapping.get(metric_class_name) + scores = [] + reasons = [] + + # Process results for this metric + for result in evaluation_results.test_results: + scores.append(1 if result.success else 0) + + # Calculate average score + if scores: + results[deepeval_metric_name] = np.mean(scores) + + finally: + # Restore stdout + if not verbose_mode: + sys.stdout.close() + sys.stdout = original_stdout + print("================================================") + print("Results:") + print(results) + print("================================================") + + return results diff --git a/src/atgen/run_scripts/run_active_learning.py b/src/atgen/run_scripts/run_active_learning.py index b009409..0752b82 100644 --- a/src/atgen/run_scripts/run_active_learning.py +++ b/src/atgen/run_scripts/run_active_learning.py @@ -9,7 +9,13 @@ from typing import Union import logging from atgen.utils.main_decorator import main_decorator -from atgen.utils.constants import DEFAULT_CONFIG_NAME, UNLABELED_DATA_SPLIT_DEFAULT_NAME, TEST_DATA_SPLIT_DEFAULT_NAME +from atgen.utils.constants import ( + DEFAULT_CONFIG_NAME, + UNLABELED_DATA_SPLIT_DEFAULT_NAME, + TEST_DATA_SPLIT_DEFAULT_NAME, + OUTPUT_FIELD_PURPOSE_TRAIN, + OUTPUT_FIELD_PURPOSE_TEST, +) log = logging.getLogger() @@ -20,9 +26,12 @@ def run_active_learning(config, workdir: Union[str, Path]): from datasets import concatenate_datasets from atgen.metrics.compute_metrics import compute_metrics - from atgen.utils.data.load_data import load_data - from atgen.utils.data.prepare_conversational_data import prepare_conversational_data - from atgen.utils.data.maybe_get_few_shot_examples import maybe_get_few_shot_examples + from atgen.utils.data import ( + load_data, + prepare_conversational_data, + maybe_get_few_shot_examples, + get_output_column_name, + ) from atgen.utils.load_model_tokenizer import load_model_tokenizer from atgen.utils.prepare_model_for_training import prepare_model_for_training from atgen.utils.training_utils import get_trainer @@ -43,8 +52,9 @@ def run_active_learning(config, workdir: Union[str, Path]): seed = config.seed cache_dir = config.cache_dir input_column_name = config.data.input_column_name - output_column_name = config.data.output_column_name dev_split_size = config.training.dev_split_size + output_column_name_train = get_output_column_name(config.data.output_column_name, purpose=OUTPUT_FIELD_PURPOSE_TRAIN) + output_column_name_test = get_output_column_name(config.data.output_column_name, purpose=OUTPUT_FIELD_PURPOSE_TEST) model_name = config.model.checkpoint @@ -124,7 +134,7 @@ def run_active_learning(config, workdir: Union[str, Path]): # TODO: unsure whether need to log here since may be confusing for a human labeller labeller: BaseLabeler = get_labeller( config.labeller, - output_column_name, + output_column_name=output_column_name_train, cache_dir=cache_dir, budget=budget, workdir=workdir, # if labeller is a human @@ -153,7 +163,7 @@ def run_active_learning(config, workdir: Union[str, Path]): query_ids = al_strategy( model=model, tokenizer=tokenizer, - unlabeled_pool=unlabeled_data.remove_columns(output_column_name), + unlabeled_pool=unlabeled_data.remove_columns(output_column_name_train), labeled_pool=None, num_to_label=al_query_size, batch_size=config.inference.batch_size, @@ -217,7 +227,7 @@ def run_active_learning(config, workdir: Union[str, Path]): metrics = compute_metrics( generated_texts=generations, - reference_texts=test_data[output_column_name], + reference_texts=test_data[output_column_name_test], original_texts=test_data[input_column_name], config=config.evaluation, cache_dir=cache_dir, @@ -328,7 +338,7 @@ def run_active_learning(config, workdir: Union[str, Path]): metrics = compute_metrics( generated_texts=generations, - reference_texts=test_data[output_column_name], + reference_texts=test_data[output_column_name_test], original_texts=test_data[input_column_name], config=config.evaluation, cache_dir=cache_dir, @@ -360,7 +370,7 @@ def run_active_learning(config, workdir: Union[str, Path]): query_ids = al_strategy( model=model, tokenizer=tokenizer, - unlabeled_pool=unlabeled_data.remove_columns(output_column_name), + unlabeled_pool=unlabeled_data.remove_columns(output_column_name_train), labeled_pool=labeled_data, num_to_label=al_query_size, batch_size=config.inference.batch_size, diff --git a/src/atgen/utils/constants.py b/src/atgen/utils/constants.py index a10fadc..95fd7e9 100644 --- a/src/atgen/utils/constants.py +++ b/src/atgen/utils/constants.py @@ -10,3 +10,6 @@ APPROX_NUM_SYSTEM_TOKENS_PER_MESSAGE = 10 DEFAULT_CONFIG_NAME = "base" + +OUTPUT_FIELD_PURPOSE_TRAIN = "train" +OUTPUT_FIELD_PURPOSE_TEST = "test" diff --git a/src/atgen/utils/data/__init__.py b/src/atgen/utils/data/__init__.py index 454678c..6b440b0 100644 --- a/src/atgen/utils/data/__init__.py +++ b/src/atgen/utils/data/__init__.py @@ -1,3 +1,5 @@ +from .get_output_column_name import get_output_column_name +from .get_preprocess_function import get_preprocess_function from .prepare_conversational_data import prepare_conversational_data from .load_data import load_data from .maybe_get_few_shot_examples import maybe_get_few_shot_examples diff --git a/src/atgen/utils/data/get_output_column_name.py b/src/atgen/utils/data/get_output_column_name.py new file mode 100644 index 0000000..c3947f1 --- /dev/null +++ b/src/atgen/utils/data/get_output_column_name.py @@ -0,0 +1,19 @@ +from typing import Union +from omegaconf import DictConfig, ListConfig + +from ..constants import OUTPUT_FIELD_PURPOSE_TRAIN + + +def get_output_column_name( + output_column_name: Union[DictConfig, ListConfig, dict[str, Union[str, list[str]]], list[str], str], + purpose: str = OUTPUT_FIELD_PURPOSE_TRAIN, +) -> str: + if isinstance(output_column_name, (list, ListConfig)): + return '_'.join(output_column_name) + elif isinstance(output_column_name, (dict, DictConfig)): + return '_'.join(output_column_name[purpose]) + elif isinstance(output_column_name, str): + return output_column_name + else: + raise NotImplementedError(f"Unexpected type {type(output_column_name)} of the output column name.") + \ No newline at end of file diff --git a/src/atgen/utils/data/get_output_field.py b/src/atgen/utils/data/get_output_field.py deleted file mode 100644 index ecb06eb..0000000 --- a/src/atgen/utils/data/get_output_field.py +++ /dev/null @@ -1,11 +0,0 @@ -from typing import Union - -from datasets import Dataset - - -def _get_output_field(data: Union[Dataset, dict], output_field: list[str]) -> str: - """Access nested dictionary using a tuple path.""" - result = data - for key in output_field: - result = result[key] - return result \ No newline at end of file diff --git a/src/atgen/utils/data/get_preprocess_function.py b/src/atgen/utils/data/get_preprocess_function.py index 5955336..6ae2182 100644 --- a/src/atgen/utils/data/get_preprocess_function.py +++ b/src/atgen/utils/data/get_preprocess_function.py @@ -1,8 +1,6 @@ from typing import Optional -from omegaconf import DictConfig - -from ..constants import MESSAGES_COLUMN_NAME +from ..constants import MESSAGES_COLUMN_NAME, OUTPUT_FIELD_PURPOSE_TRAIN def get_preprocess_function( @@ -92,9 +90,7 @@ def preprocess_fn(instance): elif split == "train": # For training, add the assistant's response if assistant_response_start: - assistant_message = ( - assistant_response_start + instance[output_column_name] - ) + assistant_message = assistant_response_start + instance[output_column_name] else: assistant_message = instance[output_column_name] messages.append({"role": "assistant", "content": assistant_message}) @@ -135,9 +131,7 @@ def preprocess_fn(instance): # For training, add the assistant's response if split == "train": if assistant_response_start: - assistant_message = ( - assistant_response_start + instance[output_column_name] - ) + assistant_message = assistant_response_start + instance[output_column_name] else: assistant_message = instance[output_column_name] messages.append({"role": "assistant", "content": assistant_message}) diff --git a/src/atgen/utils/data/load_data.py b/src/atgen/utils/data/load_data.py index b3201be..f811a1d 100644 --- a/src/atgen/utils/data/load_data.py +++ b/src/atgen/utils/data/load_data.py @@ -4,6 +4,8 @@ from datasets import load_dataset, load_from_disk, Dataset, DatasetDict from omegaconf import DictConfig, ListConfig +from .get_output_column_name import get_output_column_name + def _fetch_dataset( dataset_name_or_path: Union[str, list[str]], @@ -57,6 +59,24 @@ def _take_subset(dataset_subset: Dataset, size: int, seed: int) -> Dataset: ) return dataset_subset +def _preprocess_multicolumn_labels(dataset: Dataset, output_column_names: Union[DictConfig, ListConfig, dict[str, Union[str, list[str]]], list[str], str]) -> Dataset: + if isinstance(output_column_names, (list, ListConfig)): + new_column_name = get_output_column_name(output_column_names) + values = [] + for inst in dataset: + for col_name in output_column_names: + inst = inst[col_name] + values.append(inst) + dataset = dataset.add_column(new_column_name, values) + elif isinstance(output_column_names, (dict, DictConfig)): + for _, column_name in output_column_names.items(): + dataset = _preprocess_multicolumn_labels(dataset, column_name) + # Nothing to preprocess in this case + elif isinstance(output_column_names, str): + pass + else: + raise NotImplementedError(f"Unexpected type {type(output_column_names)} of the output column names.") + return dataset def load_data( data_config: DictConfig, @@ -75,9 +95,13 @@ def load_data( f"Unexpected split {split}; Please specify either `train` or `test`." ) dataset = _fetch_dataset( - data_config.dataset, - subset_name, - dict(data_config.fetch_kwargs, cache_dir=cache_dir), + dataset_name_or_path=data_config.dataset, + subset_name=subset_name, + fetch_kwargs=dict(data_config.fetch_kwargs, cache_dir=cache_dir), + ) + dataset = _preprocess_multicolumn_labels( + dataset=dataset, + output_column_names=data_config.output_column_name ) # Add `id` column to the dataset (practical use) or to train subset (benchmarking) diff --git a/src/atgen/utils/data/prepare_conversational_data.py b/src/atgen/utils/data/prepare_conversational_data.py index eb82713..9ab0e52 100644 --- a/src/atgen/utils/data/prepare_conversational_data.py +++ b/src/atgen/utils/data/prepare_conversational_data.py @@ -1,9 +1,9 @@ from itertools import chain from datasets import Dataset from omegaconf import DictConfig -from transformers import PreTrainedTokenizer from .get_preprocess_function import get_preprocess_function +from .get_output_column_name import get_output_column_name def prepare_conversational_data( @@ -14,8 +14,8 @@ def prepare_conversational_data( model_name: str = "kek", ) -> Dataset: input_column_name = data_config.input_column_name - output_column_name = data_config.output_column_name - + output_column_name = get_output_column_name(output_column_name=data_config.output_column_name, purpose=split) + if not few_shot_examples: few_shot_messages = [] else: @@ -29,8 +29,6 @@ def prepare_conversational_data( for (fs_input, fs_output) in zip( few_shot_examples[input_column_name], few_shot_examples[output_column_name] - if isinstance(output_column_name, str) - else few_shot_examples[output_column_name][0], ) ] ) @@ -46,6 +44,15 @@ def prepare_conversational_data( output_column_name=output_column_name, assistant_response_start=data_config.assistant_response_start, ) + try: + preprocess_fn(dataset[0]) + except Exception as e: + print(e) + import pdb + import sys + + exc_type, exc_value, exc_traceback = sys.exc_info() + pdb.post_mortem(exc_traceback) dataset = dataset.map( preprocess_fn, batched=False, From 3e01b9bc0dded0403a3fafa726ef12171558ad4a Mon Sep 17 00:00:00 2001 From: Akim Tsvigun Date: Mon, 9 Jun 2025 14:42:38 +0000 Subject: [PATCH 10/39] code blacked --- configs/data/trivia_qa.yaml | 2 +- pages/0_Configure_Experiment.py | 150 ++++++++++-------- src/atgen/labellers/golden_labeller.py | 22 ++- src/atgen/metrics/compute_metrics.py | 9 +- src/atgen/metrics/metrics.py | 4 +- src/atgen/run_scripts/run_active_learning.py | 8 +- .../utils/data/get_output_column_name.py | 13 +- .../utils/data/get_preprocess_function.py | 8 +- src/atgen/utils/data/load_data.py | 16 +- .../utils/data/prepare_conversational_data.py | 8 +- 10 files changed, 149 insertions(+), 91 deletions(-) diff --git a/configs/data/trivia_qa.yaml b/configs/data/trivia_qa.yaml index 4ee7533..27f0f58 100644 --- a/configs/data/trivia_qa.yaml +++ b/configs/data/trivia_qa.yaml @@ -9,5 +9,5 @@ input_max_length: 1024 output_max_length: 20 fetch_kwargs: {} is_in_conversational_format: false -system_prompt: "Please answer the question:\n\n" +system_prompt: "Please answer the question very laconically:\n\n" assistant_response_start: ${model.assistant_response_start} diff --git a/pages/0_Configure_Experiment.py b/pages/0_Configure_Experiment.py index 3a65e4d..8971159 100644 --- a/pages/0_Configure_Experiment.py +++ b/pages/0_Configure_Experiment.py @@ -44,6 +44,7 @@ DATA_SOURCE_UPLOAD = "Upload file" DATA_SOURCE_HUGGINGFACE = "Huggingface dataset" + # Create a cached resource for experiment status tracking @st.cache_resource def get_experiment_status_tracker(): @@ -54,9 +55,10 @@ def get_experiment_status_tracker(): "experiment_name": None, "experiment_dir": None, "current_iteration": 0, - "total_iterations": 0 + "total_iterations": 0, } + # Apply custom CSS st.markdown( """ @@ -237,12 +239,15 @@ def check_experiment_status(): return status_tracker return None -def update_experiment_status(status, experiment_name=None, experiment_dir=None, total_iterations=None): + +def update_experiment_status( + status, experiment_name=None, experiment_dir=None, total_iterations=None +): """Update the experiment status with the current state.""" status_tracker = get_experiment_status_tracker() status_tracker["status"] = status status_tracker["timestamp"] = datetime.now().isoformat() - + if experiment_name: status_tracker["experiment_name"] = experiment_name if experiment_dir: @@ -252,53 +257,51 @@ def update_experiment_status(status, experiment_name=None, experiment_dir=None, return status_tracker + def create_progress_tracker(progress_container, num_iterations): """Create and return a function to update progress tracking.""" progress_bar = progress_container.progress(0) status_text = progress_container.empty() - + def update_progress(iteration): # Set initial status on first update if iteration == 0: # Update the status - update_experiment_status( - STATUS_RUNNING, - total_iterations=num_iterations - ) + update_experiment_status(STATUS_RUNNING, total_iterations=num_iterations) return - + if iteration > num_iterations: progress = 1.0 else: progress = iteration / num_iterations if num_iterations > 0 else 1.0 - + progress_bar.progress(progress) status_text.text(f"Active Learning Iteration: {iteration}/{num_iterations}") - + # Update the status - update_experiment_status( - STATUS_RUNNING, - total_iterations=num_iterations - ) - + update_experiment_status(STATUS_RUNNING, total_iterations=num_iterations) + return update_progress -def process_uploaded_datasets(train_dataset_path, test_dataset_path, output_dir, config, status): + +def process_uploaded_datasets( + train_dataset_path, test_dataset_path, output_dir, config, status +): """Process uploaded datasets and update config. - + Args: train_dataset_path: Path to the uploaded training dataset test_dataset_path: Path to the uploaded test dataset output_dir: Directory to save the processed dataset config: Configuration dictionary to update status: Streamlit status element for progress updates - + Returns: Updated configuration dictionary """ # Both files uploaded successfully status.info("Processing uploaded datasets...") - + # Load train dataset if train_dataset_path.endswith(".csv"): train_dataset = Dataset.from_csv(train_dataset_path) @@ -311,10 +314,8 @@ def process_uploaded_datasets(train_dataset_path, test_dataset_path, output_dir, # Add ID column if not present if "id" not in train_dataset.column_names: - train_dataset = train_dataset.add_column( - "id", list(range(len(train_dataset))) - ) - + train_dataset = train_dataset.add_column("id", list(range(len(train_dataset)))) + # Load test dataset if test_dataset_path is not None: if test_dataset_path.endswith(".csv"): @@ -326,29 +327,30 @@ def process_uploaded_datasets(train_dataset_path, test_dataset_path, output_dir, f"Unsupported file format for test dataset: {test_dataset_path}" ) if "id" not in test_dataset.column_names: - test_dataset = test_dataset.add_column( - "id", list(range(len(test_dataset))) - ) - dataset_dict = DatasetDict( - {"train": train_dataset, "test": test_dataset} - ) + test_dataset = test_dataset.add_column("id", list(range(len(test_dataset)))) + dataset_dict = DatasetDict({"train": train_dataset, "test": test_dataset}) else: - st.warning("No test dataset uploaded. Consider uploading the test dataset for better results.") + st.warning( + "No test dataset uploaded. Consider uploading the test dataset for better results." + ) test_dataset = None dataset_dict = DatasetDict({"train": train_dataset}) - + dataset_path = os.path.join(output_dir, "dataset_dict") dataset_dict.save_to_disk(dataset_path) # Set the dataset in the config config["data"]["dataset"] = dataset_path config["data"]["unlabeled_data_split_name"] = UNLABELED_DATA_SPLIT_DEFAULT_NAME - config["data"]["test_split_name"] = TEST_DATA_SPLIT_DEFAULT_NAME if test_dataset is not None else None + config["data"]["test_split_name"] = ( + TEST_DATA_SPLIT_DEFAULT_NAME if test_dataset is not None else None + ) status.success("Uploaded datasets processed successfully!") - + return config + # Wrapper for run_active_learning to track progress def run_active_learning_with_progress(config, progress_callback=None): """Run active learning with progress tracking""" @@ -358,19 +360,19 @@ def run_active_learning_with_progress(config, progress_callback=None): STATUS_RUNNING, experiment_name=config.get("experiment_name", "Active Learning Experiment"), experiment_dir=config.get("output_dir", None), - total_iterations=config["al"]["num_iterations"] + total_iterations=config["al"]["num_iterations"], ) - + # Call the progress callback immediately to set initial status if progress_callback: progress_callback(0) - + # Run the actual experiment result = run_active_learning(config) - + # Update status to completed update_experiment_status(STATUS_COMPLETED) - + return result except KeyboardInterrupt: update_experiment_status(STATUS_CANCELLED) @@ -379,6 +381,7 @@ def run_active_learning_with_progress(config, progress_callback=None): update_experiment_status(STATUS_FAILED) raise + def main(): # Display header with project logo/title st.markdown( @@ -394,32 +397,45 @@ def main(): try: experiment_dir = running_experiment["experiment_dir"] # Check for iteration directories (iter_X) and find the highest X - iteration_dirs = [d for d in os.listdir(experiment_dir) - if os.path.isdir(os.path.join(experiment_dir, d)) - and d.startswith("iter_")] - + iteration_dirs = [ + d + for d in os.listdir(experiment_dir) + if os.path.isdir(os.path.join(experiment_dir, d)) + and d.startswith("iter_") + ] + if iteration_dirs: # Extract iteration numbers from directory names - iteration_numbers = [int(d.split("_")[1]) for d in iteration_dirs if d.split("_")[1].isdigit()] + iteration_numbers = [ + int(d.split("_")[1]) + for d in iteration_dirs + if d.split("_")[1].isdigit() + ] if iteration_numbers: # Current iteration is the highest existing iter_X + 1 - running_experiment["current_iteration"] = max(iteration_numbers) + 1 + running_experiment["current_iteration"] = ( + max(iteration_numbers) + 1 + ) else: running_experiment["current_iteration"] = 0 else: running_experiment["current_iteration"] = 0 except (FileNotFoundError, PermissionError, OSError): # In case of any file system errors, keep the current value - import pdb; pdb.set_trace() + import pdb + + pdb.set_trace() st.warning( f"āš ļø An experiment '{running_experiment.get('experiment_name', 'Unknown')}' is already running! " f"Current iteration: {running_experiment.get('current_iteration', '?')}/{running_experiment.get('total_iterations', '?')}. " f"The experiment was most likely started by one of the reviewers, so kindly wait for it to finish." ) - + # Show option to force reset the status (in case of stale status) - if st.button("āš ļø Reset experiment status (Use only if you're sure no experiment is running)"): + if st.button( + "āš ļø Reset experiment status (Use only if you're sure no experiment is running)" + ): update_experiment_status(STATUS_IDLE) st.success("Status reset. Refresh the page to configure a new experiment.") st.stop() @@ -437,7 +453,7 @@ def main(): if st.button("šŸ‘©ā€šŸŽØ Annotate Examples", use_container_width=True): st.switch_page("3_Annotation") st.stop() - + st.markdown( """
@@ -614,7 +630,7 @@ def main(): """, unsafe_allow_html=True, ) - + col1, col2 = st.columns(2) with col1: provider = st.radio( @@ -1159,9 +1175,11 @@ def main(): # Check again if an experiment is already running if check_experiment_status(): - st.error("Another experiment is already running. Please wait for it to complete.") + st.error( + "Another experiment is already running. Please wait for it to complete." + ) st.stop() - + # Create a progress container outside of any spinner progress_container = st.container() with progress_container: @@ -1174,7 +1192,7 @@ def main(): """, unsafe_allow_html=True, ) - + # Add progress tracking elements progress_bar = st.progress(0) progress_status = st.empty() @@ -1235,14 +1253,16 @@ def main(): # Process uploaded datasets if available if train_dataset_path is not None: config = process_uploaded_datasets( - train_dataset_path=train_dataset_path, - test_dataset_path=test_dataset_path, - output_dir=output_dir, - config=config, - status=status + train_dataset_path=train_dataset_path, + test_dataset_path=test_dataset_path, + output_dir=output_dir, + config=config, + status=status, ) else: - st.error("No training dataset uploaded. Please upload a training dataset.") + st.error( + "No training dataset uploaded. Please upload a training dataset." + ) st.stop() else: # Using standard dataset (HuggingFace or local path) @@ -1283,7 +1303,9 @@ def main(): ] = price_input_per_example elif labeller == "custom_llm": if "model_checkpoint" in locals(): - config["labeller"]["model"]["checkpoint"] = labeller_checkpoint + config["labeller"]["model"][ + "checkpoint" + ] = labeller_checkpoint elif labeller == "api_llm": config["labeller"]["api_key"] = api_key config["labeller"]["provider"] = provider @@ -1387,8 +1409,7 @@ def main(): ) # Create progress tracking callback progress_update_callback = create_progress_tracker( - progress_container, - num_iterations + progress_container, num_iterations ) # Clear the status before running the experiment @@ -1397,13 +1418,12 @@ def main(): # Run the active learning experiment with progress tracking with st.spinner("Running active learning experiment..."): run_active_learning_with_progress( - config, - progress_callback=progress_update_callback + config, progress_callback=progress_update_callback ) # Update status to completed update_experiment_status(STATUS_COMPLETED) - + # Show success indicators after completion st.balloons() diff --git a/src/atgen/labellers/golden_labeller.py b/src/atgen/labellers/golden_labeller.py index 5e20589..cf02190 100644 --- a/src/atgen/labellers/golden_labeller.py +++ b/src/atgen/labellers/golden_labeller.py @@ -8,12 +8,16 @@ # Labels rows by using their label (should be used when evaluating strategies) class GoldenLabeler(BaseLabeler): def __call__(self, dataset: Dataset) -> Dataset: - _check_output_column_in_dataset(column_names=self.output_column_name, dataset=dataset) + _check_output_column_in_dataset( + column_names=self.output_column_name, dataset=dataset + ) return dataset - + def _check_output_column_in_dataset( - column_names: Union[DictConfig, ListConfig, dict[str, Union[str, list[str]]], list[str], str], + column_names: Union[ + DictConfig, ListConfig, dict[str, Union[str, list[str]]], list[str], str + ], dataset: Union[Dataset, dict[str, Union[str, dict[str, str]]]], ) -> None: if isinstance(column_names, str): @@ -22,12 +26,18 @@ def _check_output_column_in_dataset( ), f"Column {column_names} with labels was not found in dataset" elif isinstance(column_names, (ListConfig, list)): if isinstance(dataset, Dataset): - assert column_names[0] in dataset.column_names, f"Column {column_names[0]} with labels was not found in dataset" + assert ( + column_names[0] in dataset.column_names + ), f"Column {column_names[0]} with labels was not found in dataset" dataset = dataset[0] else: - assert column_names[0] in dataset.keys(), f"Column {column_names[0]} with labels was not found in dataset" + assert ( + column_names[0] in dataset.keys() + ), f"Column {column_names[0]} with labels was not found in dataset" if len(column_names) > 1: - _check_output_column_in_dataset(column_names=column_names[1:], dataset=dataset[column_names[0]]) + _check_output_column_in_dataset( + column_names=column_names[1:], dataset=dataset[column_names[0]] + ) elif isinstance(column_names, (DictConfig, dict)): for purpose, column_name in column_names.items(): _check_output_column_in_dataset(column_names=column_name, dataset=dataset) diff --git a/src/atgen/metrics/compute_metrics.py b/src/atgen/metrics/compute_metrics.py index cd81a62..698c5a2 100644 --- a/src/atgen/metrics/compute_metrics.py +++ b/src/atgen/metrics/compute_metrics.py @@ -293,11 +293,16 @@ def get_comprehensive_config() -> MetricsConfig: ) sacrebleu_scores.append(sacrebleu_result.pop("score")) result["sacrebleu"] = sacrebleu_scores - + time_dict["time_sacrebleu"] = time() - start_time # Lengths if isinstance(reference_texts[0], list): - ref_word_lengths = np.array([np.mean([len(text.split()) for text in ref]) for ref in reference_texts]) + ref_word_lengths = np.array( + [ + np.mean([len(text.split()) for text in ref]) + for ref in reference_texts + ] + ) else: ref_word_lengths = np.array([len(ref.split()) for ref in reference_texts]) # Avoid division by zero diff --git a/src/atgen/metrics/metrics.py b/src/atgen/metrics/metrics.py index 9659eee..a22ec76 100644 --- a/src/atgen/metrics/metrics.py +++ b/src/atgen/metrics/metrics.py @@ -344,7 +344,9 @@ def calculate_alignscore( if isinstance(references[0], list): scores_baseline = [] for orig_text, refs in zip(original_texts, references): - inst_baseline_scores = scorer.score(contexts=[orig_text] * len(refs), claims=refs) + inst_baseline_scores = scorer.score( + contexts=[orig_text] * len(refs), claims=refs + ) scores_baseline.append(max(inst_baseline_scores)) else: scores_baseline = scorer.score(contexts=original_texts, claims=references) diff --git a/src/atgen/run_scripts/run_active_learning.py b/src/atgen/run_scripts/run_active_learning.py index 0752b82..172635c 100644 --- a/src/atgen/run_scripts/run_active_learning.py +++ b/src/atgen/run_scripts/run_active_learning.py @@ -53,8 +53,12 @@ def run_active_learning(config, workdir: Union[str, Path]): cache_dir = config.cache_dir input_column_name = config.data.input_column_name dev_split_size = config.training.dev_split_size - output_column_name_train = get_output_column_name(config.data.output_column_name, purpose=OUTPUT_FIELD_PURPOSE_TRAIN) - output_column_name_test = get_output_column_name(config.data.output_column_name, purpose=OUTPUT_FIELD_PURPOSE_TEST) + output_column_name_train = get_output_column_name( + config.data.output_column_name, purpose=OUTPUT_FIELD_PURPOSE_TRAIN + ) + output_column_name_test = get_output_column_name( + config.data.output_column_name, purpose=OUTPUT_FIELD_PURPOSE_TEST + ) model_name = config.model.checkpoint diff --git a/src/atgen/utils/data/get_output_column_name.py b/src/atgen/utils/data/get_output_column_name.py index c3947f1..2c24d54 100644 --- a/src/atgen/utils/data/get_output_column_name.py +++ b/src/atgen/utils/data/get_output_column_name.py @@ -5,15 +5,18 @@ def get_output_column_name( - output_column_name: Union[DictConfig, ListConfig, dict[str, Union[str, list[str]]], list[str], str], + output_column_name: Union[ + DictConfig, ListConfig, dict[str, Union[str, list[str]]], list[str], str + ], purpose: str = OUTPUT_FIELD_PURPOSE_TRAIN, ) -> str: if isinstance(output_column_name, (list, ListConfig)): - return '_'.join(output_column_name) + return "_".join(output_column_name) elif isinstance(output_column_name, (dict, DictConfig)): - return '_'.join(output_column_name[purpose]) + return "_".join(output_column_name[purpose]) elif isinstance(output_column_name, str): return output_column_name else: - raise NotImplementedError(f"Unexpected type {type(output_column_name)} of the output column name.") - \ No newline at end of file + raise NotImplementedError( + f"Unexpected type {type(output_column_name)} of the output column name." + ) diff --git a/src/atgen/utils/data/get_preprocess_function.py b/src/atgen/utils/data/get_preprocess_function.py index 6ae2182..0110654 100644 --- a/src/atgen/utils/data/get_preprocess_function.py +++ b/src/atgen/utils/data/get_preprocess_function.py @@ -90,7 +90,9 @@ def preprocess_fn(instance): elif split == "train": # For training, add the assistant's response if assistant_response_start: - assistant_message = assistant_response_start + instance[output_column_name] + assistant_message = ( + assistant_response_start + instance[output_column_name] + ) else: assistant_message = instance[output_column_name] messages.append({"role": "assistant", "content": assistant_message}) @@ -131,7 +133,9 @@ def preprocess_fn(instance): # For training, add the assistant's response if split == "train": if assistant_response_start: - assistant_message = assistant_response_start + instance[output_column_name] + assistant_message = ( + assistant_response_start + instance[output_column_name] + ) else: assistant_message = instance[output_column_name] messages.append({"role": "assistant", "content": assistant_message}) diff --git a/src/atgen/utils/data/load_data.py b/src/atgen/utils/data/load_data.py index f811a1d..732cac3 100644 --- a/src/atgen/utils/data/load_data.py +++ b/src/atgen/utils/data/load_data.py @@ -59,7 +59,13 @@ def _take_subset(dataset_subset: Dataset, size: int, seed: int) -> Dataset: ) return dataset_subset -def _preprocess_multicolumn_labels(dataset: Dataset, output_column_names: Union[DictConfig, ListConfig, dict[str, Union[str, list[str]]], list[str], str]) -> Dataset: + +def _preprocess_multicolumn_labels( + dataset: Dataset, + output_column_names: Union[ + DictConfig, ListConfig, dict[str, Union[str, list[str]]], list[str], str + ], +) -> Dataset: if isinstance(output_column_names, (list, ListConfig)): new_column_name = get_output_column_name(output_column_names) values = [] @@ -75,9 +81,12 @@ def _preprocess_multicolumn_labels(dataset: Dataset, output_column_names: Union[ elif isinstance(output_column_names, str): pass else: - raise NotImplementedError(f"Unexpected type {type(output_column_names)} of the output column names.") + raise NotImplementedError( + f"Unexpected type {type(output_column_names)} of the output column names." + ) return dataset + def load_data( data_config: DictConfig, split: str, @@ -100,8 +109,7 @@ def load_data( fetch_kwargs=dict(data_config.fetch_kwargs, cache_dir=cache_dir), ) dataset = _preprocess_multicolumn_labels( - dataset=dataset, - output_column_names=data_config.output_column_name + dataset=dataset, output_column_names=data_config.output_column_name ) # Add `id` column to the dataset (practical use) or to train subset (benchmarking) diff --git a/src/atgen/utils/data/prepare_conversational_data.py b/src/atgen/utils/data/prepare_conversational_data.py index 9ab0e52..ede81e3 100644 --- a/src/atgen/utils/data/prepare_conversational_data.py +++ b/src/atgen/utils/data/prepare_conversational_data.py @@ -14,8 +14,10 @@ def prepare_conversational_data( model_name: str = "kek", ) -> Dataset: input_column_name = data_config.input_column_name - output_column_name = get_output_column_name(output_column_name=data_config.output_column_name, purpose=split) - + output_column_name = get_output_column_name( + output_column_name=data_config.output_column_name, purpose=split + ) + if not few_shot_examples: few_shot_messages = [] else: @@ -28,7 +30,7 @@ def prepare_conversational_data( ] for (fs_input, fs_output) in zip( few_shot_examples[input_column_name], - few_shot_examples[output_column_name] + few_shot_examples[output_column_name], ) ] ) From bcebe04ecef816df723b24bdf489f727d7cb09f1 Mon Sep 17 00:00:00 2001 From: SpeedOfMagic Date: Mon, 9 Jun 2025 21:24:31 +0000 Subject: [PATCH 11/39] fix several bugs --- requirements.txt | 2 +- .../utils/data/prepare_conversational_data.py | 23 +++++-------------- src/atgen/utils/generate.py | 2 +- 3 files changed, 8 insertions(+), 19 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3742987..11b1ce5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -27,7 +27,7 @@ spacy==3.7.5 streamlit==1.37.0 streamlit-authenticator==0.4.2 tabulate==0.9.0 -transformers==4.49.0 +transformers==4.52.4 trl==0.15.2 torchmetrics==1.4.1 unsloth==2025.3.17 diff --git a/src/atgen/utils/data/prepare_conversational_data.py b/src/atgen/utils/data/prepare_conversational_data.py index ede81e3..41c00f9 100644 --- a/src/atgen/utils/data/prepare_conversational_data.py +++ b/src/atgen/utils/data/prepare_conversational_data.py @@ -18,23 +18,12 @@ def prepare_conversational_data( output_column_name=data_config.output_column_name, purpose=split ) - if not few_shot_examples: - few_shot_messages = [] - else: - few_shot_messages = list( - chain.from_iterable( - [ - [ - {"role": "user", "content": fs_input}, - {"role": "assistant", "content": fs_output}, - ] - for (fs_input, fs_output) in zip( - few_shot_examples[input_column_name], - few_shot_examples[output_column_name], - ) - ] - ) - ) + few_shot_messages = [] + if few_shot_examples: + for (fs_input, fs_output) in zip(few_shot_examples[input_column_name], few_shot_examples[output_column_name]): + few_shot_messages.append({"role": "user", "content": fs_input}) + response_start = data_config.assistant_response_start if data_config.assistant_response_start else "" + few_shot_messages.append({"role": "assistant", "content": response_start + fs_output}) # Get appropriate preprocessing function based on all parameters preprocess_fn = get_preprocess_function( model_name=model_name, diff --git a/src/atgen/utils/generate.py b/src/atgen/utils/generate.py index 922ffc7..f7263ed 100644 --- a/src/atgen/utils/generate.py +++ b/src/atgen/utils/generate.py @@ -325,7 +325,7 @@ def generate( def tokenize_conversational_example( example: dict[str, Any], tokenizer: PreTrainedTokenizer ) -> dict[str, list[int]]: - input_ids = tokenizer.apply_chat_template(example["messages"]) + input_ids = tokenizer.apply_chat_template(example["messages"], continue_final_message=True) attention_mask = [1 for _ in range(len(input_ids))] return {"input_ids": input_ids, "attention_mask": attention_mask} From 217583f6ab0afc81e2671bbfe0eaf4471cb89347 Mon Sep 17 00:00:00 2001 From: Akim Tsvigun Date: Thu, 12 Jun 2025 09:19:37 +0000 Subject: [PATCH 12/39] bugs fixed --- src/atgen/utils/generate.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/atgen/utils/generate.py b/src/atgen/utils/generate.py index f7263ed..89dccf3 100644 --- a/src/atgen/utils/generate.py +++ b/src/atgen/utils/generate.py @@ -226,7 +226,7 @@ def generate_transformers( data = data.map( tokenize_conversational_example, batched=False, - fn_kwargs={"tokenizer": tokenizer}, + fn_kwargs={"tokenizer": tokenizer, "data_config": data_config}, ) data_collator = _get_data_collator( @@ -323,9 +323,12 @@ def generate( def tokenize_conversational_example( - example: dict[str, Any], tokenizer: PreTrainedTokenizer + example: dict[str, Any], tokenizer: PreTrainedTokenizer, data_config: DictConfig ) -> dict[str, list[int]]: - input_ids = tokenizer.apply_chat_template(example["messages"], continue_final_message=True) + if data_config.assistant_response_start: + input_ids = tokenizer.apply_chat_template(example["messages"], continue_final_message=True) + else: + input_ids = tokenizer.apply_chat_template(example["messages"], add_generation_prompt=True) attention_mask = [1 for _ in range(len(input_ids))] return {"input_ids": input_ids, "attention_mask": attention_mask} From 181af9fd0d6b14ed67a07ddfe8dc9c2c03317c62 Mon Sep 17 00:00:00 2001 From: Akim Tsvigun Date: Thu, 12 Jun 2025 11:30:46 +0000 Subject: [PATCH 13/39] bugs fixed --- src/atgen/utils/generate.py | 70 ++++++++++----------- src/atgen/utils/post_process_generations.py | 16 +++++ 2 files changed, 51 insertions(+), 35 deletions(-) create mode 100644 src/atgen/utils/post_process_generations.py diff --git a/src/atgen/utils/generate.py b/src/atgen/utils/generate.py index 89dccf3..63599e4 100644 --- a/src/atgen/utils/generate.py +++ b/src/atgen/utils/generate.py @@ -8,7 +8,7 @@ from omegaconf import DictConfig from torch.utils.data import DataLoader -from torch import bfloat16, float16, cuda, no_grad, LongTensor +from torch import bfloat16, cuda from tqdm import tqdm from transformers import PreTrainedModel, PreTrainedTokenizer from vllm import LLM @@ -21,11 +21,11 @@ DEFAULT_TEMPERATURE, DEFAULT_TOP_P, MESSAGES_COLUMN_NAME, - REASONING_END_TOKEN, ) from .training_utils import _get_data_collator from .find_response_token_ids_in_text import find_response_token_ids_in_text +from .post_process_generations import post_process_generations log = logging.getLogger() @@ -101,11 +101,11 @@ def generate_vllm( batch_generations = [x.outputs[0].text for x in out] generations += batch_generations - # Remove reasoning tokens from DeepSeek-R1 - if "deepseek-r1" in llm_runner.llm_engine.model_config.model: - for i, generation in enumerate(generations): - generations[i] = _remove_thinking_part(generation) - + generations = post_process_generations( + generations=generations, + data_config=data_config, + model_name=llm_runner.llm_engine.model_config.model + ) if delete_vllm_after_inference: del llm_runner gc.collect() @@ -247,31 +247,35 @@ def generate_transformers( FastLanguageModel.for_inference(model) generations = [] - with no_grad(): - for batch in tqdm(dataloader): - try: - out = model.generate( - batch["input_ids"].to(model.device), - attention_mask=batch["attention_mask"].to(model.device), - max_new_tokens=inference_config.max_new_tokens, - temperature=inference_config.get( - "temperature", DEFAULT_TEMPERATURE - ), - top_p=inference_config.get("top_p", DEFAULT_TOP_P), - return_dict_in_generate=True, - output_scores=True, + for batch in tqdm(dataloader): + try: + out = model.generate( + batch["input_ids"].to(model.device), + attention_mask=batch["attention_mask"].to(model.device), + max_new_tokens=inference_config.max_new_tokens, + temperature=inference_config.get( + "temperature", DEFAULT_TEMPERATURE + ), + top_p=inference_config.get("top_p", DEFAULT_TOP_P), + return_dict_in_generate=True, + output_scores=True, + ) + outputs_only = [] + for output in out.sequences: + seq_only = find_response_token_ids_in_text( + output.tolist(), data_collator.response_token_ids ) - outputs_only = [] - for output in out.sequences: - seq_only = find_response_token_ids_in_text( - output.tolist(), data_collator.response_token_ids - ) - outputs_only.append(tokenizer.decode(seq_only, True).strip()) - generations += outputs_only - except Exception as e: - print(f"Error in model.generate: {e}") - # Add empty strings for this batch to maintain alignment with input data - generations += [""] * len(batch["input_ids"]) + outputs_only.append(tokenizer.decode(seq_only, True).strip()) + generations += outputs_only + except Exception as e: + print(f"Error in model.generate: {e}") + # Add empty strings for this batch to maintain alignment with input data + generations += [""] * len(batch["input_ids"]) + generations = post_process_generations( + generations=generations, + data_config=data_config, + model_name=model.name_or_path + ) return generations @@ -331,7 +335,3 @@ def tokenize_conversational_example( input_ids = tokenizer.apply_chat_template(example["messages"], add_generation_prompt=True) attention_mask = [1 for _ in range(len(input_ids))] return {"input_ids": input_ids, "attention_mask": attention_mask} - - -def _remove_thinking_part(text: str) -> str: - return REASONING_END_TOKEN.join(text.split(REASONING_END_TOKEN)[1:]).strip() diff --git a/src/atgen/utils/post_process_generations.py b/src/atgen/utils/post_process_generations.py new file mode 100644 index 0000000..333a92b --- /dev/null +++ b/src/atgen/utils/post_process_generations.py @@ -0,0 +1,16 @@ +from typing import Optional +from omegaconf import DictConfig + +from .constants import REASONING_END_TOKEN + +def post_process_generations(generations: list[str], data_config: DictConfig, model_name: Optional[str] = None) -> list[str]: + if data_config.assistant_response_start: + return [_remove_thinking_part(generation) for generation in generations] + # Remove reasoning tokens from DeepSeek-R1 + elif model_name and "deepseek-r1" in model_name: + return [_remove_thinking_part(generation) for generation in generations] + return generations + + +def _remove_thinking_part(text: str) -> str: + return REASONING_END_TOKEN.join(text.split(REASONING_END_TOKEN)[1:]).strip() From 2d699a0d5dac93b7f723cf5800e540793e87da8d Mon Sep 17 00:00:00 2001 From: Akim Tsvigun Date: Thu, 12 Jun 2025 13:21:01 +0000 Subject: [PATCH 14/39] bugs fixed --- .../metrics/semantic/bart_score_metric.py | 2 +- src/atgen/utils/generate.py | 30 +++++++++++-------- src/atgen/utils/load_model_tokenizer.py | 4 +-- src/atgen/utils/post_process_generations.py | 17 +++++++---- test/test_benchmark.py | 2 +- 5 files changed, 32 insertions(+), 23 deletions(-) diff --git a/src/atgen/metrics/semantic/bart_score_metric.py b/src/atgen/metrics/semantic/bart_score_metric.py index f03ee7e..45ef549 100644 --- a/src/atgen/metrics/semantic/bart_score_metric.py +++ b/src/atgen/metrics/semantic/bart_score_metric.py @@ -38,7 +38,7 @@ def __init__( def score(self, srcs, tgts, batch_size=4): """Score a batch of examples""" score_list = [] - for i in tqdm(range(0, len(srcs), batch_size), desc="Calculating BARTScore..."): + for i in range(0, len(srcs), batch_size): src_list = srcs[i : i + batch_size] tgt_list = tgts[i : i + batch_size] try: diff --git a/src/atgen/utils/generate.py b/src/atgen/utils/generate.py index 63599e4..ef080c2 100644 --- a/src/atgen/utils/generate.py +++ b/src/atgen/utils/generate.py @@ -30,6 +30,10 @@ log = logging.getLogger() +VLLM_FRAMEWORK = "vllm" +SGLANG_FRAMEWORK = "sglang" +TRANSFORMERS_FRAMEWORK = "transformers" + def generate_vllm( inference_config: DictConfig, @@ -104,7 +108,8 @@ def generate_vllm( generations = post_process_generations( generations=generations, data_config=data_config, - model_name=llm_runner.llm_engine.model_config.model + model_name=llm_runner.llm_engine.model_config.model, + framework=VLLM_FRAMEWORK ) if delete_vllm_after_inference: del llm_runner @@ -113,14 +118,6 @@ def generate_vllm( return generations -def generate_sglang() -> list[str]: - """ - Function for generating with the SGLang framework. - Requires either model + tokenizer or the path to the saved model and tokenizer. - """ - pass - - def generate_sglang( inference_config: DictConfig, data: Dataset, @@ -208,6 +205,12 @@ def generate_response(s, messages): batch_generations = [result.outputs["response"] for result in batch_results] generations += batch_generations + generations = post_process_generations( + generations=generations, + data_config=data_config, + model_name=model_path, + framework=SGLANG_FRAMEWORK + ) # Clean up engine.shutdown() return generations @@ -274,7 +277,8 @@ def generate_transformers( generations = post_process_generations( generations=generations, data_config=data_config, - model_name=model.name_or_path + model_name=model.name_or_path, + framework=TRANSFORMERS_FRAMEWORK ) return generations @@ -290,7 +294,7 @@ def generate( **kwargs, ) -> list[str]: framework = inference_config.framework - if framework == "vllm": + if framework == VLLM_FRAMEWORK: return generate_vllm( inference_config=inference_config, data=data, @@ -300,7 +304,7 @@ def generate( data_config=data_config, **kwargs, ) - elif framework == "transformers": + elif framework == TRANSFORMERS_FRAMEWORK: return generate_transformers( inference_config=inference_config, data=data, @@ -311,7 +315,7 @@ def generate( model_config=model_config, **kwargs, ) - elif framework == "sglang": + elif framework == SGLANG_FRAMEWORK: return generate_sglang( inference_config=inference_config, data=data, diff --git a/src/atgen/utils/load_model_tokenizer.py b/src/atgen/utils/load_model_tokenizer.py index 2711240..f33a5ac 100644 --- a/src/atgen/utils/load_model_tokenizer.py +++ b/src/atgen/utils/load_model_tokenizer.py @@ -53,6 +53,7 @@ def load_model_tokenizer( trust_remote_code=True, ) tokenizer.model_max_length = model_config.model_max_length + tokenizer.padding_side = "left" else: kwargs = { "cache_dir": cache_dir, @@ -65,8 +66,7 @@ def load_model_tokenizer( model_config.checkpoint, model_max_length=model_config.model_max_length, cache_dir=cache_dir, - # TODO: choose automatically - # padding_side="left", + padding_side="left" ) if tokenizer.pad_token is None: diff --git a/src/atgen/utils/post_process_generations.py b/src/atgen/utils/post_process_generations.py index 333a92b..fd5425a 100644 --- a/src/atgen/utils/post_process_generations.py +++ b/src/atgen/utils/post_process_generations.py @@ -1,14 +1,19 @@ -from typing import Optional +from typing import Optional, Literal from omegaconf import DictConfig from .constants import REASONING_END_TOKEN -def post_process_generations(generations: list[str], data_config: DictConfig, model_name: Optional[str] = None) -> list[str]: - if data_config.assistant_response_start: - return [_remove_thinking_part(generation) for generation in generations] - # Remove reasoning tokens from DeepSeek-R1 +def post_process_generations( + generations: list[str], + data_config: DictConfig, + model_name: Optional[str] = None, + framework: Literal["vllm", "sglang", "transformers"] = "vllm" +) -> list[str]: + # Remove assistant response start from transformers generations + if framework == "transformers" and (ass_resp_start := data_config.assistant_response_start): + return [ass_resp_start.join(gen.split(ass_resp_start)[1:]).strip() for gen in generations] elif model_name and "deepseek-r1" in model_name: - return [_remove_thinking_part(generation) for generation in generations] + return [_remove_thinking_part(gen) for gen in generations] return generations diff --git a/test/test_benchmark.py b/test/test_benchmark.py index d5b517c..799cc81 100644 --- a/test/test_benchmark.py +++ b/test/test_benchmark.py @@ -17,7 +17,7 @@ def exec_bash(s): def test_just_works(): exec_result = exec_bash( - "HYDRA_CONFIG_NAME=test run-al +debug=false +save_model=false" + "HYDRA_CONFIG_NAME=test run-al +save_model=false" ) assert exec_result.returncode == 0 From bd810d3015580569b58059150f56879848abe8c7 Mon Sep 17 00:00:00 2001 From: Akim Tsvigun Date: Thu, 12 Jun 2025 19:05:11 +0000 Subject: [PATCH 15/39] bugs fixed --- configs/data/aeslc.yaml | 1 + configs/data/{base.yaml => example.yaml} | 1 + configs/data/gigaword.yaml | 1 + configs/data/test.yaml | 1 + configs/data/trivia_qa.yaml | 1 + configs/data/user_data.yaml | 1 + configs/data/xsum.yaml | 1 + src/atgen/metrics/compute_metrics.py | 12 ++++ src/atgen/run_scripts/run_active_learning.py | 64 +++++++++---------- src/atgen/strategies/base_strategy.py | 2 +- src/atgen/strategies/bleuvar.py | 4 +- src/atgen/strategies/dual.py | 4 +- src/atgen/strategies/graph_cut.py | 4 +- src/atgen/strategies/hadas.py | 26 ++++---- src/atgen/strategies/huds.py | 4 +- src/atgen/strategies/idds.py | 4 +- src/atgen/strategies/nsp.py | 4 +- src/atgen/strategies/random_strategy.py | 4 +- src/atgen/strategies/te_delfy.py | 4 +- src/atgen/utils/check_performance_metrics.py | 12 ++-- src/atgen/utils/data/__init__.py | 2 +- ...py => get_output_column_name_for_phase.py} | 2 +- src/atgen/utils/data/load_data.py | 30 ++++++--- .../utils/data/prepare_conversational_data.py | 13 +--- src/atgen/utils/generate.py | 1 + src/atgen/utils/resolvers.py | 7 +- src/atgen/utils/validate_and_fill_config.py | 20 +++++- 27 files changed, 135 insertions(+), 95 deletions(-) rename configs/data/{base.yaml => example.yaml} (93%) rename src/atgen/utils/data/{get_output_column_name.py => get_output_column_name_for_phase.py} (95%) diff --git a/configs/data/aeslc.yaml b/configs/data/aeslc.yaml index 87f3a27..b69281a 100644 --- a/configs/data/aeslc.yaml +++ b/configs/data/aeslc.yaml @@ -11,3 +11,4 @@ fetch_kwargs: {} is_in_conversational_format: false system_prompt: "Write a subject text for the email.\n\n" assistant_response_start: ${model.assistant_response_start} +use_test_benchmark: false diff --git a/configs/data/base.yaml b/configs/data/example.yaml similarity index 93% rename from configs/data/base.yaml rename to configs/data/example.yaml index 9b55985..7f14ee6 100644 --- a/configs/data/base.yaml +++ b/configs/data/example.yaml @@ -14,3 +14,4 @@ train_subset_size: int system_prompt: str assistant_response_start: str is_in_conversational_format: bool +use_test_benchmark: bool diff --git a/configs/data/gigaword.yaml b/configs/data/gigaword.yaml index f7ab534..bcca145 100644 --- a/configs/data/gigaword.yaml +++ b/configs/data/gigaword.yaml @@ -9,3 +9,4 @@ input_max_length: 89 output_max_length: 21 fetch_kwargs: {} system_prompt: "Generate a headline for the article in lowercase." +use_test_benchmark: false diff --git a/configs/data/test.yaml b/configs/data/test.yaml index faa4787..b87f481 100644 --- a/configs/data/test.yaml +++ b/configs/data/test.yaml @@ -13,3 +13,4 @@ test_subset_size: 3 train_subset_size: 100 is_in_conversational_format: false assistant_response_start: ${model.assistant_response_start} +use_test_benchmark: false diff --git a/configs/data/trivia_qa.yaml b/configs/data/trivia_qa.yaml index 27f0f58..aa09f15 100644 --- a/configs/data/trivia_qa.yaml +++ b/configs/data/trivia_qa.yaml @@ -11,3 +11,4 @@ fetch_kwargs: {} is_in_conversational_format: false system_prompt: "Please answer the question very laconically:\n\n" assistant_response_start: ${model.assistant_response_start} +use_test_benchmark: false diff --git a/configs/data/user_data.yaml b/configs/data/user_data.yaml index 964056f..e18be97 100644 --- a/configs/data/user_data.yaml +++ b/configs/data/user_data.yaml @@ -10,3 +10,4 @@ output_max_length: 20 system_prompt: "Write a summary for the article in lowercase." test_subset_size: 100 train_subset_size: 100 +use_test_benchmark: false diff --git a/configs/data/xsum.yaml b/configs/data/xsum.yaml index ab1f5d4..ea93a93 100644 --- a/configs/data/xsum.yaml +++ b/configs/data/xsum.yaml @@ -11,3 +11,4 @@ fetch_kwargs: trust_remote_code: true is_in_conversational_format: false system_prompt: "Summarize the article in one sentence." +use_test_benchmark: false diff --git a/src/atgen/metrics/compute_metrics.py b/src/atgen/metrics/compute_metrics.py index 698c5a2..e3682e8 100644 --- a/src/atgen/metrics/compute_metrics.py +++ b/src/atgen/metrics/compute_metrics.py @@ -1,9 +1,12 @@ from time import time <<<<<<< HEAD +<<<<<<< HEAD from typing import List, Dict, Union, Optional ======= from typing import Dict, Union >>>>>>> eaad08f (fixed version for multi-ref datasets) +======= +>>>>>>> 7fd04b7 (bugs fixed) import logging import numpy as np from omegaconf import DictConfig @@ -30,6 +33,7 @@ def compute_metrics( +<<<<<<< HEAD generated_texts: List[str], reference_texts: Optional[List[Union[str, List[str]]]] = None, original_texts: Optional[List[str]] = None, @@ -38,6 +42,14 @@ def compute_metrics( tokenizer=None, cache_dir: Optional[str] = None, ) -> Dict[str, float]: +======= + generated_texts, + reference_texts, + original_texts, + config: DictConfig, + cache_dir: str = "cache", +) -> dict[str, float]: +>>>>>>> 7fd04b7 (bugs fixed) """ Compute various metrics for generated texts using the new architecture. diff --git a/src/atgen/run_scripts/run_active_learning.py b/src/atgen/run_scripts/run_active_learning.py index 172635c..728f1ef 100644 --- a/src/atgen/run_scripts/run_active_learning.py +++ b/src/atgen/run_scripts/run_active_learning.py @@ -23,14 +23,14 @@ @main_decorator def run_active_learning(config, workdir: Union[str, Path]): from transformers import set_seed - from datasets import concatenate_datasets + from datasets import concatenate_datasets, Dataset from atgen.metrics.compute_metrics import compute_metrics from atgen.utils.data import ( load_data, prepare_conversational_data, maybe_get_few_shot_examples, - get_output_column_name, + get_output_column_name_for_phase, ) from atgen.utils.load_model_tokenizer import load_model_tokenizer from atgen.utils.prepare_model_for_training import prepare_model_for_training @@ -44,6 +44,7 @@ def run_active_learning(config, workdir: Union[str, Path]): from atgen.utils.get_initial_labeled_data import ( get_initial_labeled_data_with_few_shot, ) + from atgen.strategies.base_strategy import BaseStrategy from atgen.labellers.base_labeller import BaseLabeler from atgen.utils.check_performance_metrics import ( check_performance_against_requirements, @@ -53,12 +54,8 @@ def run_active_learning(config, workdir: Union[str, Path]): cache_dir = config.cache_dir input_column_name = config.data.input_column_name dev_split_size = config.training.dev_split_size - output_column_name_train = get_output_column_name( - config.data.output_column_name, purpose=OUTPUT_FIELD_PURPOSE_TRAIN - ) - output_column_name_test = get_output_column_name( - config.data.output_column_name, purpose=OUTPUT_FIELD_PURPOSE_TEST - ) + output_column_name_train = config.data.train_output_column_name + output_column_name_test = config.data.test_output_column_name model_name = config.model.checkpoint @@ -76,7 +73,7 @@ def run_active_learning(config, workdir: Union[str, Path]): # Initialize variables for tracking available metrics available_metrics = {} - metrics_availability_checked = False + is_metrics_availability_checked = False has_test = ( config.data.test_split_name is not None and config.data.test_split_name != "" @@ -121,7 +118,7 @@ def run_active_learning(config, workdir: Union[str, Path]): ) log.info("Loading AL strategy.") - al_strategy = get_strategy( + al_strategy: BaseStrategy = get_strategy( config.al.strategy, subsample_size=config.al.subsample_size, unlabeled_pool=unlabeled_data[input_column_name], @@ -164,7 +161,7 @@ def run_active_learning(config, workdir: Union[str, Path]): lambda x: x["id"] not in set(labeled_ids) ) else: - query_ids = al_strategy( + query_ids: list[str] = al_strategy( model=model, tokenizer=tokenizer, unlabeled_pool=unlabeled_data.remove_columns(output_column_name_train), @@ -199,7 +196,7 @@ def run_active_learning(config, workdir: Union[str, Path]): labeled_data = unlabeled_data.select(range(0, 0)) labeled_ids = [] - unlabeled_data = prepare_conversational_data( + unlabeled_data: Dataset = prepare_conversational_data( dataset=unlabeled_data, data_config=config.data, split="test", @@ -208,16 +205,17 @@ def run_active_learning(config, workdir: Union[str, Path]): ) if has_test: - test_data = prepare_conversational_data( - dataset=test_data, - data_config=config.data, - split="test", - few_shot_examples=few_shot_examples, - model_name=model_name, - ) + if not config.data.use_test_benchmark: + test_data: Dataset = prepare_conversational_data( + dataset=test_data, + data_config=config.data, + split="test", + few_shot_examples=few_shot_examples, + model_name=model_name, + ) # Evaluate the initial model before any training if init_query_size_is_positive and config.al.evaluate_zero_iteration: - generations = generate( + generations: list[str] = generate( config.inference, data=test_data, model=model, @@ -229,7 +227,7 @@ def run_active_learning(config, workdir: Union[str, Path]): if os.path.exists(save_dir): rmtree(save_dir) - metrics = compute_metrics( + metrics: dict[str, float] = compute_metrics( generated_texts=generations, reference_texts=test_data[output_column_name_test], original_texts=test_data[input_column_name], @@ -238,11 +236,11 @@ def run_active_learning(config, workdir: Union[str, Path]): ) # Check required performance metrics - is_performance_reached, metrics_availability_checked, available_metrics = ( + is_performance_reached, is_metrics_availability_checked, available_metrics = ( check_performance_against_requirements( metrics=metrics, required_performance_dict=required_performance_dict, - metrics_availability_checked=metrics_availability_checked, + is_metrics_availability_checked=is_metrics_availability_checked, available_metrics=available_metrics, ) ) @@ -298,7 +296,7 @@ def run_active_learning(config, workdir: Union[str, Path]): # Set seed for reproducibility set_seed(seed) - trainer = get_trainer( + trainer: SFTTrainer = get_trainer( config=config, model=model, tokenizer=tokenizer, @@ -328,7 +326,7 @@ def run_active_learning(config, workdir: Union[str, Path]): if dev_split_size > 0: test_data = eval_data else: - generations = generate( + generations: list[str] = generate( config.inference, data=test_data, model=model, @@ -340,7 +338,7 @@ def run_active_learning(config, workdir: Union[str, Path]): if os.path.exists(save_dir): rmtree(save_dir) - metrics = compute_metrics( + metrics: dict[str, float] = compute_metrics( generated_texts=generations, reference_texts=test_data[output_column_name_test], original_texts=test_data[input_column_name], @@ -349,11 +347,11 @@ def run_active_learning(config, workdir: Union[str, Path]): ) # Check required performance metrics - is_performance_reached, metrics_availability_checked, available_metrics = ( + is_performance_reached, is_metrics_availability_checked, available_metrics = ( check_performance_against_requirements( metrics=metrics, required_performance_dict=required_performance_dict, - metrics_availability_checked=metrics_availability_checked, + is_metrics_availability_checked=is_metrics_availability_checked, available_metrics=available_metrics, ) ) @@ -371,7 +369,7 @@ def run_active_learning(config, workdir: Union[str, Path]): # Make AL query for the next round if we have not run out of iterations if al_iter != num_al_iterations + 1: log.info(f"Making AL query at iteration {al_iter}.") - query_ids = al_strategy( + query_ids: list[str] = al_strategy( model=model, tokenizer=tokenizer, unlabeled_pool=unlabeled_data.remove_columns(output_column_name_train), @@ -381,12 +379,12 @@ def run_active_learning(config, workdir: Union[str, Path]): max_new_tokens=config.inference.max_new_tokens, ) - query = unlabeled_data.filter(lambda x: x["id"] in query_ids) - unlabeled_data = unlabeled_data.filter(lambda x: x["id"] not in query_ids) - labeled_query = labeller(query) + query: Dataset = unlabeled_data.filter(lambda x: x["id"] in query_ids) + unlabeled_data: Dataset = unlabeled_data.filter(lambda x: x["id"] not in query_ids) + labeled_query: Dataset = labeller(query) if labeller.is_out_of_budget: log.info(f"Labeler ran out of budget at iteration {al_iter}.") - labeled_data = concatenate_datasets([labeled_data, labeled_query]) + labeled_data: Dataset = concatenate_datasets([labeled_data, labeled_query]) labeled_ids += query_ids log.info(f"Saving labeled data at iteration #{al_iter}.") diff --git a/src/atgen/strategies/base_strategy.py b/src/atgen/strategies/base_strategy.py index 232fc66..eb96174 100644 --- a/src/atgen/strategies/base_strategy.py +++ b/src/atgen/strategies/base_strategy.py @@ -8,7 +8,7 @@ ) -class Strategy(ABC): +class BaseStrategy(ABC): def __init__(self, subsample_size: int | float = -1): self.subsample_size = subsample_size diff --git a/src/atgen/strategies/bleuvar.py b/src/atgen/strategies/bleuvar.py index cd6e845..e9bccb6 100644 --- a/src/atgen/strategies/bleuvar.py +++ b/src/atgen/strategies/bleuvar.py @@ -1,11 +1,11 @@ -from .base_strategy import Strategy +from .base_strategy import BaseStrategy import numpy as np from datasets import Dataset from transformers import set_seed, PreTrainedModel -class BLEUVarStrategy(Strategy): +class BLEUVarStrategy(BaseStrategy): def __init__(self): super().__init__() diff --git a/src/atgen/strategies/dual.py b/src/atgen/strategies/dual.py index a6ed8b1..0382fcf 100644 --- a/src/atgen/strategies/dual.py +++ b/src/atgen/strategies/dual.py @@ -1,7 +1,7 @@ from datasets import Dataset from transformers import PreTrainedModel -from .base_strategy import Strategy +from .base_strategy import BaseStrategy from .idds import idds_sampling from .bleuvar import bleuvar from .random_strategy import random_strategy @@ -11,7 +11,7 @@ """ -class DualStrategy(Strategy): +class DualStrategy(BaseStrategy): def __init__(self, seed=None): super().__init__() self.seed = seed diff --git a/src/atgen/strategies/graph_cut.py b/src/atgen/strategies/graph_cut.py index c6120ce..d03ebc9 100644 --- a/src/atgen/strategies/graph_cut.py +++ b/src/atgen/strategies/graph_cut.py @@ -5,7 +5,7 @@ from submodlib import GraphCutFunction -from .base_strategy import Strategy +from .base_strategy import BaseStrategy from ..utils.get_embeddings import get_embeddings """ @@ -15,7 +15,7 @@ """ -class GraphCutStrategy(Strategy): +class GraphCutStrategy(BaseStrategy): def __init__( self, unlabeled_pool: list[str], diff --git a/src/atgen/strategies/hadas.py b/src/atgen/strategies/hadas.py index f98a215..a18ca2b 100644 --- a/src/atgen/strategies/hadas.py +++ b/src/atgen/strategies/hadas.py @@ -1,8 +1,8 @@ -from typing import Optional +from typing import Optional, Union import numpy as np from evaluate import EvaluationModule, load from scipy.spatial.distance import jensenshannon -from omegaconf import DictConfig +from omegaconf import DictConfig, ListConfig import logging from datasets import Dataset @@ -18,16 +18,16 @@ ) from .unieval import SumEvaluator, convert_to_json -from .base_strategy import Strategy +from .base_strategy import BaseStrategy from ..utils.generate import generate -from ..utils.data.prepare_conversational_data import prepare_conversational_data +from ..utils.data import prepare_conversational_data, get_output_column_name_for_phase from ..utils.constants import MESSAGES_COLUMN_NAME log = logging.getLogger() -class HadasStrategy(Strategy): +class HadasStrategy(BaseStrategy): def __init__( self, subsample_size: int | float = -1, @@ -81,7 +81,7 @@ def __call__( unlabeled_pool=unlabeled_pool, labeled_pool=labeled_pool, input_column_name=self.data_config.input_column_name, - output_column_name=self.data_config.output_column_name, + output_column_name=self.data_config.train_output_column_name, num_to_label=num_to_label, inference_config=self.inference_config, model_config=self.model_config, @@ -221,13 +221,13 @@ def hadas( h_halu = (U_unlabeled @ weights).flatten().numpy() U_labeled = hallucination_distribution( - entailment_model, - entailment_tokenizer, - unieval, - bertscore, - labeled_pool[input_column_name], - labeled_pool[output_column_name], - inference_config.batch_size, + entailment_model=entailment_model, + entailment_tokenizer=entailment_tokenizer, + unieval=unieval, + bertscore=bertscore, + documents=labeled_pool[input_column_name], + summaries=labeled_pool[output_column_name], + batch_size=inference_config.batch_size, ) h_div = np.array( [ diff --git a/src/atgen/strategies/huds.py b/src/atgen/strategies/huds.py index ccf9449..473b729 100644 --- a/src/atgen/strategies/huds.py +++ b/src/atgen/strategies/huds.py @@ -13,13 +13,13 @@ PreTrainedTokenizerBase, ) -from .base_strategy import Strategy +from .base_strategy import BaseStrategy from ..utils.get_embeddings import get_embeddings from ..utils.constants import MESSAGES_COLUMN_NAME from ..utils.data.prepare_conversational_data import prepare_conversational_data -class HudsStrategy(Strategy): +class HudsStrategy(BaseStrategy): def __init__( self, unlabeled_pool: list[str], diff --git a/src/atgen/strategies/idds.py b/src/atgen/strategies/idds.py index 626cea3..2dcf987 100644 --- a/src/atgen/strategies/idds.py +++ b/src/atgen/strategies/idds.py @@ -1,4 +1,4 @@ -from .base_strategy import Strategy +from .base_strategy import BaseStrategy from datasets import Dataset from transformers import PreTrainedModel @@ -10,7 +10,7 @@ import logging -class IDDSStrategy(Strategy): +class IDDSStrategy(BaseStrategy): def __init__(self, seed=None): super().__init__() self.seed = seed diff --git a/src/atgen/strategies/nsp.py b/src/atgen/strategies/nsp.py index 3c6438b..fdcaba5 100644 --- a/src/atgen/strategies/nsp.py +++ b/src/atgen/strategies/nsp.py @@ -1,4 +1,4 @@ -from .base_strategy import Strategy +from .base_strategy import BaseStrategy from typing import Union import numpy as np @@ -6,7 +6,7 @@ from transformers import PreTrainedModel -class NSPStrategy(Strategy): +class NSPStrategy(BaseStrategy): def __init__(self): super().__init__() diff --git a/src/atgen/strategies/random_strategy.py b/src/atgen/strategies/random_strategy.py index 019a0f9..556391f 100644 --- a/src/atgen/strategies/random_strategy.py +++ b/src/atgen/strategies/random_strategy.py @@ -1,7 +1,7 @@ import numpy as np from datasets import Dataset -from .base_strategy import Strategy +from .base_strategy import BaseStrategy def random_strategy( @@ -15,7 +15,7 @@ def random_strategy( return ids[:num_to_label] -class RandomStrategy(Strategy): +class RandomStrategy(BaseStrategy): def __init__(self, subsample_size: int = -1, seed: int = 42): super().__init__() self.seed = seed diff --git a/src/atgen/strategies/te_delfy.py b/src/atgen/strategies/te_delfy.py index 2e98f77..08c1da0 100644 --- a/src/atgen/strategies/te_delfy.py +++ b/src/atgen/strategies/te_delfy.py @@ -17,13 +17,13 @@ PreTrainedTokenizerBase, ) -from .base_strategy import Strategy +from .base_strategy import BaseStrategy log = logging.getLogger() -class TeDelfyStrategy(Strategy): +class TeDelfyStrategy(BaseStrategy): def __init__(self, subsample_size: int = -1, inference_config: DictConfig = None): super().__init__(subsample_size) self.random_init = True diff --git a/src/atgen/utils/check_performance_metrics.py b/src/atgen/utils/check_performance_metrics.py index 8a8b0a8..9702cc0 100644 --- a/src/atgen/utils/check_performance_metrics.py +++ b/src/atgen/utils/check_performance_metrics.py @@ -4,7 +4,7 @@ def check_performance_against_requirements( - metrics, required_performance_dict, metrics_availability_checked, available_metrics + metrics, required_performance_dict, is_metrics_availability_checked, available_metrics ): """ Check if the computed metrics meet the required performance thresholds. @@ -12,17 +12,17 @@ def check_performance_against_requirements( Args: metrics (dict): The computed metrics from the current evaluation required_performance_dict (dict): Dictionary of required metric thresholds - metrics_availability_checked (bool): Whether metrics availability has been checked + is_metrics_availability_checked (bool): Whether metrics availability has been checked available_metrics (dict): Previously identified available metrics and their thresholds Returns: - tuple: (is_performance_reached, metrics_availability_checked, available_metrics) + tuple: (is_performance_reached, is_metrics_availability_checked, available_metrics) """ is_performance_reached = False if required_performance_dict is not None: # Only check which metrics are available on the first iteration with valid metrics - if not metrics_availability_checked: + if not is_metrics_availability_checked: # Determine which required metrics are available in computed metrics available_metrics = { metric: threshold @@ -44,7 +44,7 @@ def check_performance_against_requirements( "None of the required metrics are available. Cannot evaluate required performance." ) - metrics_availability_checked = True + is_metrics_availability_checked = True # Check if required performance is reached based on available metrics if available_metrics: @@ -75,4 +75,4 @@ def check_performance_against_requirements( else: is_performance_reached = False - return is_performance_reached, metrics_availability_checked, available_metrics + return is_performance_reached, is_metrics_availability_checked, available_metrics diff --git a/src/atgen/utils/data/__init__.py b/src/atgen/utils/data/__init__.py index 6b440b0..33c7776 100644 --- a/src/atgen/utils/data/__init__.py +++ b/src/atgen/utils/data/__init__.py @@ -1,4 +1,4 @@ -from .get_output_column_name import get_output_column_name +from .get_output_column_name_for_phase import get_output_column_name_for_phase from .get_preprocess_function import get_preprocess_function from .prepare_conversational_data import prepare_conversational_data from .load_data import load_data diff --git a/src/atgen/utils/data/get_output_column_name.py b/src/atgen/utils/data/get_output_column_name_for_phase.py similarity index 95% rename from src/atgen/utils/data/get_output_column_name.py rename to src/atgen/utils/data/get_output_column_name_for_phase.py index 2c24d54..84a34e2 100644 --- a/src/atgen/utils/data/get_output_column_name.py +++ b/src/atgen/utils/data/get_output_column_name_for_phase.py @@ -4,7 +4,7 @@ from ..constants import OUTPUT_FIELD_PURPOSE_TRAIN -def get_output_column_name( +def get_output_column_name_for_phase( output_column_name: Union[ DictConfig, ListConfig, dict[str, Union[str, list[str]]], list[str], str ], diff --git a/src/atgen/utils/data/load_data.py b/src/atgen/utils/data/load_data.py index 732cac3..dbc5a5b 100644 --- a/src/atgen/utils/data/load_data.py +++ b/src/atgen/utils/data/load_data.py @@ -4,7 +4,8 @@ from datasets import load_dataset, load_from_disk, Dataset, DatasetDict from omegaconf import DictConfig, ListConfig -from .get_output_column_name import get_output_column_name +from .get_output_column_name_for_phase import get_output_column_name_for_phase +from ..constants import OUTPUT_FIELD_PURPOSE_TRAIN, OUTPUT_FIELD_PURPOSE_TEST def _fetch_dataset( @@ -60,14 +61,23 @@ def _take_subset(dataset_subset: Dataset, size: int, seed: int) -> Dataset: return dataset_subset -def _preprocess_multicolumn_labels( +def _preprocess_multicolumn_labels_if_needed( dataset: Dataset, output_column_names: Union[ DictConfig, ListConfig, dict[str, Union[str, list[str]]], list[str], str ], + data_config: DictConfig, + phase: str = OUTPUT_FIELD_PURPOSE_TRAIN, ) -> Dataset: if isinstance(output_column_names, (list, ListConfig)): - new_column_name = get_output_column_name(output_column_names) + # Get preprocessed column name + if phase == OUTPUT_FIELD_PURPOSE_TRAIN: + new_column_name = data_config.train_output_column_name + elif phase == OUTPUT_FIELD_PURPOSE_TEST: + new_column_name = data_config.test_output_column_name + else: + raise NotImplementedError(f"Unexpected phase {phase}") + # Get preprocessed column values values = [] for inst in dataset: for col_name in output_column_names: @@ -75,8 +85,10 @@ def _preprocess_multicolumn_labels( values.append(inst) dataset = dataset.add_column(new_column_name, values) elif isinstance(output_column_names, (dict, DictConfig)): - for _, column_name in output_column_names.items(): - dataset = _preprocess_multicolumn_labels(dataset, column_name) + for phase, column_name in output_column_names.items(): + dataset = _preprocess_multicolumn_labels_if_needed( + dataset, column_name, data_config, phase + ) # Nothing to preprocess in this case elif isinstance(output_column_names, str): pass @@ -108,10 +120,12 @@ def load_data( subset_name=subset_name, fetch_kwargs=dict(data_config.fetch_kwargs, cache_dir=cache_dir), ) - dataset = _preprocess_multicolumn_labels( - dataset=dataset, output_column_names=data_config.output_column_name + dataset = _preprocess_multicolumn_labels_if_needed( + dataset=dataset, + output_column_names=data_config.output_column_name, + data_config=data_config, + phase=split, ) - # Add `id` column to the dataset (practical use) or to train subset (benchmarking) dataset = _add_id_column(dataset) if subset_size is not None: diff --git a/src/atgen/utils/data/prepare_conversational_data.py b/src/atgen/utils/data/prepare_conversational_data.py index 41c00f9..432d9c3 100644 --- a/src/atgen/utils/data/prepare_conversational_data.py +++ b/src/atgen/utils/data/prepare_conversational_data.py @@ -3,7 +3,7 @@ from omegaconf import DictConfig from .get_preprocess_function import get_preprocess_function -from .get_output_column_name import get_output_column_name +from .get_output_column_name_for_phase import get_output_column_name_for_phase def prepare_conversational_data( @@ -14,7 +14,7 @@ def prepare_conversational_data( model_name: str = "kek", ) -> Dataset: input_column_name = data_config.input_column_name - output_column_name = get_output_column_name( + output_column_name = get_output_column_name_for_phase( output_column_name=data_config.output_column_name, purpose=split ) @@ -35,15 +35,6 @@ def prepare_conversational_data( output_column_name=output_column_name, assistant_response_start=data_config.assistant_response_start, ) - try: - preprocess_fn(dataset[0]) - except Exception as e: - print(e) - import pdb - import sys - - exc_type, exc_value, exc_traceback = sys.exc_info() - pdb.post_mortem(exc_traceback) dataset = dataset.map( preprocess_fn, batched=False, diff --git a/src/atgen/utils/generate.py b/src/atgen/utils/generate.py index ef080c2..87bf617 100644 --- a/src/atgen/utils/generate.py +++ b/src/atgen/utils/generate.py @@ -105,6 +105,7 @@ def generate_vllm( batch_generations = [x.outputs[0].text for x in out] generations += batch_generations + import pdb; pdb.set_trace() generations = post_process_generations( generations=generations, data_config=data_config, diff --git a/src/atgen/utils/resolvers.py b/src/atgen/utils/resolvers.py index 891c7e1..983cfce 100644 --- a/src/atgen/utils/resolvers.py +++ b/src/atgen/utils/resolvers.py @@ -2,10 +2,9 @@ Custom Hydra resolvers for configuration calculations """ -from typing import Any +from typing import Union -from hydra.core.config_store import ConfigStore -from omegaconf import OmegaConf +from omegaconf import OmegaConf, DictConfig, ListConfig def multiply_with_few_shot(input_max_length: int, few_shot_count: int) -> int: @@ -28,13 +27,13 @@ def to_string(model_name: str): """ return model_name.replace("/", "__") - def register_resolvers() -> None: """Register all custom resolvers with OmegaConf""" # Register resolvers only if they are not already registered resolvers_to_register = { "multiply_with_few_shot": multiply_with_few_shot, "to_string": to_string, + "get_output_column_name_for_phase": get_output_column_name_for_phase, } for name, resolver_fn in resolvers_to_register.items(): diff --git a/src/atgen/utils/validate_and_fill_config.py b/src/atgen/utils/validate_and_fill_config.py index 4206679..9fd982e 100644 --- a/src/atgen/utils/validate_and_fill_config.py +++ b/src/atgen/utils/validate_and_fill_config.py @@ -1,7 +1,10 @@ import os -from omegaconf import DictConfig, open_dict +from omegaconf import OmegaConf, DictConfig, open_dict import logging +from .data.get_output_column_name_for_phase import get_output_column_name_for_phase +from .constants import OUTPUT_FIELD_PURPOSE_TRAIN, OUTPUT_FIELD_PURPOSE_TEST + log = logging.getLogger(__name__) @@ -58,6 +61,21 @@ def validate_and_fill_config(config: DictConfig) -> DictConfig: config.data.setdefault("few_shot", {}) config.data.few_shot.setdefault("count", 0) config.data.few_shot.setdefault("separator", "\n\n") + # Add train and test output column names if not provided + if "train_output_column_name" not in config.data: + OmegaConf.update( + config, + "data.train_output_column_name", + get_output_column_name_for_phase(config.data.output_column_name, OUTPUT_FIELD_PURPOSE_TRAIN), + force_add=True, + ) + if "test_output_column_name" not in config.data: + OmegaConf.update( + config, + "data.test_output_column_name", + get_output_column_name_for_phase(config.data.output_column_name, OUTPUT_FIELD_PURPOSE_TEST), + force_add=True, + ) # Model validate_field(config, "model") From 3b87d7322b18b0e0192df9ea9c4fd90ce2045b2f Mon Sep 17 00:00:00 2001 From: Akim Tsvigun Date: Thu, 12 Jun 2025 19:13:56 +0000 Subject: [PATCH 16/39] debugging trace removed --- src/atgen/utils/generate.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/atgen/utils/generate.py b/src/atgen/utils/generate.py index 87bf617..ef080c2 100644 --- a/src/atgen/utils/generate.py +++ b/src/atgen/utils/generate.py @@ -105,7 +105,6 @@ def generate_vllm( batch_generations = [x.outputs[0].text for x in out] generations += batch_generations - import pdb; pdb.set_trace() generations = post_process_generations( generations=generations, data_config=data_config, From 9ee60a28707a52cab8102655a3a846a9da5876ab Mon Sep 17 00:00:00 2001 From: Akim Tsvigun Date: Fri, 13 Jun 2025 06:49:24 +0000 Subject: [PATCH 17/39] bugs fixed --- src/atgen/utils/resolvers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/atgen/utils/resolvers.py b/src/atgen/utils/resolvers.py index 983cfce..b123220 100644 --- a/src/atgen/utils/resolvers.py +++ b/src/atgen/utils/resolvers.py @@ -33,7 +33,6 @@ def register_resolvers() -> None: resolvers_to_register = { "multiply_with_few_shot": multiply_with_few_shot, "to_string": to_string, - "get_output_column_name_for_phase": get_output_column_name_for_phase, } for name, resolver_fn in resolvers_to_register.items(): From 117448fe06fc1e6cd92220caee4c8803c70a6615 Mon Sep 17 00:00:00 2001 From: alfekka Date: Mon, 9 Jun 2025 00:39:57 +0300 Subject: [PATCH 18/39] refactoring metrics --- demo_all_metrics.py | 163 ++++++++++++++++++++++++++++++++++++++++++++ demo_new_metrics.py | 97 ++++++++++++++++++++++++++ 2 files changed, 260 insertions(+) create mode 100644 demo_all_metrics.py create mode 100644 demo_new_metrics.py diff --git a/demo_all_metrics.py b/demo_all_metrics.py new file mode 100644 index 0000000..bf0f057 --- /dev/null +++ b/demo_all_metrics.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +""" +Comprehensive demonstration of the refactored metrics system including DeepEval metrics. +""" + +import sys +import os +sys.path.append('src') + +from atgen.metrics.base import BaseMetric, MetricConfig +from atgen.metrics.lexical import BleuMetric, RougeMetric +from atgen.metrics.semantic import BartScoreMetric, AlignScoreMetric, SentBertMetric +from atgen.metrics.linguistic import ColaMetric +# Note: llm-based directory needs to be renamed to llm_based for Python imports +# from atgen.metrics.llm_based import ( +# DeepEvalAnswerRelevancyMetric, +# DeepEvalFaithfulnessMetric, +# DeepEvalSummarizationMetric, +# DeepEvalPromptAlignmentMetric, +# EvaluationLLM +# ) + + +def demo_all_metrics(): + """Demonstrate all metrics in the new system.""" + print("šŸŽÆ Comprehensive Metrics System Demonstration") + print("=" * 80) + + # Sample data + predictions = [ + "The cat is sitting on the mat peacefully.", + "Python is an excellent programming language for machine learning.", + "Deep learning has revolutionized artificial intelligence applications." + ] + + references = [ + "A cat is resting on the mat.", + "Python is great for ML development.", + "Deep learning has transformed AI." + ] + + original_texts = [ + "There is a cat on a mat", + "What makes Python good for ML?", + "How has deep learning impacted AI?" + ] + + # Create different configurations for different metric types + base_config = MetricConfig( + batch_size=2, + device="cpu", # Use CPU for demo + cache_dir="cache", + aggregate=True + ) + + # DeepEval config (requires API key) + deepeval_config = MetricConfig( + batch_size=2, + device="cpu", + cache_dir="cache", + aggregate=True, + api_key=os.getenv("OPENROUTER_API_KEY"), # Set this in environment + base_url="https://openrouter.ai/api/v1", + model="openai/gpt-4o-mini", # Cheaper model for demo + threshold=0.5, + include_reason=False, + strict_mode=False, + async_mode=False, # Synchronous for simpler demo + verbose_mode=False + ) + + # Organize metrics by category + metrics_by_category = { + "šŸ“ Lexical Metrics": [ + ("BLEU", BleuMetric(base_config)), + ("ROUGE", RougeMetric(base_config)), + ], + "šŸ§  Semantic Metrics": [ + ("SentenceBERT", SentBertMetric(base_config)), + # ("BARTScore", BartScoreMetric(base_config)), # Uncomment if you have dependencies + # ("AlignScore", AlignScoreMetric(base_config)), # Uncomment if you have dependencies + ], + "šŸ—£ļø Linguistic Quality": [ + ("CoLA (Grammaticality)", ColaMetric(base_config)), + ], + # "šŸ¤– LLM-based (DeepEval)": [ + # ("Answer Relevancy", DeepEvalAnswerRelevancyMetric(deepeval_config)), + # ("Faithfulness", DeepEvalFaithfulnessMetric(deepeval_config)), + # ("Summarization", DeepEvalSummarizationMetric(deepeval_config)), + # ("Prompt Alignment", DeepEvalPromptAlignmentMetric(deepeval_config)), + # ] + } + + print(f"šŸ“Š Testing with {len(predictions)} samples\n") + + # Test each category + for category, metrics in metrics_by_category.items(): + print(f"{category}") + print("-" * 60) + + for metric_name, metric in metrics: + print(f" šŸ” {metric_name} ({metric.__class__.__name__})") + print(f" Available: {metric.is_available()}") + + if metric.is_available(): + try: + # Configure test based on metric requirements + if "DeepEval" in metric.__class__.__name__: + if deepeval_config.api_key is None: + print(f" āš ļø API key required (set OPENROUTER_API_KEY)") + print() + continue + + # Different DeepEval metrics need different inputs + if "PromptAlignment" in metric.__class__.__name__: + results = metric.calculate(predictions, references, original_texts) + else: + results = metric.calculate(predictions, None, original_texts) + + elif metric_name == "CoLA (Grammaticality)": + # CoLA only needs predictions + results = metric.calculate(predictions) + + elif metric_name == "SentenceBERT": + # SentBERT can use both references and original texts + results = metric.calculate(predictions, references, original_texts) + + else: + # Standard metrics (BLEU, ROUGE) need references + results = metric.calculate(predictions, references) + + print(f" Results: {results}") + + except Exception as e: + print(f" āŒ Error: {e}") + else: + print(f" āš ļø Dependencies not available") + + print() + + print() + + print("šŸŒŸ System Architecture Highlights:") + print(" āœ… BaseMetric abstract class with consistent interface") + print(" āœ… MetricConfig for centralized configuration") + print(" āœ… Category-based organization (lexical, semantic, linguistic, llm-based)") + print(" āœ… Automatic dependency checking") + print(" āœ… Lazy model initialization") + print(" āœ… Proper error handling and logging") + print(" āœ… Support for multiple references") + print(" āœ… API-based metrics with custom LLM implementation") + print(" āœ… Configurable aggregation") + print(" āœ… Type safety with full type hints") + + print("\nšŸ”§ Next Steps:") + print(" ā€¢ Strategy factory pattern for metric selection") + print(" ā€¢ Configuration-based metric instantiation") + print(" ā€¢ Integration with existing compute_metrics.py") + print(" ā€¢ Performance optimization and caching") + + +if __name__ == "__main__": + demo_all_metrics() \ No newline at end of file diff --git a/demo_new_metrics.py b/demo_new_metrics.py new file mode 100644 index 0000000..e2b47d6 --- /dev/null +++ b/demo_new_metrics.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +""" +Demonstration of the new refactored metrics system. +""" + +import sys +import os +sys.path.append('src') + +from atgen.metrics.base import BaseMetric, MetricConfig +from atgen.metrics.lexical import BleuMetric, RougeMetric +from atgen.metrics.semantic import BartScoreMetric, AlignScoreMetric, SentBertMetric +from atgen.metrics.linguistic import ColaMetric + + +def demo_metrics(): + """Demonstrate the new metrics system.""" + print("šŸŽÆ New Metrics System Demonstration") + print("=" * 60) + + # Sample data + predictions = [ + "The cat is on the mat", + "I love programming in Python", + "Machine learning is fascinating" + ] + + references = [ + "A cat is sitting on the mat", + "I enjoy coding with Python", + "Machine learning is very interesting" + ] + + original_texts = [ + "There is a cat on a mat", + "Programming with Python is great", + "ML technology is amazing" + ] + + # Create metrics with custom config + config = MetricConfig( + batch_size=8, + device="cpu", # Use CPU for demo + cache_dir="cache", + aggregate=True + ) + + metrics = [ + ("BLEU", BleuMetric(config)), + ("ROUGE", RougeMetric(config)), + ("CoLA (Grammaticality)", ColaMetric(config)), + ("SentenceBERT", SentBertMetric(config)), + # ("BARTScore", BartScoreMetric(config)), # Uncomment if you have the dependencies + # ("AlignScore", AlignScoreMetric(config)), # Uncomment if you have the dependencies + ] + + print(f"šŸ“Š Testing with {len(predictions)} samples\n") + + for metric_name, metric in metrics: + print(f"šŸ” Testing {metric_name} ({metric.__class__.__name__})") + print(f" Available: {metric.is_available()}") + + if metric.is_available(): + try: + # Test different scenarios based on metric requirements + if metric_name == "CoLA (Grammaticality)": + # CoLA only needs predictions + results = metric.calculate(predictions) + elif metric_name == "SentenceBERT": + # SentBERT can use both references and original texts + results = metric.calculate(predictions, references, original_texts) + else: + # Standard metrics (BLEU, ROUGE) need references + results = metric.calculate(predictions, references) + + print(f" Results: {results}") + + except Exception as e: + print(f" āŒ Error: {e}") + else: + print(f" āš ļø Dependencies not available") + + print() + + print("šŸŒŸ Key Features of the New System:") + print(" āœ… Clean inheritance from BaseMetric") + print(" āœ… Consistent configuration via MetricConfig") + print(" āœ… Automatic dependency checking") + print(" āœ… Lazy model initialization") + print(" āœ… Proper error handling") + print(" āœ… Organized by category (lexical, semantic, linguistic)") + print(" āœ… Type hints throughout") + print(" āœ… Configurable aggregation") + + +if __name__ == "__main__": + demo_metrics() \ No newline at end of file From 9d5e5d9160baa937b2b5b401188672d2a7ee5f6d Mon Sep 17 00:00:00 2001 From: alfekka Date: Mon, 9 Jun 2025 00:55:59 +0300 Subject: [PATCH 19/39] removed dependencies chek --- demo_all_metrics.py | 40 ++++++++++------ example_bigbench_usage.py | 97 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+), 14 deletions(-) create mode 100644 example_bigbench_usage.py diff --git a/demo_all_metrics.py b/demo_all_metrics.py index bf0f057..f32c864 100644 --- a/demo_all_metrics.py +++ b/demo_all_metrics.py @@ -11,14 +11,14 @@ from atgen.metrics.lexical import BleuMetric, RougeMetric from atgen.metrics.semantic import BartScoreMetric, AlignScoreMetric, SentBertMetric from atgen.metrics.linguistic import ColaMetric -# Note: llm-based directory needs to be renamed to llm_based for Python imports -# from atgen.metrics.llm_based import ( -# DeepEvalAnswerRelevancyMetric, -# DeepEvalFaithfulnessMetric, -# DeepEvalSummarizationMetric, -# DeepEvalPromptAlignmentMetric, -# EvaluationLLM -# ) +from atgen.metrics.llm_based import ( + DeepEvalAnswerRelevancyMetric, + DeepEvalFaithfulnessMetric, + DeepEvalSummarizationMetric, + DeepEvalPromptAlignmentMetric, + BigBenchHardMetric, + EvaluationLLM +) def demo_all_metrics(): @@ -83,12 +83,15 @@ def demo_all_metrics(): "šŸ—£ļø Linguistic Quality": [ ("CoLA (Grammaticality)", ColaMetric(base_config)), ], - # "šŸ¤– LLM-based (DeepEval)": [ - # ("Answer Relevancy", DeepEvalAnswerRelevancyMetric(deepeval_config)), - # ("Faithfulness", DeepEvalFaithfulnessMetric(deepeval_config)), - # ("Summarization", DeepEvalSummarizationMetric(deepeval_config)), - # ("Prompt Alignment", DeepEvalPromptAlignmentMetric(deepeval_config)), - # ] + "šŸ¤– LLM-based (DeepEval)": [ + ("Answer Relevancy", DeepEvalAnswerRelevancyMetric(deepeval_config)), + ("Faithfulness", DeepEvalFaithfulnessMetric(deepeval_config)), + ("Summarization", DeepEvalSummarizationMetric(deepeval_config)), + ("Prompt Alignment", DeepEvalPromptAlignmentMetric(deepeval_config)), + ], + "šŸ§® Benchmark Metrics": [ + ("BigBenchHard", BigBenchHardMetric(deepeval_config)), + ] } print(f"šŸ“Š Testing with {len(predictions)} samples\n") @@ -117,6 +120,14 @@ def demo_all_metrics(): else: results = metric.calculate(predictions, None, original_texts) + elif metric_name == "BigBenchHard": + # BigBenchHard needs a model and tokenizer, skip for demo + print(f" āš ļø Requires model and tokenizer (use set_model_and_tokenizer())") + print(f" šŸ’” Example: metric.set_model_and_tokenizer(model, tokenizer)") + print(f" score = metric.evaluate_benchmark(model, tokenizer)") + print() + continue + elif metric_name == "CoLA (Grammaticality)": # CoLA only needs predictions results = metric.calculate(predictions) @@ -149,6 +160,7 @@ def demo_all_metrics(): print(" āœ… Proper error handling and logging") print(" āœ… Support for multiple references") print(" āœ… API-based metrics with custom LLM implementation") + print(" āœ… Benchmark metrics (BigBenchHard)") print(" āœ… Configurable aggregation") print(" āœ… Type safety with full type hints") diff --git a/example_bigbench_usage.py b/example_bigbench_usage.py new file mode 100644 index 0000000..aebd473 --- /dev/null +++ b/example_bigbench_usage.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +""" +Example usage of BigBenchHard metric with the new architecture. +""" + +import sys +sys.path.append('src') + +from atgen.metrics.base import MetricConfig +from atgen.metrics.llm_based import BigBenchHardMetric + + +def example_bigbench_usage(): + """Example of how to use BigBenchHard metric.""" + print("šŸ§® BigBenchHard Metric Usage Example") + print("=" * 60) + + # Create configuration for BigBenchHard + config = MetricConfig( + batch_size=8, + device="cuda", # or "cpu" + cache_dir="cache", + model_name="MyAwesomeModel", + # Benchmark-specific parameters + benchmark_params={ + "n_shots": 3, # Number of few-shot examples + "enable_cot": True, # Enable chain-of-thought + }, + # Generation parameters for the model + generation_params={ + "max_new_tokens": 100, + "temperature": 0.1, + "do_sample": False, + } + ) + + # Create the metric + metric = BigBenchHardMetric(config) + + print(f"āœ… Metric created: {metric.name}") + print(f"šŸ“¦ Dependencies available: {metric.is_available()}") + + if not metric.is_available(): + print("āŒ BigBenchHard dependencies not available") + print(" Install with: pip install deepeval") + return + + print("\nšŸ”§ Usage Options:") + print("\n1. Option 1: Using the standard calculate() interface:") + print(" ```python") + print(" # First set the model and tokenizer") + print(" metric.set_model_and_tokenizer(model, tokenizer)") + print(" ") + print(" # Then calculate (predictions/references are ignored)") + print(" results = metric.calculate([], [], [])") + print(" print(results) # {'bigbench_hard_score': 0.85}") + print(" ```") + + print("\n2. Option 2: Using the convenience method:") + print(" ```python") + print(" # Direct evaluation") + print(" score = metric.evaluate_benchmark(model, tokenizer, batch_size=16)") + print(" print(f'BigBenchHard score: {score}')") + print(" ```") + + print("\n3. Option 3: Using the original interface style:") + print(" ```python") + print(" # Set model first") + print(" metric.set_model_and_tokenizer(model, tokenizer)") + print(" ") + print(" # Get the overall score") + print(" score = metric.benchmark.overall_score") + print(" ```") + + print("\nšŸ“ Configuration Details:") + print(f" ā€¢ Batch size: {config.batch_size}") + print(f" ā€¢ Device: {config.device}") + print(f" ā€¢ Model name: {config.model_name}") + print(f" ā€¢ Benchmark params: {config.benchmark_params}") + print(f" ā€¢ Generation params: {config.generation_params}") + + print("\nšŸŽÆ What BigBenchHard Evaluates:") + print(" ā€¢ Challenging reasoning tasks from BIG-bench") + print(" ā€¢ Mathematical reasoning") + print(" ā€¢ Logical reasoning") + print(" ā€¢ Multi-step problem solving") + print(" ā€¢ Chain-of-thought reasoning (if enabled)") + + print("\nšŸ’” Integration Notes:") + print(" ā€¢ Follows the same BaseMetric interface as other metrics") + print(" ā€¢ Supports the standard MetricConfig system") + print(" ā€¢ Can be used in metric factories and pipelines") + print(" ā€¢ Provides both direct and standard interfaces") + + +if __name__ == "__main__": + example_bigbench_usage() \ No newline at end of file From 89031f5f475e6405494a9aef71d787d3f321b790 Mon Sep 17 00:00:00 2001 From: alfekka Date: Mon, 9 Jun 2025 00:56:17 +0300 Subject: [PATCH 20/39] added default factory --- src/atgen/metrics/base/base_metric.py | 3 +-- src/atgen/metrics/compute_metrics.py | 36 ++++++--------------------- 2 files changed, 8 insertions(+), 31 deletions(-) diff --git a/src/atgen/metrics/base/base_metric.py b/src/atgen/metrics/base/base_metric.py index 69e6595..acb6cd5 100644 --- a/src/atgen/metrics/base/base_metric.py +++ b/src/atgen/metrics/base/base_metric.py @@ -14,8 +14,7 @@ class MetricConfig: checkpoint: Optional[str] = None model_name: Optional[str] = None - # API configuration parameters - provider: Optional[str] = None + # API-based parameters (for LLM-based metrics) api_key: Optional[str] = None base_url: Optional[str] = None model: Optional[str] = None diff --git a/src/atgen/metrics/compute_metrics.py b/src/atgen/metrics/compute_metrics.py index e3682e8..613d35c 100644 --- a/src/atgen/metrics/compute_metrics.py +++ b/src/atgen/metrics/compute_metrics.py @@ -267,36 +267,14 @@ def get_comprehensive_config() -> MetricsConfig: ] ) else: - result["exact_match"] = np.array( - [pred == ref for pred, ref in zip(generated_texts, reference_texts)] - ) - # BLEU - start_time = time() - result["bleu"] = np.array( - [ - pair_bleu(references=ref, prediction=pred) - for pred, ref in tqdm(zip(generated_texts, reference_texts)) + ref_lengths = np.array([len(text.split()) for text in references]) + + gen_lengths = np.array([len(text.split()) for text in predictions]) + if isinstance(references[0], list): + exact_matches = [ + any(pred == ref for ref in ref_list) + for pred, ref_list in zip(predictions, references) ] - ) - time_dict["time_bleu"] = time() - start_time - # ROUGE - start_time = time() - result.update( - rouge.compute( - predictions=generated_texts, - references=reference_texts, - use_stemmer=True, - ) - ) - time_dict["time_rouge"] = time() - start_time - # Sacrebleu - start_time = time() - if not isinstance(reference_texts[0], list): - sacrebleu_references = [[ref] for ref in reference_texts] - sacrebleu_result = sacrebleu.compute( - predictions=generated_texts, references=sacrebleu_references - ) - result["sacrebleu"] = sacrebleu_result.pop("score") else: sacrebleu_scores = [] for pred, ref in zip(generated_texts, reference_texts): From 48915433d3cc4e65071f7d0e533070174e430f79 Mon Sep 17 00:00:00 2001 From: alfekka Date: Fri, 13 Jun 2025 16:27:03 +0300 Subject: [PATCH 21/39] fixes of deepeval --- .../metrics/llm_based/bigbench_hard_metric.py | 14 -------------- .../metrics/llm_based/prompt_alignment_metric.py | 4 +--- .../metrics/llm_based/summarization_metric.py | 4 +--- src/atgen/metrics/semantic/bart_score_metric.py | 2 -- 4 files changed, 2 insertions(+), 22 deletions(-) diff --git a/src/atgen/metrics/llm_based/bigbench_hard_metric.py b/src/atgen/metrics/llm_based/bigbench_hard_metric.py index 236d5e2..72dad02 100644 --- a/src/atgen/metrics/llm_based/bigbench_hard_metric.py +++ b/src/atgen/metrics/llm_based/bigbench_hard_metric.py @@ -114,20 +114,6 @@ def __init__(self, config: Optional[MetricConfig] = None): self.benchmark_params = getattr(config, 'benchmark_params', {}) if config else {} self.model_name = getattr(config, 'model_name', 'Model') if config else 'Model' - def _check_dependencies(self) -> bool: - """Check if BigBenchHard dependencies are available.""" - try: - from deepeval.benchmarks import BigBenchHard - from transformers import GenerationMixin, PreTrainedTokenizerBase - return True - except ImportError: - return False - - def _validate_config(self): - """Validate BigBenchHard configuration.""" - super()._validate_config() - # BigBenchHard doesn't use the standard prediction/reference format - # so we don't need the usual validation def set_model_and_tokenizer(self, model: GenerationMixin, tokenizer: PreTrainedTokenizerBase): """ diff --git a/src/atgen/metrics/llm_based/prompt_alignment_metric.py b/src/atgen/metrics/llm_based/prompt_alignment_metric.py index 0805159..e549d18 100644 --- a/src/atgen/metrics/llm_based/prompt_alignment_metric.py +++ b/src/atgen/metrics/llm_based/prompt_alignment_metric.py @@ -24,9 +24,7 @@ def calculate(self, predictions: List[str], references: Optional[List[Union[str, Returns: Dictionary with Prompt Alignment scores """ - if not self.is_available(): - raise RuntimeError("DeepEval dependencies not available") - + if original_texts is None: raise ValueError("Prompt Alignment metric requires original texts") diff --git a/src/atgen/metrics/llm_based/summarization_metric.py b/src/atgen/metrics/llm_based/summarization_metric.py index 08d0624..e5b94c6 100644 --- a/src/atgen/metrics/llm_based/summarization_metric.py +++ b/src/atgen/metrics/llm_based/summarization_metric.py @@ -24,9 +24,7 @@ def calculate(self, predictions: List[str], references: Optional[List[Union[str, Returns: Dictionary with Summarization scores """ - if not self.is_available(): - raise RuntimeError("DeepEval dependencies not available") - + if original_texts is None: raise ValueError("Summarization metric requires original texts") diff --git a/src/atgen/metrics/semantic/bart_score_metric.py b/src/atgen/metrics/semantic/bart_score_metric.py index 45ef549..a33b168 100644 --- a/src/atgen/metrics/semantic/bart_score_metric.py +++ b/src/atgen/metrics/semantic/bart_score_metric.py @@ -112,8 +112,6 @@ def calculate(self, predictions: List[str], references: Optional[List[Union[str, Returns: Dictionary with BARTScore results """ - if not self.is_available(): - raise RuntimeError("BARTScore dependencies not available") self._initialize_scorer() From 347d59921c2cbd7e7d111a94b2b020bf5fc75e45 Mon Sep 17 00:00:00 2001 From: alfekka Date: Fri, 13 Jun 2025 17:47:28 +0300 Subject: [PATCH 22/39] hell --- configs/base.yaml | 4 ++-- src/atgen/metrics/factory.py | 11 +++++++---- src/atgen/metrics/lexical/rouge_metric.py | 5 ++--- src/atgen/metrics/llm_based/base_deepeval_metric.py | 2 -- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/configs/base.yaml b/configs/base.yaml index fae2e4c..0a074a8 100644 --- a/configs/base.yaml +++ b/configs/base.yaml @@ -26,11 +26,11 @@ al: evaluate_zero_iteration: True subsample_size: -1 required_performance: - rouge1: 0.5 + rouge: 0.5 budget: model: - checkpoint: Qwen/Qwen3-1.7B + checkpoint: Qwen/Qwen2.5-1.5B quantize: False model_max_length: ${multiply_with_few_shot:${data.input_max_length},${data.few_shot.count}} dtype: bfloat16 diff --git a/src/atgen/metrics/factory.py b/src/atgen/metrics/factory.py index 501bf4c..595a7dd 100644 --- a/src/atgen/metrics/factory.py +++ b/src/atgen/metrics/factory.py @@ -85,22 +85,25 @@ class MetricsFactory: _metric_registry = { "bleu": BleuMetric, - "rouge": RougeMetric, + "rouge1": RougeMetric, + "rouge2": RougeMetric, + "rougeL": RougeMetric, + "rougeLsum": RougeMetric, "bartscore": BartScoreMetric, "alignscore": AlignScoreMetric, "sentbert": SentBertMetric, - "sentence_bert": SentBertMetric, # Alias + "sentence_bert": SentBertMetric, "cola": ColaMetric, - "grammaticality": ColaMetric, # Alias + "grammaticality": ColaMetric, "deepeval_answer_relevance": DeepEvalAnswerRelevancyMetric, "deepeval_faithfulness": DeepEvalFaithfulnessMetric, "deepeval_summarization": DeepEvalSummarizationMetric, "deepeval_prompt_alignment": DeepEvalPromptAlignmentMetric, "bigbench_hard": BigBenchHardMetric, - "big_bench_hard": BigBenchHardMetric, # Alias + "big_bench_hard": BigBenchHardMetric, } @classmethod diff --git a/src/atgen/metrics/lexical/rouge_metric.py b/src/atgen/metrics/lexical/rouge_metric.py index 2a11a4c..61f686f 100644 --- a/src/atgen/metrics/lexical/rouge_metric.py +++ b/src/atgen/metrics/lexical/rouge_metric.py @@ -16,6 +16,7 @@ def _initialize_rouge(self): """Initialize ROUGE if not already initialized.""" if self.rouge is None: self.rouge = load("rouge", cache_dir=self.config.cache_dir) + def calculate(self, predictions: List[str], references: Optional[List[Union[str, List[str]]]] = None, original_texts: Optional[List[str]] = None) -> Dict[str, float]: """ @@ -54,13 +55,11 @@ def calculate(self, predictions: List[str], references: Optional[List[Union[str, for key, value in results.items(): if isinstance(value, (int, float)): scores[key] = float(value) - elif hasattr(value, 'item'): # For numpy scalars + elif hasattr(value, 'item'): # scores[key] = float(value.item()) else: scores[key] = float(value) - # ROUGE already aggregates by default, but we need to handle the case - # where it returns arrays (though this is typically rare) if self.config.aggregate: for key, value in scores.items(): if isinstance(value, np.ndarray): diff --git a/src/atgen/metrics/llm_based/base_deepeval_metric.py b/src/atgen/metrics/llm_based/base_deepeval_metric.py index c7e1f3d..d7408fd 100644 --- a/src/atgen/metrics/llm_based/base_deepeval_metric.py +++ b/src/atgen/metrics/llm_based/base_deepeval_metric.py @@ -84,13 +84,11 @@ def _run_evaluation(self, test_cases: List[LLMTestCase], metric, metric_name: st if not test_cases: return {} - # Disable printing to console during evaluation if not verbose original_stdout = sys.stdout if not self.config.verbose_mode: sys.stdout = open(os.devnull, "w") try: - # Run evaluation evaluation_results = evaluate( test_cases=test_cases, metrics=[metric], From 98bab28e83708632b74a4c9b0c46877bbbf6d4a6 Mon Sep 17 00:00:00 2001 From: alfekka Date: Fri, 13 Jun 2025 20:03:17 +0300 Subject: [PATCH 23/39] fixes of metrics --- .deepeval-cache.json | 1 + .deepeval_telemetry.txt | 4 + src/atgen/metrics/__init__.py | 21 ++- src/atgen/metrics/base/base_metric.py | 5 + src/atgen/metrics/compute_metrics.py | 171 ++----------------------- src/atgen/metrics/factory.py | 116 +++++++++++++---- src/atgen/metrics/semantic/__init__.py | 34 +++-- 7 files changed, 152 insertions(+), 200 deletions(-) create mode 100644 .deepeval-cache.json create mode 100644 .deepeval_telemetry.txt diff --git a/.deepeval-cache.json b/.deepeval-cache.json new file mode 100644 index 0000000..82cd467 --- /dev/null +++ b/.deepeval-cache.json @@ -0,0 +1 @@ +{"test_cases_lookup_map": {"{\"actual_output\": \"Machine learning models learn patterns from training data to make predictions.\", \"context\": null, \"expected_output\": null, \"hyperparameters\": null, \"input\": \"How do machine learning models work?\", \"retrieval_context\": null}": {"cached_metrics_data": [{"metric_data": {"name": "Answer Relevancy", "threshold": 0.5, "success": true, "score": 1.0, "strictMode": false, "evaluationModel": "EvaluationLLM: openai/gpt-4o-mini", "evaluationCost": 0, "verboseLogs": "Statements:\n[\n \"Machine learning models learn patterns from training data.\",\n \"Machine learning models make predictions.\"\n] \n \nVerdicts:\n[\n {\n \"verdict\": \"yes\",\n \"reason\": null\n },\n {\n \"verdict\": \"yes\",\n \"reason\": null\n }\n]"}, "metric_configuration": {"threshold": 0.5, "evaluation_model": "EvaluationLLM: openai/gpt-4o-mini", "strict_mode": false, "include_reason": false}}]}, "{\"actual_output\": \"Deep learning uses neural networks with multiple layers to process information.\", \"context\": null, \"expected_output\": null, \"hyperparameters\": null, \"input\": \"What is deep learning?\", \"retrieval_context\": null}": {"cached_metrics_data": [{"metric_data": {"name": "Answer Relevancy", "threshold": 0.5, "success": true, "score": 1.0, "strictMode": false, "evaluationModel": "EvaluationLLM: openai/gpt-4o-mini", "evaluationCost": 0, "verboseLogs": "Statements:\n[\n \"Deep learning uses neural networks.\",\n \"Neural networks have multiple layers.\",\n \"Neural networks process information.\"\n] \n \nVerdicts:\n[\n {\n \"verdict\": \"yes\",\n \"reason\": null\n },\n {\n \"verdict\": \"yes\",\n \"reason\": null\n },\n {\n \"verdict\": \"yes\",\n \"reason\": null\n }\n]"}, "metric_configuration": {"threshold": 0.5, "evaluation_model": "EvaluationLLM: openai/gpt-4o-mini", "strict_mode": false, "include_reason": false}}]}, "{\"actual_output\": \"Natural language processing helps computers understand and generate human language.\", \"context\": null, \"expected_output\": null, \"hyperparameters\": null, \"input\": \"Explain natural language processing.\", \"retrieval_context\": null}": {"cached_metrics_data": [{"metric_data": {"name": "Answer Relevancy", "threshold": 0.5, "success": true, "score": 1.0, "strictMode": false, "evaluationModel": "EvaluationLLM: openai/gpt-4o-mini", "evaluationCost": 0, "verboseLogs": "Statements:\n[\n \"Natural language processing helps computers understand human language.\",\n \"Natural language processing helps computers generate human language.\"\n] \n \nVerdicts:\n[\n {\n \"verdict\": \"yes\",\n \"reason\": null\n },\n {\n \"verdict\": \"yes\",\n \"reason\": null\n }\n]"}, "metric_configuration": {"threshold": 0.5, "evaluation_model": "EvaluationLLM: openai/gpt-4o-mini", "strict_mode": false, "include_reason": false}}]}, "{\"actual_output\": \"The capital of France is Paris.\", \"context\": null, \"expected_output\": null, \"hyperparameters\": null, \"input\": \"What is the capital of France?\", \"retrieval_context\": null}": {"cached_metrics_data": [{"metric_data": {"name": "Answer Relevancy", "threshold": 0.5, "success": true, "score": 1.0, "strictMode": false, "evaluationModel": "EvaluationLLM: openai/gpt-4o-mini", "evaluationCost": 0, "verboseLogs": "Statements:\n[\n \"The capital of France is Paris.\"\n] \n \nVerdicts:\n[\n {\n \"verdict\": \"yes\",\n \"reason\": null\n }\n]"}, "metric_configuration": {"threshold": 0.5, "evaluation_model": "EvaluationLLM: openai/gpt-4o-mini", "strict_mode": false, "include_reason": false}}]}, "{\"actual_output\": \"The capital of France is Paris.\", \"context\": null, \"expected_output\": null, \"hyperparameters\": null, \"input\": \"What is the capital of France?\", \"retrieval_context\": [\"What is the capital of France?\"]}": {"cached_metrics_data": [{"metric_data": {"name": "Faithfulness", "threshold": 0.5, "success": true, "score": 1.0, "strictMode": false, "evaluationModel": "EvaluationLLM: openai/gpt-4o-mini", "evaluationCost": 0, "verboseLogs": "Truths (limit=None):\n[\n \"The capital of France is Paris.\"\n] \n \nClaims:\n[\n \"The capital of France is Paris.\"\n] \n \nVerdicts:\n[\n {\n \"verdict\": \"yes\",\n \"reason\": null\n }\n]"}, "metric_configuration": {"threshold": 0.5, "evaluation_model": "EvaluationLLM: openai/gpt-4o-mini", "strict_mode": false, "include_reason": false}}]}}} \ No newline at end of file diff --git a/.deepeval_telemetry.txt b/.deepeval_telemetry.txt new file mode 100644 index 0000000..99c6285 --- /dev/null +++ b/.deepeval_telemetry.txt @@ -0,0 +1,4 @@ +DEEPEVAL_ID=5a40c1ca-390b-4d9c-8f2e-704b78207dc2 +DEEPEVAL_STATUS=old +DEEPEVAL_LAST_FEATURE=evaluation +DEEPEVAL_EVALUATION_STATUS=old diff --git a/src/atgen/metrics/__init__.py b/src/atgen/metrics/__init__.py index 7142cfa..695aaff 100644 --- a/src/atgen/metrics/__init__.py +++ b/src/atgen/metrics/__init__.py @@ -1,4 +1,3 @@ - # Base classes from .base import BaseMetric, MetricConfig @@ -86,15 +85,27 @@ "get_comprehensive_config", "get_deepeval_config", + # Helper functions + "get_available_metrics", + "get_all_possible_metric_keys", + "create_metric", + "create_metrics_from_config", + "AVAILABLE_METRICS", + "METRICS", ] def get_available_metrics(): - """Get all available metrics.""" + """Get all available metric names.""" return MetricsFactory.get_available_metrics() +def get_all_possible_metric_keys(): + """Get all possible metric keys that can be returned by compute_metrics.""" + return MetricsFactory.get_all_possible_metric_keys() + + def create_metric(name: str, config: MetricConfig = None): """Create a metric by name.""" return MetricsFactory.create_metric(name, config) @@ -104,4 +115,8 @@ def create_metrics_from_config(config): """Create metrics from configuration.""" return MetricsFactory.create_from_config(config) -AVAILABLE_METRICS = MetricsFactory.get_available_metrics() \ No newline at end of file +# For backward compatibility, still provide the basic metric names +METRICS = MetricsFactory.get_available_metrics() + +# But AVAILABLE_METRICS should include all possible keys for performance checking +AVAILABLE_METRICS = MetricsFactory.get_all_possible_metric_keys() \ No newline at end of file diff --git a/src/atgen/metrics/base/base_metric.py b/src/atgen/metrics/base/base_metric.py index acb6cd5..0bd1d66 100644 --- a/src/atgen/metrics/base/base_metric.py +++ b/src/atgen/metrics/base/base_metric.py @@ -15,6 +15,7 @@ class MetricConfig: model_name: Optional[str] = None # API-based parameters (for LLM-based metrics) + provider: Optional[str] = None api_key: Optional[str] = None base_url: Optional[str] = None model: Optional[str] = None @@ -46,4 +47,8 @@ def _validate_config(self): @abstractmethod def calculate(self, predictions, references, original_texts): pass + + def is_available(self) -> bool: + """Check if the metric is available for use.""" + return True \ No newline at end of file diff --git a/src/atgen/metrics/compute_metrics.py b/src/atgen/metrics/compute_metrics.py index 613d35c..fe07684 100644 --- a/src/atgen/metrics/compute_metrics.py +++ b/src/atgen/metrics/compute_metrics.py @@ -1,12 +1,5 @@ from time import time -<<<<<<< HEAD -<<<<<<< HEAD from typing import List, Dict, Union, Optional -======= -from typing import Dict, Union ->>>>>>> eaad08f (fixed version for multi-ref datasets) -======= ->>>>>>> 7fd04b7 (bugs fixed) import logging import numpy as np from omegaconf import DictConfig @@ -14,9 +7,6 @@ from .factory import MetricsFactory, MetricsConfig, get_metric_requirements from .base import MetricConfig -<<<<<<< HEAD -logger = logging.getLogger(__name__) -======= from .metrics import ( pair_bleu, calculate_bart_score, @@ -29,11 +19,9 @@ log = logging.getLogger() ->>>>>>> eaad08f (fixed version for multi-ref datasets) def compute_metrics( -<<<<<<< HEAD generated_texts: List[str], reference_texts: Optional[List[Union[str, List[str]]]] = None, original_texts: Optional[List[str]] = None, @@ -42,14 +30,6 @@ def compute_metrics( tokenizer=None, cache_dir: Optional[str] = None, ) -> Dict[str, float]: -======= - generated_texts, - reference_texts, - original_texts, - config: DictConfig, - cache_dir: str = "cache", -) -> dict[str, float]: ->>>>>>> 7fd04b7 (bugs fixed) """ Compute various metrics for generated texts using the new architecture. @@ -104,7 +84,7 @@ def compute_metrics( metrics = MetricsFactory.create_metrics(metrics_config) if not metrics: - logger.warning("No metrics were successfully created") + log.warning("No metrics were successfully created") return {} results = {} @@ -140,23 +120,23 @@ def compute_metrics( results["word_length_rel"] = float(np.mean(gen_lengths / ref_lengths_safe)) for metric_name, metric in metrics.items(): - logger.info(f"Computing {metric_name}...") + log.info(f"Computing {metric_name}...") start_time = time() try: req = requirements.get(metric_name, {}) if req.get("requires_references", False) and references is None: - logger.warning(f"Skipping {metric_name}: requires references but none provided") + log.warning(f"Skipping {metric_name}: requires references but none provided") continue if req.get("requires_original_texts", False) and original_texts is None: - logger.warning(f"Skipping {metric_name}: requires original texts but none provided") + log.warning(f"Skipping {metric_name}: requires original texts but none provided") continue if metric_name in ["bigbench_hard", "big_bench_hard"]: if model is None or tokenizer is None: - logger.warning(f"Skipping {metric_name}: requires model and tokenizer") + log.warning(f"Skipping {metric_name}: requires model and tokenizer") continue metric.set_model_and_tokenizer(model, tokenizer) @@ -168,17 +148,17 @@ def compute_metrics( results[key] = value timing_info[f"time_{metric_name}"] = time() - start_time - logger.info(f"Completed {metric_name} in {timing_info[f'time_{metric_name}']:.2f}s") + log.info(f"Completed {metric_name} in {timing_info[f'time_{metric_name}']:.2f}s") except Exception as e: - logger.error(f"Error computing {metric_name}: {e}") + log.error(f"Error computing {metric_name}: {e}") timing_info[f"time_{metric_name}"] = time() - start_time # Add total timing timing_info["time_total"] = time() - start_total results.update(timing_info) - logger.info(f"Computed {len(metrics)} metrics in {timing_info['time_total']:.2f}s") + log.info(f"Computed {len(metrics)} metrics in {timing_info['time_total']:.2f}s") return results @@ -224,7 +204,6 @@ def get_default_config() -> MetricsConfig: aggregate=True ) -<<<<<<< HEAD def get_comprehensive_config() -> MetricsConfig: """Get a comprehensive metrics configuration with all metrics.""" @@ -239,80 +218,8 @@ def get_comprehensive_config() -> MetricsConfig: cache_dir="cache", aggregate=True ) -======= - # Avoid division by zero - src_word_lengths_safe = np.where(src_word_lengths > 0, src_word_lengths, 1) - result["word_length_src_rel"] = result["word_length_gen"] / src_word_lengths_safe - if "bartscore" in config.additional_metrics and is_bart_score_available: - log.info("Calculating BARTScore scores...") - start_time = time() - result.update( - calculate_bart_score( - preds=generated_texts, - texts=original_texts, - refs=reference_texts, - batch_size=4, - cache_dir=cache_dir, - ) - ) - time_dict["time_bartscore"] = time() - start_time - # Metrics that use both the generated texts and the reference texts - if reference_texts is not None: - # Exact match - if isinstance(reference_texts[0], list): - result["exact_match"] = np.array( - [ - any(pred == one_ref for one_ref in ref) - for pred, ref in zip(generated_texts, reference_texts) - ] - ) - else: - ref_lengths = np.array([len(text.split()) for text in references]) - - gen_lengths = np.array([len(text.split()) for text in predictions]) - if isinstance(references[0], list): - exact_matches = [ - any(pred == ref for ref in ref_list) - for pred, ref_list in zip(predictions, references) - ] - else: - sacrebleu_scores = [] - for pred, ref in zip(generated_texts, reference_texts): - sacrebleu_result = sacrebleu.compute( - predictions=[pred], references=[ref] - ) - sacrebleu_scores.append(sacrebleu_result.pop("score")) - result["sacrebleu"] = sacrebleu_scores - - time_dict["time_sacrebleu"] = time() - start_time - # Lengths - if isinstance(reference_texts[0], list): - ref_word_lengths = np.array( - [ - np.mean([len(text.split()) for text in ref]) - for ref in reference_texts - ] - ) - else: - ref_word_lengths = np.array([len(ref.split()) for ref in reference_texts]) - # Avoid division by zero - ref_word_lengths_safe = np.where(ref_word_lengths > 0, ref_word_lengths, 1) - result["word_length_rel"] = result["word_length_gen"] / ref_word_lengths_safe - - # AlignScore - if "alignscore" in config.additional_metrics and is_alignscore_available: - log.info("Calculating AlignScore scores...") - start_time = time() - alignscores = calculate_alignscore( - generated_texts, reference_texts, original_texts - ) - if alignscores is not None: - result.update(alignscores) - time_dict["time_alignscore"] = time() - start_time ->>>>>>> eaad08f (fixed version for multi-ref datasets) -<<<<<<< HEAD def get_deepeval_config(api_key: str, model: str = "openai/gpt-4o-mini") -> MetricsConfig: return MetricsConfig( metrics=[ @@ -331,64 +238,4 @@ def get_deepeval_config(api_key: str, model: str = "openai/gpt-4o-mini") -> Metr threshold=0.5, async_mode=True, verbose_mode=False - ) -======= - if deepeval_metrics_to_calculate: - if isinstance(reference_texts[0], list): - log.error("DeepEval does not support multiple references. Skipping...") - else: - # Validate OpenRouter model - only warn if not in predefined list, but still use it - provider = config["provider"] - if config.model not in API_MODELS.get(provider): - log.warning( - f"Using custom model: {config.model}. " - + ( - f"Available models: {API_MODELS[provider]}" - if provider in API_MODELS - else "" - ) - ) - log.info( - f"Calculating DeepEval metrics: {', '.join(deepeval_metrics_to_calculate)}..." - ) - start_time = time() - result.update( - calculate_deepeval_metrics( - predictions=generated_texts, - references=reference_texts, - original_texts=original_texts, - metrics_to_calculate=deepeval_metrics_to_calculate, - base_url=config.base_url, - api_key=config.api_key, - model=config.model, - threshold=config.deepeval_threshold, - include_reason=config.deepeval_include_reason, - strict_mode=config.deepeval_strict_mode, - async_mode=config.deepeval_async_mode, - verbose_mode=config.deepeval_verbose_mode, - truths_extraction_limit=config.deepeval_truths_extraction_limit, - ) - ) - time_dict["time_deepeval"] = time() - start_time - - for key, value in result.items(): - if isinstance(value, np.ndarray): - result[key] = float(np.mean(value)) - elif isinstance(value, (int, float)): - # Ensure numerical values are converted to float - result[key] = float(value) - # Make sure non-numerical values that aren't reasons are preserved - elif not key.endswith("_reasons") and not "_reason" in key.lower(): - continue - - # Filter out reason fields from the final aggregated results - more robust filtering - result = { - key: value - for key, value in sorted(result.items()) - if not key.endswith("_reasons") - and not "_reason" in key.lower() - and isinstance(value, (int, float)) # Ensure we only keep numerical metrics - } - - return result ->>>>>>> eaad08f (fixed version for multi-ref datasets) + ) diff --git a/src/atgen/metrics/factory.py b/src/atgen/metrics/factory.py index 595a7dd..1a42947 100644 --- a/src/atgen/metrics/factory.py +++ b/src/atgen/metrics/factory.py @@ -83,43 +83,106 @@ def __post_init__(self): class MetricsFactory: """Factory for creating metric instances.""" - _metric_registry = { - "bleu": BleuMetric, - "rouge1": RougeMetric, - "rouge2": RougeMetric, - "rougeL": RougeMetric, - "rougeLsum": RougeMetric, - - "bartscore": BartScoreMetric, - "alignscore": AlignScoreMetric, - "sentbert": SentBertMetric, - "sentence_bert": SentBertMetric, - - "cola": ColaMetric, - "grammaticality": ColaMetric, - - "deepeval_answer_relevance": DeepEvalAnswerRelevancyMetric, - "deepeval_faithfulness": DeepEvalFaithfulnessMetric, - "deepeval_summarization": DeepEvalSummarizationMetric, - "deepeval_prompt_alignment": DeepEvalPromptAlignmentMetric, - "bigbench_hard": BigBenchHardMetric, - "big_bench_hard": BigBenchHardMetric, - } + # Build registry dynamically to handle None values from import failures + _metric_registry = {} + + @classmethod + def _build_registry(cls): + """Build the metric registry, excluding None values from failed imports.""" + if not cls._metric_registry: # Only build once + registry = { + "bleu": BleuMetric, + "rouge1": RougeMetric, + "rouge2": RougeMetric, + "rougeL": RougeMetric, + "rougeLsum": RougeMetric, + + "cola": ColaMetric, + "grammaticality": ColaMetric, + + "deepeval_answer_relevance": DeepEvalAnswerRelevancyMetric, + "deepeval_faithfulness": DeepEvalFaithfulnessMetric, + "deepeval_summarization": DeepEvalSummarizationMetric, + "deepeval_prompt_alignment": DeepEvalPromptAlignmentMetric, + "bigbench_hard": BigBenchHardMetric, + "big_bench_hard": BigBenchHardMetric, + } + + # Add semantic metrics if they imported successfully + if BartScoreMetric is not None: + registry["bartscore"] = BartScoreMetric + if AlignScoreMetric is not None: + registry["alignscore"] = AlignScoreMetric + if SentBertMetric is not None: + registry["sentbert"] = SentBertMetric + registry["sentence_bert"] = SentBertMetric + + cls._metric_registry = registry @classmethod def register_metric(cls, name: str, metric_class: type): """Register a new metric class.""" + cls._build_registry() cls._metric_registry[name] = metric_class logger.info(f"Registered metric: {name} -> {metric_class.__name__}") @classmethod def get_available_metrics(cls) -> List[str]: """Get list of all available metric names.""" + cls._build_registry() return list(cls._metric_registry.keys()) + @classmethod + def get_all_possible_metric_keys(cls) -> List[str]: + """ + Get list of all possible metric keys that can be returned by compute_metrics. + This includes not just metric names, but all the specific keys that metrics can return. + """ + cls._build_registry() + + # Base metric names + metric_keys = list(cls._metric_registry.keys()) + + # Add specific keys that metrics return (not just their names) + additional_keys = [ + # ROUGE variants + "rouge1", "rouge2", "rougeL", "rougeLsum", + + # SentBERT variants + "sentbert_pred_ref", "sentbert_pred_src", + + # BLEU + "bleu", + + # CoLA/Grammaticality + "cola", "grammaticality", "grammaticality_score", + + # BARTScore variants (if available) + "bartscore", "bartscore_pred_ref", "bartscore_pred_src", "bartscore_ref_pred", + + # AlignScore variants (if available) + "alignscore", "alignscore_pred_ref", "alignscore_pred_src", + + # DeepEval metrics + "deepeval_answer_relevance", "deepeval_faithfulness", + "deepeval_summarization", "deepeval_prompt_alignment", + + # BigBench variants + "bigbench_hard", "big_bench_hard", + + # Statistical metrics that compute_metrics always adds + "word_length_gen", "word_length_src_rel", "word_length_rel", "exact_match", + ] + + # Combine and deduplicate + all_keys = list(set(metric_keys + additional_keys)) + all_keys.sort() + + return all_keys + @classmethod def create_metric(cls, name: str, config: Optional[MetricConfig] = None) -> BaseMetric: - + cls._build_registry() name = name.lower().strip() if name not in cls._metric_registry: @@ -199,8 +262,9 @@ def get_metric_requirements() -> Dict[str, Dict[str, bool]]: "sentbert": {"requires_references": False, "requires_original_texts": False}, "cola": {"requires_references": False, "requires_original_texts": False}, "deepeval_answer_relevance": {"requires_references": False, "requires_original_texts": True}, - "deepeval_faithfulness": {"requires_references": False, "requires_original_texts": True}, - "deepeval_summarization": {"requires_references": False, "requires_original_texts": True}, - "deepeval_prompt_alignment": {"requires_references": True, "requires_original_texts": True}, + "deepeval_faithfulness": {"requires_references": True, "requires_original_texts": True}, + "deepeval_summarization": {"requires_references": True, "requires_original_texts": True}, + "deepeval_prompt_alignment": {"requires_references": False, "requires_original_texts": True}, "bigbench_hard": {"requires_references": False, "requires_original_texts": False}, + "big_bench_hard": {"requires_references": False, "requires_original_texts": False}, } \ No newline at end of file diff --git a/src/atgen/metrics/semantic/__init__.py b/src/atgen/metrics/semantic/__init__.py index 8d848a1..1d8da41 100644 --- a/src/atgen/metrics/semantic/__init__.py +++ b/src/atgen/metrics/semantic/__init__.py @@ -1,11 +1,27 @@ """Semantic metrics for text evaluation.""" -from .bart_score_metric import BartScoreMetric -from .alignscore_metric import AlignScoreMetric -from .sentbert_metric import SentBertMetric - -__all__ = [ - "BartScoreMetric", - "AlignScoreMetric", - "SentBertMetric", -] \ No newline at end of file +import warnings + +# Import with error handling for optional dependencies +__all__ = [] + +try: + from .bart_score_metric import BartScoreMetric + __all__.append("BartScoreMetric") +except ImportError as e: + warnings.warn(f"BartScore metric not available due to import error: {e}") + BartScoreMetric = None + +try: + from .alignscore_metric import AlignScoreMetric + __all__.append("AlignScoreMetric") +except ImportError as e: + warnings.warn(f"AlignScore metric not available due to import error: {e}") + AlignScoreMetric = None + +try: + from .sentbert_metric import SentBertMetric + __all__.append("SentBertMetric") +except ImportError as e: + warnings.warn(f"SentBert metric not available due to import error: {e}") + SentBertMetric = None \ No newline at end of file From ff3e0ab9606407b65c0245c99d2106b0420f981d Mon Sep 17 00:00:00 2001 From: Akim Tsvigun Date: Mon, 9 Jun 2025 14:37:46 +0000 Subject: [PATCH 24/39] fixed version for multi-ref datasets --- .../utils/data/get_output_column_name.py | 19 +++++++++++++++++++ .../utils/data/get_preprocess_function.py | 8 ++------ 2 files changed, 21 insertions(+), 6 deletions(-) create mode 100644 src/atgen/utils/data/get_output_column_name.py diff --git a/src/atgen/utils/data/get_output_column_name.py b/src/atgen/utils/data/get_output_column_name.py new file mode 100644 index 0000000..c3947f1 --- /dev/null +++ b/src/atgen/utils/data/get_output_column_name.py @@ -0,0 +1,19 @@ +from typing import Union +from omegaconf import DictConfig, ListConfig + +from ..constants import OUTPUT_FIELD_PURPOSE_TRAIN + + +def get_output_column_name( + output_column_name: Union[DictConfig, ListConfig, dict[str, Union[str, list[str]]], list[str], str], + purpose: str = OUTPUT_FIELD_PURPOSE_TRAIN, +) -> str: + if isinstance(output_column_name, (list, ListConfig)): + return '_'.join(output_column_name) + elif isinstance(output_column_name, (dict, DictConfig)): + return '_'.join(output_column_name[purpose]) + elif isinstance(output_column_name, str): + return output_column_name + else: + raise NotImplementedError(f"Unexpected type {type(output_column_name)} of the output column name.") + \ No newline at end of file diff --git a/src/atgen/utils/data/get_preprocess_function.py b/src/atgen/utils/data/get_preprocess_function.py index 0110654..6ae2182 100644 --- a/src/atgen/utils/data/get_preprocess_function.py +++ b/src/atgen/utils/data/get_preprocess_function.py @@ -90,9 +90,7 @@ def preprocess_fn(instance): elif split == "train": # For training, add the assistant's response if assistant_response_start: - assistant_message = ( - assistant_response_start + instance[output_column_name] - ) + assistant_message = assistant_response_start + instance[output_column_name] else: assistant_message = instance[output_column_name] messages.append({"role": "assistant", "content": assistant_message}) @@ -133,9 +131,7 @@ def preprocess_fn(instance): # For training, add the assistant's response if split == "train": if assistant_response_start: - assistant_message = ( - assistant_response_start + instance[output_column_name] - ) + assistant_message = assistant_response_start + instance[output_column_name] else: assistant_message = instance[output_column_name] messages.append({"role": "assistant", "content": assistant_message}) From e78d0be20b7ebc48b881984ae3011edb38e10f7e Mon Sep 17 00:00:00 2001 From: Akim Tsvigun Date: Thu, 12 Jun 2025 19:05:11 +0000 Subject: [PATCH 25/39] bugs fixed --- .../utils/data/get_output_column_name.py | 19 ------------------- src/atgen/utils/generate.py | 1 + src/atgen/utils/resolvers.py | 1 + 3 files changed, 2 insertions(+), 19 deletions(-) delete mode 100644 src/atgen/utils/data/get_output_column_name.py diff --git a/src/atgen/utils/data/get_output_column_name.py b/src/atgen/utils/data/get_output_column_name.py deleted file mode 100644 index c3947f1..0000000 --- a/src/atgen/utils/data/get_output_column_name.py +++ /dev/null @@ -1,19 +0,0 @@ -from typing import Union -from omegaconf import DictConfig, ListConfig - -from ..constants import OUTPUT_FIELD_PURPOSE_TRAIN - - -def get_output_column_name( - output_column_name: Union[DictConfig, ListConfig, dict[str, Union[str, list[str]]], list[str], str], - purpose: str = OUTPUT_FIELD_PURPOSE_TRAIN, -) -> str: - if isinstance(output_column_name, (list, ListConfig)): - return '_'.join(output_column_name) - elif isinstance(output_column_name, (dict, DictConfig)): - return '_'.join(output_column_name[purpose]) - elif isinstance(output_column_name, str): - return output_column_name - else: - raise NotImplementedError(f"Unexpected type {type(output_column_name)} of the output column name.") - \ No newline at end of file diff --git a/src/atgen/utils/generate.py b/src/atgen/utils/generate.py index ef080c2..87bf617 100644 --- a/src/atgen/utils/generate.py +++ b/src/atgen/utils/generate.py @@ -105,6 +105,7 @@ def generate_vllm( batch_generations = [x.outputs[0].text for x in out] generations += batch_generations + import pdb; pdb.set_trace() generations = post_process_generations( generations=generations, data_config=data_config, diff --git a/src/atgen/utils/resolvers.py b/src/atgen/utils/resolvers.py index b123220..983cfce 100644 --- a/src/atgen/utils/resolvers.py +++ b/src/atgen/utils/resolvers.py @@ -33,6 +33,7 @@ def register_resolvers() -> None: resolvers_to_register = { "multiply_with_few_shot": multiply_with_few_shot, "to_string": to_string, + "get_output_column_name_for_phase": get_output_column_name_for_phase, } for name, resolver_fn in resolvers_to_register.items(): From 747ba90c6ee1aabb120117f34b9ead74e475d969 Mon Sep 17 00:00:00 2001 From: SpeedOfMagic Date: Sun, 8 Jun 2025 22:39:25 +0000 Subject: [PATCH 26/39] dev2 --- install.sh | 37 -------------------- pyproject.toml | 4 +-- requirements.txt | 1 + src/atgen/metrics/metrics.py | 2 +- src/atgen/run_scripts/run_active_learning.py | 5 +++ src/atgen/utils/installers.py | 10 ++++++ 6 files changed, 19 insertions(+), 40 deletions(-) delete mode 100755 install.sh create mode 100644 src/atgen/utils/installers.py diff --git a/install.sh b/install.sh deleted file mode 100755 index 2a30fa3..0000000 --- a/install.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash - -# The PyPI version submodlib seems to be broken, so -# one has to resort to manual installation. -mkdir external_metrics -cd external_metrics -echo "Installing submodlib" -git clone https://github.com/decile-team/submodlib.git -cd submodlib -# Cannot run `pip install -r requirements.txt` due to versions mismatch -pip install sphinxcontrib-bibtex pybind11>=2.6.0 scikit-learn scipy -pip install . --no-deps -cd .. -rm -rf submodlib - -# Install external metrics -echo "Installing AlignScore..." -git clone https://github.com/yuh-zha/AlignScore -cd AlignScore - -# Fix the classmethod bug in inference.py -echo "Patching AlignScore code to fix classmethod bug..." -sed -i 's/BERTAlignModel(model=model).load_from_checkpoint(checkpoint_path=ckpt_path, strict=False)/BERTAlignModel.load_from_checkpoint(checkpoint_path=ckpt_path, model=model, strict=False)/g' src/alignscore/inference.py - -pip install . --no-deps -mkdir model -wget https://huggingface.co/yzha/AlignScore/resolve/main/AlignScore-base.ckpt -P model -cd ../.. - -echo "Installing the package..." -pip install -e . - -echo "Downloading NLP packages data..." -python -m spacy download en_core_web_sm -python3 -c "import nltk ; nltk.download('punkt'); nltk.download('punkt_tab')" - -echo "Done!" diff --git a/pyproject.toml b/pyproject.toml index 125ac23..23b4409 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "atgen" -version = "0.0.0" +version = "0.0.1.dev2" authors = [ { name = "List of contributors: https://github.com/Aktsvigun/atgen/graphs/contributors", email = "artemshelmanov@gmail.com" }, ] @@ -12,7 +12,7 @@ description = "Toolkit for Active Learning in Generative Tasks" readme = "README.md" keywords = ["NLP", "deep learning", "transformer", "pytorch", "peft", "inference", "active learning"] -license = {file = "LICENSE.md"} +license-files = ["LICENSE.md"] requires-python = ">=3.10" dynamic = ["dependencies"] diff --git a/requirements.txt b/requirements.txt index 11b1ce5..216563c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +alignscore-SpeedOfMagic accelerate==1.5.2 anthropic==0.49.0 benepar==0.2.0 diff --git a/src/atgen/metrics/metrics.py b/src/atgen/metrics/metrics.py index a22ec76..52f9504 100644 --- a/src/atgen/metrics/metrics.py +++ b/src/atgen/metrics/metrics.py @@ -62,7 +62,7 @@ # Going up 3 levels from metrics.py: src/atgen/metrics -> repository root os.path.join( Path(__file__).parents[3], - "external_metrics/AlignScore/model/AlignScore-base.ckpt", + "cache/AlignScore-base.ckpt", ), ) diff --git a/src/atgen/run_scripts/run_active_learning.py b/src/atgen/run_scripts/run_active_learning.py index 728f1ef..a3747a1 100644 --- a/src/atgen/run_scripts/run_active_learning.py +++ b/src/atgen/run_scripts/run_active_learning.py @@ -32,6 +32,7 @@ def run_active_learning(config, workdir: Union[str, Path]): maybe_get_few_shot_examples, get_output_column_name_for_phase, ) + from atgen.utils.installers import install_spacy, install_nltk from atgen.utils.load_model_tokenizer import load_model_tokenizer from atgen.utils.prepare_model_for_training import prepare_model_for_training from atgen.utils.training_utils import get_trainer @@ -50,6 +51,10 @@ def run_active_learning(config, workdir: Union[str, Path]): check_performance_against_requirements, ) + # TODO Figure out how to stop downloading it every time + install_spacy() + install_nltk() + seed = config.seed cache_dir = config.cache_dir input_column_name = config.data.input_column_name diff --git a/src/atgen/utils/installers.py b/src/atgen/utils/installers.py new file mode 100644 index 0000000..2dda0ef --- /dev/null +++ b/src/atgen/utils/installers.py @@ -0,0 +1,10 @@ +import subprocess + + +def install_spacy(self): + subprocess.run("python -m spacy download en_core_web_sm", shell=True) + +def install_nltk(self): + import nltk + nltk.download('punkt') + nltk.download('punkt_tab') From e96c3dc1c54c66e900f2010160bb5ce8610f368a Mon Sep 17 00:00:00 2001 From: SpeedOfMagic Date: Fri, 13 Jun 2025 15:55:41 +0000 Subject: [PATCH 27/39] fix --- src/atgen/utils/installers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/atgen/utils/installers.py b/src/atgen/utils/installers.py index 2dda0ef..b540fb9 100644 --- a/src/atgen/utils/installers.py +++ b/src/atgen/utils/installers.py @@ -1,10 +1,10 @@ import subprocess -def install_spacy(self): +def install_spacy(): subprocess.run("python -m spacy download en_core_web_sm", shell=True) -def install_nltk(self): +def install_nltk(): import nltk nltk.download('punkt') nltk.download('punkt_tab') From ddbdbf1e50e73dff10a87103be505037a07bb49b Mon Sep 17 00:00:00 2001 From: SpeedOfMagic Date: Fri, 13 Jun 2025 15:57:29 +0000 Subject: [PATCH 28/39] fix --- README.md | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index e387c5f..578c617 100644 --- a/README.md +++ b/README.md @@ -21,21 +21,7 @@ A comprehensive toolkit for applying active learning techniques to natural langu ## šŸ”§ Installation -1. Clone the repository: - ```bash - git clone https://github.com/Aktsvigun/atgen.git - cd atgen - ``` - -2. Run the installation script: - ```bash - bash install.sh - ``` - - This will install: - - All required Python packages - - External metrics (submodlib, AlignScore) - - Required NLP resources +`pip install atgen` ## šŸš€ Usage @@ -115,4 +101,4 @@ If you use this toolkit in your research, please cite: url = {https://github.com/Aktsvigun/atgen}, year = {2025}, } -``` \ No newline at end of file +``` From 876b03ce1652c3e5c5de9072094155fdc0073d9a Mon Sep 17 00:00:00 2001 From: SpeedOfMagic Date: Fri, 13 Jun 2025 15:57:45 +0000 Subject: [PATCH 29/39] fix --- .github/workflows/python-app.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index adae95b..822e71e 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -26,9 +26,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - chmod +x install.sh pip install -e . - ./install.sh pip install flake8 pytest - name: Test with pytest run: | From d56d9d86fe6bf6925ef3a455c281e00ac14d70f7 Mon Sep 17 00:00:00 2001 From: SpeedOfMagic Date: Fri, 13 Jun 2025 16:06:06 +0000 Subject: [PATCH 30/39] fix version --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 23b4409..bcfe10d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "atgen" -version = "0.0.1.dev2" +version = "0.1.0" authors = [ { name = "List of contributors: https://github.com/Aktsvigun/atgen/graphs/contributors", email = "artemshelmanov@gmail.com" }, ] From 8de8055841258c4d1f542725edf7dae9a2bbb16c Mon Sep 17 00:00:00 2001 From: alfekka Date: Mon, 9 Jun 2025 00:56:17 +0300 Subject: [PATCH 31/39] added default factory --- src/atgen/metrics/base/base_metric.py | 6 ++++++ src/atgen/metrics/lexical/bleu_metric.py | 14 +++++++++++++- src/atgen/metrics/semantic/sentbert_metric.py | 4 ++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/src/atgen/metrics/base/base_metric.py b/src/atgen/metrics/base/base_metric.py index 0bd1d66..1c844f3 100644 --- a/src/atgen/metrics/base/base_metric.py +++ b/src/atgen/metrics/base/base_metric.py @@ -14,8 +14,11 @@ class MetricConfig: checkpoint: Optional[str] = None model_name: Optional[str] = None +<<<<<<< HEAD # API-based parameters (for LLM-based metrics) provider: Optional[str] = None +======= +>>>>>>> dabcd83 (added default factory) api_key: Optional[str] = None base_url: Optional[str] = None model: Optional[str] = None @@ -47,8 +50,11 @@ def _validate_config(self): @abstractmethod def calculate(self, predictions, references, original_texts): pass +<<<<<<< HEAD def is_available(self) -> bool: """Check if the metric is available for use.""" return True +======= +>>>>>>> dabcd83 (added default factory) \ No newline at end of file diff --git a/src/atgen/metrics/lexical/bleu_metric.py b/src/atgen/metrics/lexical/bleu_metric.py index d6af063..77e8c5c 100644 --- a/src/atgen/metrics/lexical/bleu_metric.py +++ b/src/atgen/metrics/lexical/bleu_metric.py @@ -47,7 +47,11 @@ def supports_multiple_references(self) -> bool: return True +<<<<<<< HEAD def calculate( +======= + def _compute( +>>>>>>> dabcd83 (added default factory) self, predictions: List[str], references: Optional[List[Union[str, List[str]]]] = None, @@ -76,15 +80,23 @@ def calculate( tok_pred = word_tokenize(pred) try: +<<<<<<< HEAD score = corpus_bleu([tok_ref], [tok_pred], smoothing_function=smoothing_function) +======= + score = corpus_bleu(tok_ref, [tok_pred], smoothing_function=smoothing_function) +>>>>>>> dabcd83 (added default factory) scores.append(score) except (KeyError, ZeroDivisionError): scores.append(0.0) +<<<<<<< HEAD scores_dict = {"bleu": np.array(scores)} # Apply aggregation if requested if self.config.aggregate: scores_dict = {key: float(np.mean(value)) for key, value in scores_dict.items()} - return scores_dict \ No newline at end of file + return scores_dict +======= + return {"bleu": np.array(scores)} +>>>>>>> dabcd83 (added default factory) diff --git a/src/atgen/metrics/semantic/sentbert_metric.py b/src/atgen/metrics/semantic/sentbert_metric.py index d872405..b86740a 100644 --- a/src/atgen/metrics/semantic/sentbert_metric.py +++ b/src/atgen/metrics/semantic/sentbert_metric.py @@ -13,7 +13,11 @@ def __init__(self, config: Optional[MetricConfig] = None): super().__init__(config) self.model = None self.tokenizer = None +<<<<<<< HEAD self.checkpoint = (config.checkpoint if config and config.checkpoint else 'sentence-transformers/all-mpnet-base-v2') +======= + self.checkpoint = getattr(config, 'checkpoint', 'sentence-transformers/all-mpnet-base-v2') if config else 'sentence-transformers/all-mpnet-base-v2' +>>>>>>> dabcd83 (added default factory) def _initialize_model(self): From 8c079dc73b96e9690c66e1348c303a58109cc706 Mon Sep 17 00:00:00 2001 From: Akim Tsvigun Date: Mon, 9 Jun 2025 14:37:46 +0000 Subject: [PATCH 32/39] fixed version for multi-ref datasets --- .../utils/data/get_output_column_name.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 src/atgen/utils/data/get_output_column_name.py diff --git a/src/atgen/utils/data/get_output_column_name.py b/src/atgen/utils/data/get_output_column_name.py new file mode 100644 index 0000000..c3947f1 --- /dev/null +++ b/src/atgen/utils/data/get_output_column_name.py @@ -0,0 +1,19 @@ +from typing import Union +from omegaconf import DictConfig, ListConfig + +from ..constants import OUTPUT_FIELD_PURPOSE_TRAIN + + +def get_output_column_name( + output_column_name: Union[DictConfig, ListConfig, dict[str, Union[str, list[str]]], list[str], str], + purpose: str = OUTPUT_FIELD_PURPOSE_TRAIN, +) -> str: + if isinstance(output_column_name, (list, ListConfig)): + return '_'.join(output_column_name) + elif isinstance(output_column_name, (dict, DictConfig)): + return '_'.join(output_column_name[purpose]) + elif isinstance(output_column_name, str): + return output_column_name + else: + raise NotImplementedError(f"Unexpected type {type(output_column_name)} of the output column name.") + \ No newline at end of file From df81546c53848b187bd2741ad73c5cc6b6ed86d6 Mon Sep 17 00:00:00 2001 From: alfekka Date: Fri, 13 Jun 2025 20:12:57 +0300 Subject: [PATCH 33/39] fixes of metrics --- .deepeval-cache.json | 2 +- src/atgen/metrics/compute_metrics.py | 6 +++--- test/test_metrics.py | 32 ++++++++++++++++------------ 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/.deepeval-cache.json b/.deepeval-cache.json index 82cd467..18605fe 100644 --- a/.deepeval-cache.json +++ b/.deepeval-cache.json @@ -1 +1 @@ -{"test_cases_lookup_map": {"{\"actual_output\": \"Machine learning models learn patterns from training data to make predictions.\", \"context\": null, \"expected_output\": null, \"hyperparameters\": null, \"input\": \"How do machine learning models work?\", \"retrieval_context\": null}": {"cached_metrics_data": [{"metric_data": {"name": "Answer Relevancy", "threshold": 0.5, "success": true, "score": 1.0, "strictMode": false, "evaluationModel": "EvaluationLLM: openai/gpt-4o-mini", "evaluationCost": 0, "verboseLogs": "Statements:\n[\n \"Machine learning models learn patterns from training data.\",\n \"Machine learning models make predictions.\"\n] \n \nVerdicts:\n[\n {\n \"verdict\": \"yes\",\n \"reason\": null\n },\n {\n \"verdict\": \"yes\",\n \"reason\": null\n }\n]"}, "metric_configuration": {"threshold": 0.5, "evaluation_model": "EvaluationLLM: openai/gpt-4o-mini", "strict_mode": false, "include_reason": false}}]}, "{\"actual_output\": \"Deep learning uses neural networks with multiple layers to process information.\", \"context\": null, \"expected_output\": null, \"hyperparameters\": null, \"input\": \"What is deep learning?\", \"retrieval_context\": null}": {"cached_metrics_data": [{"metric_data": {"name": "Answer Relevancy", "threshold": 0.5, "success": true, "score": 1.0, "strictMode": false, "evaluationModel": "EvaluationLLM: openai/gpt-4o-mini", "evaluationCost": 0, "verboseLogs": "Statements:\n[\n \"Deep learning uses neural networks.\",\n \"Neural networks have multiple layers.\",\n \"Neural networks process information.\"\n] \n \nVerdicts:\n[\n {\n \"verdict\": \"yes\",\n \"reason\": null\n },\n {\n \"verdict\": \"yes\",\n \"reason\": null\n },\n {\n \"verdict\": \"yes\",\n \"reason\": null\n }\n]"}, "metric_configuration": {"threshold": 0.5, "evaluation_model": "EvaluationLLM: openai/gpt-4o-mini", "strict_mode": false, "include_reason": false}}]}, "{\"actual_output\": \"Natural language processing helps computers understand and generate human language.\", \"context\": null, \"expected_output\": null, \"hyperparameters\": null, \"input\": \"Explain natural language processing.\", \"retrieval_context\": null}": {"cached_metrics_data": [{"metric_data": {"name": "Answer Relevancy", "threshold": 0.5, "success": true, "score": 1.0, "strictMode": false, "evaluationModel": "EvaluationLLM: openai/gpt-4o-mini", "evaluationCost": 0, "verboseLogs": "Statements:\n[\n \"Natural language processing helps computers understand human language.\",\n \"Natural language processing helps computers generate human language.\"\n] \n \nVerdicts:\n[\n {\n \"verdict\": \"yes\",\n \"reason\": null\n },\n {\n \"verdict\": \"yes\",\n \"reason\": null\n }\n]"}, "metric_configuration": {"threshold": 0.5, "evaluation_model": "EvaluationLLM: openai/gpt-4o-mini", "strict_mode": false, "include_reason": false}}]}, "{\"actual_output\": \"The capital of France is Paris.\", \"context\": null, \"expected_output\": null, \"hyperparameters\": null, \"input\": \"What is the capital of France?\", \"retrieval_context\": null}": {"cached_metrics_data": [{"metric_data": {"name": "Answer Relevancy", "threshold": 0.5, "success": true, "score": 1.0, "strictMode": false, "evaluationModel": "EvaluationLLM: openai/gpt-4o-mini", "evaluationCost": 0, "verboseLogs": "Statements:\n[\n \"The capital of France is Paris.\"\n] \n \nVerdicts:\n[\n {\n \"verdict\": \"yes\",\n \"reason\": null\n }\n]"}, "metric_configuration": {"threshold": 0.5, "evaluation_model": "EvaluationLLM: openai/gpt-4o-mini", "strict_mode": false, "include_reason": false}}]}, "{\"actual_output\": \"The capital of France is Paris.\", \"context\": null, \"expected_output\": null, \"hyperparameters\": null, \"input\": \"What is the capital of France?\", \"retrieval_context\": [\"What is the capital of France?\"]}": {"cached_metrics_data": [{"metric_data": {"name": "Faithfulness", "threshold": 0.5, "success": true, "score": 1.0, "strictMode": false, "evaluationModel": "EvaluationLLM: openai/gpt-4o-mini", "evaluationCost": 0, "verboseLogs": "Truths (limit=None):\n[\n \"The capital of France is Paris.\"\n] \n \nClaims:\n[\n \"The capital of France is Paris.\"\n] \n \nVerdicts:\n[\n {\n \"verdict\": \"yes\",\n \"reason\": null\n }\n]"}, "metric_configuration": {"threshold": 0.5, "evaluation_model": "EvaluationLLM: openai/gpt-4o-mini", "strict_mode": false, "include_reason": false}}]}}} \ No newline at end of file +{"test_cases_lookup_map": {"{\"actual_output\": \"The capital of France is Paris, a beautiful city known for its culture and history.\", \"context\": null, \"expected_output\": null, \"hyperparameters\": null, \"input\": \"What is the capital of France?\", \"retrieval_context\": null}": {"cached_metrics_data": [{"metric_data": {"name": "Answer Relevancy", "threshold": 0.5, "success": false, "score": 0.25, "strictMode": false, "evaluationModel": "EvaluationLLM: openai/gpt-4o-mini", "evaluationCost": 0, "verboseLogs": "Statements:\n[\n \"The capital of France is Paris.\",\n \"Paris is known for its culture.\",\n \"Paris is known for its history.\",\n \"Paris is a beautiful city.\"\n] \n \nVerdicts:\n[\n {\n \"verdict\": \"yes\",\n \"reason\": null\n },\n {\n \"verdict\": \"no\",\n \"reason\": \"The statement about Paris's culture is not directly answering what the capital of France is.\"\n },\n {\n \"verdict\": \"no\",\n \"reason\": \"The statement about Paris's history does not directly state that it is the capital of France.\"\n },\n {\n \"verdict\": \"no\",\n \"reason\": \"The statement about Paris being a beautiful city does not answer the question regarding its status as the capital of France.\"\n }\n]"}, "metric_configuration": {"threshold": 0.5, "evaluation_model": "EvaluationLLM: openai/gpt-4o-mini", "strict_mode": false, "include_reason": false}}]}, "{\"actual_output\": \"Machine learning algorithms can learn patterns from data to make accurate predictions.\", \"context\": null, \"expected_output\": null, \"hyperparameters\": null, \"input\": \"How do machine learning algorithms work?\", \"retrieval_context\": null}": {"cached_metrics_data": [{"metric_data": {"name": "Answer Relevancy", "threshold": 0.5, "success": true, "score": 1.0, "strictMode": false, "evaluationModel": "EvaluationLLM: openai/gpt-4o-mini", "evaluationCost": 0, "verboseLogs": "Statements:\n[\n \"Machine learning algorithms can learn patterns from data.\",\n \"Machine learning algorithms can make accurate predictions.\"\n] \n \nVerdicts:\n[\n {\n \"verdict\": \"yes\",\n \"reason\": null\n },\n {\n \"verdict\": \"yes\",\n \"reason\": null\n }\n]"}, "metric_configuration": {"threshold": 0.5, "evaluation_model": "EvaluationLLM: openai/gpt-4o-mini", "strict_mode": false, "include_reason": false}}]}}} \ No newline at end of file diff --git a/src/atgen/metrics/compute_metrics.py b/src/atgen/metrics/compute_metrics.py index fe07684..7532d2f 100644 --- a/src/atgen/metrics/compute_metrics.py +++ b/src/atgen/metrics/compute_metrics.py @@ -59,7 +59,7 @@ def compute_metrics( if config is None: # Default configuration metrics_config = MetricsConfig( - metrics=["bleu", "rouge"], + metrics=["bleu", "rouge1"], batch_size=32, device="cuda", aggregate=True @@ -197,7 +197,7 @@ def compute_metrics_from_config( def get_default_config() -> MetricsConfig: """Get a default metrics configuration.""" return MetricsConfig( - metrics=["bleu", "rouge"], + metrics=["bleu", "rouge1"], batch_size=32, device="cuda", cache_dir="cache", @@ -209,7 +209,7 @@ def get_comprehensive_config() -> MetricsConfig: """Get a comprehensive metrics configuration with all metrics.""" return MetricsConfig( metrics=[ - "bleu", "rouge", + "bleu", "rouge1", "rouge2", "rougeL", "sentbert", "cola", ], diff --git a/test/test_metrics.py b/test/test_metrics.py index 42382c3..2a38532 100644 --- a/test/test_metrics.py +++ b/test/test_metrics.py @@ -110,7 +110,7 @@ def test_get_available_metrics(self): assert isinstance(metrics, list) assert len(metrics) > 0 assert "bleu" in metrics - assert "rouge" in metrics + assert "rouge1" in metrics assert "sentbert" in metrics assert "cola" in metrics @@ -137,21 +137,21 @@ def test_create_metric_unknown(self): def test_create_metrics_from_config(self): """Test creating multiple metrics from config.""" config = MetricsConfig( - metrics=["bleu", "rouge", "sentbert"], + metrics=["bleu", "rouge1", "sentbert"], device="cpu" ) metrics = MetricsFactory.create_metrics(config) assert len(metrics) == 3 assert "bleu" in metrics - assert "rouge" in metrics + assert "rouge1" in metrics assert "sentbert" in metrics assert all(isinstance(m, BaseMetric) for m in metrics.values()) def test_create_from_config_dict(self): """Test creating metrics from dictionary config.""" config_dict = { - "metrics": ["bleu", "rouge"], + "metrics": ["bleu", "rouge1"], "device": "cpu", "batch_size": 16 } @@ -159,7 +159,7 @@ def test_create_from_config_dict(self): assert len(metrics) == 2 assert "bleu" in metrics - assert "rouge" in metrics + assert "rouge1" in metrics def test_register_custom_metric(self): """Test registering a custom metric.""" @@ -307,6 +307,10 @@ def test_bartscore_identical_strings(self, mock_cuda, identical_data, temp_cache @patch('torch.cuda.is_available', return_value=False) def test_alignscore_identical_strings(self, mock_cuda, identical_data, temp_cache_dir): """Test AlignScore metric with identical strings.""" + # Skip test if AlignScore is not available + if AlignScoreMetric is None: + pytest.skip("AlignScore metric not available due to import issues") + config = MetricConfig(device="cpu", cache_dir=temp_cache_dir, batch_size=2) metric = AlignScoreMetric(config) @@ -376,7 +380,7 @@ def test_compute_metrics_default_config(self, sample_data, temp_cache_dir): def test_compute_metrics_custom_config(self, sample_data, temp_cache_dir): """Test compute_metrics with custom configuration.""" config = MetricsConfig( - metrics=["bleu", "rouge"], + metrics=["bleu", "rouge1"], device="cpu", cache_dir=temp_cache_dir, batch_size=16 @@ -420,7 +424,7 @@ def test_compute_metrics_empty_predictions(self): def test_compute_metrics_from_config(self, sample_data, temp_cache_dir): """Test compute_metrics_from_config function.""" config_dict = { - "metrics": ["bleu", "rouge"], + "metrics": ["bleu", "rouge1"], "device": "cpu", "cache_dir": temp_cache_dir } @@ -481,7 +485,7 @@ def test_get_default_config(self): assert isinstance(config, MetricsConfig) assert "bleu" in config.metrics - assert "rouge" in config.metrics + assert "rouge1" in config.metrics assert config.batch_size == 32 assert config.device == "cuda" @@ -492,7 +496,7 @@ def test_get_comprehensive_config(self): assert isinstance(config, MetricsConfig) assert len(config.metrics) > 2 # Should include more metrics assert "bleu" in config.metrics - assert "rouge" in config.metrics + assert any("rouge" in metric for metric in config.metrics) assert "sentbert" in config.metrics assert "cola" in config.metrics @@ -510,7 +514,7 @@ def temp_cache_dir(self): def test_empty_strings(self, temp_cache_dir): """Test metrics with empty strings.""" config = MetricsConfig( - metrics=["bleu", "rouge"], + metrics=["bleu", "rouge1"], device="cpu", cache_dir=temp_cache_dir ) @@ -528,7 +532,7 @@ def test_empty_strings(self, temp_cache_dir): def test_single_item_lists(self, temp_cache_dir): """Test metrics with single item lists.""" config = MetricsConfig( - metrics=["bleu", "rouge"], + metrics=["bleu", "rouge1"], device="cpu", cache_dir=temp_cache_dir ) @@ -546,7 +550,7 @@ def test_single_item_lists(self, temp_cache_dir): def test_multiple_references(self, temp_cache_dir): """Test metrics with multiple references.""" config = MetricsConfig( - metrics=["bleu", "rouge"], + metrics=["bleu", "rouge1"], device="cpu", cache_dir=temp_cache_dir ) @@ -597,7 +601,7 @@ def temp_cache_dir(self): def test_all_lexical_metrics(self, temp_cache_dir): """Test all lexical metrics together.""" config = MetricsConfig( - metrics=["bleu", "rouge"], + metrics=["bleu", "rouge1"], device="cpu", cache_dir=temp_cache_dir, aggregate=True @@ -622,7 +626,7 @@ def test_all_lexical_metrics(self, temp_cache_dir): def test_comprehensive_metrics_suite(self, mock_cuda, temp_cache_dir): """Test a comprehensive suite of metrics.""" config = MetricsConfig( - metrics=["bleu", "rouge", "sentbert", "cola"], + metrics=["bleu", "rouge1", "sentbert", "cola"], device="cpu", cache_dir=temp_cache_dir, batch_size=8, From 2a1ab46bf12fd7623161087508cd7cc50ac24b74 Mon Sep 17 00:00:00 2001 From: alfekka Date: Fri, 13 Jun 2025 20:31:49 +0300 Subject: [PATCH 34/39] fixes of metrics --- src/atgen/metrics/compute_metrics.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/atgen/metrics/compute_metrics.py b/src/atgen/metrics/compute_metrics.py index 7532d2f..fe07684 100644 --- a/src/atgen/metrics/compute_metrics.py +++ b/src/atgen/metrics/compute_metrics.py @@ -59,7 +59,7 @@ def compute_metrics( if config is None: # Default configuration metrics_config = MetricsConfig( - metrics=["bleu", "rouge1"], + metrics=["bleu", "rouge"], batch_size=32, device="cuda", aggregate=True @@ -197,7 +197,7 @@ def compute_metrics_from_config( def get_default_config() -> MetricsConfig: """Get a default metrics configuration.""" return MetricsConfig( - metrics=["bleu", "rouge1"], + metrics=["bleu", "rouge"], batch_size=32, device="cuda", cache_dir="cache", @@ -209,7 +209,7 @@ def get_comprehensive_config() -> MetricsConfig: """Get a comprehensive metrics configuration with all metrics.""" return MetricsConfig( metrics=[ - "bleu", "rouge1", "rouge2", "rougeL", + "bleu", "rouge", "sentbert", "cola", ], From ce4185c779e6a0a0050e5af0f83bd2236d9f8120 Mon Sep 17 00:00:00 2001 From: alfekka Date: Fri, 13 Jun 2025 20:57:00 +0300 Subject: [PATCH 35/39] fixes of tests --- .deepeval-cache.json | 2 +- src/atgen/metrics/__init__.py | 2 +- src/atgen/metrics/base/base_metric.py | 8 +- src/atgen/metrics/lexical/bleu_metric.py | 12 - src/atgen/metrics/semantic/sentbert_metric.py | 4 - test/test_metrics.py | 639 ++++++++++-------- 6 files changed, 359 insertions(+), 308 deletions(-) diff --git a/.deepeval-cache.json b/.deepeval-cache.json index 18605fe..8e33836 100644 --- a/.deepeval-cache.json +++ b/.deepeval-cache.json @@ -1 +1 @@ -{"test_cases_lookup_map": {"{\"actual_output\": \"The capital of France is Paris, a beautiful city known for its culture and history.\", \"context\": null, \"expected_output\": null, \"hyperparameters\": null, \"input\": \"What is the capital of France?\", \"retrieval_context\": null}": {"cached_metrics_data": [{"metric_data": {"name": "Answer Relevancy", "threshold": 0.5, "success": false, "score": 0.25, "strictMode": false, "evaluationModel": "EvaluationLLM: openai/gpt-4o-mini", "evaluationCost": 0, "verboseLogs": "Statements:\n[\n \"The capital of France is Paris.\",\n \"Paris is known for its culture.\",\n \"Paris is known for its history.\",\n \"Paris is a beautiful city.\"\n] \n \nVerdicts:\n[\n {\n \"verdict\": \"yes\",\n \"reason\": null\n },\n {\n \"verdict\": \"no\",\n \"reason\": \"The statement about Paris's culture is not directly answering what the capital of France is.\"\n },\n {\n \"verdict\": \"no\",\n \"reason\": \"The statement about Paris's history does not directly state that it is the capital of France.\"\n },\n {\n \"verdict\": \"no\",\n \"reason\": \"The statement about Paris being a beautiful city does not answer the question regarding its status as the capital of France.\"\n }\n]"}, "metric_configuration": {"threshold": 0.5, "evaluation_model": "EvaluationLLM: openai/gpt-4o-mini", "strict_mode": false, "include_reason": false}}]}, "{\"actual_output\": \"Machine learning algorithms can learn patterns from data to make accurate predictions.\", \"context\": null, \"expected_output\": null, \"hyperparameters\": null, \"input\": \"How do machine learning algorithms work?\", \"retrieval_context\": null}": {"cached_metrics_data": [{"metric_data": {"name": "Answer Relevancy", "threshold": 0.5, "success": true, "score": 1.0, "strictMode": false, "evaluationModel": "EvaluationLLM: openai/gpt-4o-mini", "evaluationCost": 0, "verboseLogs": "Statements:\n[\n \"Machine learning algorithms can learn patterns from data.\",\n \"Machine learning algorithms can make accurate predictions.\"\n] \n \nVerdicts:\n[\n {\n \"verdict\": \"yes\",\n \"reason\": null\n },\n {\n \"verdict\": \"yes\",\n \"reason\": null\n }\n]"}, "metric_configuration": {"threshold": 0.5, "evaluation_model": "EvaluationLLM: openai/gpt-4o-mini", "strict_mode": false, "include_reason": false}}]}}} \ No newline at end of file +{"test_cases_lookup_map": {"{\"actual_output\": \"This is a test.\", \"context\": null, \"expected_output\": null, \"hyperparameters\": null, \"input\": \"Source text one.\", \"retrieval_context\": null}": {"cached_metrics_data": [{"metric_data": {"name": "Answer Relevancy", "threshold": 0.5, "success": false, "score": 0.0, "strictMode": false, "evaluationModel": "EvaluationLLM: openai/gpt-4o-mini", "evaluationCost": 0, "verboseLogs": "Statements:\n[\n \"This is a test.\"\n] \n \nVerdicts:\n[\n {\n \"verdict\": \"no\",\n \"reason\": \"The statement does not provide any relevant information related to the input, which appears to have no context or meaning related to 'Source text one.'\"\n }\n]"}, "metric_configuration": {"threshold": 0.5, "evaluation_model": "EvaluationLLM: openai/gpt-4o-mini", "strict_mode": false, "include_reason": false}}]}, "{\"actual_output\": \"Another test.\", \"context\": null, \"expected_output\": null, \"hyperparameters\": null, \"input\": \"Source text two.\", \"retrieval_context\": null}": {"cached_metrics_data": [{"metric_data": {"name": "Answer Relevancy", "threshold": 0.5, "success": false, "score": 0.0, "strictMode": false, "evaluationModel": "EvaluationLLM: openai/gpt-4o-mini", "evaluationCost": 0, "verboseLogs": "Statements:\n[\n \"Another test.\"\n] \n \nVerdicts:\n[\n {\n \"verdict\": \"no\",\n \"reason\": \"The statement 'Another test.' does not provide any relevant information to the input 'Source text two.'\"\n }\n]"}, "metric_configuration": {"threshold": 0.5, "evaluation_model": "EvaluationLLM: openai/gpt-4o-mini", "strict_mode": false, "include_reason": false}}]}}} \ No newline at end of file diff --git a/src/atgen/metrics/__init__.py b/src/atgen/metrics/__init__.py index 695aaff..8f9078e 100644 --- a/src/atgen/metrics/__init__.py +++ b/src/atgen/metrics/__init__.py @@ -119,4 +119,4 @@ def create_metrics_from_config(config): METRICS = MetricsFactory.get_available_metrics() # But AVAILABLE_METRICS should include all possible keys for performance checking -AVAILABLE_METRICS = MetricsFactory.get_all_possible_metric_keys() \ No newline at end of file +AVAILABLE_METRICS = MetricsFactory.get_all_possible_metric_keys() diff --git a/src/atgen/metrics/base/base_metric.py b/src/atgen/metrics/base/base_metric.py index 1c844f3..c3473c6 100644 --- a/src/atgen/metrics/base/base_metric.py +++ b/src/atgen/metrics/base/base_metric.py @@ -14,11 +14,8 @@ class MetricConfig: checkpoint: Optional[str] = None model_name: Optional[str] = None -<<<<<<< HEAD - # API-based parameters (for LLM-based metrics) + # API configuration parameters provider: Optional[str] = None -======= ->>>>>>> dabcd83 (added default factory) api_key: Optional[str] = None base_url: Optional[str] = None model: Optional[str] = None @@ -50,11 +47,8 @@ def _validate_config(self): @abstractmethod def calculate(self, predictions, references, original_texts): pass -<<<<<<< HEAD def is_available(self) -> bool: """Check if the metric is available for use.""" return True -======= ->>>>>>> dabcd83 (added default factory) \ No newline at end of file diff --git a/src/atgen/metrics/lexical/bleu_metric.py b/src/atgen/metrics/lexical/bleu_metric.py index 77e8c5c..91355c8 100644 --- a/src/atgen/metrics/lexical/bleu_metric.py +++ b/src/atgen/metrics/lexical/bleu_metric.py @@ -47,11 +47,7 @@ def supports_multiple_references(self) -> bool: return True -<<<<<<< HEAD def calculate( -======= - def _compute( ->>>>>>> dabcd83 (added default factory) self, predictions: List[str], references: Optional[List[Union[str, List[str]]]] = None, @@ -80,16 +76,11 @@ def _compute( tok_pred = word_tokenize(pred) try: -<<<<<<< HEAD score = corpus_bleu([tok_ref], [tok_pred], smoothing_function=smoothing_function) -======= - score = corpus_bleu(tok_ref, [tok_pred], smoothing_function=smoothing_function) ->>>>>>> dabcd83 (added default factory) scores.append(score) except (KeyError, ZeroDivisionError): scores.append(0.0) -<<<<<<< HEAD scores_dict = {"bleu": np.array(scores)} # Apply aggregation if requested @@ -97,6 +88,3 @@ def _compute( scores_dict = {key: float(np.mean(value)) for key, value in scores_dict.items()} return scores_dict -======= - return {"bleu": np.array(scores)} ->>>>>>> dabcd83 (added default factory) diff --git a/src/atgen/metrics/semantic/sentbert_metric.py b/src/atgen/metrics/semantic/sentbert_metric.py index b86740a..d872405 100644 --- a/src/atgen/metrics/semantic/sentbert_metric.py +++ b/src/atgen/metrics/semantic/sentbert_metric.py @@ -13,11 +13,7 @@ def __init__(self, config: Optional[MetricConfig] = None): super().__init__(config) self.model = None self.tokenizer = None -<<<<<<< HEAD self.checkpoint = (config.checkpoint if config and config.checkpoint else 'sentence-transformers/all-mpnet-base-v2') -======= - self.checkpoint = getattr(config, 'checkpoint', 'sentence-transformers/all-mpnet-base-v2') if config else 'sentence-transformers/all-mpnet-base-v2' ->>>>>>> dabcd83 (added default factory) def _initialize_model(self): diff --git a/test/test_metrics.py b/test/test_metrics.py index 2a38532..53125e2 100644 --- a/test/test_metrics.py +++ b/test/test_metrics.py @@ -4,6 +4,7 @@ import os import tempfile import shutil +import warnings from atgen.metrics import ( BaseMetric, @@ -22,7 +23,13 @@ get_comprehensive_config, get_metric_categories, get_metric_requirements, + AVAILABLE_METRICS, + get_available_metrics, + get_all_possible_metric_keys, ) +from atgen.utils.check_required_performance import check_required_performance +from atgen.utils.check_performance_metrics import check_performance_against_requirements +from omegaconf import DictConfig class TestMetricConfig: @@ -36,6 +43,8 @@ def test_default_config(self): assert config.cache_dir == "cache" assert config.aggregate is True assert config.threshold == 0.5 + assert config.provider is None + assert config.api_key is None def test_custom_config(self): """Test custom configuration values.""" @@ -44,13 +53,17 @@ def test_custom_config(self): device="cpu", cache_dir="custom_cache", aggregate=False, - threshold=0.7 + threshold=0.7, + provider="openrouter", + api_key="test-key" ) assert config.batch_size == 16 assert config.device == "cpu" assert config.cache_dir == "custom_cache" assert config.aggregate is False assert config.threshold == 0.7 + assert config.provider == "openrouter" + assert config.api_key == "test-key" class TestMetricsConfig: @@ -68,19 +81,19 @@ def test_default_config(self): def test_additional_metrics_merge(self): """Test that additional_metrics are merged into metrics.""" config = MetricsConfig( - metrics=["bleu", "rouge"], + metrics=["bleu", "rouge1"], additional_metrics=["sentbert", "cola"] ) - expected_metrics = ["bleu", "rouge", "sentbert", "cola"] + expected_metrics = ["bleu", "rouge1", "sentbert", "cola"] assert config.metrics == expected_metrics def test_duplicate_metrics_removed(self): """Test that duplicate metrics are removed while preserving order.""" config = MetricsConfig( - metrics=["bleu", "rouge", "bleu"], - additional_metrics=["sentbert", "rouge"] + metrics=["bleu", "rouge1", "bleu"], + additional_metrics=["sentbert", "rouge1"] ) - expected_metrics = ["bleu", "rouge", "sentbert"] + expected_metrics = ["bleu", "rouge1", "sentbert"] assert config.metrics == expected_metrics def test_deepeval_legacy_params(self): @@ -114,6 +127,28 @@ def test_get_available_metrics(self): assert "sentbert" in metrics assert "cola" in metrics + def test_get_all_possible_metric_keys(self): + """Test getting all possible metric keys.""" + keys = MetricsFactory.get_all_possible_metric_keys() + assert isinstance(keys, list) + assert len(keys) > len(MetricsFactory.get_available_metrics()) + + # Should include specific metric keys + assert "sentbert_pred_ref" in keys + assert "sentbert_pred_src" in keys + assert "rouge1" in keys + assert "rouge2" in keys + assert "rougeL" in keys + assert "exact_match" in keys + assert "word_length_gen" in keys + + def test_available_metrics_global(self): + """Test AVAILABLE_METRICS global variable.""" + assert isinstance(AVAILABLE_METRICS, list) + assert len(AVAILABLE_METRICS) > 20 # Should be comprehensive + assert "sentbert_pred_ref" in AVAILABLE_METRICS + assert "deepeval_answer_relevance" in AVAILABLE_METRICS + def test_create_metric_success(self): """Test successful metric creation.""" config = MetricConfig(device="cpu") @@ -160,37 +195,91 @@ def test_create_from_config_dict(self): assert len(metrics) == 2 assert "bleu" in metrics assert "rouge1" in metrics + + +class TestPerformanceCheckingIntegration: + """Test integration with performance checking system.""" + + def test_check_required_performance_valid_metrics(self): + """Test check_required_performance with valid metrics.""" + performance_dict = DictConfig({ + 'bleu': 0.3, + 'rouge1': 0.5, + 'sentbert_pred_ref': 0.7, + 'deepeval_answer_relevance': 0.8 + }) + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + result = check_required_performance(performance_dict) + + # Should have no warnings for valid metrics + assert len(w) == 0 + assert dict(result) == dict(performance_dict) + + def test_check_required_performance_invalid_metrics(self): + """Test check_required_performance with invalid metrics.""" + performance_dict = DictConfig({ + 'bleu': 0.3, + 'nonexistent_metric': 0.5, + 'rouge1': 1.5 # Invalid value > 1 + }) + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + result = check_required_performance(performance_dict) + + # Should have warnings for invalid metrics + assert len(w) == 2 + assert dict(result) == {'bleu': 0.3} # Only valid metric remains + + def test_performance_checking_pipeline(self): + """Test complete performance checking pipeline.""" + # Simulate computed metrics + metrics = { + 'bleu': 0.4, + 'rouge1': 0.6, + 'sentbert_pred_ref': 0.8, + 'word_length_gen': 10.5, + 'exact_match': 0.2, + 'time_total': 5.2 + } - def test_register_custom_metric(self): - """Test registering a custom metric.""" - class CustomMetric(BaseMetric): - def calculate(self, predictions, references, original_texts): - return {"custom": 0.5} - - MetricsFactory.register_metric("custom", CustomMetric) + # Set performance requirements + required_performance = DictConfig({ + 'bleu': 0.3, + 'rouge1': 0.5, + 'sentbert_pred_ref': 0.7 + }) + + # Check requirements + checked_requirements = check_required_performance(required_performance) + + # Test performance pipeline + is_performance_reached, is_metrics_availability_checked, available_metrics = ( + check_performance_against_requirements( + metrics=metrics, + required_performance_dict=checked_requirements, + is_metrics_availability_checked=False, + available_metrics={} + ) + ) - try: - available = MetricsFactory.get_available_metrics() - assert "custom" in available - - metric = MetricsFactory.create_metric("custom") - assert isinstance(metric, CustomMetric) - finally: - # Clean up - if "custom" in MetricsFactory._metric_registry: - del MetricsFactory._metric_registry["custom"] + assert is_performance_reached is True + assert is_metrics_availability_checked is True + assert len(available_metrics) == 3 -class TestIdenticalStringsBasic: - """Test all metrics with identical strings to ensure basic functionality.""" +class TestBasicMetricFunctionality: + """Test basic functionality of individual metrics.""" @pytest.fixture - def identical_data(self): - """Sample data with identical predictions and references.""" + def sample_data(self): + """Sample data for testing.""" return { "predictions": ["This is a test sentence.", "Another test sentence here."], "references": ["This is a test sentence.", "Another test sentence here."], - "original_texts": ["Original text one.", "Original text two."] + "original_texts": ["What is this?", "What is that?"] } @pytest.fixture @@ -200,141 +289,88 @@ def temp_cache_dir(self): yield temp_dir shutil.rmtree(temp_dir, ignore_errors=True) - def test_bleu_identical_strings(self, identical_data, temp_cache_dir): - """Test BLEU metric with identical strings.""" + def test_bleu_metric(self, sample_data, temp_cache_dir): + """Test BLEU metric functionality.""" config = MetricConfig(device="cpu", cache_dir=temp_cache_dir) metric = BleuMetric(config) results = metric.calculate( - identical_data["predictions"], - identical_data["references"], - identical_data["original_texts"] + sample_data["predictions"], + sample_data["references"], + sample_data["original_texts"] ) assert "bleu" in results if isinstance(results["bleu"], np.ndarray): - # Should be perfect scores for identical strings - assert all(score == 1.0 for score in results["bleu"]) + assert all(score == 1.0 for score in results["bleu"]) # Identical strings else: assert results["bleu"] == 1.0 - def test_rouge_identical_strings(self, identical_data, temp_cache_dir): - """Test ROUGE metric with identical strings.""" + def test_rouge_metric(self, sample_data, temp_cache_dir): + """Test ROUGE metric functionality.""" config = MetricConfig(device="cpu", cache_dir=temp_cache_dir) metric = RougeMetric(config) results = metric.calculate( - identical_data["predictions"], - identical_data["references"], - identical_data["original_texts"] + sample_data["predictions"], + sample_data["references"], + sample_data["original_texts"] ) - # ROUGE should return perfect scores for identical strings - rouge_keys = ["rouge1", "rouge2", "rougeL"] - for key in rouge_keys: - if key in results: - if isinstance(results[key], np.ndarray): - assert all(score == 1.0 for score in results[key]) - else: - assert results[key] == 1.0 - + # Should return multiple ROUGE variants + rouge_keys = ["rouge1", "rouge2", "rougeL", "rougeLsum"] + found_keys = [key for key in rouge_keys if key in results] + assert len(found_keys) > 0 + + # Perfect scores for identical strings + for key in found_keys: + if isinstance(results[key], np.ndarray): + assert all(score == 1.0 for score in results[key]) + else: + assert results[key] == 1.0 + @patch('torch.cuda.is_available', return_value=False) - def test_sentbert_identical_strings(self, mock_cuda, identical_data, temp_cache_dir): - """Test SentBERT metric with identical strings.""" + def test_sentbert_metric(self, mock_cuda, sample_data, temp_cache_dir): + """Test SentBERT metric functionality.""" config = MetricConfig(device="cpu", cache_dir=temp_cache_dir) metric = SentBertMetric(config) results = metric.calculate( - identical_data["predictions"], - identical_data["references"], - identical_data["original_texts"] + sample_data["predictions"], + sample_data["references"], + sample_data["original_texts"] ) - # SentBERT should return high similarity scores for identical strings - if "sentbert_pred_ref" in results: - if isinstance(results["sentbert_pred_ref"], np.ndarray): - assert all(score > 0.99 for score in results["sentbert_pred_ref"]) - else: - assert results["sentbert_pred_ref"] > 0.99 + # Should return both pred-ref and pred-src similarities + assert "sentbert_pred_ref" in results + assert "sentbert_pred_src" in results + + # High similarity for identical strings + if isinstance(results["sentbert_pred_ref"], np.ndarray): + assert all(score > 0.99 for score in results["sentbert_pred_ref"]) + else: + assert results["sentbert_pred_ref"] > 0.99 @patch('torch.cuda.is_available', return_value=False) - def test_cola_identical_strings(self, mock_cuda, identical_data, temp_cache_dir): - """Test CoLA metric with identical strings.""" + def test_cola_metric(self, mock_cuda, sample_data, temp_cache_dir): + """Test CoLA metric functionality.""" config = MetricConfig(device="cpu", cache_dir=temp_cache_dir) metric = ColaMetric(config) results = metric.calculate( - identical_data["predictions"], - identical_data["references"], - identical_data["original_texts"] + sample_data["predictions"], + sample_data["references"], + sample_data["original_texts"] ) - # CoLA should return grammaticality scores - assert "cola" in results - if isinstance(results["cola"], np.ndarray): - assert all(0 <= score <= 1 for score in results["cola"]) + assert "cola" in results or "grammaticality" in results + + # Should return reasonable grammaticality scores + score_key = "cola" if "cola" in results else "grammaticality" + if isinstance(results[score_key], np.ndarray): + assert all(0 <= score <= 1 for score in results[score_key]) else: - assert 0 <= results["cola"] <= 1 - - @patch('torch.cuda.is_available', return_value=False) - def test_bartscore_identical_strings(self, mock_cuda, identical_data, temp_cache_dir): - """Test BARTScore metric with identical strings.""" - config = MetricConfig(device="cpu", cache_dir=temp_cache_dir, batch_size=2) - metric = BartScoreMetric(config) - - try: - results = metric.calculate( - identical_data["predictions"], - identical_data["references"], - identical_data["original_texts"] - ) - - # BARTScore should return high scores for identical strings - assert isinstance(results, dict) - assert len(results) > 0 - - # Check that scores are reasonable (BARTScore can vary but should be positive for identical strings) - for key, value in results.items(): - if isinstance(value, np.ndarray): - assert all(isinstance(score, (int, float)) for score in value) - else: - assert isinstance(value, (int, float)) - - except Exception as e: - # BARTScore might not be available or have dependency issues - pytest.skip(f"BARTScore test skipped due to: {e}") - - @patch('torch.cuda.is_available', return_value=False) - def test_alignscore_identical_strings(self, mock_cuda, identical_data, temp_cache_dir): - """Test AlignScore metric with identical strings.""" - # Skip test if AlignScore is not available - if AlignScoreMetric is None: - pytest.skip("AlignScore metric not available due to import issues") - - config = MetricConfig(device="cpu", cache_dir=temp_cache_dir, batch_size=2) - metric = AlignScoreMetric(config) - - try: - results = metric.calculate( - identical_data["predictions"], - identical_data["references"], - identical_data["original_texts"] - ) - - # AlignScore should return high scores for identical strings - assert isinstance(results, dict) - assert len(results) > 0 - - # Check that scores are reasonable - for key, value in results.items(): - if isinstance(value, np.ndarray): - assert all(isinstance(score, (int, float)) for score in value) - else: - assert isinstance(value, (int, float)) - - except Exception as e: - # AlignScore might not be available or have dependency issues - pytest.skip(f"AlignScore test skipped due to: {e}") + assert 0 <= results[score_key] <= 1 class TestComputeMetrics: @@ -365,7 +401,6 @@ def test_compute_metrics_default_config(self, sample_data, temp_cache_dir): cache_dir=temp_cache_dir ) - # Should include basic metrics assert isinstance(results, dict) assert len(results) > 0 @@ -377,15 +412,60 @@ def test_compute_metrics_default_config(self, sample_data, temp_cache_dir): assert "word_length_gen" in results assert "exact_match" in results - def test_compute_metrics_custom_config(self, sample_data, temp_cache_dir): - """Test compute_metrics with custom configuration.""" - config = MetricsConfig( - metrics=["bleu", "rouge1"], - device="cpu", - cache_dir=temp_cache_dir, - batch_size=16 + # Exact match should be 1.0 for identical strings + assert results["exact_match"] == 1.0 + + def test_compute_metrics_comprehensive_config(self, sample_data, temp_cache_dir): + """Test compute_metrics with comprehensive configuration.""" + config = DictConfig({ + 'metrics': ['bleu', 'rouge1', 'sentbert', 'cola'], + 'device': 'cpu', + 'cache_dir': temp_cache_dir, + 'batch_size': 16, + 'aggregate': True + }) + + results = compute_metrics( + generated_texts=sample_data["predictions"], + reference_texts=sample_data["references"], + original_texts=sample_data["original_texts"], + config=config ) + assert isinstance(results, dict) + assert len(results) > 10 # Should have many metrics + + # Check key metrics are present + assert "bleu" in results + assert "rouge1" in results + assert "sentbert_pred_ref" in results + assert any(key in results for key in ["cola", "grammaticality"]) + + # Perfect scores for identical strings + assert results["bleu"] == 1.0 + assert results["rouge1"] == 1.0 + assert results["exact_match"] == 1.0 + + def test_compute_metrics_with_deepeval_config(self, sample_data, temp_cache_dir): + """Test compute_metrics with DeepEval configuration.""" + # Skip if no API key available + api_key = os.environ.get('OPENROUTER_API_KEY') + if not api_key: + pytest.skip("No OpenRouter API key available for DeepEval testing") + + config = DictConfig({ + 'metrics': ['bleu', 'deepeval_answer_relevance'], + 'provider': 'openrouter', + 'base_url': 'https://openrouter.ai/api/v1', + 'api_key': api_key, + 'model': 'openai/gpt-4o-mini', + 'threshold': 0.5, + 'async_mode': False, + 'verbose_mode': False, + 'device': 'cpu', + 'cache_dir': temp_cache_dir + }) + results = compute_metrics( generated_texts=sample_data["predictions"], reference_texts=sample_data["references"], @@ -395,15 +475,21 @@ def test_compute_metrics_custom_config(self, sample_data, temp_cache_dir): assert isinstance(results, dict) assert "bleu" in results - assert any(key.startswith("rouge") for key in results.keys()) + + # Check if DeepEval worked + deepeval_keys = [k for k in results.keys() if 'deepeval' in k and not k.startswith('time_')] + if deepeval_keys: + assert len(deepeval_keys) > 0 + for key in deepeval_keys: + assert 0 <= results[key] <= 1 def test_compute_metrics_no_references(self, sample_data, temp_cache_dir): """Test compute_metrics without references.""" - config = MetricsConfig( - metrics=["sentbert", "cola"], - device="cpu", - cache_dir=temp_cache_dir - ) + config = DictConfig({ + 'metrics': ['sentbert', 'cola'], + 'device': 'cpu', + 'cache_dir': temp_cache_dir + }) results = compute_metrics( generated_texts=sample_data["predictions"], @@ -413,8 +499,8 @@ def test_compute_metrics_no_references(self, sample_data, temp_cache_dir): ) assert isinstance(results, dict) - # Should still include basic statistics assert "word_length_gen" in results + assert "sentbert_pred_src" in results # Should have pred-src similarity def test_compute_metrics_empty_predictions(self): """Test compute_metrics with empty predictions.""" @@ -438,6 +524,8 @@ def test_compute_metrics_from_config(self, sample_data, temp_cache_dir): assert isinstance(results, dict) assert len(results) > 0 + assert "bleu" in results + assert "rouge1" in results class TestMetricRequirements: @@ -485,7 +573,7 @@ def test_get_default_config(self): assert isinstance(config, MetricsConfig) assert "bleu" in config.metrics - assert "rouge1" in config.metrics + assert "rouge1" in config.metrics or any("rouge" in m for m in config.metrics) assert config.batch_size == 32 assert config.device == "cuda" @@ -494,7 +582,7 @@ def test_get_comprehensive_config(self): config = get_comprehensive_config() assert isinstance(config, MetricsConfig) - assert len(config.metrics) > 2 # Should include more metrics + assert len(config.metrics) > 2 assert "bleu" in config.metrics assert any("rouge" in metric for metric in config.metrics) assert "sentbert" in config.metrics @@ -513,11 +601,11 @@ def temp_cache_dir(self): def test_empty_strings(self, temp_cache_dir): """Test metrics with empty strings.""" - config = MetricsConfig( - metrics=["bleu", "rouge1"], - device="cpu", - cache_dir=temp_cache_dir - ) + config = DictConfig({ + 'metrics': ['bleu', 'rouge1'], + 'device': 'cpu', + 'cache_dir': temp_cache_dir + }) results = compute_metrics( generated_texts=["", ""], @@ -531,11 +619,11 @@ def test_empty_strings(self, temp_cache_dir): def test_single_item_lists(self, temp_cache_dir): """Test metrics with single item lists.""" - config = MetricsConfig( - metrics=["bleu", "rouge1"], - device="cpu", - cache_dir=temp_cache_dir - ) + config = DictConfig({ + 'metrics': ['bleu', 'rouge1'], + 'device': 'cpu', + 'cache_dir': temp_cache_dir + }) results = compute_metrics( generated_texts=["Single test sentence."], @@ -546,14 +634,15 @@ def test_single_item_lists(self, temp_cache_dir): assert isinstance(results, dict) assert len(results) > 0 + assert results["exact_match"] == 1.0 # Identical strings def test_multiple_references(self, temp_cache_dir): """Test metrics with multiple references.""" - config = MetricsConfig( - metrics=["bleu", "rouge1"], - device="cpu", - cache_dir=temp_cache_dir - ) + config = DictConfig({ + 'metrics': ['bleu', 'rouge1'], + 'device': 'cpu', + 'cache_dir': temp_cache_dir + }) results = compute_metrics( generated_texts=["Test sentence."], @@ -564,32 +653,10 @@ def test_multiple_references(self, temp_cache_dir): assert isinstance(results, dict) assert len(results) > 0 - - def test_mismatched_lengths(self, temp_cache_dir): - """Test metrics with mismatched input lengths.""" - config = MetricsConfig( - metrics=["bleu"], - device="cpu", - cache_dir=temp_cache_dir - ) - - # This should handle gracefully or raise appropriate error - try: - results = compute_metrics( - generated_texts=["Test sentence.", "Another sentence."], - reference_texts=["Test sentence."], # Shorter list - original_texts=["Source text."], - config=config - ) - # If it doesn't raise an error, check that results are reasonable - assert isinstance(results, dict) - except (ValueError, IndexError, AssertionError): - # These are acceptable errors for mismatched lengths - pass -class TestMetricIntegration: - """Integration tests for the complete metrics system.""" +class TestRealWorldScenarios: + """Test realistic scenarios that would occur in active learning.""" @pytest.fixture def temp_cache_dir(self): @@ -598,115 +665,121 @@ def temp_cache_dir(self): yield temp_dir shutil.rmtree(temp_dir, ignore_errors=True) - def test_all_lexical_metrics(self, temp_cache_dir): - """Test all lexical metrics together.""" - config = MetricsConfig( - metrics=["bleu", "rouge1"], - device="cpu", - cache_dir=temp_cache_dir, - aggregate=True - ) - + def test_active_learning_integration(self, temp_cache_dir): + """Test complete active learning integration scenario.""" + # Realistic generated texts + generated_texts = [ + "Machine learning algorithms learn patterns from training data to make predictions.", + "Deep learning uses neural networks with multiple layers to process information.", + "Natural language processing helps computers understand human language." + ] + + reference_texts = [ + "ML algorithms learn from data to make predictions on new examples.", + "Deep learning employs multi-layered neural networks for complex data processing.", + "NLP enables computers to work with human language effectively." + ] + + original_texts = [ + "How do machine learning algorithms work?", + "What is deep learning?", + "Explain natural language processing." + ] + + # Comprehensive config similar to active learning setup + config = DictConfig({ + 'metrics': ['bleu', 'rouge1', 'rouge2', 'rougeL', 'sentbert'], + 'additional_metrics': ['cola'], + 'device': 'cpu', + 'cache_dir': temp_cache_dir, + 'batch_size': 16, + 'aggregate': True + }) + + # Compute metrics results = compute_metrics( - generated_texts=["This is a comprehensive test sentence.", "Another test for metrics."], - reference_texts=["This is a comprehensive test sentence.", "Another test for metrics."], - original_texts=["Source text one.", "Source text two."], + generated_texts=generated_texts, + reference_texts=reference_texts, + original_texts=original_texts, config=config ) - assert "bleu" in results - assert any(key.startswith("rouge") for key in results.keys()) - - # All scores should be 1.0 for identical strings - # BLEU is aggregated so should be a single value - assert isinstance(results["bleu"], (int, float)) - assert results["bleu"] == 1.0 + # Verify comprehensive results + assert isinstance(results, dict) + assert len(results) > 10 - @patch('torch.cuda.is_available', return_value=False) - def test_comprehensive_metrics_suite(self, mock_cuda, temp_cache_dir): - """Test a comprehensive suite of metrics.""" - config = MetricsConfig( - metrics=["bleu", "rouge1", "sentbert", "cola"], - device="cpu", - cache_dir=temp_cache_dir, - batch_size=8, - aggregate=True + # Check key performance metrics + performance_keys = ['bleu', 'rouge1', 'rouge2', 'rougeL', 'sentbert_pred_ref'] + for key in performance_keys: + if key in results: + assert isinstance(results[key], (int, float)) + assert 0 <= results[key] <= 1 + + # Test performance checking integration + required_performance = DictConfig({ + 'bleu': 0.15, + 'rouge1': 0.35, + 'sentbert_pred_ref': 0.55 + }) + + checked_requirements = check_required_performance(required_performance) + assert len(checked_requirements) == 3 # All should be valid + + # Test complete pipeline + is_performance_reached, is_metrics_availability_checked, available_metrics = ( + check_performance_against_requirements( + metrics=results, + required_performance_dict=checked_requirements, + is_metrics_availability_checked=False, + available_metrics={} + ) ) + assert is_metrics_availability_checked is True + assert len(available_metrics) == 3 + + def test_high_quality_generation_scenario(self, temp_cache_dir): + """Test scenario with high-quality generated text.""" + # High-quality generated texts (should score well) + generated_texts = [ + "The capital of France is Paris, which is also its largest city.", + "Artificial intelligence is transforming various industries worldwide." + ] + + reference_texts = [ + "Paris is the capital and largest city of France.", + "AI is revolutionizing industries across the globe." + ] + + original_texts = [ + "What is the capital of France?", + "How is AI impacting industries?" + ] + + config = DictConfig({ + 'metrics': ['bleu', 'rouge1', 'sentbert', 'cola'], + 'device': 'cpu', + 'cache_dir': temp_cache_dir, + 'aggregate': True + }) + results = compute_metrics( - generated_texts=[ - "This is a well-formed grammatical sentence.", - "Another properly structured sentence here." - ], - reference_texts=[ - "This is a well-formed grammatical sentence.", - "Another properly structured sentence here." - ], - original_texts=[ - "Source text for the first sentence.", - "Source text for the second sentence." - ], + generated_texts=generated_texts, + reference_texts=reference_texts, + original_texts=original_texts, config=config ) - # Should include results from all metric types - assert "bleu" in results - assert any(key.startswith("rouge") for key in results.keys()) - assert any(key.startswith("sentbert") for key in results.keys()) - assert "cola" in results + # Should get reasonable scores for good quality text + assert results['bleu'] > 0.1 # Some lexical overlap + assert results['rouge1'] > 0.2 # Some word overlap + assert results['sentbert_pred_ref'] > 0.5 # Good semantic similarity - # Should include timing and basic stats - assert "time_total" in results - assert "word_length_gen" in results - assert "exact_match" in results - - # Exact match should be 1.0 for identical strings - assert results["exact_match"] == 1.0 - - @patch('torch.cuda.is_available', return_value=False) - def test_semantic_metrics_suite(self, mock_cuda, temp_cache_dir): - """Test semantic metrics including BARTScore and AlignScore.""" - config = MetricsConfig( - metrics=["sentbert", "bartscore", "alignscore"], - device="cpu", - cache_dir=temp_cache_dir, - batch_size=4, - aggregate=True - ) - - try: - results = compute_metrics( - generated_texts=[ - "This is a semantic similarity test.", - "Another sentence for testing." - ], - reference_texts=[ - "This is a semantic similarity test.", - "Another sentence for testing." - ], - original_texts=[ - "Source text for semantic testing.", - "Another source text here." - ], - config=config - ) - - # Should include results from semantic metrics - assert isinstance(results, dict) - assert len(results) > 0 - - # Should include timing and basic stats - assert "time_total" in results - assert "word_length_gen" in results - assert "exact_match" in results - - # Exact match should be 1.0 for identical strings - assert results["exact_match"] == 1.0 - - except Exception as e: - # Some semantic metrics might not be available - pytest.skip(f"Semantic metrics test skipped due to: {e}") + # CoLA should give high grammaticality scores + cola_key = 'cola' if 'cola' in results else 'grammaticality' + if cola_key in results: + assert results[cola_key] > 0.7 # Well-formed sentences if __name__ == "__main__": - pytest.main([__file__]) \ No newline at end of file + pytest.main([__file__, "-v"]) \ No newline at end of file From 0d0a91bd80a199f608105e2e665da400eb2a870f Mon Sep 17 00:00:00 2001 From: alfekka Date: Mon, 9 Jun 2025 00:55:59 +0300 Subject: [PATCH 36/39] removed dependencies chek --- src/atgen/metrics/base/base_metric.py | 45 +++++++++++++++++++ src/atgen/metrics/lexical/bleu_metric.py | 20 +++++++++ src/atgen/metrics/lexical/rouge_metric.py | 10 +++++ src/atgen/metrics/linguistic/cola_metric.py | 8 ++++ .../metrics/llm_based/base_deepeval_metric.py | 15 +++++++ .../metrics/semantic/bart_score_metric.py | 9 ++++ src/atgen/metrics/semantic/sentbert_metric.py | 42 +++++++++++++++++ 7 files changed, 149 insertions(+) diff --git a/src/atgen/metrics/base/base_metric.py b/src/atgen/metrics/base/base_metric.py index c3473c6..8c1fb84 100644 --- a/src/atgen/metrics/base/base_metric.py +++ b/src/atgen/metrics/base/base_metric.py @@ -6,26 +6,48 @@ @dataclass class MetricConfig: +<<<<<<< HEAD +======= + """Base configuration for metrics.""" + # General parameters +>>>>>>> a24e0f2 (removed dependencies chek) batch_size: int = 32 device: str = "cuda" cache_dir: str = "cache" aggregate: bool = True +<<<<<<< HEAD checkpoint: Optional[str] = None model_name: Optional[str] = None # API configuration parameters provider: Optional[str] = None +======= + # Model-specific parameters (for local models) + checkpoint: Optional[str] = None + model_name: Optional[str] = None + + # API-based parameters (for LLM-based metrics) +>>>>>>> a24e0f2 (removed dependencies chek) api_key: Optional[str] = None base_url: Optional[str] = None model: Optional[str] = None +<<<<<<< HEAD +======= + # DeepEval specific parameters +>>>>>>> a24e0f2 (removed dependencies chek) threshold: float = 0.5 include_reason: bool = False strict_mode: bool = False async_mode: bool = True verbose_mode: bool = False truths_extraction_limit: Optional[int] = None +<<<<<<< HEAD +======= + + # BigBenchHard specific parameters +>>>>>>> a24e0f2 (removed dependencies chek) benchmark_params: Dict[str, Any] = field(default_factory=dict) generation_params: Dict[str, Any] = field(default_factory=dict) @@ -34,6 +56,7 @@ class BaseMetric(ABC): def __init__(self, config: Optional[MetricConfig] = None): self.config = config or MetricConfig() self.logger = logging.getLogger(self.__class__.__name__) +<<<<<<< HEAD self._validate_config() @@ -42,13 +65,35 @@ def name(self) -> str: return self.__class__.__name__.replace("Metric", "").lower() def _validate_config(self): +======= + self._is_available = None + self._model = None + + # Validate configuration + self._validate_config() + + # Check dependencies + if not self.is_available(): + self.logger.warning(f"{self.__class__.__name__} dependencies not available") + + @property + def name(self) -> str: + """Return the metric name (defaults to class name).""" + return self.__class__.__name__.replace("Metric", "").lower() + + def _validate_config(self): + """Validate configuration - can be overridden by subclasses.""" +>>>>>>> a24e0f2 (removed dependencies chek) pass @abstractmethod def calculate(self, predictions, references, original_texts): pass +<<<<<<< HEAD def is_available(self) -> bool: """Check if the metric is available for use.""" return True +======= +>>>>>>> a24e0f2 (removed dependencies chek) \ No newline at end of file diff --git a/src/atgen/metrics/lexical/bleu_metric.py b/src/atgen/metrics/lexical/bleu_metric.py index 91355c8..4440ccf 100644 --- a/src/atgen/metrics/lexical/bleu_metric.py +++ b/src/atgen/metrics/lexical/bleu_metric.py @@ -7,6 +7,10 @@ from ..base.base_metric import BaseMetric, MetricConfig +<<<<<<< HEAD +======= +# Ensure nltk data is downloaded +>>>>>>> a24e0f2 (removed dependencies chek) try: word_tokenize("test") except LookupError: @@ -37,6 +41,10 @@ def smoothing_function(p_n, references, hypothesis, hyp_len): class BleuMetric(BaseMetric): +<<<<<<< HEAD +======= + """BLEU (Bilingual Evaluation Understudy) metric implementation.""" +>>>>>>> a24e0f2 (removed dependencies chek) @property def category(self) -> str: @@ -47,7 +55,11 @@ def supports_multiple_references(self) -> bool: return True +<<<<<<< HEAD def calculate( +======= + def _compute( +>>>>>>> a24e0f2 (removed dependencies chek) self, predictions: List[str], references: Optional[List[Union[str, List[str]]]] = None, @@ -76,11 +88,16 @@ def calculate( tok_pred = word_tokenize(pred) try: +<<<<<<< HEAD score = corpus_bleu([tok_ref], [tok_pred], smoothing_function=smoothing_function) +======= + score = corpus_bleu(tok_ref, [tok_pred], smoothing_function=smoothing_function) +>>>>>>> a24e0f2 (removed dependencies chek) scores.append(score) except (KeyError, ZeroDivisionError): scores.append(0.0) +<<<<<<< HEAD scores_dict = {"bleu": np.array(scores)} # Apply aggregation if requested @@ -88,3 +105,6 @@ def calculate( scores_dict = {key: float(np.mean(value)) for key, value in scores_dict.items()} return scores_dict +======= + return {"bleu": np.array(scores)} +>>>>>>> a24e0f2 (removed dependencies chek) diff --git a/src/atgen/metrics/lexical/rouge_metric.py b/src/atgen/metrics/lexical/rouge_metric.py index 61f686f..661dd08 100644 --- a/src/atgen/metrics/lexical/rouge_metric.py +++ b/src/atgen/metrics/lexical/rouge_metric.py @@ -16,7 +16,10 @@ def _initialize_rouge(self): """Initialize ROUGE if not already initialized.""" if self.rouge is None: self.rouge = load("rouge", cache_dir=self.config.cache_dir) +<<<<<<< HEAD +======= +>>>>>>> a24e0f2 (removed dependencies chek) def calculate(self, predictions: List[str], references: Optional[List[Union[str, List[str]]]] = None, original_texts: Optional[List[str]] = None) -> Dict[str, float]: """ @@ -55,14 +58,21 @@ def calculate(self, predictions: List[str], references: Optional[List[Union[str, for key, value in results.items(): if isinstance(value, (int, float)): scores[key] = float(value) +<<<<<<< HEAD elif hasattr(value, 'item'): # +======= + elif hasattr(value, 'item'): # For numpy scalars +>>>>>>> a24e0f2 (removed dependencies chek) scores[key] = float(value.item()) else: scores[key] = float(value) +<<<<<<< HEAD if self.config.aggregate: for key, value in scores.items(): if isinstance(value, np.ndarray): scores[key] = float(np.mean(value)) +======= +>>>>>>> a24e0f2 (removed dependencies chek) return scores \ No newline at end of file diff --git a/src/atgen/metrics/linguistic/cola_metric.py b/src/atgen/metrics/linguistic/cola_metric.py index b333dbb..ec7e08a 100644 --- a/src/atgen/metrics/linguistic/cola_metric.py +++ b/src/atgen/metrics/linguistic/cola_metric.py @@ -12,6 +12,10 @@ from ..base.base_metric import BaseMetric, MetricConfig +<<<<<<< HEAD +======= +# Ensure NLTK data is downloaded +>>>>>>> a24e0f2 (removed dependencies chek) try: nltk.data.find('tokenizers/punkt') except LookupError: @@ -25,7 +29,11 @@ def __init__(self, config: Optional[MetricConfig] = None): super().__init__(config) self.model = None self.tokenizer = None +<<<<<<< HEAD self.checkpoint = (config.checkpoint if config and config.checkpoint else 'Aktsvigun/electra-large-cola') +======= + self.checkpoint = getattr(config, 'checkpoint', 'Aktsvigun/electra-large-cola') if config else 'Aktsvigun/electra-large-cola' +>>>>>>> a24e0f2 (removed dependencies chek) def _initialize_model(self): diff --git a/src/atgen/metrics/llm_based/base_deepeval_metric.py b/src/atgen/metrics/llm_based/base_deepeval_metric.py index d7408fd..26b56b1 100644 --- a/src/atgen/metrics/llm_based/base_deepeval_metric.py +++ b/src/atgen/metrics/llm_based/base_deepeval_metric.py @@ -29,6 +29,7 @@ def _validate_config(self): def _initialize_llm(self): """Initialize the LLM for evaluation if not already initialized.""" if self.llm is None: +<<<<<<< HEAD # Determine base_url based on provider if not explicitly set base_url = self.config.base_url if base_url is None and self.config.provider: @@ -64,6 +65,12 @@ def _initialize_llm(self): api_key=self.config.api_key, model=model, base_url=base_url, +======= + self.llm = EvaluationLLM( + api_key=self.config.api_key, + model=self.config.model or "openai/gpt-4o-2024-11-20", + base_url=self.config.base_url or "https://openrouter.ai/api/v1", +>>>>>>> a24e0f2 (removed dependencies chek) ) def _create_deepeval_metric(self, metric_class, **kwargs): @@ -84,11 +91,19 @@ def _run_evaluation(self, test_cases: List[LLMTestCase], metric, metric_name: st if not test_cases: return {} +<<<<<<< HEAD +======= + # Disable printing to console during evaluation if not verbose +>>>>>>> a24e0f2 (removed dependencies chek) original_stdout = sys.stdout if not self.config.verbose_mode: sys.stdout = open(os.devnull, "w") try: +<<<<<<< HEAD +======= + # Run evaluation +>>>>>>> a24e0f2 (removed dependencies chek) evaluation_results = evaluate( test_cases=test_cases, metrics=[metric], diff --git a/src/atgen/metrics/semantic/bart_score_metric.py b/src/atgen/metrics/semantic/bart_score_metric.py index a33b168..afea7a3 100644 --- a/src/atgen/metrics/semantic/bart_score_metric.py +++ b/src/atgen/metrics/semantic/bart_score_metric.py @@ -38,7 +38,11 @@ def __init__( def score(self, srcs, tgts, batch_size=4): """Score a batch of examples""" score_list = [] +<<<<<<< HEAD for i in range(0, len(srcs), batch_size): +======= + for i in tqdm(range(0, len(srcs), batch_size), desc="Calculating BARTScore..."): +>>>>>>> a24e0f2 (removed dependencies chek) src_list = srcs[i : i + batch_size] tgt_list = tgts[i : i + batch_size] try: @@ -112,6 +116,11 @@ def calculate(self, predictions: List[str], references: Optional[List[Union[str, Returns: Dictionary with BARTScore results """ +<<<<<<< HEAD +======= + if not self.is_available(): + raise RuntimeError("BARTScore dependencies not available") +>>>>>>> a24e0f2 (removed dependencies chek) self._initialize_scorer() diff --git a/src/atgen/metrics/semantic/sentbert_metric.py b/src/atgen/metrics/semantic/sentbert_metric.py index d872405..dff5450 100644 --- a/src/atgen/metrics/semantic/sentbert_metric.py +++ b/src/atgen/metrics/semantic/sentbert_metric.py @@ -8,15 +8,27 @@ class SentBertMetric(BaseMetric): +<<<<<<< HEAD +======= + """SentenceBERT semantic similarity metric.""" +>>>>>>> a24e0f2 (removed dependencies chek) def __init__(self, config: Optional[MetricConfig] = None): super().__init__(config) self.model = None self.tokenizer = None +<<<<<<< HEAD self.checkpoint = (config.checkpoint if config and config.checkpoint else 'sentence-transformers/all-mpnet-base-v2') def _initialize_model(self): +======= + self.checkpoint = getattr(config, 'checkpoint', 'sentence-transformers/all-mpnet-base-v2') if config else 'sentence-transformers/all-mpnet-base-v2' + + + def _initialize_model(self): + """Initialize the SentenceBERT model if not already initialized.""" +>>>>>>> a24e0f2 (removed dependencies chek) if self.model is None: self.tokenizer = AutoTokenizer.from_pretrained( self.checkpoint, @@ -56,6 +68,10 @@ def _compute_similarity(self, source_texts: List[str], ref_texts: List[str]) -> end += half_batch_size batch_idx = slice(start, end) +<<<<<<< HEAD +======= + # Tokenize sentences +>>>>>>> a24e0f2 (removed dependencies chek) encoded_input = self.tokenizer( source_texts[batch_idx] + ref_texts[batch_idx], padding=True, @@ -66,11 +82,22 @@ def _compute_similarity(self, source_texts: List[str], ref_texts: List[str]) -> key: value.to(self.config.device) for key, value in encoded_input.items() } +<<<<<<< HEAD with torch.no_grad(): model_output = self.model(**encoded_input) sent_embs = self.mean_pooling(model_output, encoded_input["attention_mask"]) +======= + # Calculate embeddings + with torch.no_grad(): + model_output = self.model(**encoded_input) + + # Perform pooling + sent_embs = self.mean_pooling(model_output, encoded_input["attention_mask"]) + + # Normalize embeddings +>>>>>>> a24e0f2 (removed dependencies chek) sent_embs = F.normalize(sent_embs, p=2, dim=1) n_source_embs = len(sent_embs) // 2 @@ -101,8 +128,15 @@ def calculate(self, predictions: List[str], references: Optional[List[Union[str, scores = {} +<<<<<<< HEAD + if references is not None: + if isinstance(references[0], list): +======= + # Similarity between predictions and references if references is not None: if isinstance(references[0], list): + # Handle multiple references - compute average similarity +>>>>>>> a24e0f2 (removed dependencies chek) ref_scores = [] for pred, ref_list in zip(predictions, references): pred_list = [pred] * len(ref_list) @@ -112,9 +146,17 @@ def calculate(self, predictions: List[str], references: Optional[List[Union[str, else: scores["sentbert_pred_ref"] = self._compute_similarity(predictions, references) +<<<<<<< HEAD + if original_texts is not None: + scores["sentbert_pred_src"] = self._compute_similarity(predictions, original_texts) + +======= + # Similarity between predictions and original texts if original_texts is not None: scores["sentbert_pred_src"] = self._compute_similarity(predictions, original_texts) + # Aggregate if requested +>>>>>>> a24e0f2 (removed dependencies chek) if self.config.aggregate: scores = {key: float(np.mean(value)) for key, value in scores.items()} From 8c08ebade289960d03b8e62ef906e8f788cac51e Mon Sep 17 00:00:00 2001 From: alfekka Date: Fri, 13 Jun 2025 21:30:40 +0300 Subject: [PATCH 37/39] conflict resolve --- configs/base.yaml | 4 +- src/atgen/metrics/bart_score.py | 118 +++++ src/atgen/metrics/compute_metrics.py | 412 +++++++++--------- .../utils/data/get_preprocess_function.py | 8 +- src/atgen/utils/generate.py | 1 - 5 files changed, 333 insertions(+), 210 deletions(-) create mode 100644 src/atgen/metrics/bart_score.py diff --git a/configs/base.yaml b/configs/base.yaml index 0a074a8..fae2e4c 100644 --- a/configs/base.yaml +++ b/configs/base.yaml @@ -26,11 +26,11 @@ al: evaluate_zero_iteration: True subsample_size: -1 required_performance: - rouge: 0.5 + rouge1: 0.5 budget: model: - checkpoint: Qwen/Qwen2.5-1.5B + checkpoint: Qwen/Qwen3-1.7B quantize: False model_max_length: ${multiply_with_few_shot:${data.input_max_length},${data.few_shot.count}} dtype: bfloat16 diff --git a/src/atgen/metrics/bart_score.py b/src/atgen/metrics/bart_score.py new file mode 100644 index 0000000..6133693 --- /dev/null +++ b/src/atgen/metrics/bart_score.py @@ -0,0 +1,118 @@ +# %% +import traceback +from typing import List +from tqdm import tqdm + +import numpy as np +import torch +import torch.nn as nn +from transformers import BartTokenizer, BartForConditionalGeneration + + +class BARTScorer: + def __init__( + self, + device="cuda", # device="cuda:0", + max_length=1024, + checkpoint="facebook/bart-large-cnn", + cache_dir: str = "cache", + ): + # Set up model + self.device = device + self.max_length = max_length + self.tokenizer = BartTokenizer.from_pretrained(checkpoint, cache_dir=cache_dir) + self.model = BartForConditionalGeneration.from_pretrained( + checkpoint, cache_dir=cache_dir + ) + self.model.eval() + self.model.to(device) + + # Set up loss + self.loss_fct = nn.NLLLoss( + reduction="none", ignore_index=self.model.config.pad_token_id + ) + self.lsm = nn.LogSoftmax(dim=1) + + def load(self, path=None): + """Load model from paraphrase finetuning""" + if path is None: + path = "models/bart.pth" + self.model.load_state_dict(torch.load(path, map_location=self.device)) + + def score(self, srcs, tgts, batch_size=4): + """Score a batch of examples""" + score_list = [] + for i in range(0, len(srcs), batch_size): + src_list = srcs[i : i + batch_size] + tgt_list = tgts[i : i + batch_size] + try: + with torch.no_grad(): + encoded_src = self.tokenizer( + src_list, + max_length=self.max_length, + truncation=True, + padding=True, + return_tensors="pt", + ) + encoded_tgt = self.tokenizer( + tgt_list, + max_length=self.max_length, + truncation=True, + padding=True, + return_tensors="pt", + ) + src_tokens = encoded_src["input_ids"].to(self.device) + src_mask = encoded_src["attention_mask"].to(self.device) + + tgt_tokens = encoded_tgt["input_ids"].to(self.device) + tgt_mask = encoded_tgt["attention_mask"] + tgt_len = tgt_mask.sum(dim=1).to(self.device) + + output = self.model( + input_ids=src_tokens, attention_mask=src_mask, labels=tgt_tokens + ) + logits = output.logits.view(-1, self.model.config.vocab_size) + loss = self.loss_fct(self.lsm(logits), tgt_tokens.view(-1)) + loss = loss.view(tgt_tokens.shape[0], -1) + loss = loss.sum(dim=1) / tgt_len + curr_score_list = [-x.item() for x in loss] + score_list += curr_score_list + + except RuntimeError: + traceback.print_exc() + print(f"source: {src_list}") + print(f"target: {tgt_list}") + exit(0) + return score_list + + def multi_ref_score(self, srcs, tgts: List[List[str]], agg="mean", batch_size=4): + # Assert we have the same number of references + ref_nums = [len(x) for x in tgts] + if len(set(ref_nums)) > 1: + raise Exception("You have different number of references per test sample.") + + ref_num = len(tgts[0]) + score_matrix = [] + for i in range(ref_num): + curr_tgts = [x[i] for x in tgts] + scores = self.score(srcs, curr_tgts, batch_size) + score_matrix.append(scores) + if agg == "mean": + score_list = np.mean(score_matrix, axis=0) + elif agg == "max": + score_list = np.max(score_matrix, axis=0) + else: + raise NotImplementedError + return list(score_list) + + def test(self, batch_size=3): + """Test""" + src_list = [ + "This is a very good idea. Although simple, but very insightful.", + "Can I take a look?", + "Do not trust him, he is a liar.", + ] + + tgt_list = ["That's stupid.", "What's the problem?", "He is trustworthy."] + + print(self.score(src_list, tgt_list, batch_size)) diff --git a/src/atgen/metrics/compute_metrics.py b/src/atgen/metrics/compute_metrics.py index fe07684..f55c175 100644 --- a/src/atgen/metrics/compute_metrics.py +++ b/src/atgen/metrics/compute_metrics.py @@ -1,11 +1,10 @@ from time import time -from typing import List, Dict, Union, Optional import logging -import numpy as np from omegaconf import DictConfig -from .factory import MetricsFactory, MetricsConfig, get_metric_requirements -from .base import MetricConfig +import numpy as np +from evaluate import load +from tqdm import tqdm from .metrics import ( pair_bleu, @@ -22,220 +21,223 @@ def compute_metrics( - generated_texts: List[str], - reference_texts: Optional[List[Union[str, List[str]]]] = None, - original_texts: Optional[List[str]] = None, - config: Optional[Union[Dict, DictConfig, MetricsConfig]] = None, - model=None, - tokenizer=None, - cache_dir: Optional[str] = None, -) -> Dict[str, float]: + generated_texts, + reference_texts, + original_texts, + config: DictConfig, + cache_dir: str = "cache", +) -> dict[str, float]: """ - Compute various metrics for generated texts using the new architecture. - + Compute various metrics for generated texts. + Args: generated_texts: List of generated texts to evaluate - reference_texts: List of reference texts (ground truth) or list of lists for multiple references + reference_texts: List of reference texts (ground truth) or list of lists of reference texts original_texts: List of source texts - config: Configuration for metrics (dict, DictConfig, or MetricsConfig) - model: Model instance (required for BigBenchHard) - tokenizer: Tokenizer instance (required for BigBenchHard) - cache_dir: Cache directory for storing intermediate results - + config: Configuration for evaluation + - additional_metrics: List of additional metrics to use. Options include: + - "bartscore": BARTScore metrics + - "alignscore": AlignScore metrics + - DeepEval metrics (requires API key): + - "deepeval_answer_relevance": Evaluates how well the output answers the input + - "deepeval_faithfulness": Evaluates factual consistency with the input + - "deepeval_summarization": Evaluates summarization quality + - "deepeval_prompt_alignment": Evaluates alignment with the expected output + - provider: API key for the provider + - api_key: Model identifier to use + - model: Provider name (openai, anthropic, openrouter, or custom) + - base_url: API base URL (if None, uses default for the provider) + - deepeval_threshold: Threshold for DeepEval metrics (default: 0.5) + - deepeval_include_reason: Include reason for evaluation score (default: False) + - deepeval_strict_mode: Enforce binary metric score (default: False) + - deepeval_async_mode: Enable concurrent execution (default: True) + - deepeval_verbose_mode: Print intermediate steps (default: False) + - deepeval_truths_extraction_limit: Maximum number of factual truths to extract (default: None) Returns: - Dictionary with metric scores and timing information + Dictionary with metric scores + + Note: + The OpenRouterLLM class is also available for direct use with DeepEval metrics: + + ```python + from atgen.metrics import OpenRouterLLM + from deepeval.metrics import AnswerRelevanceMetric + + llm = OpenRouterLLM( + api_key="your_openrouter_api_key", + model="openai/gpt-4o-2024-11-20" + ) + + metric = AnswerRelevanceMetric(model=llm) + ``` """ - start_total = time() - - predictions = generated_texts - references = reference_texts - - # Handle cache_dir in the config - if cache_dir is not None and isinstance(config, dict): - config = dict(config) # Make a copy - config["cache_dir"] = cache_dir - - # Handle different config types - if config is None: - # Default configuration - metrics_config = MetricsConfig( - metrics=["bleu", "rouge"], - batch_size=32, - device="cuda", - aggregate=True + # Load metrics that are always used + sacrebleu = load("sacrebleu", cache_dir=cache_dir) + rouge = load("rouge", cache_dir=cache_dir) + + result = {} + result["word_length_gen"] = np.array( + [len(text.split()) for text in generated_texts] + ) + + time_dict = {} + + # Metrics that use both the generated texts and the original texts and + # those that do not require reference texts + src_word_lengths = np.array([len(text.split()) for text in original_texts]) + + # Avoid division by zero + src_word_lengths_safe = np.where(src_word_lengths > 0, src_word_lengths, 1) + result["word_length_src_rel"] = result["word_length_gen"] / src_word_lengths_safe + if "bartscore" in config.additional_metrics and is_bart_score_available: + log.info("Calculating BARTScore scores...") + start_time = time() + result.update( + calculate_bart_score( + preds=generated_texts, + texts=original_texts, + refs=reference_texts, + batch_size=4, + cache_dir=cache_dir, + ) ) - elif isinstance(config, dict): - metrics_config = MetricsConfig(**config) - elif isinstance(config, DictConfig): - # Convert OmegaConf to dict then to MetricsConfig - config_dict = dict(config) - metrics_config = MetricsConfig(**config_dict) - else: - metrics_config = config - - # Validate inputs - if not predictions: - raise ValueError("predictions cannot be empty") - - # Get metric requirements - requirements = get_metric_requirements() - - # Create metrics using factory - metrics = MetricsFactory.create_metrics(metrics_config) - - if not metrics: - log.warning("No metrics were successfully created") - return {} - - results = {} - timing_info = {} - - # Basic statistics - results["word_length_gen"] = float(np.mean([len(text.split()) for text in predictions])) - - if original_texts: - src_lengths = np.array([len(text.split()) for text in original_texts]) - gen_lengths = np.array([len(text.split()) for text in predictions]) - # Avoid division by zero - src_lengths_safe = np.where(src_lengths > 0, src_lengths, 1) - results["word_length_src_rel"] = float(np.mean(gen_lengths / src_lengths_safe)) - - if references: - if isinstance(references[0], list): - ref_lengths = np.array([len(refs[0].split()) for refs in references]) + time_dict["time_bartscore"] = time() - start_time + # Metrics that use both the generated texts and the reference texts + if reference_texts is not None: + # Exact match + if isinstance(reference_texts[0], list): + result["exact_match"] = np.array( + [ + any(pred == one_ref for one_ref in ref) + for pred, ref in zip(generated_texts, reference_texts) + ] + ) else: - ref_lengths = np.array([len(text.split()) for text in references]) - - gen_lengths = np.array([len(text.split()) for text in predictions]) - if isinstance(references[0], list): - exact_matches = [ - any(pred == ref for ref in ref_list) - for pred, ref_list in zip(predictions, references) + result["exact_match"] = np.array( + [pred == ref for pred, ref in zip(generated_texts, reference_texts)] + ) + # BLEU + start_time = time() + result["bleu"] = np.array( + [ + pair_bleu(references=ref, prediction=pred) + for pred, ref in tqdm(zip(generated_texts, reference_texts)) ] - else: - exact_matches = [pred == ref for pred, ref in zip(predictions, references)] - results["exact_match"] = float(np.mean(exact_matches)) - - ref_lengths_safe = np.where(ref_lengths > 0, ref_lengths, 1) - results["word_length_rel"] = float(np.mean(gen_lengths / ref_lengths_safe)) - - for metric_name, metric in metrics.items(): - log.info(f"Computing {metric_name}...") + ) + time_dict["time_bleu"] = time() - start_time + # ROUGE start_time = time() - - try: - req = requirements.get(metric_name, {}) - - if req.get("requires_references", False) and references is None: - log.warning(f"Skipping {metric_name}: requires references but none provided") - continue - - if req.get("requires_original_texts", False) and original_texts is None: - log.warning(f"Skipping {metric_name}: requires original texts but none provided") - continue - - if metric_name in ["bigbench_hard", "big_bench_hard"]: - if model is None or tokenizer is None: - log.warning(f"Skipping {metric_name}: requires model and tokenizer") - continue - - metric.set_model_and_tokenizer(model, tokenizer) - metric_results = metric.calculate([], [], []) - else: - metric_results = metric.calculate(predictions, references, original_texts) - - for key, value in metric_results.items(): - results[key] = value - - timing_info[f"time_{metric_name}"] = time() - start_time - log.info(f"Completed {metric_name} in {timing_info[f'time_{metric_name}']:.2f}s") - - except Exception as e: - log.error(f"Error computing {metric_name}: {e}") - timing_info[f"time_{metric_name}"] = time() - start_time - - # Add total timing - timing_info["time_total"] = time() - start_total - results.update(timing_info) - - log.info(f"Computed {len(metrics)} metrics in {timing_info['time_total']:.2f}s") - return results - - -def compute_metrics_from_config( - predictions: List[str], - references: Optional[List[Union[str, List[str]]]] = None, - original_texts: Optional[List[str]] = None, - config_dict: Dict = None, - model=None, - tokenizer=None, -) -> Dict[str, float]: - """ - Convenience function to compute metrics from a configuration dictionary. - - Args: - predictions: List of generated texts - references: List of reference texts - original_texts: List of source texts - config_dict: Configuration dictionary - model: Model instance (for BigBenchHard) - tokenizer: Tokenizer instance (for BigBenchHard) - - Returns: - Dictionary with metric scores - """ - return compute_metrics( - generated_texts=predictions, - reference_texts=references, - original_texts=original_texts, - config=config_dict, - model=model, - tokenizer=tokenizer - ) + result.update( + rouge.compute( + predictions=generated_texts, + references=reference_texts, + use_stemmer=True, + ) + ) + time_dict["time_rouge"] = time() - start_time + # Sacrebleu + start_time = time() + if not isinstance(reference_texts[0], list): + sacrebleu_references = [[ref] for ref in reference_texts] + sacrebleu_result = sacrebleu.compute( + predictions=generated_texts, references=sacrebleu_references + ) + result["sacrebleu"] = sacrebleu_result.pop("score") + else: + sacrebleu_scores = [] + for pred, ref in zip(generated_texts, reference_texts): + sacrebleu_result = sacrebleu.compute( + predictions=[pred], references=[ref] + ) + sacrebleu_scores.append(sacrebleu_result.pop("score")) + result["sacrebleu"] = sacrebleu_scores + time_dict["time_sacrebleu"] = time() - start_time + # Lengths + if isinstance(reference_texts[0], list): + ref_word_lengths = np.array( + [ + np.mean([len(text.split()) for text in ref]) + for ref in reference_texts + ] + ) + else: + ref_word_lengths = np.array([len(ref.split()) for ref in reference_texts]) + # Avoid division by zero + ref_word_lengths_safe = np.where(ref_word_lengths > 0, ref_word_lengths, 1) + result["word_length_rel"] = result["word_length_gen"] / ref_word_lengths_safe -def get_default_config() -> MetricsConfig: - """Get a default metrics configuration.""" - return MetricsConfig( - metrics=["bleu", "rouge"], - batch_size=32, - device="cuda", - cache_dir="cache", - aggregate=True - ) + # AlignScore + if "alignscore" in config.additional_metrics and is_alignscore_available: + log.info("Calculating AlignScore scores...") + start_time = time() + alignscores = calculate_alignscore( + generated_texts, reference_texts, original_texts + ) + if alignscores is not None: + result.update(alignscores) + time_dict["time_alignscore"] = time() - start_time + # DeepEval metrics + deepeval_metrics_to_calculate = [ + metric for metric in DEEPEVAL_METRICS if metric in config.additional_metrics + ] -def get_comprehensive_config() -> MetricsConfig: - """Get a comprehensive metrics configuration with all metrics.""" - return MetricsConfig( - metrics=[ - "bleu", "rouge", - "sentbert", - "cola", - ], - batch_size=16, - device="cuda", - cache_dir="cache", - aggregate=True - ) + if deepeval_metrics_to_calculate: + if isinstance(reference_texts[0], list): + log.error("DeepEval does not support multiple references. Skipping...") + else: + # Validate OpenRouter model - only warn if not in predefined list, but still use it + provider = config["provider"] + if config.model not in API_MODELS.get(provider): + log.warning( + f"Using custom model: {config.model}. " + + ( + f"Available models: {API_MODELS[provider]}" + if provider in API_MODELS + else "" + ) + ) + log.info( + f"Calculating DeepEval metrics: {', '.join(deepeval_metrics_to_calculate)}..." + ) + start_time = time() + result.update( + calculate_deepeval_metrics( + predictions=generated_texts, + references=reference_texts, + original_texts=original_texts, + metrics_to_calculate=deepeval_metrics_to_calculate, + base_url=config.base_url, + api_key=config.api_key, + model=config.model, + threshold=config.deepeval_threshold, + include_reason=config.deepeval_include_reason, + strict_mode=config.deepeval_strict_mode, + async_mode=config.deepeval_async_mode, + verbose_mode=config.deepeval_verbose_mode, + truths_extraction_limit=config.deepeval_truths_extraction_limit, + ) + ) + time_dict["time_deepeval"] = time() - start_time + for key, value in result.items(): + if isinstance(value, np.ndarray): + result[key] = float(np.mean(value)) + elif isinstance(value, (int, float)): + # Ensure numerical values are converted to float + result[key] = float(value) + # Make sure non-numerical values that aren't reasons are preserved + elif not key.endswith("_reasons") and not "_reason" in key.lower(): + continue -def get_deepeval_config(api_key: str, model: str = "openai/gpt-4o-mini") -> MetricsConfig: - return MetricsConfig( - metrics=[ - "deepeval_answer_relevance", - "deepeval_faithfulness", - "deepeval_summarization", - "deepeval_prompt_alignment" - ], - batch_size=8, - device="cpu", - cache_dir="cache", - aggregate=True, - api_key=api_key, - base_url="https://openrouter.ai/api/v1", - model=model, - threshold=0.5, - async_mode=True, - verbose_mode=False - ) + # Filter out reason fields from the final aggregated results - more robust filtering + result = { + key: value + for key, value in sorted(result.items()) + if not key.endswith("_reasons") + and not "_reason" in key.lower() + and isinstance(value, (int, float)) # Ensure we only keep numerical metrics + } + + return result diff --git a/src/atgen/utils/data/get_preprocess_function.py b/src/atgen/utils/data/get_preprocess_function.py index 6ae2182..0110654 100644 --- a/src/atgen/utils/data/get_preprocess_function.py +++ b/src/atgen/utils/data/get_preprocess_function.py @@ -90,7 +90,9 @@ def preprocess_fn(instance): elif split == "train": # For training, add the assistant's response if assistant_response_start: - assistant_message = assistant_response_start + instance[output_column_name] + assistant_message = ( + assistant_response_start + instance[output_column_name] + ) else: assistant_message = instance[output_column_name] messages.append({"role": "assistant", "content": assistant_message}) @@ -131,7 +133,9 @@ def preprocess_fn(instance): # For training, add the assistant's response if split == "train": if assistant_response_start: - assistant_message = assistant_response_start + instance[output_column_name] + assistant_message = ( + assistant_response_start + instance[output_column_name] + ) else: assistant_message = instance[output_column_name] messages.append({"role": "assistant", "content": assistant_message}) diff --git a/src/atgen/utils/generate.py b/src/atgen/utils/generate.py index 87bf617..ef080c2 100644 --- a/src/atgen/utils/generate.py +++ b/src/atgen/utils/generate.py @@ -105,7 +105,6 @@ def generate_vllm( batch_generations = [x.outputs[0].text for x in out] generations += batch_generations - import pdb; pdb.set_trace() generations = post_process_generations( generations=generations, data_config=data_config, From 7f5515d9b22371490ab6e684972a42713b33494e Mon Sep 17 00:00:00 2001 From: alfekka Date: Fri, 13 Jun 2025 21:33:43 +0300 Subject: [PATCH 38/39] conflict resolve --- src/atgen/metrics/bart_score.py | 118 -------------------------------- 1 file changed, 118 deletions(-) delete mode 100644 src/atgen/metrics/bart_score.py diff --git a/src/atgen/metrics/bart_score.py b/src/atgen/metrics/bart_score.py deleted file mode 100644 index 6133693..0000000 --- a/src/atgen/metrics/bart_score.py +++ /dev/null @@ -1,118 +0,0 @@ -# %% -import traceback -from typing import List -from tqdm import tqdm - -import numpy as np -import torch -import torch.nn as nn -from transformers import BartTokenizer, BartForConditionalGeneration - - -class BARTScorer: - def __init__( - self, - device="cuda", # device="cuda:0", - max_length=1024, - checkpoint="facebook/bart-large-cnn", - cache_dir: str = "cache", - ): - # Set up model - self.device = device - self.max_length = max_length - self.tokenizer = BartTokenizer.from_pretrained(checkpoint, cache_dir=cache_dir) - self.model = BartForConditionalGeneration.from_pretrained( - checkpoint, cache_dir=cache_dir - ) - self.model.eval() - self.model.to(device) - - # Set up loss - self.loss_fct = nn.NLLLoss( - reduction="none", ignore_index=self.model.config.pad_token_id - ) - self.lsm = nn.LogSoftmax(dim=1) - - def load(self, path=None): - """Load model from paraphrase finetuning""" - if path is None: - path = "models/bart.pth" - self.model.load_state_dict(torch.load(path, map_location=self.device)) - - def score(self, srcs, tgts, batch_size=4): - """Score a batch of examples""" - score_list = [] - for i in range(0, len(srcs), batch_size): - src_list = srcs[i : i + batch_size] - tgt_list = tgts[i : i + batch_size] - try: - with torch.no_grad(): - encoded_src = self.tokenizer( - src_list, - max_length=self.max_length, - truncation=True, - padding=True, - return_tensors="pt", - ) - encoded_tgt = self.tokenizer( - tgt_list, - max_length=self.max_length, - truncation=True, - padding=True, - return_tensors="pt", - ) - src_tokens = encoded_src["input_ids"].to(self.device) - src_mask = encoded_src["attention_mask"].to(self.device) - - tgt_tokens = encoded_tgt["input_ids"].to(self.device) - tgt_mask = encoded_tgt["attention_mask"] - tgt_len = tgt_mask.sum(dim=1).to(self.device) - - output = self.model( - input_ids=src_tokens, attention_mask=src_mask, labels=tgt_tokens - ) - logits = output.logits.view(-1, self.model.config.vocab_size) - loss = self.loss_fct(self.lsm(logits), tgt_tokens.view(-1)) - loss = loss.view(tgt_tokens.shape[0], -1) - loss = loss.sum(dim=1) / tgt_len - curr_score_list = [-x.item() for x in loss] - score_list += curr_score_list - - except RuntimeError: - traceback.print_exc() - print(f"source: {src_list}") - print(f"target: {tgt_list}") - exit(0) - return score_list - - def multi_ref_score(self, srcs, tgts: List[List[str]], agg="mean", batch_size=4): - # Assert we have the same number of references - ref_nums = [len(x) for x in tgts] - if len(set(ref_nums)) > 1: - raise Exception("You have different number of references per test sample.") - - ref_num = len(tgts[0]) - score_matrix = [] - for i in range(ref_num): - curr_tgts = [x[i] for x in tgts] - scores = self.score(srcs, curr_tgts, batch_size) - score_matrix.append(scores) - if agg == "mean": - score_list = np.mean(score_matrix, axis=0) - elif agg == "max": - score_list = np.max(score_matrix, axis=0) - else: - raise NotImplementedError - return list(score_list) - - def test(self, batch_size=3): - """Test""" - src_list = [ - "This is a very good idea. Although simple, but very insightful.", - "Can I take a look?", - "Do not trust him, he is a liar.", - ] - - tgt_list = ["That's stupid.", "What's the problem?", "He is trustworthy."] - - print(self.score(src_list, tgt_list, batch_size)) From dadaf0ba318f041357ec3d0b960f6ee5ad260638 Mon Sep 17 00:00:00 2001 From: alfekka Date: Fri, 13 Jun 2025 21:35:13 +0300 Subject: [PATCH 39/39] conflict resolve --- src/atgen/metrics/bart_score.py | 118 ++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 src/atgen/metrics/bart_score.py diff --git a/src/atgen/metrics/bart_score.py b/src/atgen/metrics/bart_score.py new file mode 100644 index 0000000..6133693 --- /dev/null +++ b/src/atgen/metrics/bart_score.py @@ -0,0 +1,118 @@ +# %% +import traceback +from typing import List +from tqdm import tqdm + +import numpy as np +import torch +import torch.nn as nn +from transformers import BartTokenizer, BartForConditionalGeneration + + +class BARTScorer: + def __init__( + self, + device="cuda", # device="cuda:0", + max_length=1024, + checkpoint="facebook/bart-large-cnn", + cache_dir: str = "cache", + ): + # Set up model + self.device = device + self.max_length = max_length + self.tokenizer = BartTokenizer.from_pretrained(checkpoint, cache_dir=cache_dir) + self.model = BartForConditionalGeneration.from_pretrained( + checkpoint, cache_dir=cache_dir + ) + self.model.eval() + self.model.to(device) + + # Set up loss + self.loss_fct = nn.NLLLoss( + reduction="none", ignore_index=self.model.config.pad_token_id + ) + self.lsm = nn.LogSoftmax(dim=1) + + def load(self, path=None): + """Load model from paraphrase finetuning""" + if path is None: + path = "models/bart.pth" + self.model.load_state_dict(torch.load(path, map_location=self.device)) + + def score(self, srcs, tgts, batch_size=4): + """Score a batch of examples""" + score_list = [] + for i in range(0, len(srcs), batch_size): + src_list = srcs[i : i + batch_size] + tgt_list = tgts[i : i + batch_size] + try: + with torch.no_grad(): + encoded_src = self.tokenizer( + src_list, + max_length=self.max_length, + truncation=True, + padding=True, + return_tensors="pt", + ) + encoded_tgt = self.tokenizer( + tgt_list, + max_length=self.max_length, + truncation=True, + padding=True, + return_tensors="pt", + ) + src_tokens = encoded_src["input_ids"].to(self.device) + src_mask = encoded_src["attention_mask"].to(self.device) + + tgt_tokens = encoded_tgt["input_ids"].to(self.device) + tgt_mask = encoded_tgt["attention_mask"] + tgt_len = tgt_mask.sum(dim=1).to(self.device) + + output = self.model( + input_ids=src_tokens, attention_mask=src_mask, labels=tgt_tokens + ) + logits = output.logits.view(-1, self.model.config.vocab_size) + loss = self.loss_fct(self.lsm(logits), tgt_tokens.view(-1)) + loss = loss.view(tgt_tokens.shape[0], -1) + loss = loss.sum(dim=1) / tgt_len + curr_score_list = [-x.item() for x in loss] + score_list += curr_score_list + + except RuntimeError: + traceback.print_exc() + print(f"source: {src_list}") + print(f"target: {tgt_list}") + exit(0) + return score_list + + def multi_ref_score(self, srcs, tgts: List[List[str]], agg="mean", batch_size=4): + # Assert we have the same number of references + ref_nums = [len(x) for x in tgts] + if len(set(ref_nums)) > 1: + raise Exception("You have different number of references per test sample.") + + ref_num = len(tgts[0]) + score_matrix = [] + for i in range(ref_num): + curr_tgts = [x[i] for x in tgts] + scores = self.score(srcs, curr_tgts, batch_size) + score_matrix.append(scores) + if agg == "mean": + score_list = np.mean(score_matrix, axis=0) + elif agg == "max": + score_list = np.max(score_matrix, axis=0) + else: + raise NotImplementedError + return list(score_list) + + def test(self, batch_size=3): + """Test""" + src_list = [ + "This is a very good idea. Although simple, but very insightful.", + "Can I take a look?", + "Do not trust him, he is a liar.", + ] + + tgt_list = ["That's stupid.", "What's the problem?", "He is trustworthy."] + + print(self.score(src_list, tgt_list, batch_size))