Metris

src/atgen/metrics/__init__.py

@@ -1,12 +1,122 @@
-AVAILABLE_METRICS = [
-    "rouge1",
-    "rouge2",
-    "rougeL",
-    "rougeLsum",
-    "bleu",
-    "sacrebleu",
-    "BARTScore-hr",
-    "BARTScore-sh",
-    "alignscore",
-    "BigBenchHardScore",
+# Base classes
+from .base import BaseMetric, MetricConfig
+
+# Lexical metrics
+from .lexical import BleuMetric, RougeMetric
+
+# Semantic metrics  
+from .semantic import BartScoreMetric, AlignScoreMetric, SentBertMetric
+
+# Linguistic metrics
+from .linguistic import ColaMetric
+
+# LLM-based metrics
+from .llm_based import (
+    EvaluationLLM,
+    BaseDeepEvalMetric,
+    DeepEvalAnswerRelevancyMetric,
+    DeepEvalFaithfulnessMetric,
+    DeepEvalSummarizationMetric,
+    DeepEvalPromptAlignmentMetric,
+    BigBenchHardMetric,
+)
+
+# Factory system
+from .factory import (
+    MetricsFactory,
+    MetricsConfig,
+    get_metric_categories,
+    get_metric_requirements,
+)
+
+# Compute system
+from .compute_metrics import (
+    compute_metrics,
+    compute_metrics_from_config,
+    get_default_config,
+    get_comprehensive_config,
+    get_deepeval_config,
+)
+
+
+
+
+
+# Version
+__version__ = "2.0.0"
+
+# Main exports
+__all__ = [
+    # Base classes
+    "BaseMetric",
+    "MetricConfig",
+
+    # Lexical metrics
+    "BleuMetric",
+    "RougeMetric",
+
+    # Semantic metrics
+    "BartScoreMetric",
+    "AlignScoreMetric", 
+    "SentBertMetric",
+
+    # Linguistic metrics
+    "ColaMetric",
+
+    # LLM-based metrics
+    "EvaluationLLM",
+    "BaseDeepEvalMetric",
+    "DeepEvalAnswerRelevancyMetric",
+    "DeepEvalFaithfulnessMetric",
+    "DeepEvalSummarizationMetric",
+    "DeepEvalPromptAlignmentMetric",
+    "BigBenchHardMetric",
+
+    # Factory system
+    "MetricsFactory",
+    "MetricsConfig",
+    "get_metric_categories",
+    "get_metric_requirements",
+
+    # New compute system  
+    "compute_metrics",
+    "compute_metrics_from_config",
+    "get_default_config",
+    "get_comprehensive_config",
+    "get_deepeval_config",
+
+    # Helper functions
+    "get_available_metrics",
+    "get_all_possible_metric_keys",
+    "create_metric",
+    "create_metrics_from_config",
+
+    "AVAILABLE_METRICS",
+    "METRICS",
 ]
+
+
+def get_available_metrics():
+    """Get all available metric names."""
+    return MetricsFactory.get_available_metrics()
+
+
+def get_all_possible_metric_keys():
+    """Get all possible metric keys that can be returned by compute_metrics."""
+    return MetricsFactory.get_all_possible_metric_keys()
+
+
+def create_metric(name: str, config: MetricConfig = None):
+    """Create a metric by name."""
+    return MetricsFactory.create_metric(name, config)
+
+
+def create_metrics_from_config(config):
+    """Create metrics from configuration."""
+    return MetricsFactory.create_from_config(config)
+
+# For backward compatibility, still provide the basic metric names
+METRICS = MetricsFactory.get_available_metrics()
+
+# But AVAILABLE_METRICS should include all possible keys for performance checking
+AVAILABLE_METRICS = MetricsFactory.get_all_possible_metric_keys() 

src/atgen/metrics/base/__init__.py

@@ -0,0 +1,8 @@
+"""Base classes and configuration for metrics."""
+
+from .base_metric import BaseMetric, MetricConfig
+
+__all__ = [
+    "BaseMetric",
+    "MetricConfig",
+] 

src/atgen/metrics/base/base_metric.py

@@ -0,0 +1,99 @@
+from abc import ABC, abstractmethod
+from typing import Optional, Dict, Any
+import logging
+from dataclasses import dataclass, field
+
+
+@dataclass
+class MetricConfig:
+<<<<<<< HEAD
+=======
+    """Base configuration for metrics."""
+    # General parameters
+>>>>>>> a24e0f2 (removed dependencies chek)
+    batch_size: int = 32
+    device: str = "cuda"
+    cache_dir: str = "cache"
+    aggregate: bool = True
+
+<<<<<<< HEAD
+    checkpoint: Optional[str] = None
+    model_name: Optional[str] = None
+
+    # API configuration parameters  
+    provider: Optional[str] = None
+=======
+    # Model-specific parameters (for local models)
+    checkpoint: Optional[str] = None
+    model_name: Optional[str] = None
+
+    # API-based parameters (for LLM-based metrics)
+>>>>>>> a24e0f2 (removed dependencies chek)
+    api_key: Optional[str] = None
+    base_url: Optional[str] = None
+    model: Optional[str] = None
+
+<<<<<<< HEAD
+=======
+    # DeepEval specific parameters
+>>>>>>> a24e0f2 (removed dependencies chek)
+    threshold: float = 0.5
+    include_reason: bool = False
+    strict_mode: bool = False
+    async_mode: bool = True
+    verbose_mode: bool = False
+    truths_extraction_limit: Optional[int] = None
+<<<<<<< HEAD
+=======
+
+    # BigBenchHard specific parameters
+>>>>>>> a24e0f2 (removed dependencies chek)
+    benchmark_params: Dict[str, Any] = field(default_factory=dict)
+    generation_params: Dict[str, Any] = field(default_factory=dict)
+
+
+class BaseMetric(ABC):
+    def __init__(self, config: Optional[MetricConfig] = None):
+        self.config = config or MetricConfig()
+        self.logger = logging.getLogger(self.__class__.__name__)
+<<<<<<< HEAD
+
+        self._validate_config()
+
+    @property
+    def name(self) -> str:
+        return self.__class__.__name__.replace("Metric", "").lower()
+
+    def _validate_config(self):
+=======
+        self._is_available = None
+        self._model = None
+
+        # Validate configuration
+        self._validate_config()
+
+        # Check dependencies
+        if not self.is_available():
+            self.logger.warning(f"{self.__class__.__name__} dependencies not available")
+
+    @property
+    def name(self) -> str:
+        """Return the metric name (defaults to class name)."""
+        return self.__class__.__name__.replace("Metric", "").lower()
+
+    def _validate_config(self):
+        """Validate configuration - can be overridden by subclasses."""
+>>>>>>> a24e0f2 (removed dependencies chek)
+        pass
+
+    @abstractmethod
+    def calculate(self, predictions, references, original_texts):
+        pass
+<<<<<<< HEAD
+
+    def is_available(self) -> bool:
+        """Check if the metric is available for use."""
+        return True
+=======
+>>>>>>> a24e0f2 (removed dependencies chek)
+