Feature/metrics refactor

.deepeval-cache.json

@@ -0,0 +1 @@
+{"test_cases_lookup_map": {"{\"actual_output\": \"This is a test.\", \"context\": null, \"expected_output\": null, \"hyperparameters\": null, \"input\": \"Source text one.\", \"retrieval_context\": null}": {"cached_metrics_data": [{"metric_data": {"name": "Answer Relevancy", "threshold": 0.5, "success": false, "score": 0.0, "strictMode": false, "evaluationModel": "EvaluationLLM: openai/gpt-4o-mini", "evaluationCost": 0, "verboseLogs": "Statements:\n[\n    \"This is a test.\"\n] \n \nVerdicts:\n[\n    {\n        \"verdict\": \"no\",\n        \"reason\": \"The statement does not provide any relevant information related to the input, which appears to have no context or meaning related to 'Source text one.'\"\n    }\n]"}, "metric_configuration": {"threshold": 0.5, "evaluation_model": "EvaluationLLM: openai/gpt-4o-mini", "strict_mode": false, "include_reason": false}}]}, "{\"actual_output\": \"Another test.\", \"context\": null, \"expected_output\": null, \"hyperparameters\": null, \"input\": \"Source text two.\", \"retrieval_context\": null}": {"cached_metrics_data": [{"metric_data": {"name": "Answer Relevancy", "threshold": 0.5, "success": false, "score": 0.0, "strictMode": false, "evaluationModel": "EvaluationLLM: openai/gpt-4o-mini", "evaluationCost": 0, "verboseLogs": "Statements:\n[\n    \"Another test.\"\n] \n \nVerdicts:\n[\n    {\n        \"verdict\": \"no\",\n        \"reason\": \"The statement 'Another test.' does not provide any relevant information to the input 'Source text two.'\"\n    }\n]"}, "metric_configuration": {"threshold": 0.5, "evaluation_model": "EvaluationLLM: openai/gpt-4o-mini", "strict_mode": false, "include_reason": false}}]}}}

.deepeval_telemetry.txt

@@ -0,0 +1,4 @@
+DEEPEVAL_ID=5a40c1ca-390b-4d9c-8f2e-704b78207dc2
+DEEPEVAL_STATUS=old
+DEEPEVAL_LAST_FEATURE=evaluation
+DEEPEVAL_EVALUATION_STATUS=old

.github/workflows/python-app.yml

@@ -26,9 +26,7 @@ jobs:
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        chmod +x install.sh
         pip install -e .
-        ./install.sh
         pip install flake8 pytest
     - name: Test with pytest
       run: |

LICENSE.md

@@ -1,4 +1,4 @@
-Copyright (c) 2023 MBZUAI
+Copyright (c) 2025 MBZUAI
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

README.md

@@ -1,8 +1,104 @@
-# ATGen: Active Text Generation
+# ATGen: Active Learning for Natural Language Generation
 
-## How to launch without config
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+
+A comprehensive toolkit for applying active learning techniques to natural language generation tasks. This repository contains implementations of various active learning strategies specifically designed for text generation models, helping to reduce annotation costs while maximizing model performance.
+
+## 🌟 Features
+
+- **Multiple Active Learning Strategies**: Implementation of strategies like HUDS, HADAS, FAC-LOC, IDDS, and more
+- **Flexible Model Support**: Compatible with various language models (Qwen, Llama, etc.)
+- **Comprehensive Evaluation**: Supports multiple evaluation metrics including ROUGE, BLEU, BERTScore, AlignScore, etc.
+- **Interactive Visualization**: Streamlit dashboard for exploring results and comparing strategies
+- **Hydra Configuration**: Easily configurable experiments through Hydra's YAML-based configuration system
+- **PEFT Integration**: Efficient fine-tuning using Parameter-Efficient Fine-Tuning methods
+
+## 📋 Requirements
+
+- Python 3.10+
+- CUDA-compatible GPU (for model training)
+- Dependencies listed in `requirements.txt`
+
+## 🔧 Installation
+
+`pip install atgen`
+
+## 🚀 Usage
+
+### Running Active Learning Experiments
+
+Experiments can be launched using the `run-al` command:
+
+```bash
+CUDA_VISIBLE_DEVICES=0 HYDRA_CONFIG_NAME=base run-al
+```
+
+Parameters:
+- `CUDA_VISIBLE_DEVICES`: Specify which GPU to use
+- `HYDRA_CONFIG_NAME`: Configuration file (e.g., `base`, `custom`, `test`)
+
+Additional parameters can be overridden via the command line following Hydra's syntax:
+
+```bash
+CUDA_VISIBLE_DEVICES=0 HYDRA_CONFIG_NAME=base run-al al.strategy=huds model.checkpoint=Qwen/Qwen2.5-7B
+```
+
+### Interactive Dashboard
+
+Launch the Streamlit application to explore and visualize your experiments:
 
 ```bash
-HYDRA_CONFIG_NAME=base HYDRA_CONFIG_PATH=./../../../configs python3 src/atgen/run_scripts/run_active_learning.py al.query_size=10 al.num_iterations=5 a
-l.query_size=10 data.dataset=Harvard/gigaword data.input_column_name=document data.output_column_name=summary labeler.type=golden al.strategy=random
-```
+streamlit run Welcome.py
+```
+
+Navigate to `http://localhost:8501` in your web browser to access the dashboard.
+
+## 📁 Project Structure
+
+- `configs/`: Configuration files for experiments
+  - `al/`: Active learning strategy configurations
+  - `data/`: Dataset configurations
+  - `labeller/`: Labeller configurations
+- `src/atgen/`: Main package
+  - `strategies/`: Implementation of active learning strategies
+  - `metrics/`: Code for evaluation metrics
+  - `utils/`: Utility functions
+  - `run_scripts/`: Scripts for running experiments
+  - `labellers/`: Labelling mechanisms
+  - `visualize/`: Visualization tools
+- `pages/`: Streamlit application pages
+- `outputs/`: Experimental results storage
+- `cache/`: Cached computations to speed up repeated runs
+
+## 📚 Supported Active Learning Strategies
+
+- `huds`: Hypothetical Document Scoring
+- `hadas`: Harmonic Diversity Scoring
+- `random`: Random sampling baseline
+- `fac-loc`: Facility Location strategy
+- `idds`: Improved Diverse Density Scoring
+- And more...
+
+## 📊 Supported Datasets
+
+The toolkit comes pre-configured for several datasets including summarization, question answering, and other generative tasks. Custom datasets can be added by creating new configuration files.
+
+## 🤝 Contributing
+
+Contributions are welcome! Please feel free to submit a Pull Request.
+
+## 📜 License
+
+This project is licensed under the MIT License - see the [LICENSE.md](LICENSE.md) file for details.
+
+## 🔗 Citation
+
+If you use this toolkit in your research, please cite:
+
+```
+@software{atgen,
+  title = {ATGen: Active Learning for Natural Language Generation},
+  url = {https://github.com/Aktsvigun/atgen},
+  year = {2025},
+}
+```

configs/base.yaml

@@ -30,11 +30,12 @@ al:
     budget:
 
 model:
-    checkpoint: Qwen/Qwen2.5-1.5B-Instruct
+    checkpoint: Qwen/Qwen3-1.7B
     quantize: False
     model_max_length: ${multiply_with_few_shot:${data.input_max_length},${data.few_shot.count}}
     dtype: bfloat16
     save_in_fp_32: false
+    assistant_response_start: "<think>\n\n</think>\n\n"
     peft:
         use: True
         r: 16
@@ -80,7 +81,7 @@ evaluation:
     provider: openrouter
     base_url: https://openrouter.ai/api/v1
     api_key:
-    model: anthropic/claude-3-7-sonnet
+    model: nebius/Qwen/Qwen3-235B-A22B
     deepeval_threshold: 0.5
     deepeval_include_reason: False
     deepeval_strict_mode: False

configs/data/aeslc.yaml

@@ -9,4 +9,6 @@ input_max_length: 1024
 output_max_length: 20
 fetch_kwargs: {}
 is_in_conversational_format: false
-system_prompt: "Write a subject text for the email.\nEmail:"
+system_prompt: "Write a subject text for the email.\n\n"
+assistant_response_start: ${model.assistant_response_start}
+use_test_benchmark: false

configs/data/example.yaml

@@ -0,0 +1,17 @@
+dataset: str or list[str]
+few_shot_examples: list[dict[str, str]]
+fetch_kwargs: {}
+few_shot:
+  count: 0
+unlabeled_data_split_name: str
+test_split_name: str
+input_column_name: str
+input_max_length: int
+output_column_name: str or list[str]
+output_max_length: int
+test_subset_size: int
+train_subset_size: int
+system_prompt: str
+assistant_response_start: str
+is_in_conversational_format: bool
+use_test_benchmark: bool

configs/data/gigaword.yaml

@@ -9,3 +9,4 @@ input_max_length: 89
 output_max_length: 21
 fetch_kwargs: {}
 system_prompt: "Generate a headline for the article in lowercase."
+use_test_benchmark: false

configs/data/test.yaml

@@ -12,3 +12,5 @@ system_prompt: "Write a headline for the article in lowercase."
 test_subset_size: 3
 train_subset_size: 100
 is_in_conversational_format: false
+assistant_response_start: ${model.assistant_response_start}
+use_test_benchmark: false

configs/data/trivia_qa.yaml

@@ -0,0 +1,14 @@
+dataset: ['mandarjoshi/trivia_qa', 'rc.wikipedia']
+input_column_name: 'question'
+output_column_name: {'train': ['answer', 'value'], 'test': ['answer', 'aliases']}
+unlabeled_data_split_name: train
+test_split_name: validation
+train_subset_size: null
+test_subset_size: 1_000
+input_max_length: 1024
+output_max_length: 20
+fetch_kwargs: {}
+is_in_conversational_format: false
+system_prompt: "Please answer the question very laconically:\n\n"
+assistant_response_start: ${model.assistant_response_start}
+use_test_benchmark: false

configs/data/user_data.yaml

@@ -10,3 +10,4 @@ output_max_length: 20
 system_prompt: "Write a summary for the article in lowercase."
 test_subset_size: 100
 train_subset_size: 100
+use_test_benchmark: false

configs/data/xsum.yaml

@@ -11,3 +11,4 @@ fetch_kwargs:
   trust_remote_code: true
 is_in_conversational_format: false
 system_prompt: "Summarize the article in one sentence."
+use_test_benchmark: false

configs/test.yaml

@@ -20,11 +20,12 @@ al:
     budget:
 
 model:
-    checkpoint:  Qwen/Qwen2.5-0.5B-Instruct
+    checkpoint: Qwen/Qwen3-0.6B
     quantize: False
     model_max_length: ${multiply_with_few_shot:${data.input_max_length},${data.few_shot.count}}
     dtype: bfloat16
     save_in_fp_32: false
+    assistant_response_start: "<think>\n\n</think>\n\n"
     peft:
         use: True
         r: 64