Revert "Fix"

.gitignore

@@ -1,6 +1,5 @@
 venv
 venv/*
-repos
 TensorRT-LLM
 */dataset
 cache

configs/base.yaml

@@ -17,15 +17,14 @@ name: base
 output_dir: outputs
 offline_mode: False
 cache_dir: cache
-save_model: False
 seed: 42
 al:
     # Strategy configuration now loaded from configs/al/<strategy>.yaml
     # Other AL settings that aren't strategy-specific
     init_query_size: 10
     query_size: 10
     num_iterations: 5
-    eval_zero_iteration: True
+    evaluate_zero_iteration: True
     subsample_size: -1
     required_performance:
         rouge1: 0.5
@@ -40,9 +39,9 @@ model:
     assistant_response_start: "<think>\n\n</think>\n\n"
     peft:
         use: True
-        r: 32
-        lora_alpha: 32
-        lora_dropout: 0.
+        r: 64
+        lora_alpha: 64
+        lora_dropout: 0.1
         bias: 'none'
         seed: ${seed}
         use_gradient_checkpointing: 'unsloth'
@@ -51,8 +50,8 @@ model:
 training:
     dev_split_size: 0.2
     hyperparameters:
-        num_epochs: 15
-        train_batch_size: 8
+        num_epochs: 5
+        train_batch_size: 6
         eval_batch_size: 4
         gradient_accumulation_steps: 2
         lr: 0.00003
@@ -63,7 +62,7 @@ training:
         model_max_length: ${model.model_max_length}
         dataset_num_proc: ${data.num_proc}
         packing: False
-        lr_scheduler_type: linear
+        lr_scheduler_type: cosine
         gradient_checkpointing: True
 
 
@@ -76,7 +75,6 @@ inference:
     temperature: 0.6
     model: ${model.checkpoint}
     num_display_generations: 5
-    num_threads_for_bfcl: 32
 
 evaluation:
     additional_metrics: []
@@ -90,3 +88,13 @@ evaluation:
     deepeval_async_mode: True
     deepeval_verbose_mode: False
     deepeval_truths_extraction_limit: 10
+
+data:
+    assistant_response_start: ${model.assistant_response_start}
+    # test_subset_size: 3
+    # train_subset_size: 100
+    dataset: SpeedOfMagic/gigaword_tiny
+    input_max_length: 100
+    output_max_length: 20
+    is_in_conversational_format: false
+    use_test_benchmark: false

configs/test.yaml

@@ -10,7 +10,6 @@ output_dir: outputs
 offline_mode: False
 cache_dir: cache
 seed: 42
-save_model: False
 al:
     init_query_size: 2
     query_size: 2
@@ -65,7 +64,6 @@ inference:
     top_p: 0.1
     model: ${model.checkpoint}
     num_display_generations: 0
-    num_threads_for_bfcl: 32
 
 evaluation:
     additional_metrics: []

pyproject.toml

@@ -39,7 +39,6 @@ classifiers = [
 
 [project.scripts]
 run-al = "atgen.run_scripts.run_active_learning:main"
-run-ss = "atgen.run_scripts.run_subset_selection:main"
 
 [tool.setuptools.dynamic]
 dependencies = {file = ["requirements.txt"]}

requirements.txt

@@ -1,22 +1,22 @@
 alignscore-SpeedOfMagic
-accelerate>=1.5.2
-anthropic>=0.49.0
+accelerate==1.5.2
+anthropic==0.49.0
 benepar==0.2.0
 bert-score==0.3.13
 bitsandbytes==0.45.3
 ctc_score==0.1.3
-datasets>=3.6.0
+datasets==3.4.1
 deepeval==2.5.5
 evaluate==0.4.3
 hydra-core==1.3.2
 nltk==3.9.1
 kaleido==0.2.1
 omegaconf==2.3.0
-openai>=1.90.0
+openai==1.66.3
 openpyxl==3.1.5
 peft==0.14.0
 plotly==5.23.0
-protobuf>=3.20.3
+protobuf==3.20.3
 pytest==8.3.2
 pytorch_lightning==2.5.0.post0
 rake_nltk==1.0.6
@@ -28,9 +28,9 @@ spacy==3.7.5
 streamlit==1.37.0
 streamlit-authenticator==0.4.2
 tabulate==0.9.0
-transformers>=4.52.4
-trl==0.19.1
+transformers==4.52.4
+trl==0.15.2
 torchmetrics==1.4.1
 unsloth==2025.8.1
-# vllm==0.10.0
+vllm==0.8.1
 xlrd==1.2.0

src/atgen/labellers/api_labellers/openai_labeller.py

@@ -22,7 +22,6 @@
 MAX_NUM_TRIES = 3
 UPDATE_TIME_IN_SECONDS = 10  # update time when checking for the completion
 
-
 class OpenAILabeller(BaseLabeler):
     def __init__(
         self,
@@ -106,7 +105,7 @@ def _sync_call(self, dataset: Dataset) -> Dataset:
         # TODO: make a parameter
         for i, annotation in enumerate(annotations[:2]):
             print(f"Annotation of the {i+1}th instance: {annotation}")
-            print("-" * 100)
+            print("-"*100)
         return dataset
 
     def _batched_call(self, dataset: Dataset) -> Dataset:

src/atgen/labellers/get_labeller.py

@@ -12,7 +12,7 @@
 def get_labeller(
     config: DictConfig,
     output_column_name: str = "output",
-    budget: int | float = 1_000_000,
+    budget: int = 1_000_000,
     workdir: str | Path = "tmp",
     **kwargs,
 ):

src/atgen/metrics/compute_metrics.py

@@ -81,24 +81,13 @@ def compute_metrics(
     elif task == "open-qa":
         metrics_to_calculate = ["exact_match"] + list(config.additional_metrics)
     elif task == "summarization":
-        metrics_to_calculate = [
-            "exact_match",
-            "sacrebleu",
-            "bleu",
-            "rouge",
-            "word_length",
-        ] + list(config.additional_metrics)
+        metrics_to_calculate = ["exact_match", "sacrebleu", "bleu", "rouge", "word_length"] + list(config.additional_metrics)
     elif task == "translation":
-        metrics_to_calculate = [
-            "exact_match",
-            "sacrebleu",
-            "bleu",
-            "word_length",
-        ] + list(config.additional_metrics)
+        metrics_to_calculate = ["exact_match", "sacrebleu", "bleu", "word_length"] + list(config.additional_metrics)
     elif task == "math":
         metrics_to_calculate = ["exact_match_math"] + list(config.additional_metrics)
     else:
-        raise NotImplementedError(f"Task {task} not implemented")
+        raise NotImplementedError(f"Task {task} not implemented")   
 
     if "sacrebleu" in metrics_to_calculate:
         sacrebleu = load("sacrebleu", cache_dir=cache_dir)
@@ -135,19 +124,13 @@ def compute_metrics(
             if isinstance(reference_texts[0], list):
                 result["exact_match"] = np.array(
                     [
-                        any(
-                            _preprocess_text(pred) == _preprocess_text(one_ref)
-                            for one_ref in ref
-                        )
+                        any(_preprocess_text(pred) == _preprocess_text(one_ref) for one_ref in ref)
                         for pred, ref in zip(generated_texts, reference_texts)
                     ]
                 )
             else:
                 result["exact_match"] = np.array(
-                    [
-                        _preprocess_text(pred) == _preprocess_text(ref)
-                        for pred, ref in zip(generated_texts, reference_texts)
-                    ]
+                    [_preprocess_text(pred) == _preprocess_text(ref) for pred, ref in zip(generated_texts, reference_texts)]
                 )
         if "exact_match_math" in metrics_to_calculate:
             # result["exact_match_math"] = np.array(
@@ -212,14 +195,10 @@ def compute_metrics(
                     ]
                 )
             else:
-                ref_word_lengths = np.array(
-                    [len(ref.split()) for ref in reference_texts]
-                )
+                ref_word_lengths = np.array([len(ref.split()) for ref in reference_texts])
             # Avoid division by zero
             ref_word_lengths_safe = np.where(ref_word_lengths > 0, ref_word_lengths, 1)
-            result["word_length_rel"] = (
-                result["word_length_gen"] / ref_word_lengths_safe
-            )
+            result["word_length_rel"] = result["word_length_gen"] / ref_word_lengths_safe
 
         # AlignScore
         if "alignscore" in metrics_to_calculate and is_alignscore_available:
@@ -296,42 +275,31 @@ def compute_metrics(
 
     return result
 
-
-def _preprocess_text(
-    text: str,
-    do_lowercase: bool = True,
-    do_remove_punctuation: bool = True,
-    do_remove_extra_spaces: bool = True,
-    do_remove_stopwords: bool = False,
-    stopwords: Optional[list[str]] = None,
-) -> str:
-    # Convert to lowercase
-    if do_lowercase:
-        text = text.lower()
-
-    # Remove punctuation
-    if do_remove_punctuation:
-        # Keep hyphens within words, remove other punctuation
-        text = re.sub(r"(?<!\w)-|-(?!\w)", " ", text)  # Replace standalone hyphens
-        translator = str.maketrans("", "", string.punctuation.replace("-", ""))
-        text = text.translate(translator)
-        text = re.sub(
-            r"(?<!\w)-(?!\w)", "", text
-        )  # Remove remaining standalone hyphens
-
-    # Normalize whitespace
-    if do_remove_extra_spaces:
-        text = " ".join(text.split())
-
-    # Remove stopwords
-    if do_remove_stopwords:
-        if stopwords is None:
-            import nltk
-
-            nltk.download("stopwords")
-            stopwords = nltk.corpus.stopwords.words("english")
-        words = text.split()
-        words = [w for w in words if w not in stopwords]
-        text = " ".join(words)
-
-    return text.strip()
+def _preprocess_text(text: str, do_lowercase: bool = True, do_remove_punctuation: bool = True, do_remove_extra_spaces: bool = True, do_remove_stopwords: bool = False, stopwords: Optional[list[str]] = None) -> str:
+        # Convert to lowercase
+        if do_lowercase:
+            text = text.lower()
+
+        # Remove punctuation
+        if do_remove_punctuation:
+            # Keep hyphens within words, remove other punctuation
+            text = re.sub(r'(?<!\w)-|-(?!\w)', ' ', text)  # Replace standalone hyphens
+            translator = str.maketrans('', '', string.punctuation.replace('-', ''))
+            text = text.translate(translator)
+            text = re.sub(r'(?<!\w)-(?!\w)', '', text)  # Remove remaining standalone hyphens
+
+        # Normalize whitespace
+        if do_remove_extra_spaces:
+            text = ' '.join(text.split())
+
+        # Remove stopwords
+        if do_remove_stopwords:
+            if stopwords is None:
+                import nltk
+                nltk.download('stopwords')
+                stopwords = nltk.corpus.stopwords.words('english')
+            words = text.split()
+            words = [w for w in words if w not in stopwords]
+            text = ' '.join(words)
+
+        return text.strip()