Fix

.gitignore

@@ -1,5 +1,6 @@
 venv
 venv/*
+repos
 TensorRT-LLM
 */dataset
 cache

configs/base.yaml

@@ -17,14 +17,15 @@ name: base
 output_dir: outputs
 offline_mode: False
 cache_dir: cache
+save_model: False
 seed: 42
 al:
     # Strategy configuration now loaded from configs/al/<strategy>.yaml
     # Other AL settings that aren't strategy-specific
     init_query_size: 10
     query_size: 10
     num_iterations: 5
-    evaluate_zero_iteration: True
+    eval_zero_iteration: True
     subsample_size: -1
     required_performance:
         rouge1: 0.5
@@ -39,9 +40,9 @@ model:
     assistant_response_start: "<think>\n\n</think>\n\n"
     peft:
         use: True
-        r: 64
-        lora_alpha: 64
-        lora_dropout: 0.1
+        r: 32
+        lora_alpha: 32
+        lora_dropout: 0.
         bias: 'none'
         seed: ${seed}
         use_gradient_checkpointing: 'unsloth'
@@ -50,8 +51,8 @@ model:
 training:
     dev_split_size: 0.2
     hyperparameters:
-        num_epochs: 5
-        train_batch_size: 6
+        num_epochs: 15
+        train_batch_size: 8
         eval_batch_size: 4
         gradient_accumulation_steps: 2
         lr: 0.00003
@@ -62,7 +63,7 @@ training:
         model_max_length: ${model.model_max_length}
         dataset_num_proc: ${data.num_proc}
         packing: False
-        lr_scheduler_type: cosine
+        lr_scheduler_type: linear
         gradient_checkpointing: True
 
 
@@ -75,6 +76,7 @@ inference:
     temperature: 0.6
     model: ${model.checkpoint}
     num_display_generations: 5
+    num_threads_for_bfcl: 32
 
 evaluation:
     additional_metrics: []
@@ -88,13 +90,3 @@ evaluation:
     deepeval_async_mode: True
     deepeval_verbose_mode: False
     deepeval_truths_extraction_limit: 10
-
-data:
-    assistant_response_start: ${model.assistant_response_start}
-    # test_subset_size: 3
-    # train_subset_size: 100
-    dataset: SpeedOfMagic/gigaword_tiny
-    input_max_length: 100
-    output_max_length: 20
-    is_in_conversational_format: false
-    use_test_benchmark: false

configs/data/hermes_tool.yaml

@@ -0,0 +1,15 @@
+dataset: 'Aktsvigun/xlam-function-calling-60k'
+input_column_name: 'input'
+output_column_name: 'answers'
+unlabeled_data_split_name: train
+test_split_name: bfcl_python
+train_subset_size: null
+test_subset_size: null
+input_max_length: 4096
+output_max_length: 512
+fetch_kwargs: {}
+is_in_conversational_format: false
+system_prompt: ""
+assistant_response_start: ${model.assistant_response_start}
+use_test_benchmark: true
+task: 'fc'

configs/test.yaml

@@ -10,6 +10,7 @@ output_dir: outputs
 offline_mode: False
 cache_dir: cache
 seed: 42
+save_model: False
 al:
     init_query_size: 2
     query_size: 2
@@ -64,6 +65,7 @@ inference:
     top_p: 0.1
     model: ${model.checkpoint}
     num_display_generations: 0
+    num_threads_for_bfcl: 32
 
 evaluation:
     additional_metrics: []

pyproject.toml

@@ -39,6 +39,7 @@ classifiers = [
 
 [project.scripts]
 run-al = "atgen.run_scripts.run_active_learning:main"
+run-ss = "atgen.run_scripts.run_subset_selection:main"
 
 [tool.setuptools.dynamic]
 dependencies = {file = ["requirements.txt"]}

requirements.txt

@@ -1,22 +1,22 @@
 alignscore-SpeedOfMagic
-accelerate==1.5.2
-anthropic==0.49.0
+accelerate>=1.5.2
+anthropic>=0.49.0
 benepar==0.2.0
 bert-score==0.3.13
 bitsandbytes==0.45.3
 ctc_score==0.1.3
-datasets==3.4.1
+datasets>=3.6.0
 deepeval==2.5.5
 evaluate==0.4.3
 hydra-core==1.3.2
 nltk==3.9.1
 kaleido==0.2.1
 omegaconf==2.3.0
-openai==1.66.3
+openai>=1.90.0
 openpyxl==3.1.5
 peft==0.14.0
 plotly==5.23.0
-protobuf==3.20.3
+protobuf>=3.20.3
 pytest==8.3.2
 pytorch_lightning==2.5.0.post0
 rake_nltk==1.0.6
@@ -28,9 +28,9 @@ spacy==3.7.5
 streamlit==1.37.0
 streamlit-authenticator==0.4.2
 tabulate==0.9.0
-transformers==4.52.4
-trl==0.15.2
+transformers>=4.52.4
+trl==0.19.1
 torchmetrics==1.4.1
 unsloth==2025.8.1
-vllm==0.8.1
+# vllm==0.10.0
 xlrd==1.2.0

src/atgen/labellers/api_labellers/openai_labeller.py

@@ -22,6 +22,7 @@
 MAX_NUM_TRIES = 3
 UPDATE_TIME_IN_SECONDS = 10  # update time when checking for the completion
 
+
 class OpenAILabeller(BaseLabeler):
     def __init__(
         self,
@@ -105,7 +106,7 @@ def _sync_call(self, dataset: Dataset) -> Dataset:
         # TODO: make a parameter
         for i, annotation in enumerate(annotations[:2]):
             print(f"Annotation of the {i+1}th instance: {annotation}")
-            print("-"*100)
+            print("-" * 100)
         return dataset
 
     def _batched_call(self, dataset: Dataset) -> Dataset:

src/atgen/labellers/get_labeller.py

@@ -12,7 +12,7 @@
 def get_labeller(
     config: DictConfig,
     output_column_name: str = "output",
-    budget: int = 1_000_000,
+    budget: int | float = 1_000_000,
     workdir: str | Path = "tmp",
     **kwargs,
 ):

src/atgen/metrics/compute_metrics.py

@@ -81,13 +81,24 @@ def compute_metrics(
     elif task == "open-qa":
         metrics_to_calculate = ["exact_match"] + list(config.additional_metrics)
     elif task == "summarization":
-        metrics_to_calculate = ["exact_match", "sacrebleu", "bleu", "rouge", "word_length"] + list(config.additional_metrics)
+        metrics_to_calculate = [
+            "exact_match",
+            "sacrebleu",
+            "bleu",
+            "rouge",
+            "word_length",
+        ] + list(config.additional_metrics)
     elif task == "translation":
-        metrics_to_calculate = ["exact_match", "sacrebleu", "bleu", "word_length"] + list(config.additional_metrics)
+        metrics_to_calculate = [
+            "exact_match",
+            "sacrebleu",
+            "bleu",
+            "word_length",
+        ] + list(config.additional_metrics)
     elif task == "math":
         metrics_to_calculate = ["exact_match_math"] + list(config.additional_metrics)
     else:
-        raise NotImplementedError(f"Task {task} not implemented")   
+        raise NotImplementedError(f"Task {task} not implemented")
 
     if "sacrebleu" in metrics_to_calculate:
         sacrebleu = load("sacrebleu", cache_dir=cache_dir)
@@ -124,13 +135,19 @@ def compute_metrics(
             if isinstance(reference_texts[0], list):
                 result["exact_match"] = np.array(
                     [
-                        any(_preprocess_text(pred) == _preprocess_text(one_ref) for one_ref in ref)
+                        any(
+                            _preprocess_text(pred) == _preprocess_text(one_ref)
+                            for one_ref in ref
+                        )
                         for pred, ref in zip(generated_texts, reference_texts)
                     ]
                 )
             else:
                 result["exact_match"] = np.array(
-                    [_preprocess_text(pred) == _preprocess_text(ref) for pred, ref in zip(generated_texts, reference_texts)]
+                    [
+                        _preprocess_text(pred) == _preprocess_text(ref)
+                        for pred, ref in zip(generated_texts, reference_texts)
+                    ]
                 )
         if "exact_match_math" in metrics_to_calculate:
             # result["exact_match_math"] = np.array(
@@ -195,10 +212,14 @@ def compute_metrics(
                     ]
                 )
             else:
-                ref_word_lengths = np.array([len(ref.split()) for ref in reference_texts])
+                ref_word_lengths = np.array(
+                    [len(ref.split()) for ref in reference_texts]
+                )
             # Avoid division by zero
             ref_word_lengths_safe = np.where(ref_word_lengths > 0, ref_word_lengths, 1)
-            result["word_length_rel"] = result["word_length_gen"] / ref_word_lengths_safe
+            result["word_length_rel"] = (
+                result["word_length_gen"] / ref_word_lengths_safe
+            )
 
         # AlignScore
         if "alignscore" in metrics_to_calculate and is_alignscore_available:
@@ -275,31 +296,42 @@ def compute_metrics(
 
     return result
 
-def _preprocess_text(text: str, do_lowercase: bool = True, do_remove_punctuation: bool = True, do_remove_extra_spaces: bool = True, do_remove_stopwords: bool = False, stopwords: Optional[list[str]] = None) -> str:
-        # Convert to lowercase
-        if do_lowercase:
-            text = text.lower()
-
-        # Remove punctuation
-        if do_remove_punctuation:
-            # Keep hyphens within words, remove other punctuation
-            text = re.sub(r'(?<!\w)-|-(?!\w)', ' ', text)  # Replace standalone hyphens
-            translator = str.maketrans('', '', string.punctuation.replace('-', ''))
-            text = text.translate(translator)
-            text = re.sub(r'(?<!\w)-(?!\w)', '', text)  # Remove remaining standalone hyphens
-
-        # Normalize whitespace
-        if do_remove_extra_spaces:
-            text = ' '.join(text.split())
-
-        # Remove stopwords
-        if do_remove_stopwords:
-            if stopwords is None:
-                import nltk
-                nltk.download('stopwords')
-                stopwords = nltk.corpus.stopwords.words('english')
-            words = text.split()
-            words = [w for w in words if w not in stopwords]
-            text = ' '.join(words)
-
-        return text.strip()
+
+def _preprocess_text(
+    text: str,
+    do_lowercase: bool = True,
+    do_remove_punctuation: bool = True,
+    do_remove_extra_spaces: bool = True,
+    do_remove_stopwords: bool = False,
+    stopwords: Optional[list[str]] = None,
+) -> str:
+    # Convert to lowercase
+    if do_lowercase:
+        text = text.lower()
+
+    # Remove punctuation
+    if do_remove_punctuation:
+        # Keep hyphens within words, remove other punctuation
+        text = re.sub(r"(?<!\w)-|-(?!\w)", " ", text)  # Replace standalone hyphens
+        translator = str.maketrans("", "", string.punctuation.replace("-", ""))
+        text = text.translate(translator)
+        text = re.sub(
+            r"(?<!\w)-(?!\w)", "", text
+        )  # Remove remaining standalone hyphens
+
+    # Normalize whitespace
+    if do_remove_extra_spaces:
+        text = " ".join(text.split())
+
+    # Remove stopwords
+    if do_remove_stopwords:
+        if stopwords is None:
+            import nltk
+
+            nltk.download("stopwords")
+            stopwords = nltk.corpus.stopwords.words("english")
+        words = text.split()
+        words = [w for w in words if w not in stopwords]
+        text = " ".join(words)
+
+    return text.strip()