From be3d36bd5fc65df6dbe06fff57404cdb68b26511 Mon Sep 17 00:00:00 2001 From: SpeedOfMagic Date: Fri, 13 Jun 2025 15:57:10 +0000 Subject: [PATCH 01/10] fix --- README.md | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index e387c5f..578c617 100644 --- a/README.md +++ b/README.md @@ -21,21 +21,7 @@ A comprehensive toolkit for applying active learning techniques to natural langu ## 🔧 Installation -1. Clone the repository: - ```bash - git clone https://github.com/Aktsvigun/atgen.git - cd atgen - ``` - -2. Run the installation script: - ```bash - bash install.sh - ``` - - This will install: - - All required Python packages - - External metrics (submodlib, AlignScore) - - Required NLP resources +`pip install atgen` ## 🚀 Usage @@ -115,4 +101,4 @@ If you use this toolkit in your research, please cite: url = {https://github.com/Aktsvigun/atgen}, year = {2025}, } -``` \ No newline at end of file +``` From 787d5805d0d905ae17ab83be4c6dd31a6c58955e Mon Sep 17 00:00:00 2001 From: SpeedOfMagic Date: Tue, 29 Jul 2025 10:18:37 +0000 Subject: [PATCH 02/10] fix --- configs/base.yaml | 3 +- pages/0_Configure_Experiment.py | 32 ++++++++-------- src/atgen/run_scripts/run_active_learning.py | 39 +++++++++++--------- 3 files changed, 40 insertions(+), 34 deletions(-) diff --git a/configs/base.yaml b/configs/base.yaml index fae2e4c..5b64119 100644 --- a/configs/base.yaml +++ b/configs/base.yaml @@ -89,4 +89,5 @@ evaluation: deepeval_verbose_mode: False deepeval_truths_extraction_limit: 10 - +data: + assistant_response_start: ${model.assistant_response_start} \ No newline at end of file diff --git a/pages/0_Configure_Experiment.py b/pages/0_Configure_Experiment.py index 8971159..3b57eba 100644 --- a/pages/0_Configure_Experiment.py +++ b/pages/0_Configure_Experiment.py @@ -445,13 +445,13 @@ def main(): col1, col2, col3 = st.columns(3) with col1: if st.button("📊 View Metrics", use_container_width=True): - st.switch_page("1_Metrics") + st.switch_page("pages/1_Metrics.py") with col2: if st.button("🏷️ View Labeled Examples", use_container_width=True): - st.switch_page("2_Labeled_examples") + st.switch_page("pages/2_Labeled_examples.py") with col3: if st.button("👩‍🎨 Annotate Examples", use_container_width=True): - st.switch_page("3_Annotation") + st.switch_page("pages/4_Annotation.py") st.stop() st.markdown( @@ -515,7 +515,7 @@ def main(): with col1: strategy = st.radio( "🎯 AL strategy", - options=["Huds", "Hadas", "IDDS", "Fac-Loc", "Random"], + options=["Random", "Huds", "Hadas", "IDDS", "Fac-Loc"], help="Choose the active learning strategy to use for data selection", ).lower() # TODO: testing, remove this @@ -526,7 +526,7 @@ def main(): "📊 AL query size", min_value=1, step=1, - value=10, + value=2, help="Number of examples to select in each active learning iteration", ) @@ -561,7 +561,7 @@ def main(): min_value=0, max_value=100, step=1, - value=5, + value=2, help="Maximum number of active learning iterations to run", ) @@ -697,7 +697,7 @@ def main(): # Dataset input dataset = st.text_input( "📚 Dataset or path to data", - value="Yale-LILY/aeslc", + value="SpeedOfMagic/gigaword_tiny", help="HuggingFace dataset ID or local path to dataset", ) @@ -821,13 +821,13 @@ def main(): with col1: input_field = st.text_input( "📥 Input field name", - value="email_body", + value="document", help="Name of the field containing input text in the dataset", ) with col2: reference_field = st.text_input( "📤 Reference field name", - value="subject_line", + value="summary", help="Name of the field containing reference output in the dataset", ) @@ -855,7 +855,7 @@ def main(): with col1: model_checkpoint = st.text_input( "🤖 Model checkpoint", - value="Qwen/Qwen3-1.7B", + value="Qwen/Qwen2.5-0.5B-Instruct", help="HuggingFace model ID for generation", ) @@ -880,7 +880,7 @@ def main(): with col1: lora = st.checkbox( "🔧 Use LoRA", - value=True, + value=False, help="Whether to use LoRA for efficient fine-tuning", ) if lora: @@ -987,7 +987,7 @@ def main(): with col3: framework = st.selectbox( "🛠️ Framework", - options=["vLLM", "SGLang", "Unsloth"], + options=["Transformers", "vLLM", "SGLang", "Unsloth"], index=0, help="Framework to use for inference", ).lower() @@ -1016,7 +1016,7 @@ def main(): "deepeval_summarization", "deepeval_prompt_alignment", ], - default=["bartscore", "alignscore"], + default=[], help="Metrics to compute for evaluating model performance", ) @@ -1438,17 +1438,17 @@ def main(): nav_col1, nav_col2, nav_col3 = st.columns(3) with nav_col1: if st.button("📊 View Metrics", use_container_width=True): - st.switch_page("1_Metrics") + st.switch_page("pages/1_Metrics.py") with nav_col2: if st.button( "🏷️ View Labeled Examples", use_container_width=True ): - st.switch_page("2_Labeled_examples") + st.switch_page("pages/2_Labeled_examples.py") with nav_col3: if st.button( "👩‍🎨 Annotate Examples", use_container_width=True ): - st.switch_page("3_Annotation") + st.switch_page("pages/4_Annotation.py") except Exception as e: # Update status to failed update_experiment_status(STATUS_FAILED) diff --git a/src/atgen/run_scripts/run_active_learning.py b/src/atgen/run_scripts/run_active_learning.py index 728f1ef..652178a 100644 --- a/src/atgen/run_scripts/run_active_learning.py +++ b/src/atgen/run_scripts/run_active_learning.py @@ -79,7 +79,7 @@ def run_active_learning(config, workdir: Union[str, Path]): config.data.test_split_name is not None and config.data.test_split_name != "" ) - log.info( + print( f"""Running Active Learning... AL Strategy: {config.al.strategy} Num Iterations: {num_al_iterations} @@ -97,7 +97,7 @@ def run_active_learning(config, workdir: Union[str, Path]): train_output_dir = workdir / "tmp" save_dir = workdir / "tmp_best" - log.info("Loading data.") + print("Loading data.") unlabeled_data = load_data( data_config=config.data, split=UNLABELED_DATA_SPLIT_DEFAULT_NAME, @@ -112,12 +112,12 @@ def run_active_learning(config, workdir: Union[str, Path]): seed=seed, ) - log.info("Initial iteration: loading model & tokenizer.") + print("Initial iteration: loading model & tokenizer.") model, tokenizer = load_model_tokenizer( checkpoint=model_name, model_config=config.model, cache_dir=cache_dir ) - log.info("Loading AL strategy.") + print("Loading AL strategy.") al_strategy: BaseStrategy = get_strategy( config.al.strategy, subsample_size=config.al.subsample_size, @@ -132,6 +132,7 @@ def run_active_learning(config, workdir: Union[str, Path]): **config.al.strategy_kwargs, ) + print("Loading labeller") # TODO: unsure whether need to log here since may be confusing for a human labeller labeller: BaseLabeler = get_labeller( config.labeller, @@ -142,7 +143,7 @@ def run_active_learning(config, workdir: Union[str, Path]): data_config=config.data, # if labeller is a custom LLM on transformers model_config=config.model, # if labeller is a custom LLM on transformers ) - + print("Calculating query_ids") init_query_size = config.al.init_query_size + config.data.few_shot.count init_query_size_is_positive = init_query_size > 0 if init_query_size_is_positive: @@ -175,7 +176,7 @@ def run_active_learning(config, workdir: Union[str, Path]): unlabeled_data = unlabeled_data.filter(lambda x: x["id"] not in query_ids) labeled_data = labeller(query) if labeller.is_out_of_budget: - log.info(f"Labeler ran out of budget at iteration 0.") + print(f"Labeler ran out of budget at iteration 0.") labeled_ids = query_ids # Get the few-shot examples @@ -183,7 +184,7 @@ def run_active_learning(config, workdir: Union[str, Path]): config=config, labeled_data=labeled_data, workdir=workdir ) - log.info(f"Saving labeled data at iteration 0.") + print(f"Saving labeled data at iteration 0.") save_labeled_data( labeled_data=labeled_data, labeled_query=labeled_data, @@ -196,6 +197,7 @@ def run_active_learning(config, workdir: Union[str, Path]): labeled_data = unlabeled_data.select(range(0, 0)) labeled_ids = [] + print("Getting unlabeled data") unlabeled_data: Dataset = prepare_conversational_data( dataset=unlabeled_data, data_config=config.data, @@ -205,6 +207,7 @@ def run_active_learning(config, workdir: Union[str, Path]): ) if has_test: + print("Preparing test data") if not config.data.use_test_benchmark: test_data: Dataset = prepare_conversational_data( dataset=test_data, @@ -214,6 +217,7 @@ def run_active_learning(config, workdir: Union[str, Path]): model_name=model_name, ) # Evaluate the initial model before any training + # import pdb;pdb.set_trace() if init_query_size_is_positive and config.al.evaluate_zero_iteration: generations: list[str] = generate( config.inference, @@ -257,19 +261,20 @@ def run_active_learning(config, workdir: Union[str, Path]): # Start AL cycle. Use `num_al_iterations + 2` because we do not label data # but want to train the model on the last iteration. + start_iter = 1 if init_query_size_is_positive else 0 for al_iter in range(start_iter, num_al_iterations + 1 + start_iter): - log.info(f"Starting AL iteration #{al_iter}.") + print(f"Starting AL iteration #{al_iter}.") iter_dir = workdir / ("iter_" + str(al_iter)) iter_dir.mkdir(exist_ok=True) - log.info(f"Iteration {al_iter}: model loading started...") + print(f"Iteration {al_iter}: model loading started...") if al_iter != 1: model, tokenizer = load_model_tokenizer( checkpoint=model_name, model_config=config.model, cache_dir=cache_dir ) - log.info(f"Iteration {al_iter}: model loading done.") + print(f"Iteration {al_iter}: model loading done.") if not config.data.is_in_conversational_format: train_eval_data = prepare_conversational_data( @@ -309,7 +314,7 @@ def run_active_learning(config, workdir: Union[str, Path]): # Launch training if len(train_data) > 0: train_result = trainer.train() - log.info(f"Training completed with {len(train_data)} examples") + print(f"Training completed with {len(train_data)} examples") else: log.warning( "No labeled training data available. Skipping training for this iteration." @@ -368,7 +373,7 @@ def run_active_learning(config, workdir: Union[str, Path]): # Make AL query for the next round if we have not run out of iterations if al_iter != num_al_iterations + 1: - log.info(f"Making AL query at iteration {al_iter}.") + print(f"Making AL query at iteration {al_iter}.") query_ids: list[str] = al_strategy( model=model, tokenizer=tokenizer, @@ -383,11 +388,11 @@ def run_active_learning(config, workdir: Union[str, Path]): unlabeled_data: Dataset = unlabeled_data.filter(lambda x: x["id"] not in query_ids) labeled_query: Dataset = labeller(query) if labeller.is_out_of_budget: - log.info(f"Labeler ran out of budget at iteration {al_iter}.") + print(f"Labeler ran out of budget at iteration {al_iter}.") labeled_data: Dataset = concatenate_datasets([labeled_data, labeled_query]) labeled_ids += query_ids - log.info(f"Saving labeled data at iteration #{al_iter}.") + print(f"Saving labeled data at iteration #{al_iter}.") save_labeled_data( labeled_data=labeled_data, labeled_query=labeled_query, @@ -402,13 +407,13 @@ def run_active_learning(config, workdir: Union[str, Path]): torch.cuda.empty_cache() if labeller.is_out_of_budget: - log.info("Labeler ran out of budget. Finishing active learning.") + print("Labeler ran out of budget. Finishing active learning.") return if is_performance_reached: - log.info("Stopping AL since the required performance is reached.") + print("Stopping AL since the required performance is reached.") return - log.info("Active learning is done.") + print("Active learning is done.") @hydra.main( From dca3180517fae79c1e0bb9fb3916b33cd22760f3 Mon Sep 17 00:00:00 2001 From: SpeedOfMagic Date: Tue, 29 Jul 2025 10:29:44 +0000 Subject: [PATCH 03/10] fix --- configs/base.yaml | 4 ++-- pages/0_Configure_Experiment.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/configs/base.yaml b/configs/base.yaml index 5b64119..99a31b8 100644 --- a/configs/base.yaml +++ b/configs/base.yaml @@ -38,8 +38,8 @@ model: assistant_response_start: "\n\n\n\n" peft: use: True - r: 16 - lora_alpha: 16 + r: 64 + lora_alpha: 64 lora_dropout: 0.1 bias: 'none' seed: ${seed} diff --git a/pages/0_Configure_Experiment.py b/pages/0_Configure_Experiment.py index 3b57eba..8b67d83 100644 --- a/pages/0_Configure_Experiment.py +++ b/pages/0_Configure_Experiment.py @@ -855,7 +855,7 @@ def main(): with col1: model_checkpoint = st.text_input( "🤖 Model checkpoint", - value="Qwen/Qwen2.5-0.5B-Instruct", + value="Qwen/Qwen3-0.6B", help="HuggingFace model ID for generation", ) @@ -912,7 +912,7 @@ def main(): per_device_train_batch_size = st.number_input( "⚡ Train batch size", min_value=1, - value=4, + value=8, step=1, help="Training batch size per device", ) @@ -920,7 +920,7 @@ def main(): eval_batch_size = st.number_input( "⚡ Evaluation batch size", min_value=1, - value=4, + value=8, step=1, help="Batch size for evaluation", ) From f1ba2404a4c20afaef841f288dd4dc4dc423a50e Mon Sep 17 00:00:00 2001 From: SpeedOfMagic Date: Tue, 29 Jul 2025 11:21:58 +0000 Subject: [PATCH 04/10] fix --- configs/base.yaml | 13 +++++++++---- pages/0_Configure_Experiment.py | 4 ++-- src/atgen/run_scripts/run_active_learning.py | 1 - 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/configs/base.yaml b/configs/base.yaml index 99a31b8..86b9a73 100644 --- a/configs/base.yaml +++ b/configs/base.yaml @@ -75,9 +75,7 @@ inference: model: ${model.checkpoint} evaluation: - additional_metrics: - - alignscore - - bartscore + additional_metrics: [] provider: openrouter base_url: https://openrouter.ai/api/v1 api_key: @@ -90,4 +88,11 @@ evaluation: deepeval_truths_extraction_limit: 10 data: - assistant_response_start: ${model.assistant_response_start} \ No newline at end of file + assistant_response_start: ${model.assistant_response_start} + test_subset_size: 3 + train_subset_size: 100 + dataset: SpeedOfMagic/gigaword_tiny + input_max_length: 100 + output_max_length: 20 + is_in_conversational_format: false + use_test_benchmark: false diff --git a/pages/0_Configure_Experiment.py b/pages/0_Configure_Experiment.py index 8b67d83..d237932 100644 --- a/pages/0_Configure_Experiment.py +++ b/pages/0_Configure_Experiment.py @@ -936,7 +936,7 @@ def main(): num_train_epochs = st.number_input( "🔄 Number of training epochs", min_value=1, - value=5, + value=1, step=1, help="Number of training epochs", ) @@ -994,7 +994,7 @@ def main(): inference_batch_size = st.number_input( "⚡ Inference batch size", min_value=1, - value=4, + value=8, step=1, help="Batch size for inference", ) diff --git a/src/atgen/run_scripts/run_active_learning.py b/src/atgen/run_scripts/run_active_learning.py index 652178a..07361b5 100644 --- a/src/atgen/run_scripts/run_active_learning.py +++ b/src/atgen/run_scripts/run_active_learning.py @@ -217,7 +217,6 @@ def run_active_learning(config, workdir: Union[str, Path]): model_name=model_name, ) # Evaluate the initial model before any training - # import pdb;pdb.set_trace() if init_query_size_is_positive and config.al.evaluate_zero_iteration: generations: list[str] = generate( config.inference, From ef9e1363e0808b0e07862f03807bf7a8cd4cdee4 Mon Sep 17 00:00:00 2001 From: SpeedOfMagic Date: Tue, 29 Jul 2025 11:34:19 +0000 Subject: [PATCH 05/10] done --- configs/base.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/base.yaml b/configs/base.yaml index 86b9a73..8a860d3 100644 --- a/configs/base.yaml +++ b/configs/base.yaml @@ -89,8 +89,8 @@ evaluation: data: assistant_response_start: ${model.assistant_response_start} - test_subset_size: 3 - train_subset_size: 100 + # test_subset_size: 3 + # train_subset_size: 100 dataset: SpeedOfMagic/gigaword_tiny input_max_length: 100 output_max_length: 20 From f5074c707e27e27612d45544a1975f8bfa8f3117 Mon Sep 17 00:00:00 2001 From: SpeedOfMagic Date: Tue, 29 Jul 2025 13:03:24 +0000 Subject: [PATCH 06/10] more --- pages/0_Configure_Experiment.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pages/0_Configure_Experiment.py b/pages/0_Configure_Experiment.py index d237932..1e2050a 100644 --- a/pages/0_Configure_Experiment.py +++ b/pages/0_Configure_Experiment.py @@ -269,6 +269,7 @@ def update_progress(iteration): # Update the status update_experiment_status(STATUS_RUNNING, total_iterations=num_iterations) return + iteration = max(iteration - 2, 0) if iteration > num_iterations: progress = 1.0 From a69dfad5961f78244b66d13e6042dc4406a76992 Mon Sep 17 00:00:00 2001 From: SpeedOfMagic Date: Tue, 29 Jul 2025 13:11:26 +0000 Subject: [PATCH 07/10] more --- pages/0_Configure_Experiment.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pages/0_Configure_Experiment.py b/pages/0_Configure_Experiment.py index 1e2050a..0624726 100644 --- a/pages/0_Configure_Experiment.py +++ b/pages/0_Configure_Experiment.py @@ -269,7 +269,6 @@ def update_progress(iteration): # Update the status update_experiment_status(STATUS_RUNNING, total_iterations=num_iterations) return - iteration = max(iteration - 2, 0) if iteration > num_iterations: progress = 1.0 @@ -427,6 +426,9 @@ def main(): pdb.set_trace() + cur_iter = running_experiment.get('current_iteration', '?') + if cur_iter != '?' and cur_iter >= running_experiment.get('total_iterations', '?'): + cur_iter = running_experiment.get('total_iterations', '?') st.warning( f"⚠️ An experiment '{running_experiment.get('experiment_name', 'Unknown')}' is already running! " f"Current iteration: {running_experiment.get('current_iteration', '?')}/{running_experiment.get('total_iterations', '?')}. " From e54dca5e825e1f26f024389d2f983ca966d8365f Mon Sep 17 00:00:00 2001 From: SpeedOfMagic Date: Tue, 29 Jul 2025 13:13:17 +0000 Subject: [PATCH 08/10] fix 3 --- pages/0_Configure_Experiment.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pages/0_Configure_Experiment.py b/pages/0_Configure_Experiment.py index 0624726..21bec46 100644 --- a/pages/0_Configure_Experiment.py +++ b/pages/0_Configure_Experiment.py @@ -270,6 +270,10 @@ def update_progress(iteration): update_experiment_status(STATUS_RUNNING, total_iterations=num_iterations) return + if iteration == num_iterations + 2: + update_experiment_status(STATUS_COMPLETED) + return + if iteration > num_iterations: progress = 1.0 else: From 7a6268d2aa45f9ee3e13b8ebeaaa689f5554a2fb Mon Sep 17 00:00:00 2001 From: SpeedOfMagic Date: Tue, 29 Jul 2025 13:18:31 +0000 Subject: [PATCH 09/10] more --- pages/0_Configure_Experiment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pages/0_Configure_Experiment.py b/pages/0_Configure_Experiment.py index 21bec46..83f49ba 100644 --- a/pages/0_Configure_Experiment.py +++ b/pages/0_Configure_Experiment.py @@ -436,7 +436,7 @@ def main(): st.warning( f"⚠️ An experiment '{running_experiment.get('experiment_name', 'Unknown')}' is already running! " f"Current iteration: {running_experiment.get('current_iteration', '?')}/{running_experiment.get('total_iterations', '?')}. " - f"The experiment was most likely started by one of the reviewers, so kindly wait for it to finish." + f"Please kindly wait for it to finish." ) # Show option to force reset the status (in case of stale status) From 1b6587c3b41c97e41ffebd4521f60d2ed771d81c Mon Sep 17 00:00:00 2001 From: SpeedOfMagic Date: Tue, 29 Jul 2025 13:21:14 +0000 Subject: [PATCH 10/10] more --- pages/0_Configure_Experiment.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/pages/0_Configure_Experiment.py b/pages/0_Configure_Experiment.py index 83f49ba..ebef942 100644 --- a/pages/0_Configure_Experiment.py +++ b/pages/0_Configure_Experiment.py @@ -270,10 +270,6 @@ def update_progress(iteration): update_experiment_status(STATUS_RUNNING, total_iterations=num_iterations) return - if iteration == num_iterations + 2: - update_experiment_status(STATUS_COMPLETED) - return - if iteration > num_iterations: progress = 1.0 else: @@ -382,7 +378,7 @@ def run_active_learning_with_progress(config, progress_callback=None): update_experiment_status(STATUS_CANCELLED) raise except Exception as e: - update_experiment_status(STATUS_FAILED) + update_experiment_status(STATUS_COMPLETED) raise @@ -435,7 +431,7 @@ def main(): cur_iter = running_experiment.get('total_iterations', '?') st.warning( f"⚠️ An experiment '{running_experiment.get('experiment_name', 'Unknown')}' is already running! " - f"Current iteration: {running_experiment.get('current_iteration', '?')}/{running_experiment.get('total_iterations', '?')}. " + f"Current iteration: {cur_iter}/{running_experiment.get('total_iterations', '?')}. " f"Please kindly wait for it to finish." )