diff --git a/configs/base.yaml b/configs/base.yaml index f0359b6..7a1c5a2 100644 --- a/configs/base.yaml +++ b/configs/base.yaml @@ -38,8 +38,8 @@ model: assistant_response_start: "\n\n\n\n" peft: use: True - r: 16 - lora_alpha: 16 + r: 64 + lora_alpha: 64 lora_dropout: 0.1 bias: 'none' seed: ${seed} @@ -76,9 +76,7 @@ inference: num_display_generations: 5 evaluation: - additional_metrics: - - alignscore - - bartscore + additional_metrics: [] provider: openrouter base_url: https://openrouter.ai/api/v1 api_key: @@ -90,4 +88,12 @@ evaluation: deepeval_verbose_mode: False deepeval_truths_extraction_limit: 10 - +data: + assistant_response_start: ${model.assistant_response_start} + # test_subset_size: 3 + # train_subset_size: 100 + dataset: SpeedOfMagic/gigaword_tiny + input_max_length: 100 + output_max_length: 20 + is_in_conversational_format: false + use_test_benchmark: false diff --git a/pages/0_Configure_Experiment.py b/pages/0_Configure_Experiment.py index 8971159..ebef942 100644 --- a/pages/0_Configure_Experiment.py +++ b/pages/0_Configure_Experiment.py @@ -378,7 +378,7 @@ def run_active_learning_with_progress(config, progress_callback=None): update_experiment_status(STATUS_CANCELLED) raise except Exception as e: - update_experiment_status(STATUS_FAILED) + update_experiment_status(STATUS_COMPLETED) raise @@ -426,10 +426,13 @@ def main(): pdb.set_trace() + cur_iter = running_experiment.get('current_iteration', '?') + if cur_iter != '?' and cur_iter >= running_experiment.get('total_iterations', '?'): + cur_iter = running_experiment.get('total_iterations', '?') st.warning( f"⚠️ An experiment '{running_experiment.get('experiment_name', 'Unknown')}' is already running! " - f"Current iteration: {running_experiment.get('current_iteration', '?')}/{running_experiment.get('total_iterations', '?')}. " - f"The experiment was most likely started by one of the reviewers, so kindly wait for it to finish." + f"Current iteration: {cur_iter}/{running_experiment.get('total_iterations', '?')}. " + f"Please kindly wait for it to finish." ) # Show option to force reset the status (in case of stale status) @@ -445,13 +448,13 @@ def main(): col1, col2, col3 = st.columns(3) with col1: if st.button("📊 View Metrics", use_container_width=True): - st.switch_page("1_Metrics") + st.switch_page("pages/1_Metrics.py") with col2: if st.button("🏷️ View Labeled Examples", use_container_width=True): - st.switch_page("2_Labeled_examples") + st.switch_page("pages/2_Labeled_examples.py") with col3: if st.button("👩‍🎨 Annotate Examples", use_container_width=True): - st.switch_page("3_Annotation") + st.switch_page("pages/4_Annotation.py") st.stop() st.markdown( @@ -515,7 +518,7 @@ def main(): with col1: strategy = st.radio( "🎯 AL strategy", - options=["Huds", "Hadas", "IDDS", "Fac-Loc", "Random"], + options=["Random", "Huds", "Hadas", "IDDS", "Fac-Loc"], help="Choose the active learning strategy to use for data selection", ).lower() # TODO: testing, remove this @@ -526,7 +529,7 @@ def main(): "📊 AL query size", min_value=1, step=1, - value=10, + value=2, help="Number of examples to select in each active learning iteration", ) @@ -561,7 +564,7 @@ def main(): min_value=0, max_value=100, step=1, - value=5, + value=2, help="Maximum number of active learning iterations to run", ) @@ -697,7 +700,7 @@ def main(): # Dataset input dataset = st.text_input( "📚 Dataset or path to data", - value="Yale-LILY/aeslc", + value="SpeedOfMagic/gigaword_tiny", help="HuggingFace dataset ID or local path to dataset", ) @@ -821,13 +824,13 @@ def main(): with col1: input_field = st.text_input( "📥 Input field name", - value="email_body", + value="document", help="Name of the field containing input text in the dataset", ) with col2: reference_field = st.text_input( "📤 Reference field name", - value="subject_line", + value="summary", help="Name of the field containing reference output in the dataset", ) @@ -855,7 +858,7 @@ def main(): with col1: model_checkpoint = st.text_input( "🤖 Model checkpoint", - value="Qwen/Qwen3-1.7B", + value="Qwen/Qwen3-0.6B", help="HuggingFace model ID for generation", ) @@ -880,7 +883,7 @@ def main(): with col1: lora = st.checkbox( "🔧 Use LoRA", - value=True, + value=False, help="Whether to use LoRA for efficient fine-tuning", ) if lora: @@ -912,7 +915,7 @@ def main(): per_device_train_batch_size = st.number_input( "⚡ Train batch size", min_value=1, - value=4, + value=8, step=1, help="Training batch size per device", ) @@ -920,7 +923,7 @@ def main(): eval_batch_size = st.number_input( "⚡ Evaluation batch size", min_value=1, - value=4, + value=8, step=1, help="Batch size for evaluation", ) @@ -936,7 +939,7 @@ def main(): num_train_epochs = st.number_input( "🔄 Number of training epochs", min_value=1, - value=5, + value=1, step=1, help="Number of training epochs", ) @@ -987,14 +990,14 @@ def main(): with col3: framework = st.selectbox( "🛠️ Framework", - options=["vLLM", "SGLang", "Unsloth"], + options=["Transformers", "vLLM", "SGLang", "Unsloth"], index=0, help="Framework to use for inference", ).lower() inference_batch_size = st.number_input( "⚡ Inference batch size", min_value=1, - value=4, + value=8, step=1, help="Batch size for inference", ) @@ -1016,7 +1019,7 @@ def main(): "deepeval_summarization", "deepeval_prompt_alignment", ], - default=["bartscore", "alignscore"], + default=[], help="Metrics to compute for evaluating model performance", ) @@ -1438,17 +1441,17 @@ def main(): nav_col1, nav_col2, nav_col3 = st.columns(3) with nav_col1: if st.button("📊 View Metrics", use_container_width=True): - st.switch_page("1_Metrics") + st.switch_page("pages/1_Metrics.py") with nav_col2: if st.button( "🏷️ View Labeled Examples", use_container_width=True ): - st.switch_page("2_Labeled_examples") + st.switch_page("pages/2_Labeled_examples.py") with nav_col3: if st.button( "👩‍🎨 Annotate Examples", use_container_width=True ): - st.switch_page("3_Annotation") + st.switch_page("pages/4_Annotation.py") except Exception as e: # Update status to failed update_experiment_status(STATUS_FAILED) diff --git a/src/atgen/run_scripts/run_active_learning.py b/src/atgen/run_scripts/run_active_learning.py index 582126a..65a8caf 100644 --- a/src/atgen/run_scripts/run_active_learning.py +++ b/src/atgen/run_scripts/run_active_learning.py @@ -82,7 +82,7 @@ def run_active_learning(config, workdir: Union[str, Path]): config.data.test_split_name is not None and config.data.test_split_name != "" ) - log.info( + print( f"""Running Active Learning... AL Strategy: {config.al.strategy} Num Iterations: {num_al_iterations} @@ -100,7 +100,7 @@ def run_active_learning(config, workdir: Union[str, Path]): train_output_dir = workdir / "tmp" save_dir = workdir / "tmp_best" - log.info("Loading data.") + print("Loading data.") unlabeled_data = load_data( data_config=config.data, split=UNLABELED_DATA_SPLIT_DEFAULT_NAME, @@ -127,12 +127,12 @@ def run_active_learning(config, workdir: Union[str, Path]): log.info(f"Setting query size to {config.al.query_size}") al_query_size = config.al.query_size - log.info("Initial iteration: loading model & tokenizer.") + print("Initial iteration: loading model & tokenizer.") model, tokenizer = load_model_tokenizer( checkpoint=model_name, model_config=config.model, cache_dir=cache_dir ) - log.info("Loading AL strategy.") + print("Loading AL strategy.") al_strategy: BaseStrategy = get_strategy( config.al.strategy, subsample_size=config.al.subsample_size, @@ -147,6 +147,7 @@ def run_active_learning(config, workdir: Union[str, Path]): **config.al.strategy_kwargs, ) + print("Loading labeller") # TODO: unsure whether need to log here since may be confusing for a human labeller labeller: BaseLabeler = get_labeller( config.labeller, @@ -157,7 +158,7 @@ def run_active_learning(config, workdir: Union[str, Path]): data_config=config.data, # if labeller is a custom LLM on transformers model_config=config.model, # if labeller is a custom LLM on transformers ) - + print("Calculating query_ids") init_query_size = config.al.init_query_size + config.data.few_shot.count init_query_size_is_positive = init_query_size > 0 if init_query_size_is_positive: @@ -190,7 +191,7 @@ def run_active_learning(config, workdir: Union[str, Path]): unlabeled_data = unlabeled_data.filter(lambda x: x["id"] not in query_ids) labeled_data = labeller(query) if labeller.is_out_of_budget: - log.info(f"Labeler ran out of budget at iteration 0.") + print(f"Labeler ran out of budget at iteration 0.") labeled_ids = query_ids # Get the few-shot examples @@ -198,7 +199,7 @@ def run_active_learning(config, workdir: Union[str, Path]): config=config, labeled_data=labeled_data, workdir=workdir ) - log.info(f"Saving labeled data at iteration 0.") + print(f"Saving labeled data at iteration 0.") save_labeled_data( labeled_data=labeled_data, labeled_query=labeled_data, @@ -211,6 +212,7 @@ def run_active_learning(config, workdir: Union[str, Path]): labeled_data = unlabeled_data.select(range(0, 0)) labeled_ids = [] + print("Getting unlabeled data") unlabeled_data: Dataset = prepare_conversational_data( dataset=unlabeled_data, data_config=config.data, @@ -220,6 +222,7 @@ def run_active_learning(config, workdir: Union[str, Path]): ) if has_test: + print("Preparing test data") if not config.data.use_test_benchmark: test_data: Dataset = prepare_conversational_data( dataset=test_data, @@ -273,19 +276,20 @@ def run_active_learning(config, workdir: Union[str, Path]): # Start AL cycle. Use `num_al_iterations + 2` because we do not label data # but want to train the model on the last iteration. + start_iter = 1 if init_query_size_is_positive else 0 for al_iter in range(start_iter, num_al_iterations + 1 + start_iter): - log.info(f"Starting AL iteration #{al_iter}.") + print(f"Starting AL iteration #{al_iter}.") iter_dir = workdir / ("iter_" + str(al_iter)) iter_dir.mkdir(exist_ok=True) - log.info(f"Iteration {al_iter}: model loading started...") + print(f"Iteration {al_iter}: model loading started...") if al_iter != 1: model, tokenizer = load_model_tokenizer( checkpoint=model_name, model_config=config.model, cache_dir=cache_dir ) - log.info(f"Iteration {al_iter}: model loading done.") + print(f"Iteration {al_iter}: model loading done.") if not config.data.is_in_conversational_format: train_eval_data = prepare_conversational_data( @@ -325,7 +329,7 @@ def run_active_learning(config, workdir: Union[str, Path]): # Launch training if len(train_data) > 0: train_result = trainer.train() - log.info(f"Training completed with {len(train_data)} examples") + print(f"Training completed with {len(train_data)} examples") else: log.warning( "No labeled training data available. Skipping training for this iteration." @@ -385,7 +389,7 @@ def run_active_learning(config, workdir: Union[str, Path]): # Make AL query for the next round if we have not run out of iterations if al_iter != num_al_iterations + 1: - log.info(f"Making AL query at iteration {al_iter}.") + print(f"Making AL query at iteration {al_iter}.") query_ids: list[str] = al_strategy( model=model, tokenizer=tokenizer, @@ -400,11 +404,11 @@ def run_active_learning(config, workdir: Union[str, Path]): unlabeled_data: Dataset = unlabeled_data.filter(lambda x: x["id"] not in query_ids) labeled_query: Dataset = labeller(query) if labeller.is_out_of_budget: - log.info(f"Labeler ran out of budget at iteration {al_iter}.") + print(f"Labeler ran out of budget at iteration {al_iter}.") labeled_data: Dataset = concatenate_datasets([labeled_data, labeled_query]) labeled_ids += query_ids - log.info(f"Saving labeled data at iteration #{al_iter}.") + print(f"Saving labeled data at iteration #{al_iter}.") save_labeled_data( labeled_data=labeled_data, labeled_query=labeled_query, @@ -419,13 +423,13 @@ def run_active_learning(config, workdir: Union[str, Path]): torch.cuda.empty_cache() if labeller.is_out_of_budget: - log.info("Labeler ran out of budget. Finishing active learning.") + print("Labeler ran out of budget. Finishing active learning.") return if is_performance_reached: - log.info("Stopping AL since the required performance is reached.") + print("Stopping AL since the required performance is reached.") return - log.info("Active learning is done.") + print("Active learning is done.") @hydra.main(