run_all_experiments.py

import os
import sys
import time
import argparse

def run_command(command):
    """Execute shell command and print output"""
    print(f"\n{'='*80}")
    print(f"Running: {command}")
    print(f"{'='*80}\n")
    
    start_time = time.time()
    result = os.system(command)
    elapsed = time.time() - start_time
    
    if result != 0:
        print(f"\nCommand failed with exit code {result}")
        return False
    else:
        print(f"\nCommand completed successfully in {elapsed:.2f} seconds")
        return True

def main():
    parser = argparse.ArgumentParser(description="Run all model experiments")
    parser.add_argument("--skip-baseline", action="store_true", help="Skip baseline evaluation")
    parser.add_argument("--skip-training", action="store_true", help="Skip post-training")
    parser.add_argument("--skip-post-eval", action="store_true", help="Skip post-trained evaluation")
    parser.add_argument("--skip-self-eval", action="store_true", help="Skip self-evaluation")
    parser.add_argument("--skip-combined", action="store_true", help="Skip combined approach")
    parser.add_argument("--skip-viz", action="store_true", help="Skip visualization")
    parser.add_argument("--num-problems", type=int, default=164, help="Number of problems to evaluate")
    args = parser.parse_args()
    
    # Create results directory
    os.makedirs("results", exist_ok=True)
    os.makedirs("results/figures", exist_ok=True)
    
    # Run experiments
    if not args.skip_baseline:
        print("Running baseline evaluation...")
        run_command(f"python run_baseline.py --num-problems {args.num_problems}")
        run_command("python run_self_eval.py")
    
    if not args.skip_training:
        print("Running post-training on multiple data sizes...")
        run_command("python run_post_training.py")
    
    if not args.skip_post_eval:
        print("Evaluating all post-trained models...")
        # First evaluate all percentages individually
        for percentage in [0.1, 1, 5, 10, 30, 50, 100]:
            model_path = f'results/post_trained_{percentage}pct'
            if os.path.exists(model_path):
                print(f"Evaluating {percentage}% post-trained model...")
                run_command(f"python evaluate_post_trained.py --percentage {percentage} --no-compare")
            else:
                print(f"Skipping {percentage}% evaluation - model not found at {model_path}")
        
        # Then run comparison once at the end
        print("Comparing all models...")
        run_command("python evaluate_post_trained.py --compare-only")

    if not args.skip_combined:
        print("Running combined approaches...")
        # First run combined approach for all percentages individually
        for percentage in [0.1, 1, 5, 10, 30, 50, 100]:
            model_path = f'results/post_trained_{percentage}pct'
            if os.path.exists(model_path):
                print(f"Running combined approach with {percentage}% post-trained model...")
                run_command(f"python run_combined_approach.py --percentage {percentage}")
            else:
                print(f"Skipping {percentage}% combined approach - model not found at {model_path}")
        
        # Then run comparison once at the end
        print("Comparing all combined approaches...")
        run_command("python run_combined_approach.py --compare-only")
    
    if not args.skip_viz:
        print("Generating visualizations...")
        run_command("python visualize_results.py")
    
    print("\nAll experiments completed!")

if __name__ == "__main__":
    main()