eval_simple.py

"""Simple evaluation script."""
import asyncio
import numpy as np
from datetime import datetime
from predictor import predict_outages, load_eaglei_data

TEST_DATES = [
    # Normal days
    ('2024-01-15', 14, 'Winter'),
    ('2024-03-12', 14, 'Spring'),
    ('2024-05-22', 14, 'Late Spring'),
    ('2024-06-10', 14, 'Early Summer'),
    ('2024-07-15', 14, 'Mid Summer'),
    ('2024-08-20', 14, 'Late Summer'),
    ('2024-10-08', 14, 'Fall'),
    # Storms
    ('2024-09-27', 18, 'Hurricane Helene'),
    ('2024-07-08', 18, 'Hurricane Beryl'),
    ('2024-05-27', 18, 'TX Storms'),
    ('2023-08-30', 18, 'Hurricane Idalia'),
]

async def main():
    results = []

    for date_str, hour, desc in TEST_DATES:
        print(f"\n{'='*60}")
        print(f"Testing {date_str} {hour:02d}:00 - {desc}")
        print('='*60)

        try:
            target_time = datetime.strptime(f"{date_str} {hour:02d}:00:00", "%Y-%m-%d %H:%M:%S")

            # Get predictions
            preds = await predict_outages(target_time, horizon=6)
            pred_total = preds.get('total_predicted', 0)

            # Get actuals
            year = target_time.year
            df = load_eaglei_data(year)
            df['time_diff'] = abs((df['run_start_time'] - target_time).dt.total_seconds())
            snapshot = df[df['time_diff'] <= 900].copy()
            if len(snapshot) == 0:
                nearest_idx = df['time_diff'].idxmin()
                nearest_time = df.loc[nearest_idx, 'run_start_time']
                snapshot = df[df['run_start_time'] == nearest_time].copy()

            actual_total = int(snapshot['customers_out'].sum())

            # State correlation
            pred_by_state = {p['state']: p['predicted_outages'] for p in preds.get('by_state', [])}
            actual_by_state = snapshot.groupby('state')['customers_out'].sum().to_dict()

            common = set(pred_by_state.keys()) & set(actual_by_state.keys())
            if len(common) > 5:
                p_vals = [pred_by_state[s] for s in common]
                a_vals = [actual_by_state[s] for s in common]
                corr = np.corrcoef(p_vals, a_vals)[0, 1]
            else:
                corr = 0

            ratio = pred_total / actual_total if actual_total > 0 else 0

            print(f"  Predicted: {pred_total:>12,}")
            print(f"  Actual:    {actual_total:>12,}")
            print(f"  Ratio:     {ratio:>12.2f}x")
            print(f"  Corr:      {corr:>12.3f}")

            results.append({
                'date': date_str, 'desc': desc,
                'predicted': pred_total, 'actual': actual_total,
                'ratio': ratio, 'corr': corr
            })

        except Exception as e:
            print(f"  ERROR: {e}")
            import traceback
            traceback.print_exc()

    # Summary
    print("\n" + "="*80)
    print("SUMMARY")
    print("="*80)
    print(f"{'Date':<12} {'Description':<20} {'Predicted':>12} {'Actual':>12} {'Ratio':>8} {'Corr':>6}")
    print("-"*80)

    for r in results:
        print(f"{r['date']:<12} {r['desc']:<20} {r['predicted']:>12,} {r['actual']:>12,} {r['ratio']:>7.2f}x {r['corr']:>6.2f}")

    # Aggregate stats
    normal = [r for r in results if 'Hurricane' not in r['desc'] and 'Storm' not in r['desc']]
    storms = [r for r in results if 'Hurricane' in r['desc'] or 'Storm' in r['desc']]

    print("\n" + "-"*40)
    if normal:
        ratios = [r['ratio'] for r in normal]
        corrs = [r['corr'] for r in normal]
        print(f"NORMAL DAYS (n={len(normal)}):")
        print(f"  Mean Ratio: {np.mean(ratios):.2f}x (ideal: 1.0)")
        print(f"  Mean Corr:  {np.mean(corrs):.3f}")

    if storms:
        ratios = [r['ratio'] for r in storms]
        corrs = [r['corr'] for r in storms]
        print(f"STORMS (n={len(storms)}):")
        print(f"  Mean Ratio: {np.mean(ratios):.2f}x (ideal: 1.0)")
        print(f"  Mean Corr:  {np.mean(corrs):.3f}")

if __name__ == '__main__':
    asyncio.run(main())