diff --git a/configs/config.yaml b/configs/config.yaml new file mode 100644 index 0000000..938e8a2 --- /dev/null +++ b/configs/config.yaml @@ -0,0 +1,16 @@ +# DeepDiet Training Configuration +# Use Hydra to manage experiments: python src/train_hydra.py +# Override from command line: python src/train_hydra.py model.side_aggregation=attention training.lr=5e-5 + +defaults: + - _self_ + - model: default + - training: default + - data: default + - logging: default + +# Experiment metadata +experiment: + name: null # Auto-generated if null + tags: [] + notes: "" \ No newline at end of file diff --git a/configs/logging/default.yaml b/configs/logging/default.yaml new file mode 100644 index 0000000..dcf3c80 --- /dev/null +++ b/configs/logging/default.yaml @@ -0,0 +1,13 @@ +# Logging configuration +# Override: python src/train_hydra.py logging.wandb=true + +# Weights & Biases +wandb: true +wandb_project: deepdiet +wandb_entity: null # Your W&B username/team (null = default) + +# TensorBoard (always enabled) +tensorboard: true + +# Advanced metrics frequency +advanced_metrics_freq: 2 \ No newline at end of file diff --git a/configs/logging/local.yaml b/configs/logging/local.yaml new file mode 100644 index 0000000..380c193 --- /dev/null +++ b/configs/logging/local.yaml @@ -0,0 +1,9 @@ +# Local-only logging (no W&B) +# Use: python src/train_hydra.py logging=local + +wandb: false +wandb_project: deepdiet +wandb_entity: null + +tensorboard: true +advanced_metrics_freq: 2 \ No newline at end of file diff --git a/configs/model/attention.yaml b/configs/model/attention.yaml new file mode 100644 index 0000000..3c4b286 --- /dev/null +++ b/configs/model/attention.yaml @@ -0,0 +1,21 @@ +# Attention-based model configuration +# Use: python src/train_hydra.py model=attention + +# Modalities +use_side_frames: true +use_overhead: true +use_depth: true +allow_missing_modalities: false + +# Use temporal attention for side frames +side_aggregation: attention + +# LSTM hidden (not used with attention, but kept for compatibility) +lstm_hidden: 640 + +# Encoder settings +freeze_encoders: false +unfreeze_epoch: 10 +encoder_lr_multiplier: 0.1 + +chunk_size: 4 \ No newline at end of file diff --git a/configs/model/default.yaml b/configs/model/default.yaml new file mode 100644 index 0000000..2f7bf25 --- /dev/null +++ b/configs/model/default.yaml @@ -0,0 +1,22 @@ +# Model configuration +# Override: python src/train_hydra.py model=attention + +# Modalities to use +use_side_frames: true +use_overhead: true +use_depth: true +allow_missing_modalities: false + +# Side frame aggregation: lstm, attention, or mean +side_aggregation: lstm + +# LSTM-specific (ignored if using attention/mean) +lstm_hidden: 640 + +# Encoder settings +freeze_encoders: false +unfreeze_epoch: 10 +encoder_lr_multiplier: 0.1 + +# Processing +chunk_size: 4 # Frames processed at once (lower = less memory) \ No newline at end of file diff --git a/configs/model/side_only.yaml b/configs/model/side_only.yaml new file mode 100644 index 0000000..2be1db6 --- /dev/null +++ b/configs/model/side_only.yaml @@ -0,0 +1,16 @@ +# Side frames only configuration (ablation study) +# Use: python src/train_hydra.py model=side_only + +use_side_frames: true +use_overhead: false +use_depth: false +allow_missing_modalities: false + +side_aggregation: attention +lstm_hidden: 640 + +freeze_encoders: false +unfreeze_epoch: 10 +encoder_lr_multiplier: 0.1 + +chunk_size: 4 \ No newline at end of file diff --git a/configs/training/default.yaml b/configs/training/default.yaml new file mode 100644 index 0000000..ed12711 --- /dev/null +++ b/configs/training/default.yaml @@ -0,0 +1,17 @@ +# Training configuration +# Override: python src/train_hydra.py training.lr=5e-5 + +epochs: 20 +batch_size: 4 +lr: 1e-4 +weight_decay: 1e-4 + +# Learning rate schedule +lr_decay_epochs: 5 +lr_decay_factor: 0.5 + +# Gradient clipping +max_grad_norm: 5.0 + +# Data loading +num_workers: 2 \ No newline at end of file diff --git a/configs/training/fast.yaml b/configs/training/fast.yaml new file mode 100644 index 0000000..d946a1a --- /dev/null +++ b/configs/training/fast.yaml @@ -0,0 +1,13 @@ +# Fast training configuration for debugging/testing +# Use: python src/train_hydra.py training=fast + +epochs: 3 +batch_size: 2 +lr: 1e-4 +weight_decay: 1e-4 + +lr_decay_epochs: 5 +lr_decay_factor: 0.5 +max_grad_norm: 5.0 + +num_workers: 0 # Easier debugging \ No newline at end of file diff --git a/configs/training/long.yaml b/configs/training/long.yaml new file mode 100644 index 0000000..a3f4f86 --- /dev/null +++ b/configs/training/long.yaml @@ -0,0 +1,13 @@ +# Long training configuration +# Use: python src/train_hydra.py training=long + +epochs: 50 +batch_size: 8 +lr: 1e-4 +weight_decay: 1e-4 + +lr_decay_epochs: 10 +lr_decay_factor: 0.5 +max_grad_norm: 5.0 + +num_workers: 4 \ No newline at end of file diff --git a/docs/EXPERIMENT_TRACKING.md b/docs/EXPERIMENT_TRACKING.md new file mode 100644 index 0000000..df50e5b --- /dev/null +++ b/docs/EXPERIMENT_TRACKING.md @@ -0,0 +1,423 @@ +# Experiment Tracking & Configuration Guide + +This document covers the experiment tracking, batch running, and configuration management tools available in DeepDiet. + +## Table of Contents +1. [Quick Start](#quick-start) +2. [Weights & Biases Integration](#weights--biases-integration) +3. [Experiment Runner](#experiment-runner) +4. [Hydra Configuration](#hydra-configuration) +5. [GCP Workflow](#gcp-workflow) + +--- + +## Quick Start + +### First-Time Setup + +```bash +# Install dependencies +pip install -r requirements.txt + +# Login to W&B (one-time) +wandb login +``` + +### Run a Single Experiment with W&B + +```bash +python -m src.train \ + --use-side-frames \ + --use-overhead \ + --use-depth \ + --side-aggregation attention \ + --wandb +``` + +### Run Multiple Experiments + +```bash +python scripts/run_experiments.py --experiments attention_baseline lstm_baseline +``` + +### Use Hydra Config + +```bash +python src/train_hydra.py model=attention training=fast +``` + +--- + +## Weights & Biases Integration + +W&B is integrated into `src/train.py` for experiment tracking, replacing the need for manual spreadsheets. + +### Command Line Options + +| Flag | Description | Default | +|------|-------------|---------| +| `--wandb` | Enable W&B logging | Disabled | +| `--wandb-project` | Project name in W&B | `deepdiet` | +| `--wandb-run-name` | Custom run name | Auto-generated | +| `--wandb-tags` | Tags for filtering runs | Auto + custom | + +### Example Usage + +```bash +# Basic W&B logging +python -m src.train --use-side-frames --wandb + +# Custom run name and tags +python -m src.train \ + --use-side-frames \ + --side-aggregation attention \ + --wandb \ + --wandb-run-name "attention_lr_sweep_1" \ + --wandb-tags experiment1 attention + +# Different project +python -m src.train --use-side-frames --wandb --wandb-project deepdiet-ablations +``` + +### What Gets Logged + +**Automatically tracked:** +- All hyperparameters (model, training, data config) +- Git commit hash +- System info (GPU, Python version) +- Training/validation loss per epoch +- Per-task MAE (calories, mass, fat, carb, protein) +- MAE as percentage of mean +- Timing metrics (epoch time, data loading, forward/backward) +- Best validation loss + +**View results at:** `https://wandb.ai//deepdiet` + +--- + +## Experiment Runner + +The experiment runner (`scripts/run_experiments.py`) enables batch execution of multiple experiments. + +### List Available Experiments + +```bash +python scripts/run_experiments.py --list +``` + +**Pre-defined experiments:** + +| Name | Description | +|------|-------------| +| `lstm_baseline` | LSTM aggregation (default) | +| `attention_baseline` | Temporal attention aggregation | +| `mean_baseline` | Mean pooling aggregation | +| `attention_lr_high` | Attention with LR=5e-4 | +| `attention_lr_low` | Attention with LR=5e-5 | +| `attention_frozen` | Attention with frozen encoders | +| `side_only_attention` | Side frames only (ablation) | +| `overhead_only` | Overhead RGB+depth only (ablation) | + +### Running Experiments + +```bash +# Run all pre-defined experiments +python scripts/run_experiments.py + +# Run specific experiments +python scripts/run_experiments.py --experiments attention_baseline lstm_baseline mean_baseline + +# Dry run (see commands without executing) +python scripts/run_experiments.py --dry-run + +# Custom W&B project +python scripts/run_experiments.py --wandb-project my-project +``` + +### Output + +- Logs saved to `experiment_logs/_.log` +- Results summary saved to `experiment_logs/results_.json` +- All runs visible in W&B dashboard + +### Adding Custom Experiments + +Edit `DEFAULT_EXPERIMENTS` in `scripts/run_experiments.py`: + +```python +"my_experiment": { + "description": "My custom experiment", + "args": { + "use_side_frames": True, + "use_overhead": True, + "use_depth": True, + "side_aggregation": "attention", + "epochs": 30, + "lr": 2e-4, + }, + "tags": ["custom", "experiment"], +}, +``` + +Or create a YAML config file: + +```yaml +# experiments.yaml +my_experiment: + description: "My custom experiment" + args: + use_side_frames: true + side_aggregation: attention + epochs: 30 + tags: ["custom"] +``` + +```bash +python scripts/run_experiments.py --config experiments.yaml +``` + +--- + +## Hydra Configuration + +Hydra provides YAML-based configuration with easy overrides and sweeps. + +### Config Structure + +``` +configs/ +├── config.yaml # Main config (composes others) +├── model/ +│ ├── default.yaml # LSTM baseline +│ ├── attention.yaml # Temporal attention +│ └── side_only.yaml # Side frames only +├── training/ +│ ├── default.yaml # 20 epochs, batch=4 +│ ├── fast.yaml # 3 epochs (debugging) +│ └── long.yaml # 50 epochs +├── data/ +│ ├── default.yaml # 256px, 16 frames +│ └── small.yaml # 128px, 8 frames (faster) +└── logging/ + ├── default.yaml # W&B enabled + └── local.yaml # TensorBoard only +``` + +### Basic Usage + +```bash +# Default configuration +python src/train_hydra.py + +# Use attention model config +python src/train_hydra.py model=attention + +# Combine configs +python src/train_hydra.py model=attention training=fast data=small + +# Quick debug run (no W&B) +python src/train_hydra.py training=fast logging=local +``` + +### Override Specific Values + +```bash +# Override learning rate +python src/train_hydra.py training.lr=5e-5 + +# Multiple overrides +python src/train_hydra.py model.side_aggregation=attention training.epochs=30 training.lr=2e-4 + +# Override nested values +python src/train_hydra.py model.freeze_encoders=true model.unfreeze_epoch=5 +``` + +### Hyperparameter Sweeps + +```bash +# Sweep over aggregation methods +python src/train_hydra.py --multirun model.side_aggregation=lstm,attention,mean + +# Sweep over learning rates +python src/train_hydra.py --multirun training.lr=1e-4,5e-5,1e-5 + +# Grid search (runs all combinations) +python src/train_hydra.py --multirun \ + model.side_aggregation=lstm,attention \ + training.lr=1e-4,5e-5 +``` + +### Config Options Reference + +**Model (`configs/model/`)** +```yaml +use_side_frames: true # Use side angle frames +use_overhead: true # Use overhead RGB +use_depth: true # Use depth images +side_aggregation: lstm # lstm, attention, or mean +lstm_hidden: 640 # LSTM hidden size +freeze_encoders: false # Freeze EfficientNet initially +unfreeze_epoch: 10 # When to unfreeze +encoder_lr_multiplier: 0.1 # LR multiplier for encoders +chunk_size: 4 # Frames per forward pass +``` + +**Training (`configs/training/`)** +```yaml +epochs: 20 +batch_size: 4 +lr: 1e-4 +weight_decay: 1e-4 +lr_decay_epochs: 5 # Decay LR every N epochs +lr_decay_factor: 0.5 # Multiply LR by this +max_grad_norm: 5.0 # Gradient clipping +num_workers: 2 # Data loader workers +``` + +**Data (`configs/data/`)** +```yaml +image_size: 256 # Input image size +max_frames: 16 # Max side frames per dish +data_root: data/nutrition5k_dataset +train_csv: indexes/train_official.csv +test_csv: indexes/test_official.csv +``` + +**Logging (`configs/logging/`)** +```yaml +wandb: true # Enable W&B +wandb_project: deepdiet # W&B project name +wandb_entity: null # W&B team (null = personal) +tensorboard: true # Always enabled +``` + +--- + +## GCP Workflow + +### SSH into Instance + +```bash +gcloud compute ssh --zone "us-central1-c" "instance-20251130-082004" --project "cs230-project-478801" +``` + +### Typical Training Session + +```bash +# SSH in +gcloud compute ssh --zone "us-central1-c" "instance-20251130-082004" --project "cs230-project-478801" + +# Navigate and update +cd ~/deepdiet +git pull + +# Install any new dependencies +pip install -r requirements.txt + +# Run training with W&B +python -m src.train \ + --use-side-frames \ + --use-overhead \ + --use-depth \ + --side-aggregation attention \ + --epochs 20 \ + --wandb + +# Or run in background +nohup python -m src.train \ + --use-side-frames \ + --side-aggregation attention \ + --wandb \ + > training.log 2>&1 & + +# Monitor +tail -f training.log +``` + +### Running Batch Experiments on GCP + +```bash +# SSH in +gcloud compute ssh ... + +cd ~/deepdiet && git pull + +# Run all experiments (will take hours) +nohup python scripts/run_experiments.py > experiments.log 2>&1 & + +# Monitor progress +tail -f experiments.log + +# Or check W&B dashboard for real-time updates +``` + +### Using tmux for Long Runs + +```bash +# Start tmux session +tmux new -s training + +# Run training +python -m src.train --use-side-frames --wandb + +# Detach: Ctrl+B, then D + +# Reattach later +tmux attach -t training +``` + +--- + +## Comparing Results + +### In W&B + +1. Go to `https://wandb.ai//deepdiet` +2. Select runs to compare +3. Use the comparison view to see metrics side-by-side +4. Filter by tags (e.g., "attention", "baseline") + +### In TensorBoard + +```bash +tensorboard --logdir runs/ +``` + +Then open `http://localhost:6006` in your browser. + +--- + +## Troubleshooting + +### W&B Not Logging + +```bash +# Check if logged in +wandb login + +# Verify API key +cat ~/.netrc | grep wandb +``` + +### CUDA Out of Memory + +```bash +# Reduce batch size +python -m src.train --batch-size 2 --wandb + +# Or reduce chunk size (frames processed at once) +python -m src.train --chunk-size 2 --wandb + +# Or use smaller images +python src/train_hydra.py data=small +``` + +### Hydra Working Directory Issues + +Hydra changes the working directory. If you have path issues: + +```python +# In your code +import hydra +orig_cwd = hydra.utils.get_original_cwd() +``` \ No newline at end of file diff --git a/indexes/multimodal_test.csv b/indexes/multimodal_test.csv new file mode 100644 index 0000000..4009352 --- /dev/null +++ b/indexes/multimodal_test.csv @@ -0,0 +1,710 @@ +dish_id +dish_1550704903 +dish_1550705288 +dish_1550705370 +dish_1550705445 +dish_1550705477 +dish_1550705504 +dish_1550705786 +dish_1550705888 +dish_1550705939 +dish_1550705995 +dish_1550706448 +dish_1550706583 +dish_1550706639 +dish_1550706705 +dish_1550707638 +dish_1550707756 +dish_1550707850 +dish_1550707914 +dish_1550707958 +dish_1550708366 +dish_1550711788 +dish_1550713566 +dish_1550769948 +dish_1550770222 +dish_1550770265 +dish_1550770447 +dish_1550770482 +dish_1550771421 +dish_1550772314 +dish_1550773995 +dish_1550777025 +dish_1550777124 +dish_1550777181 +dish_1550777218 +dish_1550777860 +dish_1550778172 +dish_1550778535 +dish_1550778583 +dish_1550778619 +dish_1550778646 +dish_1550779058 +dish_1550785697 +dish_1550785808 +dish_1550785876 +dish_1550785915 +dish_1550792864 +dish_1550859694 +dish_1550859927 +dish_1550859977 +dish_1550860041 +dish_1550860107 +dish_1550860188 +dish_1550860307 +dish_1550860361 +dish_1550860444 +dish_1550860631 +dish_1550860747 +dish_1550860787 +dish_1550860820 +dish_1550873338 +dish_1550873434 +dish_1550873486 +dish_1550873572 +dish_1550873668 +dish_1551124407 +dish_1551125258 +dish_1551125305 +dish_1551125357 +dish_1551125384 +dish_1551136396 +dish_1551138312 +dish_1551222311 +dish_1551223114 +dish_1551224208 +dish_1551224647 +dish_1551224689 +dish_1551224751 +dish_1551224787 +dish_1551227380 +dish_1551228478 +dish_1551234185 +dish_1551234214 +dish_1551234252 +dish_1551234288 +dish_1551234316 +dish_1551235561 +dish_1551235600 +dish_1551235669 +dish_1551235699 +dish_1551237079 +dish_1551237180 +dish_1551237231 +dish_1551237274 +dish_1551237321 +dish_1551307394 +dish_1551314022 +dish_1551314048 +dish_1551314095 +dish_1551314121 +dish_1551314589 +dish_1551316737 +dish_1551316788 +dish_1551316838 +dish_1551316878 +dish_1551374604 +dish_1551374665 +dish_1551374712 +dish_1551374746 +dish_1551374783 +dish_1551375252 +dish_1551375783 +dish_1551376623 +dish_1551377521 +dish_1551378762 +dish_1551381921 +dish_1551382179 +dish_1551390930 +dish_1551390979 +dish_1551391034 +dish_1551391083 +dish_1551391438 +dish_1551391477 +dish_1551391505 +dish_1551391543 +dish_1551391672 +dish_1551391710 +dish_1551391733 +dish_1551391758 +dish_1551395909 +dish_1551395987 +dish_1551396061 +dish_1551396096 +dish_1551488485 +dish_1551491413 +dish_1551491452 +dish_1551492023 +dish_1551492056 +dish_1551493311 +dish_1551493942 +dish_1551564084 +dish_1551564131 +dish_1551564176 +dish_1551564221 +dish_1551564255 +dish_1551564352 +dish_1551564381 +dish_1551564423 +dish_1551564456 +dish_1551564480 +dish_1551564947 +dish_1551564992 +dish_1551565034 +dish_1551565068 +dish_1551565945 +dish_1551566159 +dish_1551566781 +dish_1551566851 +dish_1551566887 +dish_1551566922 +dish_1551566974 +dish_1551567084 +dish_1551567106 +dish_1551567141 +dish_1551567170 +dish_1551567471 +dish_1551567508 +dish_1551567541 +dish_1551567573 +dish_1551567848 +dish_1551567896 +dish_1551567937 +dish_1551567964 +dish_1556575327 +dish_1557861216 +dish_1557862345 +dish_1557862696 +dish_1557862738 +dish_1557862783 +dish_1557936555 +dish_1557936599 +dish_1557936645 +dish_1557937037 +dish_1557937079 +dish_1557937157 +dish_1557937335 +dish_1557937427 +dish_1557937574 +dish_1557937615 +dish_1557937715 +dish_1557937758 +dish_1558030182 +dish_1558030289 +dish_1558030383 +dish_1558030435 +dish_1558031133 +dish_1558031237 +dish_1558031302 +dish_1558031351 +dish_1558031394 +dish_1558110297 +dish_1558110335 +dish_1558110385 +dish_1558110436 +dish_1558110478 +dish_1558113430 +dish_1558113477 +dish_1558113941 +dish_1558114086 +dish_1558114145 +dish_1558114209 +dish_1558114284 +dish_1558372948 +dish_1558373008 +dish_1558373074 +dish_1558373159 +dish_1558375667 +dish_1558375732 +dish_1558375833 +dish_1558375886 +dish_1558376212 +dish_1558376306 +dish_1558376364 +dish_1558376437 +dish_1558376483 +dish_1558376734 +dish_1558376768 +dish_1558376801 +dish_1558376874 +dish_1558376938 +dish_1558376984 +dish_1558461431 +dish_1558461607 +dish_1558461692 +dish_1558461760 +dish_1558461792 +dish_1558544663 +dish_1558545738 +dish_1558546434 +dish_1558546558 +dish_1558546663 +dish_1558549413 +dish_1558549605 +dish_1558549773 +dish_1558549806 +dish_1558630176 +dish_1558630207 +dish_1558630288 +dish_1558630325 +dish_1558635400 +dish_1558635489 +dish_1558635523 +dish_1558635546 +dish_1558635618 +dish_1558639412 +dish_1558639440 +dish_1558639482 +dish_1558639526 +dish_1558639555 +dish_1558639787 +dish_1558639818 +dish_1558639849 +dish_1558639876 +dish_1558641164 +dish_1558641200 +dish_1558641253 +dish_1558641287 +dish_1558641320 +dish_1558717604 +dish_1558718587 +dish_1558718610 +dish_1558718649 +dish_1558718680 +dish_1558718906 +dish_1558719570 +dish_1558719994 +dish_1558720097 +dish_1558720133 +dish_1558720236 +dish_1558722125 +dish_1558722171 +dish_1558722207 +dish_1558722247 +dish_1559157454 +dish_1559157777 +dish_1559157919 +dish_1559233445 +dish_1559233570 +dish_1559233627 +dish_1559233715 +dish_1559233756 +dish_1559238993 +dish_1559239151 +dish_1559239256 +dish_1559239369 +dish_1559578551 +dish_1559578579 +dish_1559578599 +dish_1559841950 +dish_1559841979 +dish_1559842047 +dish_1559842136 +dish_1559844272 +dish_1559844323 +dish_1559844366 +dish_1559844458 +dish_1559844490 +dish_1559846688 +dish_1559849020 +dish_1559849053 +dish_1559849097 +dish_1559849412 +dish_1559849466 +dish_1559849497 +dish_1559849537 +dish_1559850283 +dish_1559850314 +dish_1559850338 +dish_1559934477 +dish_1560203400 +dish_1560203424 +dish_1560367904 +dish_1560367926 +dish_1560367952 +dish_1560367980 +dish_1560368019 +dish_1560368395 +dish_1560368464 +dish_1560368506 +dish_1560368570 +dish_1560444869 +dish_1560444892 +dish_1560444914 +dish_1560444931 +dish_1560444948 +dish_1560444967 +dish_1560444985 +dish_1560445004 +dish_1560456305 +dish_1560456326 +dish_1560456348 +dish_1560456370 +dish_1560526408 +dish_1560526447 +dish_1560526472 +dish_1560526490 +dish_1560526514 +dish_1560526530 +dish_1560526552 +dish_1560800918 +dish_1560800951 +dish_1560800988 +dish_1560801020 +dish_1560801041 +dish_1560801773 +dish_1560801797 +dish_1560801829 +dish_1560801859 +dish_1560801883 +dish_1560975491 +dish_1560975517 +dish_1560975572 +dish_1560975601 +dish_1560975628 +dish_1561147944 +dish_1561147967 +dish_1561147997 +dish_1561148021 +dish_1561148042 +dish_1561393715 +dish_1561393741 +dish_1561393764 +dish_1561405332 +dish_1561405356 +dish_1561405389 +dish_1561405411 +dish_1561574368 +dish_1561574394 +dish_1561574428 +dish_1561574456 +dish_1561574479 +dish_1561574783 +dish_1561574815 +dish_1561574848 +dish_1561577848 +dish_1561577947 +dish_1561578775 +dish_1561578800 +dish_1561578839 +dish_1561578865 +dish_1561578898 +dish_1561654326 +dish_1561654346 +dish_1561654374 +dish_1561654392 +dish_1561661324 +dish_1561661347 +dish_1561661377 +dish_1561662160 +dish_1561662190 +dish_1561662216 +dish_1561662814 +dish_1561662842 +dish_1561662869 +dish_1561662926 +dish_1561663541 +dish_1561663580 +dish_1561663606 +dish_1561667207 +dish_1561667238 +dish_1561739160 +dish_1561739188 +dish_1561739212 +dish_1561739238 +dish_1561739265 +dish_1561741063 +dish_1561741090 +dish_1561997248 +dish_1561997274 +dish_1561997299 +dish_1561999624 +dish_1561999706 +dish_1562008936 +dish_1562008979 +dish_1562009006 +dish_1562009810 +dish_1562009894 +dish_1562009934 +dish_1562085672 +dish_1562099053 +dish_1562099076 +dish_1562099102 +dish_1562099134 +dish_1562099157 +dish_1562168648 +dish_1562168673 +dish_1562168696 +dish_1562168800 +dish_1562172561 +dish_1562172600 +dish_1562172643 +dish_1562617703 +dish_1562617939 +dish_1562617972 +dish_1562618000 +dish_1562690886 +dish_1562690917 +dish_1562690950 +dish_1562691000 +dish_1562691032 +dish_1562691064 +dish_1562700496 +dish_1562700524 +dish_1562700568 +dish_1562700593 +dish_1562703345 +dish_1562703382 +dish_1562703408 +dish_1562703447 +dish_1562774099 +dish_1562774129 +dish_1562774159 +dish_1562774180 +dish_1562774206 +dish_1562776428 +dish_1562776465 +dish_1562776497 +dish_1562786103 +dish_1562788506 +dish_1562788536 +dish_1562788568 +dish_1562788601 +dish_1562790775 +dish_1562790824 +dish_1562790855 +dish_1562876494 +dish_1562876531 +dish_1562963246 +dish_1562963289 +dish_1562963325 +dish_1563208412 +dish_1563208439 +dish_1563208459 +dish_1563218408 +dish_1563220987 +dish_1563221025 +dish_1563221054 +dish_1563221086 +dish_1563221157 +dish_1563294987 +dish_1563295016 +dish_1563295044 +dish_1563305513 +dish_1563378002 +dish_1563378029 +dish_1563378051 +dish_1563378073 +dish_1563378110 +dish_1563379132 +dish_1563379163 +dish_1563389548 +dish_1563389600 +dish_1563389626 +dish_1563478751 +dish_1563551105 +dish_1563551134 +dish_1563551164 +dish_1563551194 +dish_1563551220 +dish_1563565289 +dish_1563565323 +dish_1563565354 +dish_1563565415 +dish_1563565454 +dish_1563565508 +dish_1563566909 +dish_1563566939 +dish_1563566965 +dish_1563897047 +dish_1563897087 +dish_1563897938 +dish_1563897981 +dish_1563898019 +dish_1563898055 +dish_1563898084 +dish_1563898107 +dish_1563909469 +dish_1563909507 +dish_1563909550 +dish_1563909580 +dish_1563984140 +dish_1563984167 +dish_1563984199 +dish_1563984242 +dish_1563984296 +dish_1563996441 +dish_1563996464 +dish_1563996485 +dish_1563996509 +dish_1563998210 +dish_1563998244 +dish_1563998279 +dish_1563998323 +dish_1563998513 +dish_1564160043 +dish_1564427406 +dish_1564427430 +dish_1564427455 +dish_1564428102 +dish_1564428144 +dish_1564428181 +dish_1564588820 +dish_1564588859 +dish_1564588889 +dish_1564588919 +dish_1564675380 +dish_1564675411 +dish_1564686966 +dish_1564687013 +dish_1564687046 +dish_1564687090 +dish_1564689078 +dish_1564689108 +dish_1564689138 +dish_1564689168 +dish_1564689219 +dish_1564773826 +dish_1564773847 +dish_1564773876 +dish_1564773903 +dish_1564773942 +dish_1564773976 +dish_1565020755 +dish_1565020796 +dish_1565020822 +dish_1565020867 +dish_1565020895 +dish_1565020926 +dish_1565030350 +dish_1565030391 +dish_1565034605 +dish_1565034635 +dish_1565034661 +dish_1565034681 +dish_1565035715 +dish_1565035746 +dish_1565035774 +dish_1565035802 +dish_1565035830 +dish_1565035858 +dish_1565117429 +dish_1565117463 +dish_1565118700 +dish_1565118729 +dish_1565118782 +dish_1565123783 +dish_1565123816 +dish_1565123845 +dish_1565123881 +dish_1565638478 +dish_1565638515 +dish_1565638550 +dish_1565638599 +dish_1565711453 +dish_1565711498 +dish_1565711527 +dish_1565711570 +dish_1565810969 +dish_1565811000 +dish_1565811025 +dish_1565811061 +dish_1565811091 +dish_1565811139 +dish_1565894858 +dish_1565894889 +dish_1565894910 +dish_1565896662 +dish_1565896701 +dish_1565896745 +dish_1565898128 +dish_1565898158 +dish_1565898193 +dish_1565898230 +dish_1565898331 +dish_1565898361 +dish_1565898402 +dish_1565898432 +dish_1565974375 +dish_1565974409 +dish_1565986702 +dish_1565986733 +dish_1565986761 +dish_1566246415 +dish_1566246478 +dish_1566246513 +dish_1566246541 +dish_1566246573 +dish_1566246626 +dish_1566316681 +dish_1566316726 +dish_1566316757 +dish_1566328724 +dish_1566328745 +dish_1566328776 +dish_1566328805 +dish_1566328831 +dish_1566414225 +dish_1566414291 +dish_1566414342 +dish_1566414412 +dish_1566416701 +dish_1566501575 +dish_1566501594 +dish_1566502437 +dish_1566502472 +dish_1566502505 +dish_1566502537 +dish_1566502573 +dish_1566587102 +dish_1566587143 +dish_1566587182 +dish_1566589933 +dish_1566589967 +dish_1566590007 +dish_1566590056 +dish_1566838286 +dish_1566838317 +dish_1566838351 +dish_1566838378 +dish_1566838407 +dish_1566849862 +dish_1566849895 +dish_1566849923 +dish_1566849955 +dish_1566849987 +dish_1566850031 +dish_1566850099 +dish_1566850142 +dish_1566851549 +dish_1566851584 +dish_1566851616 +dish_1566851677 +dish_1566851704 +dish_1566920304 +dish_1566920339 +dish_1566920365 +dish_1567007719 +dish_1567007751 +dish_1567007783 +dish_1567007814 +dish_1567007848 +dish_1567107839 +dish_1567785622 +dish_1567785654 +dish_1567785709 +dish_1567785739 +dish_1567785778 +dish_1568146905 +dish_1568146942 +dish_1568146977 +dish_1568147009 +dish_1568147044 +dish_1568305198 +dish_1568305225 +dish_1568305257 +dish_1568401167 +dish_1568401199 +dish_1568401233 +dish_1568401261 +dish_1568401302 diff --git a/indexes/multimodal_train.csv b/indexes/multimodal_train.csv new file mode 100644 index 0000000..9c4ad30 --- /dev/null +++ b/indexes/multimodal_train.csv @@ -0,0 +1,4048 @@ +dish_id +dish_1550704750 +dish_1550705535 +dish_1550705580 +dish_1550705623 +dish_1550705669 +dish_1550705724 +dish_1550706037 +dish_1550706096 +dish_1550706177 +dish_1550706237 +dish_1550706283 +dish_1550706325 +dish_1550706766 +dish_1550706891 +dish_1550706945 +dish_1550707056 +dish_1550707110 +dish_1550707156 +dish_1550707236 +dish_1550707357 +dish_1550707504 +dish_1550707578 +dish_1550707995 +dish_1550708152 +dish_1550708221 +dish_1550708280 +dish_1550708327 +dish_1550708440 +dish_1550708487 +dish_1550708556 +dish_1550708638 +dish_1550708702 +dish_1550708875 +dish_1550708945 +dish_1550708985 +dish_1550709028 +dish_1550709129 +dish_1550710177 +dish_1550710207 +dish_1550710279 +dish_1550710318 +dish_1550710342 +dish_1550710402 +dish_1550710448 +dish_1550710507 +dish_1550710542 +dish_1550710577 +dish_1550710612 +dish_1550710664 +dish_1550710715 +dish_1550710758 +dish_1550710793 +dish_1550710845 +dish_1550710917 +dish_1550711017 +dish_1550711066 +dish_1550711094 +dish_1550711129 +dish_1550711187 +dish_1550711244 +dish_1550711314 +dish_1550711385 +dish_1550711488 +dish_1550711598 +dish_1550711688 +dish_1550711746 +dish_1550711845 +dish_1550711973 +dish_1550712024 +dish_1550712079 +dish_1550712143 +dish_1550712315 +dish_1550712369 +dish_1550712403 +dish_1550712459 +dish_1550712486 +dish_1550712524 +dish_1550712556 +dish_1550712585 +dish_1550712622 +dish_1550712654 +dish_1550712681 +dish_1550712729 +dish_1550712805 +dish_1550712839 +dish_1550712877 +dish_1550712913 +dish_1550712961 +dish_1550713065 +dish_1550713114 +dish_1550713174 +dish_1550713388 +dish_1550713423 +dish_1550713462 +dish_1550713503 +dish_1550769483 +dish_1550769725 +dish_1550769827 +dish_1550769882 +dish_1550770511 +dish_1550770616 +dish_1550770699 +dish_1550770751 +dish_1550770802 +dish_1550770866 +dish_1550771191 +dish_1550771267 +dish_1550771330 +dish_1550771387 +dish_1550771487 +dish_1550771562 +dish_1550771606 +dish_1550771648 +dish_1550771707 +dish_1550771796 +dish_1550771869 +dish_1550771938 +dish_1550771984 +dish_1550772027 +dish_1550772088 +dish_1550772168 +dish_1550772199 +dish_1550772270 +dish_1550772367 +dish_1550772454 +dish_1550772505 +dish_1550772551 +dish_1550772617 +dish_1550772701 +dish_1550772753 +dish_1550772809 +dish_1550772849 +dish_1550772892 +dish_1550772960 +dish_1550773038 +dish_1550773162 +dish_1550773200 +dish_1550773284 +dish_1550773349 +dish_1550773394 +dish_1550773430 +dish_1550773506 +dish_1550773591 +dish_1550773771 +dish_1550773839 +dish_1550773917 +dish_1550773949 +dish_1550774244 +dish_1550774296 +dish_1550774815 +dish_1550774895 +dish_1550775126 +dish_1550775219 +dish_1550775273 +dish_1550775318 +dish_1550775363 +dish_1550775393 +dish_1550775472 +dish_1550776767 +dish_1550776854 +dish_1550776907 +dish_1550776957 +dish_1550777256 +dish_1550777319 +dish_1550777418 +dish_1550777469 +dish_1550777511 +dish_1550777566 +dish_1550777646 +dish_1550777713 +dish_1550777791 +dish_1550777827 +dish_1550777919 +dish_1550778007 +dish_1550778058 +dish_1550778118 +dish_1550778312 +dish_1550778355 +dish_1550778401 +dish_1550778435 +dish_1550778469 +dish_1550778694 +dish_1550778856 +dish_1550778922 +dish_1550778981 +dish_1550779014 +dish_1550779153 +dish_1550779219 +dish_1550779282 +dish_1550779336 +dish_1550779388 +dish_1550780717 +dish_1550781280 +dish_1550781317 +dish_1550781347 +dish_1550781372 +dish_1550785292 +dish_1550785404 +dish_1550785481 +dish_1550785526 +dish_1550785599 +dish_1550785975 +dish_1550788191 +dish_1550788264 +dish_1550788398 +dish_1550788505 +dish_1550788591 +dish_1550788827 +dish_1550788957 +dish_1550789186 +dish_1550789255 +dish_1550789321 +dish_1550789601 +dish_1550789705 +dish_1550789814 +dish_1550789860 +dish_1550790042 +dish_1550790252 +dish_1550790347 +dish_1550790442 +dish_1550790511 +dish_1550790554 +dish_1550790874 +dish_1550791105 +dish_1550791160 +dish_1550791214 +dish_1550791302 +dish_1550791644 +dish_1550791701 +dish_1550791822 +dish_1550791884 +dish_1550791942 +dish_1550792023 +dish_1550792143 +dish_1550792194 +dish_1550792315 +dish_1550792365 +dish_1550792493 +dish_1550792595 +dish_1550792734 +dish_1550792824 +dish_1550793023 +dish_1550793093 +dish_1550793192 +dish_1550793245 +dish_1550793300 +dish_1550793485 +dish_1550793525 +dish_1550793582 +dish_1550793621 +dish_1550793673 +dish_1550795331 +dish_1550795361 +dish_1550795423 +dish_1550795455 +dish_1550795497 +dish_1550795690 +dish_1550795856 +dish_1550796058 +dish_1550796152 +dish_1550796330 +dish_1550796634 +dish_1550796686 +dish_1550796731 +dish_1550796763 +dish_1550796794 +dish_1550796973 +dish_1550797031 +dish_1550797094 +dish_1550797130 +dish_1550797169 +dish_1550797474 +dish_1550797692 +dish_1550797753 +dish_1550797791 +dish_1550797833 +dish_1550859356 +dish_1550859396 +dish_1550859544 +dish_1550859578 +dish_1550859622 +dish_1550860513 +dish_1550860858 +dish_1550860919 +dish_1550861027 +dish_1550861074 +dish_1550861124 +dish_1550861187 +dish_1550861324 +dish_1550861388 +dish_1550861432 +dish_1550861458 +dish_1550861520 +dish_1550861584 +dish_1550861605 +dish_1550861670 +dish_1550861703 +dish_1550861835 +dish_1550861903 +dish_1550861950 +dish_1550861994 +dish_1550862036 +dish_1550862106 +dish_1550862190 +dish_1550862217 +dish_1550862272 +dish_1550862349 +dish_1550862399 +dish_1550862495 +dish_1550862580 +dish_1550862648 +dish_1550862681 +dish_1550862733 +dish_1550862840 +dish_1550862908 +dish_1550862993 +dish_1550863037 +dish_1550863078 +dish_1550863155 +dish_1550863247 +dish_1550863285 +dish_1550863309 +dish_1550863381 +dish_1550873044 +dish_1550873218 +dish_1550873264 +dish_1550873303 +dish_1550873788 +dish_1550873952 +dish_1550874079 +dish_1550874153 +dish_1550874191 +dish_1550874267 +dish_1550874464 +dish_1550874640 +dish_1550874679 +dish_1550874729 +dish_1550874756 +dish_1550874829 +dish_1550874872 +dish_1550874918 +dish_1550874950 +dish_1550875058 +dish_1550875168 +dish_1550875229 +dish_1550875293 +dish_1550875347 +dish_1550875400 +dish_1550875891 +dish_1550875931 +dish_1550875968 +dish_1550876012 +dish_1550876055 +dish_1550876249 +dish_1550876298 +dish_1550876376 +dish_1550876416 +dish_1550876467 +dish_1550876619 +dish_1550876748 +dish_1550876819 +dish_1550876892 +dish_1550876969 +dish_1551122493 +dish_1551122567 +dish_1551122614 +dish_1551122662 +dish_1551122713 +dish_1551122795 +dish_1551122844 +dish_1551122871 +dish_1551122914 +dish_1551122952 +dish_1551123023 +dish_1551123072 +dish_1551123396 +dish_1551123455 +dish_1551123633 +dish_1551123904 +dish_1551123986 +dish_1551124062 +dish_1551124116 +dish_1551124157 +dish_1551124221 +dish_1551124267 +dish_1551124322 +dish_1551124362 +dish_1551124481 +dish_1551124591 +dish_1551124637 +dish_1551124687 +dish_1551124723 +dish_1551125436 +dish_1551133789 +dish_1551133870 +dish_1551134053 +dish_1551134138 +dish_1551134208 +dish_1551134442 +dish_1551134622 +dish_1551134757 +dish_1551134824 +dish_1551134895 +dish_1551134987 +dish_1551135179 +dish_1551135297 +dish_1551135363 +dish_1551135415 +dish_1551135536 +dish_1551135590 +dish_1551135656 +dish_1551135718 +dish_1551135767 +dish_1551135896 +dish_1551136043 +dish_1551136264 +dish_1551136315 +dish_1551136531 +dish_1551136610 +dish_1551136683 +dish_1551136769 +dish_1551136814 +dish_1551136891 +dish_1551137091 +dish_1551137173 +dish_1551137277 +dish_1551137348 +dish_1551137431 +dish_1551138110 +dish_1551138180 +dish_1551138237 +dish_1551139066 +dish_1551139192 +dish_1551139260 +dish_1551139325 +dish_1551139367 +dish_1551139415 +dish_1551139487 +dish_1551139535 +dish_1551139576 +dish_1551139606 +dish_1551139675 +dish_1551139764 +dish_1551139831 +dish_1551139874 +dish_1551139956 +dish_1551141628 +dish_1551141670 +dish_1551141710 +dish_1551141747 +dish_1551141779 +dish_1551141849 +dish_1551141892 +dish_1551141931 +dish_1551142001 +dish_1551142046 +dish_1551142128 +dish_1551142459 +dish_1551142509 +dish_1551142547 +dish_1551142589 +dish_1551221581 +dish_1551221801 +dish_1551221853 +dish_1551221892 +dish_1551221935 +dish_1551222036 +dish_1551222068 +dish_1551222188 +dish_1551222225 +dish_1551222363 +dish_1551222472 +dish_1551222527 +dish_1551222558 +dish_1551222587 +dish_1551222638 +dish_1551222712 +dish_1551222759 +dish_1551222791 +dish_1551222833 +dish_1551222877 +dish_1551222935 +dish_1551222982 +dish_1551223023 +dish_1551223195 +dish_1551223249 +dish_1551223311 +dish_1551223346 +dish_1551223394 +dish_1551224001 +dish_1551224089 +dish_1551224138 +dish_1551224175 +dish_1551224283 +dish_1551224380 +dish_1551224447 +dish_1551224483 +dish_1551224534 +dish_1551224833 +dish_1551224928 +dish_1551225021 +dish_1551225067 +dish_1551225106 +dish_1551225155 +dish_1551225281 +dish_1551225381 +dish_1551225467 +dish_1551225501 +dish_1551225541 +dish_1551225609 +dish_1551225666 +dish_1551225714 +dish_1551225764 +dish_1551225793 +dish_1551225836 +dish_1551225948 +dish_1551226013 +dish_1551226049 +dish_1551226101 +dish_1551226308 +dish_1551226363 +dish_1551226409 +dish_1551226442 +dish_1551226475 +dish_1551226840 +dish_1551226892 +dish_1551226948 +dish_1551226999 +dish_1551227050 +dish_1551227165 +dish_1551227216 +dish_1551227280 +dish_1551227325 +dish_1551227435 +dish_1551227539 +dish_1551227581 +dish_1551227628 +dish_1551227669 +dish_1551228296 +dish_1551228355 +dish_1551228396 +dish_1551228438 +dish_1551228551 +dish_1551228582 +dish_1551228630 +dish_1551228669 +dish_1551228715 +dish_1551228830 +dish_1551228876 +dish_1551228930 +dish_1551228971 +dish_1551229028 +dish_1551232627 +dish_1551232686 +dish_1551232713 +dish_1551232748 +dish_1551232782 +dish_1551232851 +dish_1551232905 +dish_1551232936 +dish_1551232973 +dish_1551233007 +dish_1551233095 +dish_1551233150 +dish_1551233180 +dish_1551233210 +dish_1551233248 +dish_1551233313 +dish_1551233345 +dish_1551233375 +dish_1551233403 +dish_1551233434 +dish_1551233491 +dish_1551233523 +dish_1551233573 +dish_1551233610 +dish_1551233633 +dish_1551233708 +dish_1551233745 +dish_1551233802 +dish_1551233827 +dish_1551233859 +dish_1551233938 +dish_1551233982 +dish_1551234017 +dish_1551234046 +dish_1551234106 +dish_1551235109 +dish_1551235171 +dish_1551235211 +dish_1551235240 +dish_1551235270 +dish_1551235759 +dish_1551235868 +dish_1551235946 +dish_1551235983 +dish_1551236011 +dish_1551236159 +dish_1551236417 +dish_1551236466 +dish_1551236495 +dish_1551236521 +dish_1551236550 +dish_1551236621 +dish_1551236657 +dish_1551236712 +dish_1551236775 +dish_1551236827 +dish_1551236949 +dish_1551236981 +dish_1551237016 +dish_1551237046 +dish_1551237348 +dish_1551237436 +dish_1551237463 +dish_1551237498 +dish_1551237527 +dish_1551237558 +dish_1551237607 +dish_1551237710 +dish_1551237778 +dish_1551237814 +dish_1551237854 +dish_1551303168 +dish_1551303285 +dish_1551303338 +dish_1551303437 +dish_1551303476 +dish_1551303644 +dish_1551303727 +dish_1551303770 +dish_1551303822 +dish_1551303866 +dish_1551303912 +dish_1551303984 +dish_1551304110 +dish_1551304155 +dish_1551304197 +dish_1551304349 +dish_1551304456 +dish_1551304571 +dish_1551304608 +dish_1551304741 +dish_1551304854 +dish_1551304973 +dish_1551305168 +dish_1551305209 +dish_1551305257 +dish_1551305410 +dish_1551306241 +dish_1551306290 +dish_1551306339 +dish_1551306375 +dish_1551306452 +dish_1551306491 +dish_1551306561 +dish_1551306647 +dish_1551306691 +dish_1551306798 +dish_1551306860 +dish_1551306958 +dish_1551307013 +dish_1551307090 +dish_1551307170 +dish_1551307258 +dish_1551307325 +dish_1551307361 +dish_1551313782 +dish_1551313822 +dish_1551313872 +dish_1551313896 +dish_1551313924 +dish_1551314163 +dish_1551314254 +dish_1551314290 +dish_1551314337 +dish_1551314360 +dish_1551314390 +dish_1551314442 +dish_1551314496 +dish_1551314531 +dish_1551314560 +dish_1551314669 +dish_1551314810 +dish_1551314855 +dish_1551314927 +dish_1551314967 +dish_1551317350 +dish_1551317449 +dish_1551317508 +dish_1551317589 +dish_1551317628 +dish_1551317743 +dish_1551317998 +dish_1551318068 +dish_1551318183 +dish_1551318377 +dish_1551318422 +dish_1551318504 +dish_1551318543 +dish_1551318603 +dish_1551318673 +dish_1551318716 +dish_1551319895 +dish_1551320684 +dish_1551321715 +dish_1551321757 +dish_1551321793 +dish_1551322577 +dish_1551322652 +dish_1551322714 +dish_1551322755 +dish_1551322804 +dish_1551322890 +dish_1551322934 +dish_1551322988 +dish_1551323035 +dish_1551323073 +dish_1551323219 +dish_1551323262 +dish_1551323309 +dish_1551323342 +dish_1551323381 +dish_1551323433 +dish_1551323487 +dish_1551323574 +dish_1551323618 +dish_1551323662 +dish_1551323794 +dish_1551323866 +dish_1551323937 +dish_1551323969 +dish_1551324010 +dish_1551324079 +dish_1551324180 +dish_1551324246 +dish_1551324284 +dish_1551324331 +dish_1551373181 +dish_1551373253 +dish_1551373647 +dish_1551373738 +dish_1551373813 +dish_1551374021 +dish_1551374155 +dish_1551374189 +dish_1551374222 +dish_1551374332 +dish_1551374440 +dish_1551374463 +dish_1551374502 +dish_1551374527 +dish_1551374560 +dish_1551374831 +dish_1551374895 +dish_1551374946 +dish_1551374982 +dish_1551375027 +dish_1551375086 +dish_1551375142 +dish_1551375189 +dish_1551375218 +dish_1551375297 +dish_1551375366 +dish_1551375411 +dish_1551375443 +dish_1551375480 +dish_1551375596 +dish_1551375650 +dish_1551375715 +dish_1551375747 +dish_1551375848 +dish_1551375933 +dish_1551375981 +dish_1551376575 +dish_1551376692 +dish_1551376850 +dish_1551376892 +dish_1551376933 +dish_1551376970 +dish_1551377052 +dish_1551377094 +dish_1551377135 +dish_1551377177 +dish_1551377209 +dish_1551377316 +dish_1551377388 +dish_1551377438 +dish_1551377473 +dish_1551377674 +dish_1551377737 +dish_1551377782 +dish_1551377828 +dish_1551377874 +dish_1551378018 +dish_1551378054 +dish_1551378099 +dish_1551378151 +dish_1551378179 +dish_1551378291 +dish_1551378344 +dish_1551378397 +dish_1551378449 +dish_1551378490 +dish_1551378584 +dish_1551378635 +dish_1551378684 +dish_1551378719 +dish_1551378829 +dish_1551378898 +dish_1551378955 +dish_1551378996 +dish_1551379045 +dish_1551379162 +dish_1551379252 +dish_1551379331 +dish_1551379379 +dish_1551379408 +dish_1551379478 +dish_1551379524 +dish_1551379566 +dish_1551379605 +dish_1551379654 +dish_1551379741 +dish_1551379788 +dish_1551379852 +dish_1551379881 +dish_1551379922 +dish_1551380856 +dish_1551380910 +dish_1551380950 +dish_1551381050 +dish_1551381103 +dish_1551381173 +dish_1551381208 +dish_1551381249 +dish_1551381285 +dish_1551381321 +dish_1551381365 +dish_1551381404 +dish_1551381450 +dish_1551381496 +dish_1551381555 +dish_1551381661 +dish_1551381730 +dish_1551381829 +dish_1551381877 +dish_1551381990 +dish_1551382060 +dish_1551382115 +dish_1551382149 +dish_1551382255 +dish_1551384024 +dish_1551384080 +dish_1551384115 +dish_1551384167 +dish_1551388984 +dish_1551389118 +dish_1551389176 +dish_1551389218 +dish_1551389268 +dish_1551389387 +dish_1551389458 +dish_1551389506 +dish_1551389551 +dish_1551389588 +dish_1551389684 +dish_1551389749 +dish_1551390006 +dish_1551390051 +dish_1551390087 +dish_1551390215 +dish_1551390265 +dish_1551390304 +dish_1551390341 +dish_1551390382 +dish_1551390429 +dish_1551390474 +dish_1551390528 +dish_1551390560 +dish_1551390594 +dish_1551390651 +dish_1551390758 +dish_1551390802 +dish_1551390837 +dish_1551390872 +dish_1551391122 +dish_1551391197 +dish_1551391257 +dish_1551391302 +dish_1551391337 +dish_1551391379 +dish_1551391586 +dish_1551391795 +dish_1551392271 +dish_1551392316 +dish_1551392347 +dish_1551392376 +dish_1551392424 +dish_1551392519 +dish_1551392557 +dish_1551392590 +dish_1551392654 +dish_1551392701 +dish_1551392752 +dish_1551392833 +dish_1551392884 +dish_1551392919 +dish_1551392958 +dish_1551393021 +dish_1551393069 +dish_1551393113 +dish_1551393161 +dish_1551393210 +dish_1551394614 +dish_1551394781 +dish_1551394835 +dish_1551394874 +dish_1551394927 +dish_1551395022 +dish_1551395088 +dish_1551395194 +dish_1551395233 +dish_1551395269 +dish_1551395396 +dish_1551395512 +dish_1551395580 +dish_1551395766 +dish_1551395800 +dish_1551396309 +dish_1551396629 +dish_1551396678 +dish_1551397543 +dish_1551397591 +dish_1551397629 +dish_1551397714 +dish_1551397752 +dish_1551399031 +dish_1551399100 +dish_1551399151 +dish_1551399293 +dish_1551399340 +dish_1551399402 +dish_1551399449 +dish_1551399484 +dish_1551485983 +dish_1551486108 +dish_1551486158 +dish_1551486217 +dish_1551486277 +dish_1551486372 +dish_1551486411 +dish_1551486571 +dish_1551486628 +dish_1551486680 +dish_1551487117 +dish_1551487158 +dish_1551487288 +dish_1551487472 +dish_1551487560 +dish_1551487735 +dish_1551487779 +dish_1551488103 +dish_1551488137 +dish_1551488197 +dish_1551488324 +dish_1551488371 +dish_1551488415 +dish_1551488448 +dish_1551488572 +dish_1551488691 +dish_1551488737 +dish_1551488776 +dish_1551488807 +dish_1551488861 +dish_1551489812 +dish_1551489885 +dish_1551489931 +dish_1551489961 +dish_1551490399 +dish_1551490433 +dish_1551490469 +dish_1551490503 +dish_1551490544 +dish_1551490670 +dish_1551490701 +dish_1551490740 +dish_1551490767 +dish_1551490819 +dish_1551490896 +dish_1551490962 +dish_1551491020 +dish_1551491048 +dish_1551491090 +dish_1551491179 +dish_1551491216 +dish_1551491271 +dish_1551491297 +dish_1551491326 +dish_1551492093 +dish_1551492252 +dish_1551492287 +dish_1551492318 +dish_1551492348 +dish_1551492369 +dish_1551492468 +dish_1551492506 +dish_1551492534 +dish_1551492561 +dish_1551492590 +dish_1551492666 +dish_1551492702 +dish_1551492732 +dish_1551492762 +dish_1551492790 +dish_1551492866 +dish_1551492908 +dish_1551492935 +dish_1551492975 +dish_1551493005 +dish_1551493123 +dish_1551493159 +dish_1551493236 +dish_1551493278 +dish_1551493680 +dish_1551493698 +dish_1551493723 +dish_1551493749 +dish_1551493773 +dish_1551493823 +dish_1551493854 +dish_1551493883 +dish_1551493916 +dish_1551494035 +dish_1551494058 +dish_1551494095 +dish_1551494147 +dish_1551494178 +dish_1551562881 +dish_1551563001 +dish_1551563119 +dish_1551563170 +dish_1551563213 +dish_1551563294 +dish_1551563337 +dish_1551563370 +dish_1551563400 +dish_1551563429 +dish_1551563511 +dish_1551563550 +dish_1551563591 +dish_1551563627 +dish_1551563660 +dish_1551563723 +dish_1551563773 +dish_1551563824 +dish_1551563853 +dish_1551563890 +dish_1551563925 +dish_1551563964 +dish_1551564016 +dish_1551564053 +dish_1551564282 +dish_1551564543 +dish_1551564577 +dish_1551564637 +dish_1551564857 +dish_1551564907 +dish_1551565096 +dish_1551565168 +dish_1551565204 +dish_1551565237 +dish_1551565289 +dish_1551565321 +dish_1551565379 +dish_1551565425 +dish_1551565464 +dish_1551565495 +dish_1551565529 +dish_1551565570 +dish_1551565613 +dish_1551565653 +dish_1551565720 +dish_1551565752 +dish_1551565797 +dish_1551565845 +dish_1551565884 +dish_1551565909 +dish_1551565996 +dish_1551566047 +dish_1551566083 +dish_1551566130 +dish_1551566208 +dish_1551566236 +dish_1551566277 +dish_1551566316 +dish_1551566356 +dish_1551566421 +dish_1551566464 +dish_1551566499 +dish_1551566524 +dish_1551566547 +dish_1551566592 +dish_1551566642 +dish_1551566687 +dish_1551566753 +dish_1551567004 +dish_1551567230 +dish_1551567293 +dish_1551567329 +dish_1551567368 +dish_1551567396 +dish_1551567426 +dish_1551567604 +dish_1551567650 +dish_1551567694 +dish_1551567731 +dish_1551567775 +dish_1551567805 +dish_1551567992 +dish_1551568059 +dish_1551568156 +dish_1551568200 +dish_1551568240 +dish_1551568285 +dish_1556572657 +dish_1556573514 +dish_1556575014 +dish_1556575083 +dish_1556575124 +dish_1556575273 +dish_1556575386 +dish_1556575446 +dish_1556575499 +dish_1556575558 +dish_1556575700 +dish_1557853154 +dish_1557853229 +dish_1557853314 +dish_1557853429 +dish_1557861605 +dish_1557861697 +dish_1557861795 +dish_1557861837 +dish_1557862384 +dish_1557862829 +dish_1557863012 +dish_1557863056 +dish_1557863104 +dish_1557863170 +dish_1558026623 +dish_1558026684 +dish_1558026714 +dish_1558026756 +dish_1558027001 +dish_1558027057 +dish_1558027164 +dish_1558027243 +dish_1558028007 +dish_1558028057 +dish_1558028142 +dish_1558028194 +dish_1558028492 +dish_1558028606 +dish_1558028690 +dish_1558028772 +dish_1558029148 +dish_1558029230 +dish_1558029278 +dish_1558029324 +dish_1558029518 +dish_1558029686 +dish_1558029842 +dish_1558029923 +dish_1558030724 +dish_1558030832 +dish_1558030910 +dish_1558030971 +dish_1558031019 +dish_1558031526 +dish_1558031566 +dish_1558031628 +dish_1558031675 +dish_1558032156 +dish_1558109511 +dish_1558109714 +dish_1558109864 +dish_1558109907 +dish_1558109945 +dish_1558109986 +dish_1558110245 +dish_1558113154 +dish_1558113204 +dish_1558113255 +dish_1558113304 +dish_1558114441 +dish_1558114484 +dish_1558114523 +dish_1558114569 +dish_1558114609 +dish_1558114875 +dish_1558114937 +dish_1558115011 +dish_1558115047 +dish_1558115169 +dish_1558115241 +dish_1558115282 +dish_1558115320 +dish_1558115364 +dish_1558115448 +dish_1558115512 +dish_1558115601 +dish_1558115641 +dish_1558115764 +dish_1558115825 +dish_1558115867 +dish_1558115915 +dish_1558115960 +dish_1558116001 +dish_1558116090 +dish_1558116146 +dish_1558116201 +dish_1558116260 +dish_1558116298 +dish_1558116399 +dish_1558116436 +dish_1558116486 +dish_1558116547 +dish_1558122583 +dish_1558122740 +dish_1558372433 +dish_1558372525 +dish_1558372650 +dish_1558372695 +dish_1558372771 +dish_1558373284 +dish_1558373352 +dish_1558373405 +dish_1558373449 +dish_1558373492 +dish_1558373639 +dish_1558373687 +dish_1558375506 +dish_1558375583 +dish_1558377105 +dish_1558378516 +dish_1558378553 +dish_1558378634 +dish_1558379128 +dish_1558379182 +dish_1558379273 +dish_1558379324 +dish_1558379391 +dish_1558379588 +dish_1558379639 +dish_1558379712 +dish_1558379876 +dish_1558380014 +dish_1558380058 +dish_1558380152 +dish_1558380181 +dish_1558380414 +dish_1558380464 +dish_1558380527 +dish_1558380557 +dish_1558380804 +dish_1558381032 +dish_1558381084 +dish_1558381143 +dish_1558381206 +dish_1558381665 +dish_1558381708 +dish_1558381742 +dish_1558381819 +dish_1558381840 +dish_1558381860 +dish_1558381943 +dish_1558382263 +dish_1558457415 +dish_1558457468 +dish_1558457530 +dish_1558457577 +dish_1558457665 +dish_1558458438 +dish_1558458496 +dish_1558458541 +dish_1558458584 +dish_1558458737 +dish_1558458847 +dish_1558458903 +dish_1558459020 +dish_1558459059 +dish_1558459115 +dish_1558459276 +dish_1558459395 +dish_1558459462 +dish_1558459528 +dish_1558459602 +dish_1558459664 +dish_1558459945 +dish_1558459994 +dish_1558460061 +dish_1558460138 +dish_1558460205 +dish_1558460908 +dish_1558462036 +dish_1558462106 +dish_1558462228 +dish_1558462531 +dish_1558463113 +dish_1558463362 +dish_1558463429 +dish_1558463562 +dish_1558463690 +dish_1558468285 +dish_1558468401 +dish_1558468435 +dish_1558468494 +dish_1558468547 +dish_1558468589 +dish_1558468632 +dish_1558468731 +dish_1558544427 +dish_1558544482 +dish_1558544870 +dish_1558546918 +dish_1558547008 +dish_1558547068 +dish_1558547124 +dish_1558547205 +dish_1558548666 +dish_1558548958 +dish_1558549008 +dish_1558549086 +dish_1558549150 +dish_1558549214 +dish_1558550079 +dish_1558550127 +dish_1558550181 +dish_1558550218 +dish_1558628339 +dish_1558628515 +dish_1558628711 +dish_1558628760 +dish_1558628888 +dish_1558628921 +dish_1558628956 +dish_1558629013 +dish_1558629041 +dish_1558629398 +dish_1558629444 +dish_1558629488 +dish_1558629517 +dish_1558629578 +dish_1558629644 +dish_1558629677 +dish_1558629715 +dish_1558629818 +dish_1558629878 +dish_1558630006 +dish_1558630038 +dish_1558630072 +dish_1558630450 +dish_1558630488 +dish_1558630600 +dish_1558630649 +dish_1558630737 +dish_1558630814 +dish_1558630867 +dish_1558630932 +dish_1558631825 +dish_1558631872 +dish_1558631904 +dish_1558631932 +dish_1558632080 +dish_1558632130 +dish_1558632198 +dish_1558632233 +dish_1558632445 +dish_1558632475 +dish_1558632799 +dish_1558632921 +dish_1558632959 +dish_1558632990 +dish_1558633158 +dish_1558633221 +dish_1558635865 +dish_1558635933 +dish_1558635962 +dish_1558635994 +dish_1558636028 +dish_1558636388 +dish_1558636483 +dish_1558636514 +dish_1558636544 +dish_1558636582 +dish_1558636763 +dish_1558636813 +dish_1558636868 +dish_1558636902 +dish_1558636928 +dish_1558636980 +dish_1558637214 +dish_1558637247 +dish_1558637351 +dish_1558637393 +dish_1558637423 +dish_1558637698 +dish_1558637769 +dish_1558637865 +dish_1558637896 +dish_1558637932 +dish_1558638137 +dish_1558638163 +dish_1558638216 +dish_1558638246 +dish_1558638292 +dish_1558638844 +dish_1558638896 +dish_1558638924 +dish_1558638954 +dish_1558638983 +dish_1558639012 +dish_1558639045 +dish_1558639089 +dish_1558639138 +dish_1558639167 +dish_1558639195 +dish_1558639221 +dish_1558639266 +dish_1558639306 +dish_1558639335 +dish_1558639362 +dish_1558639588 +dish_1558639630 +dish_1558639662 +dish_1558639673 +dish_1558639683 +dish_1558639752 +dish_1558639907 +dish_1558639949 +dish_1558639979 +dish_1558640008 +dish_1558640039 +dish_1558640074 +dish_1558640105 +dish_1558640145 +dish_1558640190 +dish_1558640238 +dish_1558640267 +dish_1558640294 +dish_1558640311 +dish_1558640373 +dish_1558640418 +dish_1558640450 +dish_1558640477 +dish_1558640513 +dish_1558640563 +dish_1558640593 +dish_1558640604 +dish_1558640614 +dish_1558640671 +dish_1558640705 +dish_1558640748 +dish_1558640781 +dish_1558640808 +dish_1558640849 +dish_1558640882 +dish_1558640917 +dish_1558640941 +dish_1558640979 +dish_1558641007 +dish_1558641037 +dish_1558641075 +dish_1558641108 +dish_1558641211 +dish_1558641358 +dish_1558641391 +dish_1558641416 +dish_1558641442 +dish_1558641897 +dish_1558641926 +dish_1558641954 +dish_1558641989 +dish_1558642021 +dish_1558642089 +dish_1558642420 +dish_1558642458 +dish_1558642490 +dish_1558717886 +dish_1558717969 +dish_1558718009 +dish_1558718051 +dish_1558718088 +dish_1558718974 +dish_1558719040 +dish_1558719066 +dish_1558719097 +dish_1558719145 +dish_1558719215 +dish_1558719298 +dish_1558719336 +dish_1558719377 +dish_1558719401 +dish_1558719514 +dish_1558720289 +dish_1558720335 +dish_1558720364 +dish_1558720396 +dish_1558720427 +dish_1558721272 +dish_1558721323 +dish_1558721345 +dish_1558721380 +dish_1558721434 +dish_1558721484 +dish_1558721533 +dish_1558721559 +dish_1558721587 +dish_1558721636 +dish_1558721669 +dish_1558721700 +dish_1558721726 +dish_1558721757 +dish_1558721827 +dish_1558721904 +dish_1558721931 +dish_1558721959 +dish_1558721980 +dish_1558722068 +dish_1558722103 +dish_1558722322 +dish_1558722370 +dish_1558722398 +dish_1558722434 +dish_1558722473 +dish_1558722636 +dish_1558722665 +dish_1558722685 +dish_1558722707 +dish_1558722748 +dish_1558722780 +dish_1558722842 +dish_1558722871 +dish_1558722883 +dish_1558722906 +dish_1558722931 +dish_1558722968 +dish_1558723015 +dish_1558723047 +dish_1558723073 +dish_1558723096 +dish_1558723123 +dish_1558723414 +dish_1558723449 +dish_1558723475 +dish_1558723512 +dish_1558723537 +dish_1558723568 +dish_1558723639 +dish_1558723671 +dish_1558723695 +dish_1558723719 +dish_1558723742 +dish_1558723777 +dish_1558723818 +dish_1558723847 +dish_1558723866 +dish_1558723894 +dish_1558723926 +dish_1558723960 +dish_1558723982 +dish_1558724005 +dish_1558724031 +dish_1558724574 +dish_1558724607 +dish_1558724633 +dish_1558724658 +dish_1558724689 +dish_1558724723 +dish_1558724766 +dish_1558724799 +dish_1558724853 +dish_1558724891 +dish_1558724912 +dish_1558724932 +dish_1558724959 +dish_1558724992 +dish_1558725049 +dish_1558725085 +dish_1558725112 +dish_1558725193 +dish_1558725226 +dish_1558725253 +dish_1558725286 +dish_1558725353 +dish_1558725403 +dish_1558725576 +dish_1558725621 +dish_1558725702 +dish_1558725734 +dish_1558725760 +dish_1558725786 +dish_1558725829 +dish_1559059924 +dish_1559059954 +dish_1559059992 +dish_1559060055 +dish_1559060106 +dish_1559061675 +dish_1559061752 +dish_1559062005 +dish_1559062051 +dish_1559062082 +dish_1559149615 +dish_1559149650 +dish_1559149688 +dish_1559149719 +dish_1559158263 +dish_1559158354 +dish_1559158425 +dish_1559172356 +dish_1559232986 +dish_1559233055 +dish_1559233102 +dish_1559233144 +dish_1559235639 +dish_1559235690 +dish_1559235771 +dish_1559235815 +dish_1559238435 +dish_1559238481 +dish_1559238566 +dish_1559238608 +dish_1559239753 +dish_1559239912 +dish_1559239985 +dish_1559240089 +dish_1559240216 +dish_1559240256 +dish_1559240382 +dish_1559240441 +dish_1559240510 +dish_1559240875 +dish_1559240908 +dish_1559240987 +dish_1559241099 +dish_1559242066 +dish_1559242155 +dish_1559242221 +dish_1559242324 +dish_1559242568 +dish_1559243139 +dish_1559243704 +dish_1559243887 +dish_1559243974 +dish_1559244305 +dish_1559244417 +dish_1559244546 +dish_1559244598 +dish_1559244649 +dish_1559245340 +dish_1559245589 +dish_1559245654 +dish_1559245697 +dish_1559245801 +dish_1559245848 +dish_1559245920 +dish_1559245979 +dish_1559246053 +dish_1559319826 +dish_1559319860 +dish_1559319902 +dish_1559319971 +dish_1559320019 +dish_1559332418 +dish_1559332487 +dish_1559332795 +dish_1559332822 +dish_1559581198 +dish_1559581236 +dish_1559581333 +dish_1559581356 +dish_1559581378 +dish_1559585533 +dish_1559589873 +dish_1559589902 +dish_1559589942 +dish_1559590005 +dish_1559590031 +dish_1559590099 +dish_1559590130 +dish_1559590157 +dish_1559590202 +dish_1559593646 +dish_1559593722 +dish_1559593746 +dish_1559593794 +dish_1559667948 +dish_1559667973 +dish_1559667994 +dish_1559668016 +dish_1559668238 +dish_1559668272 +dish_1559668314 +dish_1559674074 +dish_1559674689 +dish_1559674781 +dish_1559674866 +dish_1559675101 +dish_1559675146 +dish_1559675178 +dish_1559675234 +dish_1559675268 +dish_1559678083 +dish_1559678104 +dish_1559678127 +dish_1559678146 +dish_1559678330 +dish_1559678367 +dish_1559678405 +dish_1559678457 +dish_1559678629 +dish_1559678651 +dish_1559700851 +dish_1559751458 +dish_1559751552 +dish_1559751710 +dish_1559751744 +dish_1559751775 +dish_1559751801 +dish_1559838214 +dish_1559838279 +dish_1559838402 +dish_1559841023 +dish_1559841061 +dish_1559841091 +dish_1559841127 +dish_1559841163 +dish_1559841571 +dish_1559841781 +dish_1559841808 +dish_1559841870 +dish_1559841891 +dish_1559841908 +dish_1559842274 +dish_1559842326 +dish_1559842374 +dish_1559842409 +dish_1559843030 +dish_1559843052 +dish_1559843185 +dish_1559843222 +dish_1559843335 +dish_1559843492 +dish_1559843690 +dish_1559843715 +dish_1559844030 +dish_1559844063 +dish_1559844090 +dish_1559844113 +dish_1559844479 +dish_1559844506 +dish_1559844611 +dish_1559844657 +dish_1559844683 +dish_1559844928 +dish_1559844951 +dish_1559845018 +dish_1559845046 +dish_1559845290 +dish_1559845370 +dish_1559845399 +dish_1559845513 +dish_1559845637 +dish_1559845745 +dish_1559845767 +dish_1559846034 +dish_1559846073 +dish_1559846099 +dish_1559846311 +dish_1559846334 +dish_1559846363 +dish_1559846450 +dish_1559846504 +dish_1559846528 +dish_1559846585 +dish_1559846825 +dish_1559848857 +dish_1559848882 +dish_1559848929 +dish_1559925037 +dish_1559925083 +dish_1559925174 +dish_1559932674 +dish_1559933793 +dish_1559933818 +dish_1559933834 +dish_1559934645 +dish_1559934679 +dish_1559934708 +dish_1560270537 +dish_1560270585 +dish_1560270630 +dish_1560356314 +dish_1560356337 +dish_1560356355 +dish_1560356523 +dish_1560356566 +dish_1560356627 +dish_1560357267 +dish_1560357297 +dish_1560357427 +dish_1560357792 +dish_1560357832 +dish_1560360030 +dish_1560360055 +dish_1560360102 +dish_1560360112 +dish_1560360148 +dish_1560366389 +dish_1560366410 +dish_1560366438 +dish_1560366460 +dish_1560366963 +dish_1560366998 +dish_1560367085 +dish_1560367116 +dish_1560367153 +dish_1560367193 +dish_1560367258 +dish_1560367296 +dish_1560367333 +dish_1560367676 +dish_1560367698 +dish_1560367715 +dish_1560367733 +dish_1560368110 +dish_1560368138 +dish_1560368162 +dish_1560368691 +dish_1560368712 +dish_1560368738 +dish_1560368764 +dish_1560368815 +dish_1560368944 +dish_1560368972 +dish_1560368990 +dish_1560369008 +dish_1560369080 +dish_1560369102 +dish_1560369122 +dish_1560369146 +dish_1560369255 +dish_1560369273 +dish_1560369294 +dish_1560369313 +dish_1560369403 +dish_1560369421 +dish_1560370114 +dish_1560370137 +dish_1560370804 +dish_1560370833 +dish_1560441247 +dish_1560441270 +dish_1560441288 +dish_1560441311 +dish_1560441339 +dish_1560441464 +dish_1560441487 +dish_1560441511 +dish_1560441529 +dish_1560442226 +dish_1560442258 +dish_1560442279 +dish_1560442303 +dish_1560442374 +dish_1560442450 +dish_1560443437 +dish_1560443474 +dish_1560443537 +dish_1560443565 +dish_1560443607 +dish_1560444570 +dish_1560444592 +dish_1560444616 +dish_1560444637 +dish_1560444701 +dish_1560444724 +dish_1560444756 +dish_1560444785 +dish_1560444815 +dish_1560445069 +dish_1560453132 +dish_1560453150 +dish_1560453174 +dish_1560453275 +dish_1560453294 +dish_1560453328 +dish_1560453356 +dish_1560453380 +dish_1560453412 +dish_1560454488 +dish_1560454513 +dish_1560454539 +dish_1560454562 +dish_1560454970 +dish_1560455000 +dish_1560455030 +dish_1560455059 +dish_1560455090 +dish_1560455158 +dish_1560455179 +dish_1560455203 +dish_1560455224 +dish_1560455263 +dish_1560455343 +dish_1560455380 +dish_1560455399 +dish_1560455421 +dish_1560455442 +dish_1560456794 +dish_1560456814 +dish_1560456833 +dish_1560456853 +dish_1560527070 +dish_1560527105 +dish_1560527140 +dish_1560527168 +dish_1560527191 +dish_1560527225 +dish_1560527745 +dish_1560527950 +dish_1560529804 +dish_1560529825 +dish_1560529844 +dish_1560539314 +dish_1560539340 +dish_1560539368 +dish_1560539396 +dish_1560541680 +dish_1560541706 +dish_1560541760 +dish_1560541793 +dish_1560543465 +dish_1560543505 +dish_1560543525 +dish_1560543557 +dish_1560543583 +dish_1560543605 +dish_1560543625 +dish_1560543682 +dish_1560543704 +dish_1560543729 +dish_1560543755 +dish_1560787696 +dish_1560787754 +dish_1560787781 +dish_1560787817 +dish_1560788338 +dish_1560788349 +dish_1560788384 +dish_1560798656 +dish_1560798707 +dish_1560798744 +dish_1560798785 +dish_1560798815 +dish_1560798854 +dish_1560798928 +dish_1560798960 +dish_1560799113 +dish_1560799277 +dish_1560799368 +dish_1560799407 +dish_1560799558 +dish_1560799590 +dish_1560799615 +dish_1560799647 +dish_1560800836 +dish_1560802006 +dish_1560802048 +dish_1560802092 +dish_1560802149 +dish_1560802562 +dish_1560802730 +dish_1560802764 +dish_1560802853 +dish_1560874548 +dish_1560874667 +dish_1560874726 +dish_1560874751 +dish_1560887408 +dish_1560887435 +dish_1560887457 +dish_1560887480 +dish_1560888817 +dish_1560888849 +dish_1560888893 +dish_1560888930 +dish_1560889008 +dish_1560889032 +dish_1560889061 +dish_1560961012 +dish_1560961059 +dish_1560961107 +dish_1560961179 +dish_1560961202 +dish_1560961261 +dish_1560961385 +dish_1560973936 +dish_1560973958 +dish_1560973984 +dish_1560974009 +dish_1560974707 +dish_1560974769 +dish_1560975346 +dish_1560976044 +dish_1561049320 +dish_1561049373 +dish_1561049409 +dish_1561049435 +dish_1561060448 +dish_1561060477 +dish_1561060503 +dish_1561060531 +dish_1561060558 +dish_1561060613 +dish_1561060642 +dish_1561060703 +dish_1561060787 +dish_1561060925 +dish_1561060976 +dish_1561061123 +dish_1561061593 +dish_1561061630 +dish_1561061658 +dish_1561061687 +dish_1561062171 +dish_1561062237 +dish_1561062286 +dish_1561062320 +dish_1561062555 +dish_1561062586 +dish_1561134102 +dish_1561134128 +dish_1561134156 +dish_1561145732 +dish_1561146758 +dish_1561146781 +dish_1561148303 +dish_1561148336 +dish_1561148481 +dish_1561392386 +dish_1561392410 +dish_1561392430 +dish_1561392459 +dish_1561392512 +dish_1561392543 +dish_1561392570 +dish_1561392593 +dish_1561392632 +dish_1561392710 +dish_1561392734 +dish_1561393550 +dish_1561393619 +dish_1561393649 +dish_1561404438 +dish_1561404458 +dish_1561404485 +dish_1561404505 +dish_1561404924 +dish_1561404956 +dish_1561405008 +dish_1561405034 +dish_1561405836 +dish_1561405864 +dish_1561405887 +dish_1561405912 +dish_1561405934 +dish_1561406762 +dish_1561406789 +dish_1561406817 +dish_1561407208 +dish_1561407249 +dish_1561407283 +dish_1561407312 +dish_1561480391 +dish_1561480414 +dish_1561480439 +dish_1561480460 +dish_1561481577 +dish_1561481606 +dish_1561481634 +dish_1561481673 +dish_1561491121 +dish_1561491154 +dish_1561491180 +dish_1561491210 +dish_1561492161 +dish_1561492182 +dish_1561492204 +dish_1561492228 +dish_1561492447 +dish_1561492519 +dish_1561492628 +dish_1561492665 +dish_1561492687 +dish_1561492703 +dish_1561493975 +dish_1561494001 +dish_1561494032 +dish_1561567053 +dish_1561567086 +dish_1561574215 +dish_1561574600 +dish_1561574623 +dish_1561574974 +dish_1561575033 +dish_1561575068 +dish_1561575170 +dish_1561575192 +dish_1561575274 +dish_1561575300 +dish_1561575327 +dish_1561575347 +dish_1561575410 +dish_1561575445 +dish_1561575474 +dish_1561575506 +dish_1561575613 +dish_1561575661 +dish_1561575688 +dish_1561575908 +dish_1561575951 +dish_1561575977 +dish_1561575996 +dish_1561576026 +dish_1561576051 +dish_1561576074 +dish_1561576212 +dish_1561576257 +dish_1561576295 +dish_1561576320 +dish_1561576449 +dish_1561576483 +dish_1561576590 +dish_1561576618 +dish_1561576652 +dish_1561576680 +dish_1561576707 +dish_1561576737 +dish_1561576767 +dish_1561576794 +dish_1561576887 +dish_1561576916 +dish_1561576954 +dish_1561576978 +dish_1561577083 +dish_1561577119 +dish_1561577146 +dish_1561577177 +dish_1561577198 +dish_1561577219 +dish_1561577549 +dish_1561577573 +dish_1561577598 +dish_1561577678 +dish_1561577701 +dish_1561577731 +dish_1561578156 +dish_1561578188 +dish_1561578226 +dish_1561578323 +dish_1561578344 +dish_1561578373 +dish_1561578394 +dish_1561578417 +dish_1561578441 +dish_1561578461 +dish_1561578570 +dish_1561578603 +dish_1561578623 +dish_1561578676 +dish_1561579505 +dish_1561579576 +dish_1561579617 +dish_1561579673 +dish_1561579706 +dish_1561579824 +dish_1561579845 +dish_1561579867 +dish_1561579902 +dish_1561579933 +dish_1561580278 +dish_1561580322 +dish_1561580370 +dish_1561580406 +dish_1561580431 +dish_1561580552 +dish_1561580608 +dish_1561580628 +dish_1561580659 +dish_1561654707 +dish_1561654732 +dish_1561654756 +dish_1561661449 +dish_1561661479 +dish_1561661500 +dish_1561661552 +dish_1561661806 +dish_1561661850 +dish_1561661972 +dish_1561662022 +dish_1561662054 +dish_1561662358 +dish_1561662458 +dish_1561662501 +dish_1561662532 +dish_1561662562 +dish_1561662948 +dish_1561663035 +dish_1561663068 +dish_1561663786 +dish_1561663817 +dish_1561663855 +dish_1561663891 +dish_1561663914 +dish_1561663990 +dish_1561664013 +dish_1561664042 +dish_1561664061 +dish_1561664977 +dish_1561665024 +dish_1561665065 +dish_1561665378 +dish_1561665422 +dish_1561665452 +dish_1561665626 +dish_1561665671 +dish_1561665706 +dish_1561665747 +dish_1561666619 +dish_1561666640 +dish_1561666761 +dish_1561666794 +dish_1561666821 +dish_1561666849 +dish_1561666875 +dish_1561666897 +dish_1561666923 +dish_1561667430 +dish_1561736715 +dish_1561736736 +dish_1561737176 +dish_1561737200 +dish_1561737223 +dish_1561737248 +dish_1561737271 +dish_1561737293 +dish_1561737776 +dish_1561737839 +dish_1561739576 +dish_1561739623 +dish_1561739649 +dish_1561739719 +dish_1561739760 +dish_1561739805 +dish_1561739930 +dish_1561739954 +dish_1561739973 +dish_1561740321 +dish_1561740353 +dish_1561740385 +dish_1561740411 +dish_1561741139 +dish_1561741174 +dish_1561746876 +dish_1561746912 +dish_1561746970 +dish_1561747075 +dish_1561748274 +dish_1561748862 +dish_1561748950 +dish_1561749016 +dish_1561749061 +dish_1561749348 +dish_1561749392 +dish_1561749664 +dish_1561749711 +dish_1561749750 +dish_1561749778 +dish_1561749818 +dish_1561749857 +dish_1561749908 +dish_1561751897 +dish_1561751958 +dish_1561752035 +dish_1561752449 +dish_1561752481 +dish_1561752509 +dish_1561752535 +dish_1561752563 +dish_1561752680 +dish_1561752970 +dish_1561752996 +dish_1561753021 +dish_1561753075 +dish_1561753168 +dish_1561753203 +dish_1561753238 +dish_1561753275 +dish_1561753371 +dish_1561753400 +dish_1561753432 +dish_1561753477 +dish_1561753589 +dish_1561753623 +dish_1561753649 +dish_1561996084 +dish_1561996122 +dish_1561996369 +dish_1561996401 +dish_1561996428 +dish_1561996535 +dish_1561996554 +dish_1561996579 +dish_1561996612 +dish_1561996647 +dish_1561996750 +dish_1561996786 +dish_1561996813 +dish_1561996836 +dish_1561996859 +dish_1561996893 +dish_1561997365 +dish_1561997401 +dish_1561997900 +dish_1561997933 +dish_1561999287 +dish_1561999309 +dish_1561999332 +dish_1561999355 +dish_1562007739 +dish_1562007763 +dish_1562007789 +dish_1562007817 +dish_1562007842 +dish_1562007866 +dish_1562007904 +dish_1562008394 +dish_1562008429 +dish_1562008476 +dish_1562008517 +dish_1562008549 +dish_1562008695 +dish_1562008738 +dish_1562008780 +dish_1562008813 +dish_1562008842 +dish_1562009098 +dish_1562009120 +dish_1562009148 +dish_1562009182 +dish_1562009314 +dish_1562010048 +dish_1562010085 +dish_1562010130 +dish_1562010266 +dish_1562010293 +dish_1562010315 +dish_1562010369 +dish_1562010394 +dish_1562010418 +dish_1562010440 +dish_1562010821 +dish_1562010865 +dish_1562010891 +dish_1562010929 +dish_1562011727 +dish_1562011765 +dish_1562011794 +dish_1562012048 +dish_1562012076 +dish_1562085132 +dish_1562085163 +dish_1562085185 +dish_1562085604 +dish_1562086037 +dish_1562086067 +dish_1562086116 +dish_1562086501 +dish_1562086564 +dish_1562086595 +dish_1562086702 +dish_1562086740 +dish_1562094621 +dish_1562094694 +dish_1562094730 +dish_1562094877 +dish_1562094931 +dish_1562095283 +dish_1562095340 +dish_1562096343 +dish_1562096372 +dish_1562096401 +dish_1562096512 +dish_1562096542 +dish_1562096584 +dish_1562096608 +dish_1562096633 +dish_1562096907 +dish_1562096933 +dish_1562096973 +dish_1562097001 +dish_1562097624 +dish_1562097653 +dish_1562097673 +dish_1562168365 +dish_1562168399 +dish_1562168427 +dish_1562168450 +dish_1562168471 +dish_1562169729 +dish_1562169762 +dish_1562169818 +dish_1562169840 +dish_1562170163 +dish_1562170207 +dish_1562170236 +dish_1562170331 +dish_1562170356 +dish_1562173111 +dish_1562173175 +dish_1562173200 +dish_1562173247 +dish_1562173267 +dish_1562173381 +dish_1562173411 +dish_1562173742 +dish_1562179966 +dish_1562180025 +dish_1562180089 +dish_1562180157 +dish_1562181217 +dish_1562181247 +dish_1562181285 +dish_1562183065 +dish_1562183096 +dish_1562183125 +dish_1562183151 +dish_1562183175 +dish_1562183201 +dish_1562184604 +dish_1562184628 +dish_1562184663 +dish_1562184705 +dish_1562602589 +dish_1562602627 +dish_1562602657 +dish_1562602687 +dish_1562602803 +dish_1562602832 +dish_1562602860 +dish_1562602888 +dish_1562602918 +dish_1562602945 +dish_1562603536 +dish_1562603569 +dish_1562603598 +dish_1562603626 +dish_1562603813 +dish_1562603842 +dish_1562603867 +dish_1562603895 +dish_1562605315 +dish_1562605352 +dish_1562605384 +dish_1562605411 +dish_1562611522 +dish_1562611580 +dish_1562611620 +dish_1562611649 +dish_1562611680 +dish_1562611824 +dish_1562611853 +dish_1562611878 +dish_1562612974 +dish_1562613010 +dish_1562613045 +dish_1562613285 +dish_1562613340 +dish_1562613376 +dish_1562613403 +dish_1562613867 +dish_1562613898 +dish_1562613918 +dish_1562613974 +dish_1562614088 +dish_1562614121 +dish_1562614158 +dish_1562614895 +dish_1562614921 +dish_1562614952 +dish_1562614980 +dish_1562615070 +dish_1562615103 +dish_1562615128 +dish_1562615153 +dish_1562615283 +dish_1562615324 +dish_1562615360 +dish_1562615388 +dish_1562617370 +dish_1562617395 +dish_1562617416 +dish_1562617446 +dish_1562617470 +dish_1562617492 +dish_1562686492 +dish_1562686520 +dish_1562686548 +dish_1562686577 +dish_1562686664 +dish_1562687246 +dish_1562687274 +dish_1562687299 +dish_1562687527 +dish_1562687554 +dish_1562688382 +dish_1562688426 +dish_1562688482 +dish_1562688524 +dish_1562688552 +dish_1562689519 +dish_1562689548 +dish_1562689601 +dish_1562689620 +dish_1562689645 +dish_1562690047 +dish_1562690074 +dish_1562690112 +dish_1562690144 +dish_1562691383 +dish_1562691409 +dish_1562691441 +dish_1562691466 +dish_1562691538 +dish_1562691566 +dish_1562691613 +dish_1562691645 +dish_1562691674 +dish_1562691705 +dish_1562691737 +dish_1562698100 +dish_1562698132 +dish_1562698209 +dish_1562698239 +dish_1562698276 +dish_1562699570 +dish_1562699612 +dish_1562699652 +dish_1562699699 +dish_1562700685 +dish_1562700718 +dish_1562700760 +dish_1562700885 +dish_1562700906 +dish_1562700931 +dish_1562700957 +dish_1562701956 +dish_1562702007 +dish_1562702044 +dish_1562702091 +dish_1562702132 +dish_1562702355 +dish_1562702379 +dish_1562702412 +dish_1562702441 +dish_1562702469 +dish_1562702813 +dish_1562702853 +dish_1562702892 +dish_1562704528 +dish_1562704564 +dish_1562704616 +dish_1562704640 +dish_1562704665 +dish_1562774346 +dish_1562774456 +dish_1562774475 +dish_1562774504 +dish_1562774531 +dish_1562774980 +dish_1562775047 +dish_1562775110 +dish_1562775212 +dish_1562775241 +dish_1562775275 +dish_1562775356 +dish_1562775373 +dish_1562775456 +dish_1562775488 +dish_1562775510 +dish_1562775549 +dish_1562775574 +dish_1562777000 +dish_1562777037 +dish_1562777068 +dish_1562777099 +dish_1562777566 +dish_1562783605 +dish_1562783691 +dish_1562784242 +dish_1562784266 +dish_1562784307 +dish_1562784347 +dish_1562784388 +dish_1562785132 +dish_1562785159 +dish_1562785184 +dish_1562785328 +dish_1562785369 +dish_1562787745 +dish_1562787783 +dish_1562787817 +dish_1562787848 +dish_1562787956 +dish_1562787983 +dish_1562788010 +dish_1562788032 +dish_1562788732 +dish_1562788760 +dish_1562788794 +dish_1562788816 +dish_1562788845 +dish_1562789207 +dish_1562789240 +dish_1562789268 +dish_1562789301 +dish_1562789328 +dish_1562790170 +dish_1562790224 +dish_1562790259 +dish_1562790295 +dish_1562790321 +dish_1562859116 +dish_1562860045 +dish_1562860073 +dish_1562860700 +dish_1562860723 +dish_1562861103 +dish_1562861125 +dish_1562861174 +dish_1562862493 +dish_1562862529 +dish_1562862550 +dish_1562871297 +dish_1562871363 +dish_1562871408 +dish_1562871448 +dish_1562871488 +dish_1562871537 +dish_1562871731 +dish_1562872150 +dish_1562872182 +dish_1562872223 +dish_1562872365 +dish_1562872396 +dish_1562872432 +dish_1562872470 +dish_1562872498 +dish_1562872662 +dish_1562872746 +dish_1562872914 +dish_1562872945 +dish_1562872981 +dish_1562873002 +dish_1562873028 +dish_1562873155 +dish_1562873187 +dish_1562873224 +dish_1562873264 +dish_1562873416 +dish_1562873495 +dish_1562873544 +dish_1562873790 +dish_1562873814 +dish_1562873838 +dish_1562874049 +dish_1562874203 +dish_1562874250 +dish_1562874282 +dish_1562875159 +dish_1562875211 +dish_1562875241 +dish_1562875273 +dish_1562875320 +dish_1562875833 +dish_1562875870 +dish_1562875898 +dish_1562875940 +dish_1562876618 +dish_1562876645 +dish_1562876674 +dish_1562876709 +dish_1562876732 +dish_1562876756 +dish_1562945131 +dish_1562945186 +dish_1562945332 +dish_1562945393 +dish_1562945445 +dish_1562945499 +dish_1562945779 +dish_1562945803 +dish_1562945823 +dish_1562945851 +dish_1562945879 +dish_1562945900 +dish_1562945967 +dish_1562946006 +dish_1562946075 +dish_1562946106 +dish_1562947020 +dish_1562947376 +dish_1562947409 +dish_1562947436 +dish_1562947469 +dish_1562947503 +dish_1562947598 +dish_1562947885 +dish_1562949162 +dish_1562949205 +dish_1562949242 +dish_1562949467 +dish_1562949494 +dish_1562949515 +dish_1562949541 +dish_1562950018 +dish_1562950057 +dish_1562950091 +dish_1562950895 +dish_1562950934 +dish_1562956486 +dish_1562956510 +dish_1562958445 +dish_1562958563 +dish_1562958603 +dish_1562958629 +dish_1562958721 +dish_1562958775 +dish_1562958809 +dish_1562958998 +dish_1562959024 +dish_1562959059 +dish_1562959103 +dish_1562959129 +dish_1562959190 +dish_1562959295 +dish_1562959344 +dish_1562959379 +dish_1562959485 +dish_1562959517 +dish_1562959547 +dish_1562959570 +dish_1562959599 +dish_1562959700 +dish_1562959732 +dish_1562959766 +dish_1562959799 +dish_1562960437 +dish_1562960472 +dish_1562960520 +dish_1562961588 +dish_1562961609 +dish_1562961650 +dish_1562963006 +dish_1562963033 +dish_1562963059 +dish_1562963084 +dish_1562963109 +dish_1562963153 +dish_1562963484 +dish_1562963517 +dish_1562963583 +dish_1562963612 +dish_1562963648 +dish_1562963676 +dish_1562963704 +dish_1563205877 +dish_1563205903 +dish_1563205927 +dish_1563205953 +dish_1563205982 +dish_1563206409 +dish_1563206562 +dish_1563206582 +dish_1563206601 +dish_1563207277 +dish_1563207301 +dish_1563207332 +dish_1563207364 +dish_1563207389 +dish_1563207406 +dish_1563208094 +dish_1563208129 +dish_1563216381 +dish_1563216412 +dish_1563216440 +dish_1563216625 +dish_1563216659 +dish_1563216689 +dish_1563216717 +dish_1563216739 +dish_1563217736 +dish_1563217774 +dish_1563217804 +dish_1563217895 +dish_1563217939 +dish_1563217968 +dish_1563217993 +dish_1563218031 +dish_1563218201 +dish_1563218959 +dish_1563218987 +dish_1563219009 +dish_1563219035 +dish_1563219056 +dish_1563220058 +dish_1563220087 +dish_1563220109 +dish_1563220143 +dish_1563220171 +dish_1563220767 +dish_1563220802 +dish_1563221443 +dish_1563221495 +dish_1563221519 +dish_1563221562 +dish_1563221602 +dish_1563221640 +dish_1563222745 +dish_1563222775 +dish_1563222800 +dish_1563222830 +dish_1563222850 +dish_1563294408 +dish_1563294441 +dish_1563294465 +dish_1563294491 +dish_1563294522 +dish_1563294547 +dish_1563294575 +dish_1563295292 +dish_1563295334 +dish_1563295360 +dish_1563295400 +dish_1563302315 +dish_1563302345 +dish_1563302391 +dish_1563302420 +dish_1563302447 +dish_1563303298 +dish_1563303350 +dish_1563303386 +dish_1563304261 +dish_1563304301 +dish_1563304333 +dish_1563304369 +dish_1563304881 +dish_1563304925 +dish_1563304958 +dish_1563305005 +dish_1563305047 +dish_1563305083 +dish_1563305321 +dish_1563305354 +dish_1563305401 +dish_1563305653 +dish_1563305689 +dish_1563305722 +dish_1563305837 +dish_1563305883 +dish_1563305913 +dish_1563305948 +dish_1563305982 +dish_1563306039 +dish_1563306392 +dish_1563307488 +dish_1563307519 +dish_1563307575 +dish_1563308323 +dish_1563308353 +dish_1563308386 +dish_1563308428 +dish_1563309189 +dish_1563309237 +dish_1563309285 +dish_1563309315 +dish_1563309353 +dish_1563309398 +dish_1563378344 +dish_1563378372 +dish_1563378406 +dish_1563378435 +dish_1563378660 +dish_1563378690 +dish_1563379494 +dish_1563381461 +dish_1563381486 +dish_1563381512 +dish_1563381680 +dish_1563381706 +dish_1563381738 +dish_1563381760 +dish_1563382911 +dish_1563382953 +dish_1563382987 +dish_1563389042 +dish_1563389079 +dish_1563389114 +dish_1563389153 +dish_1563389199 +dish_1563389413 +dish_1563389446 +dish_1563389693 +dish_1563389719 +dish_1563389742 +dish_1563389786 +dish_1563391333 +dish_1563391375 +dish_1563391413 +dish_1563391453 +dish_1563391647 +dish_1563391681 +dish_1563391708 +dish_1563391922 +dish_1563391959 +dish_1563391989 +dish_1563392128 +dish_1563393148 +dish_1563393182 +dish_1563393217 +dish_1563393244 +dish_1563393341 +dish_1563393366 +dish_1563393403 +dish_1563395341 +dish_1563395374 +dish_1563395415 +dish_1563395449 +dish_1563395480 +dish_1563395510 +dish_1563464389 +dish_1563464429 +dish_1563464457 +dish_1563464480 +dish_1563464543 +dish_1563464627 +dish_1563464651 +dish_1563464670 +dish_1563465805 +dish_1563465847 +dish_1563466280 +dish_1563468199 +dish_1563468240 +dish_1563468269 +dish_1563468299 +dish_1563468327 +dish_1563468362 +dish_1563468776 +dish_1563475788 +dish_1563475826 +dish_1563475874 +dish_1563475921 +dish_1563475953 +dish_1563476267 +dish_1563476323 +dish_1563476408 +dish_1563476463 +dish_1563476551 +dish_1563476577 +dish_1563476646 +dish_1563477659 +dish_1563477701 +dish_1563477736 +dish_1563479526 +dish_1563479568 +dish_1563479596 +dish_1563479625 +dish_1563479655 +dish_1563480476 +dish_1563480505 +dish_1563480534 +dish_1563480608 +dish_1563481030 +dish_1563481061 +dish_1563550584 +dish_1563550614 +dish_1563550922 +dish_1563550950 +dish_1563550976 +dish_1563551006 +dish_1563551360 +dish_1563551397 +dish_1563551432 +dish_1563552016 +dish_1563553735 +dish_1563553756 +dish_1563553780 +dish_1563555072 +dish_1563555100 +dish_1563555126 +dish_1563555160 +dish_1563555510 +dish_1563555543 +dish_1563555563 +dish_1563555605 +dish_1563563240 +dish_1563563264 +dish_1563563289 +dish_1563563317 +dish_1563563348 +dish_1563563378 +dish_1563563467 +dish_1563563492 +dish_1563563527 +dish_1563564676 +dish_1563564707 +dish_1563564760 +dish_1563564797 +dish_1563564828 +dish_1563565683 +dish_1563565720 +dish_1563565761 +dish_1563565805 +dish_1563566626 +dish_1563566674 +dish_1563566725 +dish_1563566810 +dish_1563567319 +dish_1563567372 +dish_1563567599 +dish_1563567626 +dish_1563567663 +dish_1563567722 +dish_1563567978 +dish_1563568003 +dish_1563568025 +dish_1563568118 +dish_1563568148 +dish_1563568263 +dish_1563568292 +dish_1563568319 +dish_1563568338 +dish_1563568357 +dish_1563811412 +dish_1563811439 +dish_1563811463 +dish_1563811490 +dish_1563811515 +dish_1563811611 +dish_1563811639 +dish_1563811662 +dish_1563811686 +dish_1563812508 +dish_1563812535 +dish_1563812570 +dish_1563812615 +dish_1563813221 +dish_1563813308 +dish_1563813360 +dish_1563813395 +dish_1563813436 +dish_1563813501 +dish_1563813806 +dish_1563813833 +dish_1563820775 +dish_1563820806 +dish_1563820834 +dish_1563822546 +dish_1563822597 +dish_1563822636 +dish_1563822663 +dish_1563823292 +dish_1563823359 +dish_1563823390 +dish_1563823423 +dish_1563823814 +dish_1563823860 +dish_1563823894 +dish_1563824065 +dish_1563824092 +dish_1563824130 +dish_1563824155 +dish_1563824250 +dish_1563824279 +dish_1563824304 +dish_1563824339 +dish_1563824377 +dish_1563824406 +dish_1563826223 +dish_1563826272 +dish_1563826306 +dish_1563827060 +dish_1563827086 +dish_1563827110 +dish_1563827132 +dish_1563827162 +dish_1563827188 +dish_1563827425 +dish_1563827454 +dish_1563827482 +dish_1563897338 +dish_1563897369 +dish_1563897395 +dish_1563897439 +dish_1563897492 +dish_1563897794 +dish_1563897837 +dish_1563897870 +dish_1563898688 +dish_1563898722 +dish_1563898786 +dish_1563898813 +dish_1563898880 +dish_1563900120 +dish_1563900150 +dish_1563900172 +dish_1563900205 +dish_1563908923 +dish_1563908957 +dish_1563908996 +dish_1563909919 +dish_1563909951 +dish_1563909982 +dish_1563910106 +dish_1563910137 +dish_1563910171 +dish_1563910276 +dish_1563910530 +dish_1563910639 +dish_1563910676 +dish_1563910708 +dish_1563911156 +dish_1563911185 +dish_1563911215 +dish_1563911255 +dish_1563913588 +dish_1563913627 +dish_1563913675 +dish_1563913709 +dish_1563913741 +dish_1563984701 +dish_1563984729 +dish_1563984771 +dish_1563984835 +dish_1563986062 +dish_1563986093 +dish_1563996251 +dish_1563996279 +dish_1563996308 +dish_1563996596 +dish_1563996625 +dish_1563996656 +dish_1563997327 +dish_1563997386 +dish_1563997418 +dish_1564000202 +dish_1564000417 +dish_1564000459 +dish_1564000490 +dish_1564000536 +dish_1564073641 +dish_1564073677 +dish_1564073716 +dish_1564073860 +dish_1564082178 +dish_1564082212 +dish_1564082323 +dish_1564082462 +dish_1564083459 +dish_1564083480 +dish_1564083508 +dish_1564083534 +dish_1564086602 +dish_1564086624 +dish_1564086658 +dish_1564086709 +dish_1564086748 +dish_1564086778 +dish_1564159520 +dish_1564159543 +dish_1564159588 +dish_1564159636 +dish_1564160161 +dish_1564168695 +dish_1564168739 +dish_1564169197 +dish_1564169254 +dish_1564169298 +dish_1564169340 +dish_1564169644 +dish_1564169878 +dish_1564169932 +dish_1564169958 +dish_1564170010 +dish_1564170071 +dish_1564170098 +dish_1564170141 +dish_1564170173 +dish_1564170283 +dish_1564170474 +dish_1564170542 +dish_1564170836 +dish_1564170880 +dish_1564170923 +dish_1564170967 +dish_1564171888 +dish_1564171935 +dish_1564171970 +dish_1564171996 +dish_1564428404 +dish_1564428444 +dish_1564428475 +dish_1564428515 +dish_1564429184 +dish_1564429235 +dish_1564429270 +dish_1564429702 +dish_1564429744 +dish_1564429808 +dish_1564429843 +dish_1564429894 +dish_1564432094 +dish_1564432128 +dish_1564432165 +dish_1564432191 +dish_1564432238 +dish_1564502063 +dish_1564503184 +dish_1564503219 +dish_1564514906 +dish_1564514953 +dish_1564514997 +dish_1564515049 +dish_1564516366 +dish_1564516397 +dish_1564516429 +dish_1564516454 +dish_1564516483 +dish_1564516732 +dish_1564516760 +dish_1564518454 +dish_1564518488 +dish_1564518512 +dish_1564518542 +dish_1564518586 +dish_1564518850 +dish_1564588483 +dish_1564588533 +dish_1564588567 +dish_1564588688 +dish_1564588719 +dish_1564589025 +dish_1564589072 +dish_1564589308 +dish_1564589366 +dish_1564590115 +dish_1564590146 +dish_1564590190 +dish_1564590226 +dish_1564590254 +dish_1564590288 +dish_1564590352 +dish_1564601470 +dish_1564601528 +dish_1564601569 +dish_1564601600 +dish_1564601638 +dish_1564605293 +dish_1564605328 +dish_1564675060 +dish_1564675118 +dish_1564675157 +dish_1564675193 +dish_1564675240 +dish_1564685144 +dish_1564685183 +dish_1564685229 +dish_1564685360 +dish_1564685406 +dish_1564685450 +dish_1564685482 +dish_1564686657 +dish_1564686702 +dish_1564686740 +dish_1564686791 +dish_1564686832 +dish_1564686855 +dish_1564688969 +dish_1564690038 +dish_1564690085 +dish_1564690138 +dish_1564690167 +dish_1564690194 +dish_1564691380 +dish_1564691420 +dish_1564691450 +dish_1564761388 +dish_1564761426 +dish_1564761457 +dish_1564761488 +dish_1564774773 +dish_1564774805 +dish_1564775147 +dish_1564775182 +dish_1564775216 +dish_1564777933 +dish_1564777960 +dish_1564777986 +dish_1564778030 +dish_1564778057 +dish_1565021054 +dish_1565021104 +dish_1565021141 +dish_1565021185 +dish_1565021220 +dish_1565021599 +dish_1565021640 +dish_1565021675 +dish_1565021714 +dish_1565021965 +dish_1565022027 +dish_1565022062 +dish_1565023128 +dish_1565023157 +dish_1565023182 +dish_1565024675 +dish_1565024707 +dish_1565024741 +dish_1565031283 +dish_1565031338 +dish_1565031376 +dish_1565032680 +dish_1565032714 +dish_1565032748 +dish_1565033085 +dish_1565033110 +dish_1565033139 +dish_1565033189 +dish_1565033220 +dish_1565033265 +dish_1565034234 +dish_1565034281 +dish_1565034330 +dish_1565036788 +dish_1565036821 +dish_1565036865 +dish_1565036890 +dish_1565036922 +dish_1565036953 +dish_1565107066 +dish_1565107092 +dish_1565107119 +dish_1565107143 +dish_1565107171 +dish_1565107211 +dish_1565109606 +dish_1565109637 +dish_1565109659 +dish_1565117892 +dish_1565117934 +dish_1565117962 +dish_1565117990 +dish_1565118029 +dish_1565118067 +dish_1565118096 +dish_1565118999 +dish_1565119418 +dish_1565119439 +dish_1565119464 +dish_1565119493 +dish_1565119642 +dish_1565119669 +dish_1565119698 +dish_1565119739 +dish_1565194684 +dish_1565194752 +dish_1565194781 +dish_1565194823 +dish_1565194852 +dish_1565194906 +dish_1565195362 +dish_1565195398 +dish_1565195436 +dish_1565195493 +dish_1565196260 +dish_1565203015 +dish_1565203094 +dish_1565204855 +dish_1565204891 +dish_1565204931 +dish_1565207631 +dish_1565207662 +dish_1565207694 +dish_1565207727 +dish_1565207815 +dish_1565207865 +dish_1565207896 +dish_1565208562 +dish_1565208611 +dish_1565208637 +dish_1565209922 +dish_1565209963 +dish_1565210009 +dish_1565210037 +dish_1565210073 +dish_1565369938 +dish_1565369971 +dish_1565370004 +dish_1565378680 +dish_1565378731 +dish_1565378770 +dish_1565379251 +dish_1565379320 +dish_1565379377 +dish_1565379788 +dish_1565379827 +dish_1565379868 +dish_1565382733 +dish_1565382762 +dish_1565382803 +dish_1565382833 +dish_1565382856 +dish_1565382886 +dish_1565382969 +dish_1565382995 +dish_1565383026 +dish_1565383062 +dish_1565383097 +dish_1565383128 +dish_1565383159 +dish_1565627751 +dish_1565627800 +dish_1565637123 +dish_1565637162 +dish_1565637504 +dish_1565637548 +dish_1565637578 +dish_1565637608 +dish_1565638287 +dish_1565638323 +dish_1565638365 +dish_1565640195 +dish_1565640227 +dish_1565640272 +dish_1565640365 +dish_1565640427 +dish_1565640528 +dish_1565640549 +dish_1565640576 +dish_1565640601 +dish_1565640723 +dish_1565640772 +dish_1565640812 +dish_1565640842 +dish_1565640868 +dish_1565640894 +dish_1565640914 +dish_1565641844 +dish_1565641885 +dish_1565641915 +dish_1565641943 +dish_1565711083 +dish_1565711115 +dish_1565711154 +dish_1565711183 +dish_1565711270 +dish_1565711296 +dish_1565711326 +dish_1565711358 +dish_1565721800 +dish_1565721828 +dish_1565721855 +dish_1565724050 +dish_1565724087 +dish_1565724196 +dish_1565724235 +dish_1565724269 +dish_1565724509 +dish_1565724552 +dish_1565724588 +dish_1565724620 +dish_1565725009 +dish_1565725047 +dish_1565725071 +dish_1565725095 +dish_1565725123 +dish_1565725156 +dish_1565725188 +dish_1565725244 +dish_1565726812 +dish_1565726848 +dish_1565726887 +dish_1565726915 +dish_1565726949 +dish_1565727738 +dish_1565727765 +dish_1565727791 +dish_1565727821 +dish_1565727851 +dish_1565727875 +dish_1565727921 +dish_1565728388 +dish_1565728415 +dish_1565728453 +dish_1565728494 +dish_1565728562 +dish_1565728591 +dish_1565728634 +dish_1565728684 +dish_1565797645 +dish_1565797671 +dish_1565797699 +dish_1565797723 +dish_1565798531 +dish_1565798558 +dish_1565798591 +dish_1565798617 +dish_1565799507 +dish_1565799565 +dish_1565799597 +dish_1565799617 +dish_1565799663 +dish_1565808943 +dish_1565808975 +dish_1565808998 +dish_1565809033 +dish_1565809080 +dish_1565809120 +dish_1565810146 +dish_1565810205 +dish_1565810237 +dish_1565811287 +dish_1565811335 +dish_1565811390 +dish_1565811423 +dish_1565811529 +dish_1565811588 +dish_1565812116 +dish_1565812146 +dish_1565812186 +dish_1565813386 +dish_1565813414 +dish_1565813446 +dish_1565884692 +dish_1565884738 +dish_1565884766 +dish_1565884833 +dish_1565895808 +dish_1565895879 +dish_1565895966 +dish_1565896039 +dish_1565897569 +dish_1565897617 +dish_1565897680 +dish_1565898489 +dish_1565898511 +dish_1565898544 +dish_1565898590 +dish_1565898659 +dish_1565898723 +dish_1565898758 +dish_1565972363 +dish_1565972384 +dish_1565972429 +dish_1565972591 +dish_1565972628 +dish_1565972674 +dish_1565972715 +dish_1565981769 +dish_1565981802 +dish_1565981848 +dish_1565981898 +dish_1565984245 +dish_1565984299 +dish_1565984345 +dish_1565984434 +dish_1565987015 +dish_1565987054 +dish_1565987081 +dish_1565987116 +dish_1565987142 +dish_1565987201 +dish_1565987269 +dish_1565987298 +dish_1566230222 +dish_1566230258 +dish_1566230300 +dish_1566230377 +dish_1566242066 +dish_1566242111 +dish_1566242164 +dish_1566242195 +dish_1566242234 +dish_1566242273 +dish_1566242298 +dish_1566244815 +dish_1566244863 +dish_1566244904 +dish_1566244964 +dish_1566245365 +dish_1566245398 +dish_1566245437 +dish_1566246902 +dish_1566246946 +dish_1566246972 +dish_1566247003 +dish_1566316496 +dish_1566316525 +dish_1566316558 +dish_1566316587 +dish_1566316612 +dish_1566317711 +dish_1566317742 +dish_1566317775 +dish_1566317801 +dish_1566328894 +dish_1566328929 +dish_1566328961 +dish_1566328988 +dish_1566329021 +dish_1566329049 +dish_1566329203 +dish_1566329234 +dish_1566329257 +dish_1566330399 +dish_1566330428 +dish_1566330466 +dish_1566331801 +dish_1566331828 +dish_1566331868 +dish_1566331914 +dish_1566332040 +dish_1566332085 +dish_1566332116 +dish_1566332154 +dish_1566332193 +dish_1566332268 +dish_1566333118 +dish_1566333167 +dish_1566333194 +dish_1566333228 +dish_1566333255 +dish_1566333291 +dish_1566333324 +dish_1566402041 +dish_1566402095 +dish_1566402123 +dish_1566402160 +dish_1566402197 +dish_1566402232 +dish_1566413321 +dish_1566413356 +dish_1566413388 +dish_1566413419 +dish_1566413445 +dish_1566413469 +dish_1566413510 +dish_1566414611 +dish_1566414642 +dish_1566414686 +dish_1566414723 +dish_1566414754 +dish_1566414780 +dish_1566414862 +dish_1566414891 +dish_1566414924 +dish_1566415520 +dish_1566415558 +dish_1566415591 +dish_1566416299 +dish_1566416339 +dish_1566416388 +dish_1566416421 +dish_1566416455 +dish_1566417398 +dish_1566417442 +dish_1566417475 +dish_1566417516 +dish_1566419510 +dish_1566419546 +dish_1566419577 +dish_1566419603 +dish_1566419635 +dish_1566488905 +dish_1566488928 +dish_1566488956 +dish_1566488981 +dish_1566499564 +dish_1566499633 +dish_1566499686 +dish_1566500963 +dish_1566500988 +dish_1566501012 +dish_1566501041 +dish_1566501067 +dish_1566501126 +dish_1566501153 +dish_1566501258 +dish_1566501293 +dish_1566501324 +dish_1566501352 +dish_1566501382 +dish_1566501447 +dish_1566501473 +dish_1566502000 +dish_1566502033 +dish_1566502061 +dish_1566502211 +dish_1566502271 +dish_1566502315 +dish_1566587662 +dish_1566587689 +dish_1566587714 +dish_1566587958 +dish_1566588016 +dish_1566588066 +dish_1566590536 +dish_1566590566 +dish_1566590946 +dish_1566590987 +dish_1566591022 +dish_1566591048 +dish_1566591080 +dish_1566592456 +dish_1566835603 +dish_1566835641 +dish_1566835701 +dish_1566844766 +dish_1566844803 +dish_1566844850 +dish_1566846997 +dish_1566847043 +dish_1566847431 +dish_1566847454 +dish_1566847483 +dish_1566849208 +dish_1566849233 +dish_1566849269 +dish_1566849295 +dish_1566849327 +dish_1566849387 +dish_1566851115 +dish_1566851155 +dish_1566851198 +dish_1566851222 +dish_1566922978 +dish_1566923004 +dish_1566923030 +dish_1566923060 +dish_1566931116 +dish_1566931140 +dish_1566931179 +dish_1566931217 +dish_1566931482 +dish_1566931519 +dish_1566931548 +dish_1566931586 +dish_1566931620 +dish_1566931650 +dish_1566931674 +dish_1566933180 +dish_1566933226 +dish_1566933401 +dish_1566933454 +dish_1566933498 +dish_1566934057 +dish_1566934085 +dish_1566934120 +dish_1566934152 +dish_1566934366 +dish_1566934401 +dish_1566934430 +dish_1566934455 +dish_1566934487 +dish_1566936266 +dish_1566936297 +dish_1566936323 +dish_1566936352 +dish_1566937992 +dish_1566938026 +dish_1566938059 +dish_1566938088 +dish_1567019993 +dish_1567020054 +dish_1567020091 +dish_1567020340 +dish_1567020366 +dish_1567020394 +dish_1567020417 +dish_1567021799 +dish_1567021831 +dish_1567021864 +dish_1567022114 +dish_1567022147 +dish_1567022180 +dish_1567022212 +dish_1567022251 +dish_1567022296 +dish_1567022338 +dish_1567022383 +dish_1567023110 +dish_1567023132 +dish_1567023161 +dish_1567023184 +dish_1567023262 +dish_1567023296 +dish_1567023335 +dish_1567023375 +dish_1567024066 +dish_1567024112 +dish_1567024135 +dish_1567024187 +dish_1567024224 +dish_1567024252 +dish_1567106594 +dish_1567106620 +dish_1567106646 +dish_1567106668 +dish_1567106693 +dish_1567106794 +dish_1567106825 +dish_1567106870 +dish_1567107049 +dish_1567107989 +dish_1567108037 +dish_1567110519 +dish_1567110556 +dish_1567110631 +dish_1567110665 +dish_1567110756 +dish_1567110783 +dish_1567110811 +dish_1567110848 +dish_1567525637 +dish_1567525681 +dish_1567525712 +dish_1567525744 +dish_1567526738 +dish_1567526785 +dish_1567526814 +dish_1567526845 +dish_1567526888 +dish_1567537185 +dish_1567537212 +dish_1567537252 +dish_1567537280 +dish_1567537352 +dish_1567537387 +dish_1567537422 +dish_1567537457 +dish_1567538263 +dish_1567538303 +dish_1567542778 +dish_1567542813 +dish_1567542843 +dish_1567542876 +dish_1567542914 +dish_1567542953 +dish_1567542986 +dish_1567612954 +dish_1567612986 +dish_1567613019 +dish_1567613057 +dish_1567625294 +dish_1567625343 +dish_1567625392 +dish_1567625434 +dish_1567628069 +dish_1567628099 +dish_1567628157 +dish_1567628193 +dish_1567629081 +dish_1567629106 +dish_1567629129 +dish_1567711272 +dish_1567711356 +dish_1567711402 +dish_1567714745 +dish_1567714780 +dish_1567714813 +dish_1567714839 +dish_1567714875 +dish_1567714934 +dish_1567785130 +dish_1567785165 +dish_1567785223 +dish_1567789154 +dish_1567789190 +dish_1567789230 +dish_1567789256 +dish_1567798511 +dish_1567801468 +dish_1567801520 +dish_1567801716 +dish_1567801748 +dish_1567801791 +dish_1567801830 +dish_1568044496 +dish_1568044528 +dish_1568044581 +dish_1568044634 +dish_1568047932 +dish_1568047973 +dish_1568048018 +dish_1568048053 +dish_1568055963 +dish_1568055986 +dish_1568056019 +dish_1568056046 +dish_1568056217 +dish_1568056258 +dish_1568060414 +dish_1568060461 +dish_1568060501 +dish_1568060542 +dish_1568060616 +dish_1568060646 +dish_1568060697 +dish_1568060722 +dish_1568060746 +dish_1568061097 +dish_1568061145 +dish_1568061174 +dish_1568061202 +dish_1568061234 +dish_1568061272 +dish_1568061320 +dish_1568132484 +dish_1568132521 +dish_1568132561 +dish_1568132601 +dish_1568141871 +dish_1568141915 +dish_1568141951 +dish_1568143252 +dish_1568143288 +dish_1568143326 +dish_1568143362 +dish_1568143527 +dish_1568143572 +dish_1568143612 +dish_1568144612 +dish_1568144643 +dish_1568144688 +dish_1568144774 +dish_1568144803 +dish_1568144828 +dish_1568144855 +dish_1568144879 +dish_1568144925 +dish_1568144954 +dish_1568145027 +dish_1568145059 +dish_1568145100 +dish_1568145136 +dish_1568145177 +dish_1568217267 +dish_1568217325 +dish_1568217372 +dish_1568217417 +dish_1568217465 +dish_1568229399 +dish_1568229442 +dish_1568233403 +dish_1568233444 +dish_1568233478 +dish_1568233509 +dish_1568313487 +dish_1568313527 +dish_1568313575 +dish_1568313640 +dish_1568313676 +dish_1568314823 +dish_1568314857 +dish_1568314907 +dish_1568314951 +dish_1568315001 +dish_1568315040 +dish_1568315088 +dish_1568315132 +dish_1568315177 +dish_1568315988 +dish_1568316041 +dish_1568316085 +dish_1568318286 +dish_1568318316 +dish_1568318361 +dish_1568318408 +dish_1568318480 +dish_1568319725 +dish_1568319769 +dish_1568319838 +dish_1568319899 +dish_1568320296 +dish_1568320338 +dish_1568320383 +dish_1568320427 +dish_1568320470 +dish_1568400935 +dish_1568400967 +dish_1568401018 +dish_1568401064 +dish_1568401090 +dish_1568404850 +dish_1568404889 +dish_1568405183 +dish_1568405214 +dish_1568405273 +dish_1568405306 +dish_1568405405 +dish_1568405430 +dish_1568405466 +dish_1568405499 +dish_1568649312 +dish_1568649354 +dish_1568649387 +dish_1568649430 +dish_1568660810 +dish_1568664931 +dish_1568664972 +dish_1568665012 +dish_1568665052 +dish_1568665935 +dish_1568665970 +dish_1568666000 +dish_1568666032 +dish_1568666061 +dish_1568666089 +dish_1568666184 +dish_1568666230 +dish_1568666263 +dish_1568666303 +dish_1568666329 +dish_1568666357 diff --git a/report/runs.txt b/report/runs.txt new file mode 100644 index 0000000..9eca80a --- /dev/null +++ b/report/runs.txt @@ -0,0 +1,255 @@ +Experiment 1 +Multi-Modal CNN–LSTM + +Training val and loss: +overhead_depth_20251130_230108 + + +Objective: Establish a baseline model using a two-stream fusion architecture that processes overhead RGB and depth as separate modalities, rather than concatenating depth as a 4th channel + +Results: A summary of baseline performance compared to the Nutrition5k reference models is provided in Appendix 5. Our fusion model performs competitively on calories and substantially improves over the “Depth as 4th Channel” approach on mass prediction. Macronutrient estimates lag behind the Nutrition5k portion-independent models, consistent with the increased difficulty of joint end-to-end optimization. + +Interpretation: The two-stream fusion architecture achieved competitive results on calorie pre- +diction (49.9 vs. 47.6 MAE) and substantially outperformed the “Depth as 4th Channel” approach on mass prediction (33.0 vs. 40.7 MAE), approaching the more complex “Volume Scalar” pipeline (29.4 MAE). This suggests that treating RGB and depth as separate modalities with dedicated en-coders allows the network to learn portion-relevant features more effectively than naive channel concatenation. The fusion layer can capture complementary geometric and appearance cues, similar 6 to how the Volume Scalar method benefits from explicit geometric reasoning about depth. However, macronutrient predictions (fat, carbs, protein) lag behind the paper’s results by approximately 2×. In the Nutrition5k paper, the strongest macronutrient models were portion-independent density predictors trained separately and multiplied by predicted mass. Our end-to-end approach must jointly learn portion estimation and density prediction, making optimization more difficult. + +Experiment 2 +First run +overhead_depth_side_20251201_091912 + +Overhead_depth_side_20251130_132911 + + Training Curve Analysis + + Side + Overhead + Depth (100 epochs) + + | Epoch | Train Loss | Val Loss | Gap | + |-------|------------|----------|-------------------| + | 1 | 256 | 203 | -53 (val better!) | + | 5 | 140 | 238 | +98 | + | 10 | 104 | 223 | +119 | + | 25 | 82 | 203 | +121 | + | 50 | 80 | 199 | +119 | + | 100 | 80 | 165 | +85 | + + Overhead + Depth Only (100 epochs) + + | Epoch | Train Loss | Val Loss | Gap | + |-------|------------|----------|-------------------| + | 1 | 265 | 174 | -91 (val better!) | + | 5 | 144 | 136 | -8 | + | 10 | 106 | 117 | +11 | + | 25 | 87 | 109 | +22 | + | 50 | 80 | 104 | +24 | + | 100 | 82 | 106 | +24 | + + --- + Key Observations + + 1. Massive overfitting with side frames: The val loss never improves after epoch 1 for the side model - it starts at 203 and stays around 170-240 for 100 epochs. Meanwhile train loss drops from 256 → 80. + 2. Overhead-only generalizes well: Val loss steadily decreases from 174 → 106, tracking train loss reasonably. + 3. Strange epoch 1 behavior: Both models have validation loss better than train loss at epoch 1. This is unusual but can happen with dropout/augmentation being on during training but off during validation. + 4. Side model plateaus quickly: Train loss for side model hits ~80 by epoch 25 and stays flat. The model has effectively memorized the training set but learned nothing generalizable. + + --- + Diagnosis: The Side Frame Branch is Learning Noise + + The side frames are providing the model a "shortcut" to memorize training dishes rather than learning generalizable visual features about food. This could be because: + + 1. Too much capacity: 8 frames × EfficientNet + LSTM = lots of parameters to memorize with + 2. Frame-specific memorization: The model might be recognizing specific frames/angles rather than food content + 3. Inconsistent frame quality: If side frames vary significantly between dishes, they add noise + + --- + Questions to narrow down the fix: + + 1. Have you tried side-frames ONLY (no overhead/depth)? This would tell us if side frames are inherently problematic or if it's the combination. + 2. What happens if you freeze the side encoder? Keep EfficientNet weights frozen and only train the LSTM + fusion layers. + 3. What if you use fewer frames? Try --max-frames 1 or --max-frames 4 to reduce capacity. + +Experiment 3 +Proper side sampling +overhead_depth_side_20251201_004538 + + +6. Side Frame Sampling Strategy + + Initial Implementation + + - Original approach: Alphabetically sorted, take first N frames + - Problem: Frame names like camera_A_frame_001.jpeg sort by camera first, resulting in all frames from camera A, then B, etc. + - Result: Only seeing 1-2 camera angles instead of 360° coverage + + Camera Diversity Fix (First Iteration) + + - Solution: Round-robin interleaving across 4 cameras + - Result: Ensures all 4 camera angles represented + - Example (max_frames=8): A1, B1, C1, D1, A2, B2, C2, D2 + + Temporal Diversity Fix (Second Iteration) + + - Problem: Round-robin still takes consecutive frames from each camera (frames 1, 2, 3...) + - Issue: Camera is rotating, so consecutive frames are similar + - Final Solution: Uniform sampling within each camera's timeline + frames_per_camera = max_side_frames // num_cameras # e.g., 8 // 4 = 2 + indices = [int(i * num_available / frames_per_camera) for i in range(frames_per_camera)] + - Result: Both camera diversity AND temporal diversity + - 4 cameras: 90° viewing angle differences + - Evenly spaced frames: Maximum temporal coverage per camera + - Example: If 7 frames per camera, takes frames at indices [0, 3] → early + late frames + + + +Validation Loss Comparison (First 10 Epochs) + + | Epoch | Old Side Sampling | New Side Sampling | Improvement | + |-------|-------------------|-------------------|-------------| + | 1 | 203 | 192 | -5% | + | 2 | 206 | 182 | -12% | + | 3 | 237 | 191 | -19% | + | 4 | 221 | 223 | +1% | + | 5 | 238 | 160 | -33% | + | 6 | 207 | 161 | -22% + | 7 | 202 | 143 | -29% | + | 8 | 189 | 142 | -25% + | 9 | 174 | 137 | -21% | + | 10 | 223 | 135 | -39% | + + Key Observations + + 1. Much better trajectory: Val loss is actually decreasing consistently now (192 → 135) instead of bouncing around (203 → 223) + 2. Less variance: The old sampling had wild swings (174 → 223 between epochs 9-10). New sampling is smoother. + 3. Already competitive: At epoch 10, new side sampling (135) is approaching the overhead-only model's epoch 10 performance (117). + 4. No severe overfitting yet: The old model had val loss increasing after epoch 1. This one is trending down. + + --- + Next Steps + + Let this run continue for another 15-25 epochs to see: + 1. Does val loss continue to decrease or plateau? + 2. Does it eventually beat the overhead-only baseline (~105-109)? + 3. Does overfitting eventually kick in? + + + +Hmm, the improvement has stalled. Let me analyze: + + Analysis + + | Metric | Value | + |------------------------|------------------------------| + | Best val loss | 134.5 (epoch 17) | + | Current val loss | 142.3 (epoch 28) | + | Trend after epoch 10 | Plateaued with high variance | + | Overhead-only baseline | ~105-109 | + + The Problem + + The model hit a floor around 134-137 and is now oscillating (134 → 184 → 136 → 169 → 140...). This suggests: + + 1. Still overfitting - the high variance indicates the model isn't learning stable generalizable features + 2. Better than before - old sampling never got below 165, so the frame diversity helped + 3. But not enough - still ~25-30% worse than overhead-only + + Recommendations + + Now is a good time to try the encoder freezing approach. The side frames are providing some signal, but the encoder is likely still memorizing training-specific patterns. + + Try this run: + + PYTHONPATH=. python src/train.py \ + --use-side-frames \ + --use-overhead \ + --use-depth \ + --epochs 50 \ + --batch-size 8 \ + --lr 1e-4 \ + --max-frames 8 \ + --freeze-encoders \ + --unfreeze-epoch 20 \ + --encoder-lr-multiplier 0.1 + + Rationale: + - Freeze encoders for first 20 epochs → forces LSTM + fusion layers to learn good aggregation + - Unfreeze with 10× lower encoder LR → gentle fine-tuning without destroying pretrained features + - If val loss is still improving at epoch 20, the unfreezing might push it past the 105 barrier + + Alternatively, you could also try side-frames only (no overhead/depth) to see if the side frames alone can match overhead performance - that would tell us if the fusion is the problem vs. the side features themselves. + + + + +Experiment 4 +Architecture improvements. freeze feature layers in early epochs, discriminative learning rates + +overhead_depth_side_20251201_220514 + + Encoder freezing | Overfitting, unstable training +More stable convergence, better generalization +7. Optimization +Gradient Clipping +Value: Max norm of 1.0 +Rationale: Prevent exploding gradients, especially important with LSTM +Code location: src/train.py:720, 733 +Optimizer & Learning Rate +Optimizer: Adam with weight_decay=1e-4 +Initial LR: 1e-4 (configurable via --lr) +Schedule: Decay by 0.5 every 5 epochs +Rationale: Standard choices for vision models, aggressive decay for fine-tuning +Code location: src/train.py:641, 856-858 + +Experiment 5 +overhead_depth_side_20251203_131326 + Problem: The dataset has uneven modality coverage: + - 3,930 dishes have side frames + - 2,755 dishes have overhead RGB + - ~2,758 dishes have depth + - Only 2,638 dishes have all three modalities + + Previously, training required all requested modalities to be present, discarding ~1,300 dishes with partial data. + +2. Why We Did It + + | Motivation | Explanation | + |------------------------------|--------------------------------------------------------------------------------------------------------| + | More training data | 4,047 dishes instead of 2,638 (+53% more data) | + | Side encoder was overfitting | Previous runs showed training loss decreasing but validation loss plateauing; more data should help | + | Utilize full dataset | The Nutrition5k dataset has dishes with varying sensor availability; we were throwing away useful data | + | Implicit regularization | Randomly encountering missing modalities during training acts like dropout at the modality level | + + --- + 3. How It Works + + When a modality is missing for a dish: + + Forward pass with all modalities: + side_frames → side_encoder → [1280-dim] ─┐ + overhead_rgb → overhead_encoder → [1280-dim] ─┼─→ concat → fusion → predictions + overhead_depth → depth_encoder → [1280-dim] ─┘ + + Forward pass with missing overhead: + side_frames → side_encoder → [1280-dim] ─┐ + missing_overhead_embed (learned) → [1280-dim] ─┼─→ concat → fusion → predictions + missing_depth_embed (learned) → [1280-dim] ─┘ + + The learned embeddings are initialized as small random vectors (torch.randn(...) * 0.02) and trained alongside the model. They learn to represent "no data available for this modality." + + --- + 4. Expected Results + + | Aspect | Expected Impact | + |-------------------------|------------------------------------------------------------------------------------------------------| + | Reduced overfitting | +53% more training data should help the model generalize better | + | Better side encoder | Gets 3,930 training examples instead of 2,638 | + | Comparable test results | Test set is identical (same 709 dishes), so results are directly comparable to previous runs | + | Learned embeddings | Model learns what "missing" means; may learn to rely more on available modalities when one is absent | + | Training dynamics | Batches will have mixed modality availability, forcing the model to be robust | + + 6. Data Breakdown + + | Subset | Train | Test | + |----------------------|-------|------| + | All three modalities | 2,638 | 474 | + | Side frames only | 1,292 | 202 | + | Overhead only | 117 | 33 | + | Total | 4,047 | 709 | + diff --git a/requirements.txt b/requirements.txt index aea9bca..1e4e996 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,7 @@ google-cloud-storage>=2.10.0 # Visualization matplotlib>=3.7.0 seaborn>=0.12.0 +tensorboard>=2.14.0 # Utilities click>=8.1.0 @@ -48,4 +49,11 @@ pyflakes>=3.0.0 # Additional utilities six>=1.16.0 -tqdm>=4.65.0 # Progress bars for training \ No newline at end of file +tqdm>=4.65.0 # Progress bars for training + +# Experiment tracking +wandb>=0.16.0 + +# Config management +hydra-core>=1.3.0 +omegaconf>=2.3.0 \ No newline at end of file diff --git a/runs/report_charts.txt b/runs/report_charts.txt new file mode 100644 index 0000000..910391d --- /dev/null +++ b/runs/report_charts.txt @@ -0,0 +1,50 @@ + +Experiment 1: +overhead_depth_20251130_230108 +python ../scripts/plot_training_runs.py --labels "Baseline (overhead+depth)" "Multi-modal (side view+overhead+depth)" --runs overhead_depth_20251130_230108_loss_val.csv overhead_depth_side_20251201_004538_loss_val.csv --ylabel "Valuation Loss" --max-steps 30 --output valuation_loss_exp1.png +python ../scripts/plot_training_runs.py --labels "Baseline (overhead+depth)" "Multi-modal (side view+overhead+depth)" --runs overhead_depth_20251130_230108_loss_train.csv overhead_depth_side_20251201_004538_loss_train.csv --ylabel "Training Loss" --max-steps 30 --output training_loss_exp1.png + +Experiment 2: +first attempt at all modalities +overhead_depth_side_20251201_091912 +overhead_depth_side_20251201_091912_loss_train.csv +overhead_depth_side_20251201_091912_loss_val.csv + +Experiment 3: Data improvements (sampling) +overhead_depth_side_20251201_004538 +overhead_depth_side_20251201_004538_loss_train.csv +overhead_depth_side_20251201_004538_loss_val.csv + + +Experiment 4: Architecture changes (layer freezing, variable lr) +overhead_depth_side_20251201_220514 +overhead_depth_side_20251201_220514_loss_train.csv +overhead_depth_side_20251201_220514_loss_val.csv + + +Experiment 5: Missing modalities +overhead_depth_side_20251203_131326 +overhead_depth_side_20251203_131326_loss_train.csv +overhead_depth_side_20251203_131326_loss_val.csv + + +python ../scripts/plot_training_runs.py \ +--labels "Experiment 1" "Experiment 2" "Experiment 3" "Experiment 4" "Experiment 5" \ +--runs overhead_depth_20251130_230108_loss_train.csv overhead_depth_side_20251201_091912_loss_train.csv overhead_depth_side_20251201_004538_loss_train.csv overhead_depth_side_20251201_220514_loss_train.csv overhead_depth_side_20251203_131326_loss_train.csv \ +--smoothing 0.6 --ylabel "MAE" --max-steps 30 --output training_loss_exp5.png + +python ../scripts/plot_training_runs.py \ +--labels "Experiment 1" "Experiment 2" "Experiment 3" "Experiment 4" "Experiment 5" \ +--runs overhead_depth_20251130_230108_loss_val.csv overhead_depth_side_20251201_091912_loss_val.csv overhead_depth_side_20251201_004538_loss_val.csv overhead_depth_side_20251201_220514_loss_val.csv overhead_depth_side_20251203_131326_loss_val.csv \ +--smoothing 0.6 --ylabel "MAE" --max-steps 30 --output valuation_loss_exp5.png + + +python ../scripts/plot_training_runs.py \ +--labels "Experiment 1" "Experiment 2" "Experiment 3" "Experiment 4" \ +--runs overhead_depth_20251130_230108_loss_train.csv overhead_depth_side_20251201_004538_loss_train.csv overhead_depth_side_20251201_220514_loss_train.csv overhead_depth_side_20251203_131326_loss_train.csv \ +--smoothing 0.6 --ylabel "MAE" --max-steps 30 --output training_loss_exp5.png + +python ../scripts/plot_training_runs.py \ +--labels "Experiment 1" "Experiment 2" "Experiment 3" "Experiment 4" \ +--runs overhead_depth_20251130_230108_loss_val.csv overhead_depth_side_20251201_004538_loss_val.csv overhead_depth_side_20251201_220514_loss_val.csv overhead_depth_side_20251203_131326_loss_val.csv \ +--smoothing 0.6 --ylabel "MAE" --max-steps 30 --output valuation_loss_exp5.png \ No newline at end of file diff --git a/scripts/run_experiments.py b/scripts/run_experiments.py new file mode 100644 index 0000000..b931697 --- /dev/null +++ b/scripts/run_experiments.py @@ -0,0 +1,399 @@ +#!/usr/bin/env python3 +""" +Experiment runner for DeepDiet training. + +Runs multiple experiments sequentially or in parallel, with automatic +logging to W&B and optional Slack/email notifications. + +Usage: + # Run all defined experiments + python scripts/run_experiments.py + + # Run specific experiments by name + python scripts/run_experiments.py --experiments attention_baseline lstm_baseline + + # Dry run (print commands without executing) + python scripts/run_experiments.py --dry-run + + # Run with custom config file + python scripts/run_experiments.py --config experiments.yaml +""" + +import argparse +import subprocess +import sys +import time +import yaml +from pathlib import Path +from datetime import datetime +from typing import Dict, List, Optional +import json + +REPO = Path(__file__).resolve().parents[1] + +# Default experiments to run +DEFAULT_EXPERIMENTS = { + # Aggregation method comparison + "lstm_baseline": { + "description": "LSTM aggregation baseline", + "args": { + "use_side_frames": True, + "use_overhead": True, + "use_depth": True, + "side_aggregation": "lstm", + "epochs": 20, + "batch_size": 4, + "lr": 1e-4, + }, + "tags": ["baseline", "lstm"], + }, + "attention_baseline": { + "description": "Temporal attention aggregation", + "args": { + "use_side_frames": True, + "use_overhead": True, + "use_depth": True, + "side_aggregation": "attention", + "epochs": 20, + "batch_size": 4, + "lr": 1e-4, + }, + "tags": ["baseline", "attention"], + }, + "mean_baseline": { + "description": "Mean pooling aggregation", + "args": { + "use_side_frames": True, + "use_overhead": True, + "use_depth": True, + "side_aggregation": "mean", + "epochs": 20, + "batch_size": 4, + "lr": 1e-4, + }, + "tags": ["baseline", "mean"], + }, + + # Learning rate sweep for attention + "attention_lr_high": { + "description": "Attention with higher LR", + "args": { + "use_side_frames": True, + "use_overhead": True, + "use_depth": True, + "side_aggregation": "attention", + "epochs": 20, + "lr": 5e-4, + }, + "tags": ["lr_sweep", "attention"], + }, + "attention_lr_low": { + "description": "Attention with lower LR", + "args": { + "use_side_frames": True, + "use_overhead": True, + "use_depth": True, + "side_aggregation": "attention", + "epochs": 20, + "lr": 5e-5, + }, + "tags": ["lr_sweep", "attention"], + }, + + # Frozen encoder experiments + "attention_frozen": { + "description": "Attention with frozen encoders", + "args": { + "use_side_frames": True, + "use_overhead": True, + "use_depth": True, + "side_aggregation": "attention", + "epochs": 20, + "freeze_encoders": True, + "unfreeze_epoch": 10, + }, + "tags": ["frozen", "attention"], + }, + + # Single modality ablations + "side_only_attention": { + "description": "Side frames only with attention", + "args": { + "use_side_frames": True, + "use_overhead": False, + "use_depth": False, + "side_aggregation": "attention", + "epochs": 20, + }, + "tags": ["ablation", "side_only"], + }, + "overhead_only": { + "description": "Overhead RGB + depth only", + "args": { + "use_side_frames": False, + "use_overhead": True, + "use_depth": True, + "epochs": 20, + }, + "tags": ["ablation", "overhead_only"], + }, +} + + +def build_command(experiment_name: str, config: Dict, wandb_project: str = "deepdiet") -> List[str]: + """Build the training command from experiment config.""" + cmd = [sys.executable, "-m", "src.train"] + + args = config.get("args", {}) + + # Boolean flags + if args.get("use_side_frames"): + cmd.append("--use-side-frames") + if args.get("use_overhead"): + cmd.append("--use-overhead") + if args.get("use_depth"): + cmd.append("--use-depth") + if args.get("freeze_encoders"): + cmd.append("--freeze-encoders") + if args.get("allow_missing_modalities"): + cmd.append("--allow-missing-modalities") + + # Value arguments + if "side_aggregation" in args: + cmd.extend(["--side-aggregation", args["side_aggregation"]]) + if "epochs" in args: + cmd.extend(["--epochs", str(args["epochs"])]) + if "batch_size" in args: + cmd.extend(["--batch-size", str(args["batch_size"])]) + if "lr" in args: + cmd.extend(["--lr", str(args["lr"])]) + if "max_frames" in args: + cmd.extend(["--max-frames", str(args["max_frames"])]) + if "image_size" in args: + cmd.extend(["--image-size", str(args["image_size"])]) + if "unfreeze_epoch" in args: + cmd.extend(["--unfreeze-epoch", str(args["unfreeze_epoch"])]) + if "encoder_lr_multiplier" in args: + cmd.extend(["--encoder-lr-multiplier", str(args["encoder_lr_multiplier"])]) + + # W&B integration + cmd.append("--wandb") + cmd.extend(["--wandb-project", wandb_project]) + cmd.extend(["--wandb-run-name", experiment_name]) + + # Tags + tags = config.get("tags", []) + if tags: + cmd.append("--wandb-tags") + cmd.extend(tags) + + return cmd + + +def run_experiment( + name: str, + config: Dict, + wandb_project: str, + dry_run: bool = False, + log_dir: Optional[Path] = None, +) -> Dict: + """Run a single experiment and return results.""" + cmd = build_command(name, config, wandb_project) + cmd_str = " ".join(cmd) + + result = { + "name": name, + "description": config.get("description", ""), + "command": cmd_str, + "started_at": datetime.now().isoformat(), + "status": "pending", + } + + print(f"\n{'='*70}") + print(f"Experiment: {name}") + print(f"Description: {config.get('description', 'N/A')}") + print(f"Command: {cmd_str}") + print(f"{'='*70}") + + if dry_run: + result["status"] = "dry_run" + print("[DRY RUN] Would execute the above command") + return result + + # Set up log file + log_file = None + if log_dir: + log_file = log_dir / f"{name}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.log" + print(f"Log file: {log_file}") + + try: + start_time = time.time() + + if log_file: + with open(log_file, "w") as f: + process = subprocess.run( + cmd, + cwd=REPO, + stdout=f, + stderr=subprocess.STDOUT, + text=True, + ) + else: + process = subprocess.run( + cmd, + cwd=REPO, + text=True, + ) + + elapsed = time.time() - start_time + result["elapsed_seconds"] = elapsed + result["elapsed_human"] = f"{elapsed/60:.1f} minutes" + result["return_code"] = process.returncode + result["finished_at"] = datetime.now().isoformat() + + if process.returncode == 0: + result["status"] = "success" + print(f"\n✓ Experiment '{name}' completed successfully in {elapsed/60:.1f} minutes") + else: + result["status"] = "failed" + print(f"\n✗ Experiment '{name}' failed with return code {process.returncode}") + + except KeyboardInterrupt: + result["status"] = "interrupted" + print(f"\n! Experiment '{name}' interrupted by user") + raise + + except Exception as e: + result["status"] = "error" + result["error"] = str(e) + print(f"\n✗ Experiment '{name}' error: {e}") + + return result + + +def load_experiments(config_file: Optional[Path]) -> Dict: + """Load experiments from config file or use defaults.""" + if config_file and config_file.exists(): + with open(config_file) as f: + return yaml.safe_load(f) + return DEFAULT_EXPERIMENTS + + +def main(): + parser = argparse.ArgumentParser(description="Run DeepDiet experiments") + parser.add_argument( + "--experiments", + nargs="*", + help="Specific experiments to run (default: all)", + ) + parser.add_argument( + "--config", + type=Path, + help="YAML config file with experiment definitions", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Print commands without executing", + ) + parser.add_argument( + "--wandb-project", + default="deepdiet", + help="W&B project name (default: deepdiet)", + ) + parser.add_argument( + "--log-dir", + type=Path, + default=REPO / "experiment_logs", + help="Directory for experiment logs", + ) + parser.add_argument( + "--list", + action="store_true", + help="List available experiments and exit", + ) + args = parser.parse_args() + + # Load experiments + experiments = load_experiments(args.config) + + # List experiments and exit + if args.list: + print("\nAvailable experiments:") + print("-" * 70) + for name, config in experiments.items(): + desc = config.get("description", "No description") + tags = ", ".join(config.get("tags", [])) + print(f" {name:<25} {desc}") + if tags: + print(f" {'':25} Tags: {tags}") + return + + # Filter experiments if specific ones requested + if args.experiments: + experiments = {k: v for k, v in experiments.items() if k in args.experiments} + if not experiments: + print(f"Error: No matching experiments found for: {args.experiments}") + print("Use --list to see available experiments") + sys.exit(1) + + # Create log directory + if not args.dry_run: + args.log_dir.mkdir(parents=True, exist_ok=True) + + print(f"\n{'#'*70}") + print(f"# DeepDiet Experiment Runner") + print(f"# Running {len(experiments)} experiment(s)") + print(f"# W&B Project: {args.wandb_project}") + print(f"# Log Directory: {args.log_dir}") + print(f"{'#'*70}") + + results = [] + start_time = time.time() + + try: + for name, config in experiments.items(): + result = run_experiment( + name=name, + config=config, + wandb_project=args.wandb_project, + dry_run=args.dry_run, + log_dir=args.log_dir if not args.dry_run else None, + ) + results.append(result) + + except KeyboardInterrupt: + print("\n\nExperiment run interrupted by user") + + finally: + # Summary + total_time = time.time() - start_time + print(f"\n{'='*70}") + print("EXPERIMENT SUMMARY") + print(f"{'='*70}") + print(f"Total time: {total_time/60:.1f} minutes") + print(f"\nResults:") + + for r in results: + status_icon = { + "success": "✓", + "failed": "✗", + "interrupted": "!", + "error": "✗", + "dry_run": "○", + "pending": "?", + }.get(r["status"], "?") + + elapsed = r.get("elapsed_human", "N/A") + print(f" {status_icon} {r['name']:<25} {r['status']:<12} {elapsed}") + + # Save results + if not args.dry_run and results: + results_file = args.log_dir / f"results_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json" + with open(results_file, "w") as f: + json.dump(results, f, indent=2) + print(f"\nResults saved to: {results_file}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/src/config.py b/src/config.py index 170783f..3a38654 100644 --- a/src/config.py +++ b/src/config.py @@ -28,6 +28,9 @@ class TrainingConfig: # Side frame aggregation side_aggregation: str = 'lstm' # 'lstm' or 'mean' + # Missing modality handling + allow_missing_modalities: bool = False # Allow training with partial modalities + # Inputs use_side_frames: bool = False use_overhead: bool = True @@ -89,6 +92,9 @@ def create_config(args, repo_root: Path) -> TrainingConfig: # Side frame aggregation side_aggregation=getattr(args, 'side_aggregation', 'lstm'), + # Missing modality handling + allow_missing_modalities=getattr(args, 'allow_missing_modalities', False), + # Inputs use_side_frames=args.use_side_frames, use_overhead=args.use_overhead, diff --git a/src/dataset.py b/src/dataset.py index 607b198..90ec7d8 100644 --- a/src/dataset.py +++ b/src/dataset.py @@ -8,6 +8,51 @@ TARGETS = ["cal", "mass", "fat", "carb", "protein"] +def multimodal_collate_fn(batch): + """ + Custom collate function for batches with potentially missing modalities. + + When allow_missing_modalities=True, different samples may have different keys. + This collate function handles that by only including keys that all samples have, + or by padding missing modalities with None values. + """ + # Get all keys present in all samples + all_keys = set(batch[0].keys()) + + # Collect values for each key + result = {} + + # Always stack targets (required for all samples) + result['targets'] = torch.stack([item['targets'] for item in batch]) + + # Collect dish_ids as list + result['dish_id'] = [item['dish_id'] for item in batch] + + # For optional modalities, check if any sample has them + for key in ['side_frames', 'overhead_rgb', 'overhead_depth']: + values = [item.get(key) for item in batch] + if any(v is not None for v in values): + # If any sample has this modality, stack the ones that have it + # and use zeros for those that don't + if all(v is not None for v in values): + result[key] = torch.stack(values) + else: + # Mixed batch - need to handle missing values + # Get the shape from a non-None value + template = next(v for v in values if v is not None) + stacked = [] + for v in values: + if v is not None: + stacked.append(v) + else: + stacked.append(torch.zeros_like(template)) + result[key] = torch.stack(stacked) + # Track which samples have this modality + result[f'{key}_mask'] = torch.tensor([v is not None for v in values]) + + return result + + class PadToSquare: """Pad image to square by adding black bars to shorter dimension.""" def __call__(self, img): @@ -21,7 +66,8 @@ def __call__(self, img): class MultiViewDataset(Dataset): def __init__(self, split_file, data_root, train=True, max_side_frames=16, - use_side_frames=True, use_overhead=False, use_depth=False, image_size=256): + use_side_frames=True, use_overhead=False, use_depth=False, image_size=256, + allow_missing_modalities=False): self.data_root = Path(data_root) self.train = train self.max_side_frames = max_side_frames @@ -29,6 +75,7 @@ def __init__(self, split_file, data_root, train=True, max_side_frames=16, self.use_overhead = use_overhead self.use_depth = use_depth self.image_size = image_size + self.allow_missing_modalities = allow_missing_modalities split = pd.read_csv(split_file) all_dish_ids = split['dish_id'].tolist() @@ -90,43 +137,81 @@ def pick(*cands): self.metadata = metadata valid_dish_ids = [] + self.dish_modalities = {} # Track which modalities each dish has for dish_id in all_dish_ids: if dish_id not in self.metadata.index: continue - is_valid = True + # Check which modalities are available for this dish + has_side = False + has_overhead = False + has_depth = False if self.use_side_frames: side_dir = self.data_root / "imagery" / "side_angles" / dish_id / "frames_sampled5" - if not side_dir.is_dir() or not list(side_dir.glob("*.jpeg")): - is_valid = False + has_side = side_dir.is_dir() and bool(list(side_dir.glob("*.jpeg"))) if self.use_overhead: overhead_path = self.data_root / "imagery" / "realsense_overhead" / dish_id / "rgb.png" - if not overhead_path.is_file() or overhead_path.stat().st_size == 0: - is_valid = False + has_overhead = overhead_path.is_file() and overhead_path.stat().st_size > 0 if self.use_depth: depth_path = self.data_root / "imagery" / "realsense_overhead" / dish_id / "depth_raw.png" - if not depth_path.is_file() or depth_path.stat().st_size == 0: + has_depth = depth_path.is_file() and depth_path.stat().st_size > 0 + + if self.allow_missing_modalities: + # Accept dish if it has at least one requested modality + has_any = ( + (self.use_side_frames and has_side) or + (self.use_overhead and has_overhead) or + (self.use_depth and has_depth) + ) + if has_any: + valid_dish_ids.append(dish_id) + self.dish_modalities[dish_id] = { + 'side': has_side, + 'overhead': has_overhead, + 'depth': has_depth + } + else: + # Require all requested modalities + is_valid = True + if self.use_side_frames and not has_side: + is_valid = False + if self.use_overhead and not has_overhead: + is_valid = False + if self.use_depth and not has_depth: is_valid = False - if is_valid: - valid_dish_ids.append(dish_id) + if is_valid: + valid_dish_ids.append(dish_id) + self.dish_modalities[dish_id] = { + 'side': has_side, + 'overhead': has_overhead, + 'depth': has_depth + } self.dish_ids = valid_dish_ids - print(f"Loaded {len(self.dish_ids)} dishes from dataset.") + + # Print statistics + if self.allow_missing_modalities: + side_count = sum(1 for d in self.dish_modalities.values() if d['side']) + overhead_count = sum(1 for d in self.dish_modalities.values() if d['overhead']) + depth_count = sum(1 for d in self.dish_modalities.values() if d['depth']) + print(f"Loaded {len(self.dish_ids)} dishes (side: {side_count}, overhead: {overhead_count}, depth: {depth_count})") + else: + print(f"Loaded {len(self.dish_ids)} dishes from dataset.") # Calculate resize dimensions (slightly larger for cropping) resize_size = int(image_size * 1.125) # 12.5% larger for crop - # Side frames: pad to square first to preserve full 16:9 content - # 1920x1080 -> 1920x1920 (black bars top/bottom) -> 224x224 + # Side frames: Resize + CenterCrop (same as overhead) + # Food is centered, and rotating cameras provide redundancy for edge content if train: self.side_transform = transforms.Compose([ - PadToSquare(), # Pad to square, preserving all content - transforms.Resize((image_size, image_size)), # Resize to target + transforms.Resize(resize_size), + transforms.CenterCrop(image_size), transforms.RandomHorizontalFlip(), transforms.ColorJitter(brightness=0.2, contrast=0.2, saturation=0.2), transforms.ToTensor(), @@ -135,8 +220,8 @@ def pick(*cands): ]) else: self.side_transform = transforms.Compose([ - PadToSquare(), # Pad to square, preserving all content - transforms.Resize((image_size, image_size)), # Resize to target + transforms.Resize(resize_size), + transforms.CenterCrop(image_size), transforms.ToTensor(), transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), @@ -187,13 +272,10 @@ def _load_side_frames(self, dish_id): indices = [int(i * n / frames_per_camera) for i in range(frames_per_camera)] sampled_per_camera[camera] = [available[i] for i in indices] - # Interleave cameras: [A0, B0, C0, D0, A1, B1, C1, D1, ...] + # Sequential by camera: [A0, A1, B0, B1, C0, C1, D0, D1, ...] selected_frames = [] - max_len = max(len(v) for v in sampled_per_camera.values()) - for i in range(max_len): - for camera in cameras: - if i < len(sampled_per_camera.get(camera, [])): - selected_frames.append(sampled_per_camera[camera][i]) + for camera in cameras: + selected_frames.extend(sampled_per_camera.get(camera, [])) return selected_frames[:self.max_side_frames] @@ -205,21 +287,36 @@ def __getitem__(self, idx): row = self.metadata.loc[dish_id] targets = torch.tensor([row[t] for t in TARGETS], dtype=torch.float32) - if self.use_overhead: + # Check which modalities this dish actually has + if self.allow_missing_modalities: + modalities = self.dish_modalities.get(dish_id, {}) + has_side = modalities.get('side', False) + has_overhead = modalities.get('overhead', False) + has_depth = modalities.get('depth', False) + else: + # If not allowing missing, assume all requested modalities exist + has_side = self.use_side_frames + has_overhead = self.use_overhead + has_depth = self.use_depth + + # Load overhead RGB if requested and available + if self.use_overhead and has_overhead: overhead_path = self.data_root / "imagery" / "realsense_overhead" / dish_id / "rgb.png" overhead_img = Image.open(overhead_path).convert("RGB") overhead_img = self.overhead_transform(overhead_img) else: overhead_img = None - if self.use_depth: + # Load depth if requested and available + if self.use_depth and has_depth: depth_path = self.data_root / "imagery" / "realsense_overhead" / dish_id / "depth_raw.png" depth_img = Image.open(depth_path).convert("L") depth_img = self.depth_transform(depth_img) else: depth_img = None - if self.use_side_frames: + # Load side frames if requested and available + if self.use_side_frames and has_side: frame_paths = self._load_side_frames(dish_id) side_frames = [] diff --git a/src/model.py b/src/model.py index 1f13d90..5f14ad6 100644 --- a/src/model.py +++ b/src/model.py @@ -23,33 +23,62 @@ class DeepDietModel(nn.Module): """ - def __init__(self, use_side_frames=True, use_overhead=True, use_depth=True, chunk_size=4, lstm_hidden=640, side_aggregation='lstm'): + def __init__(self, use_side_frames=True, use_overhead=True, use_depth=True, chunk_size=4, lstm_hidden=640, side_aggregation='lstm', allow_missing_modalities=False): super().__init__() self.use_side_frames = use_side_frames self.use_overhead = use_overhead self.use_depth = use_depth and use_overhead self.chunk_size = chunk_size self.lstm_hidden = lstm_hidden - self.side_aggregation = side_aggregation # 'lstm' or 'mean' + self.side_aggregation = side_aggregation # 'lstm', 'attention', or 'mean' + self.allow_missing_modalities = allow_missing_modalities total_features = 0 + # Side frames encoder if self.use_side_frames: self.side_encoder = models.efficientnet_b0(weights=models.EfficientNet_B0_Weights.IMAGENET1K_V1) self.side_encoder.classifier = nn.Identity() if self.side_aggregation == 'lstm': self.side_aggregator = nn.LSTM(1280, self.lstm_hidden, batch_first=True, bidirectional=True) - total_features += self.lstm_hidden * 2 + side_feat_dim = self.lstm_hidden * 2 + elif self.side_aggregation == 'attention': + # Temporal self-attention over frames + encoder_layer = nn.TransformerEncoderLayer( + d_model=1280, + nhead=8, + dim_feedforward=2048, + dropout=0.1, + activation='gelu', + batch_first=True + ) + self.side_aggregator = nn.TransformerEncoder(encoder_layer, num_layers=2) + # Learnable [CLS] token for aggregation + self.side_cls_token = nn.Parameter(torch.randn(1, 1, 1280) * 0.02) + # Learnable positional embeddings (max 17 positions: 1 CLS + 16 frames) + self.side_pos_embed = nn.Parameter(torch.randn(1, 17, 1280) * 0.02) + side_feat_dim = 1280 else: # Mean pooling: output is raw EfficientNet features (1280) - total_features += 1280 + side_feat_dim = 1280 + total_features += side_feat_dim + # Learned embedding for missing side frames + if self.allow_missing_modalities: + self.missing_side_embed = nn.Parameter(torch.randn(side_feat_dim) * 0.02) + + # Overhead RGB encoder if self.use_overhead: self.overhead_encoder = models.efficientnet_b0(weights=models.EfficientNet_B0_Weights.IMAGENET1K_V1) self.overhead_encoder.classifier = nn.Identity() total_features += 1280 + # Learned embedding for missing overhead RGB + if self.allow_missing_modalities: + self.missing_overhead_embed = nn.Parameter(torch.randn(1280) * 0.02) + + # Depth encoder if self.use_depth: self.depth_encoder = models.efficientnet_b0(weights=models.EfficientNet_B0_Weights.IMAGENET1K_V1) # Since pretrained weights are based on 3 channels, convert this to 1 channel by averaging the RGB channels. @@ -58,6 +87,10 @@ def __init__(self, use_side_frames=True, use_overhead=True, use_depth=True, chun self.depth_encoder.classifier = nn.Identity() total_features += 1280 + # Learned embedding for missing depth + if self.allow_missing_modalities: + self.missing_depth_embed = nn.Parameter(torch.randn(1280) * 0.02) + self.fusion_layers = nn.Sequential( nn.Linear(total_features, 1024), nn.ReLU(), @@ -150,40 +183,81 @@ def get_param_groups(self, base_lr, encoder_lr_multiplier=0.1): return param_groups + def _get_batch_size(self, side_frames, overhead_rgb, overhead_depth): + """Get batch size from whichever input is available.""" + if side_frames is not None: + return side_frames.size(0) + elif overhead_rgb is not None: + return overhead_rgb.size(0) + elif overhead_depth is not None: + return overhead_depth.size(0) + else: + raise ValueError("At least one modality must be provided") + def forward(self, side_frames, overhead_rgb=None, overhead_depth=None): features = [] + batch_size = self._get_batch_size(side_frames, overhead_rgb, overhead_depth) + device = next(self.parameters()).device - if self.use_side_frames and side_frames is not None: - batch_size = side_frames.size(0) - num_frames = side_frames.size(1) - - # Flatten: [B, T, C, H, W] to [B * T, C, H, W] - side_frames_flat = side_frames.view(batch_size * num_frames, *side_frames.shape[2:]) - side_features = [] - - for i in range(0, side_frames_flat.size(0), self.chunk_size): - chunk = side_frames_flat[i:i + self.chunk_size] - chunk_feat = self.side_encoder(chunk) - side_features.append(chunk_feat) - - # Reshape: [B * T, 1280] -> [B, T, 1280] - side_features = torch.cat(side_features, dim=0).view(batch_size, num_frames, -1) - - # Aggregate frame features - if self.side_aggregation == 'lstm': - side_feat, _ = self.side_aggregator(side_features) - side_feat = side_feat.mean(dim=1) - else: - # Simple mean pooling over frames - side_feat = side_features.mean(dim=1) - - features.append(side_feat) - - if self.use_overhead and overhead_rgb is not None: - features.append(self.overhead_encoder(overhead_rgb)) - - if self.use_depth and overhead_depth is not None: - features.append(self.depth_encoder(overhead_depth)) + # Side frames processing + if self.use_side_frames: + if side_frames is not None: + num_frames = side_frames.size(1) + + # Flatten: [B, T, C, H, W] to [B * T, C, H, W] + side_frames_flat = side_frames.view(batch_size * num_frames, *side_frames.shape[2:]) + side_features = [] + + for i in range(0, side_frames_flat.size(0), self.chunk_size): + chunk = side_frames_flat[i:i + self.chunk_size] + chunk_feat = self.side_encoder(chunk) + side_features.append(chunk_feat) + + # Reshape: [B * T, 1280] -> [B, T, 1280] + side_features = torch.cat(side_features, dim=0).view(batch_size, num_frames, -1) + + # Aggregate frame features + if self.side_aggregation == 'lstm': + side_feat, _ = self.side_aggregator(side_features) + side_feat = side_feat.mean(dim=1) + elif self.side_aggregation == 'attention': + # Prepend [CLS] token to frame features + cls_tokens = self.side_cls_token.expand(batch_size, -1, -1) + side_features_with_cls = torch.cat([cls_tokens, side_features], dim=1) + # Add positional embeddings (slice to match sequence length: 1 CLS + num_frames) + seq_len = side_features_with_cls.size(1) + side_features_with_cls = side_features_with_cls + self.side_pos_embed[:, :seq_len, :] + # Apply transformer encoder + attended = self.side_aggregator(side_features_with_cls) + # Extract [CLS] token as aggregated representation + side_feat = attended[:, 0, :] + else: + # Simple mean pooling over frames + side_feat = side_features.mean(dim=1) + + features.append(side_feat) + elif self.allow_missing_modalities: + # Use learned embedding for missing side frames + side_feat = self.missing_side_embed.unsqueeze(0).expand(batch_size, -1) + features.append(side_feat) + + # Overhead RGB processing + if self.use_overhead: + if overhead_rgb is not None: + features.append(self.overhead_encoder(overhead_rgb)) + elif self.allow_missing_modalities: + # Use learned embedding for missing overhead RGB + overhead_feat = self.missing_overhead_embed.unsqueeze(0).expand(batch_size, -1) + features.append(overhead_feat) + + # Depth processing + if self.use_depth: + if overhead_depth is not None: + features.append(self.depth_encoder(overhead_depth)) + elif self.allow_missing_modalities: + # Use learned embedding for missing depth + depth_feat = self.missing_depth_embed.unsqueeze(0).expand(batch_size, -1) + features.append(depth_feat) concat_features = torch.cat(features, dim=1) fused_features = self.fusion_layers(concat_features) diff --git a/src/train.py b/src/train.py index 87fed4e..5d36a28 100644 --- a/src/train.py +++ b/src/train.py @@ -11,7 +11,7 @@ from src.training.epoch import train_one_epoch from src.model import DeepDietModel -from src.dataset import MultiViewDataset, TARGETS +from src.dataset import MultiViewDataset, TARGETS, multimodal_collate_fn from src.config import TrainingConfig, create_config from pathlib import Path @@ -24,6 +24,15 @@ import argparse import time import numpy as np +import os + +# Weights & Biases for experiment tracking +try: + import wandb + WANDB_AVAILABLE = True +except ImportError: + WANDB_AVAILABLE = False + print("Warning: wandb not installed. Run 'pip install wandb' for experiment tracking.") from src.metrics import ( compute_per_layer_step_sizes, compute_activation_stats, @@ -63,8 +72,8 @@ def main(): help='Maximum number of side angle frames per dish (default: 16)') parser.add_argument('--chunk-size', type=int, default=4, help='Number of frames to process at once through encoder (default: 4, lower = less memory)') - parser.add_argument('--lstm-hidden', type=int, default=384, - help='LSTM hidden size (default: 384)') + parser.add_argument('--lstm-hidden', type=int, default=640, + help='LSTM hidden size (default: 640)') parser.add_argument('--image-size', type=int, default=256, help='Image size for training (default: 256, use 128 or 192 for faster training)') parser.add_argument('--resume', type=str, default=None, @@ -79,8 +88,18 @@ def main(): help='Epoch to unfreeze encoders (default: 10)') parser.add_argument('--encoder-lr-multiplier', type=float, default=0.1, help='LR multiplier for encoders when unfrozen (default: 0.1)') - parser.add_argument('--side-aggregation', type=str, default='lstm', choices=['lstm', 'mean'], - help='Side frame aggregation method: lstm or mean (default: lstm)') + parser.add_argument('--side-aggregation', type=str, default='lstm', choices=['lstm', 'attention', 'mean'], + help='Side frame aggregation method: lstm, attention, or mean (default: lstm)') + parser.add_argument('--allow-missing-modalities', action='store_true', + help='Allow training with partial modalities (uses learned embeddings for missing inputs)') + parser.add_argument('--wandb', action='store_true', + help='Enable Weights & Biases logging (default: disabled)') + parser.add_argument('--wandb-project', type=str, default='deepdiet', + help='W&B project name (default: deepdiet)') + parser.add_argument('--wandb-run-name', type=str, default=None, + help='W&B run name (default: auto-generated)') + parser.add_argument('--wandb-tags', type=str, nargs='*', default=[], + help='W&B tags for this run (e.g., --wandb-tags baseline attention)') args = parser.parse_args() config = create_config(args, REPO) @@ -91,12 +110,55 @@ def main(): # Create TensorBoard logger timestamp = time.strftime('%Y%m%d_%H%M%S') enabled_inputs = config.get_input_channels() - run_name = f"{'_'.join(enabled_inputs)}_{timestamp}" + run_name = f"{'_'.join(enabled_inputs)}_{args.side_aggregation}_{timestamp}" log_dir = REPO / "runs" / run_name writer = SummaryWriter(log_dir) print(f"TensorBoard logs: {log_dir}") print(f"Run: tensorboard --logdir runs/") + # Initialize Weights & Biases if enabled + use_wandb = args.wandb and WANDB_AVAILABLE + if args.wandb and not WANDB_AVAILABLE: + print("Warning: --wandb flag set but wandb not installed. Skipping W&B logging.") + + if use_wandb: + wandb_run_name = args.wandb_run_name or run_name + wandb_config = { + # Model architecture + "side_aggregation": args.side_aggregation, + "use_side_frames": args.use_side_frames, + "use_overhead": args.use_overhead, + "use_depth": args.use_depth, + "lstm_hidden": args.lstm_hidden, + "max_frames": args.max_frames, + "image_size": args.image_size, + # Training params + "epochs": args.epochs, + "batch_size": args.batch_size, + "learning_rate": args.lr, + "freeze_encoders": args.freeze_encoders, + "unfreeze_epoch": args.unfreeze_epoch, + "encoder_lr_multiplier": args.encoder_lr_multiplier, + "allow_missing_modalities": args.allow_missing_modalities, + # Environment + "device": str(device), + } + + # Auto-generate tags based on config + auto_tags = list(enabled_inputs) + [args.side_aggregation] + if args.freeze_encoders: + auto_tags.append("frozen_encoders") + all_tags = auto_tags + args.wandb_tags + + wandb.init( + project=args.wandb_project, + name=wandb_run_name, + config=wandb_config, + tags=all_tags, + sync_tensorboard=True, # Also sync TensorBoard logs + ) + print(f"W&B run: {wandb.run.url}") + if not enabled_inputs: print("ERROR: At least one input type must be enabled!") return @@ -109,6 +171,8 @@ def main(): # Create datasets print("\nLoading datasets...") + if config.allow_missing_modalities: + print("Allow missing modalities: ON (using learned embeddings for missing inputs)") train_ds = MultiViewDataset( split_file=train_csv, data_root=config.data_root, @@ -117,7 +181,8 @@ def main(): use_side_frames=config.use_side_frames, use_overhead=config.use_overhead, use_depth=config.use_depth, - image_size=config.image_size + image_size=config.image_size, + allow_missing_modalities=config.allow_missing_modalities ) val_ds = MultiViewDataset( split_file=test_csv, @@ -127,14 +192,19 @@ def main(): use_side_frames=config.use_side_frames, use_overhead=config.use_overhead, use_depth=config.use_depth, - image_size=config.image_size + image_size=config.image_size, + allow_missing_modalities=config.allow_missing_modalities ) # Use 2 workers for data loading + # Use custom collate function when allowing missing modalities + collate_fn = multimodal_collate_fn if config.allow_missing_modalities else None train_dl = DataLoader(train_ds, batch_size=config.batch_size, shuffle=True, - num_workers=config.num_workers, pin_memory=(device.type == 'cuda')) + num_workers=config.num_workers, pin_memory=(device.type == 'cuda'), + collate_fn=collate_fn) val_dl = DataLoader(val_ds, batch_size=config.batch_size, shuffle=False, - num_workers=config.num_workers, pin_memory=(device.type == 'cuda')) + num_workers=config.num_workers, pin_memory=(device.type == 'cuda'), + collate_fn=collate_fn) # Compute mean target values for percentage calculations (matching paper methodology) print("\nComputing dataset statistics...") @@ -153,6 +223,7 @@ def main(): chunk_size=config.chunk_size, lstm_hidden=config.lstm_hidden, side_aggregation=config.side_aggregation, + allow_missing_modalities=config.allow_missing_modalities, ).to(device) # Load checkpoint if resuming @@ -338,6 +409,26 @@ def main(): # Flush TensorBoard writer to disk writer.flush() + + # Log to W&B (in addition to TensorBoard sync) + if use_wandb: + wandb_metrics = { + "epoch": epoch, + "train/loss": train_total_loss, + "val/loss": val_total_loss, + "time/epoch": epoch_time, + "time/data_load": data_load_time, + "time/forward": forward_time, + "time/backward": backward_time, + "best_val_loss": best_val_loss, + } + for task in TARGETS: + wandb_metrics[f"train/mae_{task}"] = train_losses[task] + wandb_metrics[f"val/mae_{task}"] = val_losses[task] + wandb_metrics[f"train/mae_pct_{task}"] = train_losses[task] / target_means[task] * 100 + wandb_metrics[f"val/mae_pct_{task}"] = val_losses[task] / target_means[task] * 100 + wandb.log(wandb_metrics, step=epoch) + print(f" [DEBUG] Logged {len(TARGETS)*4 + 6} metrics to TensorBoard for epoch {epoch}") # Compute additional metrics (do this every few epochs to minimize overhead) @@ -454,12 +545,22 @@ def main(): print(f"Best model saved at: {REPO / 'indexes' / model_name}") print("=" * 70) writer.close() + if use_wandb: + wandb.finish(exit_code=1) return print("=" * 70) print(f"Training complete! Best validation MAE: {best_val_loss:.3f}") writer.close() + # Finish W&B run and log final summary + if use_wandb: + wandb.summary["best_val_loss"] = best_val_loss + wandb.summary["final_train_loss"] = train_total_loss + for task in TARGETS: + wandb.summary[f"best_val_mae_{task}"] = val_losses[task] + wandb.finish() + if __name__ == "__main__": main() diff --git a/src/train_hydra.py b/src/train_hydra.py new file mode 100644 index 0000000..ce40690 --- /dev/null +++ b/src/train_hydra.py @@ -0,0 +1,349 @@ +#!/usr/bin/env python3 +""" +Hydra-based training script for DeepDiet. + +This script uses Hydra for configuration management, enabling: +- YAML-based config files with overrides +- Automatic experiment tracking with W&B +- Easy hyperparameter sweeps + +Usage: + # Default training + python src/train_hydra.py + + # Use attention model + python src/train_hydra.py model=attention + + # Override specific values + python src/train_hydra.py model.side_aggregation=attention training.lr=5e-5 + + # Fast debug run + python src/train_hydra.py training=fast data=small logging=local + + # Multirun sweep + python src/train_hydra.py --multirun model.side_aggregation=lstm,attention,mean training.lr=1e-4,5e-5 +""" +import warnings + +warnings.filterwarnings('ignore', category=UserWarning, module='torchvision.io') +warnings.filterwarnings('ignore', message='.*Failed to load image Python extension.*') + +import hydra +from omegaconf import DictConfig, OmegaConf +from pathlib import Path +import torch +import torch.nn as nn +import torch.optim as optim +from torch.utils.data import DataLoader +from torch.utils.tensorboard import SummaryWriter +import math +import time +import os + +# Weights & Biases +try: + import wandb + WANDB_AVAILABLE = True +except ImportError: + WANDB_AVAILABLE = False + +from src.training.epoch import train_one_epoch +from src.model import DeepDietModel +from src.dataset import MultiViewDataset, TARGETS, multimodal_collate_fn +from src.metrics import compute_target_means + + +def get_device(): + if torch.backends.mps.is_available(): + return torch.device("mps") + return torch.device("cuda" if torch.cuda.is_available() else "cpu") + + +@hydra.main(version_base=None, config_path="../configs", config_name="config") +def main(cfg: DictConfig): + # Print resolved config + print(OmegaConf.to_yaml(cfg)) + + # Get original working directory (Hydra changes cwd) + orig_cwd = hydra.utils.get_original_cwd() + repo = Path(orig_cwd) + + device = get_device() + print(f"Device: {device}") + + # Generate run name + timestamp = time.strftime('%Y%m%d_%H%M%S') + modalities = [] + if cfg.model.use_side_frames: + modalities.append("side") + if cfg.model.use_overhead: + modalities.append("overhead") + if cfg.model.use_depth: + modalities.append("depth") + + run_name = cfg.experiment.name or f"{'_'.join(modalities)}_{cfg.model.side_aggregation}_{timestamp}" + + # TensorBoard setup + log_dir = repo / "runs" / run_name + writer = SummaryWriter(log_dir) + print(f"TensorBoard logs: {log_dir}") + + # W&B setup + use_wandb = cfg.logging.wandb and WANDB_AVAILABLE + if cfg.logging.wandb and not WANDB_AVAILABLE: + print("Warning: wandb not installed. Skipping W&B logging.") + + if use_wandb: + wandb_config = OmegaConf.to_container(cfg, resolve=True) + tags = modalities + [cfg.model.side_aggregation] + list(cfg.experiment.tags) + + wandb.init( + project=cfg.logging.wandb_project, + entity=cfg.logging.wandb_entity, + name=run_name, + config=wandb_config, + tags=tags, + notes=cfg.experiment.notes, + sync_tensorboard=True, + ) + print(f"W&B run: {wandb.run.url}") + + # Data paths + data_root = repo / cfg.data.data_root + train_csv = repo / cfg.data.train_csv + test_csv = repo / cfg.data.test_csv + + # Datasets + print("\nLoading datasets...") + train_ds = MultiViewDataset( + split_file=train_csv, + data_root=data_root, + train=True, + max_side_frames=cfg.data.max_frames, + use_side_frames=cfg.model.use_side_frames, + use_overhead=cfg.model.use_overhead, + use_depth=cfg.model.use_depth, + image_size=cfg.data.image_size, + allow_missing_modalities=cfg.model.allow_missing_modalities, + ) + val_ds = MultiViewDataset( + split_file=test_csv, + data_root=data_root, + train=False, + max_side_frames=cfg.data.max_frames, + use_side_frames=cfg.model.use_side_frames, + use_overhead=cfg.model.use_overhead, + use_depth=cfg.model.use_depth, + image_size=cfg.data.image_size, + allow_missing_modalities=cfg.model.allow_missing_modalities, + ) + + collate_fn = multimodal_collate_fn if cfg.model.allow_missing_modalities else None + train_dl = DataLoader( + train_ds, + batch_size=cfg.training.batch_size, + shuffle=True, + num_workers=cfg.training.num_workers, + pin_memory=(device.type == 'cuda'), + collate_fn=collate_fn, + ) + val_dl = DataLoader( + val_ds, + batch_size=cfg.training.batch_size, + shuffle=False, + num_workers=cfg.training.num_workers, + pin_memory=(device.type == 'cuda'), + collate_fn=collate_fn, + ) + + # Target means for percentage metrics + print("\nComputing dataset statistics...") + target_means = compute_target_means(train_ds, TARGETS) + for task in TARGETS: + print(f" {task}: {target_means[task]:.2f}") + + # Model + print(f"\nInitializing model with {cfg.model.side_aggregation} aggregation...") + model = DeepDietModel( + use_side_frames=cfg.model.use_side_frames, + use_overhead=cfg.model.use_overhead, + use_depth=cfg.model.use_depth, + chunk_size=cfg.model.chunk_size, + lstm_hidden=cfg.model.lstm_hidden, + side_aggregation=cfg.model.side_aggregation, + allow_missing_modalities=cfg.model.allow_missing_modalities, + ).to(device) + + # Freeze encoders if configured + encoders_frozen = False + if cfg.model.freeze_encoders: + model.freeze_encoders() + encoders_frozen = True + print(f"Encoders frozen until epoch {cfg.model.unfreeze_epoch}") + + # Optimizer and criterion + criterion = nn.L1Loss(reduction='none') + optimizer = optim.Adam( + filter(lambda p: p.requires_grad, model.parameters()), + lr=cfg.training.lr, + weight_decay=cfg.training.weight_decay, + ) + + scaler = torch.cuda.amp.GradScaler() if device.type == 'cuda' else None + task_weights = {task: 1.0 for task in TARGETS} + + # Training loop + best_val_loss = math.inf + print(f"\nStarting training for {cfg.training.epochs} epochs...") + print("=" * 70) + + for epoch in range(1, cfg.training.epochs + 1): + epoch_start = time.time() + + # Unfreeze encoders + if encoders_frozen and epoch == cfg.model.unfreeze_epoch: + print(f"\n *** Unfreezing encoders at epoch {epoch} ***") + model.unfreeze_encoders() + encoders_frozen = False + param_groups = model.get_param_groups( + base_lr=cfg.training.lr, + encoder_lr_multiplier=cfg.model.encoder_lr_multiplier, + ) + optimizer = optim.Adam(param_groups, weight_decay=cfg.training.weight_decay) + + # Train + model.train() + train_losses = {task: 0.0 for task in TARGETS} + train_total_loss = 0.0 + + for batch in train_dl: + targets = batch['targets'].to(device) + side_frames = batch.get('side_frames') + overhead_rgb = batch.get('overhead_rgb') + overhead_depth = batch.get('overhead_depth') + + if side_frames is not None: + side_frames = side_frames.to(device) + if overhead_rgb is not None: + overhead_rgb = overhead_rgb.to(device) + if overhead_depth is not None: + overhead_depth = overhead_depth.to(device) + + optimizer.zero_grad() + + if scaler: + with torch.cuda.amp.autocast(): + pred = model(side_frames, overhead_rgb, overhead_depth) + task_losses = criterion(pred, targets).mean(dim=0) + loss = sum(task_losses[i] * task_weights[TARGETS[i]] for i in range(len(TARGETS))) + scaler.scale(loss).backward() + scaler.unscale_(optimizer) + torch.nn.utils.clip_grad_norm_(model.parameters(), cfg.training.max_grad_norm) + scaler.step(optimizer) + scaler.update() + else: + pred = model(side_frames, overhead_rgb, overhead_depth) + task_losses = criterion(pred, targets).mean(dim=0) + loss = sum(task_losses[i] * task_weights[TARGETS[i]] for i in range(len(TARGETS))) + loss.backward() + torch.nn.utils.clip_grad_norm_(model.parameters(), cfg.training.max_grad_norm) + optimizer.step() + + batch_size = targets.size(0) + train_total_loss += loss.item() * batch_size + for i, task in enumerate(TARGETS): + train_losses[task] += task_losses[i].item() * batch_size + + train_total_loss /= len(train_ds) + for task in TARGETS: + train_losses[task] /= len(train_ds) + + # Validate + model.eval() + val_losses = {task: 0.0 for task in TARGETS} + val_total_loss = 0.0 + + with torch.no_grad(): + for batch in val_dl: + targets = batch['targets'].to(device) + side_frames = batch.get('side_frames') + overhead_rgb = batch.get('overhead_rgb') + overhead_depth = batch.get('overhead_depth') + + if side_frames is not None: + side_frames = side_frames.to(device) + if overhead_rgb is not None: + overhead_rgb = overhead_rgb.to(device) + if overhead_depth is not None: + overhead_depth = overhead_depth.to(device) + + if scaler: + with torch.cuda.amp.autocast(): + pred = model(side_frames, overhead_rgb, overhead_depth) + task_losses = criterion(pred, targets).mean(dim=0) + loss = sum(task_losses[i] * task_weights[TARGETS[i]] for i in range(len(TARGETS))) + else: + pred = model(side_frames, overhead_rgb, overhead_depth) + task_losses = criterion(pred, targets).mean(dim=0) + loss = sum(task_losses[i] * task_weights[TARGETS[i]] for i in range(len(TARGETS))) + + batch_size = targets.size(0) + val_total_loss += loss.item() * batch_size + for i, task in enumerate(TARGETS): + val_losses[task] += task_losses[i].item() * batch_size + + val_total_loss /= len(val_ds) + for task in TARGETS: + val_losses[task] /= len(val_ds) + + epoch_time = time.time() - epoch_start + + # Print summary + print(f"\n[Epoch {epoch:02d}/{cfg.training.epochs}] Time: {epoch_time:.1f}s") + print(f" Loss: {train_total_loss:.3f} (train) / {val_total_loss:.3f} (val)") + + # Log to TensorBoard + writer.add_scalar('Loss/train', train_total_loss, epoch) + writer.add_scalar('Loss/val', val_total_loss, epoch) + for task in TARGETS: + writer.add_scalar(f'MAE_train/{task}', train_losses[task], epoch) + writer.add_scalar(f'MAE_val/{task}', val_losses[task], epoch) + writer.flush() + + # Log to W&B + if use_wandb: + wandb_metrics = { + "epoch": epoch, + "train/loss": train_total_loss, + "val/loss": val_total_loss, + } + for task in TARGETS: + wandb_metrics[f"train/mae_{task}"] = train_losses[task] + wandb_metrics[f"val/mae_{task}"] = val_losses[task] + wandb.log(wandb_metrics, step=epoch) + + # Save best model + if val_total_loss < best_val_loss: + best_val_loss = val_total_loss + save_path = repo / "indexes" / "multiview_best.pt" + torch.save(model.state_dict(), save_path) + print(f" → Saved best model (val_loss: {best_val_loss:.3f})") + + # LR decay + if epoch % cfg.training.lr_decay_epochs == 0: + for param_group in optimizer.param_groups: + param_group['lr'] *= cfg.training.lr_decay_factor + + print("=" * 70) + print(f"Training complete! Best val loss: {best_val_loss:.3f}") + writer.close() + + if use_wandb: + wandb.summary["best_val_loss"] = best_val_loss + wandb.finish() + + return best_val_loss + + +if __name__ == "__main__": + main() \ No newline at end of file