diff --git a/.gitignore b/.gitignore index 6606d1085..598282f54 100644 --- a/.gitignore +++ b/.gitignore @@ -153,9 +153,15 @@ dmypy.json # Results folders **/results/ +**/data +**/wandb # Log folders **/log/ # Mac OS -.DS_Store \ No newline at end of file +.DS_Store + +EEGNet_BNCI2014001_* +ShallowConvNet_BNCI2014001_hopt* +finetune_* \ No newline at end of file diff --git a/benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/EEGNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/EEGNet.yaml index 7dca191ac..0e09edff4 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/EEGNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/EEGNet.yaml @@ -47,12 +47,12 @@ cm: !name:sklearn.metrics.confusion_matrix metrics: f1: !ref acc: !ref - cm: !ref + # cm: !ref # TRAINING HPARS n_train_examples: 100 # it will be replaced in the train script # checkpoints to average avg_models: 10 # @orion_step1: --avg_models~"uniform(1, 15,discrete=True)" -number_of_epochs: 862 # @orion_step1: --number_of_epochs~"uniform(250, 1000, discrete=True)" +number_of_epochs: 100 lr: 0.0001 # @orion_step1: --lr~"choices([0.01, 0.005, 0.001, 0.0005, 0.0001])" # Learning rate scheduling (cyclic learning rate is used here) max_lr: !ref # Upper bound of the cycle (max value of the lr) @@ -120,6 +120,13 @@ augment: !new:speechbrain.augment.augmenter.Augmenter !ref , !ref ] +# WEIGHTS AND BIASES Configuration +project: "BNCI2014001" +prefix_name: 'EEGNet' +mode: "disabled" +entity: "neuro-gflow" +sweep_run: False # to indicate if you are running sweeps or normal wandb runs + # DATA NORMALIZATION dims_to_normalize: 1 # 1 (time) or 2 (EEG channels) normalize: !name:speechbrain.processing.signal_processing.mean_std_norm diff --git a/benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/ShallowConvNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/ShallowConvNet.yaml index 30c513649..51ab180a7 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/ShallowConvNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/ShallowConvNet.yaml @@ -47,12 +47,12 @@ cm: !name:sklearn.metrics.confusion_matrix metrics: f1: !ref acc: !ref - cm: !ref + # cm: !ref # TRAINING HPARS n_train_examples: 100 # it will be replaced in the train script # checkpoints to average avg_models: 1 # @orion_step1: --avg_models~"uniform(1, 15,discrete=True)" -number_of_epochs: 881 # @orion_step1: --number_of_epochs~"uniform(250, 1000, discrete=True)" +number_of_epochs: 100 lr: 0.001 # @orion_step1: --lr~"choices([0.01, 0.005, 0.001, 0.0005, 0.0001])" # Learning rate scheduling (cyclic learning rate is used here) max_lr: !ref # Upper bound of the cycle (max value of the lr) @@ -120,6 +120,13 @@ augment: !new:speechbrain.augment.augmenter.Augmenter !ref , !ref ] +# WEIGHTS AND BIASES Configuration +project: "BNCI2014001" +prefix_name: 'ShallowNet' +mode: "disabled" +entity: "neuro-gflow" +sweep_run: False # to indicate if you are running sweeps or normal wandb runs + # DATA NORMALIZATION dims_to_normalize: 1 # 1 (time) or 2 (EEG channels) normalize: !name:speechbrain.processing.signal_processing.mean_std_norm diff --git a/benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/EEGNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/EEGNet.yaml index e03a4d369..07b485563 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/EEGNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/EEGNet.yaml @@ -47,12 +47,12 @@ cm: !name:sklearn.metrics.confusion_matrix metrics: f1: !ref acc: !ref - cm: !ref + # cm: !ref # TRAINING HPARS n_train_examples: 100 # it will be replaced in the train script # checkpoints to average avg_models: 2 # @orion_step1: --avg_models~"uniform(1, 15,discrete=True)" -number_of_epochs: 796 # @orion_step1: --number_of_epochs~"uniform(250, 1000, discrete=True)" +number_of_epochs: 100 lr: 0.0001 # @orion_step1: --lr~"choices([0.01, 0.005, 0.001, 0.0005, 0.0001])" # Learning rate scheduling (cyclic learning rate is used here) max_lr: !ref # Upper bound of the cycle (max value of the lr) @@ -120,6 +120,13 @@ augment: !new:speechbrain.augment.augmenter.Augmenter !ref , !ref ] +# WEIGHTS AND BIASES Configuration +project: "BNCI2014004" +prefix_name: 'EEGNet' +mode: "disabled" +entity: "neuro-gflow" +sweep_run: False # to indicate if you are running sweeps or normal wandb runs + # DATA NORMALIZATION dims_to_normalize: 1 # 1 (time) or 2 (EEG channels) normalize: !name:speechbrain.processing.signal_processing.mean_std_norm diff --git a/benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/ShallowConvNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/ShallowConvNet.yaml index 5ae88aff6..b1510a8fd 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/ShallowConvNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/ShallowConvNet.yaml @@ -47,12 +47,12 @@ cm: !name:sklearn.metrics.confusion_matrix metrics: f1: !ref acc: !ref - cm: !ref + # cm: !ref # TRAINING HPARS n_train_examples: 100 # it will be replaced in the train script # checkpoints to average avg_models: 11 # @orion_step1: --avg_models~"uniform(1, 15,discrete=True)" -number_of_epochs: 262 # @orion_step1: --number_of_epochs~"uniform(250, 1000, discrete=True)" +number_of_epochs: 100 lr: 0.005 # @orion_step1: --lr~"choices([0.01, 0.005, 0.001, 0.0005, 0.0001])" # Learning rate scheduling (cyclic learning rate is used here) max_lr: !ref # Upper bound of the cycle (max value of the lr) @@ -120,6 +120,13 @@ augment: !new:speechbrain.augment.augmenter.Augmenter !ref , !ref ] +# WEIGHTS AND BIASES Configuration +project: "BNCI2014004" +prefix_name: 'ShallowNet' +mode: "disabled" +entity: "neuro-gflow" +sweep_run: False # to indicate if you are running sweeps or normal wandb runs + # DATA NORMALIZATION dims_to_normalize: 1 # 1 (time) or 2 (EEG channels) normalize: !name:speechbrain.processing.signal_processing.mean_std_norm diff --git a/benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/EEGNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/EEGNet.yaml index e2be109d5..03229ba1d 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/EEGNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/EEGNet.yaml @@ -47,12 +47,12 @@ cm: !name:sklearn.metrics.confusion_matrix metrics: f1: !ref acc: !ref - cm: !ref + # cm: !ref # TRAINING HPARS n_train_examples: 100 # it will be replaced in the train script # checkpoints to average avg_models: 12 # @orion_step1: --avg_models~"uniform(1, 15,discrete=True)" -number_of_epochs: 760 # @orion_step1: --number_of_epochs~"uniform(250, 1000, discrete=True)" +number_of_epochs: 100 lr: 0.001 # @orion_step1: --lr~"choices([0.01, 0.005, 0.001, 0.0005, 0.0001])" # Learning rate scheduling (cyclic learning rate is used here) max_lr: !ref # Upper bound of the cycle (max value of the lr) @@ -120,6 +120,13 @@ augment: !new:speechbrain.augment.augmenter.Augmenter !ref , !ref ] +# WEIGHTS AND BIASES Configuration +project: "BNCI2015001" +prefix_name: 'EEGNet' +mode: "disabled" +entity: "neuro-gflow" +sweep_run: False # to indicate if you are running sweeps or normal wandb runs + # DATA NORMALIZATION dims_to_normalize: 1 # 1 (time) or 2 (EEG channels) normalize: !name:speechbrain.processing.signal_processing.mean_std_norm diff --git a/benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/ShallowConvNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/ShallowConvNet.yaml index 5e830758d..1c1fe385b 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/ShallowConvNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/ShallowConvNet.yaml @@ -47,12 +47,12 @@ cm: !name:sklearn.metrics.confusion_matrix metrics: f1: !ref acc: !ref - cm: !ref + # cm: !ref # TRAINING HPARS n_train_examples: 100 # it will be replaced in the train script # checkpoints to average avg_models: 9 # @orion_step1: --avg_models~"uniform(1, 15,discrete=True)" -number_of_epochs: 976 # @orion_step1: --number_of_epochs~"uniform(250, 1000, discrete=True)" +number_of_epochs: 100 lr: 0.001 # @orion_step1: --lr~"choices([0.01, 0.005, 0.001, 0.0005, 0.0001])" # Learning rate scheduling (cyclic learning rate is used here) max_lr: !ref # Upper bound of the cycle (max value of the lr) @@ -120,6 +120,13 @@ augment: !new:speechbrain.augment.augmenter.Augmenter !ref , !ref ] +# WEIGHTS AND BIASES Configuration +project: "BNCI2015001" +prefix_name: 'ShallowNet' +mode: "disabled" +entity: "neuro-gflow" +sweep_run: False # to indicate if you are running sweeps or normal wandb runs + # DATA NORMALIZATION dims_to_normalize: 1 # 1 (time) or 2 (EEG channels) normalize: !name:speechbrain.processing.signal_processing.mean_std_norm diff --git a/benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/EEGNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/EEGNet.yaml index 839d6995f..a1040c0c6 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/EEGNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/EEGNet.yaml @@ -47,12 +47,12 @@ cm: !name:sklearn.metrics.confusion_matrix metrics: f1: !ref acc: !ref - cm: !ref + # cm: !ref # TRAINING HPARS n_train_examples: 100 # it will be replaced in the train script # checkpoints to average avg_models: 15 # @orion_step1: --avg_models~"uniform(1, 15,discrete=True)" -number_of_epochs: 821 # @orion_step1: --number_of_epochs~"uniform(250, 1000, discrete=True)" +number_of_epochs: 100 lr: 0.001 # @orion_step1: --lr~"choices([0.01, 0.005, 0.001, 0.0005, 0.0001])" # Learning rate scheduling (cyclic learning rate is used here) max_lr: !ref # Upper bound of the cycle (max value of the lr) @@ -120,6 +120,13 @@ augment: !new:speechbrain.augment.augmenter.Augmenter !ref , !ref ] +# WEIGHTS AND BIASES Configuration +project: "Lee2019_MI" +prefix_name: 'EEGNet' +mode: "disabled" +entity: "neuro-gflow" +sweep_run: False # to indicate if you are running sweeps or normal wandb runs + # DATA NORMALIZATION dims_to_normalize: 1 # 1 (time) or 2 (EEG channels) normalize: !name:speechbrain.processing.signal_processing.mean_std_norm diff --git a/benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/ShallowConvNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/ShallowConvNet.yaml index 8cf848256..5c6f55980 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/ShallowConvNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/ShallowConvNet.yaml @@ -47,12 +47,12 @@ cm: !name:sklearn.metrics.confusion_matrix metrics: f1: !ref acc: !ref - cm: !ref + # cm: !ref # TRAINING HPARS n_train_examples: 100 # it will be replaced in the train script # checkpoints to average avg_models: 12 # @orion_step1: --avg_models~"uniform(1, 15,discrete=True)" -number_of_epochs: 922 # @orion_step1: --number_of_epochs~"uniform(250, 1000, discrete=True)" +number_of_epochs: 100 lr: 0.001 # @orion_step1: --lr~"choices([0.01, 0.005, 0.001, 0.0005, 0.0001])" # Learning rate scheduling (cyclic learning rate is used here) max_lr: !ref # Upper bound of the cycle (max value of the lr) @@ -120,6 +120,13 @@ augment: !new:speechbrain.augment.augmenter.Augmenter !ref , !ref ] +# WEIGHTS AND BIASES Configuration +project: "Lee2019_MI" +prefix_name: 'ShallowNet' +mode: "disabled" +entity: "neuro-gflow" +sweep_run: False # to indicate if you are running sweeps or normal wandb runs + # DATA NORMALIZATION dims_to_normalize: 1 # 1 (time) or 2 (EEG channels) normalize: !name:speechbrain.processing.signal_processing.mean_std_norm diff --git a/benchmarks/MOABB/hparams/MotorImagery/Zhou2016/EEGNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/Zhou2016/EEGNet.yaml index bdae72c87..8c1ba7b5e 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/Zhou2016/EEGNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/Zhou2016/EEGNet.yaml @@ -47,12 +47,12 @@ cm: !name:sklearn.metrics.confusion_matrix metrics: f1: !ref acc: !ref - cm: !ref + # cm: !ref # TRAINING HPARS n_train_examples: 100 # it will be replaced in the train script # checkpoints to average avg_models: 10 # @orion_step1: --avg_models~"uniform(1, 15,discrete=True)" -number_of_epochs: 741 # @orion_step1: --number_of_epochs~"uniform(250, 1000, discrete=True)" +number_of_epochs: 100 lr: 0.001 # @orion_step1: --lr~"choices([0.01, 0.005, 0.001, 0.0005, 0.0001])" # Learning rate scheduling (cyclic learning rate is used here) max_lr: !ref # Upper bound of the cycle (max value of the lr) @@ -120,6 +120,13 @@ augment: !new:speechbrain.augment.augmenter.Augmenter !ref , !ref ] +# WEIGHTS AND BIASES Configuration +project: "Zhou2016" +prefix_name: 'EEGNet' +mode: "disabled" +entity: "neuro-gflow" +sweep_run: False # to indicate if you are running sweeps or normal wandb runs + # DATA NORMALIZATION dims_to_normalize: 1 # 1 (time) or 2 (EEG channels) normalize: !name:speechbrain.processing.signal_processing.mean_std_norm diff --git a/benchmarks/MOABB/hparams/MotorImagery/Zhou2016/ShallowConvNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/Zhou2016/ShallowConvNet.yaml index 3edce849f..60c4b88db 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/Zhou2016/ShallowConvNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/Zhou2016/ShallowConvNet.yaml @@ -47,12 +47,12 @@ cm: !name:sklearn.metrics.confusion_matrix metrics: f1: !ref acc: !ref - cm: !ref + # cm: !ref # TRAINING HPARS n_train_examples: 100 # it will be replaced in the train script # checkpoints to average avg_models: 13 # @orion_step1: --avg_models~"uniform(1, 15,discrete=True)" -number_of_epochs: 955 # @orion_step1: --number_of_epochs~"uniform(250, 1000, discrete=True)" +number_of_epochs: 100 lr: 0.001 # @orion_step1: --lr~"choices([0.01, 0.005, 0.001, 0.0005, 0.0001])" # Learning rate scheduling (cyclic learning rate is used here) max_lr: !ref # Upper bound of the cycle (max value of the lr) @@ -120,6 +120,13 @@ augment: !new:speechbrain.augment.augmenter.Augmenter !ref , !ref ] +# WEIGHTS AND BIASES Configuration +project: "Zhou2016" +prefix_name: 'ShallowNet' +mode: "disabled" +entity: "neuro-gflow" +sweep_run: False # to indicate if you are running sweeps or normal wandb runs + # DATA NORMALIZATION dims_to_normalize: 1 # 1 (time) or 2 (EEG channels) normalize: !name:speechbrain.processing.signal_processing.mean_std_norm diff --git a/benchmarks/MOABB/run_hparam_opt_EEGNet.sh b/benchmarks/MOABB/run_hparam_opt_EEGNet.sh new file mode 100644 index 000000000..082e2d078 --- /dev/null +++ b/benchmarks/MOABB/run_hparam_opt_EEGNet.sh @@ -0,0 +1,29 @@ +#!/bin/bash +#SBATCH --job-name=finetune +#SBATCH --output=finetune_output_%j.txt +#SBATCH --error=finetune_error_%j.txt +#SBATCH --ntasks=1 +#SBATCH --time=5:00:00 +#SBATCH --partition=unkillable +#SBATCH --cpus-per-task=6 +#SBATCH --mem=32G +#SBATCH --gres=gpu:a100l:1 + +bash run_hparam_optimization.sh --exp_name 'EEGNet_BNCI2014001_hopt' \ + --output_folder results/BNCI2014001/EEGNet/hopt \ + --data_folder data \ + --cached_data_folder data \ + --hparams hparams/MotorImagery/BNCI2014001/EEGNet.yaml \ + --nsbj_hpsearch 9 \ + --nsess_hpsearch 2 \ + --nsbj 9 \ + --nsess 2 \ + --nruns 1 \ + --nruns_eval 10 \ + --eval_metric acc \ + --train_mode leave-one-subject-out \ + --exp_max_trials 50 \ + --store_all True \ + --device 'cuda' \ + --project 'EEGNet_BNCI2014001_hopt' \ + --mode 'online' \ No newline at end of file diff --git a/benchmarks/MOABB/run_hparam_opt_shallownet.sh b/benchmarks/MOABB/run_hparam_opt_shallownet.sh new file mode 100644 index 000000000..f86f4032b --- /dev/null +++ b/benchmarks/MOABB/run_hparam_opt_shallownet.sh @@ -0,0 +1,29 @@ +#!/bin/bash +#SBATCH --job-name=finetune +#SBATCH --output=finetune_output_%j.txt +#SBATCH --error=finetune_error_%j.txt +#SBATCH --ntasks=1 +#SBATCH --time=10:00:00 +#SBATCH --partition=main +#SBATCH --cpus-per-task=8 +#SBATCH --mem=48G +#SBATCH --gres=gpu:a100l:1 + +bash run_hparam_optimization.sh --exp_name 'ShallowConvNet_BNCI2014001_hopt' \ + --output_folder results/BNCI2014001/ShallowConvNet/hopt \ + --data_folder data \ + --cached_data_folder data \ + --hparams hparams/MotorImagery/BNCI2014001/ShallowConvNet.yaml \ + --nsbj_hpsearch 9 \ + --nsess_hpsearch 2 \ + --nsbj 9 \ + --nsess 2 \ + --nruns 1 \ + --nruns_eval 10 \ + --eval_metric acc \ + --train_mode leave-one-subject-out \ + --exp_max_trials 50 \ + --store_all True \ + --device 'cuda' \ + --project 'ShallowConvNet_BNCI2014001_hopt' \ + --mode 'online' \ No newline at end of file diff --git a/benchmarks/MOABB/train.py b/benchmarks/MOABB/train.py index f1d3745df..adc18fcea 100644 --- a/benchmarks/MOABB/train.py +++ b/benchmarks/MOABB/train.py @@ -25,6 +25,9 @@ import speechbrain as sb import yaml +import wandb +import datetime +import uuid class MOABBBrain(sb.Brain): def init_model(self, model): @@ -83,6 +86,15 @@ def compute_objectives(self, predictions, batch, stage): def on_fit_start(self,): """Gets called at the beginning of ``fit()``""" + # Initialize wandb + run_name = f"{self.hparams.prefix_name}_{datetime.datetime.now().strftime('%Y%m%d_%H%M%S')}" + wandb.init( + project=self.hparams.project, + name=run_name, + mode=self.hparams.mode, + entity=self.hparams.entity, + ) + self.init_model(self.hparams.model) self.init_optimizers() in_shape = ( @@ -137,6 +149,14 @@ def on_stage_end(self, stage, stage_loss, epoch=None): train_stats={"loss": self.train_loss}, valid_stats=self.last_eval_stats, ) + # Log validation stats to wandb + wandb.log( + { + **self.last_eval_stats, + "lr": old_lr, + "train_loss": self.train_loss, + } + ) if epoch == 1: self.best_eval_stats = self.last_eval_stats @@ -181,6 +201,8 @@ def on_stage_end(self, stage, stage_loss, epoch=None): ) elif stage == sb.Stage.TEST: + # Log final test metrics to wandb + wandb.log({"test_loss": self.last_eval_stats['loss'], "test_acc": self.last_eval_stats['acc'], "test_f1": self.last_eval_stats['f1']}) self.hparams.train_logger.log_stats( stats_meta={ "epoch loaded": self.hparams.epoch_counter.current @@ -391,7 +413,17 @@ def load_hparams_and_dataset_iterators(hparams_file, run_opts, overrides): # loading hparams for the each training and evaluation processes with open(hparams_file) as fin: hparams = load_hyperpyyaml(fin, overrides) - hparams["exp_dir"] = os.path.join(hparams["output_folder"], tail_path) + + if hparams["sweep_run"]: + # Generate a random identifier + unique_output_folder = os.path.join( + hparams["output_folder"], + f"{datetime.datetime.now().strftime('%Y%m%d_%H%M%S')}_{str(uuid.uuid4())}", + tail_path, + ) + hparams["exp_dir"] = unique_output_folder + else: + hparams["exp_dir"] = os.path.join(hparams["output_folder"], tail_path) # creating experiment directory sb.create_experiment_directory(