From fc9f4788731b46172cf4fc2a0aed3d7c7785d994 Mon Sep 17 00:00:00 2001 From: Jama Hussein Mohamud Date: Mon, 5 Aug 2024 14:05:58 -0400 Subject: [PATCH 1/6] added wandb support to the default main branch --- .gitignore | 1 + .../MotorImagery/BNCI2014001/EEGNet.yaml | 7 +++++++ .../BNCI2014001/ShallowConvNet.yaml | 7 +++++++ .../MotorImagery/BNCI2014004/EEGNet.yaml | 7 +++++++ .../BNCI2014004/ShallowConvNet.yaml | 7 +++++++ .../MotorImagery/BNCI2015001/EEGNet.yaml | 7 +++++++ .../BNCI2015001/ShallowConvNet.yaml | 7 +++++++ .../MotorImagery/Lee2019_MI/EEGNet.yaml | 7 +++++++ .../Lee2019_MI/ShallowConvNet.yaml | 7 +++++++ .../hparams/MotorImagery/Zhou2016/EEGNet.yaml | 7 +++++++ .../MotorImagery/Zhou2016/ShallowConvNet.yaml | 7 +++++++ benchmarks/MOABB/train.py | 21 +++++++++++++++++++ 12 files changed, 92 insertions(+) diff --git a/.gitignore b/.gitignore index 6606d1085..3012676d2 100644 --- a/.gitignore +++ b/.gitignore @@ -153,6 +153,7 @@ dmypy.json # Results folders **/results/ +**/data # Log folders **/log/ diff --git a/benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/EEGNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/EEGNet.yaml index 7dca191ac..1725c0d3c 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/EEGNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/EEGNet.yaml @@ -120,6 +120,13 @@ augment: !new:speechbrain.augment.augmenter.Augmenter !ref , !ref ] +# WEIGHTS AND BIASES Configuration +project: "BNCI2014001" +prefix_name: 'EEGNet' +mode: "disabled" +entity: "neuro-gflow" +sweep_run: False # to indicate if you are running sweeps or normal wandb runs + # DATA NORMALIZATION dims_to_normalize: 1 # 1 (time) or 2 (EEG channels) normalize: !name:speechbrain.processing.signal_processing.mean_std_norm diff --git a/benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/ShallowConvNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/ShallowConvNet.yaml index 30c513649..8d4a6babf 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/ShallowConvNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/ShallowConvNet.yaml @@ -120,6 +120,13 @@ augment: !new:speechbrain.augment.augmenter.Augmenter !ref , !ref ] +# WEIGHTS AND BIASES Configuration +project: "BNCI2014001" +prefix_name: 'ShallowNet' +mode: "disabled" +entity: "neuro-gflow" +sweep_run: False # to indicate if you are running sweeps or normal wandb runs + # DATA NORMALIZATION dims_to_normalize: 1 # 1 (time) or 2 (EEG channels) normalize: !name:speechbrain.processing.signal_processing.mean_std_norm diff --git a/benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/EEGNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/EEGNet.yaml index e03a4d369..bfdfd1258 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/EEGNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/EEGNet.yaml @@ -120,6 +120,13 @@ augment: !new:speechbrain.augment.augmenter.Augmenter !ref , !ref ] +# WEIGHTS AND BIASES Configuration +project: "BNCI2014004" +prefix_name: 'EEGNet' +mode: "disabled" +entity: "neuro-gflow" +sweep_run: False # to indicate if you are running sweeps or normal wandb runs + # DATA NORMALIZATION dims_to_normalize: 1 # 1 (time) or 2 (EEG channels) normalize: !name:speechbrain.processing.signal_processing.mean_std_norm diff --git a/benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/ShallowConvNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/ShallowConvNet.yaml index 5ae88aff6..f316323c9 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/ShallowConvNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/ShallowConvNet.yaml @@ -120,6 +120,13 @@ augment: !new:speechbrain.augment.augmenter.Augmenter !ref , !ref ] +# WEIGHTS AND BIASES Configuration +project: "BNCI2014004" +prefix_name: 'ShallowNet' +mode: "disabled" +entity: "neuro-gflow" +sweep_run: False # to indicate if you are running sweeps or normal wandb runs + # DATA NORMALIZATION dims_to_normalize: 1 # 1 (time) or 2 (EEG channels) normalize: !name:speechbrain.processing.signal_processing.mean_std_norm diff --git a/benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/EEGNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/EEGNet.yaml index e2be109d5..da9b782b9 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/EEGNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/EEGNet.yaml @@ -120,6 +120,13 @@ augment: !new:speechbrain.augment.augmenter.Augmenter !ref , !ref ] +# WEIGHTS AND BIASES Configuration +project: "BNCI2015001" +prefix_name: 'EEGNet' +mode: "disabled" +entity: "neuro-gflow" +sweep_run: False # to indicate if you are running sweeps or normal wandb runs + # DATA NORMALIZATION dims_to_normalize: 1 # 1 (time) or 2 (EEG channels) normalize: !name:speechbrain.processing.signal_processing.mean_std_norm diff --git a/benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/ShallowConvNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/ShallowConvNet.yaml index 5e830758d..6cba9f0ba 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/ShallowConvNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/ShallowConvNet.yaml @@ -120,6 +120,13 @@ augment: !new:speechbrain.augment.augmenter.Augmenter !ref , !ref ] +# WEIGHTS AND BIASES Configuration +project: "BNCI2015001" +prefix_name: 'ShallowNet' +mode: "disabled" +entity: "neuro-gflow" +sweep_run: False # to indicate if you are running sweeps or normal wandb runs + # DATA NORMALIZATION dims_to_normalize: 1 # 1 (time) or 2 (EEG channels) normalize: !name:speechbrain.processing.signal_processing.mean_std_norm diff --git a/benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/EEGNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/EEGNet.yaml index 839d6995f..d683113b7 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/EEGNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/EEGNet.yaml @@ -120,6 +120,13 @@ augment: !new:speechbrain.augment.augmenter.Augmenter !ref , !ref ] +# WEIGHTS AND BIASES Configuration +project: "Lee2019_MI" +prefix_name: 'EEGNet' +mode: "disabled" +entity: "neuro-gflow" +sweep_run: False # to indicate if you are running sweeps or normal wandb runs + # DATA NORMALIZATION dims_to_normalize: 1 # 1 (time) or 2 (EEG channels) normalize: !name:speechbrain.processing.signal_processing.mean_std_norm diff --git a/benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/ShallowConvNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/ShallowConvNet.yaml index 8cf848256..49d348900 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/ShallowConvNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/ShallowConvNet.yaml @@ -120,6 +120,13 @@ augment: !new:speechbrain.augment.augmenter.Augmenter !ref , !ref ] +# WEIGHTS AND BIASES Configuration +project: "Lee2019_MI" +prefix_name: 'ShallowNet' +mode: "disabled" +entity: "neuro-gflow" +sweep_run: False # to indicate if you are running sweeps or normal wandb runs + # DATA NORMALIZATION dims_to_normalize: 1 # 1 (time) or 2 (EEG channels) normalize: !name:speechbrain.processing.signal_processing.mean_std_norm diff --git a/benchmarks/MOABB/hparams/MotorImagery/Zhou2016/EEGNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/Zhou2016/EEGNet.yaml index bdae72c87..a28402c5a 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/Zhou2016/EEGNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/Zhou2016/EEGNet.yaml @@ -120,6 +120,13 @@ augment: !new:speechbrain.augment.augmenter.Augmenter !ref , !ref ] +# WEIGHTS AND BIASES Configuration +project: "Zhou2016" +prefix_name: 'EEGNet' +mode: "disabled" +entity: "neuro-gflow" +sweep_run: False # to indicate if you are running sweeps or normal wandb runs + # DATA NORMALIZATION dims_to_normalize: 1 # 1 (time) or 2 (EEG channels) normalize: !name:speechbrain.processing.signal_processing.mean_std_norm diff --git a/benchmarks/MOABB/hparams/MotorImagery/Zhou2016/ShallowConvNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/Zhou2016/ShallowConvNet.yaml index 3edce849f..9d991936a 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/Zhou2016/ShallowConvNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/Zhou2016/ShallowConvNet.yaml @@ -120,6 +120,13 @@ augment: !new:speechbrain.augment.augmenter.Augmenter !ref , !ref ] +# WEIGHTS AND BIASES Configuration +project: "Zhou2016" +prefix_name: 'ShallowNet' +mode: "disabled" +entity: "neuro-gflow" +sweep_run: False # to indicate if you are running sweeps or normal wandb runs + # DATA NORMALIZATION dims_to_normalize: 1 # 1 (time) or 2 (EEG channels) normalize: !name:speechbrain.processing.signal_processing.mean_std_norm diff --git a/benchmarks/MOABB/train.py b/benchmarks/MOABB/train.py index f1d3745df..0ea48fc11 100644 --- a/benchmarks/MOABB/train.py +++ b/benchmarks/MOABB/train.py @@ -25,6 +25,8 @@ import speechbrain as sb import yaml +import wandb +import datetime class MOABBBrain(sb.Brain): def init_model(self, model): @@ -83,6 +85,15 @@ def compute_objectives(self, predictions, batch, stage): def on_fit_start(self,): """Gets called at the beginning of ``fit()``""" + # Initialize wandb + run_name = f"{self.hparams.prefix_name}_{datetime.datetime.now().strftime('%Y%m%d_%H%M%S')}" + wandb.init( + project=self.hparams.project, + name=run_name, + mode=self.hparams.mode, + entity=self.hparams.entity, + ) + self.init_model(self.hparams.model) self.init_optimizers() in_shape = ( @@ -137,6 +148,14 @@ def on_stage_end(self, stage, stage_loss, epoch=None): train_stats={"loss": self.train_loss}, valid_stats=self.last_eval_stats, ) + # Log validation stats to wandb + wandb.log( + { + **self.last_eval_stats, + "lr": old_lr, + "train_loss": self.train_loss, + } + ) if epoch == 1: self.best_eval_stats = self.last_eval_stats @@ -181,6 +200,8 @@ def on_stage_end(self, stage, stage_loss, epoch=None): ) elif stage == sb.Stage.TEST: + # Log final test metrics to wandb + wandb.log({**self.last_eval_stats}) self.hparams.train_logger.log_stats( stats_meta={ "epoch loaded": self.hparams.epoch_counter.current From 1f339ed0e2a08bef5e8450c41369f53df6b8db1c Mon Sep 17 00:00:00 2001 From: Jama Hussein Mohamud Date: Mon, 5 Aug 2024 14:20:29 -0400 Subject: [PATCH 2/6] removed confusion matrix calculation --- benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/EEGNet.yaml | 2 +- .../MOABB/hparams/MotorImagery/BNCI2014001/ShallowConvNet.yaml | 2 +- benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/EEGNet.yaml | 2 +- .../MOABB/hparams/MotorImagery/BNCI2014004/ShallowConvNet.yaml | 2 +- benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/EEGNet.yaml | 2 +- .../MOABB/hparams/MotorImagery/BNCI2015001/ShallowConvNet.yaml | 2 +- benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/EEGNet.yaml | 2 +- .../MOABB/hparams/MotorImagery/Lee2019_MI/ShallowConvNet.yaml | 2 +- benchmarks/MOABB/hparams/MotorImagery/Zhou2016/EEGNet.yaml | 2 +- .../MOABB/hparams/MotorImagery/Zhou2016/ShallowConvNet.yaml | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/EEGNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/EEGNet.yaml index 1725c0d3c..204a72890 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/EEGNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/EEGNet.yaml @@ -47,7 +47,7 @@ cm: !name:sklearn.metrics.confusion_matrix metrics: f1: !ref acc: !ref - cm: !ref + # cm: !ref # TRAINING HPARS n_train_examples: 100 # it will be replaced in the train script # checkpoints to average diff --git a/benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/ShallowConvNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/ShallowConvNet.yaml index 8d4a6babf..40ad1b4a5 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/ShallowConvNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/ShallowConvNet.yaml @@ -47,7 +47,7 @@ cm: !name:sklearn.metrics.confusion_matrix metrics: f1: !ref acc: !ref - cm: !ref + # cm: !ref # TRAINING HPARS n_train_examples: 100 # it will be replaced in the train script # checkpoints to average diff --git a/benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/EEGNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/EEGNet.yaml index bfdfd1258..9f0ebce0f 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/EEGNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/EEGNet.yaml @@ -47,7 +47,7 @@ cm: !name:sklearn.metrics.confusion_matrix metrics: f1: !ref acc: !ref - cm: !ref + # cm: !ref # TRAINING HPARS n_train_examples: 100 # it will be replaced in the train script # checkpoints to average diff --git a/benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/ShallowConvNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/ShallowConvNet.yaml index f316323c9..a01347a52 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/ShallowConvNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/ShallowConvNet.yaml @@ -47,7 +47,7 @@ cm: !name:sklearn.metrics.confusion_matrix metrics: f1: !ref acc: !ref - cm: !ref + # cm: !ref # TRAINING HPARS n_train_examples: 100 # it will be replaced in the train script # checkpoints to average diff --git a/benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/EEGNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/EEGNet.yaml index da9b782b9..fb3d8cb12 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/EEGNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/EEGNet.yaml @@ -47,7 +47,7 @@ cm: !name:sklearn.metrics.confusion_matrix metrics: f1: !ref acc: !ref - cm: !ref + # cm: !ref # TRAINING HPARS n_train_examples: 100 # it will be replaced in the train script # checkpoints to average diff --git a/benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/ShallowConvNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/ShallowConvNet.yaml index 6cba9f0ba..cc7471af5 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/ShallowConvNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/ShallowConvNet.yaml @@ -47,7 +47,7 @@ cm: !name:sklearn.metrics.confusion_matrix metrics: f1: !ref acc: !ref - cm: !ref + # cm: !ref # TRAINING HPARS n_train_examples: 100 # it will be replaced in the train script # checkpoints to average diff --git a/benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/EEGNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/EEGNet.yaml index d683113b7..c8aeba6dd 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/EEGNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/EEGNet.yaml @@ -47,7 +47,7 @@ cm: !name:sklearn.metrics.confusion_matrix metrics: f1: !ref acc: !ref - cm: !ref + # cm: !ref # TRAINING HPARS n_train_examples: 100 # it will be replaced in the train script # checkpoints to average diff --git a/benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/ShallowConvNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/ShallowConvNet.yaml index 49d348900..36e012512 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/ShallowConvNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/ShallowConvNet.yaml @@ -47,7 +47,7 @@ cm: !name:sklearn.metrics.confusion_matrix metrics: f1: !ref acc: !ref - cm: !ref + # cm: !ref # TRAINING HPARS n_train_examples: 100 # it will be replaced in the train script # checkpoints to average diff --git a/benchmarks/MOABB/hparams/MotorImagery/Zhou2016/EEGNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/Zhou2016/EEGNet.yaml index a28402c5a..38ca5208f 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/Zhou2016/EEGNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/Zhou2016/EEGNet.yaml @@ -47,7 +47,7 @@ cm: !name:sklearn.metrics.confusion_matrix metrics: f1: !ref acc: !ref - cm: !ref + # cm: !ref # TRAINING HPARS n_train_examples: 100 # it will be replaced in the train script # checkpoints to average diff --git a/benchmarks/MOABB/hparams/MotorImagery/Zhou2016/ShallowConvNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/Zhou2016/ShallowConvNet.yaml index 9d991936a..6a7402fe9 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/Zhou2016/ShallowConvNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/Zhou2016/ShallowConvNet.yaml @@ -47,7 +47,7 @@ cm: !name:sklearn.metrics.confusion_matrix metrics: f1: !ref acc: !ref - cm: !ref + # cm: !ref # TRAINING HPARS n_train_examples: 100 # it will be replaced in the train script # checkpoints to average From 2ae2eeb71abfb870d5defa21c4d78612946f5273 Mon Sep 17 00:00:00 2001 From: Jama Hussein Mohamud Date: Mon, 5 Aug 2024 14:34:51 -0400 Subject: [PATCH 3/6] add saving testset results --- benchmarks/MOABB/train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/MOABB/train.py b/benchmarks/MOABB/train.py index 0ea48fc11..dc02ea4e1 100644 --- a/benchmarks/MOABB/train.py +++ b/benchmarks/MOABB/train.py @@ -201,7 +201,7 @@ def on_stage_end(self, stage, stage_loss, epoch=None): elif stage == sb.Stage.TEST: # Log final test metrics to wandb - wandb.log({**self.last_eval_stats}) + wandb.log({"test_loss": self.last_eval_stats['loss'], "test_acc": self.last_eval_stats['acc'], "test_f1": self.last_eval_stats['f1']}) self.hparams.train_logger.log_stats( stats_meta={ "epoch loaded": self.hparams.epoch_counter.current From 6e563817806f6613244bcb2a1ae31ae02508b9d8 Mon Sep 17 00:00:00 2001 From: Jama Hussein Mohamud Date: Mon, 5 Aug 2024 19:05:49 -0400 Subject: [PATCH 4/6] fix bug in sweep run case --- benchmarks/MOABB/train.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/benchmarks/MOABB/train.py b/benchmarks/MOABB/train.py index dc02ea4e1..adc18fcea 100644 --- a/benchmarks/MOABB/train.py +++ b/benchmarks/MOABB/train.py @@ -27,6 +27,7 @@ import wandb import datetime +import uuid class MOABBBrain(sb.Brain): def init_model(self, model): @@ -412,7 +413,17 @@ def load_hparams_and_dataset_iterators(hparams_file, run_opts, overrides): # loading hparams for the each training and evaluation processes with open(hparams_file) as fin: hparams = load_hyperpyyaml(fin, overrides) - hparams["exp_dir"] = os.path.join(hparams["output_folder"], tail_path) + + if hparams["sweep_run"]: + # Generate a random identifier + unique_output_folder = os.path.join( + hparams["output_folder"], + f"{datetime.datetime.now().strftime('%Y%m%d_%H%M%S')}_{str(uuid.uuid4())}", + tail_path, + ) + hparams["exp_dir"] = unique_output_folder + else: + hparams["exp_dir"] = os.path.join(hparams["output_folder"], tail_path) # creating experiment directory sb.create_experiment_directory( From b7d50ff787db1fb28c8075568da9f4a912400f2e Mon Sep 17 00:00:00 2001 From: Jama Hussein Mohamud Date: Thu, 8 Aug 2024 12:21:30 -0400 Subject: [PATCH 5/6] updated num_epoch to 100 and removed its orion line --- benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/EEGNet.yaml | 2 +- .../MOABB/hparams/MotorImagery/BNCI2014001/ShallowConvNet.yaml | 2 +- benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/EEGNet.yaml | 2 +- .../MOABB/hparams/MotorImagery/BNCI2014004/ShallowConvNet.yaml | 2 +- benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/EEGNet.yaml | 2 +- .../MOABB/hparams/MotorImagery/BNCI2015001/ShallowConvNet.yaml | 2 +- benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/EEGNet.yaml | 2 +- .../MOABB/hparams/MotorImagery/Lee2019_MI/ShallowConvNet.yaml | 2 +- benchmarks/MOABB/hparams/MotorImagery/Zhou2016/EEGNet.yaml | 2 +- .../MOABB/hparams/MotorImagery/Zhou2016/ShallowConvNet.yaml | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/EEGNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/EEGNet.yaml index 204a72890..0e09edff4 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/EEGNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/EEGNet.yaml @@ -52,7 +52,7 @@ metrics: n_train_examples: 100 # it will be replaced in the train script # checkpoints to average avg_models: 10 # @orion_step1: --avg_models~"uniform(1, 15,discrete=True)" -number_of_epochs: 862 # @orion_step1: --number_of_epochs~"uniform(250, 1000, discrete=True)" +number_of_epochs: 100 lr: 0.0001 # @orion_step1: --lr~"choices([0.01, 0.005, 0.001, 0.0005, 0.0001])" # Learning rate scheduling (cyclic learning rate is used here) max_lr: !ref # Upper bound of the cycle (max value of the lr) diff --git a/benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/ShallowConvNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/ShallowConvNet.yaml index 40ad1b4a5..51ab180a7 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/ShallowConvNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/BNCI2014001/ShallowConvNet.yaml @@ -52,7 +52,7 @@ metrics: n_train_examples: 100 # it will be replaced in the train script # checkpoints to average avg_models: 1 # @orion_step1: --avg_models~"uniform(1, 15,discrete=True)" -number_of_epochs: 881 # @orion_step1: --number_of_epochs~"uniform(250, 1000, discrete=True)" +number_of_epochs: 100 lr: 0.001 # @orion_step1: --lr~"choices([0.01, 0.005, 0.001, 0.0005, 0.0001])" # Learning rate scheduling (cyclic learning rate is used here) max_lr: !ref # Upper bound of the cycle (max value of the lr) diff --git a/benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/EEGNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/EEGNet.yaml index 9f0ebce0f..07b485563 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/EEGNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/EEGNet.yaml @@ -52,7 +52,7 @@ metrics: n_train_examples: 100 # it will be replaced in the train script # checkpoints to average avg_models: 2 # @orion_step1: --avg_models~"uniform(1, 15,discrete=True)" -number_of_epochs: 796 # @orion_step1: --number_of_epochs~"uniform(250, 1000, discrete=True)" +number_of_epochs: 100 lr: 0.0001 # @orion_step1: --lr~"choices([0.01, 0.005, 0.001, 0.0005, 0.0001])" # Learning rate scheduling (cyclic learning rate is used here) max_lr: !ref # Upper bound of the cycle (max value of the lr) diff --git a/benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/ShallowConvNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/ShallowConvNet.yaml index a01347a52..b1510a8fd 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/ShallowConvNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/BNCI2014004/ShallowConvNet.yaml @@ -52,7 +52,7 @@ metrics: n_train_examples: 100 # it will be replaced in the train script # checkpoints to average avg_models: 11 # @orion_step1: --avg_models~"uniform(1, 15,discrete=True)" -number_of_epochs: 262 # @orion_step1: --number_of_epochs~"uniform(250, 1000, discrete=True)" +number_of_epochs: 100 lr: 0.005 # @orion_step1: --lr~"choices([0.01, 0.005, 0.001, 0.0005, 0.0001])" # Learning rate scheduling (cyclic learning rate is used here) max_lr: !ref # Upper bound of the cycle (max value of the lr) diff --git a/benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/EEGNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/EEGNet.yaml index fb3d8cb12..03229ba1d 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/EEGNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/EEGNet.yaml @@ -52,7 +52,7 @@ metrics: n_train_examples: 100 # it will be replaced in the train script # checkpoints to average avg_models: 12 # @orion_step1: --avg_models~"uniform(1, 15,discrete=True)" -number_of_epochs: 760 # @orion_step1: --number_of_epochs~"uniform(250, 1000, discrete=True)" +number_of_epochs: 100 lr: 0.001 # @orion_step1: --lr~"choices([0.01, 0.005, 0.001, 0.0005, 0.0001])" # Learning rate scheduling (cyclic learning rate is used here) max_lr: !ref # Upper bound of the cycle (max value of the lr) diff --git a/benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/ShallowConvNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/ShallowConvNet.yaml index cc7471af5..1c1fe385b 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/ShallowConvNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/BNCI2015001/ShallowConvNet.yaml @@ -52,7 +52,7 @@ metrics: n_train_examples: 100 # it will be replaced in the train script # checkpoints to average avg_models: 9 # @orion_step1: --avg_models~"uniform(1, 15,discrete=True)" -number_of_epochs: 976 # @orion_step1: --number_of_epochs~"uniform(250, 1000, discrete=True)" +number_of_epochs: 100 lr: 0.001 # @orion_step1: --lr~"choices([0.01, 0.005, 0.001, 0.0005, 0.0001])" # Learning rate scheduling (cyclic learning rate is used here) max_lr: !ref # Upper bound of the cycle (max value of the lr) diff --git a/benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/EEGNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/EEGNet.yaml index c8aeba6dd..a1040c0c6 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/EEGNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/EEGNet.yaml @@ -52,7 +52,7 @@ metrics: n_train_examples: 100 # it will be replaced in the train script # checkpoints to average avg_models: 15 # @orion_step1: --avg_models~"uniform(1, 15,discrete=True)" -number_of_epochs: 821 # @orion_step1: --number_of_epochs~"uniform(250, 1000, discrete=True)" +number_of_epochs: 100 lr: 0.001 # @orion_step1: --lr~"choices([0.01, 0.005, 0.001, 0.0005, 0.0001])" # Learning rate scheduling (cyclic learning rate is used here) max_lr: !ref # Upper bound of the cycle (max value of the lr) diff --git a/benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/ShallowConvNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/ShallowConvNet.yaml index 36e012512..5c6f55980 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/ShallowConvNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/Lee2019_MI/ShallowConvNet.yaml @@ -52,7 +52,7 @@ metrics: n_train_examples: 100 # it will be replaced in the train script # checkpoints to average avg_models: 12 # @orion_step1: --avg_models~"uniform(1, 15,discrete=True)" -number_of_epochs: 922 # @orion_step1: --number_of_epochs~"uniform(250, 1000, discrete=True)" +number_of_epochs: 100 lr: 0.001 # @orion_step1: --lr~"choices([0.01, 0.005, 0.001, 0.0005, 0.0001])" # Learning rate scheduling (cyclic learning rate is used here) max_lr: !ref # Upper bound of the cycle (max value of the lr) diff --git a/benchmarks/MOABB/hparams/MotorImagery/Zhou2016/EEGNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/Zhou2016/EEGNet.yaml index 38ca5208f..8c1ba7b5e 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/Zhou2016/EEGNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/Zhou2016/EEGNet.yaml @@ -52,7 +52,7 @@ metrics: n_train_examples: 100 # it will be replaced in the train script # checkpoints to average avg_models: 10 # @orion_step1: --avg_models~"uniform(1, 15,discrete=True)" -number_of_epochs: 741 # @orion_step1: --number_of_epochs~"uniform(250, 1000, discrete=True)" +number_of_epochs: 100 lr: 0.001 # @orion_step1: --lr~"choices([0.01, 0.005, 0.001, 0.0005, 0.0001])" # Learning rate scheduling (cyclic learning rate is used here) max_lr: !ref # Upper bound of the cycle (max value of the lr) diff --git a/benchmarks/MOABB/hparams/MotorImagery/Zhou2016/ShallowConvNet.yaml b/benchmarks/MOABB/hparams/MotorImagery/Zhou2016/ShallowConvNet.yaml index 6a7402fe9..60c4b88db 100644 --- a/benchmarks/MOABB/hparams/MotorImagery/Zhou2016/ShallowConvNet.yaml +++ b/benchmarks/MOABB/hparams/MotorImagery/Zhou2016/ShallowConvNet.yaml @@ -52,7 +52,7 @@ metrics: n_train_examples: 100 # it will be replaced in the train script # checkpoints to average avg_models: 13 # @orion_step1: --avg_models~"uniform(1, 15,discrete=True)" -number_of_epochs: 955 # @orion_step1: --number_of_epochs~"uniform(250, 1000, discrete=True)" +number_of_epochs: 100 lr: 0.001 # @orion_step1: --lr~"choices([0.01, 0.005, 0.001, 0.0005, 0.0001])" # Learning rate scheduling (cyclic learning rate is used here) max_lr: !ref # Upper bound of the cycle (max value of the lr) From 332f7ef9ec9615d83fced2d21248ef69fcc8ef13 Mon Sep 17 00:00:00 2001 From: Jama Hussein Mohamud Date: Fri, 9 Aug 2024 20:22:07 -0400 Subject: [PATCH 6/6] slurmn scripts for running hparam optimization for eegnet and shallownet --- .gitignore | 7 ++++- benchmarks/MOABB/run_hparam_opt_EEGNet.sh | 29 +++++++++++++++++++ benchmarks/MOABB/run_hparam_opt_shallownet.sh | 29 +++++++++++++++++++ 3 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 benchmarks/MOABB/run_hparam_opt_EEGNet.sh create mode 100644 benchmarks/MOABB/run_hparam_opt_shallownet.sh diff --git a/.gitignore b/.gitignore index 3012676d2..598282f54 100644 --- a/.gitignore +++ b/.gitignore @@ -154,9 +154,14 @@ dmypy.json # Results folders **/results/ **/data +**/wandb # Log folders **/log/ # Mac OS -.DS_Store \ No newline at end of file +.DS_Store + +EEGNet_BNCI2014001_* +ShallowConvNet_BNCI2014001_hopt* +finetune_* \ No newline at end of file diff --git a/benchmarks/MOABB/run_hparam_opt_EEGNet.sh b/benchmarks/MOABB/run_hparam_opt_EEGNet.sh new file mode 100644 index 000000000..082e2d078 --- /dev/null +++ b/benchmarks/MOABB/run_hparam_opt_EEGNet.sh @@ -0,0 +1,29 @@ +#!/bin/bash +#SBATCH --job-name=finetune +#SBATCH --output=finetune_output_%j.txt +#SBATCH --error=finetune_error_%j.txt +#SBATCH --ntasks=1 +#SBATCH --time=5:00:00 +#SBATCH --partition=unkillable +#SBATCH --cpus-per-task=6 +#SBATCH --mem=32G +#SBATCH --gres=gpu:a100l:1 + +bash run_hparam_optimization.sh --exp_name 'EEGNet_BNCI2014001_hopt' \ + --output_folder results/BNCI2014001/EEGNet/hopt \ + --data_folder data \ + --cached_data_folder data \ + --hparams hparams/MotorImagery/BNCI2014001/EEGNet.yaml \ + --nsbj_hpsearch 9 \ + --nsess_hpsearch 2 \ + --nsbj 9 \ + --nsess 2 \ + --nruns 1 \ + --nruns_eval 10 \ + --eval_metric acc \ + --train_mode leave-one-subject-out \ + --exp_max_trials 50 \ + --store_all True \ + --device 'cuda' \ + --project 'EEGNet_BNCI2014001_hopt' \ + --mode 'online' \ No newline at end of file diff --git a/benchmarks/MOABB/run_hparam_opt_shallownet.sh b/benchmarks/MOABB/run_hparam_opt_shallownet.sh new file mode 100644 index 000000000..f86f4032b --- /dev/null +++ b/benchmarks/MOABB/run_hparam_opt_shallownet.sh @@ -0,0 +1,29 @@ +#!/bin/bash +#SBATCH --job-name=finetune +#SBATCH --output=finetune_output_%j.txt +#SBATCH --error=finetune_error_%j.txt +#SBATCH --ntasks=1 +#SBATCH --time=10:00:00 +#SBATCH --partition=main +#SBATCH --cpus-per-task=8 +#SBATCH --mem=48G +#SBATCH --gres=gpu:a100l:1 + +bash run_hparam_optimization.sh --exp_name 'ShallowConvNet_BNCI2014001_hopt' \ + --output_folder results/BNCI2014001/ShallowConvNet/hopt \ + --data_folder data \ + --cached_data_folder data \ + --hparams hparams/MotorImagery/BNCI2014001/ShallowConvNet.yaml \ + --nsbj_hpsearch 9 \ + --nsess_hpsearch 2 \ + --nsbj 9 \ + --nsess 2 \ + --nruns 1 \ + --nruns_eval 10 \ + --eval_metric acc \ + --train_mode leave-one-subject-out \ + --exp_max_trials 50 \ + --store_all True \ + --device 'cuda' \ + --project 'ShallowConvNet_BNCI2014001_hopt' \ + --mode 'online' \ No newline at end of file