From d5fcb47f00d6712d74f424e5caa735534be2e0b8 Mon Sep 17 00:00:00 2001 From: Nick Hay <83538768+lasermanick@users.noreply.github.com> Date: Mon, 11 Jul 2022 21:55:41 +0100 Subject: [PATCH 01/30] Add tqdm status bars tqdm status bars added to the generative and supervised scripts --- scripts/generative.py | 9 +++++++-- scripts/supervised.py | 8 ++++++-- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/scripts/generative.py b/scripts/generative.py index 1d7e4c5..ac8123b 100644 --- a/scripts/generative.py +++ b/scripts/generative.py @@ -3,6 +3,9 @@ import torch +from tqdm import tqdm +from time import sleep + from pypc import utils from pypc import datasets from pypc import optim @@ -40,7 +43,8 @@ def main(cf): for epoch in range(1, cf.n_epochs + 1): print(f"\nTrain @ epoch {epoch} ({len(train_loader)} batches)") - for batch_id, (img_batch, label_batch) in enumerate(train_loader): + sleep(0.1) + for batch_id, (img_batch, label_batch) in enumerate(tqdm(train_loader)): model.train_batch_generative( img_batch, label_batch, cf.n_train_iters, fixed_preds=cf.fixed_preds_train ) @@ -53,8 +57,9 @@ def main(cf): if epoch % cf.test_every == 0: print(f"\nTest @ epoch {epoch}") + sleep(0.1) acc = 0 - for _, (img_batch, label_batch) in enumerate(test_loader): + for _, (img_batch, label_batch) in enumerate(tqdm(test_loader)): label_preds = model.test_batch_generative( img_batch, cf.n_test_iters, init_std=cf.init_std, fixed_preds=cf.fixed_preds_test ) diff --git a/scripts/supervised.py b/scripts/supervised.py index 2c33155..6a3641f 100644 --- a/scripts/supervised.py +++ b/scripts/supervised.py @@ -3,6 +3,9 @@ import torch +from tqdm import tqdm +from time import sleep + from pypc import utils from pypc import datasets from pypc import optim @@ -47,7 +50,8 @@ def main(cf): for epoch in range(1, cf.n_epochs + 1): print(f"\nTrain @ epoch {epoch} ({len(train_loader)} batches)") - for batch_id, (img_batch, label_batch) in enumerate(train_loader): + sleep(0.1) + for batch_id, (img_batch, label_batch) in enumerate(tqdm(train_loader)): model.train_batch_supervised( img_batch, label_batch, cf.n_train_iters, fixed_preds=cf.fixed_preds_train ) @@ -60,7 +64,7 @@ def main(cf): if epoch % cf.test_every == 0: acc = 0 - for _, (img_batch, label_batch) in enumerate(test_loader): + for _, (img_batch, label_batch) in enumerate(tqdm(test_loader)): label_preds = model.test_batch_supervised(img_batch) acc += datasets.accuracy(label_preds, label_batch) metrics["acc"].append(acc / len(test_loader)) From 201a2d20d8c2214b8bfbc427f82c89e94599d06a Mon Sep 17 00:00:00 2001 From: Nick Hay <83538768+lasermanick@users.noreply.github.com> Date: Thu, 14 Jul 2022 14:07:42 +0100 Subject: [PATCH 02/30] _get_transform() bug Original did not add the normalization transform to the transform list (so MNIST images were not normalised - only scaled to range [0,1] by ToTensor()). Fix is to replace '+' with '+='. --- pypc/datasets.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pypc/datasets.py b/pypc/datasets.py index c6ed966..55f7dec 100644 --- a/pypc/datasets.py +++ b/pypc/datasets.py @@ -157,9 +157,16 @@ def _preprocess_batch(batch): def _get_transform(normalize=True, mean=(0.5), std=(0.5)): + """ + Define transformation to convert PIL image or numpy.ndarray to tensor with optional normalization + :param normalize: True or False + :param mean: Transformed mean + :param std: Transformed std dev + :return: Transformation + """ transform = [transforms.ToTensor()] if normalize: - transform + [transforms.Normalize(mean=mean, std=std)] + transform += [transforms.Normalize(mean=mean, std=std)] return transforms.Compose(transform) From ea23d43fc84aab2384a65109315d220841240071 Mon Sep 17 00:00:00 2001 From: Nick Hay <83538768+lasermanick@users.noreply.github.com> Date: Thu, 14 Jul 2022 15:48:50 +0100 Subject: [PATCH 03/30] Change _get_transform() defaults Change default normalization and correct documentation --- pypc/datasets.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pypc/datasets.py b/pypc/datasets.py index 55f7dec..8e63e39 100644 --- a/pypc/datasets.py +++ b/pypc/datasets.py @@ -156,12 +156,13 @@ def _preprocess_batch(batch): return (batch[0], batch[1]) -def _get_transform(normalize=True, mean=(0.5), std=(0.5)): +def _get_transform(normalize=True, mean=0.0, std=1.0): """ Define transformation to convert PIL image or numpy.ndarray to tensor with optional normalization + :param normalize: True or False - :param mean: Transformed mean - :param std: Transformed std dev + :param mean: Input mean (after scaling to range [0,1]) + :param std: Input std dev (after scaling to range [0,1]) :return: Transformation """ transform = [transforms.ToTensor()] From aa00599aecfd4380b4f547c9c1d12f555fa1d0dd Mon Sep 17 00:00:00 2001 From: Nick Hay <83538768+lasermanick@users.noreply.github.com> Date: Fri, 15 Jul 2022 12:22:50 +0100 Subject: [PATCH 04/30] Document the MNIST class --- pypc/datasets.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/pypc/datasets.py b/pypc/datasets.py index 8e63e39..25d7fcb 100644 --- a/pypc/datasets.py +++ b/pypc/datasets.py @@ -9,13 +9,30 @@ class MNIST(datasets.MNIST): def __init__(self, train, size=None, scale=None, normalize=False): - transform = _get_transform(normalize=normalize, mean=(0.1307), std=(0.3081)) + """ + Load MNIST dataset, scale to range [0,1], optionally normalise with mean = 0 and std dev = 1, + optionally set label scale factor, optionally limit size of dataset + + :param train: True for training data, False for test data + :param size: Number of samples to keep in the dataset + :param scale: Scale factor for one-hot labels (e.g. scale=1 => 0 or 1, scale=0.5 => 0.25 or 0.75, + scale=0.1 => 0.45 or 0.55) + :param normalize: True to normalize + """ + transform = _get_transform(normalize=normalize, mean=(0.1307), std=(0.3081)) # Transform to mean=0, std=1 super().__init__("./data/mnist", download=True, transform=transform, train=train) self.scale = scale if size is not None: self._reduce(size) def __getitem__(self, index): + """ + Return image (data) and label (target) with image converted from 28x28 to 784x1 and label converted to one-hot + encoding (optionally scaled) + + :param index: Index + :return: image, label + """ data, target = super().__getitem__(index) data = _to_vector(data) target = _one_hot(target) @@ -24,6 +41,11 @@ def __getitem__(self, index): return data, target def _reduce(self, size): + """ + Crop the dataset + + :param size: Maximum sample number to retain + """ self.data = self.data[0:size] self.targets = self.targets[0:size] From 932bf899bada93832bd756a42d792638ab147f43 Mon Sep 17 00:00:00 2001 From: Nick Hay <83538768+lasermanick@users.noreply.github.com> Date: Mon, 18 Jul 2022 09:21:27 +0100 Subject: [PATCH 05/30] datasets.py documentation Added/edited doc strings --- pypc/datasets.py | 40 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/pypc/datasets.py b/pypc/datasets.py index 25d7fcb..ab7ce62 100644 --- a/pypc/datasets.py +++ b/pypc/datasets.py @@ -27,8 +27,8 @@ def __init__(self, train, size=None, scale=None, normalize=False): def __getitem__(self, index): """ - Return image (data) and label (target) with image converted from 28x28 to 784x1 and label converted to one-hot - encoding (optionally scaled) + Return image (data) and label (target) with image converted from (1,28,28) to (784,) and label converted to + one-hot encoding (optionally scaled) :param index: Index :return: image, label @@ -147,6 +147,14 @@ def _reduce(self, size): def get_dataloader(dataset, batch_size): + """ + Create PyTorch DataLoader for given dataset and batch size, perform pre-processing to move data onto the selected + cpu/cuda device with dtype=torch.float32, and return a list containing samples and labels + + :param dataset: PyTorch Dataset + :param batch_size: Batch size + :return: List of tuples with index 0 containing samples and index 1 containing labels + """ dataloader = data.DataLoader(dataset, batch_size, shuffle=True, drop_last=True) return list(map(_preprocess_batch, dataloader)) @@ -173,6 +181,12 @@ def plot_imgs(img_preds, path): def _preprocess_batch(batch): + """ + Pre-process a batch to move data onto the selected cpu/cuda device with dtype=torch.float32 + + :param batch: List of Tensor objects with index 0 containing samples and index 1 containing labels + :return: Pre-processed batch as tuple of Tensor objects + """ batch[0] = utils.set_tensor(batch[0]) batch[1] = utils.set_tensor(batch[1]) return (batch[0], batch[1]) @@ -194,14 +208,36 @@ def _get_transform(normalize=True, mean=0.0, std=1.0): def _one_hot(labels, n_classes=10): + """ + Convert categorical label to one-hot encoding (trick is to index an identity matrix) NOTE: Only used for individual + labels so consider changing parameter name to 'label') + + :param labels: Categorical label + :param n_classes: Number of classes (categories) + :return: One-hot encoded label + """ arr = torch.eye(n_classes) return arr[labels] def _scale(targets, factor): + """ + Scale one-hot labels (targets) according to: + scaled = 0.5 + factor x (original - 0.5) + (e.g. scale=1 => 0 or 1, scale=0.5 => 0.25 or 0.75, scale=0.1 => 0.45 or 0.55) + + :param targets: Labels + :param factor: Scale factor + :return: Scaled labels + """ return targets * factor + 0.5 * (1 - factor) * torch.ones_like(targets) def _to_vector(batch): + """ + Convert batch of 2D images to vector format NOTE: Currently only used for single images so naming is confusing + :param batch: Image or batch of images + :return: Image or batch of images in vector format + """ batch_size = batch.size(0) return batch.reshape(batch_size, -1).squeeze() From 11c76a66e15a088de0a2d35ad1c874570504574c Mon Sep 17 00:00:00 2001 From: Nick Hay <83538768+lasermanick@users.noreply.github.com> Date: Mon, 18 Jul 2022 11:52:48 +0100 Subject: [PATCH 06/30] Remove redundancies layers.py Remove redundant initialisation code and unused import (and add one doc string) --- pypc/layers.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/pypc/layers.py b/pypc/layers.py index de7b062..e1cfd52 100644 --- a/pypc/layers.py +++ b/pypc/layers.py @@ -1,7 +1,6 @@ import math import torch import numpy as np -from copy import deepcopy from torch import nn import torch.nn.functional as F from pypc import utils @@ -11,6 +10,16 @@ class Layer(nn.Module): def __init__( self, in_size, out_size, act_fn, use_bias=False, kaiming_init=False, is_forward=False ): + """ + Initialise layer + + :param in_size: Number of input nodes + :param out_size: Number of output nodes + :param act_fn: Activation function + :param use_bias: Include bias terms? + :param kaiming_init: Use Kaiming weight initialisation? + :param is_forward: + """ super().__init__() self.in_size = in_size self.out_size = out_size @@ -19,14 +28,8 @@ def __init__( self.is_forward = is_forward self.kaiming_init = kaiming_init - self.weights = None - self.bias = None - self.grad = {"weights": None, "bias": None} - - if kaiming_init: - self._reset_params_kaiming() - else: - self._reset_params() + self._reset_grad() + self.reset() def forward(self, *args, **kwargs): raise NotImplementedError @@ -66,7 +69,6 @@ def __init__( self, in_size, out_size, act_fn, use_bias=False, kaiming_init=False, is_forward=False ): super().__init__(in_size, out_size, act_fn, use_bias, kaiming_init, is_forward=is_forward) - self.use_bias = use_bias self.inp = None def forward(self, inp): From 6b238c84798fe0974ce0f9ee74b70e982d95acee Mon Sep 17 00:00:00 2001 From: Nick Hay <83538768+lasermanick@users.noreply.github.com> Date: Tue, 9 Aug 2022 20:38:31 +0100 Subject: [PATCH 07/30] Document FCLayer class Add doc strings to FCLayer methods --- pypc/layers.py | 47 ++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/pypc/layers.py b/pypc/layers.py index e1cfd52..990bd19 100644 --- a/pypc/layers.py +++ b/pypc/layers.py @@ -32,24 +32,43 @@ def __init__( self.reset() def forward(self, *args, **kwargs): + """ + Abstract forward pass method. Must be overridden. + + :param args: + :param kwargs: + """ raise NotImplementedError def reset(self): + """ + Reset weights and biases + """ if self.kaiming_init: self._reset_params_kaiming() else: self._reset_params() def _reset_grad(self): + """ + Reset gradients + """ self.grad = {"weights": None, "bias": None} def _reset_params(self): + """ + Reset weights to be normally distributed with mean=0.0 and std dev 0.05 and reset biases to zero + """ weights = torch.empty((self.in_size, self.out_size)).normal_(mean=0.0, std=0.05) bias = torch.zeros((self.out_size)) self.weights = utils.set_tensor(weights) self.bias = utils.set_tensor(bias) def _reset_params_kaiming(self): + """ + Reset weights using Kaiming He initialisation and reset biases to be uniformly distributed about zero + NOTE: He et al (2015) arXiv:1502.01852v1 initialise biases to zero + """ self.weights = utils.set_tensor(torch.empty((self.in_size, self.out_size))) self.bias = utils.set_tensor(torch.zeros((self.out_size))) if isinstance(self.act_fn, utils.Linear): @@ -68,10 +87,26 @@ class FCLayer(Layer): def __init__( self, in_size, out_size, act_fn, use_bias=False, kaiming_init=False, is_forward=False ): + """ + Initialise fully connected layer + + :param in_size: Number of input nodes + :param out_size: Number of output nodes + :param act_fn: Activation function + :param use_bias: Include bias terms? + :param kaiming_init: Use Kaiming weight initialisation? + :param is_forward: + """ super().__init__(in_size, out_size, act_fn, use_bias, kaiming_init, is_forward=is_forward) self.inp = None def forward(self, inp): + """ + Perform forward pass for batch + + :param inp: Node inputs for batch, Tensor:(batch_size, in_size) + :return: Node outputs for batch, Tensor:(batch_size, out_size) + """ self.inp = inp.clone() out = self.act_fn(torch.matmul(self.inp, self.weights)) if self.use_bias: @@ -79,11 +114,22 @@ def forward(self, inp): return out def backward(self, err): + """ + Perform backward pass for batch + + :param err: Errors for batch, Tensor:(batch_size, out_size) + :return: Tensor:(batch_size, in_size) + """ fn_deriv = self.act_fn.deriv(torch.matmul(self.inp, self.weights)) out = torch.matmul(err * fn_deriv, self.weights.T) return out def update_gradient(self, err): + """ + Update gradients for weights and biases + + :param err: Errors for batch, Tensor:(batch_size, out_size) + """ fn_deriv = self.act_fn.deriv(torch.matmul(self.inp, self.weights)) delta = torch.matmul(self.inp.T, err * fn_deriv) self.grad["weights"] = delta @@ -91,7 +137,6 @@ def update_gradient(self, err): self.grad["bias"] = torch.sum(err, axis=0) - class ConvLayer(Layer): def __init__(self,input_size,num_channels,num_filters,batch_size,kernel_size,learning_rate,act_fn,padding=0,stride=1,device="cpu"): self.input_size = input_size From b7ee05695864af369129fab9cd5dab0d3c71e2e8 Mon Sep 17 00:00:00 2001 From: Nick Hay <83538768+lasermanick@users.noreply.github.com> Date: Tue, 9 Aug 2022 20:39:44 +0100 Subject: [PATCH 08/30] Document PCModel class Add docstrings and other comments to PCModel class. Also remove one redundant import. --- pypc/models.py | 91 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 84 insertions(+), 7 deletions(-) diff --git a/pypc/models.py b/pypc/models.py index 541b995..6900ce4 100644 --- a/pypc/models.py +++ b/pypc/models.py @@ -1,4 +1,3 @@ -import numpy as np import torch from pypc import utils @@ -7,6 +6,17 @@ class PCModel(object): def __init__(self, nodes, mu_dt, act_fn, use_bias=False, kaiming_init=False): + """ + Define the Predictive Coding PyTorch model. All layers fully connected. All nodes using the specified activation + function except for the output layer which is linear. Bias terms are optional. Kaiming weight initialisation is + optional. + + :param nodes: List of number of nodes in each layer + :param mu_dt: + :param act_fn: Activation function + :param use_bias: Include bias terms? + :param kaiming_init: Use Kaiming weight initialisation? + """ self.nodes = nodes self.mu_dt = mu_dt @@ -27,6 +37,9 @@ def __init__(self, nodes, mu_dt, act_fn, use_bias=False, kaiming_init=False): self.layers.append(layer) def reset(self): + """ + Initialise predictions (preds), errors (errs), and variational means (mus) to empty lists + """ self.preds = [[] for _ in range(self.n_nodes)] self.errs = [[] for _ in range(self.n_nodes)] self.mus = [[] for _ in range(self.n_nodes)] @@ -38,9 +51,19 @@ def reset_mus(self, batch_size, init_std): ) def set_input(self, inp): + """ + Set input node mus for batch to input values + + :param inp: Input values, Tensor:(batch_size, nodes[0]) + """ self.mus[0] = inp.clone() def set_target(self, target): + """ + Set output node mus for batch to target values + + :param target: Target values, Tensor:(batch_size, nodes[-1]) + """ self.mus[-1] = target.clone() def forward(self, val): @@ -49,18 +72,38 @@ def forward(self, val): return val def propagate_mu(self): + """ + Perform forward pass for batch, update mus for all nodes except inputs and outputs (targets) + """ for l in range(1, self.n_layers): self.mus[l] = self.layers[l - 1].forward(self.mus[l - 1]) def train_batch_supervised(self, img_batch, label_batch, n_iters, fixed_preds=False): - self.reset() - self.set_input(img_batch) - self.propagate_mu() - self.set_target(label_batch) - self.train_updates(n_iters, fixed_preds=fixed_preds) - self.update_grads() + """ + Train the model using the (mini)batch images as inputs and labels as targets + + :param img_batch: Batch of input images, Tensor:(batch_size, nodes[0]) + :param label_batch: Batch of target labels, Tensor:(batch_size, nodes[-1]) + :param n_iters: Number of training iterations + :param fixed_preds: Fix predictions at initial values? + """ + self.reset() # Initialise the prediction, error, and mu data structures + self.set_input(img_batch) # Set the model inputs, mus[0], equal to the training *images* + self.propagate_mu() # Perform forward pass, update mus for all nodes except inputs and targets + self.set_target(label_batch) # Set the model outputs (targets), mus[-1], equal to the training *labels* + self.train_updates(n_iters, fixed_preds=fixed_preds) # Iteratively update mus, predictions and errors + self.update_grads() # Calculate gradients of weights and biases for all layers def train_batch_generative(self, img_batch, label_batch, n_iters, fixed_preds=False): + """ + Train the model using the (mini)batch labels as inputs and images as targets + (Identical to train_batch_supervised() but inputs and targets are swapped) + + :param img_batch: Batch of target images, Tensor:(batch_size, nodes[-1]) + :param label_batch: Batch of input labels, Tensor:(batch_size, nodes[0]) + :param n_iters: Number of training iterations + :param fixed_preds: Fix predictions at initial values? + """ self.reset() self.set_input(label_batch) self.propagate_mu() @@ -80,25 +123,43 @@ def test_batch_generative(self, img_batch, n_iters, init_std=0.05, fixed_preds=F return self.mus[0] def train_updates(self, n_iters, fixed_preds=False): + """ + Iteratively update mus, predictions and errors + + :param n_iters: Number of training iterations + :param fixed_preds: Fix predictions at initial values? + """ + # For batch, initialise predictions and errors for all nodes except inputs for n in range(1, self.n_nodes): self.preds[n] = self.layers[n - 1].forward(self.mus[n - 1]) self.errs[n] = self.mus[n] - self.preds[n] + # For each training iteration for itr in range(n_iters): + # For batch, update mus for all nodes except inputs and outputs for l in range(1, self.n_layers): delta = self.layers[l].backward(self.errs[l + 1]) - self.errs[l] self.mus[l] = self.mus[l] + self.mu_dt * delta + # For batch, update errors and (optionally) predictions for all nodes except inputs for n in range(1, self.n_nodes): if not fixed_preds: self.preds[n] = self.layers[n - 1].forward(self.mus[n - 1]) self.errs[n] = self.mus[n] - self.preds[n] def test_updates(self, n_iters, fixed_preds): + """ + Test model + + :param n_iters: Number of training iterations + :param fixed_preds: Fix predictions at initial values? + """ + # For batch, initialise predictions and errors for all nodes except inputs for n in range(1, self.n_nodes): self.preds[n] = self.layers[n - 1].forward(self.mus[n - 1]) self.errs[n] = self.mus[n] - self.preds[n] + # For each test iteration for itr in range(n_iters): delta = self.layers[0].backward(self.errs[1]) self.mus[0] = self.mus[0] + self.mu_dt * delta @@ -112,13 +173,29 @@ def test_updates(self, n_iters, fixed_preds): self.errs[n] = self.mus[n] - self.preds[n] def update_grads(self): + """ + Calculate gradients of weights and biases for all layers + + """ for l in range(self.n_layers): self.layers[l].update_gradient(self.errs[l + 1]) def get_target_loss(self): + """ + Calculate loss as the sum of the squares of the target errors + (Not currently used) + + :return: Loss + """ return torch.sum(self.errs[-1] ** 2).item() @property def params(self): + """ + Allows controlled and standardised access to model parameters (for passing to an optimizer). + Currently of limited use but could be expanded. + + :return: Model layers + """ return self.layers From 643efaf9d2b4fc9ccdc12336393f0c5dc02fdd36 Mon Sep 17 00:00:00 2001 From: Nick Hay <83538768+lasermanick@users.noreply.github.com> Date: Tue, 9 Aug 2022 20:40:22 +0100 Subject: [PATCH 09/30] Document utils.py Add some doc strings to utils.py --- pypc/utils.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pypc/utils.py b/pypc/utils.py index 7ab9d56..e6a845a 100644 --- a/pypc/utils.py +++ b/pypc/utils.py @@ -49,6 +49,11 @@ def deriv(self, inp): def seed(seed): + """ + Set seeds for relevant pseudorandom number generators + + :param seed: Seed value + """ torch.manual_seed(seed) if torch.cuda.is_available(): torch.cuda.manual_seed_all(seed) @@ -57,6 +62,12 @@ def seed(seed): def set_tensor(tensor): + """ + Move data onto the selected cpu/cuda device with dtype=torch.float32 + + :param tensor: Tensor object + :return: Tensor object allocated to selected device with dtype=torch.float32 + """ return tensor.to(DEVICE).float() @@ -65,6 +76,12 @@ def flatten_array(array): def save_json(obj, path): + """ + Save an object as a json file + + :param obj: Object to be saved + :param path: File path + """ with open(path, "w") as file: json.dump(obj, file) From 749eb3227674a825ec2fd1b3337951b53ac2e94b Mon Sep 17 00:00:00 2001 From: Nick Hay <83538768+lasermanick@users.noreply.github.com> Date: Tue, 9 Aug 2022 20:42:28 +0100 Subject: [PATCH 10/30] Update generative.py Add some comments, minor reformatting, add option to disable tqdm progress bars, change some of the parameters (with original values added as comments) --- scripts/generative.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/scripts/generative.py b/scripts/generative.py index ac8123b..a988d15 100644 --- a/scripts/generative.py +++ b/scripts/generative.py @@ -27,7 +27,11 @@ def main(cf): print(f"Loaded data [train batches: {len(train_loader)} test batches: {len(test_loader)}]") model = PCModel( - nodes=cf.nodes, mu_dt=cf.mu_dt, act_fn=cf.act_fn, use_bias=cf.use_bias, kaiming_init=cf.kaiming_init + nodes=cf.nodes, + mu_dt=cf.mu_dt, + act_fn=cf.act_fn, + use_bias=cf.use_bias, + kaiming_init=cf.kaiming_init, ) optimizer = optim.get_optim( model.params, @@ -38,13 +42,13 @@ def main(cf): weight_decay=cf.weight_decay, ) - with torch.no_grad(): + with torch.no_grad(): # Disable automatic gradient calculation metrics = {"acc": []} for epoch in range(1, cf.n_epochs + 1): print(f"\nTrain @ epoch {epoch} ({len(train_loader)} batches)") sleep(0.1) - for batch_id, (img_batch, label_batch) in enumerate(tqdm(train_loader)): + for batch_id, (img_batch, label_batch) in enumerate(tqdm(train_loader, disable=False)): model.train_batch_generative( img_batch, label_batch, cf.n_train_iters, fixed_preds=cf.fixed_preds_train ) @@ -59,7 +63,7 @@ def main(cf): print(f"\nTest @ epoch {epoch}") sleep(0.1) acc = 0 - for _, (img_batch, label_batch) in enumerate(tqdm(test_loader)): + for _, (img_batch, label_batch) in enumerate(tqdm(test_loader, disable=False)): label_preds = model.test_batch_generative( img_batch, cf.n_test_iters, init_std=cf.init_std, fixed_preds=cf.fixed_preds_test ) @@ -75,23 +79,23 @@ def main(cf): if __name__ == "__main__": - cf = utils.AttrDict() - cf.seeds = [0] + cf = utils.AttrDict() # Create configuration + cf.seeds = [0] # Create list of >=1 seeds for repeat runs for seed in cf.seeds: # experiment params cf.seed = seed - cf.n_epochs = 20 + cf.n_epochs = 2 # 20 cf.test_every = 1 cf.logdir = f"data/generative/{seed}/" cf.imgdir = cf.logdir + "imgs/" # dataset params - cf.train_size = None - cf.test_size = None + cf.train_size = 6000 # None + cf.test_size = 1000 # None cf.label_scale = None - cf.normalize = True + cf.normalize = False # True # optim params cf.optim = "Adam" From 2b33f5466309009b90a2a30680b30830b41ceb58 Mon Sep 17 00:00:00 2001 From: Nick Hay <83538768+lasermanick@users.noreply.github.com> Date: Tue, 9 Aug 2022 20:44:28 +0100 Subject: [PATCH 11/30] Update supervised.py Minor reformatting to be consistent with generative.py , add option to disable tqdm progress bars, change one parameter (with original value added as comment) --- scripts/supervised.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/scripts/supervised.py b/scripts/supervised.py index 6a3641f..1300776 100644 --- a/scripts/supervised.py +++ b/scripts/supervised.py @@ -19,12 +19,8 @@ def main(cf): utils.seed(cf.seed) utils.save_json({k: str(v) for (k, v) in cf.items()}, cf.logdir + "config.json") - train_dataset = datasets.MNIST( - train=True, scale=cf.label_scale, size=cf.train_size, normalize=cf.normalize - ) - test_dataset = datasets.MNIST( - train=False, scale=cf.label_scale, size=cf.test_size, normalize=cf.normalize - ) + train_dataset = datasets.MNIST(train=True, scale=cf.label_scale, size=cf.train_size, normalize=cf.normalize) + test_dataset = datasets.MNIST(train=False, scale=cf.label_scale, size=cf.test_size, normalize=cf.normalize) train_loader = datasets.get_dataloader(train_dataset, cf.batch_size) test_loader = datasets.get_dataloader(test_dataset, cf.batch_size) print(f"Loaded data [train batches: {len(train_loader)} test batches: {len(test_loader)}]") @@ -51,7 +47,7 @@ def main(cf): print(f"\nTrain @ epoch {epoch} ({len(train_loader)} batches)") sleep(0.1) - for batch_id, (img_batch, label_batch) in enumerate(tqdm(train_loader)): + for batch_id, (img_batch, label_batch) in enumerate(tqdm(train_loader, disable=False)): model.train_batch_supervised( img_batch, label_batch, cf.n_train_iters, fixed_preds=cf.fixed_preds_train ) @@ -64,7 +60,7 @@ def main(cf): if epoch % cf.test_every == 0: acc = 0 - for _, (img_batch, label_batch) in enumerate(tqdm(test_loader)): + for _, (img_batch, label_batch) in enumerate(tqdm(test_loader, disable=False)): label_preds = model.test_batch_supervised(img_batch) acc += datasets.accuracy(label_preds, label_batch) metrics["acc"].append(acc / len(test_loader)) @@ -107,7 +103,7 @@ def main(cf): # model params cf.use_bias = True - cf.kaiming_init = False + cf.kaiming_init = True # False cf.nodes = [784, 300, 100, 10] cf.act_fn = utils.ReLU() From 5e7f398808bfcadf334e7da834f42b22c1dcd18a Mon Sep 17 00:00:00 2001 From: Nick Hay <83538768+lasermanick@users.noreply.github.com> Date: Tue, 20 Sep 2022 13:52:15 +0100 Subject: [PATCH 12/30] Comment out unused layer classes Comment out unused (not implemented) layer classes and associated imports --- pypc/layers.py | 304 ++++++++++++++++++++++++------------------------- 1 file changed, 152 insertions(+), 152 deletions(-) diff --git a/pypc/layers.py b/pypc/layers.py index 990bd19..a0be318 100644 --- a/pypc/layers.py +++ b/pypc/layers.py @@ -1,8 +1,8 @@ import math import torch -import numpy as np +# import numpy as np from torch import nn -import torch.nn.functional as F +# import torch.nn.functional as F from pypc import utils @@ -137,153 +137,153 @@ def update_gradient(self, err): self.grad["bias"] = torch.sum(err, axis=0) -class ConvLayer(Layer): - def __init__(self,input_size,num_channels,num_filters,batch_size,kernel_size,learning_rate,act_fn,padding=0,stride=1,device="cpu"): - self.input_size = input_size - self.num_channels = num_channels - self.num_filters = num_filters - self.batch_size = batch_size - self.kernel_size = kernel_size - self.padding = padding - self.stride = stride - self.output_size = math.floor((self.input_size + (2 * self.padding) - self.kernel_size)/self.stride) +1 - self.learning_rate = learning_rate - self.act_fn - self.device = device - self.kernel= torch.empty(self.num_filters,self.num_channels,self.kernel_size,self.kernel_size).normal_(mean=0,std=0.05).to(self.device) - self.unfold = nn.Unfold(kernel_size=(self.kernel_size,self.kernel_size),padding=self.padding,stride=self.stride).to(self.device) - self.fold = nn.Fold(output_size=(self.input_size,self.input_size),kernel_size=(self.kernel_size,self.kernel_size),padding=self.padding,stride=self.stride).to(self.device) - - def forward(self,inp): - self.X_col = self.unfold(inp.clone()) - self.flat_weights = self.kernel.reshape(self.num_filters,-1) - out = self.flat_weights @ self.X_col - self.activations = out.reshape(self.batch_size, self.num_filters, self.output_size, self.output_size) - return self.f(self.activations) - - def update_gradient(self,e): - fn_deriv = self.act_fn.deriv(self.activations) - e = e * fn_deriv - self.dout = e.reshape(self.batch_size,self.num_filters,-1) - dW = self.dout @ self.X_col.permute(0,2,1) - dW = torch.sum(dW,dim=0) - dW = dW.reshape((self.num_filters,self.num_channels,self.kernel_size,self.kernel_size)) - self.grad["weights"] = dW - - def backward(self,e): - fn_deriv = self.act_fn.deriv(self.activations) - e = e * fn_deriv - self.dout = e.reshape(self.batch_size,self.num_filters,-1) - dX_col = self.flat_weights.T @ self.dout - dX = self.fold(dX_col) - return torch.clamp(dX,-50,50) - - def get_true_weight_grad(self): - return self.kernel.grad - - def set_weight_parameters(self): - self.kernel = nn.Parameter(self.kernel) - - def save_layer(self,logdir,i): - np.save(logdir +"/layer_"+str(i)+"_weights.npy",self.kernel.detach().cpu().numpy()) - - def load_layer(self,logdir,i): - kernel = np.load(logdir +"/layer_"+str(i)+"_weights.npy") - self.kernel = set_tensor(torch.from_numpy(kernel)) - -class MaxPool(Layer): - def __init__(self, kernel_size,device='cpu'): - self.kernel_size = kernel_size - self.device = device - self.activations = torch.empty(1) - self.weights = torch.zeros((1,1)).to(self.device) - - def forward(self,x): - out, self.idxs = F.max_pool2d(x, self.kernel_size,return_indices=True) - return out - - def backward(self, y): - return F.max_unpool2d(y,self.idxs, self.kernel_size) - - def update_gradient(self,e,update_weights=False,sign_reverse=False): - self.grad["weights"] = torch.zeros_like(self.weights) - - def get_true_weight_grad(self): - return None - - def set_weight_parameters(self): - pass - - def save_layer(self,logdir,i): - pass - - def load_layer(self,logdir,i): - pass - -class AvgPool(Layer): - def __init__(self, kernel_size,device='cpu'): - self.kernel_size = kernel_size - self.device = device - self.activations = torch.empty(1) - self.weights = torch.zeros((1,1)).to(self.device) - - - def forward(self, x): - self.B_in,self.C_in,self.H_in,self.W_in = x.shape - return F.avg_pool2d(x,self.kernel_size) - - def backward(self, y): - N,C,H,W = y.shape - print("in backward: ", y.shape) - return F.interpolate(y,scale_factor=(1,1,self.kernel_size,self.kernel_size)) - - def update_gradient(self,e,update_weights=False, sign_reverse=False): - self.grad["weights"] = torch.zeros_like(self.weights) - - def save_layer(self,logdir,i): - pass - - def load_layer(self,logdir,i): - pass - -class ProjectionLayer(Layer): - def __init__(self,input_size, output_size,act_fn,learning_rate,device='cpu'): - self.input_size = input_size - self.B, self.C, self.H, self.W = self.input_size - self.output_size =output_size - self.learning_rate = learning_rate - self.act_fn = act_fn - self.device = device - self.Hid = self.C * self.H * self.W - self.weights = torch.empty((self.Hid, self.output_size)).normal_(mean=0.0, std=0.05).to(self.device) - - def forward(self, x): - self.inp = x.detach().clone() - out = x.reshape((len(x), -1)) - self.activations = torch.matmul(out,self.weights) - return self.f(self.activations) - - def backward(self, e): - fn_deriv = self.act_fn.deriv(self.activations) - out = torch.matmul(e * fn_deriv, self.weights.T) - out = out.reshape((len(e), self.C, self.H, self.W)) - return torch.clamp(out,-50,50) - - def update_gradient(self, e,update_weights=False,sign_reverse=False): - out = self.inp.reshape((len(self.inp), -1)) - fn_deriv = self.act_fn.deriv(self.activations) - dw = torch.matmul(out.T, e * fn_deriv) - self.grad["weights"] = dw - - def get_true_weight_grad(self): - return self.weights.grad - - def set_weight_parameters(self): - self.weights = nn.Parameter(self.weights) - - def save_layer(self,logdir,i): - np.save(logdir +"/layer_"+str(i)+"_weights.npy",self.weights.detach().cpu().numpy()) - - def load_layer(self,logdir,i): - weights = np.load(logdir +"/layer_"+str(i)+"_weights.npy") - self.weights = set_tensor(torch.from_numpy(weights)) +# class ConvLayer(Layer): +# def __init__(self,input_size,num_channels,num_filters,batch_size,kernel_size,learning_rate,act_fn,padding=0,stride=1,device="cpu"): +# self.input_size = input_size +# self.num_channels = num_channels +# self.num_filters = num_filters +# self.batch_size = batch_size +# self.kernel_size = kernel_size +# self.padding = padding +# self.stride = stride +# self.output_size = math.floor((self.input_size + (2 * self.padding) - self.kernel_size)/self.stride) +1 +# self.learning_rate = learning_rate +# self.act_fn +# self.device = device +# self.kernel= torch.empty(self.num_filters,self.num_channels,self.kernel_size,self.kernel_size).normal_(mean=0,std=0.05).to(self.device) +# self.unfold = nn.Unfold(kernel_size=(self.kernel_size,self.kernel_size),padding=self.padding,stride=self.stride).to(self.device) +# self.fold = nn.Fold(output_size=(self.input_size,self.input_size),kernel_size=(self.kernel_size,self.kernel_size),padding=self.padding,stride=self.stride).to(self.device) +# +# def forward(self,inp): +# self.X_col = self.unfold(inp.clone()) +# self.flat_weights = self.kernel.reshape(self.num_filters,-1) +# out = self.flat_weights @ self.X_col +# self.activations = out.reshape(self.batch_size, self.num_filters, self.output_size, self.output_size) +# return self.f(self.activations) +# +# def update_gradient(self,e): +# fn_deriv = self.act_fn.deriv(self.activations) +# e = e * fn_deriv +# self.dout = e.reshape(self.batch_size,self.num_filters,-1) +# dW = self.dout @ self.X_col.permute(0,2,1) +# dW = torch.sum(dW,dim=0) +# dW = dW.reshape((self.num_filters,self.num_channels,self.kernel_size,self.kernel_size)) +# self.grad["weights"] = dW +# +# def backward(self,e): +# fn_deriv = self.act_fn.deriv(self.activations) +# e = e * fn_deriv +# self.dout = e.reshape(self.batch_size,self.num_filters,-1) +# dX_col = self.flat_weights.T @ self.dout +# dX = self.fold(dX_col) +# return torch.clamp(dX,-50,50) +# +# def get_true_weight_grad(self): +# return self.kernel.grad +# +# def set_weight_parameters(self): +# self.kernel = nn.Parameter(self.kernel) +# +# def save_layer(self,logdir,i): +# np.save(logdir +"/layer_"+str(i)+"_weights.npy",self.kernel.detach().cpu().numpy()) +# +# def load_layer(self,logdir,i): +# kernel = np.load(logdir +"/layer_"+str(i)+"_weights.npy") +# self.kernel = set_tensor(torch.from_numpy(kernel)) +# +# class MaxPool(Layer): +# def __init__(self, kernel_size,device='cpu'): +# self.kernel_size = kernel_size +# self.device = device +# self.activations = torch.empty(1) +# self.weights = torch.zeros((1,1)).to(self.device) +# +# def forward(self,x): +# out, self.idxs = F.max_pool2d(x, self.kernel_size,return_indices=True) +# return out +# +# def backward(self, y): +# return F.max_unpool2d(y,self.idxs, self.kernel_size) +# +# def update_gradient(self,e,update_weights=False,sign_reverse=False): +# self.grad["weights"] = torch.zeros_like(self.weights) +# +# def get_true_weight_grad(self): +# return None +# +# def set_weight_parameters(self): +# pass +# +# def save_layer(self,logdir,i): +# pass +# +# def load_layer(self,logdir,i): +# pass +# +# class AvgPool(Layer): +# def __init__(self, kernel_size,device='cpu'): +# self.kernel_size = kernel_size +# self.device = device +# self.activations = torch.empty(1) +# self.weights = torch.zeros((1,1)).to(self.device) +# +# +# def forward(self, x): +# self.B_in,self.C_in,self.H_in,self.W_in = x.shape +# return F.avg_pool2d(x,self.kernel_size) +# +# def backward(self, y): +# N,C,H,W = y.shape +# print("in backward: ", y.shape) +# return F.interpolate(y,scale_factor=(1,1,self.kernel_size,self.kernel_size)) +# +# def update_gradient(self,e,update_weights=False, sign_reverse=False): +# self.grad["weights"] = torch.zeros_like(self.weights) +# +# def save_layer(self,logdir,i): +# pass +# +# def load_layer(self,logdir,i): +# pass +# +# class ProjectionLayer(Layer): +# def __init__(self,input_size, output_size,act_fn,learning_rate,device='cpu'): +# self.input_size = input_size +# self.B, self.C, self.H, self.W = self.input_size +# self.output_size =output_size +# self.learning_rate = learning_rate +# self.act_fn = act_fn +# self.device = device +# self.Hid = self.C * self.H * self.W +# self.weights = torch.empty((self.Hid, self.output_size)).normal_(mean=0.0, std=0.05).to(self.device) +# +# def forward(self, x): +# self.inp = x.detach().clone() +# out = x.reshape((len(x), -1)) +# self.activations = torch.matmul(out,self.weights) +# return self.f(self.activations) +# +# def backward(self, e): +# fn_deriv = self.act_fn.deriv(self.activations) +# out = torch.matmul(e * fn_deriv, self.weights.T) +# out = out.reshape((len(e), self.C, self.H, self.W)) +# return torch.clamp(out,-50,50) +# +# def update_gradient(self, e,update_weights=False,sign_reverse=False): +# out = self.inp.reshape((len(self.inp), -1)) +# fn_deriv = self.act_fn.deriv(self.activations) +# dw = torch.matmul(out.T, e * fn_deriv) +# self.grad["weights"] = dw +# +# def get_true_weight_grad(self): +# return self.weights.grad +# +# def set_weight_parameters(self): +# self.weights = nn.Parameter(self.weights) +# +# def save_layer(self,logdir,i): +# np.save(logdir +"/layer_"+str(i)+"_weights.npy",self.weights.detach().cpu().numpy()) +# +# def load_layer(self,logdir,i): +# weights = np.load(logdir +"/layer_"+str(i)+"_weights.npy") +# self.weights = set_tensor(torch.from_numpy(weights)) From d6f74307129eea06c766fada89cdf16664f4a6ad Mon Sep 17 00:00:00 2001 From: Nick Hay <83538768+lasermanick@users.noreply.github.com> Date: Tue, 20 Sep 2022 13:53:51 +0100 Subject: [PATCH 13/30] PCModel comments Add comments to PCModel.train_batch_generative() --- pypc/models.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pypc/models.py b/pypc/models.py index 6900ce4..c7a3903 100644 --- a/pypc/models.py +++ b/pypc/models.py @@ -104,12 +104,12 @@ def train_batch_generative(self, img_batch, label_batch, n_iters, fixed_preds=Fa :param n_iters: Number of training iterations :param fixed_preds: Fix predictions at initial values? """ - self.reset() - self.set_input(label_batch) - self.propagate_mu() - self.set_target(img_batch) - self.train_updates(n_iters, fixed_preds=fixed_preds) - self.update_grads() + self.reset() # Initialise the prediction, error, and mu data structures + self.set_input(label_batch) # Set the model inputs, mus[0], equal to the training *labels* + self.propagate_mu() # Perform forward pass, update mus for all nodes except inputs and targets + self.set_target(img_batch) # Set the model outputs (targets), mus[-1], equal to the training *images* + self.train_updates(n_iters, fixed_preds=fixed_preds) # Iteratively update mus, predictions and errors + self.update_grads() # Calculate gradients of weights and biases for all layers def test_batch_supervised(self, img_batch): return self.forward(img_batch) From 548647c12d47e541eea2a380d5c55a8f8fbda8e9 Mon Sep 17 00:00:00 2001 From: Nick Hay <83538768+lasermanick@users.noreply.github.com> Date: Fri, 23 Sep 2022 12:17:12 +0100 Subject: [PATCH 14/30] Add fixed, diagonal precisions Added fixed precisions in the form of (identity matrix x constant) where constant is set by layer. Can be disabled. Implemented by optionally calculating precision scaled error term. Execution time increases by ~10% when enabled. --- pypc/models.py | 29 ++++++++++++++++++++++++++--- scripts/generative.py | 6 +++++- scripts/supervised.py | 4 ++++ 3 files changed, 35 insertions(+), 4 deletions(-) diff --git a/pypc/models.py b/pypc/models.py index c7a3903..9edbca4 100644 --- a/pypc/models.py +++ b/pypc/models.py @@ -5,17 +5,19 @@ class PCModel(object): - def __init__(self, nodes, mu_dt, act_fn, use_bias=False, kaiming_init=False): + def __init__(self, nodes, mu_dt, act_fn, use_bias=False, kaiming_init=False, use_precis=False, precis=None): """ Define the Predictive Coding PyTorch model. All layers fully connected. All nodes using the specified activation function except for the output layer which is linear. Bias terms are optional. Kaiming weight initialisation is - optional. + optional. Precisions are optional. :param nodes: List of number of nodes in each layer - :param mu_dt: + :param mu_dt: Timestep for updating means :param act_fn: Activation function :param use_bias: Include bias terms? :param kaiming_init: Use Kaiming weight initialisation? + :param use_precis: Use precisions? (If False, precisions are implied to be identity matrices) + :param precis: List of precision scaling per node layer """ self.nodes = nodes self.mu_dt = mu_dt @@ -36,6 +38,14 @@ def __init__(self, nodes, mu_dt, act_fn, use_bias=False, kaiming_init=False): ) self.layers.append(layer) + self.use_precis = use_precis + # If precisions used, create list of diagonal precision matrices with given scale factor + # NOTE: Precisions are fixed, so can be defined here. Not suitable for dynamic (learned) precisions + if self.use_precis: + self.precis = [[] for _ in range(self.n_nodes)] + for n in range(self.n_nodes): + self.precis[n] = utils.set_tensor(torch.diagflat(precis[n]*torch.ones(nodes[n]))) + def reset(self): """ Initialise predictions (preds), errors (errs), and variational means (mus) to empty lists @@ -130,9 +140,12 @@ def train_updates(self, n_iters, fixed_preds=False): :param fixed_preds: Fix predictions at initial values? """ # For batch, initialise predictions and errors for all nodes except inputs + # Optionally, errors are precision scaled for n in range(1, self.n_nodes): self.preds[n] = self.layers[n - 1].forward(self.mus[n - 1]) self.errs[n] = self.mus[n] - self.preds[n] + if self.use_precis: + self.errs[n] = torch.matmul(self.errs[n], self.precis[n]) # For each training iteration for itr in range(n_iters): @@ -142,10 +155,13 @@ def train_updates(self, n_iters, fixed_preds=False): self.mus[l] = self.mus[l] + self.mu_dt * delta # For batch, update errors and (optionally) predictions for all nodes except inputs + # Optionally, errors are precision scaled for n in range(1, self.n_nodes): if not fixed_preds: self.preds[n] = self.layers[n - 1].forward(self.mus[n - 1]) self.errs[n] = self.mus[n] - self.preds[n] + if self.use_precis: + self.errs[n] = torch.matmul(self.errs[n], self.precis[n]) def test_updates(self, n_iters, fixed_preds): """ @@ -155,9 +171,12 @@ def test_updates(self, n_iters, fixed_preds): :param fixed_preds: Fix predictions at initial values? """ # For batch, initialise predictions and errors for all nodes except inputs + # Optionally, errors are precision scaled for n in range(1, self.n_nodes): self.preds[n] = self.layers[n - 1].forward(self.mus[n - 1]) self.errs[n] = self.mus[n] - self.preds[n] + if self.use_precis: + self.errs[n] = torch.matmul(self.errs[n], self.precis[n]) # For each test iteration for itr in range(n_iters): @@ -167,10 +186,14 @@ def test_updates(self, n_iters, fixed_preds): delta = self.layers[l].backward(self.errs[l + 1]) - self.errs[l] self.mus[l] = self.mus[l] + self.mu_dt * delta + # For batch, update errors and (optionally) predictions for all nodes except inputs + # Optionally, errors are precision scaled for n in range(1, self.n_nodes): if not fixed_preds: self.preds[n] = self.layers[n - 1].forward(self.mus[n - 1]) self.errs[n] = self.mus[n] - self.preds[n] + if self.use_precis: + self.errs[n] = torch.matmul(self.errs[n], self.precis[n]) def update_grads(self): """ diff --git a/scripts/generative.py b/scripts/generative.py index a988d15..23593df 100644 --- a/scripts/generative.py +++ b/scripts/generative.py @@ -32,6 +32,8 @@ def main(cf): act_fn=cf.act_fn, use_bias=cf.use_bias, kaiming_init=cf.kaiming_init, + use_precis=cf.use_precis, + precis=cf.precis, ) optimizer = optim.get_optim( model.params, @@ -86,7 +88,7 @@ def main(cf): # experiment params cf.seed = seed - cf.n_epochs = 2 # 20 + cf.n_epochs = 20 # 20 cf.test_every = 1 cf.logdir = f"data/generative/{seed}/" cf.imgdir = cf.logdir + "imgs/" @@ -118,5 +120,7 @@ def main(cf): cf.kaiming_init = False cf.nodes = [10, 100, 300, 784] cf.act_fn = utils.Tanh() + cf.use_precis = False + cf.precis = [1.0, 1.0, 1.0, 1.0] main(cf) diff --git a/scripts/supervised.py b/scripts/supervised.py index 1300776..d15ee50 100644 --- a/scripts/supervised.py +++ b/scripts/supervised.py @@ -31,6 +31,8 @@ def main(cf): act_fn=cf.act_fn, use_bias=cf.use_bias, kaiming_init=cf.kaiming_init, + use_precis=cf.use_precis, + precis=cf.precis, ) optimizer = optim.get_optim( model.params, @@ -106,5 +108,7 @@ def main(cf): cf.kaiming_init = True # False cf.nodes = [784, 300, 100, 10] cf.act_fn = utils.ReLU() + cf.use_precis = False + cf.precis = [1.0, 1.0, 1.0, 1.0] main(cf) From b023ab5db0e224691823bd60da2968a467a83cd0 Mon Sep 17 00:00:00 2001 From: Nick Hay <83538768+lasermanick@users.noreply.github.com> Date: Fri, 23 Sep 2022 12:18:59 +0100 Subject: [PATCH 15/30] Update test_updates() default Add default=False to fixed_preds parameter in test_updates() (to match train_updates()) --- pypc/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pypc/models.py b/pypc/models.py index 9edbca4..a7fb5dd 100644 --- a/pypc/models.py +++ b/pypc/models.py @@ -163,7 +163,7 @@ def train_updates(self, n_iters, fixed_preds=False): if self.use_precis: self.errs[n] = torch.matmul(self.errs[n], self.precis[n]) - def test_updates(self, n_iters, fixed_preds): + def test_updates(self, n_iters, fixed_preds=False): """ Test model From 2726bd49bf5acf52fe79ca84cd6c7891b595d04c Mon Sep 17 00:00:00 2001 From: Nick Hay <83538768+lasermanick@users.noreply.github.com> Date: Fri, 23 Sep 2022 12:20:45 +0100 Subject: [PATCH 16/30] Tweak progress display Adjust progress display to match generative.py. (Also use reduced dataset by default, again matching generative.py) --- scripts/supervised.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/scripts/supervised.py b/scripts/supervised.py index d15ee50..cd6b24e 100644 --- a/scripts/supervised.py +++ b/scripts/supervised.py @@ -61,12 +61,14 @@ def main(cf): ) if epoch % cf.test_every == 0: + print(f"\nTest @ epoch {epoch}") + sleep(0.1) acc = 0 for _, (img_batch, label_batch) in enumerate(tqdm(test_loader, disable=False)): label_preds = model.test_batch_supervised(img_batch) acc += datasets.accuracy(label_preds, label_batch) metrics["acc"].append(acc / len(test_loader)) - print("\nTest @ epoch {} / Accuracy: {:.4f}".format(epoch, acc / len(test_loader))) + print("\nAccuracy: {:.4f}".format(acc / len(test_loader))) utils.save_json(metrics, cf.logdir + "metrics.json") @@ -85,8 +87,8 @@ def main(cf): cf.logdir = f"data/supervised/{seed}/" # dataset params - cf.train_size = None - cf.test_size = None + cf.train_size = 6000 # None + cf.test_size = 1000 # None cf.label_scale = None cf.normalize = False From 416ae56b277501a8614b1d2103ac1123d9797c6a Mon Sep 17 00:00:00 2001 From: Nick Hay <83538768+lasermanick@users.noreply.github.com> Date: Thu, 28 Mar 2024 16:38:40 +0000 Subject: [PATCH 17/30] Create environment.yml An environment definition. For example, use as follows to define a new environment: conda env create -f environment.yml --- environment.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 environment.yml diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..66436c6 --- /dev/null +++ b/environment.yml @@ -0,0 +1,18 @@ +name: pypc +channels: + - pytorch + - nvidia + - conda-forge + - defaults +dependencies: + - python + - numpy + - matplotlib + - pandas + - plotly + - neptune + - tqdm + - pytorch + - pytorch-cuda=12.1 + - torchaudio + - torchvision \ No newline at end of file From f1e91bf59afd1a4e962264463fda3b8acbd48f4a Mon Sep 17 00:00:00 2001 From: Nick Hay <83538768+lasermanick@users.noreply.github.com> Date: Thu, 28 Mar 2024 16:49:48 +0000 Subject: [PATCH 18/30] Create figures.py Creates figures from Neptune data using Plotly. Defines a class NickFig and some standard colours. For each figure, creates a definition. Finally, creates instances of NickFig with selected definition(s). --- pypc/figures.py | 1250 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1250 insertions(+) create mode 100644 pypc/figures.py diff --git a/pypc/figures.py b/pypc/figures.py new file mode 100644 index 0000000..2be4fce --- /dev/null +++ b/pypc/figures.py @@ -0,0 +1,1250 @@ +import os +import plotly.graph_objs as go +# from dash import Dash, html, dcc +import numpy as np +import pandas as pd +import neptune.new as neptune + +from collections import namedtuple +from operator import itemgetter + +def rgb_to_rgba(rgb_value, alpha): + """ + Adds the alpha channel to an RGB Value and returns it as an RGBA Value + :param rgb_value: Input RGB Value + :param alpha: Alpha Value to add in range [0,1] + :return: RGBA Value + """ + return f"rgba{rgb_value[3:-1]}, {alpha})" + +# Colours for plots +colours = { + 'white': 'rgb(255, 255, 255)', + 'grey75': 'rgb(191, 191, 191)', + 'grey50': 'rgb(127, 127, 127)', + 'grey25': 'rgb(63, 63, 63)', + 'black': 'rgb(0, 0, 0)', + 'darkred': 'rgb(127, 0, 0)', + 'red': 'rgb(255, 0, 0)', + 'orange': 'rgb(255, 127, 0)', + 'yellow': 'rgb(255, 255, 0)', + 'green': 'rgb(0, 255, 0)', + 'darkgreen': 'rgb(0, 127, 0)', + 'cyan': 'rgb(0, 255, 255)', + 'blue': 'rgb(0, 0, 255)', + 'darkblue': 'rgb(0, 0, 127)', + 'magenta': 'rgb(255, 0, 255)', +} + +# Groups of runs to be added to dataset +RunGroup = namedtuple("RunGroup", ["start_run", "num_runs", "title", "colour", "x_value"]) + + +class NickFig: + def __init__(self, fig_type, groups, fig_num, title, yaxis_title, yaxis_scale, data_field, xaxis_title, project_name, num_datapoints, xaxis_range, yaxis_range, path, filename, showlegend, font, width, height, show_fig=False, save_fig=False): + self.fig_type = fig_type + self.groups = groups + self.fig_num = fig_num + self.title = title + self.yaxis_title = yaxis_title + self.data_field = data_field + self.xaxis_title = xaxis_title + self.project_name = project_name + self.num_datapoints = num_datapoints + self.xaxis_range = xaxis_range + self.yaxis_range = yaxis_range + self.yaxis_scale = yaxis_scale + self.path = path + self.filename = filename + self.showlegend = showlegend + self.font = font + self.width = width + self.height = height + self.show_fig = show_fig + self.save_fig = save_fig + + self.stats = pd.DataFrame() + + self.fig = go.Figure() + + self.format_figure() + if self.fig_type == "1d_multiple_curves": + self.calc_stats() + elif self.fig_type == "2d": + self.calc_stats_2dscan() + else: + assert False, "Select a valid figure type!" + + self.draw_traces() + if self.show_fig: + self.show_figure() + if self.save_fig: + self.save_figure() + + def get_runs(self, group): + df = pd.DataFrame() + for run in range(group.start_run, group.start_run + group.num_runs): + run_id = f"PYPC-{run}" + # print(f"{run_id}") + run = neptune.init_run( + project="lasermanick/PYPC", + mode="read-only", + api_token="eyJhcGlfYWRkcmVzcyI6Imh0dHBzOi8vYXBwLm5lcHR1bmUuYWkiLCJhcGlfdXJsIjoiaHR0cHM6Ly9hcHAubmVwdHVuZS5haSIsImFwaV9rZXkiOiIxMDVhMjgyMi01ZjU0LTQ3NDMtODcwOS1jNjlmNGNjNDRhYTEifQ==", + with_id=run_id, + ) + df.insert(df.shape[1], run_id, run[self.data_field].fetch_values(include_timestamp=False)["value"]) + return df + + def calc_stats(self): + self.stats.insert(self.stats.shape[1], self.xaxis_title, + np.arange(1, self.num_datapoints + 1)) # x-axis column + for g in self.groups: + df = self.get_runs(g) + self.stats.insert(self.stats.shape[1], f"{g.title}m", df.mean(1) * self.yaxis_scale) + self.stats.insert(self.stats.shape[1], f"{g.title}s", df.std(1) * self.yaxis_scale) + self.stats.insert(self.stats.shape[1], f"{g.title}l", (df.mean(1) - df.std(1)) * self.yaxis_scale) + self.stats.insert(self.stats.shape[1], f"{g.title}u", (df.mean(1) + df.std(1)) * self.yaxis_scale) + + def calc_stats_2dscan(self): + for g in self.groups: # For each group of runs in the figure + df = self.get_runs(g) + # Create the columns if needed + if f"{g.title}m" not in self.stats.columns: + self.stats.insert(self.stats.shape[1], f"{g.title}m", np.zeros(self.stats.shape[0])) + self.stats.insert(self.stats.shape[1], f"{g.title}s", np.zeros(self.stats.shape[0])) + self.stats.insert(self.stats.shape[1], f"{g.title}l", np.zeros(self.stats.shape[0])) + self.stats.insert(self.stats.shape[1], f"{g.title}u", np.zeros(self.stats.shape[0])) + # Create the row if needed + if g.x_value not in self.stats.index: + self.stats.loc[g.x_value] = np.zeros(self.stats.shape[1]) + # Add the stats to the stats dataframe + self.stats.loc[g.x_value][f"{g.title}m"] = df.mean(1) * self.yaxis_scale + self.stats.loc[g.x_value][f"{g.title}s"] = df.std(1) * self.yaxis_scale + self.stats.loc[g.x_value][f"{g.title}l"] = (df.mean(1) - df.std(1)) * self.yaxis_scale + self.stats.loc[g.x_value][f"{g.title}u"] = (df.mean(1) + df.std(1)) * self.yaxis_scale + + # Add x-axis column + self.stats.insert(self.stats.shape[1], self.xaxis_title, self.stats.index) + + def draw_traces(self): + # Extract traces (with unique titles) from the groups + key = itemgetter(3) + traces = {key(g): g for g in self.groups}.values() + for t in traces: + # Mean + self.fig.add_trace( + go.Scatter( + name=t.title, + x=self.stats[self.xaxis_title], + y=self.stats[f"{t.title}m"], + mode='lines', + line=dict(color=t.colour) + ) + ) + # Upper bound + self.fig.add_trace( + go.Scatter( + name=f"{t.title} upper", + x=self.stats[self.xaxis_title], + y=self.stats[f"{t.title}u"], + mode='lines', + line=dict(width=0), + showlegend=False + ) + ) + # Lower bound + self.fig.add_trace( + go.Scatter( + name=f"{t.title} lower", + x=self.stats[self.xaxis_title], + y=self.stats[f"{t.title}l"], + line=dict(width=0), + mode='lines', + fillcolor=rgb_to_rgba(t.colour, 0.3), + fill='tonexty', + showlegend=False + ) + ) + + def format_figure(self): + self.fig.update_layout( + font=self.font, + showlegend=self.showlegend, + xaxis_title=self.xaxis_title, + yaxis_title=self.yaxis_title, + title=self.title, + xaxis_range=self.xaxis_range, + yaxis_range=self.yaxis_range, + plot_bgcolor=colours['white'], + width=self.width, + height=self.height, + ) + self.fig.update_xaxes( + linecolor=colours['black'], + showline=True, + showgrid=False, + linewidth=2, + ) + self.fig.update_yaxes( + linecolor=colours['black'], + showline=True, + showgrid=False, + linewidth=2, + ) + + def show_figure(self): + self.fig.show() + + def save_figure(self): + figdir = self.path + os.makedirs(figdir, exist_ok=True) + self.fig.write_image(figdir + self.filename) + + def get_fig(self): + return self.fig + + +testacc_fig_cfg = { + "fig_type": "1d_multiple_curves", + "title": "Title", + "yaxis_title": "Test accuracy (%)", + "data_field": "test/acc", + "xaxis_title": "Epoch", + "project_name": "PYPC", + "num_datapoints": 20, + "xaxis_range": [0, 20], + "yaxis_range": [0, 100], + "yaxis_scale": 100.0, + "path": "figs/", + "filename": "fig.svg", + "showlegend": True, + "font": dict(family="Arial", size=20, color=colours['black']), + "width": 800, + "height": 600, +} + +fig_cfgs = {} + +fig_num = "D(a)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} node structures 00" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(226, 6, "[10, 169, 729, 784]", colours['darkred'], None), + RunGroup(232, 6, "[10, 144, 169, 784]", colours['red'], None), + RunGroup(238, 6, "[10, 100, 300, 784]", colours['orange'], None), + RunGroup(244, 6, "[10, 36, 169, 784]", colours['green'], None), + RunGroup(250, 6, "[10, 10, 169, 729, 784]", colours['cyan'], None), + RunGroup(256, 6, "[10, 676, 729, 784]", colours['blue'], None), + RunGroup(262, 6, "[10, 36, 169, 729, 784]", colours['darkblue'], None), +] + +fig_num = "D(b)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} node structures 01" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(268, 6, "0.001", colours['red'], None), + RunGroup(262, 6, "0.003", colours['darkblue'], None), + RunGroup(274, 6, "0.005", colours['green'], None), +] + +fig_num = "D(c)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} node structures 02" +fc["filename"] = f"Fig {fig_num}.svg" +fc["num_datapoints"] = 40 +fc["xaxis_range"] = [0, 40] +fc["groups"] = [ + RunGroup(280, 6, "[10, 169, 729, 784]", colours['red'], None), + RunGroup(286, 6, "[10, 144, 169, 784]", colours['darkblue'], None), + RunGroup(292, 6, "[10, 100, 300, 784]", colours['green'], None), +] + +fig_num = "G(a)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} lr scan" +fc["filename"] = f"Fig {fig_num}.svg" +fc["num_datapoints"] = 40 +fc["xaxis_range"] = [0, 40] +fc["groups"] = [ + # RunGroup(304, 6, "0.001", colours['red'], None), + RunGroup(310, 6, "0.002", colours['darkred'], None), + # RunGroup(316, 6, "0.003", colours['red'], None), + # RunGroup(322, 6, "0.004", colours['red'], None), + RunGroup(328, 6, "0.005", colours['red'], None), + RunGroup(334, 6, "0.006", colours['orange'], None), + RunGroup(340, 6, "0.008", colours['green'], None), + RunGroup(346, 6, "0.010", colours['cyan'], None), + RunGroup(364, 6, "0.015", colours['blue'], None), + RunGroup(352, 6, "0.020", colours['darkblue'], None), + # RunGroup(358, 6, "0.03", colours['red'], None), +] + +fig_num = "G(b)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} dt scan" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(370, 6, "0.001", colours['red'], None), + RunGroup(376, 6, "0.003", colours['orange'], None), + RunGroup(382, 6, "0.01", colours['green'], None), + RunGroup(388, 6, "0.03", colours['cyan'], None), + RunGroup(394, 6, "0.1", colours['blue'], None), +] + +fig_num = "I(a)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} training its scan" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(400, 6, "5", colours['red'], None), + RunGroup(406, 6, "15", colours['orange'], None), + RunGroup(412, 6, "50", colours['green'], None), + RunGroup(418, 6, "150", colours['cyan'], None), + RunGroup(424, 6, "500", colours['blue'], None), +] + +fig_num = "I(b)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} test its scan" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(430, 6, "20", colours['red'], None), + RunGroup(436, 6, "60", colours['orange'], None), + RunGroup(442, 6, "200", colours['green'], None), + RunGroup(448, 6, "600", colours['cyan'], None), + RunGroup(454, 6, "2000", colours['blue'], None), +] + +fig_num = "K(a)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} lr scan with extreme train and test its" +fc["filename"] = f"Fig {fig_num}.svg" +fc["num_datapoints"] = 40 +fc["xaxis_range"] = [0, 40] +fc["groups"] = [ + RunGroup(460, 6, "0.001", colours['red'], None), + RunGroup(466, 6, "0.002", colours['orange'], None), + RunGroup(472, 6, "0.006", colours['green'], None), + RunGroup(478, 6, "0.01", colours['cyan'], None), + RunGroup(484, 6, "0.02", colours['blue'], None), +] + +fig_num = "K(b)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} lr scan with extreme train and test its" +fc["filename"] = f"Fig {fig_num}.svg" +fc["num_datapoints"] = 40 +fc["xaxis_range"] = [0, 40] +fc["groups"] = [ + RunGroup(490, 1, "0.001", colours['red'], None), + RunGroup(492, 1, "0.002", colours['orange'], None), + RunGroup(493, 1, "0.006", colours['green'], None), + RunGroup(494, 1, "0.01", colours['cyan'], None), + RunGroup(495, 1, "0.02", colours['blue'], None), +] + +fig_num = "L" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} confirm prec change equiv to dt change" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(499, 6, "[10.0, 10.0, 10.0, 10.0]", colours['red'], None), + RunGroup(505, 6, "[3.0, 3.0, 3.0, 3.0]", colours['orange'], None), + RunGroup(511, 6, "[1.0, 1.0, 1.0, 1.0]", colours['green'], None), + RunGroup(517, 6, "[0.3, 0.3, 0.3, 0.3]", colours['cyan'], None), + RunGroup(523, 6, "[0.1, 0.1, 0.1, 0.1]", colours['blue'], None), +] + +fig_num = "M(a)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} layer 3 prec scan" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(570, 6, "20", colours['darkred'], None), + RunGroup(576, 6, "10", colours['red'], None), + RunGroup(582, 6, "5", colours['orange'], None), + RunGroup(588, 6, "2", colours['green'], None), + RunGroup(594, 6, "1", colours['cyan'], None), + RunGroup(600, 6, "0.5", colours['blue'], None), + RunGroup(606, 6, "0.2", colours['darkblue'], None), + RunGroup(612, 6, "0.1", colours['grey25'], None), + RunGroup(618, 6, "0.05", colours['grey75'], None), +] + +fig_num = "M(b)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} layer 2 prec scan" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(624, 6, "20", colours['darkred'], None), + RunGroup(630, 6, "10", colours['red'], None), + RunGroup(636, 6, "5", colours['orange'], None), + RunGroup(642, 6, "2", colours['green'], None), + RunGroup(648, 6, "1", colours['cyan'], None), + RunGroup(654, 6, "0.5", colours['blue'], None), + RunGroup(660, 6, "0.2", colours['darkblue'], None), + RunGroup(666, 6, "0.1", colours['grey25'], None), + RunGroup(672, 6, "0.05", colours['grey75'], None), +] + +fig_num = "M(c)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} layer 1 prec scan" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(678, 6, "20", colours['darkred'], None), + RunGroup(684, 6, "10", colours['red'], None), + RunGroup(690, 6, "5", colours['orange'], None), + RunGroup(696, 6, "2", colours['green'], None), + RunGroup(702, 6, "1", colours['cyan'], None), + RunGroup(708, 6, "0.5", colours['blue'], None), + RunGroup(714, 6, "0.2", colours['darkblue'], None), + RunGroup(720, 6, "0.1", colours['grey25'], None), + RunGroup(726, 6, "0.05", colours['grey75'], None), +] + +fig_num = "N(a)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} prec ratio by layer" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(1017, 1, "[1.0, 0.01, 0.01, 0.01]", colours['darkred'], None), + RunGroup(1018, 1, "[1.0, 0.04, 0.02, 0.01]", colours['red'], None), + RunGroup(1019, 1, "[1.0, 0.25, 0.05, 0.01]", colours['orange'], None), + RunGroup(1020, 1, "[1.0, 1.0, 0.10, 0.01]", colours['green'], None), + RunGroup(1021, 1, "[1.0, 4.0, 0.20, 0.01]", colours['cyan'], None), + RunGroup(1022, 1, "[1.0, 25, 0.50, 0.01]", colours['blue'], None), + RunGroup(1023, 1, "[1.0, 100, 1.0, 0.01]", colours['darkblue'], None), +] + +fig_num = "N(b)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} prec ratio by layer" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(1024, 1, "[1.0, 0.01, 0.01, 0.01]", colours['darkred'], None), + RunGroup(1025, 1, "[1.0, 0.01, 0.02, 0.04]", colours['red'], None), + RunGroup(1026, 1, "[1.0, 0.01, 0.05, 0.25]", colours['orange'], None), + RunGroup(1027, 1, "[1.0, 0.01, 0.10, 1.0]", colours['green'], None), + RunGroup(1028, 1, "[1.0, 0.01, 0.20, 4.0]", colours['cyan'], None), + RunGroup(1029, 1, "[1.0, 0.01, 0.50, 25]", colours['blue'], None), + RunGroup(1030, 1, "[1.0, 0.01, 1.0, 100]", colours['darkblue'], None), +] + +fig_num = "P(a)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} train vs test accuracy - test" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(732, 1, "0.001", colours['darkred'], None), + RunGroup(734, 1, "0.003", colours['red'], None), + RunGroup(735, 1, "0.006", colours['orange'], None), + RunGroup(736, 1, "0.010", colours['green'], None), + RunGroup(737, 1, "0.015", colours['cyan'], None), + RunGroup(738, 1, "0.030", colours['blue'], None), +] + +fig_num = "P(b)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["yaxis_title"] = "Training accuracy (%)" +fc["data_field"] = "train/acc" +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} train vs test accuracy - train" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(732, 1, "0.001", colours['darkred'], None), + RunGroup(734, 1, "0.003", colours['red'], None), + RunGroup(735, 1, "0.006", colours['orange'], None), + RunGroup(736, 1, "0.010", colours['green'], None), + RunGroup(737, 1, "0.015", colours['cyan'], None), + RunGroup(738, 1, "0.030", colours['blue'], None), +] + +fig_num = "Q" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} effect of noise on accuracy" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(801, 1, "var = 4, cov = 1.0", colours['darkgreen'], None), + RunGroup(841, 6, "var = 4, cov = 0.5", colours['green'], None), + RunGroup(800, 1, "var = 1, cov = 1.0", colours['darkred'], None), + RunGroup(813, 1, "var = 1, cov = 0.5", colours['red'], None), + RunGroup(702, 6, "No noise", colours['grey25'], None), +] + +fig_num = "R" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} bottom half noise, bottom half layer 3 prec scan" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(817, 6, "0.1", colours['darkred'], None), + RunGroup(823, 6, "0.2", colours['red'], None), + RunGroup(829, 6, "0.5", colours['orange'], None), + # RunGroup(835, 6, "0.8", colours['green'], None), + RunGroup(841, 6, "1", colours['green'], None), + RunGroup(847, 6, "2", colours['cyan'], None), + RunGroup(853, 6, "5", colours['blue'], None), + # RunGroup(859, 6, "8", colours['grey25'], None), + RunGroup(865, 6, "10", colours['darkblue'], None), + RunGroup(702, 6, "No noise", colours['grey25'], None), +] + +fig_num = "S" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} bottom half noise, full layer 3 prec scan" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(873, 1, "0.1", colours['darkred'], None), + RunGroup(874, 1, "0.2", colours['red'], None), + RunGroup(875, 1, "0.5", colours['orange'], None), + # RunGroup(876, 1, "0.8", colours['green'], None), + RunGroup(841, 6, "1", colours['green'], None), + RunGroup(878, 1, "2", colours['cyan'], None), + RunGroup(879, 1, "5", colours['blue'], None), + # RunGroup(880, 1, "8", colours['grey25'], None), + RunGroup(881, 1, "10", colours['darkblue'], None), + RunGroup(702, 6, "No noise", colours['grey25'], None), +] + +fig_num = "T" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} bottom half noise, full layer 3 prec scan" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(883, 1, "0.1", colours['darkred'], None), + RunGroup(884, 1, "0.2", colours['red'], None), + RunGroup(885, 1, "0.5", colours['orange'], None), + # RunGroup(886, 1, "0.8", colours['green'], None), + RunGroup(829, 6, "1", colours['green'], None), + RunGroup(888, 1, "2", colours['cyan'], None), + RunGroup(889, 1, "5", colours['blue'], None), + # RunGroup(890, 1, "8", colours['grey25'], None), + RunGroup(891, 1, "10", colours['darkblue'], None), + RunGroup(702, 6, "No noise", colours['grey25'], None), +] + +fig_num = "U" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} bottom half noise, full prec scan" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(892, 1, "0.1", colours['darkred'], None), + RunGroup(893, 1, "0.2", colours['red'], None), + RunGroup(894, 1, "0.5", colours['orange'], None), + RunGroup(841, 6, "1", colours['green'], None), + RunGroup(896, 1, "2", colours['cyan'], None), + RunGroup(897, 1, "5", colours['blue'], None), + RunGroup(898, 1, "10", colours['darkblue'], None), + RunGroup(702, 6, "No noise", colours['grey25'], None), +] + +fig_num = "V" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} no noise, bottom half layer 3 prec scan" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(899, 1, "0.1", colours['darkred'], None), + RunGroup(905, 1, "0.2", colours['red'], None), + RunGroup(911, 1, "0.5", colours['orange'], None), + RunGroup(702, 6, "1", colours['green'], None), + RunGroup(923, 1, "2", colours['cyan'], None), + RunGroup(929, 1, "5", colours['blue'], None), + RunGroup(935, 1, "10", colours['darkblue'], None), +] + +fig_num = "AC" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} batch size=2000 bottom half noise, bottom half layer 3 prec scan" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(975, 6, "0.1", colours['darkred'], None), + RunGroup(981, 6, "0.2", colours['red'], None), + RunGroup(987, 6, "0.5", colours['orange'], None), + RunGroup(993, 6, "1", colours['green'], None), + RunGroup(999, 6, "2", colours['cyan'], None), + RunGroup(1005, 6, "5", colours['blue'], None), + RunGroup(1011, 6, "10", colours['darkblue'], None), + RunGroup(1031, 6, "No noise", colours['grey25'], None), +] + +fig_num = "AD" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} batch size scan 03" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(946, 1, "10000", colours['red'], None), + RunGroup(947, 1, "5000", colours['orange'], None), + RunGroup(948, 1, "2000", colours['green'], None), + RunGroup(949, 1, "1000", colours['cyan'], None), + RunGroup(950, 1, "500", colours['blue'], None), +] + +fig_num = "AF" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} lr scan with batch size = 2000" +fc["filename"] = f"Fig {fig_num}.svg" +fc["num_datapoints"] = 40 +fc["xaxis_range"] = [0, 40] +fc["groups"] = [ + RunGroup(1039, 1, "0.001", colours['red'], None), + RunGroup(1040, 1, "0.002", colours['darkred'], None), + RunGroup(1041, 1, "0.005", colours['red'], None), + RunGroup(1042, 1, "0.006", colours['orange'], None), + RunGroup(1043, 1, "0.008", colours['green'], None), + RunGroup(1044, 1, "0.010", colours['cyan'], None), + RunGroup(1045, 1, "0.015", colours['blue'], None), + RunGroup(1046, 1, "0.020", colours['darkblue'], None), +] + +fig_num = "AG(a)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_type"] = "2d" +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} 2D scan of noise stddev and precision (coverage=0.5)" +fc["filename"] = f"Fig {fig_num}.svg" +fc["xaxis_title"] = "Precision (lower image)" +fc["xaxis_range"] = [0, 1.2] +fc["yaxis_range"] = [67, 88] +fc["groups"] = [ + RunGroup(1266, 6, "0", colours['red'], 0), + RunGroup(2082, 6, "0", colours['red'], 0.002), + RunGroup(2088, 6, "0", colours['red'], 0.005), + RunGroup(2094, 6, "0", colours['red'], 0.01), + RunGroup(2100, 6, "0", colours['red'], 0.02), + RunGroup(1506, 6, "0", colours['red'], 0.027), + RunGroup(1512, 6, "0", colours['red'], 0.038), + RunGroup(1272, 6, "0", colours['red'], 0.059), + RunGroup(1278, 6, "0", colours['red'], 0.1), + RunGroup(1284, 6, "0", colours['red'], 0.2), + RunGroup(1290, 6, "0", colours['red'], 0.5), + RunGroup(1296, 6, "0", colours['red'], 0.8), + RunGroup(1302, 6, "0", colours['red'], 1.0), + RunGroup(1308, 6, "0", colours['red'], 1.2), + RunGroup(1314, 6, "1/4", colours['orange'], 0), + RunGroup(2106, 6, "1/4", colours['orange'], 0.002), + RunGroup(2112, 6, "1/4", colours['orange'], 0.005), + RunGroup(2118, 6, "1/4", colours['orange'], 0.01), + RunGroup(2124, 6, "1/4", colours['orange'], 0.02), + RunGroup(1518, 6, "1/4", colours['orange'], 0.027), + RunGroup(1524, 6, "1/4", colours['orange'], 0.038), + RunGroup(1320, 6, "1/4", colours['orange'], 0.059), + RunGroup(1326, 6, "1/4", colours['orange'], 0.1), + RunGroup(1332, 6, "1/4", colours['orange'], 0.2), + RunGroup(1338, 6, "1/4", colours['orange'], 0.5), + RunGroup(1344, 6, "1/4", colours['orange'], 0.8), + RunGroup(1350, 6, "1/4", colours['orange'], 1.0), + RunGroup(1356, 6, "1/4", colours['orange'], 1.2), + RunGroup(1362, 6, "1", colours['green'], 0), + RunGroup(2130, 6, "1", colours['green'], 0.002), + RunGroup(2136, 6, "1", colours['green'], 0.005), + RunGroup(2142, 6, "1", colours['green'], 0.01), + RunGroup(2148, 6, "1", colours['green'], 0.02), + RunGroup(1530, 6, "1", colours['green'], 0.027), + RunGroup(1536, 6, "1", colours['green'], 0.038), + RunGroup(1368, 6, "1", colours['green'], 0.059), + RunGroup(1374, 6, "1", colours['green'], 0.1), + RunGroup(1380, 6, "1", colours['green'], 0.2), + RunGroup(1386, 6, "1", colours['green'], 0.5), + RunGroup(1392, 6, "1", colours['green'], 0.8), + RunGroup(1398, 6, "1", colours['green'], 1.0), + RunGroup(1404, 6, "1", colours['green'], 1.2), + RunGroup(1410, 6, "4", colours['cyan'], 0), + RunGroup(2154, 6, "4", colours['cyan'], 0.002), + RunGroup(2160, 6, "4", colours['cyan'], 0.005), + RunGroup(2166, 6, "4", colours['cyan'], 0.01), + RunGroup(2172, 6, "4", colours['cyan'], 0.02), + RunGroup(1542, 6, "4", colours['cyan'], 0.027), + RunGroup(1548, 6, "4", colours['cyan'], 0.038), + RunGroup(1416, 6, "4", colours['cyan'], 0.059), + RunGroup(1422, 6, "4", colours['cyan'], 0.1), + RunGroup(1428, 6, "4", colours['cyan'], 0.2), + RunGroup(1434, 6, "4", colours['cyan'], 0.5), + RunGroup(1440, 6, "4", colours['cyan'], 0.8), + RunGroup(1446, 6, "4", colours['cyan'], 1.0), + RunGroup(1452, 6, "4", colours['cyan'], 1.2), + RunGroup(1458, 6, "9", colours['blue'], 0), + RunGroup(2178, 6, "9", colours['blue'], 0.002), + RunGroup(2184, 6, "9", colours['blue'], 0.005), + RunGroup(2190, 6, "9", colours['blue'], 0.01), + RunGroup(2196, 6, "9", colours['blue'], 0.02), + RunGroup(1554, 6, "9", colours['blue'], 0.027), + RunGroup(1560, 6, "9", colours['blue'], 0.038), + RunGroup(1464, 6, "9", colours['blue'], 0.059), + RunGroup(1470, 6, "9", colours['blue'], 0.1), + RunGroup(1476, 6, "9", colours['blue'], 0.2), + RunGroup(1482, 6, "9", colours['blue'], 0.5), + RunGroup(1488, 6, "9", colours['blue'], 0.8), + RunGroup(1494, 6, "9", colours['blue'], 1.0), + RunGroup(1500, 6, "9", colours['blue'], 1.2), +] + +fig_num = "AH" +fc = fig_cfgs[fig_num] = fc.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} 2D scan of noise stddev and precision (coverage=0.5)" +fc["filename"] = f"Fig {fig_num}.svg" +fc["yaxis_title"] = "Free energy" +fc["data_field"] = "test/free_e" +fc["xaxis_title"] = "Precision (lower image)" +fc["xaxis_range"] = [0, 1.2] +fc["yaxis_range"] = [0.0, 0.5] +fc["yaxis_scale"] = [1.0] + +fig_num = "AG(c)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_type"] = "2d" +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} 2D scan of noise stddev and precision (coverage=1.0)" +fc["filename"] = f"Fig {fig_num}.svg" +fc["xaxis_title"] = "Precision (full image)" +fc["xaxis_range"] = [0, 1.2] +fc["yaxis_range"] = [52 , 88] +fc["groups"] = [ + RunGroup(1602, 6, "0", colours['red'], 0), + RunGroup(1962, 6, "0", colours['red'], 0.002), + RunGroup(1968, 6, "0", colours['red'], 0.005), + RunGroup(1908, 6, "0", colours['red'], 0.01), + RunGroup(1902, 6, "0", colours['red'], 0.02), + RunGroup(1608, 6, "0", colours['red'], 0.027), + RunGroup(1614, 6, "0", colours['red'], 0.038), + RunGroup(1620, 6, "0", colours['red'], 0.059), + RunGroup(1626, 6, "0", colours['red'], 0.1), + RunGroup(1632, 6, "0", colours['red'], 0.2), + RunGroup(1638, 6, "0", colours['red'], 0.5), + RunGroup(1644, 6, "0", colours['red'], 0.8), + RunGroup(1650, 6, "0", colours['red'], 1.0), + RunGroup(1656, 6, "0", colours['red'], 1.2), + RunGroup(1662, 6, "1/4", colours['orange'], 0), + RunGroup(1974, 6, "1/4", colours['orange'], 0.002), + RunGroup(1980, 6, "1/4", colours['orange'], 0.005), + RunGroup(1920, 6, "1/4", colours['orange'], 0.01), + RunGroup(1914, 6, "1/4", colours['orange'], 0.02), + RunGroup(1668, 6, "1/4", colours['orange'], 0.027), + RunGroup(1674, 6, "1/4", colours['orange'], 0.038), + RunGroup(1680, 6, "1/4", colours['orange'], 0.059), + RunGroup(1686, 6, "1/4", colours['orange'], 0.1), + RunGroup(1692, 6, "1/4", colours['orange'], 0.2), + RunGroup(1698, 6, "1/4", colours['orange'], 0.5), + RunGroup(1704, 6, "1/4", colours['orange'], 0.8), + RunGroup(1710, 6, "1/4", colours['orange'], 1.0), + RunGroup(1716, 6, "1/4", colours['orange'], 1.2), + RunGroup(1722, 6, "1", colours['green'], 0), + RunGroup(1986, 6, "1", colours['green'], 0.002), + RunGroup(1992, 6, "1", colours['green'], 0.005), + RunGroup(1932, 6, "1", colours['green'], 0.01), + RunGroup(1926, 6, "1", colours['green'], 0.02), + RunGroup(1728, 6, "1", colours['green'], 0.027), + RunGroup(1734, 6, "1", colours['green'], 0.038), + RunGroup(1740, 6, "1", colours['green'], 0.059), + RunGroup(1746, 6, "1", colours['green'], 0.1), + RunGroup(1752, 6, "1", colours['green'], 0.2), + RunGroup(1758, 6, "1", colours['green'], 0.5), + RunGroup(1764, 6, "1", colours['green'], 0.8), + RunGroup(1770, 6, "1", colours['green'], 1.0), + RunGroup(1776, 6, "1", colours['green'], 1.2), + RunGroup(1782, 6, "4", colours['cyan'], 0), + RunGroup(1998, 6, "4", colours['cyan'], 0.002), + RunGroup(2004, 6, "4", colours['cyan'], 0.005), + RunGroup(1944, 6, "4", colours['cyan'], 0.01), + RunGroup(1938, 6, "4", colours['cyan'], 0.02), + RunGroup(1788, 6, "4", colours['cyan'], 0.027), + RunGroup(1794, 6, "4", colours['cyan'], 0.038), + RunGroup(1800, 6, "4", colours['cyan'], 0.059), + RunGroup(1806, 6, "4", colours['cyan'], 0.1), + RunGroup(1812, 6, "4", colours['cyan'], 0.2), + RunGroup(1818, 6, "4", colours['cyan'], 0.5), + RunGroup(1824, 6, "4", colours['cyan'], 0.8), + RunGroup(1830, 6, "4", colours['cyan'], 1.0), + RunGroup(1836, 6, "4", colours['cyan'], 1.2), + RunGroup(1842, 6, "9", colours['blue'], 0), + RunGroup(2010, 6, "9", colours['blue'], 0.002), + RunGroup(2016, 6, "9", colours['blue'], 0.005), + RunGroup(1956, 6, "9", colours['blue'], 0.01), + RunGroup(1950, 6, "9", colours['blue'], 0.02), + RunGroup(1848, 6, "9", colours['blue'], 0.027), + RunGroup(1854, 6, "9", colours['blue'], 0.038), + RunGroup(1860, 6, "9", colours['blue'], 0.059), + RunGroup(1866, 6, "9", colours['blue'], 0.1), + RunGroup(1872, 6, "9", colours['blue'], 0.2), + RunGroup(1878, 6, "9", colours['blue'], 0.5), + RunGroup(1884, 6, "9", colours['blue'], 0.8), + RunGroup(1890, 6, "9", colours['blue'], 1.0), + RunGroup(1896, 6, "9", colours['blue'], 1.2), +] + +fig_num = "AK" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_type"] = "2d" +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} 2D scan of noise std and prec (coverage=n1.0/p0.5)" +fc["filename"] = f"Fig {fig_num}.svg" +fc["xaxis_title"] = "Precision" +fc["xaxis_range"] = [0, 1.2] +fc["yaxis_range"] = [7, 88] +fc["groups"] = [ + RunGroup(2074, 1, "0", colours['red'], 0), + RunGroup(2022, 1, "0", colours['red'], 0.002), + RunGroup(2023, 1, "0", colours['red'], 0.005), + RunGroup(2024, 1, "0", colours['red'], 0.01), + RunGroup(2025, 1, "0", colours['red'], 0.02), + RunGroup(2026, 1, "0", colours['red'], 0.027), + RunGroup(2075, 1, "0", colours['red'], 0.038), + RunGroup(2028, 1, "0", colours['red'], 0.059), + RunGroup(2029, 1, "0", colours['red'], 0.1), + RunGroup(2030, 1, "0", colours['red'], 0.2), + RunGroup(2031, 1, "0", colours['red'], 0.5), + RunGroup(2032, 1, "0", colours['red'], 0.8), + RunGroup(2033, 1, "0", colours['red'], 1.0), + RunGroup(2034, 1, "0", colours['red'], 1.2), + RunGroup(2076, 1, "1.0", colours['green'], 0), + RunGroup(2035, 1, "1.0", colours['green'], 0.002), + RunGroup(2036, 1, "1.0", colours['green'], 0.005), + RunGroup(2037, 1, "1.0", colours['green'], 0.01), + RunGroup(2038, 1, "1.0", colours['green'], 0.02), + RunGroup(2039, 1, "1.0", colours['green'], 0.027), + RunGroup(2077, 1, "1.0", colours['green'], 0.038), + RunGroup(2041, 1, "1.0", colours['green'], 0.059), + RunGroup(2042, 1, "1.0", colours['green'], 0.1), + RunGroup(2043, 1, "1.0", colours['green'], 0.2), + RunGroup(2044, 1, "1.0", colours['green'], 0.5), + RunGroup(2045, 1, "1.0", colours['green'], 0.8), + RunGroup(2046, 1, "1.0", colours['green'], 1.0), + RunGroup(2047, 1, "1.0", colours['green'], 1.2), + RunGroup(2078, 1, "2.0", colours['cyan'], 0), + RunGroup(2048, 1, "2.0", colours['cyan'], 0.002), + RunGroup(2049, 1, "2.0", colours['cyan'], 0.005), + RunGroup(2050, 1, "2.0", colours['cyan'], 0.01), + RunGroup(2051, 1, "2.0", colours['cyan'], 0.02), + RunGroup(2052, 1, "2.0", colours['cyan'], 0.027), + RunGroup(2079, 1, "2.0", colours['cyan'], 0.038), + RunGroup(2054, 1, "2.0", colours['cyan'], 0.059), + RunGroup(2055, 1, "2.0", colours['cyan'], 0.1), + RunGroup(2056, 1, "2.0", colours['cyan'], 0.2), + RunGroup(2057, 1, "2.0", colours['cyan'], 0.5), + RunGroup(2058, 1, "2.0", colours['cyan'], 0.8), + RunGroup(2059, 1, "2.0", colours['cyan'], 1.0), + RunGroup(2060, 1, "2.0", colours['cyan'], 1.2), + RunGroup(2080, 1, "3.0", colours['blue'], 0), + RunGroup(2061, 1, "3.0", colours['blue'], 0.002), + RunGroup(2062, 1, "3.0", colours['blue'], 0.005), + RunGroup(2063, 1, "3.0", colours['blue'], 0.01), + RunGroup(2064, 1, "3.0", colours['blue'], 0.02), + RunGroup(2065, 1, "3.0", colours['blue'], 0.027), + RunGroup(2081, 1, "3.0", colours['blue'], 0.038), + RunGroup(2067, 1, "3.0", colours['blue'], 0.059), + RunGroup(2068, 1, "3.0", colours['blue'], 0.1), + RunGroup(2069, 1, "3.0", colours['blue'], 0.2), + RunGroup(2070, 1, "3.0", colours['blue'], 0.5), + RunGroup(2071, 1, "3.0", colours['blue'], 0.8), + RunGroup(2072, 1, "3.0", colours['blue'], 1.0), + RunGroup(2073, 1, "3.0", colours['blue'], 1.2), +] + +fig_num = "AL" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_type"] = "2d" +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} scan of per pixel noise (coverage=0.5/1.0)" +fc["filename"] = f"Fig {fig_num}.svg" +fc["xaxis_title"] = "Noise range (=2.0 +/- x)" +fc["xaxis_range"] = [0, 3.0] +fc["yaxis_range"] = [72, 88] +fc["groups"] = [ + RunGroup(2202, 6, "0.5", colours['red'], 0), + RunGroup(2208, 6, "1.0", colours['blue'], 0), + RunGroup(2214, 6, "0.5", colours['red'], 0.2), + RunGroup(2220, 6, "1.0", colours['blue'], 0.2), + RunGroup(2226, 6, "0.5", colours['red'], 0.4), + RunGroup(2232, 6, "1.0", colours['blue'], 0.4), + RunGroup(2238, 6, "0.5", colours['red'], 0.6), + RunGroup(2244, 6, "1.0", colours['blue'], 0.6), + RunGroup(2250, 6, "0.5", colours['red'], 0.8), + RunGroup(2256, 6, "1.0", colours['blue'], 0.8), + RunGroup(2262, 6, "0.5", colours['red'], 1.0), + RunGroup(2268, 6, "1.0", colours['blue'], 1.0), + RunGroup(2274, 6, "0.5", colours['red'], 1.2), + RunGroup(2280, 6, "1.0", colours['blue'], 1.2), + RunGroup(2286, 6, "0.5", colours['red'], 1.4), + RunGroup(2292, 6, "1.0", colours['blue'], 1.4), + RunGroup(2298, 6, "0.5", colours['red'], 1.6), + RunGroup(2304, 6, "1.0", colours['blue'], 1.6), + RunGroup(2310, 6, "0.5", colours['red'], 1.8), + RunGroup(2316, 6, "1.0", colours['blue'], 1.8), + RunGroup(2322, 6, "0.5", colours['red'], 2.0), + RunGroup(2328, 6, "1.0", colours['blue'], 2.0), + RunGroup(2334, 6, "0.5", colours['red'], 2.2), + RunGroup(2340, 6, "1.0", colours['blue'], 2.2), + RunGroup(2346, 6, "0.5", colours['red'], 2.4), + RunGroup(2352, 6, "1.0", colours['blue'], 2.4), + RunGroup(2358, 6, "0.5", colours['red'], 2.6), + RunGroup(2364, 6, "1.0", colours['blue'], 2.6), + RunGroup(2370, 6, "0.5", colours['red'], 2.8), + RunGroup(2376, 6, "1.0", colours['blue'], 2.8), + RunGroup(2382, 6, "0.5", colours['red'], 3.0), + RunGroup(2388, 6, "1.0", colours['blue'], 3.0), +] + +fig_num = "AM" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_type"] = "2d" +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} 2D scan of its mult and prec (noise sd=3.0, cov=1.0)" +fc["filename"] = f"Fig {fig_num}.svg" +fc["xaxis_title"] = "Precision (full image)" +fc["xaxis_range"] = [0, 1.2] +fc["yaxis_range"] = [7, 88] +fc["groups"] = [ + RunGroup(2414, 1, "0.5", colours['red'], 0), + RunGroup(2415, 1, "0.5", colours['red'], 0.002), + RunGroup(2416, 1, "0.5", colours['red'], 0.005), + RunGroup(2417, 1, "0.5", colours['red'], 0.01), + RunGroup(2418, 1, "0.5", colours['red'], 0.02), + RunGroup(2419, 1, "0.5", colours['red'], 0.027), + RunGroup(2420, 1, "0.5", colours['red'], 0.038), + RunGroup(2421, 1, "0.5", colours['red'], 0.059), + RunGroup(2422, 1, "0.5", colours['red'], 0.1), + RunGroup(2423, 1, "0.5", colours['red'], 0.2), + RunGroup(2424, 1, "0.5", colours['red'], 0.5), + RunGroup(2440, 1, "0.5", colours['red'], 0.8), + RunGroup(2425, 1, "0.5", colours['red'], 1.0), + RunGroup(2426, 1, "0.5", colours['red'], 1.2), + RunGroup(1842, 6, "1.0", colours['blue'], 0), + RunGroup(2010, 6, "1.0", colours['blue'], 0.002), + RunGroup(2016, 6, "1.0", colours['blue'], 0.005), + RunGroup(1956, 6, "1.0", colours['blue'], 0.01), + RunGroup(1950, 6, "1.0", colours['blue'], 0.02), + RunGroup(1848, 6, "1.0", colours['blue'], 0.027), + RunGroup(1854, 6, "1.0", colours['blue'], 0.038), + RunGroup(1860, 6, "1.0", colours['blue'], 0.059), + RunGroup(1866, 6, "1.0", colours['blue'], 0.1), + RunGroup(1872, 6, "1.0", colours['blue'], 0.2), + RunGroup(1878, 6, "1.0", colours['blue'], 0.5), + RunGroup(1884, 6, "1.0", colours['blue'], 0.8), + RunGroup(1890, 6, "1.0", colours['blue'], 1.0), + RunGroup(1896, 6, "1.0", colours['blue'], 1.2), + RunGroup(2427, 1, "2.0", colours['green'], 0), + RunGroup(2428, 1, "2.0", colours['green'], 0.002), + RunGroup(2429, 1, "2.0", colours['green'], 0.005), + RunGroup(2430, 1, "2.0", colours['green'], 0.01), + RunGroup(2431, 1, "2.0", colours['green'], 0.02), + RunGroup(2432, 1, "2.0", colours['green'], 0.027), + RunGroup(2433, 1, "2.0", colours['green'], 0.038), + RunGroup(2434, 1, "2.0", colours['green'], 0.059), + RunGroup(2435, 1, "2.0", colours['green'], 0.1), + RunGroup(2436, 1, "2.0", colours['green'], 0.2), + RunGroup(2437, 1, "2.0", colours['green'], 0.5), + RunGroup(2441, 1, "2.0", colours['green'], 0.8), + RunGroup(2438, 1, "2.0", colours['green'], 1.0), + RunGroup(2439, 1, "2.0", colours['green'], 1.2), +] + +fig_num = "AN" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_type"] = "2d" +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} 2D scan of its mult and prec (noise sd=0)" +fc["filename"] = f"Fig {fig_num}.svg" +fc["xaxis_title"] = "Precision (full image)" +fc["xaxis_range"] = [0, 1.2] +fc["yaxis_range"] = [7, 88] +fc["groups"] = [ + RunGroup(2442, 1, "0.5", colours['red'], 0), + RunGroup(2443, 1, "0.5", colours['red'], 0.002), + RunGroup(2444, 1, "0.5", colours['red'], 0.005), + RunGroup(2445, 1, "0.5", colours['red'], 0.01), + RunGroup(2446, 1, "0.5", colours['red'], 0.02), + RunGroup(2447, 1, "0.5", colours['red'], 0.027), + RunGroup(2448, 1, "0.5", colours['red'], 0.038), + RunGroup(2449, 1, "0.5", colours['red'], 0.059), + RunGroup(2450, 1, "0.5", colours['red'], 0.1), + RunGroup(2451, 1, "0.5", colours['red'], 0.2), + RunGroup(2452, 1, "0.5", colours['red'], 0.5), + RunGroup(2453, 1, "0.5", colours['red'], 0.8), + RunGroup(2454, 1, "0.5", colours['red'], 1.0), + RunGroup(2455, 1, "0.5", colours['red'], 1.2), + RunGroup(1602, 6, "1.0", colours['blue'], 0), + RunGroup(1962, 6, "1.0", colours['blue'], 0.002), + RunGroup(1968, 6, "1.0", colours['blue'], 0.005), + RunGroup(1908, 6, "1.0", colours['blue'], 0.01), + RunGroup(1902, 6, "1.0", colours['blue'], 0.02), + RunGroup(1608, 6, "1.0", colours['blue'], 0.027), + RunGroup(1614, 6, "1.0", colours['blue'], 0.038), + RunGroup(1620, 6, "1.0", colours['blue'], 0.059), + RunGroup(1626, 6, "1.0", colours['blue'], 0.1), + RunGroup(1632, 6, "1.0", colours['blue'], 0.2), + RunGroup(1638, 6, "1.0", colours['blue'], 0.5), + RunGroup(1644, 6, "1.0", colours['blue'], 0.8), + RunGroup(1650, 6, "1.0", colours['blue'], 1.0), + RunGroup(1656, 6, "1.0", colours['blue'], 1.2), + RunGroup(2456, 1, "2.0", colours['green'], 0), + RunGroup(2457, 1, "2.0", colours['green'], 0.002), + RunGroup(2458, 1, "2.0", colours['green'], 0.005), + RunGroup(2459, 1, "2.0", colours['green'], 0.01), + RunGroup(2460, 1, "2.0", colours['green'], 0.02), + RunGroup(2461, 1, "2.0", colours['green'], 0.027), + RunGroup(2462, 1, "2.0", colours['green'], 0.038), + RunGroup(2463, 1, "2.0", colours['green'], 0.059), + RunGroup(2464, 1, "2.0", colours['green'], 0.1), + RunGroup(2465, 1, "2.0", colours['green'], 0.2), + RunGroup(2466, 1, "2.0", colours['green'], 0.5), + RunGroup(2467, 1, "2.0", colours['green'], 0.8), + RunGroup(2468, 1, "2.0", colours['green'], 1.0), + RunGroup(2469, 1, "2.0", colours['green'], 1.2), +] + +fig_num = "AG(b)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_type"] = "2d" +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} 2D scan of its mult and prec (noise sd=3, cov=0.5)" +fc["filename"] = f"Fig {fig_num}.svg" +fc["xaxis_title"] = "Precision (lower image)" +fc["xaxis_range"] = [0, 1.2] +fc["yaxis_range"] = [67, 88] +fc["groups"] = [ + RunGroup(2470, 1, "x 0.5", colours['red'], 0), + RunGroup(2471, 1, "x 0.5", colours['red'], 0.002), + RunGroup(2472, 1, "x 0.5", colours['red'], 0.005), + RunGroup(2473, 1, "x 0.5", colours['red'], 0.01), + RunGroup(2474, 1, "x 0.5", colours['red'], 0.02), + RunGroup(2475, 1, "x 0.5", colours['red'], 0.027), + RunGroup(2476, 1, "x 0.5", colours['red'], 0.038), + RunGroup(2477, 1, "x 0.5", colours['red'], 0.059), + RunGroup(2478, 1, "x 0.5", colours['red'], 0.1), + RunGroup(2479, 1, "x 0.5", colours['red'], 0.2), + RunGroup(2480, 1, "x 0.5", colours['red'], 0.5), + RunGroup(2481, 1, "x 0.5", colours['red'], 0.8), + RunGroup(2482, 1, "x 0.5", colours['red'], 1.0), + RunGroup(2483, 1, "x 0.5", colours['red'], 1.2), + RunGroup(1458, 6, "x 1.0", colours['blue'], 0), + RunGroup(2178, 6, "x 1.0", colours['blue'], 0.002), + RunGroup(2184, 6, "x 1.0", colours['blue'], 0.005), + RunGroup(2190, 6, "x 1.0", colours['blue'], 0.01), + RunGroup(2196, 6, "x 1.0", colours['blue'], 0.02), + RunGroup(1554, 6, "x 1.0", colours['blue'], 0.027), + RunGroup(1560, 6, "x 1.0", colours['blue'], 0.038), + RunGroup(1464, 6, "x 1.0", colours['blue'], 0.059), + RunGroup(1470, 6, "x 1.0", colours['blue'], 0.1), + RunGroup(1477, 6, "x 1.0", colours['blue'], 0.2), + RunGroup(1482, 6, "x 1.0", colours['blue'], 0.5), + RunGroup(1488, 6, "x 1.0", colours['blue'], 0.8), + RunGroup(1494, 6, "x 1.0", colours['blue'], 1.0), + RunGroup(1500, 6, "x 1.0", colours['blue'], 1.2), + RunGroup(2484, 1, "x 2.0", colours['green'], 0), + RunGroup(2485, 1, "x 2.0", colours['green'], 0.002), + RunGroup(2486, 1, "x 2.0", colours['green'], 0.005), + RunGroup(2487, 1, "x 2.0", colours['green'], 0.01), + RunGroup(2488, 1, "x 2.0", colours['green'], 0.02), + RunGroup(2489, 1, "x 2.0", colours['green'], 0.027), + RunGroup(2490, 1, "x 2.0", colours['green'], 0.038), + RunGroup(2491, 1, "x 2.0", colours['green'], 0.059), + RunGroup(2492, 1, "x 2.0", colours['green'], 0.1), + RunGroup(2493, 1, "x 2.0", colours['green'], 0.2), + RunGroup(2494, 1, "x 2.0", colours['green'], 0.5), + RunGroup(2495, 1, "x 2.0", colours['green'], 0.8), + RunGroup(2496, 1, "x 2.0", colours['green'], 1.0), + RunGroup(2497, 1, "x 2.0", colours['green'], 1.2), +] + +fig_num = "AQ" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_type"] = "2d" +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} 2D scan of its mult and prec (noise sd=0, cov=0.5)" +fc["filename"] = f"Fig {fig_num}.svg" +fc["xaxis_title"] = "Precision (lower image)" +fc["xaxis_range"] = [0, 1.2] +fc["yaxis_range"] = [7, 88] +fc["groups"] = [ + RunGroup(2498, 1, "0.5", colours['red'], 0), + RunGroup(2499, 1, "0.5", colours['red'], 0.002), + RunGroup(2500, 1, "0.5", colours['red'], 0.005), + RunGroup(2501, 1, "0.5", colours['red'], 0.01), + RunGroup(2502, 1, "0.5", colours['red'], 0.02), + RunGroup(2503, 1, "0.5", colours['red'], 0.027), + RunGroup(2504, 1, "0.5", colours['red'], 0.038), + RunGroup(2505, 1, "0.5", colours['red'], 0.059), + RunGroup(2506, 1, "0.5", colours['red'], 0.1), + RunGroup(2507, 1, "0.5", colours['red'], 0.2), + RunGroup(2508, 1, "0.5", colours['red'], 0.5), + RunGroup(2509, 1, "0.5", colours['red'], 0.8), + RunGroup(2510, 1, "0.5", colours['red'], 1.0), + RunGroup(2511, 1, "0.5", colours['red'], 1.2), + RunGroup(1266, 6, "1.0", colours['blue'], 0), + RunGroup(2082, 6, "1.0", colours['blue'], 0.002), + RunGroup(2088, 6, "1.0", colours['blue'], 0.005), + RunGroup(2094, 6, "1.0", colours['blue'], 0.01), + RunGroup(2100, 6, "1.0", colours['blue'], 0.02), + RunGroup(1506, 6, "1.0", colours['blue'], 0.027), + RunGroup(1512, 6, "1.0", colours['blue'], 0.038), + RunGroup(1272, 6, "1.0", colours['blue'], 0.059), + RunGroup(1278, 6, "1.0", colours['blue'], 0.1), + RunGroup(1284, 6, "1.0", colours['blue'], 0.2), + RunGroup(1290, 6, "1.0", colours['blue'], 0.5), + RunGroup(1296, 6, "1.0", colours['blue'], 0.8), + RunGroup(1302, 6, "1.0", colours['blue'], 1.0), + RunGroup(1308, 6, "1.0", colours['blue'], 1.2), + RunGroup(2512, 1, "2.0", colours['green'], 0), + RunGroup(2513, 1, "2.0", colours['green'], 0.002), + RunGroup(2514, 1, "2.0", colours['green'], 0.005), + RunGroup(2515, 1, "2.0", colours['green'], 0.01), + RunGroup(2516, 1, "2.0", colours['green'], 0.02), + RunGroup(2517, 1, "2.0", colours['green'], 0.027), + RunGroup(2518, 1, "2.0", colours['green'], 0.038), + RunGroup(2519, 1, "2.0", colours['green'], 0.059), + RunGroup(2520, 1, "2.0", colours['green'], 0.1), + RunGroup(2521, 1, "2.0", colours['green'], 0.2), + RunGroup(2522, 1, "2.0", colours['green'], 0.5), + RunGroup(2523, 1, "2.0", colours['green'], 0.8), + RunGroup(2524, 1, "2.0", colours['green'], 1.0), + RunGroup(2525, 1, "2.0", colours['green'], 1.2), +] + +fig_num = "AR" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_type"] = "2d" +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} 2D scan of its mult and prec B (noise sd=3, cov=0.5)" +fc["filename"] = f"Fig {fig_num}.svg" +fc["xaxis_title"] = "Precision (lower image)" +fc["xaxis_range"] = [0, 1.2] +fc["yaxis_range"] = [7, 88] +fc["groups"] = [ + RunGroup(2530, 1, "1/P", colours['red'], 0), + RunGroup(2531, 1, "1/P", colours['red'], 0.002), + RunGroup(2532, 1, "1/P", colours['red'], 0.005), + RunGroup(2533, 1, "1/P", colours['red'], 0.01), + RunGroup(2534, 1, "1/P", colours['red'], 0.02), + RunGroup(2535, 1, "1/P", colours['red'], 0.027), + RunGroup(2536, 1, "1/P", colours['red'], 0.038), + RunGroup(2537, 1, "1/P", colours['red'], 0.059), + RunGroup(2538, 1, "1/P", colours['red'], 0.1), + RunGroup(2539, 1, "1/P", colours['red'], 0.2), + RunGroup(2540, 1, "1/P", colours['red'], 0.5), + RunGroup(2541, 1, "1/P", colours['red'], 0.8), + RunGroup(2542, 1, "1/P", colours['red'], 1.0), + RunGroup(2543, 1, "1/P", colours['red'], 1.2), + RunGroup(1458, 6, "1.0", colours['blue'], 0), + RunGroup(2178, 6, "1.0", colours['blue'], 0.002), + RunGroup(2184, 6, "1.0", colours['blue'], 0.005), + RunGroup(2190, 6, "1.0", colours['blue'], 0.01), + RunGroup(2196, 6, "1.0", colours['blue'], 0.02), + RunGroup(1554, 6, "1.0", colours['blue'], 0.027), + RunGroup(1560, 6, "1.0", colours['blue'], 0.038), + RunGroup(1464, 6, "1.0", colours['blue'], 0.059), + RunGroup(1470, 6, "1.0", colours['blue'], 0.1), + RunGroup(1477, 6, "1.0", colours['blue'], 0.2), + RunGroup(1482, 6, "1.0", colours['blue'], 0.5), + RunGroup(1488, 6, "1.0", colours['blue'], 0.8), + RunGroup(1494, 6, "1.0", colours['blue'], 1.0), + RunGroup(1500, 6, "1.0", colours['blue'], 1.2), +] + +fig_num = "AS" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} bottom half noise, bottom half layer 3 prec scan, FashionMNIST" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + # RunGroup(817, 6, "0.1", colours['darkred'], None), + RunGroup(2571, 6, "0.2", colours['red'], None), + RunGroup(2565, 6, "0.5", colours['orange'], None), + # RunGroup(835, 6, "0.8", colours['green'], None), + RunGroup(2559, 6, "1", colours['green'], None), + RunGroup(2577, 1, "2", colours['cyan'], None), + RunGroup(2578, 1, "5", colours['blue'], None), + # RunGroup(859, 6, "8", colours['grey25'], None), + # RunGroup(865, 6, "10", colours['darkblue'], None), + RunGroup(2547, 6, "No noise", colours['grey25'], None), +] + +show_figs = True +save_figs = True +figs = { + # "D(a)": NickFig(**fig_cfgs["D(a)"], show_fig=show_figs, save_fig=save_figs), + # "D(b)": NickFig(**fig_cfgs["D(b)"], show_fig=show_figs, save_fig=save_figs), + # "D(c)": NickFig(**fig_cfgs["D(c)"], show_fig=show_figs, save_fig=save_figs), + # "G(a)": NickFig(**fig_cfgs["G(a)"], show_fig=show_figs, save_fig=save_figs), + # "G(b)": NickFig(**fig_cfgs["G(b)"], show_fig=show_figs, save_fig=save_figs), + # "I(a)": NickFig(**fig_cfgs["I(a)"], show_fig=show_figs, save_fig=save_figs), + # "I(b)": NickFig(**fig_cfgs["I(b)"], show_fig=show_figs, save_fig=save_figs), + # "K(a)": NickFig(**fig_cfgs["K(a)"], show_fig=show_figs, save_fig=save_figs), + # "K(b)": NickFig(**fig_cfgs["K(b)"], show_fig=show_figs, save_fig=save_figs), + # "L": NickFig(**fig_cfgs["L"], show_fig=show_figs, save_fig=save_figs), + # "M(a)": NickFig(**fig_cfgs["M(a)"], show_fig=show_figs, save_fig=save_figs), + # "M(b)": NickFig(**fig_cfgs["M(b)"], show_fig=show_figs, save_fig=save_figs), + # "M(c)": NickFig(**fig_cfgs["M(c)"], show_fig=show_figs, save_fig=save_figs), + # "N(a)": NickFig(**fig_cfgs["N(a)"], show_fig=show_figs, save_fig=save_figs), + # "N(b)": NickFig(**fig_cfgs["N(b)"], show_fig=show_figs, save_fig=save_figs), + # "P(a)": NickFig(**fig_cfgs["P(a)"], show_fig=show_figs, save_fig=save_figs), + # "P(b)": NickFig(**fig_cfgs["P(b)"], show_fig=show_figs, save_fig=save_figs), + # "Q": NickFig(**fig_cfgs["Q"], show_fig=show_figs, save_fig=save_figs), + # "R": NickFig(**fig_cfgs["R"], show_fig=show_figs, save_fig=save_figs), + # "S": NickFig(**fig_cfgs["S"], show_fig=show_figs, save_fig=save_figs), + # "T": NickFig(**fig_cfgs["T"], show_fig=show_figs, save_fig=save_figs), + # "U": NickFig(**fig_cfgs["U"], show_fig=show_figs, save_fig=save_figs), + # "V": NickFig(**fig_cfgs["V"], show_fig=show_figs, save_fig=save_figs), + "AC": NickFig(**fig_cfgs["AC"], show_fig=show_figs, save_fig=save_figs), + # "AD": NickFig(**fig_cfgs["AD"], show_fig=show_figs, save_fig=save_figs), + # "AF": NickFig(**fig_cfgs["AF"], show_fig=show_figs, save_fig=save_figs), + # "AG(a)": NickFig(**fig_cfgs["AG(a)"], show_fig=show_figs, save_fig=save_figs), + # "AH": NickFig(**fig_cfgs["AH"], show_fig=show_figs, save_fig=save_figs), + # "AG(c)": NickFig(**fig_cfgs["AG(c)"], show_fig=show_figs, save_fig=save_figs), + # "AK": NickFig(**fig_cfgs["AK"], show_fig=show_figs, save_fig=save_figs), + # "AL": NickFig(**fig_cfgs["AL"], show_fig=show_figs, save_fig=save_figs), + # "AM": NickFig(**fig_cfgs["AM"], show_fig=show_figs, save_fig=save_figs), + # "AN": NickFig(**fig_cfgs["AN"], show_fig=show_figs, save_fig=save_figs), + # "AG(b)": NickFig(**fig_cfgs["AG(b)"], show_fig=show_figs, save_fig=save_figs), + # "AQ": NickFig(**fig_cfgs["AQ"], show_fig=show_figs, save_fig=save_figs), + # "AR": NickFig(**fig_cfgs["AR"], show_fig=show_figs, save_fig=save_figs), + # "AS": NickFig(**fig_cfgs["AS"], show_fig=show_figs, save_fig=save_figs), +} + +# app = Dash(__name__) +# +# app.layout = html.Div(children=[ +# html.H1(children='Hello Dash'), +# +# html.Div(children=''' +# Dash: A web application framework for your data. +# '''), +# +# dcc.Graph( +# id='figD', +# figure=figs["D"].get_fig() +# ), +# dcc.Graph( +# id='figE', +# figure=figs["E"].get_fig() +# ), +# ]) +# +# if __name__ == '__main__': +# app.run_server(debug=False) From 6639b982d6f23e7827d7f57b47c68aa96a842d05 Mon Sep 17 00:00:00 2001 From: Nick Hay <83538768+lasermanick@users.noreply.github.com> Date: Thu, 28 Mar 2024 16:59:39 +0000 Subject: [PATCH 19/30] Update datasets.py Add transforms for gaussian additive noise and update MNIST and FashionMNIST classes to optionally use these. Add scaling to FashionMNIST (following MNIST). Add save_csv() and plot_imgs_alt() methods. Add some comments. --- pypc/datasets.py | 109 +++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 101 insertions(+), 8 deletions(-) diff --git a/pypc/datasets.py b/pypc/datasets.py index ab7ce62..22e05ff 100644 --- a/pypc/datasets.py +++ b/pypc/datasets.py @@ -1,4 +1,5 @@ import numpy as np +import pandas as pd import matplotlib.pyplot as plt import torch from torch.utils import data @@ -6,20 +7,52 @@ from pypc import utils +class GaussianAddNoise(object): + def __init__(self, mean=0.0, std=1.0, coverage=1.0): + self.std = std + self.mean = mean + self.coverage = max(0.0, min(1.0, coverage)) # Clamp to range [0, 1] + + def __call__(self, input): + noise = torch.randn(input.size()) * self.std + self.mean + rows = input.size(1) + cols = input.size(2) + first_noise_row = int(rows*(1.0 - self.coverage)) + noise[0, 0:first_noise_row, :] = torch.zeros(cols) + return input + noise + +class PerPixelGaussianAddNoise(object): + def __init__(self, per_pixel_scaling, mean=0.0, coverage=1.0): + self.mean = mean + self.coverage = max(0.0, min(1.0, coverage)) # Clamp to range [0, 1] + self.per_pixel_scaling = per_pixel_scaling + + def __call__(self, input): + noise = torch.randn(input.size()) # std = 1, mean = 0 + noise = (noise * self.per_pixel_scaling) + self.mean + rows = input.size(1) + cols = input.size(2) + first_noise_row = int(rows*(1.0 - self.coverage)) + noise[0, 0:first_noise_row, :] = torch.zeros(cols) + return input + noise class MNIST(datasets.MNIST): - def __init__(self, train, size=None, scale=None, normalize=False): + def __init__(self, train, size=None, scale=None, normalize=False, add_noise=False, noise_mean=0.0, noise_std=1.0, noise_coverage=1.0, noise_per_pixel_scaling=None): """ Load MNIST dataset, scale to range [0,1], optionally normalise with mean = 0 and std dev = 1, - optionally set label scale factor, optionally limit size of dataset + optionally set label scale factor, optionally limit size of dataset, optionally add noise :param train: True for training data, False for test data :param size: Number of samples to keep in the dataset :param scale: Scale factor for one-hot labels (e.g. scale=1 => 0 or 1, scale=0.5 => 0.25 or 0.75, scale=0.1 => 0.45 or 0.55) :param normalize: True to normalize + :param add_noise: True to add Gaussian noise + :param noise_mean: Gaussian noise mean + :param noise_std: Gaussian noise standard deviation + :param noise_coverage: Gaussian noise coverage (e.g. 1.0=full image, 0.5=bottom half, 0.0=none) """ - transform = _get_transform(normalize=normalize, mean=(0.1307), std=(0.3081)) # Transform to mean=0, std=1 + transform = _get_transform(normalize=normalize, mean=(0.1307), std=(0.3081), add_noise=add_noise, noise_mean=noise_mean, noise_std=noise_std, noise_coverage=noise_coverage, noise_per_pixel_scaling=noise_per_pixel_scaling) # Transform to mean=0, std=1 super().__init__("./data/mnist", download=True, transform=transform, train=train) self.scale = scale if size is not None: @@ -129,19 +162,48 @@ def _reduce(self, size): class FashionMNIST(datasets.FashionMNIST): - def __init__(self, train, size=None, normalize=False): - transform = _get_transform(normalize=normalize, mean=(0.5), std=(0.5)) + def __init__(self, train, size=None, scale=None, normalize=False, add_noise=False, noise_mean=0.0, noise_std=1.0, noise_coverage=1.0, noise_per_pixel_scaling=None): + """ + Load FashionMNIST dataset, scale to range [0,1], optionally normalise with mean = 0 and std dev = 1, + optionally set label scale factor, optionally limit size of dataset, optionally add noise + + :param train: True for training data, False for test data + :param size: Number of samples to keep in the dataset + :param scale: Scale factor for one-hot labels (e.g. scale=1 => 0 or 1, scale=0.5 => 0.25 or 0.75, + scale=0.1 => 0.45 or 0.55) + :param normalize: True to normalize + :param add_noise: True to add Gaussian noise + :param noise_mean: Gaussian noise mean + :param noise_std: Gaussian noise standard deviation + :param noise_coverage: Gaussian noise coverage (e.g. 1.0=full image, 0.5=bottom half, 0.0=none) + """ + transform = _get_transform(normalize=normalize, mean=(0.5), std=(0.5), add_noise=add_noise, noise_mean=noise_mean, noise_std=noise_std, noise_coverage=noise_coverage, noise_per_pixel_scaling=noise_per_pixel_scaling) # Transform to mean=0, std=1 super().__init__("./data/mnist", download=True, transform=transform, train=train) + self.scale = scale if size is not None: self._reduce(size) def __getitem__(self, index): + """ + Return image (data) and label (target) with image converted from (1,28,28) to (784,) and label converted to + one-hot encoding (optionally scaled) + + :param index: Index + :return: image, label + """ data, target = super().__getitem__(index) data = _to_vector(data) target = _one_hot(target) + if self.scale is not None: + target = _scale(target, self.scale) return data, target def _reduce(self, size): + """ + Crop the dataset + + :param size: Maximum sample number to retain + """ self.data = self.data[0:size] self.targets = self.targets[0:size] @@ -168,18 +230,43 @@ def accuracy(pred_labels, true_labels): return correct / batch_size -def plot_imgs(img_preds, path): +def save_csv(tensor_data, path): + pd.DataFrame(tensor_data.cpu().numpy()).to_csv(path) + + +def plot_imgs(img_preds, path, cmap="gray"): imgs = img_preds.cpu().detach().numpy() imgs = imgs[0:10, :] imgs = [np.reshape(imgs[i, :], [28, 28]) for i in range(imgs.shape[0])] _, axes = plt.subplots(2, 5) axes = axes.flatten() for i, img in enumerate(imgs): - axes[i].imshow(img, cmap="gray") + axes[i].imshow(img, cmap=cmap) plt.savefig(path) plt.close("all") +def plot_imgs_alt(img_preds, path=None, cmap="gray"): + images = img_preds.cpu().detach().numpy() + fig, axes = plt.subplots(2, 5) + fig.set_size_inches(8, 3) + fig.set_dpi(150) + axes = axes.flatten() + plt.setp(axes, xticks=[0, 27]) + plt.setp(axes, yticks=[0, 27]) + for i in range(10): + axes[i].tick_params(top=False, labeltop=False, bottom=False, labelbottom=False, width=2) + axes[i].tick_params(left=False, labelleft=False, right=False, labelright=False, width=2) + axes[i].imshow(images[i].reshape(28, 28), cmap=cmap) + axes[0].tick_params(top=True, labeltop=True, bottom=False, labelbottom=False, labelsize=16) + axes[0].tick_params(left=True, labelleft=True, right=False, labelright=False, labelsize=16) + + if path: + plt.savefig(path) + plt.show() + plt.close("all") + + def _preprocess_batch(batch): """ Pre-process a batch to move data onto the selected cpu/cuda device with dtype=torch.float32 @@ -192,7 +279,7 @@ def _preprocess_batch(batch): return (batch[0], batch[1]) -def _get_transform(normalize=True, mean=0.0, std=1.0): +def _get_transform(normalize=True, mean=0.0, std=1.0, add_noise=False, noise_mean=0.0, noise_std=1.0, noise_coverage=1.0, noise_per_pixel_scaling=None): """ Define transformation to convert PIL image or numpy.ndarray to tensor with optional normalization @@ -204,6 +291,12 @@ def _get_transform(normalize=True, mean=0.0, std=1.0): transform = [transforms.ToTensor()] if normalize: transform += [transforms.Normalize(mean=mean, std=std)] + if add_noise: + if noise_per_pixel_scaling is None: + transform += [GaussianAddNoise(mean=noise_mean, std=noise_std, coverage=noise_coverage)] + else: + transform += [PerPixelGaussianAddNoise(noise_per_pixel_scaling, mean=noise_mean, coverage=noise_coverage)] + return transforms.Compose(transform) From a756c1d43437acacb4b0cfcf97c0722eef3788e3 Mon Sep 17 00:00:00 2001 From: Nick Hay <83538768+lasermanick@users.noreply.github.com> Date: Thu, 28 Mar 2024 17:05:03 +0000 Subject: [PATCH 20/30] Update models.py Add comments. Add free energy calculation. Add logging to Neptune. Add code for precision scale factor and precision coverage. --- pypc/models.py | 123 +++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 104 insertions(+), 19 deletions(-) diff --git a/pypc/models.py b/pypc/models.py index a7fb5dd..09dc729 100644 --- a/pypc/models.py +++ b/pypc/models.py @@ -1,11 +1,12 @@ import torch +import numpy as np from pypc import utils from pypc.layers import FCLayer class PCModel(object): - def __init__(self, nodes, mu_dt, act_fn, use_bias=False, kaiming_init=False, use_precis=False, precis=None): + def __init__(self, nodes, mu_dt, act_fn, use_bias=False, kaiming_init=False, use_precis=False, precis_factor=None, precis_coverage=None, precis_per_pixel=None, run_log=None, log_node_its=False): """ Define the Predictive Coding PyTorch model. All layers fully connected. All nodes using the specified activation function except for the output layer which is linear. Bias terms are optional. Kaiming weight initialisation is @@ -17,8 +18,12 @@ def __init__(self, nodes, mu_dt, act_fn, use_bias=False, kaiming_init=False, use :param use_bias: Include bias terms? :param kaiming_init: Use Kaiming weight initialisation? :param use_precis: Use precisions? (If False, precisions are implied to be identity matrices) - :param precis: List of precision scaling per node layer + :param precis_factor: List of precision scaling per node layer + :param precis_coverage: List of precision coverage by node layer or None for full coverage + :param run_log: A neptune.ai run object or None if logging not required + :param log_node_its: Log stats after every node update iteration? (WARNING: Slow!) """ + self.nodes = nodes self.mu_dt = mu_dt @@ -39,12 +44,33 @@ def __init__(self, nodes, mu_dt, act_fn, use_bias=False, kaiming_init=False, use self.layers.append(layer) self.use_precis = use_precis - # If precisions used, create list of diagonal precision matrices with given scale factor + self.precis_coverage = precis_coverage + # If precisions used, create: + # List of diagonal precision matrices with given scale factor + # List of diagonal variance matrices (inverse of precision matrices) (easy for diagonal precision) + # List of variance determinants (for free energy sum) # NOTE: Precisions are fixed, so can be defined here. Not suitable for dynamic (learned) precisions if self.use_precis: self.precis = [[] for _ in range(self.n_nodes)] + # self.varis = [[] for _ in range(self.n_nodes)] + # self.vari_dets = [1.0 for _ in range(self.n_nodes)] for n in range(self.n_nodes): - self.precis[n] = utils.set_tensor(torch.diagflat(precis[n]*torch.ones(nodes[n]))) + if (n == self.n_nodes - 1) and precis_per_pixel is not None: + temp = precis_per_pixel.flatten() + else: + temp = precis_factor[n] * torch.ones(nodes[n]) + if self.precis_coverage is not None: + coverage = max(0.0, min(1.0, self.precis_coverage[n])) # Clamp to range [0, 1] + first_scaled_precision = int(self.nodes[n] * (1.0 - coverage)) + temp[0:first_scaled_precision] = torch.ones(first_scaled_precision) + self.precis[n] = utils.set_tensor(torch.diagflat(temp)) + # TODO: Need to rework below to account for precision coverage + # self.varis[n] = utils.set_tensor(torch.diagflat((1.0/precis_factor[n])*torch.ones(nodes[n]))) + # self.vari_dets[n] = (1.0/precis_factor[n])**nodes[n] + # self.loggy = np.log(2.0*np.pi*self.vari_dets[n]) + + self.run_log = run_log + self.log_node_its = log_node_its def reset(self): """ @@ -53,8 +79,15 @@ def reset(self): self.preds = [[] for _ in range(self.n_nodes)] self.errs = [[] for _ in range(self.n_nodes)] self.mus = [[] for _ in range(self.n_nodes)] + self.free_energy = [0.0 for _ in range(self.n_nodes)] def reset_mus(self, batch_size, init_std): + """ + For each layer, initialise variational means (mus) to be normally distributed with mean=0 + + :param batch_size: Number of samples in current batch + :param init_std: Initial standard deviation of mus + """ for l in range(self.n_layers): self.mus[l] = utils.set_tensor( torch.empty(batch_size, self.layers[l].in_size).normal_(mean=0, std=init_std) @@ -104,7 +137,13 @@ def train_batch_supervised(self, img_batch, label_batch, n_iters, fixed_preds=Fa self.train_updates(n_iters, fixed_preds=fixed_preds) # Iteratively update mus, predictions and errors self.update_grads() # Calculate gradients of weights and biases for all layers - def train_batch_generative(self, img_batch, label_batch, n_iters, fixed_preds=False): + # def set_precisions_by_per_pixel_variance(self, img_batch): + # per_pixel_variance = img_batch.var(dim=0) + # # per_pixel_precision = 1.0 / (per_pixel_variance + 1e-1) + # per_pixel_precision = 1.0 / per_pixel_variance.clamp(min=0.1) + # self.precis[3] = torch.diagflat(per_pixel_precision) + + def train_batch_generative(self, img_batch, label_batch, n_iters, fixed_preds=False, log_batch=False): """ Train the model using the (mini)batch labels as inputs and images as targets (Identical to train_batch_supervised() but inputs and targets are swapped) @@ -118,21 +157,46 @@ def train_batch_generative(self, img_batch, label_batch, n_iters, fixed_preds=Fa self.set_input(label_batch) # Set the model inputs, mus[0], equal to the training *labels* self.propagate_mu() # Perform forward pass, update mus for all nodes except inputs and targets self.set_target(img_batch) # Set the model outputs (targets), mus[-1], equal to the training *images* - self.train_updates(n_iters, fixed_preds=fixed_preds) # Iteratively update mus, predictions and errors + self.train_updates(n_iters, fixed_preds=fixed_preds, log_batch=log_batch) # Iteratively update mus, predictions and errors self.update_grads() # Calculate gradients of weights and biases for all layers + if self.run_log and log_batch: + self.log_layer_stats(prefix="train") + + def log_layer_stats(self, prefix="layer"): + for l in range(self.n_layers): + self.run_log[f"{prefix}/grad_weight_mean_{l}]"].log(torch.mean(self.layers[l].grad["weights"])) + self.run_log[f"{prefix}/grad_bias_mean_{l}]"].log(torch.mean(self.layers[l].grad["bias"])) + self.run_log[f"{prefix}/grad_weight_std_{l}]"].log(torch.std(self.layers[l].grad["weights"])) + self.run_log[f"{prefix}/grad_bias_std_{l}]"].log(torch.std(self.layers[l].grad["bias"])) + self.run_log[f"{prefix}/weight_mean_{l}]"].log(torch.mean(self.layers[l].weights)) + self.run_log[f"{prefix}/bias_mean_{l}]"].log(torch.mean(self.layers[l].bias)) + self.run_log[f"{prefix}/weight_std_{l}]"].log(torch.std(self.layers[l].weights)) + self.run_log[f"{prefix}/bias_std_{l}]"].log(torch.std(self.layers[l].bias)) + + def log_node_stats(self, prefix="node"): + for n in range(1, self.n_nodes): + self.run_log[f"{prefix}/err_mean_{n}]"].log(torch.mean(self.errs[n])) + self.run_log[f"{prefix}/pred_mean_{n}"].log(torch.mean(self.preds[n])) + self.run_log[f"{prefix}/mu_mean_{n}"].log(torch.mean(self.mus[n])) + self.run_log[f"{prefix}/err_std_{n}]"].log(torch.std(self.errs[n])) + self.run_log[f"{prefix}/pred_std_{n}"].log(torch.std(self.preds[n])) + self.run_log[f"{prefix}/mu_std_{n}"].log(torch.std(self.mus[n])) + self.run_log[f"{prefix}/free_e_{n}"].log(self.free_energy[n]) + self.run_log[f"{prefix}/free_e"].log(sum(self.free_energy)) def test_batch_supervised(self, img_batch): return self.forward(img_batch) - def test_batch_generative(self, img_batch, n_iters, init_std=0.05, fixed_preds=False): + def test_batch_generative(self, img_batch, n_iters, init_std=0.05, fixed_preds=False, log_batch=False): batch_size = img_batch.size(0) - self.reset() - self.reset_mus(batch_size, init_std) - self.set_target(img_batch) - self.test_updates(n_iters, fixed_preds=fixed_preds) + self.reset() # Initialise the prediction, error, and mu data structures + self.reset_mus(batch_size, init_std) # Initialise variational means (mus) + self.set_target(img_batch) # Set output node mus for batch to target values (test images) + self.test_updates(n_iters, fixed_preds=fixed_preds, log_batch=log_batch) + # self.update_grads() # Calculate gradients of weights and biases for all layers - NOT REQUIRED return self.mus[0] - def train_updates(self, n_iters, fixed_preds=False): + def train_updates(self, n_iters, fixed_preds=False, log_batch=False): """ Iteratively update mus, predictions and errors @@ -140,30 +204,40 @@ def train_updates(self, n_iters, fixed_preds=False): :param fixed_preds: Fix predictions at initial values? """ # For batch, initialise predictions and errors for all nodes except inputs - # Optionally, errors are precision scaled + # Optionally, errors are precision scaled and free energy is calculated for n in range(1, self.n_nodes): self.preds[n] = self.layers[n - 1].forward(self.mus[n - 1]) self.errs[n] = self.mus[n] - self.preds[n] if self.use_precis: + self.free_energy[n] = torch.mean(self.errs[n] @ self.precis[n] @ self.errs[n].T).item() self.errs[n] = torch.matmul(self.errs[n], self.precis[n]) + # Log values to neptune + if self.run_log and log_batch and self.log_node_its: + self.log_node_stats(prefix="train") + # For each training iteration for itr in range(n_iters): - # For batch, update mus for all nodes except inputs and outputs + # For batch, update mus for all nodes except inputs (labels) and outputs (images) for l in range(1, self.n_layers): delta = self.layers[l].backward(self.errs[l + 1]) - self.errs[l] self.mus[l] = self.mus[l] + self.mu_dt * delta # For batch, update errors and (optionally) predictions for all nodes except inputs - # Optionally, errors are precision scaled + # Optionally, errors are precision scaled and free energy is calculated for n in range(1, self.n_nodes): if not fixed_preds: self.preds[n] = self.layers[n - 1].forward(self.mus[n - 1]) self.errs[n] = self.mus[n] - self.preds[n] if self.use_precis: + self.free_energy[n] = torch.mean(self.errs[n] @ self.precis[n] @ self.errs[n].T).item() self.errs[n] = torch.matmul(self.errs[n], self.precis[n]) - def test_updates(self, n_iters, fixed_preds=False): + # Log values to neptune + if self.run_log and log_batch and (self.log_node_its or (itr == n_iters-1)): + self.log_node_stats(prefix="train") + + def test_updates(self, n_iters, fixed_preds=False, log_batch=False): """ Test model @@ -171,15 +245,22 @@ def test_updates(self, n_iters, fixed_preds=False): :param fixed_preds: Fix predictions at initial values? """ # For batch, initialise predictions and errors for all nodes except inputs - # Optionally, errors are precision scaled + # Optionally, errors are precision scaled and free energy is calculated for n in range(1, self.n_nodes): self.preds[n] = self.layers[n - 1].forward(self.mus[n - 1]) self.errs[n] = self.mus[n] - self.preds[n] if self.use_precis: + self.free_energy[n] = torch.mean(self.errs[n] @ self.precis[n] @ self.errs[n].T).item() self.errs[n] = torch.matmul(self.errs[n], self.precis[n]) + # Log values to neptune + if self.run_log and log_batch and self.log_node_its: + self.log_node_stats(prefix="test") + # For each test iteration for itr in range(n_iters): + # For batch, update mus for all nodes except outputs (images) + # NOTE: Unlike training which also does not update inputs (labels) delta = self.layers[0].backward(self.errs[1]) self.mus[0] = self.mus[0] + self.mu_dt * delta for l in range(1, self.n_layers): @@ -187,14 +268,19 @@ def test_updates(self, n_iters, fixed_preds=False): self.mus[l] = self.mus[l] + self.mu_dt * delta # For batch, update errors and (optionally) predictions for all nodes except inputs - # Optionally, errors are precision scaled + # Optionally, errors are precision scaled and free energy is calculated for n in range(1, self.n_nodes): if not fixed_preds: self.preds[n] = self.layers[n - 1].forward(self.mus[n - 1]) self.errs[n] = self.mus[n] - self.preds[n] if self.use_precis: + self.free_energy[n] = torch.mean(self.errs[n] @ self.precis[n] @ self.errs[n].T).item() self.errs[n] = torch.matmul(self.errs[n], self.precis[n]) + # Log values to neptune + if self.run_log and log_batch and (self.log_node_its or (itr == n_iters-1)): + self.log_node_stats(prefix="test") + def update_grads(self): """ Calculate gradients of weights and biases for all layers @@ -221,4 +307,3 @@ def params(self): :return: Model layers """ return self.layers - From a4a0488a064cdc9e0d2ac2a2ee4d24706bb225d1 Mon Sep 17 00:00:00 2001 From: Nick Hay <83538768+lasermanick@users.noreply.github.com> Date: Thu, 28 Mar 2024 17:06:11 +0000 Subject: [PATCH 21/30] Update optim.py Replace improper use of "is" with "==" x 2. --- pypc/optim.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pypc/optim.py b/pypc/optim.py index 650e461..8756857 100644 --- a/pypc/optim.py +++ b/pypc/optim.py @@ -3,11 +3,11 @@ def get_optim(params, optim_id, lr, q_lr=None, batch_scale=True, grad_clip=None, weight_decay=None): - if optim_id is "Adam": + if optim_id == "Adam": return Adam( params, lr=lr, q_lr=q_lr, batch_scale=batch_scale, grad_clip=grad_clip, weight_decay=weight_decay ) - elif optim_id is "SGD": + elif optim_id == "SGD": return SGD( params, lr=lr, q_lr=q_lr, batch_scale=batch_scale, grad_clip=grad_clip, weight_decay=weight_decay ) From b4bfae39e470650517b63ed96a489c857262bbbc Mon Sep 17 00:00:00 2001 From: Nick Hay <83538768+lasermanick@users.noreply.github.com> Date: Fri, 29 Mar 2024 14:51:18 +0000 Subject: [PATCH 22/30] Update supervised.py Added Neptune logging. NOTE: Needs further updates to be aligned with generative.py. --- scripts/supervised.py | 65 ++++++++++++++++++++++++++++++++++++++----- 1 file changed, 58 insertions(+), 7 deletions(-) diff --git a/scripts/supervised.py b/scripts/supervised.py index cd6b24e..8d97227 100644 --- a/scripts/supervised.py +++ b/scripts/supervised.py @@ -2,6 +2,8 @@ import pprint import torch +import torchvision +import matplotlib.pyplot as plt from tqdm import tqdm from time import sleep @@ -11,6 +13,22 @@ from pypc import optim from pypc.models import PCModel +import neptune +from pypc import constants # Defines secret API_KEY_NEPTUNE for Neptune access + +from torch.utils.tensorboard import SummaryWriter + +# helper function to show an image +# (used in the `plot_classes_preds` function below) +def matplotlib_imshow(img, one_channel=False): + if one_channel: + img = img.mean(dim=0) + img = img / 2 + 0.5 # unnormalize + npimg = img.cpu().numpy() + if one_channel: + plt.imshow(npimg, cmap="Greys") + else: + plt.imshow(np.transpose(npimg, (1, 2, 0))) def main(cf): print(f"\nStarting supervised experiment {cf.logdir}: --seed {cf.seed} --device {utils.DEVICE}") @@ -19,12 +37,17 @@ def main(cf): utils.seed(cf.seed) utils.save_json({k: str(v) for (k, v) in cf.items()}, cf.logdir + "config.json") + print("Loading data...") train_dataset = datasets.MNIST(train=True, scale=cf.label_scale, size=cf.train_size, normalize=cf.normalize) test_dataset = datasets.MNIST(train=False, scale=cf.label_scale, size=cf.test_size, normalize=cf.normalize) train_loader = datasets.get_dataloader(train_dataset, cf.batch_size) test_loader = datasets.get_dataloader(test_dataset, cf.batch_size) print(f"Loaded data [train batches: {len(train_loader)} test batches: {len(test_loader)}]") + run = neptune.init( + project="lasermanick/PYPC", + api_token=constants.API_KEY_NEPTUNE, + ) model = PCModel( nodes=cf.nodes, mu_dt=cf.mu_dt, @@ -33,6 +56,7 @@ def main(cf): kaiming_init=cf.kaiming_init, use_precis=cf.use_precis, precis=cf.precis, + run_log=run, ) optimizer = optim.get_optim( model.params, @@ -43,6 +67,26 @@ def main(cf): weight_decay=cf.weight_decay, ) + params = cf # {"learning_rate": 0.001, "optimizer": "Adam"} + run["parameters"] = params + + + # writer = SummaryWriter(f"{cf.logdir}/tensorboard") + # # get some random training images + # images, labels = train_loader[0] + # images_sq = torch.reshape(images, (6, 1, 28, 28)) + # + # # create grid of images + # img_grid = torchvision.utils.make_grid(images_sq) + # + # # show images + # matplotlib_imshow(img_grid, one_channel=True) + # + # # write to tensorboard + # writer.add_image('six_fashion_mnist_images_b', img_grid) + # + # writer.close() + with torch.no_grad(): metrics = {"acc": []} for epoch in range(1, cf.n_epochs + 1): @@ -72,6 +116,12 @@ def main(cf): utils.save_json(metrics, cf.logdir + "metrics.json") + run["train/acc"].log(acc / len(test_loader)) + + run["eval/f1_score"] = 0.66 # An example only + + run.stop() + if __name__ == "__main__": cf = utils.AttrDict() @@ -87,30 +137,31 @@ def main(cf): cf.logdir = f"data/supervised/{seed}/" # dataset params - cf.train_size = 6000 # None - cf.test_size = 1000 # None + cf.train_size = None + cf.test_size = None cf.label_scale = None cf.normalize = False # optim params cf.optim = "Adam" - cf.lr = 1e-4 - cf.batch_size = 64 + cf.lr = 5e-3 + cf.batch_size = 6400 cf.batch_scale = False cf.grad_clip = 50 cf.weight_decay = None # inference params cf.mu_dt = 0.01 - cf.n_train_iters = 50 - cf.fixed_preds_train = True + cf.n_train_iters = 200 + cf.fixed_preds_train = False # model params cf.use_bias = True - cf.kaiming_init = True # False + cf.kaiming_init = False cf.nodes = [784, 300, 100, 10] cf.act_fn = utils.ReLU() cf.use_precis = False + # cf.precis = [1.0, 625.0, 100.0, 500.0] cf.precis = [1.0, 1.0, 1.0, 1.0] main(cf) From 61a1e5737541030fd46c18131a62492b82ccdb83 Mon Sep 17 00:00:00 2001 From: Nick Hay <83538768+lasermanick@users.noreply.github.com> Date: Fri, 29 Mar 2024 14:56:37 +0000 Subject: [PATCH 23/30] Update generative.py Added Neptune logging. Added plot_example_data_MNIST(). Moved accuracy calculation to calc_accuracy(). Added print_batch_stats(). Made json logging optional. Added saving of label predictions. Added loops for 2D parameter scans. --- scripts/generative.py | 293 ++++++++++++++++++++++++++++++++---------- 1 file changed, 224 insertions(+), 69 deletions(-) diff --git a/scripts/generative.py b/scripts/generative.py index 23593df..ae0f787 100644 --- a/scripts/generative.py +++ b/scripts/generative.py @@ -1,30 +1,120 @@ import os import pprint +import sys import torch from tqdm import tqdm -from time import sleep from pypc import utils from pypc import datasets from pypc import optim from pypc.models import PCModel +import neptune +from pypc import constants # Defines secret API_KEY_NEPTUNE for Neptune access + +from datetime import datetime + +import numpy as np +import matplotlib.pyplot as plt +import pandas as pd + +def plot_example_data_MNIST(dataloader, batch_num=0, path=None, cmap="gray"): + if batch_num >= len(dataloader): + batch_num = -1 + image_batch = dataloader[batch_num][0] + label_batch = dataloader[batch_num][1] + images = image_batch.cpu().detach().numpy() + labels = label_batch.cpu().detach().numpy() + _, indices = np.unique(labels, axis=0, return_index=True) + indices = np.flip(indices) + # HACK to plot first ten images (not first examples of digits 0 to 9) + # indices = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + + fig, axes = plt.subplots(2, 5) + fig.set_size_inches(8, 3) + fig.set_dpi(150) + axes = axes.flatten() + plt.setp(axes, xticks=[0, 27]) + plt.setp(axes, yticks=[0, 27]) + for i in range(10): + axes[i].tick_params(top=False, labeltop=False, bottom=False, labelbottom=False, width=2) + axes[i].tick_params(left=False, labelleft=False, right=False, labelright=False, width=2) + axes[i].imshow(images[indices[i]].reshape(28, 28), cmap=cmap) + axes[0].tick_params(top=True, labeltop=True, bottom=False, labelbottom=False, labelsize=16) + axes[0].tick_params(left=True, labelleft=True, right=False, labelright=False, labelsize=16) + + if path: + plt.savefig(path) + plt.show() + plt.close("all") + +def calc_accuracy(data_loader, model, config, file_prefix="", log_test=False): + acc = 0 + for batch_id, (img_batch, label_batch) in enumerate(tqdm(data_loader, file=sys.stdout)): + # model.set_precisions_by_per_pixel_variance(img_batch) + label_preds = model.test_batch_generative( + img_batch, config.n_test_iters, init_std=config.init_std, fixed_preds=config.fixed_preds_test, log_batch=(log_test and (batch_id == len(data_loader)-1)) + ) + acc += datasets.accuracy(label_preds, label_batch) + datasets.save_csv(label_preds, cf.preddir + "preds_" + f"{file_prefix}b{batch_id:03}.csv") + datasets.save_csv(label_batch, cf.preddir + "actuals_" + f"{file_prefix}b{batch_id:03}.csv") + return acc / len(data_loader) + +def print_batch_stats(img_batch, cf): + first_noisy_pixel = int(img_batch.size(1) * (1.0 - cf.noise_coverage)) + last_noisy_pixel = img_batch.size(1) + print(f"Noise coverage = {cf.noise_coverage:.2f}") + print(f"First noisy pixel = {first_noisy_pixel}") + print(f"Last noisy pixel = {last_noisy_pixel}") + print(f"Full image") + print(f"Batch size = {img_batch.size()}") + print(f"Batch mean = {img_batch.mean():.4f}") + print(f"Batch variance = {img_batch.var():.4f}") + print(f"Batch stddev = {img_batch.std():.4f}") + print(f"Region without added noise") + print(f"Batch size = {img_batch[:, 0:first_noisy_pixel].size()}") + print(f"Batch mean = {img_batch[:, 0:first_noisy_pixel].mean():.4f}") + print(f"Batch variance = {img_batch[:, 0:first_noisy_pixel].var():.4f}") + print(f"Batch stddev = {img_batch[:, 0:first_noisy_pixel].std():.4f}") + print(f"Region with added noise") + print(f"Batch size = {img_batch[:, first_noisy_pixel:last_noisy_pixel].size()}") + print(f"Batch mean = {img_batch[:, first_noisy_pixel:last_noisy_pixel].mean():.4f}") + print(f"Batch variance = {img_batch[:, first_noisy_pixel:last_noisy_pixel].var():.4f}") + print(f"Batch stddev = {img_batch[:, first_noisy_pixel:last_noisy_pixel].std():.4f}") def main(cf): print(f"\nStarting generative experiment {cf.logdir}: --seed {cf.seed} --device {utils.DEVICE}") - pprint.pprint(cf) os.makedirs(cf.logdir, exist_ok=True) os.makedirs(cf.imgdir, exist_ok=True) + os.makedirs(cf.preddir, exist_ok=True) utils.seed(cf.seed) - utils.save_json({k: str(v) for (k, v) in cf.items()}, cf.logdir + "config.json") - train_dataset = datasets.MNIST(train=True, scale=cf.label_scale, size=cf.train_size, normalize=cf.normalize) - test_dataset = datasets.MNIST(train=False, scale=cf.label_scale, size=cf.test_size, normalize=cf.normalize) + pprint.pprint(cf) + if cf.json_enabled: + utils.save_json({k: str(v) for (k, v) in cf.items()}, cf.logdir + "config.json") + if cf.neptune_enabled: + cf.run_log["parameters"] = cf # Log configuration to Neptune + + # Use per pixel scaling of noise (and precision) + # noise_per_pixel_scaling = torch.rand((28, 28)) * (cf.noise_max_std - cf.noise_min_std) + cf.noise_min_std + # precis_per_pixel = 1 / (1 + noise_per_pixel_scaling ** 2.0) + noise_per_pixel_scaling = None + precis_per_pixel = None + + print("Loading data...") + # train_dataset = datasets.MNIST(train=True, scale=cf.label_scale, size=cf.train_size, normalize=cf.normalize, add_noise=cf.add_noise, noise_mean=cf.noise_mean, noise_std=cf.noise_std, noise_coverage=cf.noise_coverage, noise_per_pixel_scaling=noise_per_pixel_scaling) + # test_dataset = datasets.MNIST(train=False, scale=cf.label_scale, size=cf.test_size, normalize=cf.normalize, add_noise=cf.add_noise, noise_mean=cf.noise_mean, noise_std=cf.noise_std, noise_coverage=cf.noise_coverage, noise_per_pixel_scaling=noise_per_pixel_scaling) + train_dataset = datasets.FashionMNIST(train=True, scale=cf.label_scale, size=cf.train_size, normalize=cf.normalize, add_noise=cf.add_noise, noise_mean=cf.noise_mean, noise_std=cf.noise_std, noise_coverage=cf.noise_coverage, noise_per_pixel_scaling=noise_per_pixel_scaling) + test_dataset = datasets.FashionMNIST(train=False, scale=cf.label_scale, size=cf.test_size, normalize=cf.normalize, add_noise=cf.add_noise, noise_mean=cf.noise_mean, noise_std=cf.noise_std, noise_coverage=cf.noise_coverage, noise_per_pixel_scaling=noise_per_pixel_scaling) train_loader = datasets.get_dataloader(train_dataset, cf.batch_size) test_loader = datasets.get_dataloader(test_dataset, cf.batch_size) - print(f"Loaded data [train batches: {len(train_loader)} test batches: {len(test_loader)}]") + print(f"Loaded data [training batches: {len(train_loader)}, test batches: {len(test_loader)}]") + + plot_example_data_MNIST(train_loader, batch_num=0, path=cf.imgdir + "example_training_data.png", cmap='rainbow') + # HACK to plot test data (see also the function) + # plot_example_data_MNIST(test_loader, batch_num=0, path=cf.imgdir + "example_training_data.png", cmap='rainbow') model = PCModel( nodes=cf.nodes, @@ -33,7 +123,11 @@ def main(cf): use_bias=cf.use_bias, kaiming_init=cf.kaiming_init, use_precis=cf.use_precis, - precis=cf.precis, + precis_factor=cf.precis_factor, + precis_coverage=cf.precis_coverage, + precis_per_pixel=precis_per_pixel, + run_log=cf.run_log, + log_node_its=cf.log_node_its, ) optimizer = optim.get_optim( model.params, @@ -45,14 +139,16 @@ def main(cf): ) with torch.no_grad(): # Disable automatic gradient calculation - metrics = {"acc": []} + metrics = {"test_acc": [], "train_acc": []} for epoch in range(1, cf.n_epochs + 1): - print(f"\nTrain @ epoch {epoch} ({len(train_loader)} batches)") - sleep(0.1) - for batch_id, (img_batch, label_batch) in enumerate(tqdm(train_loader, disable=False)): + + # Train each batch + for batch_id, (img_batch, label_batch) in enumerate(tqdm(train_loader, file=sys.stdout)): + # print_batch_stats(img_batch, cf) + # model.set_precisions_by_per_pixel_variance(img_batch) model.train_batch_generative( - img_batch, label_batch, cf.n_train_iters, fixed_preds=cf.fixed_preds_train + img_batch, label_batch, cf.n_train_iters, fixed_preds=cf.fixed_preds_train, log_batch=(batch_id == len(train_loader)-1) ) optimizer.step( curr_epoch=epoch, @@ -61,66 +157,125 @@ def main(cf): batch_size=img_batch.size(0), ) - if epoch % cf.test_every == 0: - print(f"\nTest @ epoch {epoch}") - sleep(0.1) - acc = 0 - for _, (img_batch, label_batch) in enumerate(tqdm(test_loader, disable=False)): - label_preds = model.test_batch_generative( - img_batch, cf.n_test_iters, init_std=cf.init_std, fixed_preds=cf.fixed_preds_test - ) - acc += datasets.accuracy(label_preds, label_batch) - metrics["acc"].append(acc / len(test_loader)) - print("Accuracy: {:.4f}".format(acc / len(test_loader))) + # Evaluate the model at specified intervals + if epoch % cf.eval_every == 0: + print(f"\nEvaluate @ epoch {epoch}") + + # Calculate training accuracy + if cf.eval_train_acc: + acc = calc_accuracy(train_loader, model, cf, f"train_e{epoch:03}_", log_test=False) + print(f"Train accuracy: {acc:.4f}") + metrics["train_acc"].append(acc) + if cf.neptune_enabled: + cf.run_log["train/acc"].log(acc) + + # Calculate test accuracy + if cf.eval_test_acc: + acc = calc_accuracy(test_loader, model, cf, f"test_e{epoch:03}_", log_test=True) + print(f"Test accuracy: {acc:.4f}") + metrics["test_acc"].append(acc) + if cf.neptune_enabled: + cf.run_log["test/acc"].log(acc) + + # Generate image predictions + if cf.eval_images: + # _, label_batch = next(iter(test_loader)) # Use first batch + label_batch = utils.set_tensor(torch.diagflat(torch.ones(10))) # Use each of the one hot encoded labels + img_preds = model.forward(label_batch) + datasets.plot_imgs_alt(img_preds, path=cf.imgdir + f"e{epoch:03}.png", cmap="rainbow") + + # preds = pd.read_csv(r'D:\n\OneDrive\Documents\GitHub\pypc-desktop\scripts\data\230727_101349 - copy for analysis\000\preds\preds_test_e015_b000.csv', nrows=10, index_col=0) + # label_batch = utils.set_tensor(torch.tensor(preds.to_numpy())) + # img_preds = model.forward(label_batch) + # datasets.plot_imgs_alt(img_preds, path=cf.imgdir + f"alt e{epoch:03}.png", cmap="rainbow") - _, label_batch = next(iter(test_loader)) - img_preds = model.forward(label_batch) - datasets.plot_imgs(img_preds, cf.imgdir + f"{epoch}.png") + # Save metrics to json log + if cf.json_enabled: + utils.save_json(metrics, cf.logdir + "metrics.json") - utils.save_json(metrics, cf.logdir + "metrics.json") + if cf.neptune_enabled: + cf.run_log.stop() # Stop logging to Neptune.ai +# INSTRUCTIONS +# For a single point calculation, set scan_1d and scan_2d to [0] and ignore s1d and s2d +# For a 1D scan, set scan_1d to a list of parameter values, set scan_2d to [0], use s1d and ignore s2d +# For a 2D scan, set scan_1d and scan_2d to lists of parameter values, use s1d and s2d +# In all cases, use cf.seeds to repeat runs (and, for easier analysis, gather into a Neptune group if used) if __name__ == "__main__": cf = utils.AttrDict() # Create configuration - cf.seeds = [0] # Create list of >=1 seeds for repeat runs - - for seed in cf.seeds: - - # experiment params - cf.seed = seed - cf.n_epochs = 20 # 20 - cf.test_every = 1 - cf.logdir = f"data/generative/{seed}/" - cf.imgdir = cf.logdir + "imgs/" - - # dataset params - cf.train_size = 6000 # None - cf.test_size = 1000 # None - cf.label_scale = None - cf.normalize = False # True - - # optim params - cf.optim = "Adam" - cf.lr = 1e-4 - cf.batch_size = 64 - cf.batch_scale = True - cf.grad_clip = None - cf.weight_decay = None - - # inference params - cf.mu_dt = 0.01 - cf.n_train_iters = 50 - cf.n_test_iters = 200 - cf.init_std = 0.01 - cf.fixed_preds_train = False - cf.fixed_preds_test = False - - # model params - cf.use_bias = True - cf.kaiming_init = False - cf.nodes = [10, 100, 300, 784] - cf.act_fn = utils.Tanh() - cf.use_precis = False - cf.precis = [1.0, 1.0, 1.0, 1.0] - - main(cf) + # scan_1d = [0, 0.002, 0.005, 0.01, 0.02, 0.027, 0.038, 0.059, 0.1, 0.2, 0.5, 0.8, 1.0, 1.2] # List of parameters for 1D parameter scan + scan_1d = [0] # List of parameters for 1D parameter scan + scan_2d = [0] # List of parameters for 2D parameter scan + for s2d in scan_2d: + for s1d in scan_1d: + cf.seeds = [0] # Create list of >=1 seeds for repeat runs + now = datetime.now().strftime('%y%m%d_%H%M%S') + cf.run_group = "" + for seed in cf.seeds: + # logging params + cf.log_node_its = False # WARNING: Can log A LOT of data to Neptune if True + cf.json_enabled = True + cf.neptune_enabled = True + cf.neptune_mode = "async" # https://docs.neptune.ai/api/connection_modes/ + cf.neptune_project = "lasermanick/PYPC" + cf.neptune_api_token = constants.API_KEY_NEPTUNE + if cf.neptune_enabled: + cf.run_log = neptune.init_run(mode=cf.neptune_mode, project=cf.neptune_project, api_token=cf.neptune_api_token) + run_id = cf.run_log["sys/id"].fetch() + if cf.run_group == "": + cf.run_group = "grp_" + run_id + cf.logdir = f"data/generative/{now}_{cf.run_group}/{seed:03}_{run_id}/" + else: + cf.run_log = None + cf.logdir = f"data/generative/{now}/{seed:03}/" + cf.imgdir = cf.logdir + "imgs/" + cf.preddir = cf.logdir + "preds/" + + # experiment params + cf.seed = seed + cf.n_epochs = 20 # 20 + cf.eval_every = 1 + cf.eval_train_acc = False + cf.eval_test_acc = True + cf.eval_images = True + + # model params + cf.use_bias = True + cf.kaiming_init = False + cf.nodes = [10, 144, 169, 784] # [10, 100, 300, 784] + cf.act_fn = utils.Tanh() + cf.use_precis = True + cf.precis_factor = [1.0, 1.0, 1.0, 0.1] # Errors and precisions at layer 0 are not used + cf.precis_coverage = [0.0, 0.0, 0.0, 0.5] # Errors and precisions at layer 0 are not used + + # dataset params + cf.train_size = None # None + cf.test_size = None # None + cf.label_scale = None # None + cf.normalize = True # None + cf.add_noise = True # False + cf.noise_mean = 0.0 + cf.noise_std = 2.0 + cf.noise_coverage = 0.5 + # cf.noise_min_std = 3.0 + # cf.noise_max_std = 3.0 + + # optim params + cf.optim = "Adam" # "Adam" + cf.lr = 0.006 # 0.006 + cf.batch_size = 10000 # 10000 + cf.batch_scale = False + cf.grad_clip = None + cf.weight_decay = 0.01 # None + + # inference params + cf.mu_dt = 0.01 # 0.01 + cf.n_train_iters = 50 # 50 + # cf.n_test_iters = min(round(200 / (s1d + 1e-9)), 4000) # 200 + cf.n_test_iters = 200 + cf.init_std = 0.01 + cf.fixed_preds_train = False + cf.fixed_preds_test = False + + main(cf) From 2b3577c82eed7a5ccb6afe2d53a1ba245c805edd Mon Sep 17 00:00:00 2001 From: Nick Hay <83538768+lasermanick@users.noreply.github.com> Date: Fri, 29 Mar 2024 14:58:59 +0000 Subject: [PATCH 24/30] Update .gitignore Ignoring Neptune cache, generated figures, secret keys (constants.py), and temporary/scratch files. --- .gitignore | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 655664b..cde2b4e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,14 @@ __pycache__/ *.py[cod] *$py.class -data/ \ No newline at end of file +data/ +/scripts/.neptune/ +/runs/ +/pypc/figs/ +/pypc/.neptune +.idea/ +/pypc/constants.py +/pypc/PyTorch_HelloWorld.py +/pypc/PyTorch_MNIST_Example.py +/pypc/temp.py +/tensorboard_test.py From 45f2777538514b19ac78f779968cbf949431193c Mon Sep 17 00:00:00 2001 From: Nick Hay <83538768+lasermanick@users.noreply.github.com> Date: Fri, 29 Mar 2024 15:35:27 +0000 Subject: [PATCH 25/30] MacOS specific environment (no pytorch-cuda) --- environment_macos.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 environment_macos.yml diff --git a/environment_macos.yml b/environment_macos.yml new file mode 100644 index 0000000..501e54e --- /dev/null +++ b/environment_macos.yml @@ -0,0 +1,18 @@ +name: pypc +channels: + - pytorch + - nvidia + - conda-forge + - defaults +dependencies: + - python + - numpy + - matplotlib + - pandas + - plotly + - neptune + - tqdm + - pytorch + # - pytorch-cuda=12.1 + - torchaudio + - torchvision \ No newline at end of file From 784969dabbdef26f5a4f930aa76b3c74d05561b5 Mon Sep 17 00:00:00 2001 From: Nick Hay <83538768+lasermanick@users.noreply.github.com> Date: Fri, 29 Mar 2024 16:01:18 +0000 Subject: [PATCH 26/30] Update environment_windows.yml Renamed after adding environment_macos.yml --- environment.yml => environment_windows.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename environment.yml => environment_windows.yml (100%) diff --git a/environment.yml b/environment_windows.yml similarity index 100% rename from environment.yml rename to environment_windows.yml From 98e157da185002c6b16025e3bdd96ad6a4697b23 Mon Sep 17 00:00:00 2001 From: Nick Hay <83538768+lasermanick@users.noreply.github.com> Date: Fri, 29 Mar 2024 16:39:44 +0000 Subject: [PATCH 27/30] Enable Metal Use pytorch-nightly channel to enable Metal (MPS) acceleration in pytorch --- .DS_Store | Bin 0 -> 6148 bytes environment_macos.yml | 2 +- pypc/utils.py | 5 ++++- 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 .DS_Store diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..0d6f6064dd07ce5a1950d022e584830140dfd65c GIT binary patch literal 6148 zcmeHK%Z}496us^Q+A>0P!AP(`vc$G3lhN=Jn@p!&j07wu2o``!(o_(UiK`}^3RRV| zhks$skMJ+7;9T2-+78_zgeo8D`uK6L6CcNROhlqPO?;vj5j9}U-ZdnD0JpPULe182 zqtJa!NKsB_G(zg&6mSas?+WnRy`k6ZIi<9`eoN}mC#>b{BvZpUQ!zYwoA?Ur{M0~> zXm3OG0I?Abu_F{4dOBL`sZLMngpR3C&ncw->T|`NHE)l`c)u|X*Ze+j@{4E1*pA~{ zJkHXh+~5B$YCGPQt94+v{xnX)1n(qUUTV%QdxYFkK&i1O~0!^ZDZgzby~_`-`@m`~HK2wtV#PaIt7M_U_($ zays~!o#l#O9uU}0HLhE}pfAY$9{71U$upI|M;-e&=@il(QIDR1JCHwFljaA!mZz?c zsTpF@Q^;6SvFcspewV#bbl4^Z>_^tv-KIggu{Z^s0)IsT-XA24Ij~$A)>{V>eFXq} zsMbK9e-b!GS`I8%hS35D6ACt=!d@|i2}ixOeu3r6un8w&FCW4_S=bwj&`(GIQoEA~ z47=1R;1pO@V8;|&eE#=0-~U>*)cm6xb>Sgx3puT@1;dt(OMJXRU<2gE82zGOR<8 j*yC7d_$a;w(?FZ#3^=e{8Ac7@egw1(E^!L{Qw4qjW#OFr literal 0 HcmV?d00001 diff --git a/environment_macos.yml b/environment_macos.yml index 501e54e..75ec563 100644 --- a/environment_macos.yml +++ b/environment_macos.yml @@ -1,6 +1,6 @@ name: pypc channels: - - pytorch + - pytorch-nightly - nvidia - conda-forge - defaults diff --git a/pypc/utils.py b/pypc/utils.py index e6a845a..81fd136 100644 --- a/pypc/utils.py +++ b/pypc/utils.py @@ -3,7 +3,10 @@ import numpy as np import torch -DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") +DEVICE = torch.device( + "cuda" if torch.cuda.is_available() + else "mps" if torch.backends.mps.is_available() + else "cpu") class AttrDict(dict): From e51d860079d88286d659bb11c23f316c7bfb810d Mon Sep 17 00:00:00 2001 From: Nick Hay <83538768+lasermanick@users.noreply.github.com> Date: Sat, 30 Mar 2024 09:09:39 +0000 Subject: [PATCH 28/30] Ignore .DS_Store (on MacOS) --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index cde2b4e..d4bf607 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,4 @@ data/ /pypc/PyTorch_MNIST_Example.py /pypc/temp.py /tensorboard_test.py +.DS_Store From 3b5ec55bcfd68642c56947324f539b012e80215f Mon Sep 17 00:00:00 2001 From: Nick Hay <83538768+lasermanick@users.noreply.github.com> Date: Sat, 30 Mar 2024 09:10:19 +0000 Subject: [PATCH 29/30] Print start/stop timestamps --- scripts/generative.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/generative.py b/scripts/generative.py index ae0f787..59f4547 100644 --- a/scripts/generative.py +++ b/scripts/generative.py @@ -86,6 +86,7 @@ def print_batch_stats(img_batch, cf): def main(cf): print(f"\nStarting generative experiment {cf.logdir}: --seed {cf.seed} --device {utils.DEVICE}") + print(datetime.now()) os.makedirs(cf.logdir, exist_ok=True) os.makedirs(cf.imgdir, exist_ok=True) os.makedirs(cf.preddir, exist_ok=True) @@ -216,7 +217,7 @@ def main(cf): # logging params cf.log_node_its = False # WARNING: Can log A LOT of data to Neptune if True cf.json_enabled = True - cf.neptune_enabled = True + cf.neptune_enabled = False cf.neptune_mode = "async" # https://docs.neptune.ai/api/connection_modes/ cf.neptune_project = "lasermanick/PYPC" cf.neptune_api_token = constants.API_KEY_NEPTUNE @@ -279,3 +280,5 @@ def main(cf): cf.fixed_preds_test = False main(cf) +print("Finishing experiment") +print(datetime.now()) From 4dd8eef55156be34dfca26e3dc517eefb65b798d Mon Sep 17 00:00:00 2001 From: Nick Hay <83538768+lasermanick@users.noreply.github.com> Date: Sat, 30 Mar 2024 09:10:44 +0000 Subject: [PATCH 30/30] Update .DS_Store --- .DS_Store | Bin 6148 -> 6148 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/.DS_Store b/.DS_Store index 0d6f6064dd07ce5a1950d022e584830140dfd65c..fccdcbcf66fe9096f235805b0578e7e88709c9da 100644 GIT binary patch delta 43 zcmZoMXffEp$jZ2FG81bdhgfyBfsTT?!Q?tt8ODyuvsvXCJ2&rU^