diff --git a/.DS_Store b/.DS_Store new file mode 100644 index 0000000..fccdcbc Binary files /dev/null and b/.DS_Store differ diff --git a/.gitignore b/.gitignore index 655664b..d4bf607 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,15 @@ __pycache__/ *.py[cod] *$py.class -data/ \ No newline at end of file +data/ +/scripts/.neptune/ +/runs/ +/pypc/figs/ +/pypc/.neptune +.idea/ +/pypc/constants.py +/pypc/PyTorch_HelloWorld.py +/pypc/PyTorch_MNIST_Example.py +/pypc/temp.py +/tensorboard_test.py +.DS_Store diff --git a/environment_macos.yml b/environment_macos.yml new file mode 100644 index 0000000..75ec563 --- /dev/null +++ b/environment_macos.yml @@ -0,0 +1,18 @@ +name: pypc +channels: + - pytorch-nightly + - nvidia + - conda-forge + - defaults +dependencies: + - python + - numpy + - matplotlib + - pandas + - plotly + - neptune + - tqdm + - pytorch + # - pytorch-cuda=12.1 + - torchaudio + - torchvision \ No newline at end of file diff --git a/environment_windows.yml b/environment_windows.yml new file mode 100644 index 0000000..66436c6 --- /dev/null +++ b/environment_windows.yml @@ -0,0 +1,18 @@ +name: pypc +channels: + - pytorch + - nvidia + - conda-forge + - defaults +dependencies: + - python + - numpy + - matplotlib + - pandas + - plotly + - neptune + - tqdm + - pytorch + - pytorch-cuda=12.1 + - torchaudio + - torchvision \ No newline at end of file diff --git a/pypc/datasets.py b/pypc/datasets.py index c6ed966..22e05ff 100644 --- a/pypc/datasets.py +++ b/pypc/datasets.py @@ -1,4 +1,5 @@ import numpy as np +import pandas as pd import matplotlib.pyplot as plt import torch from torch.utils import data @@ -6,16 +7,65 @@ from pypc import utils +class GaussianAddNoise(object): + def __init__(self, mean=0.0, std=1.0, coverage=1.0): + self.std = std + self.mean = mean + self.coverage = max(0.0, min(1.0, coverage)) # Clamp to range [0, 1] + + def __call__(self, input): + noise = torch.randn(input.size()) * self.std + self.mean + rows = input.size(1) + cols = input.size(2) + first_noise_row = int(rows*(1.0 - self.coverage)) + noise[0, 0:first_noise_row, :] = torch.zeros(cols) + return input + noise + +class PerPixelGaussianAddNoise(object): + def __init__(self, per_pixel_scaling, mean=0.0, coverage=1.0): + self.mean = mean + self.coverage = max(0.0, min(1.0, coverage)) # Clamp to range [0, 1] + self.per_pixel_scaling = per_pixel_scaling + + def __call__(self, input): + noise = torch.randn(input.size()) # std = 1, mean = 0 + noise = (noise * self.per_pixel_scaling) + self.mean + rows = input.size(1) + cols = input.size(2) + first_noise_row = int(rows*(1.0 - self.coverage)) + noise[0, 0:first_noise_row, :] = torch.zeros(cols) + return input + noise class MNIST(datasets.MNIST): - def __init__(self, train, size=None, scale=None, normalize=False): - transform = _get_transform(normalize=normalize, mean=(0.1307), std=(0.3081)) + def __init__(self, train, size=None, scale=None, normalize=False, add_noise=False, noise_mean=0.0, noise_std=1.0, noise_coverage=1.0, noise_per_pixel_scaling=None): + """ + Load MNIST dataset, scale to range [0,1], optionally normalise with mean = 0 and std dev = 1, + optionally set label scale factor, optionally limit size of dataset, optionally add noise + + :param train: True for training data, False for test data + :param size: Number of samples to keep in the dataset + :param scale: Scale factor for one-hot labels (e.g. scale=1 => 0 or 1, scale=0.5 => 0.25 or 0.75, + scale=0.1 => 0.45 or 0.55) + :param normalize: True to normalize + :param add_noise: True to add Gaussian noise + :param noise_mean: Gaussian noise mean + :param noise_std: Gaussian noise standard deviation + :param noise_coverage: Gaussian noise coverage (e.g. 1.0=full image, 0.5=bottom half, 0.0=none) + """ + transform = _get_transform(normalize=normalize, mean=(0.1307), std=(0.3081), add_noise=add_noise, noise_mean=noise_mean, noise_std=noise_std, noise_coverage=noise_coverage, noise_per_pixel_scaling=noise_per_pixel_scaling) # Transform to mean=0, std=1 super().__init__("./data/mnist", download=True, transform=transform, train=train) self.scale = scale if size is not None: self._reduce(size) def __getitem__(self, index): + """ + Return image (data) and label (target) with image converted from (1,28,28) to (784,) and label converted to + one-hot encoding (optionally scaled) + + :param index: Index + :return: image, label + """ data, target = super().__getitem__(index) data = _to_vector(data) target = _one_hot(target) @@ -24,6 +74,11 @@ def __getitem__(self, index): return data, target def _reduce(self, size): + """ + Crop the dataset + + :param size: Maximum sample number to retain + """ self.data = self.data[0:size] self.targets = self.targets[0:size] @@ -107,24 +162,61 @@ def _reduce(self, size): class FashionMNIST(datasets.FashionMNIST): - def __init__(self, train, size=None, normalize=False): - transform = _get_transform(normalize=normalize, mean=(0.5), std=(0.5)) + def __init__(self, train, size=None, scale=None, normalize=False, add_noise=False, noise_mean=0.0, noise_std=1.0, noise_coverage=1.0, noise_per_pixel_scaling=None): + """ + Load FashionMNIST dataset, scale to range [0,1], optionally normalise with mean = 0 and std dev = 1, + optionally set label scale factor, optionally limit size of dataset, optionally add noise + + :param train: True for training data, False for test data + :param size: Number of samples to keep in the dataset + :param scale: Scale factor for one-hot labels (e.g. scale=1 => 0 or 1, scale=0.5 => 0.25 or 0.75, + scale=0.1 => 0.45 or 0.55) + :param normalize: True to normalize + :param add_noise: True to add Gaussian noise + :param noise_mean: Gaussian noise mean + :param noise_std: Gaussian noise standard deviation + :param noise_coverage: Gaussian noise coverage (e.g. 1.0=full image, 0.5=bottom half, 0.0=none) + """ + transform = _get_transform(normalize=normalize, mean=(0.5), std=(0.5), add_noise=add_noise, noise_mean=noise_mean, noise_std=noise_std, noise_coverage=noise_coverage, noise_per_pixel_scaling=noise_per_pixel_scaling) # Transform to mean=0, std=1 super().__init__("./data/mnist", download=True, transform=transform, train=train) + self.scale = scale if size is not None: self._reduce(size) def __getitem__(self, index): + """ + Return image (data) and label (target) with image converted from (1,28,28) to (784,) and label converted to + one-hot encoding (optionally scaled) + + :param index: Index + :return: image, label + """ data, target = super().__getitem__(index) data = _to_vector(data) target = _one_hot(target) + if self.scale is not None: + target = _scale(target, self.scale) return data, target def _reduce(self, size): + """ + Crop the dataset + + :param size: Maximum sample number to retain + """ self.data = self.data[0:size] self.targets = self.targets[0:size] def get_dataloader(dataset, batch_size): + """ + Create PyTorch DataLoader for given dataset and batch size, perform pre-processing to move data onto the selected + cpu/cuda device with dtype=torch.float32, and return a list containing samples and labels + + :param dataset: PyTorch Dataset + :param batch_size: Batch size + :return: List of tuples with index 0 containing samples and index 1 containing labels + """ dataloader = data.DataLoader(dataset, batch_size, shuffle=True, drop_last=True) return list(map(_preprocess_batch, dataloader)) @@ -138,40 +230,107 @@ def accuracy(pred_labels, true_labels): return correct / batch_size -def plot_imgs(img_preds, path): +def save_csv(tensor_data, path): + pd.DataFrame(tensor_data.cpu().numpy()).to_csv(path) + + +def plot_imgs(img_preds, path, cmap="gray"): imgs = img_preds.cpu().detach().numpy() imgs = imgs[0:10, :] imgs = [np.reshape(imgs[i, :], [28, 28]) for i in range(imgs.shape[0])] _, axes = plt.subplots(2, 5) axes = axes.flatten() for i, img in enumerate(imgs): - axes[i].imshow(img, cmap="gray") + axes[i].imshow(img, cmap=cmap) plt.savefig(path) plt.close("all") +def plot_imgs_alt(img_preds, path=None, cmap="gray"): + images = img_preds.cpu().detach().numpy() + fig, axes = plt.subplots(2, 5) + fig.set_size_inches(8, 3) + fig.set_dpi(150) + axes = axes.flatten() + plt.setp(axes, xticks=[0, 27]) + plt.setp(axes, yticks=[0, 27]) + for i in range(10): + axes[i].tick_params(top=False, labeltop=False, bottom=False, labelbottom=False, width=2) + axes[i].tick_params(left=False, labelleft=False, right=False, labelright=False, width=2) + axes[i].imshow(images[i].reshape(28, 28), cmap=cmap) + axes[0].tick_params(top=True, labeltop=True, bottom=False, labelbottom=False, labelsize=16) + axes[0].tick_params(left=True, labelleft=True, right=False, labelright=False, labelsize=16) + + if path: + plt.savefig(path) + plt.show() + plt.close("all") + + def _preprocess_batch(batch): + """ + Pre-process a batch to move data onto the selected cpu/cuda device with dtype=torch.float32 + + :param batch: List of Tensor objects with index 0 containing samples and index 1 containing labels + :return: Pre-processed batch as tuple of Tensor objects + """ batch[0] = utils.set_tensor(batch[0]) batch[1] = utils.set_tensor(batch[1]) return (batch[0], batch[1]) -def _get_transform(normalize=True, mean=(0.5), std=(0.5)): +def _get_transform(normalize=True, mean=0.0, std=1.0, add_noise=False, noise_mean=0.0, noise_std=1.0, noise_coverage=1.0, noise_per_pixel_scaling=None): + """ + Define transformation to convert PIL image or numpy.ndarray to tensor with optional normalization + + :param normalize: True or False + :param mean: Input mean (after scaling to range [0,1]) + :param std: Input std dev (after scaling to range [0,1]) + :return: Transformation + """ transform = [transforms.ToTensor()] if normalize: - transform + [transforms.Normalize(mean=mean, std=std)] + transform += [transforms.Normalize(mean=mean, std=std)] + if add_noise: + if noise_per_pixel_scaling is None: + transform += [GaussianAddNoise(mean=noise_mean, std=noise_std, coverage=noise_coverage)] + else: + transform += [PerPixelGaussianAddNoise(noise_per_pixel_scaling, mean=noise_mean, coverage=noise_coverage)] + return transforms.Compose(transform) def _one_hot(labels, n_classes=10): + """ + Convert categorical label to one-hot encoding (trick is to index an identity matrix) NOTE: Only used for individual + labels so consider changing parameter name to 'label') + + :param labels: Categorical label + :param n_classes: Number of classes (categories) + :return: One-hot encoded label + """ arr = torch.eye(n_classes) return arr[labels] def _scale(targets, factor): + """ + Scale one-hot labels (targets) according to: + scaled = 0.5 + factor x (original - 0.5) + (e.g. scale=1 => 0 or 1, scale=0.5 => 0.25 or 0.75, scale=0.1 => 0.45 or 0.55) + + :param targets: Labels + :param factor: Scale factor + :return: Scaled labels + """ return targets * factor + 0.5 * (1 - factor) * torch.ones_like(targets) def _to_vector(batch): + """ + Convert batch of 2D images to vector format NOTE: Currently only used for single images so naming is confusing + :param batch: Image or batch of images + :return: Image or batch of images in vector format + """ batch_size = batch.size(0) return batch.reshape(batch_size, -1).squeeze() diff --git a/pypc/figures.py b/pypc/figures.py new file mode 100644 index 0000000..2be4fce --- /dev/null +++ b/pypc/figures.py @@ -0,0 +1,1250 @@ +import os +import plotly.graph_objs as go +# from dash import Dash, html, dcc +import numpy as np +import pandas as pd +import neptune.new as neptune + +from collections import namedtuple +from operator import itemgetter + +def rgb_to_rgba(rgb_value, alpha): + """ + Adds the alpha channel to an RGB Value and returns it as an RGBA Value + :param rgb_value: Input RGB Value + :param alpha: Alpha Value to add in range [0,1] + :return: RGBA Value + """ + return f"rgba{rgb_value[3:-1]}, {alpha})" + +# Colours for plots +colours = { + 'white': 'rgb(255, 255, 255)', + 'grey75': 'rgb(191, 191, 191)', + 'grey50': 'rgb(127, 127, 127)', + 'grey25': 'rgb(63, 63, 63)', + 'black': 'rgb(0, 0, 0)', + 'darkred': 'rgb(127, 0, 0)', + 'red': 'rgb(255, 0, 0)', + 'orange': 'rgb(255, 127, 0)', + 'yellow': 'rgb(255, 255, 0)', + 'green': 'rgb(0, 255, 0)', + 'darkgreen': 'rgb(0, 127, 0)', + 'cyan': 'rgb(0, 255, 255)', + 'blue': 'rgb(0, 0, 255)', + 'darkblue': 'rgb(0, 0, 127)', + 'magenta': 'rgb(255, 0, 255)', +} + +# Groups of runs to be added to dataset +RunGroup = namedtuple("RunGroup", ["start_run", "num_runs", "title", "colour", "x_value"]) + + +class NickFig: + def __init__(self, fig_type, groups, fig_num, title, yaxis_title, yaxis_scale, data_field, xaxis_title, project_name, num_datapoints, xaxis_range, yaxis_range, path, filename, showlegend, font, width, height, show_fig=False, save_fig=False): + self.fig_type = fig_type + self.groups = groups + self.fig_num = fig_num + self.title = title + self.yaxis_title = yaxis_title + self.data_field = data_field + self.xaxis_title = xaxis_title + self.project_name = project_name + self.num_datapoints = num_datapoints + self.xaxis_range = xaxis_range + self.yaxis_range = yaxis_range + self.yaxis_scale = yaxis_scale + self.path = path + self.filename = filename + self.showlegend = showlegend + self.font = font + self.width = width + self.height = height + self.show_fig = show_fig + self.save_fig = save_fig + + self.stats = pd.DataFrame() + + self.fig = go.Figure() + + self.format_figure() + if self.fig_type == "1d_multiple_curves": + self.calc_stats() + elif self.fig_type == "2d": + self.calc_stats_2dscan() + else: + assert False, "Select a valid figure type!" + + self.draw_traces() + if self.show_fig: + self.show_figure() + if self.save_fig: + self.save_figure() + + def get_runs(self, group): + df = pd.DataFrame() + for run in range(group.start_run, group.start_run + group.num_runs): + run_id = f"PYPC-{run}" + # print(f"{run_id}") + run = neptune.init_run( + project="lasermanick/PYPC", + mode="read-only", + api_token="eyJhcGlfYWRkcmVzcyI6Imh0dHBzOi8vYXBwLm5lcHR1bmUuYWkiLCJhcGlfdXJsIjoiaHR0cHM6Ly9hcHAubmVwdHVuZS5haSIsImFwaV9rZXkiOiIxMDVhMjgyMi01ZjU0LTQ3NDMtODcwOS1jNjlmNGNjNDRhYTEifQ==", + with_id=run_id, + ) + df.insert(df.shape[1], run_id, run[self.data_field].fetch_values(include_timestamp=False)["value"]) + return df + + def calc_stats(self): + self.stats.insert(self.stats.shape[1], self.xaxis_title, + np.arange(1, self.num_datapoints + 1)) # x-axis column + for g in self.groups: + df = self.get_runs(g) + self.stats.insert(self.stats.shape[1], f"{g.title}m", df.mean(1) * self.yaxis_scale) + self.stats.insert(self.stats.shape[1], f"{g.title}s", df.std(1) * self.yaxis_scale) + self.stats.insert(self.stats.shape[1], f"{g.title}l", (df.mean(1) - df.std(1)) * self.yaxis_scale) + self.stats.insert(self.stats.shape[1], f"{g.title}u", (df.mean(1) + df.std(1)) * self.yaxis_scale) + + def calc_stats_2dscan(self): + for g in self.groups: # For each group of runs in the figure + df = self.get_runs(g) + # Create the columns if needed + if f"{g.title}m" not in self.stats.columns: + self.stats.insert(self.stats.shape[1], f"{g.title}m", np.zeros(self.stats.shape[0])) + self.stats.insert(self.stats.shape[1], f"{g.title}s", np.zeros(self.stats.shape[0])) + self.stats.insert(self.stats.shape[1], f"{g.title}l", np.zeros(self.stats.shape[0])) + self.stats.insert(self.stats.shape[1], f"{g.title}u", np.zeros(self.stats.shape[0])) + # Create the row if needed + if g.x_value not in self.stats.index: + self.stats.loc[g.x_value] = np.zeros(self.stats.shape[1]) + # Add the stats to the stats dataframe + self.stats.loc[g.x_value][f"{g.title}m"] = df.mean(1) * self.yaxis_scale + self.stats.loc[g.x_value][f"{g.title}s"] = df.std(1) * self.yaxis_scale + self.stats.loc[g.x_value][f"{g.title}l"] = (df.mean(1) - df.std(1)) * self.yaxis_scale + self.stats.loc[g.x_value][f"{g.title}u"] = (df.mean(1) + df.std(1)) * self.yaxis_scale + + # Add x-axis column + self.stats.insert(self.stats.shape[1], self.xaxis_title, self.stats.index) + + def draw_traces(self): + # Extract traces (with unique titles) from the groups + key = itemgetter(3) + traces = {key(g): g for g in self.groups}.values() + for t in traces: + # Mean + self.fig.add_trace( + go.Scatter( + name=t.title, + x=self.stats[self.xaxis_title], + y=self.stats[f"{t.title}m"], + mode='lines', + line=dict(color=t.colour) + ) + ) + # Upper bound + self.fig.add_trace( + go.Scatter( + name=f"{t.title} upper", + x=self.stats[self.xaxis_title], + y=self.stats[f"{t.title}u"], + mode='lines', + line=dict(width=0), + showlegend=False + ) + ) + # Lower bound + self.fig.add_trace( + go.Scatter( + name=f"{t.title} lower", + x=self.stats[self.xaxis_title], + y=self.stats[f"{t.title}l"], + line=dict(width=0), + mode='lines', + fillcolor=rgb_to_rgba(t.colour, 0.3), + fill='tonexty', + showlegend=False + ) + ) + + def format_figure(self): + self.fig.update_layout( + font=self.font, + showlegend=self.showlegend, + xaxis_title=self.xaxis_title, + yaxis_title=self.yaxis_title, + title=self.title, + xaxis_range=self.xaxis_range, + yaxis_range=self.yaxis_range, + plot_bgcolor=colours['white'], + width=self.width, + height=self.height, + ) + self.fig.update_xaxes( + linecolor=colours['black'], + showline=True, + showgrid=False, + linewidth=2, + ) + self.fig.update_yaxes( + linecolor=colours['black'], + showline=True, + showgrid=False, + linewidth=2, + ) + + def show_figure(self): + self.fig.show() + + def save_figure(self): + figdir = self.path + os.makedirs(figdir, exist_ok=True) + self.fig.write_image(figdir + self.filename) + + def get_fig(self): + return self.fig + + +testacc_fig_cfg = { + "fig_type": "1d_multiple_curves", + "title": "Title", + "yaxis_title": "Test accuracy (%)", + "data_field": "test/acc", + "xaxis_title": "Epoch", + "project_name": "PYPC", + "num_datapoints": 20, + "xaxis_range": [0, 20], + "yaxis_range": [0, 100], + "yaxis_scale": 100.0, + "path": "figs/", + "filename": "fig.svg", + "showlegend": True, + "font": dict(family="Arial", size=20, color=colours['black']), + "width": 800, + "height": 600, +} + +fig_cfgs = {} + +fig_num = "D(a)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} node structures 00" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(226, 6, "[10, 169, 729, 784]", colours['darkred'], None), + RunGroup(232, 6, "[10, 144, 169, 784]", colours['red'], None), + RunGroup(238, 6, "[10, 100, 300, 784]", colours['orange'], None), + RunGroup(244, 6, "[10, 36, 169, 784]", colours['green'], None), + RunGroup(250, 6, "[10, 10, 169, 729, 784]", colours['cyan'], None), + RunGroup(256, 6, "[10, 676, 729, 784]", colours['blue'], None), + RunGroup(262, 6, "[10, 36, 169, 729, 784]", colours['darkblue'], None), +] + +fig_num = "D(b)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} node structures 01" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(268, 6, "0.001", colours['red'], None), + RunGroup(262, 6, "0.003", colours['darkblue'], None), + RunGroup(274, 6, "0.005", colours['green'], None), +] + +fig_num = "D(c)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} node structures 02" +fc["filename"] = f"Fig {fig_num}.svg" +fc["num_datapoints"] = 40 +fc["xaxis_range"] = [0, 40] +fc["groups"] = [ + RunGroup(280, 6, "[10, 169, 729, 784]", colours['red'], None), + RunGroup(286, 6, "[10, 144, 169, 784]", colours['darkblue'], None), + RunGroup(292, 6, "[10, 100, 300, 784]", colours['green'], None), +] + +fig_num = "G(a)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} lr scan" +fc["filename"] = f"Fig {fig_num}.svg" +fc["num_datapoints"] = 40 +fc["xaxis_range"] = [0, 40] +fc["groups"] = [ + # RunGroup(304, 6, "0.001", colours['red'], None), + RunGroup(310, 6, "0.002", colours['darkred'], None), + # RunGroup(316, 6, "0.003", colours['red'], None), + # RunGroup(322, 6, "0.004", colours['red'], None), + RunGroup(328, 6, "0.005", colours['red'], None), + RunGroup(334, 6, "0.006", colours['orange'], None), + RunGroup(340, 6, "0.008", colours['green'], None), + RunGroup(346, 6, "0.010", colours['cyan'], None), + RunGroup(364, 6, "0.015", colours['blue'], None), + RunGroup(352, 6, "0.020", colours['darkblue'], None), + # RunGroup(358, 6, "0.03", colours['red'], None), +] + +fig_num = "G(b)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} dt scan" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(370, 6, "0.001", colours['red'], None), + RunGroup(376, 6, "0.003", colours['orange'], None), + RunGroup(382, 6, "0.01", colours['green'], None), + RunGroup(388, 6, "0.03", colours['cyan'], None), + RunGroup(394, 6, "0.1", colours['blue'], None), +] + +fig_num = "I(a)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} training its scan" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(400, 6, "5", colours['red'], None), + RunGroup(406, 6, "15", colours['orange'], None), + RunGroup(412, 6, "50", colours['green'], None), + RunGroup(418, 6, "150", colours['cyan'], None), + RunGroup(424, 6, "500", colours['blue'], None), +] + +fig_num = "I(b)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} test its scan" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(430, 6, "20", colours['red'], None), + RunGroup(436, 6, "60", colours['orange'], None), + RunGroup(442, 6, "200", colours['green'], None), + RunGroup(448, 6, "600", colours['cyan'], None), + RunGroup(454, 6, "2000", colours['blue'], None), +] + +fig_num = "K(a)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} lr scan with extreme train and test its" +fc["filename"] = f"Fig {fig_num}.svg" +fc["num_datapoints"] = 40 +fc["xaxis_range"] = [0, 40] +fc["groups"] = [ + RunGroup(460, 6, "0.001", colours['red'], None), + RunGroup(466, 6, "0.002", colours['orange'], None), + RunGroup(472, 6, "0.006", colours['green'], None), + RunGroup(478, 6, "0.01", colours['cyan'], None), + RunGroup(484, 6, "0.02", colours['blue'], None), +] + +fig_num = "K(b)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} lr scan with extreme train and test its" +fc["filename"] = f"Fig {fig_num}.svg" +fc["num_datapoints"] = 40 +fc["xaxis_range"] = [0, 40] +fc["groups"] = [ + RunGroup(490, 1, "0.001", colours['red'], None), + RunGroup(492, 1, "0.002", colours['orange'], None), + RunGroup(493, 1, "0.006", colours['green'], None), + RunGroup(494, 1, "0.01", colours['cyan'], None), + RunGroup(495, 1, "0.02", colours['blue'], None), +] + +fig_num = "L" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} confirm prec change equiv to dt change" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(499, 6, "[10.0, 10.0, 10.0, 10.0]", colours['red'], None), + RunGroup(505, 6, "[3.0, 3.0, 3.0, 3.0]", colours['orange'], None), + RunGroup(511, 6, "[1.0, 1.0, 1.0, 1.0]", colours['green'], None), + RunGroup(517, 6, "[0.3, 0.3, 0.3, 0.3]", colours['cyan'], None), + RunGroup(523, 6, "[0.1, 0.1, 0.1, 0.1]", colours['blue'], None), +] + +fig_num = "M(a)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} layer 3 prec scan" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(570, 6, "20", colours['darkred'], None), + RunGroup(576, 6, "10", colours['red'], None), + RunGroup(582, 6, "5", colours['orange'], None), + RunGroup(588, 6, "2", colours['green'], None), + RunGroup(594, 6, "1", colours['cyan'], None), + RunGroup(600, 6, "0.5", colours['blue'], None), + RunGroup(606, 6, "0.2", colours['darkblue'], None), + RunGroup(612, 6, "0.1", colours['grey25'], None), + RunGroup(618, 6, "0.05", colours['grey75'], None), +] + +fig_num = "M(b)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} layer 2 prec scan" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(624, 6, "20", colours['darkred'], None), + RunGroup(630, 6, "10", colours['red'], None), + RunGroup(636, 6, "5", colours['orange'], None), + RunGroup(642, 6, "2", colours['green'], None), + RunGroup(648, 6, "1", colours['cyan'], None), + RunGroup(654, 6, "0.5", colours['blue'], None), + RunGroup(660, 6, "0.2", colours['darkblue'], None), + RunGroup(666, 6, "0.1", colours['grey25'], None), + RunGroup(672, 6, "0.05", colours['grey75'], None), +] + +fig_num = "M(c)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} layer 1 prec scan" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(678, 6, "20", colours['darkred'], None), + RunGroup(684, 6, "10", colours['red'], None), + RunGroup(690, 6, "5", colours['orange'], None), + RunGroup(696, 6, "2", colours['green'], None), + RunGroup(702, 6, "1", colours['cyan'], None), + RunGroup(708, 6, "0.5", colours['blue'], None), + RunGroup(714, 6, "0.2", colours['darkblue'], None), + RunGroup(720, 6, "0.1", colours['grey25'], None), + RunGroup(726, 6, "0.05", colours['grey75'], None), +] + +fig_num = "N(a)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} prec ratio by layer" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(1017, 1, "[1.0, 0.01, 0.01, 0.01]", colours['darkred'], None), + RunGroup(1018, 1, "[1.0, 0.04, 0.02, 0.01]", colours['red'], None), + RunGroup(1019, 1, "[1.0, 0.25, 0.05, 0.01]", colours['orange'], None), + RunGroup(1020, 1, "[1.0, 1.0, 0.10, 0.01]", colours['green'], None), + RunGroup(1021, 1, "[1.0, 4.0, 0.20, 0.01]", colours['cyan'], None), + RunGroup(1022, 1, "[1.0, 25, 0.50, 0.01]", colours['blue'], None), + RunGroup(1023, 1, "[1.0, 100, 1.0, 0.01]", colours['darkblue'], None), +] + +fig_num = "N(b)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} prec ratio by layer" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(1024, 1, "[1.0, 0.01, 0.01, 0.01]", colours['darkred'], None), + RunGroup(1025, 1, "[1.0, 0.01, 0.02, 0.04]", colours['red'], None), + RunGroup(1026, 1, "[1.0, 0.01, 0.05, 0.25]", colours['orange'], None), + RunGroup(1027, 1, "[1.0, 0.01, 0.10, 1.0]", colours['green'], None), + RunGroup(1028, 1, "[1.0, 0.01, 0.20, 4.0]", colours['cyan'], None), + RunGroup(1029, 1, "[1.0, 0.01, 0.50, 25]", colours['blue'], None), + RunGroup(1030, 1, "[1.0, 0.01, 1.0, 100]", colours['darkblue'], None), +] + +fig_num = "P(a)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} train vs test accuracy - test" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(732, 1, "0.001", colours['darkred'], None), + RunGroup(734, 1, "0.003", colours['red'], None), + RunGroup(735, 1, "0.006", colours['orange'], None), + RunGroup(736, 1, "0.010", colours['green'], None), + RunGroup(737, 1, "0.015", colours['cyan'], None), + RunGroup(738, 1, "0.030", colours['blue'], None), +] + +fig_num = "P(b)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["yaxis_title"] = "Training accuracy (%)" +fc["data_field"] = "train/acc" +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} train vs test accuracy - train" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(732, 1, "0.001", colours['darkred'], None), + RunGroup(734, 1, "0.003", colours['red'], None), + RunGroup(735, 1, "0.006", colours['orange'], None), + RunGroup(736, 1, "0.010", colours['green'], None), + RunGroup(737, 1, "0.015", colours['cyan'], None), + RunGroup(738, 1, "0.030", colours['blue'], None), +] + +fig_num = "Q" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} effect of noise on accuracy" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(801, 1, "var = 4, cov = 1.0", colours['darkgreen'], None), + RunGroup(841, 6, "var = 4, cov = 0.5", colours['green'], None), + RunGroup(800, 1, "var = 1, cov = 1.0", colours['darkred'], None), + RunGroup(813, 1, "var = 1, cov = 0.5", colours['red'], None), + RunGroup(702, 6, "No noise", colours['grey25'], None), +] + +fig_num = "R" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} bottom half noise, bottom half layer 3 prec scan" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(817, 6, "0.1", colours['darkred'], None), + RunGroup(823, 6, "0.2", colours['red'], None), + RunGroup(829, 6, "0.5", colours['orange'], None), + # RunGroup(835, 6, "0.8", colours['green'], None), + RunGroup(841, 6, "1", colours['green'], None), + RunGroup(847, 6, "2", colours['cyan'], None), + RunGroup(853, 6, "5", colours['blue'], None), + # RunGroup(859, 6, "8", colours['grey25'], None), + RunGroup(865, 6, "10", colours['darkblue'], None), + RunGroup(702, 6, "No noise", colours['grey25'], None), +] + +fig_num = "S" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} bottom half noise, full layer 3 prec scan" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(873, 1, "0.1", colours['darkred'], None), + RunGroup(874, 1, "0.2", colours['red'], None), + RunGroup(875, 1, "0.5", colours['orange'], None), + # RunGroup(876, 1, "0.8", colours['green'], None), + RunGroup(841, 6, "1", colours['green'], None), + RunGroup(878, 1, "2", colours['cyan'], None), + RunGroup(879, 1, "5", colours['blue'], None), + # RunGroup(880, 1, "8", colours['grey25'], None), + RunGroup(881, 1, "10", colours['darkblue'], None), + RunGroup(702, 6, "No noise", colours['grey25'], None), +] + +fig_num = "T" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} bottom half noise, full layer 3 prec scan" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(883, 1, "0.1", colours['darkred'], None), + RunGroup(884, 1, "0.2", colours['red'], None), + RunGroup(885, 1, "0.5", colours['orange'], None), + # RunGroup(886, 1, "0.8", colours['green'], None), + RunGroup(829, 6, "1", colours['green'], None), + RunGroup(888, 1, "2", colours['cyan'], None), + RunGroup(889, 1, "5", colours['blue'], None), + # RunGroup(890, 1, "8", colours['grey25'], None), + RunGroup(891, 1, "10", colours['darkblue'], None), + RunGroup(702, 6, "No noise", colours['grey25'], None), +] + +fig_num = "U" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} bottom half noise, full prec scan" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(892, 1, "0.1", colours['darkred'], None), + RunGroup(893, 1, "0.2", colours['red'], None), + RunGroup(894, 1, "0.5", colours['orange'], None), + RunGroup(841, 6, "1", colours['green'], None), + RunGroup(896, 1, "2", colours['cyan'], None), + RunGroup(897, 1, "5", colours['blue'], None), + RunGroup(898, 1, "10", colours['darkblue'], None), + RunGroup(702, 6, "No noise", colours['grey25'], None), +] + +fig_num = "V" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} no noise, bottom half layer 3 prec scan" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(899, 1, "0.1", colours['darkred'], None), + RunGroup(905, 1, "0.2", colours['red'], None), + RunGroup(911, 1, "0.5", colours['orange'], None), + RunGroup(702, 6, "1", colours['green'], None), + RunGroup(923, 1, "2", colours['cyan'], None), + RunGroup(929, 1, "5", colours['blue'], None), + RunGroup(935, 1, "10", colours['darkblue'], None), +] + +fig_num = "AC" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} batch size=2000 bottom half noise, bottom half layer 3 prec scan" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(975, 6, "0.1", colours['darkred'], None), + RunGroup(981, 6, "0.2", colours['red'], None), + RunGroup(987, 6, "0.5", colours['orange'], None), + RunGroup(993, 6, "1", colours['green'], None), + RunGroup(999, 6, "2", colours['cyan'], None), + RunGroup(1005, 6, "5", colours['blue'], None), + RunGroup(1011, 6, "10", colours['darkblue'], None), + RunGroup(1031, 6, "No noise", colours['grey25'], None), +] + +fig_num = "AD" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} batch size scan 03" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + RunGroup(946, 1, "10000", colours['red'], None), + RunGroup(947, 1, "5000", colours['orange'], None), + RunGroup(948, 1, "2000", colours['green'], None), + RunGroup(949, 1, "1000", colours['cyan'], None), + RunGroup(950, 1, "500", colours['blue'], None), +] + +fig_num = "AF" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} lr scan with batch size = 2000" +fc["filename"] = f"Fig {fig_num}.svg" +fc["num_datapoints"] = 40 +fc["xaxis_range"] = [0, 40] +fc["groups"] = [ + RunGroup(1039, 1, "0.001", colours['red'], None), + RunGroup(1040, 1, "0.002", colours['darkred'], None), + RunGroup(1041, 1, "0.005", colours['red'], None), + RunGroup(1042, 1, "0.006", colours['orange'], None), + RunGroup(1043, 1, "0.008", colours['green'], None), + RunGroup(1044, 1, "0.010", colours['cyan'], None), + RunGroup(1045, 1, "0.015", colours['blue'], None), + RunGroup(1046, 1, "0.020", colours['darkblue'], None), +] + +fig_num = "AG(a)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_type"] = "2d" +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} 2D scan of noise stddev and precision (coverage=0.5)" +fc["filename"] = f"Fig {fig_num}.svg" +fc["xaxis_title"] = "Precision (lower image)" +fc["xaxis_range"] = [0, 1.2] +fc["yaxis_range"] = [67, 88] +fc["groups"] = [ + RunGroup(1266, 6, "0", colours['red'], 0), + RunGroup(2082, 6, "0", colours['red'], 0.002), + RunGroup(2088, 6, "0", colours['red'], 0.005), + RunGroup(2094, 6, "0", colours['red'], 0.01), + RunGroup(2100, 6, "0", colours['red'], 0.02), + RunGroup(1506, 6, "0", colours['red'], 0.027), + RunGroup(1512, 6, "0", colours['red'], 0.038), + RunGroup(1272, 6, "0", colours['red'], 0.059), + RunGroup(1278, 6, "0", colours['red'], 0.1), + RunGroup(1284, 6, "0", colours['red'], 0.2), + RunGroup(1290, 6, "0", colours['red'], 0.5), + RunGroup(1296, 6, "0", colours['red'], 0.8), + RunGroup(1302, 6, "0", colours['red'], 1.0), + RunGroup(1308, 6, "0", colours['red'], 1.2), + RunGroup(1314, 6, "1/4", colours['orange'], 0), + RunGroup(2106, 6, "1/4", colours['orange'], 0.002), + RunGroup(2112, 6, "1/4", colours['orange'], 0.005), + RunGroup(2118, 6, "1/4", colours['orange'], 0.01), + RunGroup(2124, 6, "1/4", colours['orange'], 0.02), + RunGroup(1518, 6, "1/4", colours['orange'], 0.027), + RunGroup(1524, 6, "1/4", colours['orange'], 0.038), + RunGroup(1320, 6, "1/4", colours['orange'], 0.059), + RunGroup(1326, 6, "1/4", colours['orange'], 0.1), + RunGroup(1332, 6, "1/4", colours['orange'], 0.2), + RunGroup(1338, 6, "1/4", colours['orange'], 0.5), + RunGroup(1344, 6, "1/4", colours['orange'], 0.8), + RunGroup(1350, 6, "1/4", colours['orange'], 1.0), + RunGroup(1356, 6, "1/4", colours['orange'], 1.2), + RunGroup(1362, 6, "1", colours['green'], 0), + RunGroup(2130, 6, "1", colours['green'], 0.002), + RunGroup(2136, 6, "1", colours['green'], 0.005), + RunGroup(2142, 6, "1", colours['green'], 0.01), + RunGroup(2148, 6, "1", colours['green'], 0.02), + RunGroup(1530, 6, "1", colours['green'], 0.027), + RunGroup(1536, 6, "1", colours['green'], 0.038), + RunGroup(1368, 6, "1", colours['green'], 0.059), + RunGroup(1374, 6, "1", colours['green'], 0.1), + RunGroup(1380, 6, "1", colours['green'], 0.2), + RunGroup(1386, 6, "1", colours['green'], 0.5), + RunGroup(1392, 6, "1", colours['green'], 0.8), + RunGroup(1398, 6, "1", colours['green'], 1.0), + RunGroup(1404, 6, "1", colours['green'], 1.2), + RunGroup(1410, 6, "4", colours['cyan'], 0), + RunGroup(2154, 6, "4", colours['cyan'], 0.002), + RunGroup(2160, 6, "4", colours['cyan'], 0.005), + RunGroup(2166, 6, "4", colours['cyan'], 0.01), + RunGroup(2172, 6, "4", colours['cyan'], 0.02), + RunGroup(1542, 6, "4", colours['cyan'], 0.027), + RunGroup(1548, 6, "4", colours['cyan'], 0.038), + RunGroup(1416, 6, "4", colours['cyan'], 0.059), + RunGroup(1422, 6, "4", colours['cyan'], 0.1), + RunGroup(1428, 6, "4", colours['cyan'], 0.2), + RunGroup(1434, 6, "4", colours['cyan'], 0.5), + RunGroup(1440, 6, "4", colours['cyan'], 0.8), + RunGroup(1446, 6, "4", colours['cyan'], 1.0), + RunGroup(1452, 6, "4", colours['cyan'], 1.2), + RunGroup(1458, 6, "9", colours['blue'], 0), + RunGroup(2178, 6, "9", colours['blue'], 0.002), + RunGroup(2184, 6, "9", colours['blue'], 0.005), + RunGroup(2190, 6, "9", colours['blue'], 0.01), + RunGroup(2196, 6, "9", colours['blue'], 0.02), + RunGroup(1554, 6, "9", colours['blue'], 0.027), + RunGroup(1560, 6, "9", colours['blue'], 0.038), + RunGroup(1464, 6, "9", colours['blue'], 0.059), + RunGroup(1470, 6, "9", colours['blue'], 0.1), + RunGroup(1476, 6, "9", colours['blue'], 0.2), + RunGroup(1482, 6, "9", colours['blue'], 0.5), + RunGroup(1488, 6, "9", colours['blue'], 0.8), + RunGroup(1494, 6, "9", colours['blue'], 1.0), + RunGroup(1500, 6, "9", colours['blue'], 1.2), +] + +fig_num = "AH" +fc = fig_cfgs[fig_num] = fc.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} 2D scan of noise stddev and precision (coverage=0.5)" +fc["filename"] = f"Fig {fig_num}.svg" +fc["yaxis_title"] = "Free energy" +fc["data_field"] = "test/free_e" +fc["xaxis_title"] = "Precision (lower image)" +fc["xaxis_range"] = [0, 1.2] +fc["yaxis_range"] = [0.0, 0.5] +fc["yaxis_scale"] = [1.0] + +fig_num = "AG(c)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_type"] = "2d" +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} 2D scan of noise stddev and precision (coverage=1.0)" +fc["filename"] = f"Fig {fig_num}.svg" +fc["xaxis_title"] = "Precision (full image)" +fc["xaxis_range"] = [0, 1.2] +fc["yaxis_range"] = [52 , 88] +fc["groups"] = [ + RunGroup(1602, 6, "0", colours['red'], 0), + RunGroup(1962, 6, "0", colours['red'], 0.002), + RunGroup(1968, 6, "0", colours['red'], 0.005), + RunGroup(1908, 6, "0", colours['red'], 0.01), + RunGroup(1902, 6, "0", colours['red'], 0.02), + RunGroup(1608, 6, "0", colours['red'], 0.027), + RunGroup(1614, 6, "0", colours['red'], 0.038), + RunGroup(1620, 6, "0", colours['red'], 0.059), + RunGroup(1626, 6, "0", colours['red'], 0.1), + RunGroup(1632, 6, "0", colours['red'], 0.2), + RunGroup(1638, 6, "0", colours['red'], 0.5), + RunGroup(1644, 6, "0", colours['red'], 0.8), + RunGroup(1650, 6, "0", colours['red'], 1.0), + RunGroup(1656, 6, "0", colours['red'], 1.2), + RunGroup(1662, 6, "1/4", colours['orange'], 0), + RunGroup(1974, 6, "1/4", colours['orange'], 0.002), + RunGroup(1980, 6, "1/4", colours['orange'], 0.005), + RunGroup(1920, 6, "1/4", colours['orange'], 0.01), + RunGroup(1914, 6, "1/4", colours['orange'], 0.02), + RunGroup(1668, 6, "1/4", colours['orange'], 0.027), + RunGroup(1674, 6, "1/4", colours['orange'], 0.038), + RunGroup(1680, 6, "1/4", colours['orange'], 0.059), + RunGroup(1686, 6, "1/4", colours['orange'], 0.1), + RunGroup(1692, 6, "1/4", colours['orange'], 0.2), + RunGroup(1698, 6, "1/4", colours['orange'], 0.5), + RunGroup(1704, 6, "1/4", colours['orange'], 0.8), + RunGroup(1710, 6, "1/4", colours['orange'], 1.0), + RunGroup(1716, 6, "1/4", colours['orange'], 1.2), + RunGroup(1722, 6, "1", colours['green'], 0), + RunGroup(1986, 6, "1", colours['green'], 0.002), + RunGroup(1992, 6, "1", colours['green'], 0.005), + RunGroup(1932, 6, "1", colours['green'], 0.01), + RunGroup(1926, 6, "1", colours['green'], 0.02), + RunGroup(1728, 6, "1", colours['green'], 0.027), + RunGroup(1734, 6, "1", colours['green'], 0.038), + RunGroup(1740, 6, "1", colours['green'], 0.059), + RunGroup(1746, 6, "1", colours['green'], 0.1), + RunGroup(1752, 6, "1", colours['green'], 0.2), + RunGroup(1758, 6, "1", colours['green'], 0.5), + RunGroup(1764, 6, "1", colours['green'], 0.8), + RunGroup(1770, 6, "1", colours['green'], 1.0), + RunGroup(1776, 6, "1", colours['green'], 1.2), + RunGroup(1782, 6, "4", colours['cyan'], 0), + RunGroup(1998, 6, "4", colours['cyan'], 0.002), + RunGroup(2004, 6, "4", colours['cyan'], 0.005), + RunGroup(1944, 6, "4", colours['cyan'], 0.01), + RunGroup(1938, 6, "4", colours['cyan'], 0.02), + RunGroup(1788, 6, "4", colours['cyan'], 0.027), + RunGroup(1794, 6, "4", colours['cyan'], 0.038), + RunGroup(1800, 6, "4", colours['cyan'], 0.059), + RunGroup(1806, 6, "4", colours['cyan'], 0.1), + RunGroup(1812, 6, "4", colours['cyan'], 0.2), + RunGroup(1818, 6, "4", colours['cyan'], 0.5), + RunGroup(1824, 6, "4", colours['cyan'], 0.8), + RunGroup(1830, 6, "4", colours['cyan'], 1.0), + RunGroup(1836, 6, "4", colours['cyan'], 1.2), + RunGroup(1842, 6, "9", colours['blue'], 0), + RunGroup(2010, 6, "9", colours['blue'], 0.002), + RunGroup(2016, 6, "9", colours['blue'], 0.005), + RunGroup(1956, 6, "9", colours['blue'], 0.01), + RunGroup(1950, 6, "9", colours['blue'], 0.02), + RunGroup(1848, 6, "9", colours['blue'], 0.027), + RunGroup(1854, 6, "9", colours['blue'], 0.038), + RunGroup(1860, 6, "9", colours['blue'], 0.059), + RunGroup(1866, 6, "9", colours['blue'], 0.1), + RunGroup(1872, 6, "9", colours['blue'], 0.2), + RunGroup(1878, 6, "9", colours['blue'], 0.5), + RunGroup(1884, 6, "9", colours['blue'], 0.8), + RunGroup(1890, 6, "9", colours['blue'], 1.0), + RunGroup(1896, 6, "9", colours['blue'], 1.2), +] + +fig_num = "AK" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_type"] = "2d" +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} 2D scan of noise std and prec (coverage=n1.0/p0.5)" +fc["filename"] = f"Fig {fig_num}.svg" +fc["xaxis_title"] = "Precision" +fc["xaxis_range"] = [0, 1.2] +fc["yaxis_range"] = [7, 88] +fc["groups"] = [ + RunGroup(2074, 1, "0", colours['red'], 0), + RunGroup(2022, 1, "0", colours['red'], 0.002), + RunGroup(2023, 1, "0", colours['red'], 0.005), + RunGroup(2024, 1, "0", colours['red'], 0.01), + RunGroup(2025, 1, "0", colours['red'], 0.02), + RunGroup(2026, 1, "0", colours['red'], 0.027), + RunGroup(2075, 1, "0", colours['red'], 0.038), + RunGroup(2028, 1, "0", colours['red'], 0.059), + RunGroup(2029, 1, "0", colours['red'], 0.1), + RunGroup(2030, 1, "0", colours['red'], 0.2), + RunGroup(2031, 1, "0", colours['red'], 0.5), + RunGroup(2032, 1, "0", colours['red'], 0.8), + RunGroup(2033, 1, "0", colours['red'], 1.0), + RunGroup(2034, 1, "0", colours['red'], 1.2), + RunGroup(2076, 1, "1.0", colours['green'], 0), + RunGroup(2035, 1, "1.0", colours['green'], 0.002), + RunGroup(2036, 1, "1.0", colours['green'], 0.005), + RunGroup(2037, 1, "1.0", colours['green'], 0.01), + RunGroup(2038, 1, "1.0", colours['green'], 0.02), + RunGroup(2039, 1, "1.0", colours['green'], 0.027), + RunGroup(2077, 1, "1.0", colours['green'], 0.038), + RunGroup(2041, 1, "1.0", colours['green'], 0.059), + RunGroup(2042, 1, "1.0", colours['green'], 0.1), + RunGroup(2043, 1, "1.0", colours['green'], 0.2), + RunGroup(2044, 1, "1.0", colours['green'], 0.5), + RunGroup(2045, 1, "1.0", colours['green'], 0.8), + RunGroup(2046, 1, "1.0", colours['green'], 1.0), + RunGroup(2047, 1, "1.0", colours['green'], 1.2), + RunGroup(2078, 1, "2.0", colours['cyan'], 0), + RunGroup(2048, 1, "2.0", colours['cyan'], 0.002), + RunGroup(2049, 1, "2.0", colours['cyan'], 0.005), + RunGroup(2050, 1, "2.0", colours['cyan'], 0.01), + RunGroup(2051, 1, "2.0", colours['cyan'], 0.02), + RunGroup(2052, 1, "2.0", colours['cyan'], 0.027), + RunGroup(2079, 1, "2.0", colours['cyan'], 0.038), + RunGroup(2054, 1, "2.0", colours['cyan'], 0.059), + RunGroup(2055, 1, "2.0", colours['cyan'], 0.1), + RunGroup(2056, 1, "2.0", colours['cyan'], 0.2), + RunGroup(2057, 1, "2.0", colours['cyan'], 0.5), + RunGroup(2058, 1, "2.0", colours['cyan'], 0.8), + RunGroup(2059, 1, "2.0", colours['cyan'], 1.0), + RunGroup(2060, 1, "2.0", colours['cyan'], 1.2), + RunGroup(2080, 1, "3.0", colours['blue'], 0), + RunGroup(2061, 1, "3.0", colours['blue'], 0.002), + RunGroup(2062, 1, "3.0", colours['blue'], 0.005), + RunGroup(2063, 1, "3.0", colours['blue'], 0.01), + RunGroup(2064, 1, "3.0", colours['blue'], 0.02), + RunGroup(2065, 1, "3.0", colours['blue'], 0.027), + RunGroup(2081, 1, "3.0", colours['blue'], 0.038), + RunGroup(2067, 1, "3.0", colours['blue'], 0.059), + RunGroup(2068, 1, "3.0", colours['blue'], 0.1), + RunGroup(2069, 1, "3.0", colours['blue'], 0.2), + RunGroup(2070, 1, "3.0", colours['blue'], 0.5), + RunGroup(2071, 1, "3.0", colours['blue'], 0.8), + RunGroup(2072, 1, "3.0", colours['blue'], 1.0), + RunGroup(2073, 1, "3.0", colours['blue'], 1.2), +] + +fig_num = "AL" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_type"] = "2d" +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} scan of per pixel noise (coverage=0.5/1.0)" +fc["filename"] = f"Fig {fig_num}.svg" +fc["xaxis_title"] = "Noise range (=2.0 +/- x)" +fc["xaxis_range"] = [0, 3.0] +fc["yaxis_range"] = [72, 88] +fc["groups"] = [ + RunGroup(2202, 6, "0.5", colours['red'], 0), + RunGroup(2208, 6, "1.0", colours['blue'], 0), + RunGroup(2214, 6, "0.5", colours['red'], 0.2), + RunGroup(2220, 6, "1.0", colours['blue'], 0.2), + RunGroup(2226, 6, "0.5", colours['red'], 0.4), + RunGroup(2232, 6, "1.0", colours['blue'], 0.4), + RunGroup(2238, 6, "0.5", colours['red'], 0.6), + RunGroup(2244, 6, "1.0", colours['blue'], 0.6), + RunGroup(2250, 6, "0.5", colours['red'], 0.8), + RunGroup(2256, 6, "1.0", colours['blue'], 0.8), + RunGroup(2262, 6, "0.5", colours['red'], 1.0), + RunGroup(2268, 6, "1.0", colours['blue'], 1.0), + RunGroup(2274, 6, "0.5", colours['red'], 1.2), + RunGroup(2280, 6, "1.0", colours['blue'], 1.2), + RunGroup(2286, 6, "0.5", colours['red'], 1.4), + RunGroup(2292, 6, "1.0", colours['blue'], 1.4), + RunGroup(2298, 6, "0.5", colours['red'], 1.6), + RunGroup(2304, 6, "1.0", colours['blue'], 1.6), + RunGroup(2310, 6, "0.5", colours['red'], 1.8), + RunGroup(2316, 6, "1.0", colours['blue'], 1.8), + RunGroup(2322, 6, "0.5", colours['red'], 2.0), + RunGroup(2328, 6, "1.0", colours['blue'], 2.0), + RunGroup(2334, 6, "0.5", colours['red'], 2.2), + RunGroup(2340, 6, "1.0", colours['blue'], 2.2), + RunGroup(2346, 6, "0.5", colours['red'], 2.4), + RunGroup(2352, 6, "1.0", colours['blue'], 2.4), + RunGroup(2358, 6, "0.5", colours['red'], 2.6), + RunGroup(2364, 6, "1.0", colours['blue'], 2.6), + RunGroup(2370, 6, "0.5", colours['red'], 2.8), + RunGroup(2376, 6, "1.0", colours['blue'], 2.8), + RunGroup(2382, 6, "0.5", colours['red'], 3.0), + RunGroup(2388, 6, "1.0", colours['blue'], 3.0), +] + +fig_num = "AM" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_type"] = "2d" +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} 2D scan of its mult and prec (noise sd=3.0, cov=1.0)" +fc["filename"] = f"Fig {fig_num}.svg" +fc["xaxis_title"] = "Precision (full image)" +fc["xaxis_range"] = [0, 1.2] +fc["yaxis_range"] = [7, 88] +fc["groups"] = [ + RunGroup(2414, 1, "0.5", colours['red'], 0), + RunGroup(2415, 1, "0.5", colours['red'], 0.002), + RunGroup(2416, 1, "0.5", colours['red'], 0.005), + RunGroup(2417, 1, "0.5", colours['red'], 0.01), + RunGroup(2418, 1, "0.5", colours['red'], 0.02), + RunGroup(2419, 1, "0.5", colours['red'], 0.027), + RunGroup(2420, 1, "0.5", colours['red'], 0.038), + RunGroup(2421, 1, "0.5", colours['red'], 0.059), + RunGroup(2422, 1, "0.5", colours['red'], 0.1), + RunGroup(2423, 1, "0.5", colours['red'], 0.2), + RunGroup(2424, 1, "0.5", colours['red'], 0.5), + RunGroup(2440, 1, "0.5", colours['red'], 0.8), + RunGroup(2425, 1, "0.5", colours['red'], 1.0), + RunGroup(2426, 1, "0.5", colours['red'], 1.2), + RunGroup(1842, 6, "1.0", colours['blue'], 0), + RunGroup(2010, 6, "1.0", colours['blue'], 0.002), + RunGroup(2016, 6, "1.0", colours['blue'], 0.005), + RunGroup(1956, 6, "1.0", colours['blue'], 0.01), + RunGroup(1950, 6, "1.0", colours['blue'], 0.02), + RunGroup(1848, 6, "1.0", colours['blue'], 0.027), + RunGroup(1854, 6, "1.0", colours['blue'], 0.038), + RunGroup(1860, 6, "1.0", colours['blue'], 0.059), + RunGroup(1866, 6, "1.0", colours['blue'], 0.1), + RunGroup(1872, 6, "1.0", colours['blue'], 0.2), + RunGroup(1878, 6, "1.0", colours['blue'], 0.5), + RunGroup(1884, 6, "1.0", colours['blue'], 0.8), + RunGroup(1890, 6, "1.0", colours['blue'], 1.0), + RunGroup(1896, 6, "1.0", colours['blue'], 1.2), + RunGroup(2427, 1, "2.0", colours['green'], 0), + RunGroup(2428, 1, "2.0", colours['green'], 0.002), + RunGroup(2429, 1, "2.0", colours['green'], 0.005), + RunGroup(2430, 1, "2.0", colours['green'], 0.01), + RunGroup(2431, 1, "2.0", colours['green'], 0.02), + RunGroup(2432, 1, "2.0", colours['green'], 0.027), + RunGroup(2433, 1, "2.0", colours['green'], 0.038), + RunGroup(2434, 1, "2.0", colours['green'], 0.059), + RunGroup(2435, 1, "2.0", colours['green'], 0.1), + RunGroup(2436, 1, "2.0", colours['green'], 0.2), + RunGroup(2437, 1, "2.0", colours['green'], 0.5), + RunGroup(2441, 1, "2.0", colours['green'], 0.8), + RunGroup(2438, 1, "2.0", colours['green'], 1.0), + RunGroup(2439, 1, "2.0", colours['green'], 1.2), +] + +fig_num = "AN" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_type"] = "2d" +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} 2D scan of its mult and prec (noise sd=0)" +fc["filename"] = f"Fig {fig_num}.svg" +fc["xaxis_title"] = "Precision (full image)" +fc["xaxis_range"] = [0, 1.2] +fc["yaxis_range"] = [7, 88] +fc["groups"] = [ + RunGroup(2442, 1, "0.5", colours['red'], 0), + RunGroup(2443, 1, "0.5", colours['red'], 0.002), + RunGroup(2444, 1, "0.5", colours['red'], 0.005), + RunGroup(2445, 1, "0.5", colours['red'], 0.01), + RunGroup(2446, 1, "0.5", colours['red'], 0.02), + RunGroup(2447, 1, "0.5", colours['red'], 0.027), + RunGroup(2448, 1, "0.5", colours['red'], 0.038), + RunGroup(2449, 1, "0.5", colours['red'], 0.059), + RunGroup(2450, 1, "0.5", colours['red'], 0.1), + RunGroup(2451, 1, "0.5", colours['red'], 0.2), + RunGroup(2452, 1, "0.5", colours['red'], 0.5), + RunGroup(2453, 1, "0.5", colours['red'], 0.8), + RunGroup(2454, 1, "0.5", colours['red'], 1.0), + RunGroup(2455, 1, "0.5", colours['red'], 1.2), + RunGroup(1602, 6, "1.0", colours['blue'], 0), + RunGroup(1962, 6, "1.0", colours['blue'], 0.002), + RunGroup(1968, 6, "1.0", colours['blue'], 0.005), + RunGroup(1908, 6, "1.0", colours['blue'], 0.01), + RunGroup(1902, 6, "1.0", colours['blue'], 0.02), + RunGroup(1608, 6, "1.0", colours['blue'], 0.027), + RunGroup(1614, 6, "1.0", colours['blue'], 0.038), + RunGroup(1620, 6, "1.0", colours['blue'], 0.059), + RunGroup(1626, 6, "1.0", colours['blue'], 0.1), + RunGroup(1632, 6, "1.0", colours['blue'], 0.2), + RunGroup(1638, 6, "1.0", colours['blue'], 0.5), + RunGroup(1644, 6, "1.0", colours['blue'], 0.8), + RunGroup(1650, 6, "1.0", colours['blue'], 1.0), + RunGroup(1656, 6, "1.0", colours['blue'], 1.2), + RunGroup(2456, 1, "2.0", colours['green'], 0), + RunGroup(2457, 1, "2.0", colours['green'], 0.002), + RunGroup(2458, 1, "2.0", colours['green'], 0.005), + RunGroup(2459, 1, "2.0", colours['green'], 0.01), + RunGroup(2460, 1, "2.0", colours['green'], 0.02), + RunGroup(2461, 1, "2.0", colours['green'], 0.027), + RunGroup(2462, 1, "2.0", colours['green'], 0.038), + RunGroup(2463, 1, "2.0", colours['green'], 0.059), + RunGroup(2464, 1, "2.0", colours['green'], 0.1), + RunGroup(2465, 1, "2.0", colours['green'], 0.2), + RunGroup(2466, 1, "2.0", colours['green'], 0.5), + RunGroup(2467, 1, "2.0", colours['green'], 0.8), + RunGroup(2468, 1, "2.0", colours['green'], 1.0), + RunGroup(2469, 1, "2.0", colours['green'], 1.2), +] + +fig_num = "AG(b)" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_type"] = "2d" +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} 2D scan of its mult and prec (noise sd=3, cov=0.5)" +fc["filename"] = f"Fig {fig_num}.svg" +fc["xaxis_title"] = "Precision (lower image)" +fc["xaxis_range"] = [0, 1.2] +fc["yaxis_range"] = [67, 88] +fc["groups"] = [ + RunGroup(2470, 1, "x 0.5", colours['red'], 0), + RunGroup(2471, 1, "x 0.5", colours['red'], 0.002), + RunGroup(2472, 1, "x 0.5", colours['red'], 0.005), + RunGroup(2473, 1, "x 0.5", colours['red'], 0.01), + RunGroup(2474, 1, "x 0.5", colours['red'], 0.02), + RunGroup(2475, 1, "x 0.5", colours['red'], 0.027), + RunGroup(2476, 1, "x 0.5", colours['red'], 0.038), + RunGroup(2477, 1, "x 0.5", colours['red'], 0.059), + RunGroup(2478, 1, "x 0.5", colours['red'], 0.1), + RunGroup(2479, 1, "x 0.5", colours['red'], 0.2), + RunGroup(2480, 1, "x 0.5", colours['red'], 0.5), + RunGroup(2481, 1, "x 0.5", colours['red'], 0.8), + RunGroup(2482, 1, "x 0.5", colours['red'], 1.0), + RunGroup(2483, 1, "x 0.5", colours['red'], 1.2), + RunGroup(1458, 6, "x 1.0", colours['blue'], 0), + RunGroup(2178, 6, "x 1.0", colours['blue'], 0.002), + RunGroup(2184, 6, "x 1.0", colours['blue'], 0.005), + RunGroup(2190, 6, "x 1.0", colours['blue'], 0.01), + RunGroup(2196, 6, "x 1.0", colours['blue'], 0.02), + RunGroup(1554, 6, "x 1.0", colours['blue'], 0.027), + RunGroup(1560, 6, "x 1.0", colours['blue'], 0.038), + RunGroup(1464, 6, "x 1.0", colours['blue'], 0.059), + RunGroup(1470, 6, "x 1.0", colours['blue'], 0.1), + RunGroup(1477, 6, "x 1.0", colours['blue'], 0.2), + RunGroup(1482, 6, "x 1.0", colours['blue'], 0.5), + RunGroup(1488, 6, "x 1.0", colours['blue'], 0.8), + RunGroup(1494, 6, "x 1.0", colours['blue'], 1.0), + RunGroup(1500, 6, "x 1.0", colours['blue'], 1.2), + RunGroup(2484, 1, "x 2.0", colours['green'], 0), + RunGroup(2485, 1, "x 2.0", colours['green'], 0.002), + RunGroup(2486, 1, "x 2.0", colours['green'], 0.005), + RunGroup(2487, 1, "x 2.0", colours['green'], 0.01), + RunGroup(2488, 1, "x 2.0", colours['green'], 0.02), + RunGroup(2489, 1, "x 2.0", colours['green'], 0.027), + RunGroup(2490, 1, "x 2.0", colours['green'], 0.038), + RunGroup(2491, 1, "x 2.0", colours['green'], 0.059), + RunGroup(2492, 1, "x 2.0", colours['green'], 0.1), + RunGroup(2493, 1, "x 2.0", colours['green'], 0.2), + RunGroup(2494, 1, "x 2.0", colours['green'], 0.5), + RunGroup(2495, 1, "x 2.0", colours['green'], 0.8), + RunGroup(2496, 1, "x 2.0", colours['green'], 1.0), + RunGroup(2497, 1, "x 2.0", colours['green'], 1.2), +] + +fig_num = "AQ" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_type"] = "2d" +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} 2D scan of its mult and prec (noise sd=0, cov=0.5)" +fc["filename"] = f"Fig {fig_num}.svg" +fc["xaxis_title"] = "Precision (lower image)" +fc["xaxis_range"] = [0, 1.2] +fc["yaxis_range"] = [7, 88] +fc["groups"] = [ + RunGroup(2498, 1, "0.5", colours['red'], 0), + RunGroup(2499, 1, "0.5", colours['red'], 0.002), + RunGroup(2500, 1, "0.5", colours['red'], 0.005), + RunGroup(2501, 1, "0.5", colours['red'], 0.01), + RunGroup(2502, 1, "0.5", colours['red'], 0.02), + RunGroup(2503, 1, "0.5", colours['red'], 0.027), + RunGroup(2504, 1, "0.5", colours['red'], 0.038), + RunGroup(2505, 1, "0.5", colours['red'], 0.059), + RunGroup(2506, 1, "0.5", colours['red'], 0.1), + RunGroup(2507, 1, "0.5", colours['red'], 0.2), + RunGroup(2508, 1, "0.5", colours['red'], 0.5), + RunGroup(2509, 1, "0.5", colours['red'], 0.8), + RunGroup(2510, 1, "0.5", colours['red'], 1.0), + RunGroup(2511, 1, "0.5", colours['red'], 1.2), + RunGroup(1266, 6, "1.0", colours['blue'], 0), + RunGroup(2082, 6, "1.0", colours['blue'], 0.002), + RunGroup(2088, 6, "1.0", colours['blue'], 0.005), + RunGroup(2094, 6, "1.0", colours['blue'], 0.01), + RunGroup(2100, 6, "1.0", colours['blue'], 0.02), + RunGroup(1506, 6, "1.0", colours['blue'], 0.027), + RunGroup(1512, 6, "1.0", colours['blue'], 0.038), + RunGroup(1272, 6, "1.0", colours['blue'], 0.059), + RunGroup(1278, 6, "1.0", colours['blue'], 0.1), + RunGroup(1284, 6, "1.0", colours['blue'], 0.2), + RunGroup(1290, 6, "1.0", colours['blue'], 0.5), + RunGroup(1296, 6, "1.0", colours['blue'], 0.8), + RunGroup(1302, 6, "1.0", colours['blue'], 1.0), + RunGroup(1308, 6, "1.0", colours['blue'], 1.2), + RunGroup(2512, 1, "2.0", colours['green'], 0), + RunGroup(2513, 1, "2.0", colours['green'], 0.002), + RunGroup(2514, 1, "2.0", colours['green'], 0.005), + RunGroup(2515, 1, "2.0", colours['green'], 0.01), + RunGroup(2516, 1, "2.0", colours['green'], 0.02), + RunGroup(2517, 1, "2.0", colours['green'], 0.027), + RunGroup(2518, 1, "2.0", colours['green'], 0.038), + RunGroup(2519, 1, "2.0", colours['green'], 0.059), + RunGroup(2520, 1, "2.0", colours['green'], 0.1), + RunGroup(2521, 1, "2.0", colours['green'], 0.2), + RunGroup(2522, 1, "2.0", colours['green'], 0.5), + RunGroup(2523, 1, "2.0", colours['green'], 0.8), + RunGroup(2524, 1, "2.0", colours['green'], 1.0), + RunGroup(2525, 1, "2.0", colours['green'], 1.2), +] + +fig_num = "AR" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_type"] = "2d" +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} 2D scan of its mult and prec B (noise sd=3, cov=0.5)" +fc["filename"] = f"Fig {fig_num}.svg" +fc["xaxis_title"] = "Precision (lower image)" +fc["xaxis_range"] = [0, 1.2] +fc["yaxis_range"] = [7, 88] +fc["groups"] = [ + RunGroup(2530, 1, "1/P", colours['red'], 0), + RunGroup(2531, 1, "1/P", colours['red'], 0.002), + RunGroup(2532, 1, "1/P", colours['red'], 0.005), + RunGroup(2533, 1, "1/P", colours['red'], 0.01), + RunGroup(2534, 1, "1/P", colours['red'], 0.02), + RunGroup(2535, 1, "1/P", colours['red'], 0.027), + RunGroup(2536, 1, "1/P", colours['red'], 0.038), + RunGroup(2537, 1, "1/P", colours['red'], 0.059), + RunGroup(2538, 1, "1/P", colours['red'], 0.1), + RunGroup(2539, 1, "1/P", colours['red'], 0.2), + RunGroup(2540, 1, "1/P", colours['red'], 0.5), + RunGroup(2541, 1, "1/P", colours['red'], 0.8), + RunGroup(2542, 1, "1/P", colours['red'], 1.0), + RunGroup(2543, 1, "1/P", colours['red'], 1.2), + RunGroup(1458, 6, "1.0", colours['blue'], 0), + RunGroup(2178, 6, "1.0", colours['blue'], 0.002), + RunGroup(2184, 6, "1.0", colours['blue'], 0.005), + RunGroup(2190, 6, "1.0", colours['blue'], 0.01), + RunGroup(2196, 6, "1.0", colours['blue'], 0.02), + RunGroup(1554, 6, "1.0", colours['blue'], 0.027), + RunGroup(1560, 6, "1.0", colours['blue'], 0.038), + RunGroup(1464, 6, "1.0", colours['blue'], 0.059), + RunGroup(1470, 6, "1.0", colours['blue'], 0.1), + RunGroup(1477, 6, "1.0", colours['blue'], 0.2), + RunGroup(1482, 6, "1.0", colours['blue'], 0.5), + RunGroup(1488, 6, "1.0", colours['blue'], 0.8), + RunGroup(1494, 6, "1.0", colours['blue'], 1.0), + RunGroup(1500, 6, "1.0", colours['blue'], 1.2), +] + +fig_num = "AS" +fc = fig_cfgs[fig_num] = testacc_fig_cfg.copy() +fc["fig_num"] = fig_num +fc["title"] = f"Fig. {fig_num} bottom half noise, bottom half layer 3 prec scan, FashionMNIST" +fc["filename"] = f"Fig {fig_num}.svg" +fc["groups"] = [ + # RunGroup(817, 6, "0.1", colours['darkred'], None), + RunGroup(2571, 6, "0.2", colours['red'], None), + RunGroup(2565, 6, "0.5", colours['orange'], None), + # RunGroup(835, 6, "0.8", colours['green'], None), + RunGroup(2559, 6, "1", colours['green'], None), + RunGroup(2577, 1, "2", colours['cyan'], None), + RunGroup(2578, 1, "5", colours['blue'], None), + # RunGroup(859, 6, "8", colours['grey25'], None), + # RunGroup(865, 6, "10", colours['darkblue'], None), + RunGroup(2547, 6, "No noise", colours['grey25'], None), +] + +show_figs = True +save_figs = True +figs = { + # "D(a)": NickFig(**fig_cfgs["D(a)"], show_fig=show_figs, save_fig=save_figs), + # "D(b)": NickFig(**fig_cfgs["D(b)"], show_fig=show_figs, save_fig=save_figs), + # "D(c)": NickFig(**fig_cfgs["D(c)"], show_fig=show_figs, save_fig=save_figs), + # "G(a)": NickFig(**fig_cfgs["G(a)"], show_fig=show_figs, save_fig=save_figs), + # "G(b)": NickFig(**fig_cfgs["G(b)"], show_fig=show_figs, save_fig=save_figs), + # "I(a)": NickFig(**fig_cfgs["I(a)"], show_fig=show_figs, save_fig=save_figs), + # "I(b)": NickFig(**fig_cfgs["I(b)"], show_fig=show_figs, save_fig=save_figs), + # "K(a)": NickFig(**fig_cfgs["K(a)"], show_fig=show_figs, save_fig=save_figs), + # "K(b)": NickFig(**fig_cfgs["K(b)"], show_fig=show_figs, save_fig=save_figs), + # "L": NickFig(**fig_cfgs["L"], show_fig=show_figs, save_fig=save_figs), + # "M(a)": NickFig(**fig_cfgs["M(a)"], show_fig=show_figs, save_fig=save_figs), + # "M(b)": NickFig(**fig_cfgs["M(b)"], show_fig=show_figs, save_fig=save_figs), + # "M(c)": NickFig(**fig_cfgs["M(c)"], show_fig=show_figs, save_fig=save_figs), + # "N(a)": NickFig(**fig_cfgs["N(a)"], show_fig=show_figs, save_fig=save_figs), + # "N(b)": NickFig(**fig_cfgs["N(b)"], show_fig=show_figs, save_fig=save_figs), + # "P(a)": NickFig(**fig_cfgs["P(a)"], show_fig=show_figs, save_fig=save_figs), + # "P(b)": NickFig(**fig_cfgs["P(b)"], show_fig=show_figs, save_fig=save_figs), + # "Q": NickFig(**fig_cfgs["Q"], show_fig=show_figs, save_fig=save_figs), + # "R": NickFig(**fig_cfgs["R"], show_fig=show_figs, save_fig=save_figs), + # "S": NickFig(**fig_cfgs["S"], show_fig=show_figs, save_fig=save_figs), + # "T": NickFig(**fig_cfgs["T"], show_fig=show_figs, save_fig=save_figs), + # "U": NickFig(**fig_cfgs["U"], show_fig=show_figs, save_fig=save_figs), + # "V": NickFig(**fig_cfgs["V"], show_fig=show_figs, save_fig=save_figs), + "AC": NickFig(**fig_cfgs["AC"], show_fig=show_figs, save_fig=save_figs), + # "AD": NickFig(**fig_cfgs["AD"], show_fig=show_figs, save_fig=save_figs), + # "AF": NickFig(**fig_cfgs["AF"], show_fig=show_figs, save_fig=save_figs), + # "AG(a)": NickFig(**fig_cfgs["AG(a)"], show_fig=show_figs, save_fig=save_figs), + # "AH": NickFig(**fig_cfgs["AH"], show_fig=show_figs, save_fig=save_figs), + # "AG(c)": NickFig(**fig_cfgs["AG(c)"], show_fig=show_figs, save_fig=save_figs), + # "AK": NickFig(**fig_cfgs["AK"], show_fig=show_figs, save_fig=save_figs), + # "AL": NickFig(**fig_cfgs["AL"], show_fig=show_figs, save_fig=save_figs), + # "AM": NickFig(**fig_cfgs["AM"], show_fig=show_figs, save_fig=save_figs), + # "AN": NickFig(**fig_cfgs["AN"], show_fig=show_figs, save_fig=save_figs), + # "AG(b)": NickFig(**fig_cfgs["AG(b)"], show_fig=show_figs, save_fig=save_figs), + # "AQ": NickFig(**fig_cfgs["AQ"], show_fig=show_figs, save_fig=save_figs), + # "AR": NickFig(**fig_cfgs["AR"], show_fig=show_figs, save_fig=save_figs), + # "AS": NickFig(**fig_cfgs["AS"], show_fig=show_figs, save_fig=save_figs), +} + +# app = Dash(__name__) +# +# app.layout = html.Div(children=[ +# html.H1(children='Hello Dash'), +# +# html.Div(children=''' +# Dash: A web application framework for your data. +# '''), +# +# dcc.Graph( +# id='figD', +# figure=figs["D"].get_fig() +# ), +# dcc.Graph( +# id='figE', +# figure=figs["E"].get_fig() +# ), +# ]) +# +# if __name__ == '__main__': +# app.run_server(debug=False) diff --git a/pypc/layers.py b/pypc/layers.py index de7b062..a0be318 100644 --- a/pypc/layers.py +++ b/pypc/layers.py @@ -1,9 +1,8 @@ import math import torch -import numpy as np -from copy import deepcopy +# import numpy as np from torch import nn -import torch.nn.functional as F +# import torch.nn.functional as F from pypc import utils @@ -11,6 +10,16 @@ class Layer(nn.Module): def __init__( self, in_size, out_size, act_fn, use_bias=False, kaiming_init=False, is_forward=False ): + """ + Initialise layer + + :param in_size: Number of input nodes + :param out_size: Number of output nodes + :param act_fn: Activation function + :param use_bias: Include bias terms? + :param kaiming_init: Use Kaiming weight initialisation? + :param is_forward: + """ super().__init__() self.in_size = in_size self.out_size = out_size @@ -19,34 +28,47 @@ def __init__( self.is_forward = is_forward self.kaiming_init = kaiming_init - self.weights = None - self.bias = None - self.grad = {"weights": None, "bias": None} - - if kaiming_init: - self._reset_params_kaiming() - else: - self._reset_params() + self._reset_grad() + self.reset() def forward(self, *args, **kwargs): + """ + Abstract forward pass method. Must be overridden. + + :param args: + :param kwargs: + """ raise NotImplementedError def reset(self): + """ + Reset weights and biases + """ if self.kaiming_init: self._reset_params_kaiming() else: self._reset_params() def _reset_grad(self): + """ + Reset gradients + """ self.grad = {"weights": None, "bias": None} def _reset_params(self): + """ + Reset weights to be normally distributed with mean=0.0 and std dev 0.05 and reset biases to zero + """ weights = torch.empty((self.in_size, self.out_size)).normal_(mean=0.0, std=0.05) bias = torch.zeros((self.out_size)) self.weights = utils.set_tensor(weights) self.bias = utils.set_tensor(bias) def _reset_params_kaiming(self): + """ + Reset weights using Kaiming He initialisation and reset biases to be uniformly distributed about zero + NOTE: He et al (2015) arXiv:1502.01852v1 initialise biases to zero + """ self.weights = utils.set_tensor(torch.empty((self.in_size, self.out_size))) self.bias = utils.set_tensor(torch.zeros((self.out_size))) if isinstance(self.act_fn, utils.Linear): @@ -65,11 +87,26 @@ class FCLayer(Layer): def __init__( self, in_size, out_size, act_fn, use_bias=False, kaiming_init=False, is_forward=False ): + """ + Initialise fully connected layer + + :param in_size: Number of input nodes + :param out_size: Number of output nodes + :param act_fn: Activation function + :param use_bias: Include bias terms? + :param kaiming_init: Use Kaiming weight initialisation? + :param is_forward: + """ super().__init__(in_size, out_size, act_fn, use_bias, kaiming_init, is_forward=is_forward) - self.use_bias = use_bias self.inp = None def forward(self, inp): + """ + Perform forward pass for batch + + :param inp: Node inputs for batch, Tensor:(batch_size, in_size) + :return: Node outputs for batch, Tensor:(batch_size, out_size) + """ self.inp = inp.clone() out = self.act_fn(torch.matmul(self.inp, self.weights)) if self.use_bias: @@ -77,11 +114,22 @@ def forward(self, inp): return out def backward(self, err): + """ + Perform backward pass for batch + + :param err: Errors for batch, Tensor:(batch_size, out_size) + :return: Tensor:(batch_size, in_size) + """ fn_deriv = self.act_fn.deriv(torch.matmul(self.inp, self.weights)) out = torch.matmul(err * fn_deriv, self.weights.T) return out def update_gradient(self, err): + """ + Update gradients for weights and biases + + :param err: Errors for batch, Tensor:(batch_size, out_size) + """ fn_deriv = self.act_fn.deriv(torch.matmul(self.inp, self.weights)) delta = torch.matmul(self.inp.T, err * fn_deriv) self.grad["weights"] = delta @@ -89,154 +137,153 @@ def update_gradient(self, err): self.grad["bias"] = torch.sum(err, axis=0) - -class ConvLayer(Layer): - def __init__(self,input_size,num_channels,num_filters,batch_size,kernel_size,learning_rate,act_fn,padding=0,stride=1,device="cpu"): - self.input_size = input_size - self.num_channels = num_channels - self.num_filters = num_filters - self.batch_size = batch_size - self.kernel_size = kernel_size - self.padding = padding - self.stride = stride - self.output_size = math.floor((self.input_size + (2 * self.padding) - self.kernel_size)/self.stride) +1 - self.learning_rate = learning_rate - self.act_fn - self.device = device - self.kernel= torch.empty(self.num_filters,self.num_channels,self.kernel_size,self.kernel_size).normal_(mean=0,std=0.05).to(self.device) - self.unfold = nn.Unfold(kernel_size=(self.kernel_size,self.kernel_size),padding=self.padding,stride=self.stride).to(self.device) - self.fold = nn.Fold(output_size=(self.input_size,self.input_size),kernel_size=(self.kernel_size,self.kernel_size),padding=self.padding,stride=self.stride).to(self.device) - - def forward(self,inp): - self.X_col = self.unfold(inp.clone()) - self.flat_weights = self.kernel.reshape(self.num_filters,-1) - out = self.flat_weights @ self.X_col - self.activations = out.reshape(self.batch_size, self.num_filters, self.output_size, self.output_size) - return self.f(self.activations) - - def update_gradient(self,e): - fn_deriv = self.act_fn.deriv(self.activations) - e = e * fn_deriv - self.dout = e.reshape(self.batch_size,self.num_filters,-1) - dW = self.dout @ self.X_col.permute(0,2,1) - dW = torch.sum(dW,dim=0) - dW = dW.reshape((self.num_filters,self.num_channels,self.kernel_size,self.kernel_size)) - self.grad["weights"] = dW - - def backward(self,e): - fn_deriv = self.act_fn.deriv(self.activations) - e = e * fn_deriv - self.dout = e.reshape(self.batch_size,self.num_filters,-1) - dX_col = self.flat_weights.T @ self.dout - dX = self.fold(dX_col) - return torch.clamp(dX,-50,50) - - def get_true_weight_grad(self): - return self.kernel.grad - - def set_weight_parameters(self): - self.kernel = nn.Parameter(self.kernel) - - def save_layer(self,logdir,i): - np.save(logdir +"/layer_"+str(i)+"_weights.npy",self.kernel.detach().cpu().numpy()) - - def load_layer(self,logdir,i): - kernel = np.load(logdir +"/layer_"+str(i)+"_weights.npy") - self.kernel = set_tensor(torch.from_numpy(kernel)) - -class MaxPool(Layer): - def __init__(self, kernel_size,device='cpu'): - self.kernel_size = kernel_size - self.device = device - self.activations = torch.empty(1) - self.weights = torch.zeros((1,1)).to(self.device) - - def forward(self,x): - out, self.idxs = F.max_pool2d(x, self.kernel_size,return_indices=True) - return out - - def backward(self, y): - return F.max_unpool2d(y,self.idxs, self.kernel_size) - - def update_gradient(self,e,update_weights=False,sign_reverse=False): - self.grad["weights"] = torch.zeros_like(self.weights) - - def get_true_weight_grad(self): - return None - - def set_weight_parameters(self): - pass - - def save_layer(self,logdir,i): - pass - - def load_layer(self,logdir,i): - pass - -class AvgPool(Layer): - def __init__(self, kernel_size,device='cpu'): - self.kernel_size = kernel_size - self.device = device - self.activations = torch.empty(1) - self.weights = torch.zeros((1,1)).to(self.device) - - - def forward(self, x): - self.B_in,self.C_in,self.H_in,self.W_in = x.shape - return F.avg_pool2d(x,self.kernel_size) - - def backward(self, y): - N,C,H,W = y.shape - print("in backward: ", y.shape) - return F.interpolate(y,scale_factor=(1,1,self.kernel_size,self.kernel_size)) - - def update_gradient(self,e,update_weights=False, sign_reverse=False): - self.grad["weights"] = torch.zeros_like(self.weights) - - def save_layer(self,logdir,i): - pass - - def load_layer(self,logdir,i): - pass - -class ProjectionLayer(Layer): - def __init__(self,input_size, output_size,act_fn,learning_rate,device='cpu'): - self.input_size = input_size - self.B, self.C, self.H, self.W = self.input_size - self.output_size =output_size - self.learning_rate = learning_rate - self.act_fn = act_fn - self.device = device - self.Hid = self.C * self.H * self.W - self.weights = torch.empty((self.Hid, self.output_size)).normal_(mean=0.0, std=0.05).to(self.device) - - def forward(self, x): - self.inp = x.detach().clone() - out = x.reshape((len(x), -1)) - self.activations = torch.matmul(out,self.weights) - return self.f(self.activations) - - def backward(self, e): - fn_deriv = self.act_fn.deriv(self.activations) - out = torch.matmul(e * fn_deriv, self.weights.T) - out = out.reshape((len(e), self.C, self.H, self.W)) - return torch.clamp(out,-50,50) - - def update_gradient(self, e,update_weights=False,sign_reverse=False): - out = self.inp.reshape((len(self.inp), -1)) - fn_deriv = self.act_fn.deriv(self.activations) - dw = torch.matmul(out.T, e * fn_deriv) - self.grad["weights"] = dw - - def get_true_weight_grad(self): - return self.weights.grad - - def set_weight_parameters(self): - self.weights = nn.Parameter(self.weights) - - def save_layer(self,logdir,i): - np.save(logdir +"/layer_"+str(i)+"_weights.npy",self.weights.detach().cpu().numpy()) - - def load_layer(self,logdir,i): - weights = np.load(logdir +"/layer_"+str(i)+"_weights.npy") - self.weights = set_tensor(torch.from_numpy(weights)) +# class ConvLayer(Layer): +# def __init__(self,input_size,num_channels,num_filters,batch_size,kernel_size,learning_rate,act_fn,padding=0,stride=1,device="cpu"): +# self.input_size = input_size +# self.num_channels = num_channels +# self.num_filters = num_filters +# self.batch_size = batch_size +# self.kernel_size = kernel_size +# self.padding = padding +# self.stride = stride +# self.output_size = math.floor((self.input_size + (2 * self.padding) - self.kernel_size)/self.stride) +1 +# self.learning_rate = learning_rate +# self.act_fn +# self.device = device +# self.kernel= torch.empty(self.num_filters,self.num_channels,self.kernel_size,self.kernel_size).normal_(mean=0,std=0.05).to(self.device) +# self.unfold = nn.Unfold(kernel_size=(self.kernel_size,self.kernel_size),padding=self.padding,stride=self.stride).to(self.device) +# self.fold = nn.Fold(output_size=(self.input_size,self.input_size),kernel_size=(self.kernel_size,self.kernel_size),padding=self.padding,stride=self.stride).to(self.device) +# +# def forward(self,inp): +# self.X_col = self.unfold(inp.clone()) +# self.flat_weights = self.kernel.reshape(self.num_filters,-1) +# out = self.flat_weights @ self.X_col +# self.activations = out.reshape(self.batch_size, self.num_filters, self.output_size, self.output_size) +# return self.f(self.activations) +# +# def update_gradient(self,e): +# fn_deriv = self.act_fn.deriv(self.activations) +# e = e * fn_deriv +# self.dout = e.reshape(self.batch_size,self.num_filters,-1) +# dW = self.dout @ self.X_col.permute(0,2,1) +# dW = torch.sum(dW,dim=0) +# dW = dW.reshape((self.num_filters,self.num_channels,self.kernel_size,self.kernel_size)) +# self.grad["weights"] = dW +# +# def backward(self,e): +# fn_deriv = self.act_fn.deriv(self.activations) +# e = e * fn_deriv +# self.dout = e.reshape(self.batch_size,self.num_filters,-1) +# dX_col = self.flat_weights.T @ self.dout +# dX = self.fold(dX_col) +# return torch.clamp(dX,-50,50) +# +# def get_true_weight_grad(self): +# return self.kernel.grad +# +# def set_weight_parameters(self): +# self.kernel = nn.Parameter(self.kernel) +# +# def save_layer(self,logdir,i): +# np.save(logdir +"/layer_"+str(i)+"_weights.npy",self.kernel.detach().cpu().numpy()) +# +# def load_layer(self,logdir,i): +# kernel = np.load(logdir +"/layer_"+str(i)+"_weights.npy") +# self.kernel = set_tensor(torch.from_numpy(kernel)) +# +# class MaxPool(Layer): +# def __init__(self, kernel_size,device='cpu'): +# self.kernel_size = kernel_size +# self.device = device +# self.activations = torch.empty(1) +# self.weights = torch.zeros((1,1)).to(self.device) +# +# def forward(self,x): +# out, self.idxs = F.max_pool2d(x, self.kernel_size,return_indices=True) +# return out +# +# def backward(self, y): +# return F.max_unpool2d(y,self.idxs, self.kernel_size) +# +# def update_gradient(self,e,update_weights=False,sign_reverse=False): +# self.grad["weights"] = torch.zeros_like(self.weights) +# +# def get_true_weight_grad(self): +# return None +# +# def set_weight_parameters(self): +# pass +# +# def save_layer(self,logdir,i): +# pass +# +# def load_layer(self,logdir,i): +# pass +# +# class AvgPool(Layer): +# def __init__(self, kernel_size,device='cpu'): +# self.kernel_size = kernel_size +# self.device = device +# self.activations = torch.empty(1) +# self.weights = torch.zeros((1,1)).to(self.device) +# +# +# def forward(self, x): +# self.B_in,self.C_in,self.H_in,self.W_in = x.shape +# return F.avg_pool2d(x,self.kernel_size) +# +# def backward(self, y): +# N,C,H,W = y.shape +# print("in backward: ", y.shape) +# return F.interpolate(y,scale_factor=(1,1,self.kernel_size,self.kernel_size)) +# +# def update_gradient(self,e,update_weights=False, sign_reverse=False): +# self.grad["weights"] = torch.zeros_like(self.weights) +# +# def save_layer(self,logdir,i): +# pass +# +# def load_layer(self,logdir,i): +# pass +# +# class ProjectionLayer(Layer): +# def __init__(self,input_size, output_size,act_fn,learning_rate,device='cpu'): +# self.input_size = input_size +# self.B, self.C, self.H, self.W = self.input_size +# self.output_size =output_size +# self.learning_rate = learning_rate +# self.act_fn = act_fn +# self.device = device +# self.Hid = self.C * self.H * self.W +# self.weights = torch.empty((self.Hid, self.output_size)).normal_(mean=0.0, std=0.05).to(self.device) +# +# def forward(self, x): +# self.inp = x.detach().clone() +# out = x.reshape((len(x), -1)) +# self.activations = torch.matmul(out,self.weights) +# return self.f(self.activations) +# +# def backward(self, e): +# fn_deriv = self.act_fn.deriv(self.activations) +# out = torch.matmul(e * fn_deriv, self.weights.T) +# out = out.reshape((len(e), self.C, self.H, self.W)) +# return torch.clamp(out,-50,50) +# +# def update_gradient(self, e,update_weights=False,sign_reverse=False): +# out = self.inp.reshape((len(self.inp), -1)) +# fn_deriv = self.act_fn.deriv(self.activations) +# dw = torch.matmul(out.T, e * fn_deriv) +# self.grad["weights"] = dw +# +# def get_true_weight_grad(self): +# return self.weights.grad +# +# def set_weight_parameters(self): +# self.weights = nn.Parameter(self.weights) +# +# def save_layer(self,logdir,i): +# np.save(logdir +"/layer_"+str(i)+"_weights.npy",self.weights.detach().cpu().numpy()) +# +# def load_layer(self,logdir,i): +# weights = np.load(logdir +"/layer_"+str(i)+"_weights.npy") +# self.weights = set_tensor(torch.from_numpy(weights)) diff --git a/pypc/models.py b/pypc/models.py index 541b995..09dc729 100644 --- a/pypc/models.py +++ b/pypc/models.py @@ -1,12 +1,29 @@ -import numpy as np import torch +import numpy as np from pypc import utils from pypc.layers import FCLayer class PCModel(object): - def __init__(self, nodes, mu_dt, act_fn, use_bias=False, kaiming_init=False): + def __init__(self, nodes, mu_dt, act_fn, use_bias=False, kaiming_init=False, use_precis=False, precis_factor=None, precis_coverage=None, precis_per_pixel=None, run_log=None, log_node_its=False): + """ + Define the Predictive Coding PyTorch model. All layers fully connected. All nodes using the specified activation + function except for the output layer which is linear. Bias terms are optional. Kaiming weight initialisation is + optional. Precisions are optional. + + :param nodes: List of number of nodes in each layer + :param mu_dt: Timestep for updating means + :param act_fn: Activation function + :param use_bias: Include bias terms? + :param kaiming_init: Use Kaiming weight initialisation? + :param use_precis: Use precisions? (If False, precisions are implied to be identity matrices) + :param precis_factor: List of precision scaling per node layer + :param precis_coverage: List of precision coverage by node layer or None for full coverage + :param run_log: A neptune.ai run object or None if logging not required + :param log_node_its: Log stats after every node update iteration? (WARNING: Slow!) + """ + self.nodes = nodes self.mu_dt = mu_dt @@ -26,21 +43,70 @@ def __init__(self, nodes, mu_dt, act_fn, use_bias=False, kaiming_init=False): ) self.layers.append(layer) + self.use_precis = use_precis + self.precis_coverage = precis_coverage + # If precisions used, create: + # List of diagonal precision matrices with given scale factor + # List of diagonal variance matrices (inverse of precision matrices) (easy for diagonal precision) + # List of variance determinants (for free energy sum) + # NOTE: Precisions are fixed, so can be defined here. Not suitable for dynamic (learned) precisions + if self.use_precis: + self.precis = [[] for _ in range(self.n_nodes)] + # self.varis = [[] for _ in range(self.n_nodes)] + # self.vari_dets = [1.0 for _ in range(self.n_nodes)] + for n in range(self.n_nodes): + if (n == self.n_nodes - 1) and precis_per_pixel is not None: + temp = precis_per_pixel.flatten() + else: + temp = precis_factor[n] * torch.ones(nodes[n]) + if self.precis_coverage is not None: + coverage = max(0.0, min(1.0, self.precis_coverage[n])) # Clamp to range [0, 1] + first_scaled_precision = int(self.nodes[n] * (1.0 - coverage)) + temp[0:first_scaled_precision] = torch.ones(first_scaled_precision) + self.precis[n] = utils.set_tensor(torch.diagflat(temp)) + # TODO: Need to rework below to account for precision coverage + # self.varis[n] = utils.set_tensor(torch.diagflat((1.0/precis_factor[n])*torch.ones(nodes[n]))) + # self.vari_dets[n] = (1.0/precis_factor[n])**nodes[n] + # self.loggy = np.log(2.0*np.pi*self.vari_dets[n]) + + self.run_log = run_log + self.log_node_its = log_node_its + def reset(self): + """ + Initialise predictions (preds), errors (errs), and variational means (mus) to empty lists + """ self.preds = [[] for _ in range(self.n_nodes)] self.errs = [[] for _ in range(self.n_nodes)] self.mus = [[] for _ in range(self.n_nodes)] + self.free_energy = [0.0 for _ in range(self.n_nodes)] def reset_mus(self, batch_size, init_std): + """ + For each layer, initialise variational means (mus) to be normally distributed with mean=0 + + :param batch_size: Number of samples in current batch + :param init_std: Initial standard deviation of mus + """ for l in range(self.n_layers): self.mus[l] = utils.set_tensor( torch.empty(batch_size, self.layers[l].in_size).normal_(mean=0, std=init_std) ) def set_input(self, inp): + """ + Set input node mus for batch to input values + + :param inp: Input values, Tensor:(batch_size, nodes[0]) + """ self.mus[0] = inp.clone() def set_target(self, target): + """ + Set output node mus for batch to target values + + :param target: Target values, Tensor:(batch_size, nodes[-1]) + """ self.mus[-1] = target.clone() def forward(self, val): @@ -49,76 +115,195 @@ def forward(self, val): return val def propagate_mu(self): + """ + Perform forward pass for batch, update mus for all nodes except inputs and outputs (targets) + """ for l in range(1, self.n_layers): self.mus[l] = self.layers[l - 1].forward(self.mus[l - 1]) def train_batch_supervised(self, img_batch, label_batch, n_iters, fixed_preds=False): - self.reset() - self.set_input(img_batch) - self.propagate_mu() - self.set_target(label_batch) - self.train_updates(n_iters, fixed_preds=fixed_preds) - self.update_grads() - - def train_batch_generative(self, img_batch, label_batch, n_iters, fixed_preds=False): - self.reset() - self.set_input(label_batch) - self.propagate_mu() - self.set_target(img_batch) - self.train_updates(n_iters, fixed_preds=fixed_preds) - self.update_grads() + """ + Train the model using the (mini)batch images as inputs and labels as targets + + :param img_batch: Batch of input images, Tensor:(batch_size, nodes[0]) + :param label_batch: Batch of target labels, Tensor:(batch_size, nodes[-1]) + :param n_iters: Number of training iterations + :param fixed_preds: Fix predictions at initial values? + """ + self.reset() # Initialise the prediction, error, and mu data structures + self.set_input(img_batch) # Set the model inputs, mus[0], equal to the training *images* + self.propagate_mu() # Perform forward pass, update mus for all nodes except inputs and targets + self.set_target(label_batch) # Set the model outputs (targets), mus[-1], equal to the training *labels* + self.train_updates(n_iters, fixed_preds=fixed_preds) # Iteratively update mus, predictions and errors + self.update_grads() # Calculate gradients of weights and biases for all layers + + # def set_precisions_by_per_pixel_variance(self, img_batch): + # per_pixel_variance = img_batch.var(dim=0) + # # per_pixel_precision = 1.0 / (per_pixel_variance + 1e-1) + # per_pixel_precision = 1.0 / per_pixel_variance.clamp(min=0.1) + # self.precis[3] = torch.diagflat(per_pixel_precision) + + def train_batch_generative(self, img_batch, label_batch, n_iters, fixed_preds=False, log_batch=False): + """ + Train the model using the (mini)batch labels as inputs and images as targets + (Identical to train_batch_supervised() but inputs and targets are swapped) + + :param img_batch: Batch of target images, Tensor:(batch_size, nodes[-1]) + :param label_batch: Batch of input labels, Tensor:(batch_size, nodes[0]) + :param n_iters: Number of training iterations + :param fixed_preds: Fix predictions at initial values? + """ + self.reset() # Initialise the prediction, error, and mu data structures + self.set_input(label_batch) # Set the model inputs, mus[0], equal to the training *labels* + self.propagate_mu() # Perform forward pass, update mus for all nodes except inputs and targets + self.set_target(img_batch) # Set the model outputs (targets), mus[-1], equal to the training *images* + self.train_updates(n_iters, fixed_preds=fixed_preds, log_batch=log_batch) # Iteratively update mus, predictions and errors + self.update_grads() # Calculate gradients of weights and biases for all layers + if self.run_log and log_batch: + self.log_layer_stats(prefix="train") + + def log_layer_stats(self, prefix="layer"): + for l in range(self.n_layers): + self.run_log[f"{prefix}/grad_weight_mean_{l}]"].log(torch.mean(self.layers[l].grad["weights"])) + self.run_log[f"{prefix}/grad_bias_mean_{l}]"].log(torch.mean(self.layers[l].grad["bias"])) + self.run_log[f"{prefix}/grad_weight_std_{l}]"].log(torch.std(self.layers[l].grad["weights"])) + self.run_log[f"{prefix}/grad_bias_std_{l}]"].log(torch.std(self.layers[l].grad["bias"])) + self.run_log[f"{prefix}/weight_mean_{l}]"].log(torch.mean(self.layers[l].weights)) + self.run_log[f"{prefix}/bias_mean_{l}]"].log(torch.mean(self.layers[l].bias)) + self.run_log[f"{prefix}/weight_std_{l}]"].log(torch.std(self.layers[l].weights)) + self.run_log[f"{prefix}/bias_std_{l}]"].log(torch.std(self.layers[l].bias)) + + def log_node_stats(self, prefix="node"): + for n in range(1, self.n_nodes): + self.run_log[f"{prefix}/err_mean_{n}]"].log(torch.mean(self.errs[n])) + self.run_log[f"{prefix}/pred_mean_{n}"].log(torch.mean(self.preds[n])) + self.run_log[f"{prefix}/mu_mean_{n}"].log(torch.mean(self.mus[n])) + self.run_log[f"{prefix}/err_std_{n}]"].log(torch.std(self.errs[n])) + self.run_log[f"{prefix}/pred_std_{n}"].log(torch.std(self.preds[n])) + self.run_log[f"{prefix}/mu_std_{n}"].log(torch.std(self.mus[n])) + self.run_log[f"{prefix}/free_e_{n}"].log(self.free_energy[n]) + self.run_log[f"{prefix}/free_e"].log(sum(self.free_energy)) def test_batch_supervised(self, img_batch): return self.forward(img_batch) - def test_batch_generative(self, img_batch, n_iters, init_std=0.05, fixed_preds=False): + def test_batch_generative(self, img_batch, n_iters, init_std=0.05, fixed_preds=False, log_batch=False): batch_size = img_batch.size(0) - self.reset() - self.reset_mus(batch_size, init_std) - self.set_target(img_batch) - self.test_updates(n_iters, fixed_preds=fixed_preds) + self.reset() # Initialise the prediction, error, and mu data structures + self.reset_mus(batch_size, init_std) # Initialise variational means (mus) + self.set_target(img_batch) # Set output node mus for batch to target values (test images) + self.test_updates(n_iters, fixed_preds=fixed_preds, log_batch=log_batch) + # self.update_grads() # Calculate gradients of weights and biases for all layers - NOT REQUIRED return self.mus[0] - def train_updates(self, n_iters, fixed_preds=False): + def train_updates(self, n_iters, fixed_preds=False, log_batch=False): + """ + Iteratively update mus, predictions and errors + + :param n_iters: Number of training iterations + :param fixed_preds: Fix predictions at initial values? + """ + # For batch, initialise predictions and errors for all nodes except inputs + # Optionally, errors are precision scaled and free energy is calculated for n in range(1, self.n_nodes): self.preds[n] = self.layers[n - 1].forward(self.mus[n - 1]) self.errs[n] = self.mus[n] - self.preds[n] + if self.use_precis: + self.free_energy[n] = torch.mean(self.errs[n] @ self.precis[n] @ self.errs[n].T).item() + self.errs[n] = torch.matmul(self.errs[n], self.precis[n]) + # Log values to neptune + if self.run_log and log_batch and self.log_node_its: + self.log_node_stats(prefix="train") + + # For each training iteration for itr in range(n_iters): + # For batch, update mus for all nodes except inputs (labels) and outputs (images) for l in range(1, self.n_layers): delta = self.layers[l].backward(self.errs[l + 1]) - self.errs[l] self.mus[l] = self.mus[l] + self.mu_dt * delta + # For batch, update errors and (optionally) predictions for all nodes except inputs + # Optionally, errors are precision scaled and free energy is calculated for n in range(1, self.n_nodes): if not fixed_preds: self.preds[n] = self.layers[n - 1].forward(self.mus[n - 1]) self.errs[n] = self.mus[n] - self.preds[n] + if self.use_precis: + self.free_energy[n] = torch.mean(self.errs[n] @ self.precis[n] @ self.errs[n].T).item() + self.errs[n] = torch.matmul(self.errs[n], self.precis[n]) + + # Log values to neptune + if self.run_log and log_batch and (self.log_node_its or (itr == n_iters-1)): + self.log_node_stats(prefix="train") + + def test_updates(self, n_iters, fixed_preds=False, log_batch=False): + """ + Test model - def test_updates(self, n_iters, fixed_preds): + :param n_iters: Number of training iterations + :param fixed_preds: Fix predictions at initial values? + """ + # For batch, initialise predictions and errors for all nodes except inputs + # Optionally, errors are precision scaled and free energy is calculated for n in range(1, self.n_nodes): self.preds[n] = self.layers[n - 1].forward(self.mus[n - 1]) self.errs[n] = self.mus[n] - self.preds[n] + if self.use_precis: + self.free_energy[n] = torch.mean(self.errs[n] @ self.precis[n] @ self.errs[n].T).item() + self.errs[n] = torch.matmul(self.errs[n], self.precis[n]) + # Log values to neptune + if self.run_log and log_batch and self.log_node_its: + self.log_node_stats(prefix="test") + + # For each test iteration for itr in range(n_iters): + # For batch, update mus for all nodes except outputs (images) + # NOTE: Unlike training which also does not update inputs (labels) delta = self.layers[0].backward(self.errs[1]) self.mus[0] = self.mus[0] + self.mu_dt * delta for l in range(1, self.n_layers): delta = self.layers[l].backward(self.errs[l + 1]) - self.errs[l] self.mus[l] = self.mus[l] + self.mu_dt * delta + # For batch, update errors and (optionally) predictions for all nodes except inputs + # Optionally, errors are precision scaled and free energy is calculated for n in range(1, self.n_nodes): if not fixed_preds: self.preds[n] = self.layers[n - 1].forward(self.mus[n - 1]) self.errs[n] = self.mus[n] - self.preds[n] + if self.use_precis: + self.free_energy[n] = torch.mean(self.errs[n] @ self.precis[n] @ self.errs[n].T).item() + self.errs[n] = torch.matmul(self.errs[n], self.precis[n]) + + # Log values to neptune + if self.run_log and log_batch and (self.log_node_its or (itr == n_iters-1)): + self.log_node_stats(prefix="test") def update_grads(self): + """ + Calculate gradients of weights and biases for all layers + + """ for l in range(self.n_layers): self.layers[l].update_gradient(self.errs[l + 1]) def get_target_loss(self): + """ + Calculate loss as the sum of the squares of the target errors + (Not currently used) + + :return: Loss + """ return torch.sum(self.errs[-1] ** 2).item() @property def params(self): - return self.layers + """ + Allows controlled and standardised access to model parameters (for passing to an optimizer). + Currently of limited use but could be expanded. + :return: Model layers + """ + return self.layers diff --git a/pypc/optim.py b/pypc/optim.py index 650e461..8756857 100644 --- a/pypc/optim.py +++ b/pypc/optim.py @@ -3,11 +3,11 @@ def get_optim(params, optim_id, lr, q_lr=None, batch_scale=True, grad_clip=None, weight_decay=None): - if optim_id is "Adam": + if optim_id == "Adam": return Adam( params, lr=lr, q_lr=q_lr, batch_scale=batch_scale, grad_clip=grad_clip, weight_decay=weight_decay ) - elif optim_id is "SGD": + elif optim_id == "SGD": return SGD( params, lr=lr, q_lr=q_lr, batch_scale=batch_scale, grad_clip=grad_clip, weight_decay=weight_decay ) diff --git a/pypc/utils.py b/pypc/utils.py index 7ab9d56..81fd136 100644 --- a/pypc/utils.py +++ b/pypc/utils.py @@ -3,7 +3,10 @@ import numpy as np import torch -DEVICE = torch.device("cuda" if torch.cuda.is_available() else "cpu") +DEVICE = torch.device( + "cuda" if torch.cuda.is_available() + else "mps" if torch.backends.mps.is_available() + else "cpu") class AttrDict(dict): @@ -49,6 +52,11 @@ def deriv(self, inp): def seed(seed): + """ + Set seeds for relevant pseudorandom number generators + + :param seed: Seed value + """ torch.manual_seed(seed) if torch.cuda.is_available(): torch.cuda.manual_seed_all(seed) @@ -57,6 +65,12 @@ def seed(seed): def set_tensor(tensor): + """ + Move data onto the selected cpu/cuda device with dtype=torch.float32 + + :param tensor: Tensor object + :return: Tensor object allocated to selected device with dtype=torch.float32 + """ return tensor.to(DEVICE).float() @@ -65,6 +79,12 @@ def flatten_array(array): def save_json(obj, path): + """ + Save an object as a json file + + :param obj: Object to be saved + :param path: File path + """ with open(path, "w") as file: json.dump(obj, file) diff --git a/scripts/generative.py b/scripts/generative.py index 1d7e4c5..59f4547 100644 --- a/scripts/generative.py +++ b/scripts/generative.py @@ -1,30 +1,134 @@ import os import pprint +import sys import torch +from tqdm import tqdm + from pypc import utils from pypc import datasets from pypc import optim from pypc.models import PCModel +import neptune +from pypc import constants # Defines secret API_KEY_NEPTUNE for Neptune access + +from datetime import datetime + +import numpy as np +import matplotlib.pyplot as plt +import pandas as pd + +def plot_example_data_MNIST(dataloader, batch_num=0, path=None, cmap="gray"): + if batch_num >= len(dataloader): + batch_num = -1 + image_batch = dataloader[batch_num][0] + label_batch = dataloader[batch_num][1] + images = image_batch.cpu().detach().numpy() + labels = label_batch.cpu().detach().numpy() + _, indices = np.unique(labels, axis=0, return_index=True) + indices = np.flip(indices) + # HACK to plot first ten images (not first examples of digits 0 to 9) + # indices = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + + fig, axes = plt.subplots(2, 5) + fig.set_size_inches(8, 3) + fig.set_dpi(150) + axes = axes.flatten() + plt.setp(axes, xticks=[0, 27]) + plt.setp(axes, yticks=[0, 27]) + for i in range(10): + axes[i].tick_params(top=False, labeltop=False, bottom=False, labelbottom=False, width=2) + axes[i].tick_params(left=False, labelleft=False, right=False, labelright=False, width=2) + axes[i].imshow(images[indices[i]].reshape(28, 28), cmap=cmap) + axes[0].tick_params(top=True, labeltop=True, bottom=False, labelbottom=False, labelsize=16) + axes[0].tick_params(left=True, labelleft=True, right=False, labelright=False, labelsize=16) + + if path: + plt.savefig(path) + plt.show() + plt.close("all") + +def calc_accuracy(data_loader, model, config, file_prefix="", log_test=False): + acc = 0 + for batch_id, (img_batch, label_batch) in enumerate(tqdm(data_loader, file=sys.stdout)): + # model.set_precisions_by_per_pixel_variance(img_batch) + label_preds = model.test_batch_generative( + img_batch, config.n_test_iters, init_std=config.init_std, fixed_preds=config.fixed_preds_test, log_batch=(log_test and (batch_id == len(data_loader)-1)) + ) + acc += datasets.accuracy(label_preds, label_batch) + datasets.save_csv(label_preds, cf.preddir + "preds_" + f"{file_prefix}b{batch_id:03}.csv") + datasets.save_csv(label_batch, cf.preddir + "actuals_" + f"{file_prefix}b{batch_id:03}.csv") + return acc / len(data_loader) + +def print_batch_stats(img_batch, cf): + first_noisy_pixel = int(img_batch.size(1) * (1.0 - cf.noise_coverage)) + last_noisy_pixel = img_batch.size(1) + print(f"Noise coverage = {cf.noise_coverage:.2f}") + print(f"First noisy pixel = {first_noisy_pixel}") + print(f"Last noisy pixel = {last_noisy_pixel}") + print(f"Full image") + print(f"Batch size = {img_batch.size()}") + print(f"Batch mean = {img_batch.mean():.4f}") + print(f"Batch variance = {img_batch.var():.4f}") + print(f"Batch stddev = {img_batch.std():.4f}") + print(f"Region without added noise") + print(f"Batch size = {img_batch[:, 0:first_noisy_pixel].size()}") + print(f"Batch mean = {img_batch[:, 0:first_noisy_pixel].mean():.4f}") + print(f"Batch variance = {img_batch[:, 0:first_noisy_pixel].var():.4f}") + print(f"Batch stddev = {img_batch[:, 0:first_noisy_pixel].std():.4f}") + print(f"Region with added noise") + print(f"Batch size = {img_batch[:, first_noisy_pixel:last_noisy_pixel].size()}") + print(f"Batch mean = {img_batch[:, first_noisy_pixel:last_noisy_pixel].mean():.4f}") + print(f"Batch variance = {img_batch[:, first_noisy_pixel:last_noisy_pixel].var():.4f}") + print(f"Batch stddev = {img_batch[:, first_noisy_pixel:last_noisy_pixel].std():.4f}") def main(cf): print(f"\nStarting generative experiment {cf.logdir}: --seed {cf.seed} --device {utils.DEVICE}") - pprint.pprint(cf) + print(datetime.now()) os.makedirs(cf.logdir, exist_ok=True) os.makedirs(cf.imgdir, exist_ok=True) + os.makedirs(cf.preddir, exist_ok=True) utils.seed(cf.seed) - utils.save_json({k: str(v) for (k, v) in cf.items()}, cf.logdir + "config.json") - train_dataset = datasets.MNIST(train=True, scale=cf.label_scale, size=cf.train_size, normalize=cf.normalize) - test_dataset = datasets.MNIST(train=False, scale=cf.label_scale, size=cf.test_size, normalize=cf.normalize) + pprint.pprint(cf) + if cf.json_enabled: + utils.save_json({k: str(v) for (k, v) in cf.items()}, cf.logdir + "config.json") + if cf.neptune_enabled: + cf.run_log["parameters"] = cf # Log configuration to Neptune + + # Use per pixel scaling of noise (and precision) + # noise_per_pixel_scaling = torch.rand((28, 28)) * (cf.noise_max_std - cf.noise_min_std) + cf.noise_min_std + # precis_per_pixel = 1 / (1 + noise_per_pixel_scaling ** 2.0) + noise_per_pixel_scaling = None + precis_per_pixel = None + + print("Loading data...") + # train_dataset = datasets.MNIST(train=True, scale=cf.label_scale, size=cf.train_size, normalize=cf.normalize, add_noise=cf.add_noise, noise_mean=cf.noise_mean, noise_std=cf.noise_std, noise_coverage=cf.noise_coverage, noise_per_pixel_scaling=noise_per_pixel_scaling) + # test_dataset = datasets.MNIST(train=False, scale=cf.label_scale, size=cf.test_size, normalize=cf.normalize, add_noise=cf.add_noise, noise_mean=cf.noise_mean, noise_std=cf.noise_std, noise_coverage=cf.noise_coverage, noise_per_pixel_scaling=noise_per_pixel_scaling) + train_dataset = datasets.FashionMNIST(train=True, scale=cf.label_scale, size=cf.train_size, normalize=cf.normalize, add_noise=cf.add_noise, noise_mean=cf.noise_mean, noise_std=cf.noise_std, noise_coverage=cf.noise_coverage, noise_per_pixel_scaling=noise_per_pixel_scaling) + test_dataset = datasets.FashionMNIST(train=False, scale=cf.label_scale, size=cf.test_size, normalize=cf.normalize, add_noise=cf.add_noise, noise_mean=cf.noise_mean, noise_std=cf.noise_std, noise_coverage=cf.noise_coverage, noise_per_pixel_scaling=noise_per_pixel_scaling) train_loader = datasets.get_dataloader(train_dataset, cf.batch_size) test_loader = datasets.get_dataloader(test_dataset, cf.batch_size) - print(f"Loaded data [train batches: {len(train_loader)} test batches: {len(test_loader)}]") + print(f"Loaded data [training batches: {len(train_loader)}, test batches: {len(test_loader)}]") + + plot_example_data_MNIST(train_loader, batch_num=0, path=cf.imgdir + "example_training_data.png", cmap='rainbow') + # HACK to plot test data (see also the function) + # plot_example_data_MNIST(test_loader, batch_num=0, path=cf.imgdir + "example_training_data.png", cmap='rainbow') model = PCModel( - nodes=cf.nodes, mu_dt=cf.mu_dt, act_fn=cf.act_fn, use_bias=cf.use_bias, kaiming_init=cf.kaiming_init + nodes=cf.nodes, + mu_dt=cf.mu_dt, + act_fn=cf.act_fn, + use_bias=cf.use_bias, + kaiming_init=cf.kaiming_init, + use_precis=cf.use_precis, + precis_factor=cf.precis_factor, + precis_coverage=cf.precis_coverage, + precis_per_pixel=precis_per_pixel, + run_log=cf.run_log, + log_node_its=cf.log_node_its, ) optimizer = optim.get_optim( model.params, @@ -35,14 +139,17 @@ def main(cf): weight_decay=cf.weight_decay, ) - with torch.no_grad(): - metrics = {"acc": []} + with torch.no_grad(): # Disable automatic gradient calculation + metrics = {"test_acc": [], "train_acc": []} for epoch in range(1, cf.n_epochs + 1): - print(f"\nTrain @ epoch {epoch} ({len(train_loader)} batches)") - for batch_id, (img_batch, label_batch) in enumerate(train_loader): + + # Train each batch + for batch_id, (img_batch, label_batch) in enumerate(tqdm(train_loader, file=sys.stdout)): + # print_batch_stats(img_batch, cf) + # model.set_precisions_by_per_pixel_variance(img_batch) model.train_batch_generative( - img_batch, label_batch, cf.n_train_iters, fixed_preds=cf.fixed_preds_train + img_batch, label_batch, cf.n_train_iters, fixed_preds=cf.fixed_preds_train, log_batch=(batch_id == len(train_loader)-1) ) optimizer.step( curr_epoch=epoch, @@ -51,63 +158,127 @@ def main(cf): batch_size=img_batch.size(0), ) - if epoch % cf.test_every == 0: - print(f"\nTest @ epoch {epoch}") - acc = 0 - for _, (img_batch, label_batch) in enumerate(test_loader): - label_preds = model.test_batch_generative( - img_batch, cf.n_test_iters, init_std=cf.init_std, fixed_preds=cf.fixed_preds_test - ) - acc += datasets.accuracy(label_preds, label_batch) - metrics["acc"].append(acc / len(test_loader)) - print("Accuracy: {:.4f}".format(acc / len(test_loader))) + # Evaluate the model at specified intervals + if epoch % cf.eval_every == 0: + print(f"\nEvaluate @ epoch {epoch}") + + # Calculate training accuracy + if cf.eval_train_acc: + acc = calc_accuracy(train_loader, model, cf, f"train_e{epoch:03}_", log_test=False) + print(f"Train accuracy: {acc:.4f}") + metrics["train_acc"].append(acc) + if cf.neptune_enabled: + cf.run_log["train/acc"].log(acc) + + # Calculate test accuracy + if cf.eval_test_acc: + acc = calc_accuracy(test_loader, model, cf, f"test_e{epoch:03}_", log_test=True) + print(f"Test accuracy: {acc:.4f}") + metrics["test_acc"].append(acc) + if cf.neptune_enabled: + cf.run_log["test/acc"].log(acc) + + # Generate image predictions + if cf.eval_images: + # _, label_batch = next(iter(test_loader)) # Use first batch + label_batch = utils.set_tensor(torch.diagflat(torch.ones(10))) # Use each of the one hot encoded labels + img_preds = model.forward(label_batch) + datasets.plot_imgs_alt(img_preds, path=cf.imgdir + f"e{epoch:03}.png", cmap="rainbow") + + # preds = pd.read_csv(r'D:\n\OneDrive\Documents\GitHub\pypc-desktop\scripts\data\230727_101349 - copy for analysis\000\preds\preds_test_e015_b000.csv', nrows=10, index_col=0) + # label_batch = utils.set_tensor(torch.tensor(preds.to_numpy())) + # img_preds = model.forward(label_batch) + # datasets.plot_imgs_alt(img_preds, path=cf.imgdir + f"alt e{epoch:03}.png", cmap="rainbow") - _, label_batch = next(iter(test_loader)) - img_preds = model.forward(label_batch) - datasets.plot_imgs(img_preds, cf.imgdir + f"{epoch}.png") + # Save metrics to json log + if cf.json_enabled: + utils.save_json(metrics, cf.logdir + "metrics.json") - utils.save_json(metrics, cf.logdir + "metrics.json") + if cf.neptune_enabled: + cf.run_log.stop() # Stop logging to Neptune.ai +# INSTRUCTIONS +# For a single point calculation, set scan_1d and scan_2d to [0] and ignore s1d and s2d +# For a 1D scan, set scan_1d to a list of parameter values, set scan_2d to [0], use s1d and ignore s2d +# For a 2D scan, set scan_1d and scan_2d to lists of parameter values, use s1d and s2d +# In all cases, use cf.seeds to repeat runs (and, for easier analysis, gather into a Neptune group if used) if __name__ == "__main__": - cf = utils.AttrDict() - cf.seeds = [0] - - for seed in cf.seeds: - - # experiment params - cf.seed = seed - cf.n_epochs = 20 - cf.test_every = 1 - cf.logdir = f"data/generative/{seed}/" - cf.imgdir = cf.logdir + "imgs/" - - # dataset params - cf.train_size = None - cf.test_size = None - cf.label_scale = None - cf.normalize = True - - # optim params - cf.optim = "Adam" - cf.lr = 1e-4 - cf.batch_size = 64 - cf.batch_scale = True - cf.grad_clip = None - cf.weight_decay = None - - # inference params - cf.mu_dt = 0.01 - cf.n_train_iters = 50 - cf.n_test_iters = 200 - cf.init_std = 0.01 - cf.fixed_preds_train = False - cf.fixed_preds_test = False - - # model params - cf.use_bias = True - cf.kaiming_init = False - cf.nodes = [10, 100, 300, 784] - cf.act_fn = utils.Tanh() - - main(cf) + cf = utils.AttrDict() # Create configuration + # scan_1d = [0, 0.002, 0.005, 0.01, 0.02, 0.027, 0.038, 0.059, 0.1, 0.2, 0.5, 0.8, 1.0, 1.2] # List of parameters for 1D parameter scan + scan_1d = [0] # List of parameters for 1D parameter scan + scan_2d = [0] # List of parameters for 2D parameter scan + for s2d in scan_2d: + for s1d in scan_1d: + cf.seeds = [0] # Create list of >=1 seeds for repeat runs + now = datetime.now().strftime('%y%m%d_%H%M%S') + cf.run_group = "" + for seed in cf.seeds: + # logging params + cf.log_node_its = False # WARNING: Can log A LOT of data to Neptune if True + cf.json_enabled = True + cf.neptune_enabled = False + cf.neptune_mode = "async" # https://docs.neptune.ai/api/connection_modes/ + cf.neptune_project = "lasermanick/PYPC" + cf.neptune_api_token = constants.API_KEY_NEPTUNE + if cf.neptune_enabled: + cf.run_log = neptune.init_run(mode=cf.neptune_mode, project=cf.neptune_project, api_token=cf.neptune_api_token) + run_id = cf.run_log["sys/id"].fetch() + if cf.run_group == "": + cf.run_group = "grp_" + run_id + cf.logdir = f"data/generative/{now}_{cf.run_group}/{seed:03}_{run_id}/" + else: + cf.run_log = None + cf.logdir = f"data/generative/{now}/{seed:03}/" + cf.imgdir = cf.logdir + "imgs/" + cf.preddir = cf.logdir + "preds/" + + # experiment params + cf.seed = seed + cf.n_epochs = 20 # 20 + cf.eval_every = 1 + cf.eval_train_acc = False + cf.eval_test_acc = True + cf.eval_images = True + + # model params + cf.use_bias = True + cf.kaiming_init = False + cf.nodes = [10, 144, 169, 784] # [10, 100, 300, 784] + cf.act_fn = utils.Tanh() + cf.use_precis = True + cf.precis_factor = [1.0, 1.0, 1.0, 0.1] # Errors and precisions at layer 0 are not used + cf.precis_coverage = [0.0, 0.0, 0.0, 0.5] # Errors and precisions at layer 0 are not used + + # dataset params + cf.train_size = None # None + cf.test_size = None # None + cf.label_scale = None # None + cf.normalize = True # None + cf.add_noise = True # False + cf.noise_mean = 0.0 + cf.noise_std = 2.0 + cf.noise_coverage = 0.5 + # cf.noise_min_std = 3.0 + # cf.noise_max_std = 3.0 + + # optim params + cf.optim = "Adam" # "Adam" + cf.lr = 0.006 # 0.006 + cf.batch_size = 10000 # 10000 + cf.batch_scale = False + cf.grad_clip = None + cf.weight_decay = 0.01 # None + + # inference params + cf.mu_dt = 0.01 # 0.01 + cf.n_train_iters = 50 # 50 + # cf.n_test_iters = min(round(200 / (s1d + 1e-9)), 4000) # 200 + cf.n_test_iters = 200 + cf.init_std = 0.01 + cf.fixed_preds_train = False + cf.fixed_preds_test = False + + main(cf) +print("Finishing experiment") +print(datetime.now()) diff --git a/scripts/supervised.py b/scripts/supervised.py index 2c33155..8d97227 100644 --- a/scripts/supervised.py +++ b/scripts/supervised.py @@ -2,12 +2,33 @@ import pprint import torch +import torchvision +import matplotlib.pyplot as plt + +from tqdm import tqdm +from time import sleep from pypc import utils from pypc import datasets from pypc import optim from pypc.models import PCModel +import neptune +from pypc import constants # Defines secret API_KEY_NEPTUNE for Neptune access + +from torch.utils.tensorboard import SummaryWriter + +# helper function to show an image +# (used in the `plot_classes_preds` function below) +def matplotlib_imshow(img, one_channel=False): + if one_channel: + img = img.mean(dim=0) + img = img / 2 + 0.5 # unnormalize + npimg = img.cpu().numpy() + if one_channel: + plt.imshow(npimg, cmap="Greys") + else: + plt.imshow(np.transpose(npimg, (1, 2, 0))) def main(cf): print(f"\nStarting supervised experiment {cf.logdir}: --seed {cf.seed} --device {utils.DEVICE}") @@ -16,22 +37,26 @@ def main(cf): utils.seed(cf.seed) utils.save_json({k: str(v) for (k, v) in cf.items()}, cf.logdir + "config.json") - train_dataset = datasets.MNIST( - train=True, scale=cf.label_scale, size=cf.train_size, normalize=cf.normalize - ) - test_dataset = datasets.MNIST( - train=False, scale=cf.label_scale, size=cf.test_size, normalize=cf.normalize - ) + print("Loading data...") + train_dataset = datasets.MNIST(train=True, scale=cf.label_scale, size=cf.train_size, normalize=cf.normalize) + test_dataset = datasets.MNIST(train=False, scale=cf.label_scale, size=cf.test_size, normalize=cf.normalize) train_loader = datasets.get_dataloader(train_dataset, cf.batch_size) test_loader = datasets.get_dataloader(test_dataset, cf.batch_size) print(f"Loaded data [train batches: {len(train_loader)} test batches: {len(test_loader)}]") + run = neptune.init( + project="lasermanick/PYPC", + api_token=constants.API_KEY_NEPTUNE, + ) model = PCModel( nodes=cf.nodes, mu_dt=cf.mu_dt, act_fn=cf.act_fn, use_bias=cf.use_bias, kaiming_init=cf.kaiming_init, + use_precis=cf.use_precis, + precis=cf.precis, + run_log=run, ) optimizer = optim.get_optim( model.params, @@ -42,12 +67,33 @@ def main(cf): weight_decay=cf.weight_decay, ) + params = cf # {"learning_rate": 0.001, "optimizer": "Adam"} + run["parameters"] = params + + + # writer = SummaryWriter(f"{cf.logdir}/tensorboard") + # # get some random training images + # images, labels = train_loader[0] + # images_sq = torch.reshape(images, (6, 1, 28, 28)) + # + # # create grid of images + # img_grid = torchvision.utils.make_grid(images_sq) + # + # # show images + # matplotlib_imshow(img_grid, one_channel=True) + # + # # write to tensorboard + # writer.add_image('six_fashion_mnist_images_b', img_grid) + # + # writer.close() + with torch.no_grad(): metrics = {"acc": []} for epoch in range(1, cf.n_epochs + 1): print(f"\nTrain @ epoch {epoch} ({len(train_loader)} batches)") - for batch_id, (img_batch, label_batch) in enumerate(train_loader): + sleep(0.1) + for batch_id, (img_batch, label_batch) in enumerate(tqdm(train_loader, disable=False)): model.train_batch_supervised( img_batch, label_batch, cf.n_train_iters, fixed_preds=cf.fixed_preds_train ) @@ -59,15 +105,23 @@ def main(cf): ) if epoch % cf.test_every == 0: + print(f"\nTest @ epoch {epoch}") + sleep(0.1) acc = 0 - for _, (img_batch, label_batch) in enumerate(test_loader): + for _, (img_batch, label_batch) in enumerate(tqdm(test_loader, disable=False)): label_preds = model.test_batch_supervised(img_batch) acc += datasets.accuracy(label_preds, label_batch) metrics["acc"].append(acc / len(test_loader)) - print("\nTest @ epoch {} / Accuracy: {:.4f}".format(epoch, acc / len(test_loader))) + print("\nAccuracy: {:.4f}".format(acc / len(test_loader))) utils.save_json(metrics, cf.logdir + "metrics.json") + run["train/acc"].log(acc / len(test_loader)) + + run["eval/f1_score"] = 0.66 # An example only + + run.stop() + if __name__ == "__main__": cf = utils.AttrDict() @@ -90,21 +144,24 @@ def main(cf): # optim params cf.optim = "Adam" - cf.lr = 1e-4 - cf.batch_size = 64 + cf.lr = 5e-3 + cf.batch_size = 6400 cf.batch_scale = False cf.grad_clip = 50 cf.weight_decay = None # inference params cf.mu_dt = 0.01 - cf.n_train_iters = 50 - cf.fixed_preds_train = True + cf.n_train_iters = 200 + cf.fixed_preds_train = False # model params cf.use_bias = True cf.kaiming_init = False cf.nodes = [784, 300, 100, 10] cf.act_fn = utils.ReLU() + cf.use_precis = False + # cf.precis = [1.0, 625.0, 100.0, 500.0] + cf.precis = [1.0, 1.0, 1.0, 1.0] main(cf)