From 26a90f04d79b4623d1a32d19f3d79853870c2479 Mon Sep 17 00:00:00 2001 From: gbatmaz <50459436+gbatmaz@users.noreply.github.com> Date: Mon, 12 Sep 2022 14:51:46 +0100 Subject: [PATCH 01/13] Update autoencoder.py add early stop --- dfencoder/autoencoder.py | 28 ++++++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/dfencoder/autoencoder.py b/dfencoder/autoencoder.py index 75a02bd..38aacbc 100644 --- a/dfencoder/autoencoder.py +++ b/dfencoder/autoencoder.py @@ -658,7 +658,7 @@ def _create_stat_dict(self, a): std = scaler.std return {'scaler': scaler, 'mean': mean, 'std': std} - def fit(self, df, epochs=1, val=None): + def fit(self, df, epochs=1, val=None, patience=5): """Does training.""" pdf = df.copy() # if val is None: @@ -687,6 +687,9 @@ def fit(self, df, epochs=1, val=None): n_updates = len(df) // self.batch_size if len(df) % self.batch_size > 0: n_updates += 1 + last_loss = 5000 + + count_es = 0 for i in range(epochs): self.train() if self.verbose: @@ -724,6 +727,26 @@ def fit(self, df, epochs=1, val=None): num, bin, cat = self.forward(slc_out_tensor) _, _, _, net_loss = self.compute_loss(num, bin, cat, slc_out, _id=True) id_loss.append(net_loss) + + #Earlystopping + current_net_loss = net_loss + print('The Current Net Loss:', current_net_loss) + + if current_net_loss > last_loss: + count_es += 1 + print('Early stop count:', count_es) + + if count_es >= patience: + print('Early stopping!\n') + break + + else: + print('set count for earlystop: 0') + count_es = 0 + + last_loss = current_net_loss + + self.logger.end_epoch() # if self.project_embeddings: @@ -740,6 +763,7 @@ def fit(self, df, epochs=1, val=None): msg += 'net validation loss, unaltered input: \n' msg += f"{round(id_loss, 4)} \n\n\n" print(msg) + #Getting training loss statistics # mse_loss, bce_loss, cce_loss, _ = self.get_anomaly_score(pdf) if pdf_val is None else self.get_anomaly_score(pd.concat([pdf, pdf_val])) @@ -1041,4 +1065,4 @@ def get_results(self, df, return_abs = False): result_cols.append('max_abs_z') result_cols.append('mean_abs_z') return pdf[result_cols] - \ No newline at end of file + From fd83537cec3c724d5898a63ccb9836fa660bf65f Mon Sep 17 00:00:00 2001 From: Michael Demoret Date: Tue, 13 Sep 2022 11:50:40 -0600 Subject: [PATCH 02/13] REL v22.09.00 alpha From 1ce2447b22ea176fd8ea9ae4096834a1d94a077d Mon Sep 17 00:00:00 2001 From: Michael Demoret <42954918+mdemoret-nv@users.noreply.github.com> Date: Fri, 16 Sep 2022 16:30:36 -0600 Subject: [PATCH 03/13] Engineering Improvements (#1) * Reducing the number of cpu -> gpu calls * Adding comparisons * MOre debugging * Fixed final bug * More fixes discovered during testing * Cleaning up the repo. Removing debugging comparisons --- dfencoder/autoencoder.py | 332 ++++++++++++++++++--------------------- dfencoder/scalers.py | 38 ++++- 2 files changed, 184 insertions(+), 186 deletions(-) diff --git a/dfencoder/autoencoder.py b/dfencoder/autoencoder.py index 75a02bd..dc63e14 100644 --- a/dfencoder/autoencoder.py +++ b/dfencoder/autoencoder.py @@ -44,17 +44,18 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -from collections import OrderedDict import gc +from collections import OrderedDict -import pandas as pd import numpy as np +import pandas as pd import torch import tqdm from .dataframe import EncoderDataFrame from .logging import BasicLogger, IpynbLogger, TensorboardXLogger -from .scalers import StandardScaler, NullScaler, GaussRankScaler +from .scalers import GaussRankScaler, NullScaler, StandardScaler + def ohe(input_vector, dim, device="cpu"): """Does one-hot encoding of input vector.""" @@ -69,6 +70,7 @@ def ohe(input_vector, dim, device="cpu"): return y_onehot + def compute_embedding_size(n_categories): """ Applies a standard formula to choose the number of feature embeddings @@ -76,23 +78,16 @@ def compute_embedding_size(n_categories): n_categories is the number of unique categories in a column. """ - val = min(600, round(1.6 * n_categories ** 0.56)) + val = min(600, round(1.6 * n_categories**0.56)) return int(val) + class CompleteLayer(torch.nn.Module): """ Impliments a layer with linear transformation and optional activation and dropout.""" - def __init__( - self, - in_dim, - out_dim, - activation=None, - dropout=None, - *args, - **kwargs - ): + def __init__(self, in_dim, out_dim, activation=None, dropout=None, *args, **kwargs): super(CompleteLayer, self).__init__(*args, **kwargs) self.layers = [] linear = torch.nn.Linear(in_dim, out_dim) @@ -137,42 +132,41 @@ def forward(self, x): x = layer(x) return x + class AutoEncoder(torch.nn.Module): - def __init__( - self, - encoder_layers=None, - decoder_layers=None, - encoder_dropout=None, - decoder_dropout=None, - encoder_activations=None, - decoder_activations=None, - activation='relu', - min_cats=10, - swap_p=.15, - lr=0.01, - batch_size=256, - eval_batch_size=1024, - optimizer='adam', - amsgrad=False, - momentum=0, - betas=(0.9, 0.999), - dampening=0, - weight_decay=0, - lr_decay=None, - nesterov=False, - verbose=False, - device=None, - logger='basic', - logdir='logdir/', - project_embeddings=True, - run=None, - progress_bar=True, - n_megabatches=1, - scaler='standard', - *args, - **kwargs - ): + def __init__(self, + encoder_layers=None, + decoder_layers=None, + encoder_dropout=None, + decoder_dropout=None, + encoder_activations=None, + decoder_activations=None, + activation='relu', + min_cats=10, + swap_p=.15, + lr=0.01, + batch_size=256, + eval_batch_size=1024, + optimizer='adam', + amsgrad=False, + momentum=0, + betas=(0.9, 0.999), + dampening=0, + weight_decay=0, + lr_decay=None, + nesterov=False, + verbose=False, + device=None, + logger='basic', + logdir='logdir/', + project_embeddings=True, + run=None, + progress_bar=True, + n_megabatches=1, + scaler='standard', + *args, + **kwargs): super(AutoEncoder, self).__init__(*args, **kwargs) self.numeric_fts = OrderedDict() self.binary_fts = OrderedDict() @@ -234,12 +228,7 @@ def __init__( self.n_megabatches = n_megabatches def get_scaler(self, name): - scalers = { - 'standard': StandardScaler, - 'gauss_rank': GaussRankScaler, - None: NullScaler, - 'none': NullScaler - } + scalers = {'standard': StandardScaler, 'gauss_rank': GaussRankScaler, None: NullScaler, 'none': NullScaler} return scalers[name] def init_numeric(self, df): @@ -255,16 +244,13 @@ def init_numeric(self, df): for ft in numeric: Scaler = self.get_scaler(scalers.get(ft, 'gauss_rank')) - feature = { - 'mean': df[ft].mean(), - 'std': df[ft].std(), - 'scaler': Scaler() - } + feature = {'mean': df[ft].mean(), 'std': df[ft].std(), 'scaler': Scaler()} feature['scaler'].fit(df[ft][~df[ft].isna()].values) self.numeric_fts[ft] = feature self.num_names = list(self.numeric_fts.keys()) - def create_numerical_col_max(self,num_names, mse_loss): + + def create_numerical_col_max(self, num_names, mse_loss): if num_names: num_df = pd.DataFrame(num_names) num_df.columns = ['num_col_max_loss'] @@ -277,8 +263,7 @@ def create_numerical_col_max(self,num_names, mse_loss): num_df = pd.DataFrame() return num_df - - def create_binary_col_max(self,bin_names, bce_loss): + def create_binary_col_max(self, bin_names, bce_loss): if bin_names: bool_df = pd.DataFrame(bin_names) bool_df.columns = ['bin_col_max_loss'] @@ -291,8 +276,7 @@ def create_binary_col_max(self,bin_names, bce_loss): bool_df = pd.DataFrame() return bool_df - - def create_categorical_col_max(self,cat_names, cce_loss): + def create_categorical_col_max(self, cat_names, cce_loss): final_list = [] if cat_names: for index, val in enumerate(cce_loss): @@ -304,16 +288,15 @@ def create_categorical_col_max(self,cat_names, cce_loss): else: cat_df = pd.DataFrame() return cat_df - - def get_variable_importance(self, num_names, cat_names, bin_names, mse_loss, bce_loss, cce_loss, - cloudtrail_df): + + def get_variable_importance(self, num_names, cat_names, bin_names, mse_loss, bce_loss, cce_loss, cloudtrail_df): # Get data in the right format num_df = self.create_numerical_col_max(num_names, mse_loss) bool_df = self.create_binary_col_max(bin_names, bce_loss) cat_df = self.create_categorical_col_max(cat_names, cce_loss) variable_importance_df = pd.concat([num_df, bool_df, cat_df], axis=1) return variable_importance_df - + def return_feature_names(self): bin_names = list(self.binary_fts.keys()) num_names = list(self.numeric_fts.keys()) @@ -413,13 +396,11 @@ def build_optimizer(self): lr = self.lr params = self.parameters() if self.optimizer == 'adam': - return torch.optim.Adam( - params, - lr=self.lr, - amsgrad=self.amsgrad, - weight_decay=self.weight_decay, - betas=self.betas - ) + return torch.optim.Adam(params, + lr=self.lr, + amsgrad=self.amsgrad, + weight_decay=self.weight_decay, + betas=self.betas) elif self.optimizer == 'sgd': return torch.optim.SGD( params, @@ -428,7 +409,6 @@ def build_optimizer(self): nesterov=self.nesterov, dampening=self.dampening, weight_decay=self.weight_decay, - ) def build_model(self, df): @@ -468,24 +448,14 @@ def build_model(self, df): for i, dim in enumerate(self.encoder_layers): activation = self.encoder_activations[i] - layer = CompleteLayer( - input_dim, - dim, - activation=activation, - dropout=self.encoder_dropout[i] - ) + layer = CompleteLayer(input_dim, dim, activation=activation, dropout=self.encoder_dropout[i]) input_dim = dim self.encoder.append(layer) self.add_module(f'encoder_{i}', layer) for i, dim in enumerate(self.decoder_layers): activation = self.decoder_activations[i] - layer = CompleteLayer( - input_dim, - dim, - activation=activation, - dropout=self.decoder_dropout[i] - ) + layer = CompleteLayer(input_dim, dim, activation=activation, dropout=self.decoder_dropout[i]) input_dim = dim self.decoder.append(layer) self.add_module(f'decoder_{i}', layer) @@ -586,7 +556,7 @@ def compute_loss(self, num, bin, cat, target_df, logging=True, _id=False): net_loss += list(mse_loss.mean(dim=0).cpu().detach().numpy()) mse_loss = mse_loss.mean() bce_loss = self.bce(bin, bin_target) - + net_loss += list(bce_loss.mean(dim=0).cpu().detach().numpy()) bce_loss = bce_loss.mean() cce_loss = [] @@ -640,13 +610,7 @@ def compute_baseline_performance(self, in_, out_): dim = len(feature['cats']) + 1 pred = ohe(cd, dim, device=self.device) * 5 codes_pred.append(pred) - mse_loss, bce_loss, cce_loss, net_loss = self.compute_loss( - num_pred, - bin_pred, - codes_pred, - out_, - logging=False - ) + mse_loss, bce_loss, cce_loss, net_loss = self.compute_loss(num_pred, bin_pred, codes_pred, out_, logging=False) if isinstance(self.logger, BasicLogger): self.logger.baseline_loss = net_loss return net_loss @@ -713,7 +677,7 @@ def fit(self, df, epochs=1, val=None): slc_in = val_in.iloc[start:stop] slc_in_tensor = self.build_input_tensor(slc_in) - + slc_out = val_df.iloc[start:stop] slc_out_tensor = self.build_input_tensor(slc_out) @@ -745,15 +709,15 @@ def fit(self, df, epochs=1, val=None): # mse_loss, bce_loss, cce_loss, _ = self.get_anomaly_score(pdf) if pdf_val is None else self.get_anomaly_score(pd.concat([pdf, pdf_val])) mse_loss, bce_loss, cce_loss, _ = self.get_anomaly_score_with_losses(pdf) for i, ft in enumerate(self.numeric_fts): - i_loss = mse_loss[:,i].cpu().numpy() + i_loss = mse_loss[:, i] self.feature_loss_stats[ft] = self._create_stat_dict(i_loss) for i, ft in enumerate(self.binary_fts): - i_loss = bce_loss[:,i].cpu().numpy() + i_loss = bce_loss[:, i] self.feature_loss_stats[ft] = self._create_stat_dict(i_loss) for i, ft in enumerate(self.categorical_fts): - i_loss = cce_loss[i].cpu().numpy() + i_loss = cce_loss[:, i] self.feature_loss_stats[ft] = self._create_stat_dict(i_loss) - + def train_epoch(self, n_updates, input_df, df, pbar=None): """Run regular epoch.""" @@ -771,10 +735,7 @@ def train_epoch(self, n_updates, input_df, df, pbar=None): in_sample_tensor = self.build_input_tensor(in_sample) target_sample = df.iloc[start:stop] num, bin, cat = self.forward(in_sample_tensor) - mse, bce, cce, net_loss = self.compute_loss( - num, bin, cat, target_sample, - logging=True - ) + mse, bce, cce, net_loss = self.compute_loss(num, bin, cat, target_sample, logging=True) self.do_backward(mse, bce, cce) self.optim.step() self.optim.zero_grad() @@ -888,60 +849,18 @@ def get_deep_stack_features(self, df): result = torch.cat(result, dim=0) return result - def get_anomaly_score_with_losses(self, df): - self.eval() - data = self.prepare_df(df) - input = self.build_input_tensor(data) - - num_target, bin_target, codes = self.compute_targets(data) - - with torch.no_grad(): - num, bin, cat = self.forward(input) - - mse_loss = self.mse(num, num_target) - net_loss = [mse_loss.data] - bce_loss = self.bce(bin, bin_target) - net_loss += [bce_loss.data] - cce_loss = [] - for i, ft in enumerate(self.categorical_fts): - loss = self.cce(cat[i], codes[i]) - cce_loss.append(loss) - net_loss += [loss.data.reshape(-1, 1)] - - net_loss = torch.cat(net_loss, dim=1).mean(dim=1) - return mse_loss, bce_loss,cce_loss,net_loss.cpu().numpy() - def get_anomaly_score(self, df): """ Returns a per-row loss of the input dataframe. Does not corrupt inputs. """ - return self.get_anomaly_score_with_losses(df)[3] - - def get_scaled_anomaly_scores(self, df): - self.eval() - data = self.prepare_df(df) - input = self.build_input_tensor(data) + mse, bce, cce = self.get_anomaly_score_losses(df) - num_target, bin_target, codes = self.compute_targets(data) - with torch.no_grad(): - num, bin, cat = self.forward(input) + combined_loss = torch.cat([mse, bce, cce], dim=1) + net_loss = combined_loss.mean(dim=1).cpu().numpy() - mse_loss = self.mse(num, num_target) - mse_scaled = torch.zeros(mse_loss.shape) - for i, ft in enumerate(self.numeric_fts): - mse_scaled[:,i] = torch.tensor(self.feature_loss_stats[ft]['scaler'].transform(mse_loss[:,i].cpu().numpy())) - bce_loss = self.bce(bin, bin_target) - bce_scaled = torch.zeros(bce_loss.shape) - for i, ft in enumerate(self.binary_fts): - bce_scaled[:,i] = torch.tensor(self.feature_loss_stats[ft]['scaler'].transform(mse_loss[:,i].cpu().numpy())) - cce_scaled = [] - for i, ft in enumerate(self.categorical_fts): - loss = torch.tensor(self.feature_loss_stats[ft]['scaler'].transform(self.cce(cat[i], codes[i]).cpu().numpy())) - cce_scaled.append(loss) - - return mse_scaled, bce_scaled, cce_scaled + return net_loss def decode_to_df(self, x, df=None): """ @@ -972,10 +891,7 @@ def decode_to_df(self, x, df=None): bin_df = bin_df.apply(lambda x: round(x)).astype(bool) for ft in bin_df.columns: feature = self.binary_fts[ft] - map = { - False: feature['cats'][0], - True: feature['cats'][1] - } + map = {False: feature['cats'][0], True: feature['cats'][1]} bin_df[ft] = bin_df[ft].apply(lambda x: map[x]) cat_df = pd.DataFrame(index=df.index) @@ -1009,36 +925,96 @@ def df_predict(self, df): return output_df - def get_results(self, df, return_abs = False): - pdf = df.copy() - orig_cols = pdf.columns + def get_anomaly_score_with_losses(self, df): + + mse, bce, cce = self.get_anomaly_score_losses(df) + + net = self.get_anomaly_score(df) + + return mse, bce, cce, net + + def get_anomaly_score_losses(self, df): self.eval() data = self.prepare_df(df) + input = self.build_input_tensor(data) + + num_target, bin_target, codes = self.compute_targets(data) + + with torch.no_grad(): + num, bin, cat = self.forward(input) + + mse_loss: torch.Tensor = self.mse(num, num_target) + bce_loss: torch.Tensor = self.bce(bin, bin_target) + cce_loss = [] + for i, ft in enumerate(self.categorical_fts): + loss = self.cce(cat[i], codes[i]) + # Convert to 2 dimensions + cce_loss.append(loss.data.reshape(-1, 1)) + + # Join all categories into a single tensor + cce_loss = torch.cat(cce_loss, dim=1) + + return mse_loss, bce_loss, cce_loss + + def scale_losses(self, mse, bce, cce): + + # Create outputs + mse_scaled = torch.zeros_like(mse) + bce_scaled = torch.zeros_like(bce) + cce_scaled = torch.zeros_like(cce) + + for i, ft in enumerate(self.numeric_fts): + mse_scaled[:, i] = self.feature_loss_stats[ft]['scaler'].transform(mse[:, i]) + + for i, ft in enumerate(self.binary_fts): + bce_scaled[:, i] = self.feature_loss_stats[ft]['scaler'].transform(bce[:, i]) + + for i, ft in enumerate(self.categorical_fts): + cce_scaled[:, i] = self.feature_loss_stats[ft]['scaler'].transform(cce[:, i]) + + return mse_scaled, bce_scaled, cce_scaled + + def get_results(self, df, return_abs=False): + pdf = pd.DataFrame() + self.eval() + + data = self.prepare_df(df) + with torch.no_grad(): num, bin, embeddings = self.encode_input(data) x = torch.cat(num + bin + embeddings, dim=1) x = self.encode(x) - output_df = self.decode_to_df(x, df=df) - mse, bce, cce, _ = self.get_anomaly_score_with_losses(df) - mse_scaled, bce_scaled, cce_scaled = self.get_scaled_anomaly_scores(df) + output_df = self.decode_to_df(x) + + mse, bce, cce = self.get_anomaly_score_losses(df) + mse_scaled, bce_scaled, cce_scaled = self.scale_losses(mse, bce, cce) + + if (return_abs): + mse_scaled = abs(mse_scaled) + bce_scaled = abs(bce_scaled) + cce_scaled = abs(cce_scaled) + + combined_loss = torch.cat([mse_scaled, bce_scaled, cce_scaled], dim=1) + for i, ft in enumerate(self.numeric_fts): - pdf[ft+'_pred'] = output_df[ft] - pdf[ft+'_loss'] = mse[:, i].cpu().numpy() - pdf[ft+'_z_loss'] = mse_scaled[:, i].cpu().numpy() if not return_abs else abs(mse_scaled[:, i].cpu().numpy()) + pdf[ft] = df[ft] + pdf[ft + '_pred'] = output_df[ft] + pdf[ft + '_loss'] = mse[:, i].cpu().numpy() + pdf[ft + '_z_loss'] = mse_scaled[:, i].cpu().numpy() + for i, ft in enumerate(self.binary_fts): - pdf[ft+'_pred'] = output_df[ft] - pdf[ft+'_loss'] = bce[:, i].cpu().numpy() - pdf[ft+'_z_loss'] = bce_scaled[:, i].cpu().numpy() if not return_abs else abs(bce_scaled[:, i].cpu().numpy()) + pdf[ft] = df[ft] + pdf[ft + '_pred'] = output_df[ft] + pdf[ft + '_loss'] = bce[:, i].cpu().numpy() + pdf[ft + '_z_loss'] = bce_scaled[:, i].cpu().numpy() + for i, ft in enumerate(self.categorical_fts): - pdf[ft+'_pred'] = output_df[ft] - pdf[ft+'_loss'] = cce[i].cpu().numpy() - pdf[ft+'_z_loss'] = cce_scaled[i].cpu().numpy() if not return_abs else abs(cce_scaled[i].cpu().numpy()) - all_cols = [[c, c+'_pred', c+'_loss', c+'_z_loss'] for c in orig_cols] - result_cols = [col for col_collection in all_cols for col in col_collection] - z_losses = [c+'_z_loss' for c in orig_cols] - pdf['max_abs_z'] = pdf[z_losses].max(axis=1) - pdf['mean_abs_z'] = pdf[z_losses].mean(axis=1) - result_cols.append('max_abs_z') - result_cols.append('mean_abs_z') - return pdf[result_cols] - \ No newline at end of file + pdf[ft] = df[ft] + pdf[ft + '_pred'] = output_df[ft] + pdf[ft + '_loss'] = cce[:, i].cpu().numpy() + pdf[ft + '_z_loss'] = cce_scaled[:, i].cpu().numpy() + + pdf['max_abs_z'] = combined_loss.max(dim=1)[0].cpu().numpy() + pdf['mean_abs_z'] = combined_loss.mean(dim=1).cpu().numpy() + + return pdf diff --git a/dfencoder/scalers.py b/dfencoder/scalers.py index e78d773..167a83d 100644 --- a/dfencoder/scalers.py +++ b/dfencoder/scalers.py @@ -1,6 +1,10 @@ +import typing + import numpy as np +import torch from sklearn.preprocessing import QuantileTransformer + class StandardScaler(object): """Impliments standard (mean/std) scaling.""" @@ -8,26 +12,43 @@ def __init__(self): self.mean = None self.std = None - def fit(self, x): - self.mean = x.mean() - self.std = x.std() + def fit(self, x: torch.Tensor): + self.mean = x.mean().item() + self.std = x.std().item() + + # Having a std == 0 (when all values are the same), breaks training. Just use 1.0 in this case + if (self.std == 0): + self.std = 1.0 + + def transform(self, x: typing.Union[torch.Tensor, np.ndarray]): + + # Ensure we are in the right floating point format + if (isinstance(x, torch.Tensor)): + result = x.to(dtype=torch.float32, copy=True) + elif (isinstance(x, np.ndarray)): + result = x.astype(float) - def transform(self, x): - result = x.astype(float) result -= self.mean result /= self.std return result - def inverse_transform(self, x): - result = x.astype(float) + def inverse_transform(self, x: torch.Tensor): + + # Ensure we are in the right floating point format + if (isinstance(x, torch.Tensor)): + result = x.to(dtype=torch.float32, copy=True) + elif (isinstance(x, np.ndarray)): + result = x.astype(float) + result *= self.std result += self.mean return result - def fit_transform(self, x): + def fit_transform(self, x: torch.Tensor): self.fit(x) return self.transform(x) + class GaussRankScaler(object): """ So-called "Gauss Rank" scaling. @@ -58,6 +79,7 @@ def fit_transform(self, x): self.fit(x) return self.transform(x) + class NullScaler(object): def __init__(self): From 168d88a5c1fc5c0ec7e318548e7fd8e9eb7d5e36 Mon Sep 17 00:00:00 2001 From: Michael Demoret Date: Fri, 16 Sep 2022 17:23:12 -0600 Subject: [PATCH 04/13] Fix decode with no categories --- dfencoder/autoencoder.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/dfencoder/autoencoder.py b/dfencoder/autoencoder.py index dc63e14..d196790 100644 --- a/dfencoder/autoencoder.py +++ b/dfencoder/autoencoder.py @@ -897,10 +897,16 @@ def decode_to_df(self, x, df=None): cat_df = pd.DataFrame(index=df.index) for i, ft in enumerate(self.categorical_fts): feature = self.categorical_fts[ft] - # get argmax excluding NaN column (impute with next-best guess) - codes = torch.argmax(cat[i][:, :-1], dim=1).cpu().numpy() - cat_df[ft] = codes cats = feature['cats'] + + if (len(cats) > 0): + # get argmax excluding NaN column (impute with next-best guess) + codes = torch.argmax(cat[i][:, :-1], dim=1).cpu().numpy() + else: + # Only one option + codes = torch.argmax(cat[i], dim=1).cpu().numpy() + cat_df[ft] = codes + cats = feature['cats'] + ["_other"] cat_df[ft] = cat_df[ft].apply(lambda x: cats[x]) # concat From 91815265e05e4e789cbd7f77df6bd17799afd8a8 Mon Sep 17 00:00:00 2001 From: Eli Fajardo Date: Mon, 10 Oct 2022 16:00:54 +0000 Subject: [PATCH 05/13] print when early stopping --- dfencoder/autoencoder.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/dfencoder/autoencoder.py b/dfencoder/autoencoder.py index cec0ec6..fb76944 100644 --- a/dfencoder/autoencoder.py +++ b/dfencoder/autoencoder.py @@ -692,7 +692,7 @@ def fit(self, df, epochs=1, val=None, patience=5): _, _, _, net_loss = self.compute_loss(num, bin, cat, slc_out, _id=True) id_loss.append(net_loss) - #Earlystopping + # Early stopping current_net_loss = net_loss if self.verbose: print('The Current Net Loss:', current_net_loss) @@ -703,13 +703,12 @@ def fit(self, df, epochs=1, val=None, patience=5): print('Early stop count:', count_es) if count_es >= patience: - if self.verbose: - print('Early stopping!\n') + print('Early stopping: early stop count({}) >= patience({})'.format(count_es, patience)) break else: if self.verbose: - print('set count for earlystop: 0') + print('Set count for earlystop: 0') count_es = 0 self.logger.end_epoch() From c6570825035efbd20af57f6961b71ed61e9ecf88 Mon Sep 17 00:00:00 2001 From: Eli Fajardo Date: Wed, 16 Nov 2022 16:35:56 +0000 Subject: [PATCH 06/13] move patience arg to constructor --- dfencoder/autoencoder.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/dfencoder/autoencoder.py b/dfencoder/autoencoder.py index fb76944..82a06e6 100644 --- a/dfencoder/autoencoder.py +++ b/dfencoder/autoencoder.py @@ -165,6 +165,7 @@ def __init__(self, progress_bar=True, n_megabatches=1, scaler='standard', + patience=5, *args, **kwargs): super(AutoEncoder, self).__init__(*args, **kwargs) @@ -224,6 +225,8 @@ def __init__(self, self.project_embeddings = project_embeddings self.scaler = scaler + + self.patience = patience self.n_megabatches = n_megabatches @@ -622,7 +625,7 @@ def _create_stat_dict(self, a): std = scaler.std return {'scaler': scaler, 'mean': mean, 'std': std} - def fit(self, df, epochs=1, val=None, patience=5): + def fit(self, df, epochs=1, val=None): """Does training.""" pdf = df.copy() # if val is None: @@ -702,8 +705,9 @@ def fit(self, df, epochs=1, val=None, patience=5): if self.verbose: print('Early stop count:', count_es) - if count_es >= patience: - print('Early stopping: early stop count({}) >= patience({})'.format(count_es, patience)) + if count_es >= self.patience: + if self.verbose: + print('Early stopping: early stop count({}) >= patience({})'.format(count_es, self.patience)) break else: From a93aaefbc13c2808d290c9bd6166318cd0612eee Mon Sep 17 00:00:00 2001 From: Eli Fajardo Date: Wed, 16 Nov 2022 12:09:30 -0500 Subject: [PATCH 07/13] Add option to preset categories (#3) * add option to preset categories * set the index of the prediction df to match the input df --- dfencoder/autoencoder.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/dfencoder/autoencoder.py b/dfencoder/autoencoder.py index d196790..250904b 100644 --- a/dfencoder/autoencoder.py +++ b/dfencoder/autoencoder.py @@ -165,6 +165,7 @@ def __init__(self, progress_bar=True, n_megabatches=1, scaler='standard', + preset_cats=None, *args, **kwargs): super(AutoEncoder, self).__init__(*args, **kwargs) @@ -180,6 +181,7 @@ def __init__(self, self.encoder_dropout = encoder_dropout self.decoder_dropout = decoder_dropout self.min_cats = min_cats + self.preset_cats = preset_cats self.encoder = [] self.decoder = [] self.train_mode = self.train @@ -330,8 +332,11 @@ def init_binary(self, df): self.bin_names = list(self.binary_fts.keys()) def init_features(self, df): + if self.preset_cats is not None: + self.categorical_fts = self.preset_cats + else: + self.init_cats(df) self.init_numeric(df) - self.init_cats(df) self.init_binary(df) def build_inputs(self): @@ -991,6 +996,9 @@ def get_results(self, df, return_abs=False): x = torch.cat(num + bin + embeddings, dim=1) x = self.encode(x) output_df = self.decode_to_df(x) + + # set the index of the prediction df to match the input df + output_df.index = df.index mse, bce, cce = self.get_anomaly_score_losses(df) mse_scaled, bce_scaled, cce_scaled = self.scale_losses(mse, bce, cce) From 3c2639915535bdd2d98aad86619e28b4ec25cc79 Mon Sep 17 00:00:00 2001 From: Michael Demoret Date: Wed, 30 Nov 2022 11:52:10 -0700 Subject: [PATCH 08/13] Creating branch for v23.01 From bf70f6aa1ba67016fd5afc4cfae6dcb5583411aa Mon Sep 17 00:00:00 2001 From: Michael Demoret Date: Wed, 30 Nov 2022 11:54:20 -0700 Subject: [PATCH 09/13] Adding CODEOWNERS and ops-bot.yaml --- .github/CODEOWNERS | 2 ++ .github/ops-bot.yaml | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 .github/CODEOWNERS create mode 100644 .github/ops-bot.yaml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..64affdd --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,2 @@ +# Default Approval rule if one of the later sections does not apply +* @nv-morpheus/dfencoder-codeowners diff --git a/.github/ops-bot.yaml b/.github/ops-bot.yaml new file mode 100644 index 0000000..fbe76f6 --- /dev/null +++ b/.github/ops-bot.yaml @@ -0,0 +1,24 @@ +# SPDX-FileCopyrightText: Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This file controls which features from the `ops-bot` repository below are enabled. +# - https://github.com/rapidsai/ops-bot + +auto_merger: true +branch_checker: true +label_checker: true +release_drafter: true +copy_prs: true +rerun_tests: true From 9da3bbdd610308234067bf0a4854e68369c9a76d Mon Sep 17 00:00:00 2001 From: Eli Fajardo Date: Wed, 30 Nov 2022 21:49:23 +0000 Subject: [PATCH 10/13] update last_loss in early stop --- dfencoder/autoencoder.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dfencoder/autoencoder.py b/dfencoder/autoencoder.py index 9e7510b..c97bd40 100644 --- a/dfencoder/autoencoder.py +++ b/dfencoder/autoencoder.py @@ -720,6 +720,8 @@ def fit(self, df, epochs=1, val=None): print('Set count for earlystop: 0') count_es = 0 + last_loss = current_net_loss + self.logger.end_epoch() # if self.project_embeddings: # self.logger.show_embeddings(self.categorical_fts) From 79fd108aab046c9a65b9a8991167a55f971675a9 Mon Sep 17 00:00:00 2001 From: David Gardner <96306125+dagardner-nv@users.noreply.github.com> Date: Mon, 19 Dec 2022 10:47:09 -0800 Subject: [PATCH 11/13] Prevent result from being an undefined variable (#4) Raises a `ValueError` when `x` isn't one of the unexpected types. Avoids an `UnboundLocalError: local variable 'result' referenced before assignment` error Not sure if this is specific to our fork or if it should be contributed upstream. Authors: - David Gardner (https://github.com/dagardner-nv) Approvers: - Michael Demoret (https://github.com/mdemoret-nv) URL: https://github.com/nv-morpheus/dfencoder/pull/4 --- dfencoder/scalers.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dfencoder/scalers.py b/dfencoder/scalers.py index 167a83d..f95cd00 100644 --- a/dfencoder/scalers.py +++ b/dfencoder/scalers.py @@ -27,6 +27,8 @@ def transform(self, x: typing.Union[torch.Tensor, np.ndarray]): result = x.to(dtype=torch.float32, copy=True) elif (isinstance(x, np.ndarray)): result = x.astype(float) + else: + raise ValueError(f"Unsupported type: {type(x)}") result -= self.mean result /= self.std From d5c7820aaf6ac171007801d534b50ce840b434ae Mon Sep 17 00:00:00 2001 From: hsin-c <109615347+hsin-c@users.noreply.github.com> Date: Fri, 13 Jan 2023 16:42:36 -0800 Subject: [PATCH 12/13] Batched implementation of get_anomaly_score_losses (#5) Addressing: https://github.com/nv-morpheus/Morpheus/issues/498 Batching the evaluation of the input dataframe in the `get_anomaly_score_losses` function helps with the high GPU memory usage issue. Authors: - https://github.com/hsin-c Approvers: - David Gardner (https://github.com/dagardner-nv) - Eli Fajardo (https://github.com/efajardo-nv) - Michael Demoret (https://github.com/mdemoret-nv) URL: https://github.com/nv-morpheus/dfencoder/pull/5 --- dfencoder/autoencoder.py | 51 +++++++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 16 deletions(-) diff --git a/dfencoder/autoencoder.py b/dfencoder/autoencoder.py index 250904b..4649894 100644 --- a/dfencoder/autoencoder.py +++ b/dfencoder/autoencoder.py @@ -945,26 +945,45 @@ def get_anomaly_score_with_losses(self, df): return mse, bce, cce, net def get_anomaly_score_losses(self, df): + """ + Run the input dataframe `df` through the autoencoder to get the recovery losses by feature type + (numerical/boolean/categorical). + """ self.eval() - data = self.prepare_df(df) - input = self.build_input_tensor(data) - num_target, bin_target, codes = self.compute_targets(data) + n_batches = len(df) // self.batch_size + if len(df) % self.batch_size > 0: + n_batches += 1 + mse_loss_slices, bce_loss_slices, cce_loss_slices = [], [], [] with torch.no_grad(): - num, bin, cat = self.forward(input) - - mse_loss: torch.Tensor = self.mse(num, num_target) - bce_loss: torch.Tensor = self.bce(bin, bin_target) - cce_loss = [] - for i, ft in enumerate(self.categorical_fts): - loss = self.cce(cat[i], codes[i]) - # Convert to 2 dimensions - cce_loss.append(loss.data.reshape(-1, 1)) - - # Join all categories into a single tensor - cce_loss = torch.cat(cce_loss, dim=1) - + for i in range(n_batches): + start = i * self.batch_size + stop = (i + 1) * self.batch_size + + df_slice = df.iloc[start:stop] + data_slice = self.prepare_df(df_slice) + num_target, bin_target, codes = self.compute_targets(data_slice) + + input_slice = self.build_input_tensor(data_slice) + + num, bin, cat = self.forward(input_slice) + mse_loss_slice: torch.Tensor = self.mse(num, num_target) + bce_loss_slice: torch.Tensor = self.bce(bin, bin_target) + cce_loss_slice_of_each_feat = [] # each entry in this list is the cce loss of a feature, ordered by the feature list self.categorical_fts + for i, ft in enumerate(self.categorical_fts): + loss = self.cce(cat[i], codes[i]) + # Convert to 2 dimensions + cce_loss_slice_of_each_feat.append(loss.data.reshape(-1, 1)) + cce_loss_slice = torch.cat(cce_loss_slice_of_each_feat, dim=1) # merge the tensors into one (n_records * n_features) tensor + + mse_loss_slices.append(mse_loss_slice) + bce_loss_slices.append(bce_loss_slice) + cce_loss_slices.append(cce_loss_slice) + + mse_loss = torch.cat(mse_loss_slices, dim=0) + bce_loss = torch.cat(bce_loss_slices, dim=0) + cce_loss = torch.cat(cce_loss_slices, dim=0) return mse_loss, bce_loss, cce_loss def scale_losses(self, mse, bce, cce): From 9938198bd678883efeb1e83a3c617d74e9eec71b Mon Sep 17 00:00:00 2001 From: hsin-c <109615347+hsin-c@users.noreply.github.com> Date: Thu, 26 Jan 2023 11:50:21 -0800 Subject: [PATCH 13/13] Add an option to use modified z score instead of z score to scale losses (#6) 1 of the 2 PRs for the issue: https://github.com/nv-morpheus/Morpheus/issues/497 This PR checks the `DFP to optionally use modified zscore (MAD)` checkbox. Authors: - https://github.com/hsin-c Approvers: - https://github.com/gbatmaz - Michael Demoret (https://github.com/mdemoret-nv) URL: https://github.com/nv-morpheus/dfencoder/pull/6 --- dfencoder/autoencoder.py | 80 ++++++++++++++++++++++++++++++---------- dfencoder/scalers.py | 73 ++++++++++++++++++++++++++++-------- 2 files changed, 118 insertions(+), 35 deletions(-) diff --git a/dfencoder/autoencoder.py b/dfencoder/autoencoder.py index 4649894..191723c 100644 --- a/dfencoder/autoencoder.py +++ b/dfencoder/autoencoder.py @@ -54,7 +54,7 @@ from .dataframe import EncoderDataFrame from .logging import BasicLogger, IpynbLogger, TensorboardXLogger -from .scalers import GaussRankScaler, NullScaler, StandardScaler +from .scalers import GaussRankScaler, NullScaler, StandardScaler, ModifiedScaler def ohe(input_vector, dim, device="cpu"): @@ -164,8 +164,9 @@ def __init__(self, run=None, progress_bar=True, n_megabatches=1, - scaler='standard', + scaler='standard', # scaler for the numerical features preset_cats=None, + loss_scaler='standard', # scaler for the losses (z score) *args, **kwargs): super(AutoEncoder, self).__init__(*args, **kwargs) @@ -226,11 +227,20 @@ def __init__(self, self.project_embeddings = project_embeddings self.scaler = scaler + # scaler class used to scale losses and collect loss stats + self.loss_scaler_str = loss_scaler + self.loss_scaler = self.get_scaler(loss_scaler) self.n_megabatches = n_megabatches def get_scaler(self, name): - scalers = {'standard': StandardScaler, 'gauss_rank': GaussRankScaler, None: NullScaler, 'none': NullScaler} + scalers = { + 'standard': StandardScaler, + 'gauss_rank': GaussRankScaler, + 'modified': ModifiedScaler, + None: NullScaler, + 'none': NullScaler + } return scalers[name] def init_numeric(self, df): @@ -621,26 +631,48 @@ def compute_baseline_performance(self, in_, out_): return net_loss def _create_stat_dict(self, a): - scaler = StandardScaler() + scaler = self.loss_scaler() scaler.fit(a) - mean = scaler.mean - std = scaler.std - return {'scaler': scaler, 'mean': mean, 'std': std} - - def fit(self, df, epochs=1, val=None): - """Does training.""" - pdf = df.copy() - # if val is None: - # pdf_val = None - # else: - # pdf_val = val.copy() + return {'scaler': scaler} + + def fit( + self, df, epochs=1, val=None, run_validation=False, use_val_for_loss_stats=False + ): + """Does training. + Args: + df: pandas df used for training + epochs: number of epochs to run training + val: optional pandas dataframe for validation or loss stats + run_validation: boolean indicating whether to collect validation loss for each + epoch during training + use_val_for_loss_stats: boolean indicating whether to use the validation set + for loss statistics collection (for z score calculation) + + Raises: + ValueError: + if run_validation or use_val_for_loss_stats is True but val is not provided + """ + if (run_validation or use_val_for_loss_stats) and val is None: + raise ValueError( + "Validation set is required if either run_validation or \ + use_val_for_loss_stats is set to True." + ) + + if use_val_for_loss_stats: + df_for_loss_stats = val.copy() + else: + # use train loss + df_for_loss_stats = df.copy() + + if run_validation and val is not None: + val = val.copy() if self.optim is None: self.build_model(df) if self.n_megabatches == 1: df = self.prepare_df(df) - if val is not None: + if run_validation and val is not None: val_df = self.prepare_df(val) val_in = val_df.swap(likelihood=self.swap_p) msg = "Validating during training.\n" @@ -671,7 +703,7 @@ def fit(self, df, epochs=1, val=None): if self.lr_decay is not None: self.lr_decay.step() - if val is not None: + if run_validation and val is not None: self.eval() with torch.no_grad(): swapped_loss = [] @@ -712,7 +744,7 @@ def fit(self, df, epochs=1, val=None): #Getting training loss statistics # mse_loss, bce_loss, cce_loss, _ = self.get_anomaly_score(pdf) if pdf_val is None else self.get_anomaly_score(pd.concat([pdf, pdf_val])) - mse_loss, bce_loss, cce_loss, _ = self.get_anomaly_score_with_losses(pdf) + mse_loss, bce_loss, cce_loss, _ = self.get_anomaly_score_with_losses(df_for_loss_stats) for i, ft in enumerate(self.numeric_fts): i_loss = mse_loss[:, i] self.feature_loss_stats[ft] = self._create_stat_dict(i_loss) @@ -1028,7 +1060,7 @@ def get_results(self, df, return_abs=False): cce_scaled = abs(cce_scaled) combined_loss = torch.cat([mse_scaled, bce_scaled, cce_scaled], dim=1) - + for i, ft in enumerate(self.numeric_fts): pdf[ft] = df[ft] pdf[ft + '_pred'] = output_df[ft] @@ -1050,4 +1082,14 @@ def get_results(self, df, return_abs=False): pdf['max_abs_z'] = combined_loss.max(dim=1)[0].cpu().numpy() pdf['mean_abs_z'] = combined_loss.mean(dim=1).cpu().numpy() + # add a column describing the scaler of the losses + if self.loss_scaler_str == 'standard': + output_scaled_loss_str = 'z' + elif self.loss_scaler_str == 'modified': + output_scaled_loss_str = 'modz' + else: + # in case other custom scaling is used + output_scaled_loss_str = f'{self.loss_scaler_str}_scaled' + pdf['z_loss_scaler_type'] = output_scaled_loss_str + return pdf diff --git a/dfencoder/scalers.py b/dfencoder/scalers.py index f95cd00..07a275f 100644 --- a/dfencoder/scalers.py +++ b/dfencoder/scalers.py @@ -4,6 +4,16 @@ import torch from sklearn.preprocessing import QuantileTransformer +def ensure_float_type(x: typing.Union[torch.Tensor, np.ndarray]): + """Ensure we are in the right floating point format. """ + if (isinstance(x, torch.Tensor)): + result = x.to(dtype=torch.float32, copy=True) + elif (isinstance(x, np.ndarray)): + result = x.astype(float) + else: + raise ValueError(f"Unsupported type: {type(x)}") + return result + class StandardScaler(object): """Impliments standard (mean/std) scaling.""" @@ -21,27 +31,13 @@ def fit(self, x: torch.Tensor): self.std = 1.0 def transform(self, x: typing.Union[torch.Tensor, np.ndarray]): - - # Ensure we are in the right floating point format - if (isinstance(x, torch.Tensor)): - result = x.to(dtype=torch.float32, copy=True) - elif (isinstance(x, np.ndarray)): - result = x.astype(float) - else: - raise ValueError(f"Unsupported type: {type(x)}") - + result = ensure_float_type(x) result -= self.mean result /= self.std return result def inverse_transform(self, x: torch.Tensor): - - # Ensure we are in the right floating point format - if (isinstance(x, torch.Tensor)): - result = x.to(dtype=torch.float32, copy=True) - elif (isinstance(x, np.ndarray)): - result = x.astype(float) - + result = ensure_float_type(x) result *= self.std result += self.mean return result @@ -50,6 +46,51 @@ def fit_transform(self, x: torch.Tensor): self.fit(x) return self.transform(x) +class ModifiedScaler(object): + """Implements scaling using modified z score. + Reference: https://www.ibm.com/docs/el/cognos-analytics/11.1.0?topic=terms-modified-z-score + """ + MAD_SCALING_FACTOR = 1.486 # 1.486 * MAD approximately equals the standard deviation + MEANAD_SCALING_FACTOR = 1.253314 # 1.253314 * MeanAD approximately equals the standard deviation + + def __init__(self): + self.median: float = None + self.mad: float = None # median absolute deviation + self.meanad: float = None # mean absolute deviation + + def fit(self, x: torch.Tensor): + med = x.median().item() + self.median = med + self.mad = (x - med).abs().median().item() + self.meanad = (x - med).abs().mean().item() + # Having a meanad == 0 (when all values are the same), breaks training. Just use 1.0 in this case + if (self.meanad == 0): + self.meanad = 1.0 + + def transform(self, x: typing.Union[torch.Tensor, np.ndarray]): + result = ensure_float_type(x) + + result -= self.median + if self.mad == 0: + result /= (self.MEANAD_SCALING_FACTOR * self.meanad) + else: + result /= (self.MAD_SCALING_FACTOR * self.mad) + return result + + def inverse_transform(self, x: torch.Tensor): + result = ensure_float_type(x) + + if self.mad == 0: + result *= (self.MEANAD_SCALING_FACTOR * self.meanad) + else: + result *= (self.MAD_SCALING_FACTOR * self.mad) + result += self.median + return result + + def fit_transform(self, x: torch.Tensor): + self.fit(x) + return self.transform(x) + class GaussRankScaler(object): """