diff --git a/.github/workflows/build_coverage.yml b/.github/workflows/build_coverage.yml index 44f73053..3c95ed08 100644 --- a/.github/workflows/build_coverage.yml +++ b/.github/workflows/build_coverage.yml @@ -23,36 +23,69 @@ jobs: matrix: os: - ubuntu-latest - python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"] # Bugs: 3.10 will become 3.1 if without quotes -> https://github.com/actions/setup-python/issues/695 # For ubuntu 22.04: https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json - pytorch-version: ["1.9.1", "1.10.1", "1.11.0", "1.12.1", "1.13.1", "2.0.1"] + pytorch-version: ["1.9.1", "1.10.1", "1.11.0", "1.12.1", "1.13.1", "2.0.1", "2.1.2", "2.2.0", "2.3.1", "2.4.1", "2.5.1"] # 1.5.1, 1.4.0 Model load error in robustbench. # 1.8.1, 1.7.1, 1.6.0 'padding==same' error in TIFGSM exclude: # https://github.com/pytorch/vision#installation + # pytorch 2.5 support python from 3.9 to 3.12 + - pytorch-version: "2.5.1" + python-version: "3.7" + - pytorch-version: "2.5.1" + python-version: "3.8" + # pytorch 2.4 support python from 3.8 to 3.12 + - pytorch-version: "2.4.1" + python-version: "3.7" + # pytorch 2.3 support python from 3.8 to 3.12 + - pytorch-version: "2.3.1" + python-version: "3.7" + # pytorch 2.2 support python from 3.8 to 3.11 + - pytorch-version: "2.2.0" + python-version: "3.7" + - pytorch-version: "2.2.0" + python-version: "3.12" + # pytorch 2.1 support python from 3.8 to 3.11 + - pytorch-version: "2.1.2" + python-version: "3.7" + - pytorch-version: "2.1.2" + python-version: "3.12" # pytorch 2.0 support python from 3.8 to 3.11 - pytorch-version: "2.0.1" python-version: "3.7" + - pytorch-version: "2.0.1" + python-version: "3.12" # pytorch 1.13 support python from 3.7.2 to 3.10 - pytorch-version: "1.13.1" python-version: "3.11" + - pytorch-version: "1.13.1" + python-version: "3.12" # pytorch 1.12 support python from 3.7 to 3.10 - pytorch-version: "1.12.1" python-version: "3.11" + - pytorch-version: "1.12.1" + python-version: "3.12" # pytorch 1.11 support python from 3.7 to 3.10 - pytorch-version: "1.11.0" python-version: "3.11" + - pytorch-version: "1.11.0" + python-version: "3.12" # pytorch 1.10 support python from 3.6 to 3.9 - pytorch-version: "1.10.1" python-version: "3.10" - pytorch-version: "1.10.1" python-version: "3.11" + - pytorch-version: "1.10.1" + python-version: "3.12" # pytorch 1.9 support python from 3.6 to 3.9 - pytorch-version: "1.9.1" python-version: "3.10" - pytorch-version: "1.9.1" python-version: "3.11" + - pytorch-version: "1.9.1" + python-version: "3.12" runs-on: ${{ matrix.os }} diff --git a/.gitignore b/.gitignore index 7b58903a..0356639a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,28 +1,28 @@ +.vscode/ __pycache__ -.ipynb_checkpoints/* +.ipynb_checkpoints/ debug.log -build/* -_build -dist/* -torchattacks.egg-info/* -data/* -models/* +build/ +dist/ +torchattacks.egg-info/ +data/ */.* MENIFEST.in setup.cfg _commit.bat +code_coverage/data/ -autoattack/* +autoattack/ +.pytest_cache/ demo/_* -demo/data/* -demo/models/* -demo/torchdefenses/* -demo/robustbench/* -demo/autoattack/* +demo/data +demo/models +demo/torchdefenses +demo/autoattack + TODO.txt -.vscode/ coverage.xml .coverage black.ipynb diff --git a/README.md b/README.md index 79528a86..b3eaeacd 100644 --- a/README.md +++ b/README.md @@ -94,10 +94,10 @@ pip install -e . ``` * By label ```python - atk.set_mode_targeted_by_label(quiet=True) # shift all class loops one to the right, 1=>2, 2=>3, .., 9=>0 target_labels = (labels + 1) % 10 - adv_images = atk(images, target_labels) + atk.set_mode_targeted_by_label(target_labels=target_labels, quiet=True) + adv_images = atk(images, labels) ``` * Return to default ```python @@ -128,12 +128,6 @@ pip install -e . atk2 = torchattacks.PGD(model, eps=8/255, alpha=2/255, iters=40, random_start=True) atk = torchattacks.MultiAttack([atk1, atk2]) ``` - * Binary search for CW - ```python - atk1 = torchattacks.CW(model, c=0.1, steps=1000, lr=0.01) - atk2 = torchattacks.CW(model, c=1, steps=1000, lr=0.01) - atk = torchattacks.MultiAttack([atk1, atk2]) - ``` * Random restarts ```python atk1 = torchattacks.PGD(model, eps=8/255, alpha=2/255, iters=40, random_start=True) @@ -160,7 +154,7 @@ The distance measure in parentheses. | **EOTPGD**
(Linf) | Comment on "Adv-BNN: Improved Adversarial Defense through Robust Bayesian Neural Network" ([Zimmermann, 2019](https://arxiv.org/abs/1907.00895)) | [EOT](https://arxiv.org/abs/1707.07397)+PGD | | **APGD**
(Linf, L2) | Reliable evaluation of adversarial robustness with an ensemble of diverse parameter-free attacks ([Croce et al., 2020](https://arxiv.org/abs/2001.03994)) | | | **APGDT**
(Linf, L2) | Reliable evaluation of adversarial robustness with an ensemble of diverse parameter-free attacks ([Croce et al., 2020](https://arxiv.org/abs/2001.03994)) | Targeted APGD | -| **FAB**
(Linf, L2, L1) | Minimally distorted Adversarial Examples with a Fast Adaptive Boundary Attack ([Croce et al., 2019](https://arxiv.org/abs/1907.02044)) | | +| **AFAB**
(Linf, L1, L2) | Minimally distorted Adversarial Examples with a Fast Adaptive Boundary Attack ([Croce et al., 2019](https://arxiv.org/abs/1907.02044)) | | | **Square**
(Linf, L2) | Square Attack: a query-efficient black-box adversarial attack via random search ([Andriushchenko et al., 2019](https://arxiv.org/abs/1912.00049)) | | | **AutoAttack**
(Linf, L2) | Reliable evaluation of adversarial robustness with an ensemble of diverse parameter-free attacks ([Croce et al., 2020](https://arxiv.org/abs/2001.03994)) | APGD+APGDT+FAB+Square | | **DeepFool**
(L2) | DeepFool: A Simple and Accurate Method to Fool Deep Neural Networks ([Moosavi-Dezfooli et al., 2016](https://arxiv.org/abs/1511.04599)) | | @@ -181,8 +175,15 @@ The distance measure in parentheses. | **EADEN**
(L1, L2) | EAD: Elastic-Net Attacks to Deep Neural Networks ([Chen, Pin-Yu, et al., 2018](https://arxiv.org/abs/1709.04114)) | :heart_eyes: Contributor [Riko Naka](https://github.com/rikonaka) | | **PIFGSM (PIM)**
(Linf) | Patch-wise Attack for Fooling Deep Neural Network ([Gao, Lianli, et al., 2020](https://arxiv.org/abs/2007.06765)) | :heart_eyes: Contributor [Riko Naka](https://github.com/rikonaka) | | **PIFGSM++ (PIM++)**
(Linf) | Patch-wise++ Perturbation for Adversarial Targeted Attacks ([Gao, Lianli, et al., 2021](https://arxiv.org/abs/2012.15503)) | :heart_eyes: Contributor [Riko Naka](https://github.com/rikonaka) | - - +| **CWL0**
(L0) | Towards Evaluating the Robustness of Neural Networks ([Carlini N, Wagner D, 2017](https://arxiv.org/abs/1608.046443)) | :heart_eyes: Contributor [Riko Naka](https://github.com/rikonaka) | +| **CWLinf**
(Linf) | Towards Evaluating the Robustness of Neural Networks ([Carlini N, Wagner D, 2017](https://arxiv.org/abs/1608.046443)) | :heart_eyes: Contributor [Riko Naka](https://github.com/rikonaka) | +| **CWBSL0 (Binary Search Version)**
(L0) | Towards Evaluating the Robustness of Neural Networks ([Carlini N, Wagner D, 2017](https://arxiv.org/abs/1608.046443)) | :heart_eyes: Contributor [Riko Naka](https://github.com/rikonaka) | +| **CWBSL2 (Binary Search Version)**
(L2) | Towards Evaluating the Robustness of Neural Networks ([Carlini N, Wagner D, 2017](https://arxiv.org/abs/1608.046443)) | :heart_eyes: Contributor [Riko Naka](https://github.com/rikonaka) | +| **CWBSLinf (Binary Search Version)**
(Linf) | Towards Evaluating the Robustness of Neural Networks ([Carlini N, Wagner D, 2017](https://arxiv.org/abs/1608.046443)) | :heart_eyes: Contributor [Riko Naka](https://github.com/rikonaka) | +| **ESPGD (Early-Stopped PGD Version)**
(Linf) | Attacks Which Do Not Kill Training Make Adversarial Learning Stronger ([Zhang, Jingfeng, 2020](https://arxiv.org/abs/2002.11242)) | :heart_eyes: Contributor [Riko Naka](https://github.com/rikonaka) | +| **FAB**
(Linf) | Minimally distorted Adversarial Examples with a Fast Adaptive Boundary Attack ([Croce et al., 2019](https://arxiv.org/abs/1907.02044)) | :heart_eyes: Contributor [Riko Naka](https://github.com/rikonaka) | +| **FABL1**
(L1) | Minimally distorted Adversarial Examples with a Fast Adaptive Boundary Attack ([Croce et al., 2019](https://arxiv.org/abs/1907.02044)) | :heart_eyes: Contributor [Riko Naka](https://github.com/rikonaka) | +| **FABL2**
(L2) | Minimally distorted Adversarial Examples with a Fast Adaptive Boundary Attack ([Croce et al., 2019](https://arxiv.org/abs/1907.02044)) | :heart_eyes: Contributor [Riko Naka](https://github.com/rikonaka) | ## :bar_chart: Performance Comparison diff --git a/code_coverage/images.pth b/code_coverage/images.pth new file mode 100644 index 00000000..9b5d7ffd Binary files /dev/null and b/code_coverage/images.pth differ diff --git a/code_coverage/labels.pth b/code_coverage/labels.pth new file mode 100644 index 00000000..0fd708ca Binary files /dev/null and b/code_coverage/labels.pth differ diff --git a/code_coverage/resnet18_eval.pth b/code_coverage/resnet18_eval.pth new file mode 100644 index 00000000..7606da16 Binary files /dev/null and b/code_coverage/resnet18_eval.pth differ diff --git a/code_coverage/script/pickle_cifar10.py b/code_coverage/script/pickle_cifar10.py new file mode 100644 index 00000000..2ece930d --- /dev/null +++ b/code_coverage/script/pickle_cifar10.py @@ -0,0 +1,25 @@ +import torch +from torchvision import datasets, transforms + +transform_test = transforms.Compose([ + transforms.ToTensor(), +]) +testset = datasets.CIFAR10( + root='../data', train=False, download=True, transform=transform_test) +test_loader = torch.utils.data.DataLoader( + testset, batch_size=10, shuffle=False) + + +def split(testloader): + for (x, y) in testloader: + torch.save(x, 'images.pth') + torch.save(y, 'labels.pth') + break + + +def main(): + split(test_loader) + + +if __name__ == '__main__': + main() diff --git a/code_coverage/script/resnet.py b/code_coverage/script/resnet.py new file mode 100644 index 00000000..43324c64 --- /dev/null +++ b/code_coverage/script/resnet.py @@ -0,0 +1,117 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class BasicBlock(nn.Module): + expansion = 1 + + def __init__(self, in_planes, planes, stride=1): + super(BasicBlock, self).__init__() + self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + self.relu = nn.ReLU() + + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion * planes: + self.shortcut = nn.Sequential( + nn.Conv2d(in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(self.expansion * planes) + ) + + def forward(self, x): + out = self.relu(self.bn1(self.conv1(x))) + out = self.bn2(self.conv2(out)) + out += self.shortcut(x) + out = F.relu(out) + return out + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, in_planes, planes, stride=1): + super(Bottleneck, self).__init__() + self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + self.conv3 = nn.Conv2d(planes, self.expansion * planes, kernel_size=1, bias=False) + self.bn3 = nn.BatchNorm2d(self.expansion * planes) + self.relu = nn.ReLU() + + self.shortcut = nn.Sequential() + if stride != 1 or in_planes != self.expansion * planes: + self.shortcut = nn.Sequential( + nn.Conv2d(in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=False), + nn.BatchNorm2d(self.expansion * planes) + ) + + def forward(self, x): + out = self.relu(self.bn1(self.conv1(x))) + out = self.relu(self.bn2(self.conv2(out))) + out = self.bn3(self.conv3(out)) + out += self.shortcut(x) + out = F.relu(out) + return out + + +class ResNet(nn.Module): + def __init__(self, block, num_blocks, num_classes=10): + super(ResNet, self).__init__() + self.in_planes = 64 + + self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(64) + self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1) + self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2) + self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2) + self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2) + self.linear = nn.Linear(512 * block.expansion, num_classes) + + def _make_layer(self, block, planes, num_blocks, stride): + strides = [stride] + [1] * (num_blocks - 1) + layers = [] + for stride in strides: + layers.append(block(self.in_planes, planes, stride)) + self.in_planes = planes * block.expansion + return nn.Sequential(*layers) + + def forward(self, x): + out = F.relu(self.bn1(self.conv1(x))) + out = self.layer1(out) + out = self.layer2(out) + out = self.layer3(out) + out = self.layer4(out) + out = F.avg_pool2d(out, 4) + out = out.view(out.size(0), -1) + out = self.linear(out) + return out + + +def ResNet18(num_classes=10): + return ResNet(BasicBlock, [2, 2, 2, 2]) + + +def ResNet34(): + return ResNet(BasicBlock, [3, 4, 6, 3]) + + +def ResNet50(): + return ResNet(Bottleneck, [3, 4, 6, 3]) + + +def ResNet101(): + return ResNet(Bottleneck, [3, 4, 23, 3]) + + +def ResNet152(): + return ResNet(Bottleneck, [3, 8, 36, 3]) + + +def test(): + net = ResNet18() + y = net(torch.randn(1, 3, 32, 32)) + print(y.size()) \ No newline at end of file diff --git a/code_coverage/script/train_simple_resnet18.py b/code_coverage/script/train_simple_resnet18.py new file mode 100644 index 00000000..99a4115d --- /dev/null +++ b/code_coverage/script/train_simple_resnet18.py @@ -0,0 +1,128 @@ +import torch +import torch.nn as nn +import torch.optim as optim +import torch.backends.cudnn as cudnn +from torchvision import datasets, transforms +from tqdm import tqdm +import time +import os + +from resnet import ResNet18 + +device = 'cuda' if torch.cuda.is_available() else 'cpu' +cudnn.benchmark = True + +transform_train = transforms.Compose([ + transforms.RandomCrop(32, padding=4), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), +]) +transform_test = transforms.Compose([ + transforms.ToTensor(), +]) + +trainset = datasets.CIFAR10( + root='./data', train=True, download=True, transform=transform_train) +testset = datasets.CIFAR10( + root='./data', train=False, download=True, transform=transform_test) + +train_loader = torch.utils.data.DataLoader( + trainset, batch_size=128, shuffle=True, num_workers=32) +test_loader = torch.utils.data.DataLoader( + testset, batch_size=100, shuffle=False, num_workers=32) + + +def train(net, loss, optimizer, trainloader, epoch): + # Training + print('\nEpoch: {}'.format(epoch)) + net.train() + train_loss = 0 + correct = 0 + total = 0 + + with tqdm(total=len(trainloader), desc='Train') as tbar: + for batch_idx, (x, y) in enumerate(trainloader): + x, y = x.to(device), y.to(device) + total += y.shape[0] + + optimizer.zero_grad() + outputs = net(x) + _loss = loss(outputs, y) + _loss.backward() + optimizer.step() + + train_loss += _loss.item() + predicted = torch.argmax(outputs, 1) + correct += torch.sum((predicted == y)).item() + + lr = optimizer.param_groups[0].get('lr') + tbar.set_postfix(loss=train_loss/(batch_idx+1), + acc=(correct/total)*100., lr=lr) + tbar.update() + + return (correct / total) * 100. + + +def test(net, loss, testloader, epoch, best_acc): + # Test + net.eval() + correct = 0 + total = 0 + test_loss = 0 + + with tqdm(total=len(testloader), desc='Test') as tbar: + for batch_idx, (x, y) in enumerate(testloader): + x, y = x.to(device), y.to(device) + total += y.shape[0] + + outputs = net(x) + _loss = loss(outputs, y) + test_loss += _loss.item() + predicted = torch.argmax(outputs, 1) + correct += torch.sum((predicted == y)).item() + + # tbar.set_description('loss: {:.3f} acc: {:.3f} aacc: {:.3f}'.format( + # test_loss/(batch_idx+1, (correct/total)*100, (adv_correct/total)*100))) + tbar.set_postfix(loss=test_loss/(batch_idx+1), + acc=(correct/total)*100.) + tbar.update() + + acc = (correct / total) * 100. + # Save checkpoint. + if acc > best_acc: + best_acc = acc + p = os.path.join('./', f'resnet18_eval.pth') + state = { + 'net': net.state_dict(), + 'acc': acc, + 'epoch': epoch, + } + torch.save(state, p) + + return best_acc + + +def main(): + lr = 0.01 + momentum = 0.9 + weight_decay = 3.5e-3 + best_acc = 0 + epochs = 100 + + net = ResNet18().to(device) + loss = nn.CrossEntropyLoss() + optimizer = optim.SGD(net.parameters(), lr=lr, + momentum=momentum, weight_decay=weight_decay) + scheduler = torch.optim.lr_scheduler.CosineAnnealingLR( + optimizer, T_max=100) + for epoch in range(1, epochs + 1): + _ = train(net, loss, optimizer, train_loader, epoch) # nopep8 + best_acc = test(net, loss, test_loader, epoch, best_acc) # nopep8 + scheduler.step() + + time_str = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) + print(f'END: {time_str}') + + +if __name__ == '__main__': + main() diff --git a/code_coverage/test_atks.py b/code_coverage/test_atks.py index f2f868dc..7c4488ca 100644 --- a/code_coverage/test_atks.py +++ b/code_coverage/test_atks.py @@ -1,44 +1,56 @@ -import sys -import os # Importing the parent directory # This line must be preceded by -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) # nopep8 +import sys +import os +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))) + +import torchattacks +import pytest +import time +import torch + +from script.resnet import ResNet18 -from robustbench.utils import load_model # nopep8 -from robustbench.utils import clean_accuracy # nopep8 -from robustbench.data import load_cifar10 # nopep8 -import torchattacks # nopep8 -import torch # nopep8 -import pytest # nopep8 -import time # nopep8 CACHE = {} -def get_model(model_name='Standard', device='cpu', model_dir='./models'): - model = load_model(model_name, model_dir=model_dir, norm='Linf') - # fsize = os.path.getsize(filePath) - return model.to(device) +def get_model(device='cpu'): + # load checkpoint. + print(os.getcwd()) + checkpoint = torch.load('./code_coverage/resnet18_eval.pth', + map_location=torch.device(device)) + net = ResNet18().to(device) + net.load_state_dict(checkpoint['net']) + return net.to(device) -def get_data(data_name='CIFAR10', device='cpu', n_examples=5, data_dir='./data'): - images, labels = load_cifar10(n_examples=n_examples, data_dir=data_dir) +def get_data(device='cpu'): + images = torch.load('./code_coverage/images.pth') # 10 images + labels = torch.load('./code_coverage/labels.pth') # 10 images return images.to(device), labels.to(device) +def clean_accuracy(model, images, labels): + model.eval() + pred = torch.argmax(model(images), dim=1) + correct = torch.sum(labels == pred) + total = images.shape[0] + return correct / total + + @torch.no_grad() @pytest.mark.parametrize('atk_class', [atk_class for atk_class in torchattacks.__all__ if atk_class not in torchattacks.__wrapper__]) -def test_atks_on_cifar10(atk_class, device='cpu', n_examples=5, model_dir='./models', data_dir='./data'): +def test_atks_on_cifar10(atk_class, device='cpu'): global CACHE if CACHE.get('model') is None: - model = get_model(device=device, model_dir=model_dir) + model = get_model(device=device) CACHE['model'] = model else: model = CACHE['model'] - if CACHE.get('images') is None: - images, labels = get_data( - device=device, n_examples=n_examples, data_dir=data_dir) + if CACHE.get('images') is None or CACHE.get('labels') is None: + images, labels = get_data() CACHE['images'] = images CACHE['labels'] = labels else: @@ -51,32 +63,38 @@ def test_atks_on_cifar10(atk_class, device='cpu', n_examples=5, model_dir='./mod else: clean_acc = CACHE['clean_acc'] - try: - kargs = {} - if atk_class in ['SPSA']: - kargs['max_batch_size'] = 5 - atk = eval("torchattacks."+atk_class)(model, **kargs) + kargs = {} + if atk_class in ['SPSA']: + kargs['max_batch_size'] = 5 + + atk = eval("torchattacks."+atk_class)(model, **kargs) + start = time.time() + with torch.enable_grad(): + adv_images = atk(images, labels) + + # non-targeted attack test + robust_acc_1 = clean_accuracy(model, adv_images, labels) + assert clean_acc >= robust_acc_1 + end = time.time() + + sec = float(end - start) + print('{0:<12}: clean_acc={1:2.2f} robust_acc={2:2.2f} sec={3:2.2f}'.format( + atk_class, clean_acc, robust_acc_1, sec)) + + # targeted attack test + start = time.time() + if 'targeted' in atk.supported_mode: + atk.set_mode_targeted_random(quiet=True) start = time.time() with torch.enable_grad(): adv_images = atk(images, labels) end = time.time() - robust_acc = clean_accuracy(model, adv_images, labels) - sec = float(end - start) - print('{0:<12}: clean_acc={1:2.2f} robust_acc={2:2.2f} sec={3:2.2f}'.format( - atk_class, clean_acc, robust_acc, sec)) - - if 'targeted' in atk.supported_mode: - atk.set_mode_targeted_random(quiet=True) - with torch.enable_grad(): - adv_images = atk(images, labels) - robust_acc = clean_accuracy(model, adv_images, labels) - sec = float(end - start) - print('{0:<12}: clean_acc={1:2.2f} robust_acc={2:2.2f} sec={3:2.2f}'.format( - "- targeted", clean_acc, robust_acc, sec)) - - except Exception as e: - robust_acc = clean_acc + 1 # It will cuase assertion. - print('{0:<12} test acc Error'.format(atk_class)) - print(e) - - assert clean_acc >= robust_acc + robust_acc_2 = clean_accuracy(model, adv_images, labels) + else: + robust_acc_2 = 0 + assert clean_acc >= robust_acc_2 + end = time.time() + + sec = float(end - start) + print('{0:<12}: clean_acc={1:2.2f} robust_acc={2:2.2f} sec={3:2.2f}'.format( + "- targeted", clean_acc, robust_acc_2, sec)) diff --git a/demo/White-box Targeted Attack on CIFAR10.ipynb b/demo/White-box Targeted Attack on CIFAR10.ipynb index 0c6542bc..55475990 100644 --- a/demo/White-box Targeted Attack on CIFAR10.ipynb +++ b/demo/White-box Targeted Attack on CIFAR10.ipynb @@ -71,28 +71,9 @@ "source": [ "from torchattacks import PGD\n", "atk = PGD(model, eps=8/255, alpha=2/225, steps=10, random_start=True)\n", - "atk.set_mode_targeted_by_label()\n", - "print(atk)" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "tensor([3, 8, 8, 0, 6])\n", - "tensor([4, 9, 9, 1, 7])\n" - ] - } - ], - "source": [ - "print(labels)\n", "new_labels = (labels + 1) % 10\n", - "print(new_labels)" + "atk.set_mode_targeted_by_label(target_labels=new_labels)\n", + "print(atk)" ] }, { @@ -111,7 +92,7 @@ } ], "source": [ - "adv_images = atk(images, new_labels)\n", + "adv_images = atk(images, labels)\n", "adv_pred = model(adv_images)\n", "print(labels)\n", "print(torch.argmax(adv_pred, 1))" diff --git a/requirements.txt b/requirements.txt index 96c1582e..c413a9b7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,4 +6,5 @@ scipy>=0.14.0 tqdm>=4.56.1 requests>=2.25.1 pandas>=1.2.4 -numpy>=1.19.4 +# The last stable 1.x version of numpy +numpy<=1.26.4 diff --git a/robustbench/README.md b/robustbench/README.md deleted file mode 100644 index 7d38a0f3..00000000 --- a/robustbench/README.md +++ /dev/null @@ -1,11 +0,0 @@ -Please refer to [https://github.com/RobustBench/robustbench](https://github.com/RobustBench/robustbench) for the original version. - -For faster coverage computation, remove some modules. - - * Remove timm. - - * Remove sodef_layers. - - * Remove xcit. - - * Disable eval.py for removing autoattack. \ No newline at end of file diff --git a/robustbench/__init__.py b/robustbench/__init__.py deleted file mode 100644 index f685f04a..00000000 --- a/robustbench/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .data import load_cifar10 -from .utils import load_model -# from .eval import benchmark diff --git a/robustbench/data.py b/robustbench/data.py deleted file mode 100644 index 5b777c8e..00000000 --- a/robustbench/data.py +++ /dev/null @@ -1,315 +0,0 @@ -from typing import Callable, Union -import os -from pathlib import Path -from typing import Callable, Dict, Optional, Sequence, Set, Tuple - -import numpy as np -import torch -import torch.utils.data as data -import torchvision.datasets as datasets -import torchvision.transforms as transforms -from torch.utils.data import Dataset - -from robustbench.model_zoo import model_dicts as all_models -from robustbench.model_zoo.enums import BenchmarkDataset, ThreatModel -from robustbench.zenodo_download import DownloadError, zenodo_download -from robustbench.loaders import CustomImageFolder - -PREPROCESSINGS = { - 'Res256Crop224': - transforms.Compose([ - transforms.Resize(256), - transforms.CenterCrop(224), - transforms.ToTensor() - ]), - 'Crop288': - transforms.Compose([transforms.CenterCrop(288), - transforms.ToTensor()]), - None: - transforms.Compose([transforms.ToTensor()]), -} - - -def get_preprocessing( - dataset: BenchmarkDataset, threat_model: ThreatModel, - model_name: Optional[str], - preprocessing: Optional[Union[str, Callable]]) -> Callable: - if isinstance(preprocessing, Callable): - return preprocessing - - if dataset == BenchmarkDataset.imagenet: - if model_name is not None and model_name in all_models[dataset][ - threat_model]: - prepr = all_models[dataset][threat_model][model_name][ - 'preprocessing'] - elif preprocessing is not None: - prepr = preprocessing - else: - raise Exception( - "Preprocessing should be specified if the model is not already in the model zoo" - ) - else: - prepr = None - - return PREPROCESSINGS[prepr] - - -def _load_dataset( - dataset: Dataset, - n_examples: Optional[int] = None) -> Tuple[torch.Tensor, torch.Tensor]: - batch_size = 100 - test_loader = data.DataLoader(dataset, - batch_size=batch_size, - shuffle=False, - num_workers=0) - - x_test, y_test = [], [] - for i, (x, y) in enumerate(test_loader): - x_test.append(x) - y_test.append(y) - if n_examples is not None and batch_size * i >= n_examples: - break - x_test_tensor = torch.cat(x_test) - y_test_tensor = torch.cat(y_test) - - if n_examples is not None: - x_test_tensor = x_test_tensor[:n_examples] - y_test_tensor = y_test_tensor[:n_examples] - - return x_test_tensor, y_test_tensor - - -def load_cifar10( - n_examples: Optional[int] = None, - data_dir: str = './data', - transforms_test: Callable = PREPROCESSINGS[None] -) -> Tuple[torch.Tensor, torch.Tensor]: - dataset = datasets.CIFAR10(root=data_dir, - train=False, - transform=transforms_test, - download=True) - return _load_dataset(dataset, n_examples) - - -def load_cifar100( - n_examples: Optional[int] = None, - data_dir: str = './data', - transforms_test: Callable = PREPROCESSINGS[None] -) -> Tuple[torch.Tensor, torch.Tensor]: - dataset = datasets.CIFAR100(root=data_dir, - train=False, - transform=transforms_test, - download=True) - return _load_dataset(dataset, n_examples) - - -def load_imagenet( - n_examples: Optional[int] = 5000, - data_dir: str = './data', - transforms_test: Callable = PREPROCESSINGS['Res256Crop224'] -) -> Tuple[torch.Tensor, torch.Tensor]: - if n_examples > 5000: - raise ValueError( - 'The evaluation is currently possible on at most 5000 points-') - - imagenet = CustomImageFolder(data_dir + '/val', transforms_test) - - test_loader = data.DataLoader(imagenet, - batch_size=n_examples, - shuffle=False, - num_workers=4) - - x_test, y_test, paths = next(iter(test_loader)) - - return x_test, y_test - - -CleanDatasetLoader = Callable[[Optional[int], str, Callable], - Tuple[torch.Tensor, torch.Tensor]] -_clean_dataset_loaders: Dict[BenchmarkDataset, CleanDatasetLoader] = { - BenchmarkDataset.cifar_10: load_cifar10, - BenchmarkDataset.cifar_100: load_cifar100, - BenchmarkDataset.imagenet: load_imagenet, -} - - -def load_clean_dataset(dataset: BenchmarkDataset, n_examples: Optional[int], - data_dir: str, - prepr: Callable) -> Tuple[torch.Tensor, torch.Tensor]: - return _clean_dataset_loaders[dataset](n_examples, data_dir, prepr) - - -CORRUPTIONS = ("shot_noise", "motion_blur", "snow", "pixelate", - "gaussian_noise", "defocus_blur", "brightness", "fog", - "zoom_blur", "frost", "glass_blur", "impulse_noise", "contrast", - "jpeg_compression", "elastic_transform") - -CORRUPTIONS_3DCC = ('near_focus', 'far_focus', 'bit_error', 'color_quant', - 'flash', 'fog_3d', 'h265_abr', 'h265_crf', 'iso_noise', - 'low_light', 'xy_motion_blur', 'z_motion_blur') - -ZENODO_CORRUPTIONS_LINKS: Dict[BenchmarkDataset, Tuple[str, Set[str]]] = { - BenchmarkDataset.cifar_10: ("2535967", {"CIFAR-10-C.tar"}), - BenchmarkDataset.cifar_100: ("3555552", {"CIFAR-100-C.tar"}) -} - -CORRUPTIONS_DIR_NAMES: Dict[BenchmarkDataset, str] = { - BenchmarkDataset.cifar_10: "CIFAR-10-C", - BenchmarkDataset.cifar_100: "CIFAR-100-C", - BenchmarkDataset.imagenet: "ImageNet-C", - # BenchmarkDataset.imagenet_3d: "ImageNet-3DCC" -} - - -def load_cifar10c( - n_examples: int, - severity: int = 5, - data_dir: str = './data', - shuffle: bool = False, - corruptions: Sequence[str] = CORRUPTIONS, - _: Callable = PREPROCESSINGS[None] -) -> Tuple[torch.Tensor, torch.Tensor]: - return load_corruptions_cifar(BenchmarkDataset.cifar_10, n_examples, - severity, data_dir, corruptions, shuffle) - - -def load_cifar100c( - n_examples: int, - severity: int = 5, - data_dir: str = './data', - shuffle: bool = False, - corruptions: Sequence[str] = CORRUPTIONS, - _: Callable = PREPROCESSINGS[None]) -> Tuple[torch.Tensor, torch.Tensor]: - return load_corruptions_cifar(BenchmarkDataset.cifar_100, n_examples, - severity, data_dir, corruptions, shuffle) - - -def load_imagenetc( - n_examples: Optional[int] = 5000, - severity: int = 5, - data_dir: str = './data', - shuffle: bool = False, - corruptions: Sequence[str] = CORRUPTIONS, - prepr: Callable = PREPROCESSINGS[None] -) -> Tuple[torch.Tensor, torch.Tensor]: - if n_examples > 5000: - raise ValueError( - 'The evaluation is currently possible on at most 5000 points.') - - assert len( - corruptions - ) == 1, "so far only one corruption is supported (that's how this function is called in eval.py" - # TODO: generalize this (although this would probably require writing a function similar to `load_corruptions_cifar` - # or alternatively creating yet another CustomImageFolder class that fetches images from multiple corruption types - # at once -- perhaps this is a cleaner solution) - - data_folder_path = Path(data_dir) / CORRUPTIONS_DIR_NAMES[ - BenchmarkDataset.imagenet] / corruptions[0] / str(severity) - imagenet = CustomImageFolder(data_folder_path, prepr) - test_loader = data.DataLoader(imagenet, - batch_size=n_examples, - shuffle=shuffle, - num_workers=2) - - x_test, y_test, paths = next(iter(test_loader)) - - return x_test, y_test - - -# def load_imagenet3dcc( -# n_examples: Optional[int] = 5000, -# severity: int = 5, -# data_dir: str = './data', -# shuffle: bool = False, -# corruptions: Sequence[str] = CORRUPTIONS_3DCC, -# prepr: Callable = PREPROCESSINGS[None] -# ) -> Tuple[torch.Tensor, torch.Tensor]: -# if n_examples > 5000: -# raise ValueError( -# 'The evaluation is currently possible on at most 5000 points.') - -# assert len( -# corruptions -# ) == 1, "so far only one corruption is supported (that's how this function is called in eval.py" -# # TODO: generalize this (although this would probably require writing a function similar to `load_corruptions_cifar` -# # or alternatively creating yet another CustomImageFolder class that fetches images from multiple corruption types -# # at once -- perhaps this is a cleaner solution) - -# data_folder_path = Path(data_dir) / CORRUPTIONS_DIR_NAMES[ -# BenchmarkDataset.imagenet_3d] / corruptions[0] / str(severity) -# imagenet = CustomImageFolder(data_folder_path, prepr) -# test_loader = data.DataLoader(imagenet, -# batch_size=n_examples, -# shuffle=shuffle, -# num_workers=2) - -# x_test, y_test, paths = next(iter(test_loader)) - -# return x_test, y_test - - -CorruptDatasetLoader = Callable[[int, int, str, bool, Sequence[str], Callable], - Tuple[torch.Tensor, torch.Tensor]] -CORRUPTION_DATASET_LOADERS: Dict[BenchmarkDataset, CorruptDatasetLoader] = { - BenchmarkDataset.cifar_10: load_cifar10c, - BenchmarkDataset.cifar_100: load_cifar100c, - BenchmarkDataset.imagenet: load_imagenetc, - # BenchmarkDataset.imagenet_3d: load_imagenet3dcc, -} - - -def load_corruptions_cifar( - dataset: BenchmarkDataset, - n_examples: int, - severity: int, - data_dir: str, - corruptions: Sequence[str] = CORRUPTIONS, - shuffle: bool = False) -> Tuple[torch.Tensor, torch.Tensor]: - assert 1 <= severity <= 5 - n_total_cifar = 10000 - - if not os.path.exists(data_dir): - os.makedirs(data_dir) - - data_dir = Path(data_dir) - data_root_dir = data_dir / CORRUPTIONS_DIR_NAMES[dataset] - - if not data_root_dir.exists(): - zenodo_download(*ZENODO_CORRUPTIONS_LINKS[dataset], save_dir=data_dir) - - # Download labels - labels_path = data_root_dir / 'labels.npy' - if not os.path.isfile(labels_path): - raise DownloadError("Labels are missing, try to re-download them.") - labels = np.load(labels_path) - - x_test_list, y_test_list = [], [] - n_pert = len(corruptions) - for corruption in corruptions: - corruption_file_path = data_root_dir / (corruption + '.npy') - if not corruption_file_path.is_file(): - raise DownloadError( - f"{corruption} file is missing, try to re-download it.") - - images_all = np.load(corruption_file_path) - images = images_all[(severity - 1) * n_total_cifar:severity * - n_total_cifar] - n_img = int(np.ceil(n_examples / n_pert)) - x_test_list.append(images[:n_img]) - # Duplicate the same labels potentially multiple times - y_test_list.append(labels[:n_img]) - - x_test, y_test = np.concatenate(x_test_list), np.concatenate(y_test_list) - if shuffle: - rand_idx = np.random.permutation(np.arange(len(x_test))) - x_test, y_test = x_test[rand_idx], y_test[rand_idx] - - # Make it in the PyTorch format - x_test = np.transpose(x_test, (0, 3, 1, 2)) - # Make it compatible with our models - x_test = x_test.astype(np.float32) / 255 - # Make sure that we get exactly n_examples but not a few samples more - x_test = torch.tensor(x_test)[:n_examples] - y_test = torch.tensor(y_test)[:n_examples] - - return x_test, y_test diff --git a/robustbench/eval.py b/robustbench/eval.py deleted file mode 100644 index 7b34952f..00000000 --- a/robustbench/eval.py +++ /dev/null @@ -1,223 +0,0 @@ -# import warnings -# from argparse import Namespace -# from pathlib import Path -# from typing import Callable, Dict, Optional, Sequence, Tuple, Union - -# import numpy as np -# import pandas as pd -# import torch -# import random -# from autoattack import AutoAttack -# from torch import nn -# from tqdm import tqdm - -# from robustbench.data import CORRUPTIONS, get_preprocessing, load_clean_dataset, \ -# CORRUPTION_DATASET_LOADERS -# from robustbench.model_zoo.enums import BenchmarkDataset, ThreatModel -# from robustbench.utils import clean_accuracy, load_model, parse_args, update_json -# from robustbench.model_zoo import model_dicts as all_models - - -# def benchmark( -# model: Union[nn.Module, Sequence[nn.Module]], -# n_examples: int = 10000, -# dataset: Union[str, BenchmarkDataset] = BenchmarkDataset.cifar_10, -# threat_model: Union[str, ThreatModel] = ThreatModel.Linf, -# to_disk: bool = False, -# model_name: Optional[str] = None, -# data_dir: str = "./data", -# device: Optional[Union[torch.device, Sequence[torch.device]]] = None, -# batch_size: int = 32, -# eps: Optional[float] = None, -# log_path: Optional[str] = None, -# preprocessing: Optional[Union[str, -# Callable]] = None) -> Tuple[float, float]: -# """Benchmarks the given model(s). - -# It is possible to benchmark on 3 different threat models, and to save the results on disk. In -# the future benchmarking multiple models in parallel is going to be possible. - -# :param model: The model to benchmark. -# :param n_examples: The number of examples to use to benchmark the model. -# :param dataset: The dataset to use to benchmark. Must be one of {cifar10, cifar100} -# :param threat_model: The threat model to use to benchmark, must be one of {L2, Linf -# corruptions} -# :param to_disk: Whether the results must be saved on disk as .json. -# :param model_name: The name of the model to use to save the results. Must be specified if -# to_json is True. -# :param data_dir: The directory where the dataset is or where the dataset must be downloaded. -# :param device: The device to run the computations. -# :param batch_size: The batch size to run the computations. The larger, the faster the -# evaluation. -# :param eps: The epsilon to use for L2 and Linf threat models. Must not be specified for -# corruptions threat model. -# :param preprocessing: The preprocessing that should be used for ImageNet benchmarking. Should be -# specified if `dataset` is `imageget`. - -# :return: A Tuple with the clean accuracy and the accuracy in the given threat model. -# """ -# if isinstance(model, Sequence) or isinstance(device, Sequence): -# # Multiple models evaluation in parallel not yet implemented -# raise NotImplementedError - -# try: -# if model.training: -# warnings.warn(Warning("The given model is *not* in eval mode.")) -# except AttributeError: -# warnings.warn( -# Warning( -# "It is not possible to asses if the model is in eval mode")) - -# dataset_: BenchmarkDataset = BenchmarkDataset(dataset) -# threat_model_: ThreatModel = ThreatModel(threat_model) - -# device = device or torch.device("cpu") -# model = model.to(device) - -# prepr = get_preprocessing(dataset_, threat_model_, model_name, -# preprocessing) - -# clean_x_test, clean_y_test = load_clean_dataset(dataset_, n_examples, -# data_dir, prepr) - -# accuracy = clean_accuracy(model, -# clean_x_test, -# clean_y_test, -# batch_size=batch_size, -# device=device) -# print(f'Clean accuracy: {accuracy:.2%}') - -# if threat_model_ in {ThreatModel.Linf, ThreatModel.L2}: -# if eps is None: -# raise ValueError( -# "If the threat model is L2 or Linf, `eps` must be specified.") - -# adversary = AutoAttack(model, -# norm=threat_model_.value, -# eps=eps, -# version='standard', -# device=device, -# log_path=log_path) -# x_adv = adversary.run_standard_evaluation(clean_x_test, -# clean_y_test, -# bs=batch_size) -# adv_accuracy = clean_accuracy(model, -# x_adv, -# clean_y_test, -# batch_size=batch_size, -# device=device) -# elif threat_model_ == ThreatModel.corruptions: -# corruptions = CORRUPTIONS -# print(f"Evaluating over {len(corruptions)} corruptions") -# # Save into a dict to make a Pandas DF with nested index -# adv_accuracy = corruptions_evaluation(batch_size, data_dir, dataset_, -# device, model, n_examples, -# to_disk, prepr, model_name) -# else: -# raise NotImplementedError -# print(f'Adversarial accuracy: {adv_accuracy:.2%}') - -# if to_disk: -# if model_name is None: -# raise ValueError( -# "If `to_disk` is True, `model_name` should be specified.") - -# update_json(dataset_, threat_model_, model_name, accuracy, -# adv_accuracy, eps) - -# return accuracy, adv_accuracy - - -# def corruptions_evaluation(batch_size: int, data_dir: str, -# dataset: BenchmarkDataset, device: torch.device, -# model: nn.Module, n_examples: int, to_disk: bool, -# prepr: str, model_name: Optional[str]) -> float: -# if to_disk and model_name is None: -# raise ValueError( -# "If `to_disk` is True, `model_name` should be specified.") - -# corruptions = CORRUPTIONS -# model_results_dict: Dict[Tuple[str, int], float] = {} -# for corruption in tqdm(corruptions): -# for severity in range(1, 6): -# x_corrupt, y_corrupt = CORRUPTION_DATASET_LOADERS[dataset]( -# n_examples, -# severity, -# data_dir, -# shuffle=False, -# corruptions=[corruption], -# prepr=prepr) - -# corruption_severity_accuracy = clean_accuracy( -# model, -# x_corrupt, -# y_corrupt, -# batch_size=batch_size, -# device=device) -# print('corruption={}, severity={}: {:.2%} accuracy'.format( -# corruption, severity, corruption_severity_accuracy)) - -# model_results_dict[(corruption, -# severity)] = corruption_severity_accuracy - -# model_results = pd.DataFrame(model_results_dict, index=[model_name]) -# adv_accuracy = model_results.values.mean() - -# if not to_disk: -# return adv_accuracy - -# # Save disaggregated results on disk -# existing_results_path = Path( -# "model_info" -# ) / dataset.value / "corruptions" / "unaggregated_results.csv" -# if not existing_results_path.parent.exists(): -# existing_results_path.parent.mkdir(parents=True, exist_ok=True) -# try: -# existing_results = pd.read_csv(existing_results_path, -# header=[0, 1], -# index_col=0) -# existing_results.columns = existing_results.columns.set_levels([ -# existing_results.columns.levels[0], -# existing_results.columns.levels[1].astype(int) -# ]) -# full_results = pd.concat([existing_results, model_results]) -# except FileNotFoundError: -# full_results = model_results -# full_results.to_csv(existing_results_path) - -# return adv_accuracy - - -# def main(args: Namespace) -> None: -# torch.manual_seed(args.seed) -# torch.cuda.manual_seed(args.seed) -# np.random.seed(args.seed) -# random.seed(args.seed) - -# model = load_model(args.model_name, -# model_dir=args.model_dir, -# dataset=args.dataset, -# threat_model=args.threat_model) - -# model.eval() - -# device = torch.device(args.device) -# benchmark(model, -# n_examples=args.n_ex, -# dataset=args.dataset, -# threat_model=args.threat_model, -# to_disk=args.to_disk, -# model_name=args.model_name, -# data_dir=args.data_dir, -# device=device, -# batch_size=args.batch_size, -# eps=args.eps) - - -# if __name__ == '__main__': -# # Example: -# # python -m robustbench.eval --n_ex=5000 --dataset=imagenet --threat_model=Linf \ -# # --model_name=Salman2020Do_R18 --data_dir=/tmldata1/andriush/imagenet/val \ -# # --batch_size=128 --eps=0.0156862745 -# args_ = parse_args() -# main(args_) diff --git a/robustbench/helper_files/imagenet_class_to_id_map.json b/robustbench/helper_files/imagenet_class_to_id_map.json deleted file mode 100644 index bf064b3b..00000000 --- a/robustbench/helper_files/imagenet_class_to_id_map.json +++ /dev/null @@ -1 +0,0 @@ -{"n01440764": 0, "n01443537": 1, "n01484850": 2, "n01491361": 3, "n01494475": 4, "n01496331": 5, "n01498041": 6, "n01514668": 7, "n01514859": 8, "n01518878": 9, "n01530575": 10, "n01531178": 11, "n01532829": 12, "n01534433": 13, "n01537544": 14, "n01558993": 15, "n01560419": 16, "n01580077": 17, "n01582220": 18, "n01592084": 19, "n01601694": 20, "n01608432": 21, "n01614925": 22, "n01616318": 23, "n01622779": 24, "n01629819": 25, "n01630670": 26, "n01631663": 27, "n01632458": 28, "n01632777": 29, "n01641577": 30, "n01644373": 31, "n01644900": 32, "n01664065": 33, "n01665541": 34, "n01667114": 35, "n01667778": 36, "n01669191": 37, "n01675722": 38, "n01677366": 39, "n01682714": 40, "n01685808": 41, "n01687978": 42, "n01688243": 43, "n01689811": 44, "n01692333": 45, "n01693334": 46, "n01694178": 47, "n01695060": 48, "n01697457": 49, "n01698640": 50, "n01704323": 51, "n01728572": 52, "n01728920": 53, "n01729322": 54, "n01729977": 55, "n01734418": 56, "n01735189": 57, "n01737021": 58, "n01739381": 59, "n01740131": 60, "n01742172": 61, "n01744401": 62, "n01748264": 63, "n01749939": 64, "n01751748": 65, "n01753488": 66, "n01755581": 67, "n01756291": 68, "n01768244": 69, "n01770081": 70, "n01770393": 71, "n01773157": 72, "n01773549": 73, "n01773797": 74, "n01774384": 75, "n01774750": 76, "n01775062": 77, "n01776313": 78, "n01784675": 79, "n01795545": 80, "n01796340": 81, "n01797886": 82, "n01798484": 83, "n01806143": 84, "n01806567": 85, "n01807496": 86, "n01817953": 87, "n01818515": 88, "n01819313": 89, "n01820546": 90, "n01824575": 91, "n01828970": 92, "n01829413": 93, "n01833805": 94, "n01843065": 95, "n01843383": 96, "n01847000": 97, "n01855032": 98, "n01855672": 99, "n01860187": 100, "n01871265": 101, "n01872401": 102, "n01873310": 103, "n01877812": 104, "n01882714": 105, "n01883070": 106, "n01910747": 107, "n01914609": 108, "n01917289": 109, "n01924916": 110, "n01930112": 111, "n01943899": 112, "n01944390": 113, "n01945685": 114, "n01950731": 115, "n01955084": 116, "n01968897": 117, "n01978287": 118, "n01978455": 119, "n01980166": 120, "n01981276": 121, "n01983481": 122, "n01984695": 123, "n01985128": 124, "n01986214": 125, "n01990800": 126, "n02002556": 127, "n02002724": 128, "n02006656": 129, "n02007558": 130, "n02009229": 131, "n02009912": 132, "n02011460": 133, "n02012849": 134, "n02013706": 135, "n02017213": 136, "n02018207": 137, "n02018795": 138, "n02025239": 139, "n02027492": 140, "n02028035": 141, "n02033041": 142, "n02037110": 143, "n02051845": 144, "n02056570": 145, "n02058221": 146, "n02066245": 147, "n02071294": 148, "n02074367": 149, "n02077923": 150, "n02085620": 151, "n02085782": 152, "n02085936": 153, "n02086079": 154, "n02086240": 155, "n02086646": 156, "n02086910": 157, "n02087046": 158, "n02087394": 159, "n02088094": 160, "n02088238": 161, "n02088364": 162, "n02088466": 163, "n02088632": 164, "n02089078": 165, "n02089867": 166, "n02089973": 167, "n02090379": 168, "n02090622": 169, "n02090721": 170, "n02091032": 171, "n02091134": 172, "n02091244": 173, "n02091467": 174, "n02091635": 175, "n02091831": 176, "n02092002": 177, "n02092339": 178, "n02093256": 179, "n02093428": 180, "n02093647": 181, "n02093754": 182, "n02093859": 183, "n02093991": 184, "n02094114": 185, "n02094258": 186, "n02094433": 187, "n02095314": 188, "n02095570": 189, "n02095889": 190, "n02096051": 191, "n02096177": 192, "n02096294": 193, "n02096437": 194, "n02096585": 195, "n02097047": 196, "n02097130": 197, "n02097209": 198, "n02097298": 199, "n02097474": 200, "n02097658": 201, "n02098105": 202, "n02098286": 203, "n02098413": 204, "n02099267": 205, "n02099429": 206, "n02099601": 207, "n02099712": 208, "n02099849": 209, "n02100236": 210, "n02100583": 211, "n02100735": 212, "n02100877": 213, "n02101006": 214, "n02101388": 215, "n02101556": 216, "n02102040": 217, "n02102177": 218, "n02102318": 219, "n02102480": 220, "n02102973": 221, "n02104029": 222, "n02104365": 223, "n02105056": 224, "n02105162": 225, "n02105251": 226, "n02105412": 227, "n02105505": 228, "n02105641": 229, "n02105855": 230, "n02106030": 231, "n02106166": 232, "n02106382": 233, "n02106550": 234, "n02106662": 235, "n02107142": 236, "n02107312": 237, "n02107574": 238, "n02107683": 239, "n02107908": 240, "n02108000": 241, "n02108089": 242, "n02108422": 243, "n02108551": 244, "n02108915": 245, "n02109047": 246, "n02109525": 247, "n02109961": 248, "n02110063": 249, "n02110185": 250, "n02110341": 251, "n02110627": 252, "n02110806": 253, "n02110958": 254, "n02111129": 255, "n02111277": 256, "n02111500": 257, "n02111889": 258, "n02112018": 259, "n02112137": 260, "n02112350": 261, "n02112706": 262, "n02113023": 263, "n02113186": 264, "n02113624": 265, "n02113712": 266, "n02113799": 267, "n02113978": 268, "n02114367": 269, "n02114548": 270, "n02114712": 271, "n02114855": 272, "n02115641": 273, "n02115913": 274, "n02116738": 275, "n02117135": 276, "n02119022": 277, "n02119789": 278, "n02120079": 279, "n02120505": 280, "n02123045": 281, "n02123159": 282, "n02123394": 283, "n02123597": 284, "n02124075": 285, "n02125311": 286, "n02127052": 287, "n02128385": 288, "n02128757": 289, "n02128925": 290, "n02129165": 291, "n02129604": 292, "n02130308": 293, "n02132136": 294, "n02133161": 295, "n02134084": 296, "n02134418": 297, "n02137549": 298, "n02138441": 299, "n02165105": 300, "n02165456": 301, "n02167151": 302, "n02168699": 303, "n02169497": 304, "n02172182": 305, "n02174001": 306, "n02177972": 307, "n02190166": 308, "n02206856": 309, "n02219486": 310, "n02226429": 311, "n02229544": 312, "n02231487": 313, "n02233338": 314, "n02236044": 315, "n02256656": 316, "n02259212": 317, "n02264363": 318, "n02268443": 319, "n02268853": 320, "n02276258": 321, "n02277742": 322, "n02279972": 323, "n02280649": 324, "n02281406": 325, "n02281787": 326, "n02317335": 327, "n02319095": 328, "n02321529": 329, "n02325366": 330, "n02326432": 331, "n02328150": 332, "n02342885": 333, "n02346627": 334, "n02356798": 335, "n02361337": 336, "n02363005": 337, "n02364673": 338, "n02389026": 339, "n02391049": 340, "n02395406": 341, "n02396427": 342, "n02397096": 343, "n02398521": 344, "n02403003": 345, "n02408429": 346, "n02410509": 347, "n02412080": 348, "n02415577": 349, "n02417914": 350, "n02422106": 351, "n02422699": 352, "n02423022": 353, "n02437312": 354, "n02437616": 355, "n02441942": 356, "n02442845": 357, "n02443114": 358, "n02443484": 359, "n02444819": 360, "n02445715": 361, "n02447366": 362, "n02454379": 363, "n02457408": 364, "n02480495": 365, "n02480855": 366, "n02481823": 367, "n02483362": 368, "n02483708": 369, "n02484975": 370, "n02486261": 371, "n02486410": 372, "n02487347": 373, "n02488291": 374, "n02488702": 375, "n02489166": 376, "n02490219": 377, "n02492035": 378, "n02492660": 379, "n02493509": 380, "n02493793": 381, "n02494079": 382, "n02497673": 383, "n02500267": 384, "n02504013": 385, "n02504458": 386, "n02509815": 387, "n02510455": 388, "n02514041": 389, "n02526121": 390, "n02536864": 391, "n02606052": 392, "n02607072": 393, "n02640242": 394, "n02641379": 395, "n02643566": 396, "n02655020": 397, "n02666196": 398, "n02667093": 399, "n02669723": 400, "n02672831": 401, "n02676566": 402, "n02687172": 403, "n02690373": 404, "n02692877": 405, "n02699494": 406, "n02701002": 407, "n02704792": 408, "n02708093": 409, "n02727426": 410, "n02730930": 411, "n02747177": 412, "n02749479": 413, "n02769748": 414, "n02776631": 415, "n02777292": 416, "n02782093": 417, "n02783161": 418, "n02786058": 419, "n02787622": 420, "n02788148": 421, "n02790996": 422, "n02791124": 423, "n02791270": 424, "n02793495": 425, "n02794156": 426, "n02795169": 427, "n02797295": 428, "n02799071": 429, "n02802426": 430, "n02804414": 431, "n02804610": 432, "n02807133": 433, "n02808304": 434, "n02808440": 435, "n02814533": 436, "n02814860": 437, "n02815834": 438, "n02817516": 439, "n02823428": 440, "n02823750": 441, "n02825657": 442, "n02834397": 443, "n02835271": 444, "n02837789": 445, "n02840245": 446, "n02841315": 447, "n02843684": 448, "n02859443": 449, "n02860847": 450, "n02865351": 451, "n02869837": 452, "n02870880": 453, "n02871525": 454, "n02877765": 455, "n02879718": 456, "n02883205": 457, "n02892201": 458, "n02892767": 459, "n02894605": 460, "n02895154": 461, "n02906734": 462, "n02909870": 463, "n02910353": 464, "n02916936": 465, "n02917067": 466, "n02927161": 467, "n02930766": 468, "n02939185": 469, "n02948072": 470, "n02950826": 471, "n02951358": 472, "n02951585": 473, "n02963159": 474, "n02965783": 475, "n02966193": 476, "n02966687": 477, "n02971356": 478, "n02974003": 479, "n02977058": 480, "n02978881": 481, "n02979186": 482, "n02980441": 483, "n02981792": 484, "n02988304": 485, "n02992211": 486, "n02992529": 487, "n02999410": 488, "n03000134": 489, "n03000247": 490, "n03000684": 491, "n03014705": 492, "n03016953": 493, "n03017168": 494, "n03018349": 495, "n03026506": 496, "n03028079": 497, "n03032252": 498, "n03041632": 499, "n03042490": 500, "n03045698": 501, "n03047690": 502, "n03062245": 503, "n03063599": 504, "n03063689": 505, "n03065424": 506, "n03075370": 507, "n03085013": 508, "n03089624": 509, "n03095699": 510, "n03100240": 511, "n03109150": 512, "n03110669": 513, "n03124043": 514, "n03124170": 515, "n03125729": 516, "n03126707": 517, "n03127747": 518, "n03127925": 519, "n03131574": 520, "n03133878": 521, "n03134739": 522, "n03141823": 523, "n03146219": 524, "n03160309": 525, "n03179701": 526, "n03180011": 527, "n03187595": 528, "n03188531": 529, "n03196217": 530, "n03197337": 531, "n03201208": 532, "n03207743": 533, "n03207941": 534, "n03208938": 535, "n03216828": 536, "n03218198": 537, "n03220513": 538, "n03223299": 539, "n03240683": 540, "n03249569": 541, "n03250847": 542, "n03255030": 543, "n03259280": 544, "n03271574": 545, "n03272010": 546, "n03272562": 547, "n03290653": 548, "n03291819": 549, "n03297495": 550, "n03314780": 551, "n03325584": 552, "n03337140": 553, "n03344393": 554, "n03345487": 555, "n03347037": 556, "n03355925": 557, "n03372029": 558, "n03376595": 559, "n03379051": 560, "n03384352": 561, "n03388043": 562, "n03388183": 563, "n03388549": 564, "n03393912": 565, "n03394916": 566, "n03400231": 567, "n03404251": 568, "n03417042": 569, "n03424325": 570, "n03425413": 571, "n03443371": 572, "n03444034": 573, "n03445777": 574, "n03445924": 575, "n03447447": 576, "n03447721": 577, "n03450230": 578, "n03452741": 579, "n03457902": 580, "n03459775": 581, "n03461385": 582, "n03467068": 583, "n03476684": 584, "n03476991": 585, "n03478589": 586, "n03481172": 587, "n03482405": 588, "n03483316": 589, "n03485407": 590, "n03485794": 591, "n03492542": 592, "n03494278": 593, "n03495258": 594, "n03496892": 595, "n03498962": 596, "n03527444": 597, "n03529860": 598, "n03530642": 599, "n03532672": 600, "n03534580": 601, "n03535780": 602, "n03538406": 603, "n03544143": 604, "n03584254": 605, "n03584829": 606, "n03590841": 607, "n03594734": 608, "n03594945": 609, "n03595614": 610, "n03598930": 611, "n03599486": 612, "n03602883": 613, "n03617480": 614, "n03623198": 615, "n03627232": 616, "n03630383": 617, "n03633091": 618, "n03637318": 619, "n03642806": 620, "n03649909": 621, "n03657121": 622, "n03658185": 623, "n03661043": 624, "n03662601": 625, "n03666591": 626, "n03670208": 627, "n03673027": 628, "n03676483": 629, "n03680355": 630, "n03690938": 631, "n03691459": 632, "n03692522": 633, "n03697007": 634, "n03706229": 635, "n03709823": 636, "n03710193": 637, "n03710637": 638, "n03710721": 639, "n03717622": 640, "n03720891": 641, "n03721384": 642, "n03724870": 643, "n03729826": 644, "n03733131": 645, "n03733281": 646, "n03733805": 647, "n03742115": 648, "n03743016": 649, "n03759954": 650, "n03761084": 651, "n03763968": 652, "n03764736": 653, "n03769881": 654, "n03770439": 655, "n03770679": 656, "n03773504": 657, "n03775071": 658, "n03775546": 659, "n03776460": 660, "n03777568": 661, "n03777754": 662, "n03781244": 663, "n03782006": 664, "n03785016": 665, "n03786901": 666, "n03787032": 667, "n03788195": 668, "n03788365": 669, "n03791053": 670, "n03792782": 671, "n03792972": 672, "n03793489": 673, "n03794056": 674, "n03796401": 675, "n03803284": 676, "n03804744": 677, "n03814639": 678, "n03814906": 679, "n03825788": 680, "n03832673": 681, "n03837869": 682, "n03838899": 683, "n03840681": 684, "n03841143": 685, "n03843555": 686, "n03854065": 687, "n03857828": 688, "n03866082": 689, "n03868242": 690, "n03868863": 691, "n03871628": 692, "n03873416": 693, "n03874293": 694, "n03874599": 695, "n03876231": 696, "n03877472": 697, "n03877845": 698, "n03884397": 699, "n03887697": 700, "n03888257": 701, "n03888605": 702, "n03891251": 703, "n03891332": 704, "n03895866": 705, "n03899768": 706, "n03902125": 707, "n03903868": 708, "n03908618": 709, "n03908714": 710, "n03916031": 711, "n03920288": 712, "n03924679": 713, "n03929660": 714, "n03929855": 715, "n03930313": 716, "n03930630": 717, "n03933933": 718, "n03935335": 719, "n03937543": 720, "n03938244": 721, "n03942813": 722, "n03944341": 723, "n03947888": 724, "n03950228": 725, "n03954731": 726, "n03956157": 727, "n03958227": 728, "n03961711": 729, "n03967562": 730, "n03970156": 731, "n03976467": 732, "n03976657": 733, "n03977966": 734, "n03980874": 735, "n03982430": 736, "n03983396": 737, "n03991062": 738, "n03992509": 739, "n03995372": 740, "n03998194": 741, "n04004767": 742, "n04005630": 743, "n04008634": 744, "n04009552": 745, "n04019541": 746, "n04023962": 747, "n04026417": 748, "n04033901": 749, "n04033995": 750, "n04037443": 751, "n04039381": 752, "n04040759": 753, "n04041544": 754, "n04044716": 755, "n04049303": 756, "n04065272": 757, "n04067472": 758, "n04069434": 759, "n04070727": 760, "n04074963": 761, "n04081281": 762, "n04086273": 763, "n04090263": 764, "n04099969": 765, "n04111531": 766, "n04116512": 767, "n04118538": 768, "n04118776": 769, "n04120489": 770, "n04125021": 771, "n04127249": 772, "n04131690": 773, "n04133789": 774, "n04136333": 775, "n04141076": 776, "n04141327": 777, "n04141975": 778, "n04146614": 779, "n04147183": 780, "n04149813": 781, "n04152593": 782, "n04153751": 783, "n04154565": 784, "n04162706": 785, "n04179913": 786, "n04192698": 787, "n04200800": 788, "n04201297": 789, "n04204238": 790, "n04204347": 791, "n04208210": 792, "n04209133": 793, "n04209239": 794, "n04228054": 795, "n04229816": 796, "n04235860": 797, "n04238763": 798, "n04239074": 799, "n04243546": 800, "n04251144": 801, "n04252077": 802, "n04252225": 803, "n04254120": 804, "n04254680": 805, "n04254777": 806, "n04258138": 807, "n04259630": 808, "n04263257": 809, "n04264628": 810, "n04265275": 811, "n04266014": 812, "n04270147": 813, "n04273569": 814, "n04275548": 815, "n04277352": 816, "n04285008": 817, "n04286575": 818, "n04296562": 819, "n04310018": 820, "n04311004": 821, "n04311174": 822, "n04317175": 823, "n04325704": 824, "n04326547": 825, "n04328186": 826, "n04330267": 827, "n04332243": 828, "n04335435": 829, "n04336792": 830, "n04344873": 831, "n04346328": 832, "n04347754": 833, "n04350905": 834, "n04355338": 835, "n04355933": 836, "n04356056": 837, "n04357314": 838, "n04366367": 839, "n04367480": 840, "n04370456": 841, "n04371430": 842, "n04371774": 843, "n04372370": 844, "n04376876": 845, "n04380533": 846, "n04389033": 847, "n04392985": 848, "n04398044": 849, "n04399382": 850, "n04404412": 851, "n04409515": 852, "n04417672": 853, "n04418357": 854, "n04423845": 855, "n04428191": 856, "n04429376": 857, "n04435653": 858, "n04442312": 859, "n04443257": 860, "n04447861": 861, "n04456115": 862, "n04458633": 863, "n04461696": 864, "n04462240": 865, "n04465501": 866, "n04467665": 867, "n04476259": 868, "n04479046": 869, "n04482393": 870, "n04483307": 871, "n04485082": 872, "n04486054": 873, "n04487081": 874, "n04487394": 875, "n04493381": 876, "n04501370": 877, "n04505470": 878, "n04507155": 879, "n04509417": 880, "n04515003": 881, "n04517823": 882, "n04522168": 883, "n04523525": 884, "n04525038": 885, "n04525305": 886, "n04532106": 887, "n04532670": 888, "n04536866": 889, "n04540053": 890, "n04542943": 891, "n04548280": 892, "n04548362": 893, "n04550184": 894, "n04552348": 895, "n04553703": 896, "n04554684": 897, "n04557648": 898, "n04560804": 899, "n04562935": 900, "n04579145": 901, "n04579432": 902, "n04584207": 903, "n04589890": 904, "n04590129": 905, "n04591157": 906, "n04591713": 907, "n04592741": 908, "n04596742": 909, "n04597913": 910, "n04599235": 911, "n04604644": 912, "n04606251": 913, "n04612504": 914, "n04613696": 915, "n06359193": 916, "n06596364": 917, "n06785654": 918, "n06794110": 919, "n06874185": 920, "n07248320": 921, "n07565083": 922, "n07579787": 923, "n07583066": 924, "n07584110": 925, "n07590611": 926, "n07613480": 927, "n07614500": 928, "n07615774": 929, "n07684084": 930, "n07693725": 931, "n07695742": 932, "n07697313": 933, "n07697537": 934, "n07711569": 935, "n07714571": 936, "n07714990": 937, "n07715103": 938, "n07716358": 939, "n07716906": 940, "n07717410": 941, "n07717556": 942, "n07718472": 943, "n07718747": 944, "n07720875": 945, "n07730033": 946, "n07734744": 947, "n07742313": 948, "n07745940": 949, "n07747607": 950, "n07749582": 951, "n07753113": 952, "n07753275": 953, "n07753592": 954, "n07754684": 955, "n07760859": 956, "n07768694": 957, "n07802026": 958, "n07831146": 959, "n07836838": 960, "n07860988": 961, "n07871810": 962, "n07873807": 963, "n07875152": 964, "n07880968": 965, "n07892512": 966, "n07920052": 967, "n07930864": 968, "n07932039": 969, "n09193705": 970, "n09229709": 971, "n09246464": 972, "n09256479": 973, "n09288635": 974, "n09332890": 975, "n09399592": 976, "n09421951": 977, "n09428293": 978, "n09468604": 979, "n09472597": 980, "n09835506": 981, "n10148035": 982, "n10565667": 983, "n11879895": 984, "n11939491": 985, "n12057211": 986, "n12144580": 987, "n12267677": 988, "n12620546": 989, "n12768682": 990, "n12985857": 991, "n12998815": 992, "n13037406": 993, "n13040303": 994, "n13044778": 995, "n13052670": 996, "n13054560": 997, "n13133613": 998, "n15075141": 999} \ No newline at end of file diff --git a/robustbench/helper_files/imagenet_test_image_ids.txt b/robustbench/helper_files/imagenet_test_image_ids.txt deleted file mode 100644 index cb6494cb..00000000 --- a/robustbench/helper_files/imagenet_test_image_ids.txt +++ /dev/null @@ -1,5000 +0,0 @@ -n09468604/ILSVRC2012_val_00015253.JPEG -n02823750/ILSVRC2012_val_00020685.JPEG -n09472597/ILSVRC2012_val_00039611.JPEG -n03983396/ILSVRC2012_val_00004059.JPEG -n03459775/ILSVRC2012_val_00029201.JPEG -n01770081/ILSVRC2012_val_00035512.JPEG -n03240683/ILSVRC2012_val_00035210.JPEG -n04070727/ILSVRC2012_val_00030280.JPEG -n02948072/ILSVRC2012_val_00014444.JPEG -n03676483/ILSVRC2012_val_00014112.JPEG -n01685808/ILSVRC2012_val_00022003.JPEG -n03814906/ILSVRC2012_val_00033046.JPEG -n04485082/ILSVRC2012_val_00036322.JPEG -n01818515/ILSVRC2012_val_00032473.JPEG -n03534580/ILSVRC2012_val_00001418.JPEG -n02807133/ILSVRC2012_val_00045678.JPEG -n02804610/ILSVRC2012_val_00021821.JPEG -n03000247/ILSVRC2012_val_00033473.JPEG -n01819313/ILSVRC2012_val_00023805.JPEG -n02666196/ILSVRC2012_val_00031659.JPEG -n02093754/ILSVRC2012_val_00024411.JPEG -n02011460/ILSVRC2012_val_00047470.JPEG -n02051845/ILSVRC2012_val_00038981.JPEG -n03792782/ILSVRC2012_val_00018074.JPEG -n04515003/ILSVRC2012_val_00023889.JPEG -n03594734/ILSVRC2012_val_00005849.JPEG -n02108422/ILSVRC2012_val_00042576.JPEG -n03657121/ILSVRC2012_val_00029482.JPEG -n03937543/ILSVRC2012_val_00025358.JPEG -n01749939/ILSVRC2012_val_00042383.JPEG -n02815834/ILSVRC2012_val_00044110.JPEG -n01518878/ILSVRC2012_val_00039355.JPEG -n07873807/ILSVRC2012_val_00026333.JPEG -n01484850/ILSVRC2012_val_00040710.JPEG -n02319095/ILSVRC2012_val_00000142.JPEG -n02120505/ILSVRC2012_val_00043918.JPEG -n02108551/ILSVRC2012_val_00045160.JPEG -n04026417/ILSVRC2012_val_00019537.JPEG -n02708093/ILSVRC2012_val_00044469.JPEG -n02769748/ILSVRC2012_val_00016010.JPEG -n03110669/ILSVRC2012_val_00038881.JPEG -n02480855/ILSVRC2012_val_00042024.JPEG -n02231487/ILSVRC2012_val_00001714.JPEG -n03047690/ILSVRC2012_val_00030355.JPEG -n03729826/ILSVRC2012_val_00023649.JPEG -n03706229/ILSVRC2012_val_00032353.JPEG -n02776631/ILSVRC2012_val_00012943.JPEG -n02087394/ILSVRC2012_val_00037577.JPEG -n04179913/ILSVRC2012_val_00015951.JPEG -n02128925/ILSVRC2012_val_00036139.JPEG -n03791053/ILSVRC2012_val_00010227.JPEG -n01983481/ILSVRC2012_val_00015749.JPEG -n01855672/ILSVRC2012_val_00002997.JPEG -n01695060/ILSVRC2012_val_00030703.JPEG -n03887697/ILSVRC2012_val_00008442.JPEG -n04111531/ILSVRC2012_val_00011779.JPEG -n03706229/ILSVRC2012_val_00005700.JPEG -n01843065/ILSVRC2012_val_00003672.JPEG -n02690373/ILSVRC2012_val_00034342.JPEG -n03444034/ILSVRC2012_val_00010927.JPEG -n01828970/ILSVRC2012_val_00037031.JPEG -n02111277/ILSVRC2012_val_00023569.JPEG -n03954731/ILSVRC2012_val_00037755.JPEG -n04254777/ILSVRC2012_val_00021862.JPEG -n01871265/ILSVRC2012_val_00005860.JPEG -n04008634/ILSVRC2012_val_00009588.JPEG -n03595614/ILSVRC2012_val_00016334.JPEG -n02105162/ILSVRC2012_val_00023289.JPEG -n02105412/ILSVRC2012_val_00027094.JPEG -n03763968/ILSVRC2012_val_00040966.JPEG -n03141823/ILSVRC2012_val_00032252.JPEG -n02445715/ILSVRC2012_val_00004601.JPEG -n02109961/ILSVRC2012_val_00035036.JPEG -n09835506/ILSVRC2012_val_00024079.JPEG -n03956157/ILSVRC2012_val_00000753.JPEG -n02951585/ILSVRC2012_val_00015663.JPEG -n03063689/ILSVRC2012_val_00036221.JPEG -n09288635/ILSVRC2012_val_00032546.JPEG -n02769748/ILSVRC2012_val_00042967.JPEG -n04273569/ILSVRC2012_val_00020353.JPEG -n01860187/ILSVRC2012_val_00007423.JPEG -n07754684/ILSVRC2012_val_00013939.JPEG -n04328186/ILSVRC2012_val_00035731.JPEG -n02980441/ILSVRC2012_val_00018553.JPEG -n02099849/ILSVRC2012_val_00017132.JPEG -n04370456/ILSVRC2012_val_00027573.JPEG -n03595614/ILSVRC2012_val_00025000.JPEG -n03598930/ILSVRC2012_val_00037231.JPEG -n03481172/ILSVRC2012_val_00024678.JPEG -n04553703/ILSVRC2012_val_00037093.JPEG -n03770439/ILSVRC2012_val_00034389.JPEG -n01616318/ILSVRC2012_val_00045589.JPEG -n01669191/ILSVRC2012_val_00027858.JPEG -n03337140/ILSVRC2012_val_00016690.JPEG -n02480495/ILSVRC2012_val_00015704.JPEG -n03026506/ILSVRC2012_val_00040019.JPEG -n04039381/ILSVRC2012_val_00008637.JPEG -n02017213/ILSVRC2012_val_00023773.JPEG -n03444034/ILSVRC2012_val_00005381.JPEG -n04286575/ILSVRC2012_val_00026988.JPEG -n04258138/ILSVRC2012_val_00028650.JPEG -n04376876/ILSVRC2012_val_00037562.JPEG -n03045698/ILSVRC2012_val_00029544.JPEG -n01537544/ILSVRC2012_val_00044540.JPEG -n04136333/ILSVRC2012_val_00018763.JPEG -n07716906/ILSVRC2012_val_00034580.JPEG -n04296562/ILSVRC2012_val_00041663.JPEG -n07697537/ILSVRC2012_val_00040060.JPEG -n01608432/ILSVRC2012_val_00027693.JPEG -n04069434/ILSVRC2012_val_00010481.JPEG -n03347037/ILSVRC2012_val_00001413.JPEG -n02094258/ILSVRC2012_val_00045518.JPEG -n03930630/ILSVRC2012_val_00024317.JPEG -n03417042/ILSVRC2012_val_00006922.JPEG -n09428293/ILSVRC2012_val_00015233.JPEG -n03459775/ILSVRC2012_val_00016371.JPEG -n02025239/ILSVRC2012_val_00034760.JPEG -n03977966/ILSVRC2012_val_00010166.JPEG -n03459775/ILSVRC2012_val_00036180.JPEG -n01742172/ILSVRC2012_val_00008849.JPEG -n06359193/ILSVRC2012_val_00013312.JPEG -n03134739/ILSVRC2012_val_00015656.JPEG -n04435653/ILSVRC2012_val_00022683.JPEG -n02110063/ILSVRC2012_val_00012935.JPEG -n02120079/ILSVRC2012_val_00037005.JPEG -n07871810/ILSVRC2012_val_00014188.JPEG -n02114548/ILSVRC2012_val_00028165.JPEG -n03272010/ILSVRC2012_val_00042093.JPEG -n04153751/ILSVRC2012_val_00039490.JPEG -n04265275/ILSVRC2012_val_00014737.JPEG -n03814906/ILSVRC2012_val_00027073.JPEG -n03627232/ILSVRC2012_val_00037520.JPEG -n02843684/ILSVRC2012_val_00011133.JPEG -n02112137/ILSVRC2012_val_00009909.JPEG -n09246464/ILSVRC2012_val_00007042.JPEG -n02951358/ILSVRC2012_val_00041830.JPEG -n13054560/ILSVRC2012_val_00037445.JPEG -n03089624/ILSVRC2012_val_00006745.JPEG -n02110063/ILSVRC2012_val_00040096.JPEG -n02817516/ILSVRC2012_val_00024828.JPEG -n03958227/ILSVRC2012_val_00002885.JPEG -n02536864/ILSVRC2012_val_00040726.JPEG -n04263257/ILSVRC2012_val_00038858.JPEG -n02815834/ILSVRC2012_val_00005508.JPEG -n03793489/ILSVRC2012_val_00040362.JPEG -n07248320/ILSVRC2012_val_00037923.JPEG -n01753488/ILSVRC2012_val_00023970.JPEG -n03379051/ILSVRC2012_val_00046816.JPEG -n01734418/ILSVRC2012_val_00008534.JPEG -n02480495/ILSVRC2012_val_00020525.JPEG -n04487081/ILSVRC2012_val_00010937.JPEG -n03445777/ILSVRC2012_val_00011581.JPEG -n04286575/ILSVRC2012_val_00001679.JPEG -n02106550/ILSVRC2012_val_00019518.JPEG -n02051845/ILSVRC2012_val_00024180.JPEG -n03041632/ILSVRC2012_val_00021297.JPEG -n01669191/ILSVRC2012_val_00036944.JPEG -n03220513/ILSVRC2012_val_00007952.JPEG -n03950228/ILSVRC2012_val_00019039.JPEG -n02870880/ILSVRC2012_val_00023672.JPEG -n06596364/ILSVRC2012_val_00047158.JPEG -n01695060/ILSVRC2012_val_00011093.JPEG -n02111129/ILSVRC2012_val_00030445.JPEG -n02056570/ILSVRC2012_val_00011100.JPEG -n04442312/ILSVRC2012_val_00025862.JPEG -n04162706/ILSVRC2012_val_00034609.JPEG -n03942813/ILSVRC2012_val_00005631.JPEG -n03627232/ILSVRC2012_val_00031229.JPEG -n02097209/ILSVRC2012_val_00008453.JPEG -n07892512/ILSVRC2012_val_00019736.JPEG -n07768694/ILSVRC2012_val_00036377.JPEG -n02493509/ILSVRC2012_val_00046527.JPEG -n02099429/ILSVRC2012_val_00003061.JPEG -n01855672/ILSVRC2012_val_00009610.JPEG -n03887697/ILSVRC2012_val_00049213.JPEG -n03692522/ILSVRC2012_val_00000999.JPEG -n03197337/ILSVRC2012_val_00007983.JPEG -n03485794/ILSVRC2012_val_00018873.JPEG -n07716358/ILSVRC2012_val_00013849.JPEG -n03535780/ILSVRC2012_val_00027475.JPEG -n02111277/ILSVRC2012_val_00048512.JPEG -n04263257/ILSVRC2012_val_00024876.JPEG -n04347754/ILSVRC2012_val_00013200.JPEG -n01580077/ILSVRC2012_val_00028440.JPEG -n03630383/ILSVRC2012_val_00016958.JPEG -n03532672/ILSVRC2012_val_00026821.JPEG -n04136333/ILSVRC2012_val_00031664.JPEG -n04332243/ILSVRC2012_val_00002981.JPEG -n02105505/ILSVRC2012_val_00003152.JPEG -n02095314/ILSVRC2012_val_00048955.JPEG -n04273569/ILSVRC2012_val_00031708.JPEG -n03594945/ILSVRC2012_val_00019849.JPEG -n04536866/ILSVRC2012_val_00021977.JPEG -n02483362/ILSVRC2012_val_00046783.JPEG -n04597913/ILSVRC2012_val_00014544.JPEG -n03063599/ILSVRC2012_val_00037302.JPEG -n02018207/ILSVRC2012_val_00002144.JPEG -n02643566/ILSVRC2012_val_00046455.JPEG -n03937543/ILSVRC2012_val_00023209.JPEG -n02007558/ILSVRC2012_val_00017796.JPEG -n02231487/ILSVRC2012_val_00021735.JPEG -n07753592/ILSVRC2012_val_00022193.JPEG -n03697007/ILSVRC2012_val_00005462.JPEG -n04532106/ILSVRC2012_val_00032645.JPEG -n02114855/ILSVRC2012_val_00029186.JPEG -n02892767/ILSVRC2012_val_00046365.JPEG -n02128757/ILSVRC2012_val_00048394.JPEG -n01665541/ILSVRC2012_val_00016779.JPEG -n02895154/ILSVRC2012_val_00013983.JPEG -n02097047/ILSVRC2012_val_00017832.JPEG -n02606052/ILSVRC2012_val_00042242.JPEG -n03014705/ILSVRC2012_val_00018061.JPEG -n02088632/ILSVRC2012_val_00049516.JPEG -n03481172/ILSVRC2012_val_00027296.JPEG -n02167151/ILSVRC2012_val_00037453.JPEG -n03584254/ILSVRC2012_val_00031376.JPEG -n02788148/ILSVRC2012_val_00025392.JPEG -n10565667/ILSVRC2012_val_00040033.JPEG -n02389026/ILSVRC2012_val_00044391.JPEG -n02672831/ILSVRC2012_val_00009600.JPEG -n02441942/ILSVRC2012_val_00042483.JPEG -n10148035/ILSVRC2012_val_00049999.JPEG -n03481172/ILSVRC2012_val_00038221.JPEG -n01530575/ILSVRC2012_val_00027507.JPEG -n02125311/ILSVRC2012_val_00000487.JPEG -n12267677/ILSVRC2012_val_00019713.JPEG -n02106662/ILSVRC2012_val_00019267.JPEG -n02510455/ILSVRC2012_val_00042190.JPEG -n04560804/ILSVRC2012_val_00005081.JPEG -n01755581/ILSVRC2012_val_00037451.JPEG -n03895866/ILSVRC2012_val_00001450.JPEG -n02011460/ILSVRC2012_val_00002242.JPEG -n03394916/ILSVRC2012_val_00042707.JPEG -n04505470/ILSVRC2012_val_00048598.JPEG -n03947888/ILSVRC2012_val_00045572.JPEG -n02939185/ILSVRC2012_val_00014853.JPEG -n02256656/ILSVRC2012_val_00041190.JPEG -n07768694/ILSVRC2012_val_00009431.JPEG -n02672831/ILSVRC2012_val_00040418.JPEG -n01737021/ILSVRC2012_val_00010522.JPEG -n01773797/ILSVRC2012_val_00023301.JPEG -n03291819/ILSVRC2012_val_00007215.JPEG -n02494079/ILSVRC2012_val_00030313.JPEG -n03787032/ILSVRC2012_val_00001035.JPEG -n02787622/ILSVRC2012_val_00022234.JPEG -n13037406/ILSVRC2012_val_00045909.JPEG -n02317335/ILSVRC2012_val_00048546.JPEG -n02268443/ILSVRC2012_val_00046425.JPEG -n04579432/ILSVRC2012_val_00004456.JPEG -n02090721/ILSVRC2012_val_00035108.JPEG -n02112018/ILSVRC2012_val_00024277.JPEG -n03796401/ILSVRC2012_val_00027133.JPEG -n03977966/ILSVRC2012_val_00046876.JPEG -n01729977/ILSVRC2012_val_00033617.JPEG -n03895866/ILSVRC2012_val_00023946.JPEG -n01530575/ILSVRC2012_val_00015223.JPEG -n02321529/ILSVRC2012_val_00018073.JPEG -n03840681/ILSVRC2012_val_00047219.JPEG -n11879895/ILSVRC2012_val_00011338.JPEG -n02494079/ILSVRC2012_val_00002011.JPEG -n01440764/ILSVRC2012_val_00031094.JPEG -n02113186/ILSVRC2012_val_00028146.JPEG -n02105505/ILSVRC2012_val_00041147.JPEG -n04275548/ILSVRC2012_val_00026078.JPEG -n03017168/ILSVRC2012_val_00036837.JPEG -n02791270/ILSVRC2012_val_00018164.JPEG -n01728572/ILSVRC2012_val_00040382.JPEG -n02018795/ILSVRC2012_val_00025273.JPEG -n01629819/ILSVRC2012_val_00009674.JPEG -n13054560/ILSVRC2012_val_00004877.JPEG -n02088466/ILSVRC2012_val_00039741.JPEG -n04099969/ILSVRC2012_val_00022208.JPEG -n03709823/ILSVRC2012_val_00022534.JPEG -n04344873/ILSVRC2012_val_00041978.JPEG -n04081281/ILSVRC2012_val_00028580.JPEG -n02808440/ILSVRC2012_val_00033106.JPEG -n02107908/ILSVRC2012_val_00041844.JPEG -n01796340/ILSVRC2012_val_00024630.JPEG -n04418357/ILSVRC2012_val_00021010.JPEG -n02841315/ILSVRC2012_val_00012679.JPEG -n02447366/ILSVRC2012_val_00035718.JPEG -n02279972/ILSVRC2012_val_00035113.JPEG -n02097474/ILSVRC2012_val_00036865.JPEG -n02769748/ILSVRC2012_val_00025286.JPEG -n02018207/ILSVRC2012_val_00008951.JPEG -n02346627/ILSVRC2012_val_00024325.JPEG -n02749479/ILSVRC2012_val_00040754.JPEG -n03877472/ILSVRC2012_val_00022479.JPEG -n02172182/ILSVRC2012_val_00029856.JPEG -n03729826/ILSVRC2012_val_00038281.JPEG -n03967562/ILSVRC2012_val_00019662.JPEG -n03075370/ILSVRC2012_val_00038524.JPEG -n02114548/ILSVRC2012_val_00002948.JPEG -n04229816/ILSVRC2012_val_00023612.JPEG -n04228054/ILSVRC2012_val_00011377.JPEG -n02101556/ILSVRC2012_val_00031411.JPEG -n07871810/ILSVRC2012_val_00026959.JPEG -n13054560/ILSVRC2012_val_00010280.JPEG -n03445777/ILSVRC2012_val_00003793.JPEG -n03983396/ILSVRC2012_val_00028328.JPEG -n02132136/ILSVRC2012_val_00035969.JPEG -n02782093/ILSVRC2012_val_00014172.JPEG -n02977058/ILSVRC2012_val_00010658.JPEG -n02966687/ILSVRC2012_val_00046479.JPEG -n01984695/ILSVRC2012_val_00037235.JPEG -n03871628/ILSVRC2012_val_00002069.JPEG -n04560804/ILSVRC2012_val_00004773.JPEG -n02100735/ILSVRC2012_val_00003077.JPEG -n01518878/ILSVRC2012_val_00015706.JPEG -n02910353/ILSVRC2012_val_00008958.JPEG -n02480495/ILSVRC2012_val_00047956.JPEG -n04263257/ILSVRC2012_val_00008511.JPEG -n02704792/ILSVRC2012_val_00010301.JPEG -n04069434/ILSVRC2012_val_00049582.JPEG -n07734744/ILSVRC2012_val_00015748.JPEG -n03770439/ILSVRC2012_val_00019644.JPEG -n03903868/ILSVRC2012_val_00010953.JPEG -n01632777/ILSVRC2012_val_00034583.JPEG -n02510455/ILSVRC2012_val_00041143.JPEG -n04552348/ILSVRC2012_val_00022103.JPEG -n03630383/ILSVRC2012_val_00040416.JPEG -n03394916/ILSVRC2012_val_00044598.JPEG -n01748264/ILSVRC2012_val_00017753.JPEG -n01582220/ILSVRC2012_val_00018183.JPEG -n02099712/ILSVRC2012_val_00048301.JPEG -n03814639/ILSVRC2012_val_00033296.JPEG -n01494475/ILSVRC2012_val_00020418.JPEG -n01872401/ILSVRC2012_val_00015560.JPEG -n04501370/ILSVRC2012_val_00009103.JPEG -n03657121/ILSVRC2012_val_00017810.JPEG -n02280649/ILSVRC2012_val_00030984.JPEG -n01734418/ILSVRC2012_val_00016897.JPEG -n04447861/ILSVRC2012_val_00038860.JPEG -n02606052/ILSVRC2012_val_00019218.JPEG -n01632458/ILSVRC2012_val_00015709.JPEG -n02105056/ILSVRC2012_val_00011573.JPEG -n03065424/ILSVRC2012_val_00028620.JPEG -n02129604/ILSVRC2012_val_00002601.JPEG -n02483708/ILSVRC2012_val_00023772.JPEG -n02488702/ILSVRC2012_val_00039808.JPEG -n02097130/ILSVRC2012_val_00005896.JPEG -n09193705/ILSVRC2012_val_00011306.JPEG -n07920052/ILSVRC2012_val_00032165.JPEG -n01843065/ILSVRC2012_val_00039988.JPEG -n02804610/ILSVRC2012_val_00018806.JPEG -n03982430/ILSVRC2012_val_00004246.JPEG -n01622779/ILSVRC2012_val_00006472.JPEG -n02113023/ILSVRC2012_val_00013105.JPEG -n02699494/ILSVRC2012_val_00012734.JPEG -n02281406/ILSVRC2012_val_00038721.JPEG -n03388549/ILSVRC2012_val_00002945.JPEG -n04579145/ILSVRC2012_val_00043865.JPEG -n03992509/ILSVRC2012_val_00038358.JPEG -n03950228/ILSVRC2012_val_00033857.JPEG -n02843684/ILSVRC2012_val_00006780.JPEG -n04507155/ILSVRC2012_val_00048492.JPEG -n03444034/ILSVRC2012_val_00013026.JPEG -n04275548/ILSVRC2012_val_00009763.JPEG -n03459775/ILSVRC2012_val_00041276.JPEG -n01728572/ILSVRC2012_val_00035265.JPEG -n02086079/ILSVRC2012_val_00041043.JPEG -n04238763/ILSVRC2012_val_00010211.JPEG -n04462240/ILSVRC2012_val_00014406.JPEG -n03630383/ILSVRC2012_val_00023509.JPEG -n04462240/ILSVRC2012_val_00002632.JPEG -n03062245/ILSVRC2012_val_00047232.JPEG -n02092339/ILSVRC2012_val_00001425.JPEG -n04136333/ILSVRC2012_val_00000615.JPEG -n03773504/ILSVRC2012_val_00027130.JPEG -n02095889/ILSVRC2012_val_00015338.JPEG -n03775071/ILSVRC2012_val_00035581.JPEG -n02093647/ILSVRC2012_val_00013873.JPEG -n01984695/ILSVRC2012_val_00012455.JPEG -n03998194/ILSVRC2012_val_00009724.JPEG -n04277352/ILSVRC2012_val_00003535.JPEG -n04336792/ILSVRC2012_val_00006911.JPEG -n01981276/ILSVRC2012_val_00002157.JPEG -n04270147/ILSVRC2012_val_00002285.JPEG -n04040759/ILSVRC2012_val_00026609.JPEG -n03792972/ILSVRC2012_val_00041295.JPEG -n02408429/ILSVRC2012_val_00022696.JPEG -n03920288/ILSVRC2012_val_00014663.JPEG -n03089624/ILSVRC2012_val_00046798.JPEG -n03777754/ILSVRC2012_val_00027890.JPEG -n03496892/ILSVRC2012_val_00030173.JPEG -n02106382/ILSVRC2012_val_00014490.JPEG -n04465501/ILSVRC2012_val_00003326.JPEG -n04252077/ILSVRC2012_val_00026759.JPEG -n03272010/ILSVRC2012_val_00039867.JPEG -n03933933/ILSVRC2012_val_00026135.JPEG -n04296562/ILSVRC2012_val_00020163.JPEG -n03476684/ILSVRC2012_val_00008548.JPEG -n01748264/ILSVRC2012_val_00010194.JPEG -n03874293/ILSVRC2012_val_00004863.JPEG -n03017168/ILSVRC2012_val_00036909.JPEG -n03670208/ILSVRC2012_val_00047170.JPEG -n04505470/ILSVRC2012_val_00005863.JPEG -n02727426/ILSVRC2012_val_00017703.JPEG -n02699494/ILSVRC2012_val_00005289.JPEG -n02096051/ILSVRC2012_val_00030238.JPEG -n02279972/ILSVRC2012_val_00036259.JPEG -n02483362/ILSVRC2012_val_00049019.JPEG -n04523525/ILSVRC2012_val_00020794.JPEG -n07716906/ILSVRC2012_val_00038315.JPEG -n02088238/ILSVRC2012_val_00034260.JPEG -n02107142/ILSVRC2012_val_00045894.JPEG -n12768682/ILSVRC2012_val_00014680.JPEG -n02120079/ILSVRC2012_val_00024465.JPEG -n03188531/ILSVRC2012_val_00021254.JPEG -n02093428/ILSVRC2012_val_00028963.JPEG -n02977058/ILSVRC2012_val_00037304.JPEG -n04597913/ILSVRC2012_val_00002685.JPEG -n04065272/ILSVRC2012_val_00002117.JPEG -n01806143/ILSVRC2012_val_00028208.JPEG -n03379051/ILSVRC2012_val_00008234.JPEG -n04286575/ILSVRC2012_val_00044452.JPEG -n02095889/ILSVRC2012_val_00030377.JPEG -n03825788/ILSVRC2012_val_00039757.JPEG -n02500267/ILSVRC2012_val_00042211.JPEG -n02607072/ILSVRC2012_val_00003033.JPEG -n02999410/ILSVRC2012_val_00017738.JPEG -n04493381/ILSVRC2012_val_00014421.JPEG -n04040759/ILSVRC2012_val_00015712.JPEG -n03930313/ILSVRC2012_val_00046309.JPEG -n01817953/ILSVRC2012_val_00001629.JPEG -n03485794/ILSVRC2012_val_00019813.JPEG -n01667778/ILSVRC2012_val_00034770.JPEG -n04550184/ILSVRC2012_val_00041408.JPEG -n03584829/ILSVRC2012_val_00022284.JPEG -n04442312/ILSVRC2012_val_00048721.JPEG -n01685808/ILSVRC2012_val_00005590.JPEG -n04254777/ILSVRC2012_val_00039044.JPEG -n03026506/ILSVRC2012_val_00043360.JPEG -n03874599/ILSVRC2012_val_00027371.JPEG -n03467068/ILSVRC2012_val_00007208.JPEG -n03770439/ILSVRC2012_val_00049078.JPEG -n02749479/ILSVRC2012_val_00000790.JPEG -n04238763/ILSVRC2012_val_00013058.JPEG -n03450230/ILSVRC2012_val_00007147.JPEG -n04501370/ILSVRC2012_val_00010488.JPEG -n03874293/ILSVRC2012_val_00030003.JPEG -n02102318/ILSVRC2012_val_00041928.JPEG -n01924916/ILSVRC2012_val_00024457.JPEG -n04041544/ILSVRC2012_val_00002756.JPEG -n04493381/ILSVRC2012_val_00026655.JPEG -n03720891/ILSVRC2012_val_00047710.JPEG -n04252077/ILSVRC2012_val_00041951.JPEG -n01498041/ILSVRC2012_val_00005642.JPEG -n03908714/ILSVRC2012_val_00027234.JPEG -n04044716/ILSVRC2012_val_00002966.JPEG -n01847000/ILSVRC2012_val_00032362.JPEG -n04532670/ILSVRC2012_val_00042666.JPEG -n04532670/ILSVRC2012_val_00012169.JPEG -n01883070/ILSVRC2012_val_00023999.JPEG -n02177972/ILSVRC2012_val_00040996.JPEG -n02795169/ILSVRC2012_val_00026180.JPEG -n02097209/ILSVRC2012_val_00003731.JPEG -n03207941/ILSVRC2012_val_00005008.JPEG -n04044716/ILSVRC2012_val_00002046.JPEG -n02699494/ILSVRC2012_val_00021797.JPEG -n02802426/ILSVRC2012_val_00030155.JPEG -n02112018/ILSVRC2012_val_00001065.JPEG -n07932039/ILSVRC2012_val_00012915.JPEG -n04536866/ILSVRC2012_val_00040936.JPEG -n02978881/ILSVRC2012_val_00047907.JPEG -n02093647/ILSVRC2012_val_00009636.JPEG -n02687172/ILSVRC2012_val_00015250.JPEG -n01773549/ILSVRC2012_val_00008316.JPEG -n03720891/ILSVRC2012_val_00033713.JPEG -n02410509/ILSVRC2012_val_00030959.JPEG -n03876231/ILSVRC2012_val_00007959.JPEG -n03733805/ILSVRC2012_val_00000163.JPEG -n02672831/ILSVRC2012_val_00043981.JPEG -n03590841/ILSVRC2012_val_00008702.JPEG -n04243546/ILSVRC2012_val_00013741.JPEG -n07802026/ILSVRC2012_val_00020287.JPEG -n04399382/ILSVRC2012_val_00020037.JPEG -n04040759/ILSVRC2012_val_00030121.JPEG -n02457408/ILSVRC2012_val_00022195.JPEG -n03777754/ILSVRC2012_val_00047569.JPEG -n02259212/ILSVRC2012_val_00030505.JPEG -n01667114/ILSVRC2012_val_00036963.JPEG -n12768682/ILSVRC2012_val_00025053.JPEG -n01978455/ILSVRC2012_val_00013423.JPEG -n03825788/ILSVRC2012_val_00001209.JPEG -n03627232/ILSVRC2012_val_00013065.JPEG -n02099849/ILSVRC2012_val_00040639.JPEG -n03602883/ILSVRC2012_val_00010992.JPEG -n01675722/ILSVRC2012_val_00039378.JPEG -n02123159/ILSVRC2012_val_00011854.JPEG -n02815834/ILSVRC2012_val_00018493.JPEG -n02480495/ILSVRC2012_val_00037212.JPEG -n09421951/ILSVRC2012_val_00026980.JPEG -n07880968/ILSVRC2012_val_00026100.JPEG -n01749939/ILSVRC2012_val_00022436.JPEG -n02172182/ILSVRC2012_val_00007375.JPEG -n07873807/ILSVRC2012_val_00005784.JPEG -n12144580/ILSVRC2012_val_00021581.JPEG -n02094114/ILSVRC2012_val_00029281.JPEG -n02113712/ILSVRC2012_val_00049715.JPEG -n12267677/ILSVRC2012_val_00029915.JPEG -n04127249/ILSVRC2012_val_00041072.JPEG -n07697537/ILSVRC2012_val_00006368.JPEG -n04149813/ILSVRC2012_val_00028121.JPEG -n02704792/ILSVRC2012_val_00006872.JPEG -n02825657/ILSVRC2012_val_00008361.JPEG -n02097658/ILSVRC2012_val_00014259.JPEG -n04116512/ILSVRC2012_val_00003867.JPEG -n04366367/ILSVRC2012_val_00011834.JPEG -n02808440/ILSVRC2012_val_00003786.JPEG -n03028079/ILSVRC2012_val_00006268.JPEG -n03958227/ILSVRC2012_val_00020845.JPEG -n03492542/ILSVRC2012_val_00023887.JPEG -n03888257/ILSVRC2012_val_00029149.JPEG -n07717410/ILSVRC2012_val_00027302.JPEG -n02236044/ILSVRC2012_val_00027153.JPEG -n04040759/ILSVRC2012_val_00018235.JPEG -n02226429/ILSVRC2012_val_00047043.JPEG -n02492035/ILSVRC2012_val_00019432.JPEG -n04023962/ILSVRC2012_val_00017190.JPEG -n02013706/ILSVRC2012_val_00019998.JPEG -n03016953/ILSVRC2012_val_00044562.JPEG -n02423022/ILSVRC2012_val_00029872.JPEG -n01950731/ILSVRC2012_val_00023865.JPEG -n04429376/ILSVRC2012_val_00012217.JPEG -n04044716/ILSVRC2012_val_00046650.JPEG -n01518878/ILSVRC2012_val_00048600.JPEG -n07716358/ILSVRC2012_val_00032041.JPEG -n03933933/ILSVRC2012_val_00041214.JPEG -n07590611/ILSVRC2012_val_00001512.JPEG -n06596364/ILSVRC2012_val_00009053.JPEG -n02974003/ILSVRC2012_val_00004412.JPEG -n04239074/ILSVRC2012_val_00018266.JPEG -n03125729/ILSVRC2012_val_00009778.JPEG -n02793495/ILSVRC2012_val_00024346.JPEG -n02190166/ILSVRC2012_val_00039673.JPEG -n01773797/ILSVRC2012_val_00009108.JPEG -n04243546/ILSVRC2012_val_00004488.JPEG -n02093647/ILSVRC2012_val_00022164.JPEG -n03388183/ILSVRC2012_val_00019194.JPEG -n04136333/ILSVRC2012_val_00006032.JPEG -n04486054/ILSVRC2012_val_00047183.JPEG -n02802426/ILSVRC2012_val_00049469.JPEG -n09421951/ILSVRC2012_val_00040169.JPEG -n07720875/ILSVRC2012_val_00027266.JPEG -n02165105/ILSVRC2012_val_00000768.JPEG -n02094433/ILSVRC2012_val_00008324.JPEG -n02109961/ILSVRC2012_val_00028523.JPEG -n04286575/ILSVRC2012_val_00040305.JPEG -n03680355/ILSVRC2012_val_00049231.JPEG -n03259280/ILSVRC2012_val_00012242.JPEG -n02006656/ILSVRC2012_val_00027667.JPEG -n03623198/ILSVRC2012_val_00038316.JPEG -n03095699/ILSVRC2012_val_00010325.JPEG -n02100877/ILSVRC2012_val_00011706.JPEG -n02105412/ILSVRC2012_val_00042494.JPEG -n02138441/ILSVRC2012_val_00016283.JPEG -n03445777/ILSVRC2012_val_00012634.JPEG -n03920288/ILSVRC2012_val_00024971.JPEG -n03649909/ILSVRC2012_val_00026117.JPEG -n02504458/ILSVRC2012_val_00003747.JPEG -n09472597/ILSVRC2012_val_00044072.JPEG -n07873807/ILSVRC2012_val_00015007.JPEG -n02110341/ILSVRC2012_val_00013861.JPEG -n02794156/ILSVRC2012_val_00032063.JPEG -n02802426/ILSVRC2012_val_00042222.JPEG -n04579432/ILSVRC2012_val_00035203.JPEG -n01796340/ILSVRC2012_val_00022769.JPEG -n03534580/ILSVRC2012_val_00000845.JPEG -n03642806/ILSVRC2012_val_00001713.JPEG -n03290653/ILSVRC2012_val_00043202.JPEG -n04591157/ILSVRC2012_val_00009550.JPEG -n01685808/ILSVRC2012_val_00022711.JPEG -n04086273/ILSVRC2012_val_00018910.JPEG -n02098286/ILSVRC2012_val_00003276.JPEG -n02769748/ILSVRC2012_val_00011933.JPEG -n02091244/ILSVRC2012_val_00018737.JPEG -n03792972/ILSVRC2012_val_00045315.JPEG -n02093991/ILSVRC2012_val_00044532.JPEG -n01910747/ILSVRC2012_val_00047328.JPEG -n02093991/ILSVRC2012_val_00007327.JPEG -n02100877/ILSVRC2012_val_00000161.JPEG -n04483307/ILSVRC2012_val_00032321.JPEG -n03131574/ILSVRC2012_val_00003036.JPEG -n01695060/ILSVRC2012_val_00034125.JPEG -n01753488/ILSVRC2012_val_00040417.JPEG -n02098105/ILSVRC2012_val_00009019.JPEG -n02100877/ILSVRC2012_val_00016589.JPEG -n04229816/ILSVRC2012_val_00009290.JPEG -n02395406/ILSVRC2012_val_00032867.JPEG -n01742172/ILSVRC2012_val_00035425.JPEG -n03841143/ILSVRC2012_val_00013824.JPEG -n03124170/ILSVRC2012_val_00038273.JPEG -n04009552/ILSVRC2012_val_00044512.JPEG -n02493793/ILSVRC2012_val_00003812.JPEG -n04065272/ILSVRC2012_val_00014054.JPEG -n07873807/ILSVRC2012_val_00015340.JPEG -n03724870/ILSVRC2012_val_00029084.JPEG -n01601694/ILSVRC2012_val_00006907.JPEG -n02509815/ILSVRC2012_val_00003158.JPEG -n03197337/ILSVRC2012_val_00003455.JPEG -n02100236/ILSVRC2012_val_00017442.JPEG -n02906734/ILSVRC2012_val_00036355.JPEG -n03447721/ILSVRC2012_val_00010239.JPEG -n03047690/ILSVRC2012_val_00037434.JPEG -n03026506/ILSVRC2012_val_00045301.JPEG -n03109150/ILSVRC2012_val_00007412.JPEG -n04332243/ILSVRC2012_val_00018089.JPEG -n03876231/ILSVRC2012_val_00016176.JPEG -n03047690/ILSVRC2012_val_00040110.JPEG -n01728920/ILSVRC2012_val_00001042.JPEG -n03393912/ILSVRC2012_val_00020863.JPEG -n02814860/ILSVRC2012_val_00045456.JPEG -n01685808/ILSVRC2012_val_00046349.JPEG -n04285008/ILSVRC2012_val_00008392.JPEG -n01616318/ILSVRC2012_val_00024514.JPEG -n01734418/ILSVRC2012_val_00019383.JPEG -n02134084/ILSVRC2012_val_00007713.JPEG -n03494278/ILSVRC2012_val_00028082.JPEG -n03970156/ILSVRC2012_val_00044213.JPEG -n03786901/ILSVRC2012_val_00006228.JPEG -n01798484/ILSVRC2012_val_00041021.JPEG -n04239074/ILSVRC2012_val_00014551.JPEG -n02793495/ILSVRC2012_val_00035179.JPEG -n01667114/ILSVRC2012_val_00001636.JPEG -n01943899/ILSVRC2012_val_00009730.JPEG -n01983481/ILSVRC2012_val_00028657.JPEG -n03759954/ILSVRC2012_val_00047866.JPEG -n03376595/ILSVRC2012_val_00038258.JPEG -n11939491/ILSVRC2012_val_00009188.JPEG -n03445924/ILSVRC2012_val_00032168.JPEG -n01986214/ILSVRC2012_val_00040059.JPEG -n01945685/ILSVRC2012_val_00016138.JPEG -n02108000/ILSVRC2012_val_00048636.JPEG -n03887697/ILSVRC2012_val_00007657.JPEG -n02108000/ILSVRC2012_val_00023760.JPEG -n02092002/ILSVRC2012_val_00013361.JPEG -n02087394/ILSVRC2012_val_00000102.JPEG -n02115641/ILSVRC2012_val_00017164.JPEG -n01820546/ILSVRC2012_val_00006715.JPEG -n02165105/ILSVRC2012_val_00038929.JPEG -n03595614/ILSVRC2012_val_00005596.JPEG -n01930112/ILSVRC2012_val_00012846.JPEG -n02134084/ILSVRC2012_val_00010742.JPEG -n03424325/ILSVRC2012_val_00009791.JPEG -n04317175/ILSVRC2012_val_00009153.JPEG -n01695060/ILSVRC2012_val_00014794.JPEG -n04604644/ILSVRC2012_val_00021878.JPEG -n01807496/ILSVRC2012_val_00003676.JPEG -n02086079/ILSVRC2012_val_00020926.JPEG -n04252225/ILSVRC2012_val_00043941.JPEG -n07768694/ILSVRC2012_val_00012211.JPEG -n04435653/ILSVRC2012_val_00027772.JPEG -n04270147/ILSVRC2012_val_00004044.JPEG -n01592084/ILSVRC2012_val_00012836.JPEG -n02834397/ILSVRC2012_val_00041817.JPEG -n03662601/ILSVRC2012_val_00016448.JPEG -n03920288/ILSVRC2012_val_00011763.JPEG -n04347754/ILSVRC2012_val_00011046.JPEG -n04328186/ILSVRC2012_val_00031291.JPEG -n03223299/ILSVRC2012_val_00004922.JPEG -n03444034/ILSVRC2012_val_00047734.JPEG -n07718747/ILSVRC2012_val_00006854.JPEG -n04311174/ILSVRC2012_val_00030265.JPEG -n03803284/ILSVRC2012_val_00020872.JPEG -n02802426/ILSVRC2012_val_00003772.JPEG -n02226429/ILSVRC2012_val_00037042.JPEG -n03691459/ILSVRC2012_val_00034656.JPEG -n09421951/ILSVRC2012_val_00036544.JPEG -n02802426/ILSVRC2012_val_00018104.JPEG -n02807133/ILSVRC2012_val_00049413.JPEG -n02966687/ILSVRC2012_val_00043450.JPEG -n01729977/ILSVRC2012_val_00007710.JPEG -n03404251/ILSVRC2012_val_00004835.JPEG -n02870880/ILSVRC2012_val_00018286.JPEG -n01914609/ILSVRC2012_val_00031110.JPEG -n03617480/ILSVRC2012_val_00018442.JPEG -n07714571/ILSVRC2012_val_00007647.JPEG -n03141823/ILSVRC2012_val_00007337.JPEG -n03218198/ILSVRC2012_val_00049755.JPEG -n07718747/ILSVRC2012_val_00040344.JPEG -n02123394/ILSVRC2012_val_00008986.JPEG -n02091134/ILSVRC2012_val_00012262.JPEG -n02097298/ILSVRC2012_val_00043974.JPEG -n04317175/ILSVRC2012_val_00032866.JPEG -n03691459/ILSVRC2012_val_00000783.JPEG -n02951585/ILSVRC2012_val_00018344.JPEG -n02870880/ILSVRC2012_val_00019950.JPEG -n02769748/ILSVRC2012_val_00008482.JPEG -n03598930/ILSVRC2012_val_00041984.JPEG -n03450230/ILSVRC2012_val_00004901.JPEG -n01697457/ILSVRC2012_val_00007266.JPEG -n04264628/ILSVRC2012_val_00024441.JPEG -n03481172/ILSVRC2012_val_00025789.JPEG -n03930630/ILSVRC2012_val_00035897.JPEG -n07579787/ILSVRC2012_val_00039997.JPEG -n04509417/ILSVRC2012_val_00026754.JPEG -n02107574/ILSVRC2012_val_00004666.JPEG -n03642806/ILSVRC2012_val_00045326.JPEG -n02112706/ILSVRC2012_val_00028694.JPEG -n02109047/ILSVRC2012_val_00003069.JPEG -n03425413/ILSVRC2012_val_00015654.JPEG -n07920052/ILSVRC2012_val_00022894.JPEG -n02268443/ILSVRC2012_val_00016926.JPEG -n01735189/ILSVRC2012_val_00039522.JPEG -n03291819/ILSVRC2012_val_00038807.JPEG -n02965783/ILSVRC2012_val_00021101.JPEG -n03633091/ILSVRC2012_val_00006333.JPEG -n04326547/ILSVRC2012_val_00001505.JPEG -n02101388/ILSVRC2012_val_00036408.JPEG -n03000247/ILSVRC2012_val_00016036.JPEG -n03792972/ILSVRC2012_val_00002528.JPEG -n04392985/ILSVRC2012_val_00023462.JPEG -n01843383/ILSVRC2012_val_00014068.JPEG -n04208210/ILSVRC2012_val_00049452.JPEG -n07860988/ILSVRC2012_val_00038923.JPEG -n07749582/ILSVRC2012_val_00040080.JPEG -n02091831/ILSVRC2012_val_00000300.JPEG -n07753592/ILSVRC2012_val_00002187.JPEG -n03527444/ILSVRC2012_val_00024999.JPEG -n01776313/ILSVRC2012_val_00003577.JPEG -n01728920/ILSVRC2012_val_00042312.JPEG -n02860847/ILSVRC2012_val_00042115.JPEG -n04154565/ILSVRC2012_val_00039125.JPEG -n01580077/ILSVRC2012_val_00022505.JPEG -n01847000/ILSVRC2012_val_00006399.JPEG -n02101556/ILSVRC2012_val_00047295.JPEG -n02666196/ILSVRC2012_val_00009782.JPEG -n03666591/ILSVRC2012_val_00015019.JPEG -n04086273/ILSVRC2012_val_00009333.JPEG -n04447861/ILSVRC2012_val_00000449.JPEG -n02105641/ILSVRC2012_val_00041351.JPEG -n03623198/ILSVRC2012_val_00016824.JPEG -n02106550/ILSVRC2012_val_00049351.JPEG -n04192698/ILSVRC2012_val_00049941.JPEG -n04263257/ILSVRC2012_val_00009658.JPEG -n07579787/ILSVRC2012_val_00017367.JPEG -n02091467/ILSVRC2012_val_00049263.JPEG -n03895866/ILSVRC2012_val_00012886.JPEG -n04532106/ILSVRC2012_val_00019133.JPEG -n02423022/ILSVRC2012_val_00046413.JPEG -n02097209/ILSVRC2012_val_00011057.JPEG -n02114367/ILSVRC2012_val_00047767.JPEG -n02110063/ILSVRC2012_val_00034291.JPEG -n01614925/ILSVRC2012_val_00046632.JPEG -n02088094/ILSVRC2012_val_00036395.JPEG -n01739381/ILSVRC2012_val_00039455.JPEG -n03775546/ILSVRC2012_val_00000173.JPEG -n02909870/ILSVRC2012_val_00042633.JPEG -n02259212/ILSVRC2012_val_00010192.JPEG -n02999410/ILSVRC2012_val_00035721.JPEG -n01664065/ILSVRC2012_val_00014316.JPEG -n04026417/ILSVRC2012_val_00026615.JPEG -n02226429/ILSVRC2012_val_00000972.JPEG -n02788148/ILSVRC2012_val_00022424.JPEG -n01774384/ILSVRC2012_val_00048406.JPEG -n07590611/ILSVRC2012_val_00014121.JPEG -n02106166/ILSVRC2012_val_00008138.JPEG -n04141076/ILSVRC2012_val_00026817.JPEG -n03347037/ILSVRC2012_val_00033001.JPEG -n02787622/ILSVRC2012_val_00018617.JPEG -n01986214/ILSVRC2012_val_00040913.JPEG -n01693334/ILSVRC2012_val_00023246.JPEG -n04515003/ILSVRC2012_val_00038019.JPEG -n02489166/ILSVRC2012_val_00011904.JPEG -n04560804/ILSVRC2012_val_00013257.JPEG -n03916031/ILSVRC2012_val_00013585.JPEG -n02085936/ILSVRC2012_val_00012291.JPEG -n04099969/ILSVRC2012_val_00037756.JPEG -n03495258/ILSVRC2012_val_00021401.JPEG -n04149813/ILSVRC2012_val_00021901.JPEG -n07745940/ILSVRC2012_val_00012728.JPEG -n02086910/ILSVRC2012_val_00021961.JPEG -n02088364/ILSVRC2012_val_00042531.JPEG -n02510455/ILSVRC2012_val_00045697.JPEG -n04465501/ILSVRC2012_val_00028200.JPEG -n07590611/ILSVRC2012_val_00039174.JPEG -n01872401/ILSVRC2012_val_00034476.JPEG -n04270147/ILSVRC2012_val_00005377.JPEG -n03929855/ILSVRC2012_val_00013102.JPEG -n07684084/ILSVRC2012_val_00043611.JPEG -n03160309/ILSVRC2012_val_00000330.JPEG -n03633091/ILSVRC2012_val_00022723.JPEG -n01871265/ILSVRC2012_val_00049994.JPEG -n02795169/ILSVRC2012_val_00014204.JPEG -n03127925/ILSVRC2012_val_00037023.JPEG -n03666591/ILSVRC2012_val_00041936.JPEG -n04264628/ILSVRC2012_val_00031902.JPEG -n04026417/ILSVRC2012_val_00006166.JPEG -n02102973/ILSVRC2012_val_00009428.JPEG -n04201297/ILSVRC2012_val_00046889.JPEG -n01983481/ILSVRC2012_val_00048921.JPEG -n02096585/ILSVRC2012_val_00035182.JPEG -n04033901/ILSVRC2012_val_00034976.JPEG -n01855672/ILSVRC2012_val_00016195.JPEG -n06596364/ILSVRC2012_val_00009212.JPEG -n03250847/ILSVRC2012_val_00011127.JPEG -n04275548/ILSVRC2012_val_00018751.JPEG -n02101388/ILSVRC2012_val_00031633.JPEG -n03930630/ILSVRC2012_val_00038049.JPEG -n13037406/ILSVRC2012_val_00000200.JPEG -n01498041/ILSVRC2012_val_00006402.JPEG -n02981792/ILSVRC2012_val_00019980.JPEG -n02120079/ILSVRC2012_val_00046934.JPEG -n02006656/ILSVRC2012_val_00000878.JPEG -n02951358/ILSVRC2012_val_00032600.JPEG -n02834397/ILSVRC2012_val_00010541.JPEG -n02169497/ILSVRC2012_val_00020652.JPEG -n02441942/ILSVRC2012_val_00028413.JPEG -n02391049/ILSVRC2012_val_00019146.JPEG -n04162706/ILSVRC2012_val_00004653.JPEG -n07684084/ILSVRC2012_val_00028029.JPEG -n02883205/ILSVRC2012_val_00015595.JPEG -n03424325/ILSVRC2012_val_00027617.JPEG -n01978287/ILSVRC2012_val_00020675.JPEG -n04265275/ILSVRC2012_val_00037218.JPEG -n04228054/ILSVRC2012_val_00006208.JPEG -n04596742/ILSVRC2012_val_00007451.JPEG -n04532670/ILSVRC2012_val_00016295.JPEG -n03899768/ILSVRC2012_val_00023647.JPEG -n01664065/ILSVRC2012_val_00000958.JPEG -n07749582/ILSVRC2012_val_00035888.JPEG -n02229544/ILSVRC2012_val_00004706.JPEG -n02797295/ILSVRC2012_val_00017307.JPEG -n04590129/ILSVRC2012_val_00015963.JPEG -n03947888/ILSVRC2012_val_00032856.JPEG -n02134418/ILSVRC2012_val_00032576.JPEG -n01641577/ILSVRC2012_val_00033863.JPEG -n02018207/ILSVRC2012_val_00038948.JPEG -n02988304/ILSVRC2012_val_00041129.JPEG -n02480855/ILSVRC2012_val_00042839.JPEG -n03417042/ILSVRC2012_val_00005744.JPEG -n04536866/ILSVRC2012_val_00025372.JPEG -n04286575/ILSVRC2012_val_00006882.JPEG -n02110806/ILSVRC2012_val_00045859.JPEG -n02281406/ILSVRC2012_val_00005117.JPEG -n01689811/ILSVRC2012_val_00024252.JPEG -n03291819/ILSVRC2012_val_00009905.JPEG -n01795545/ILSVRC2012_val_00007126.JPEG -n01806567/ILSVRC2012_val_00035426.JPEG -n04501370/ILSVRC2012_val_00024489.JPEG -n02119789/ILSVRC2012_val_00018821.JPEG -n02840245/ILSVRC2012_val_00018669.JPEG -n04525038/ILSVRC2012_val_00012324.JPEG -n02279972/ILSVRC2012_val_00023084.JPEG -n02814860/ILSVRC2012_val_00016685.JPEG -n04209239/ILSVRC2012_val_00044909.JPEG -n01687978/ILSVRC2012_val_00040115.JPEG -n07730033/ILSVRC2012_val_00010660.JPEG -n07697537/ILSVRC2012_val_00023660.JPEG -n03590841/ILSVRC2012_val_00028060.JPEG -n02966193/ILSVRC2012_val_00047099.JPEG -n03733131/ILSVRC2012_val_00034620.JPEG -n03733281/ILSVRC2012_val_00011827.JPEG -n03956157/ILSVRC2012_val_00002620.JPEG -n03763968/ILSVRC2012_val_00011906.JPEG -n06359193/ILSVRC2012_val_00013443.JPEG -n02916936/ILSVRC2012_val_00019479.JPEG -n02280649/ILSVRC2012_val_00035809.JPEG -n01824575/ILSVRC2012_val_00047094.JPEG -n03478589/ILSVRC2012_val_00044229.JPEG -n02655020/ILSVRC2012_val_00008212.JPEG -n02109525/ILSVRC2012_val_00035820.JPEG -n03777568/ILSVRC2012_val_00013826.JPEG -n04335435/ILSVRC2012_val_00049437.JPEG -n02111129/ILSVRC2012_val_00042400.JPEG -n01728920/ILSVRC2012_val_00027773.JPEG -n03483316/ILSVRC2012_val_00026886.JPEG -n01883070/ILSVRC2012_val_00032499.JPEG -n03944341/ILSVRC2012_val_00004364.JPEG -n02095889/ILSVRC2012_val_00038237.JPEG -n12768682/ILSVRC2012_val_00036532.JPEG -n03594734/ILSVRC2012_val_00045325.JPEG -n01753488/ILSVRC2012_val_00025598.JPEG -n07697313/ILSVRC2012_val_00026756.JPEG -n03085013/ILSVRC2012_val_00014436.JPEG -n04597913/ILSVRC2012_val_00034343.JPEG -n01675722/ILSVRC2012_val_00008226.JPEG -n02492660/ILSVRC2012_val_00016829.JPEG -n03291819/ILSVRC2012_val_00023324.JPEG -n03063689/ILSVRC2012_val_00045069.JPEG -n02441942/ILSVRC2012_val_00022744.JPEG -n07873807/ILSVRC2012_val_00035536.JPEG -n01817953/ILSVRC2012_val_00012003.JPEG -n02090721/ILSVRC2012_val_00023769.JPEG -n01817953/ILSVRC2012_val_00002791.JPEG -n03770439/ILSVRC2012_val_00020532.JPEG -n02099712/ILSVRC2012_val_00016148.JPEG -n04443257/ILSVRC2012_val_00032397.JPEG -n02093991/ILSVRC2012_val_00032239.JPEG -n02101388/ILSVRC2012_val_00031014.JPEG -n04542943/ILSVRC2012_val_00021295.JPEG -n04589890/ILSVRC2012_val_00011099.JPEG -n12057211/ILSVRC2012_val_00009076.JPEG -n03792972/ILSVRC2012_val_00038176.JPEG -n02104365/ILSVRC2012_val_00005937.JPEG -n04277352/ILSVRC2012_val_00004080.JPEG -n07684084/ILSVRC2012_val_00046907.JPEG -n03250847/ILSVRC2012_val_00006941.JPEG -n03125729/ILSVRC2012_val_00000041.JPEG -n04004767/ILSVRC2012_val_00031132.JPEG -n03127747/ILSVRC2012_val_00023927.JPEG -n02727426/ILSVRC2012_val_00018587.JPEG -n04552348/ILSVRC2012_val_00047529.JPEG -n02165105/ILSVRC2012_val_00019643.JPEG -n04532670/ILSVRC2012_val_00037270.JPEG -n03796401/ILSVRC2012_val_00040512.JPEG -n03146219/ILSVRC2012_val_00003217.JPEG -n03259280/ILSVRC2012_val_00014647.JPEG -n02667093/ILSVRC2012_val_00029439.JPEG -n01530575/ILSVRC2012_val_00012814.JPEG -n03717622/ILSVRC2012_val_00034613.JPEG -n03485407/ILSVRC2012_val_00005121.JPEG -n03666591/ILSVRC2012_val_00010010.JPEG -n03000684/ILSVRC2012_val_00035266.JPEG -n04482393/ILSVRC2012_val_00038027.JPEG -n04515003/ILSVRC2012_val_00042717.JPEG -n04479046/ILSVRC2012_val_00007996.JPEG -n03995372/ILSVRC2012_val_00027123.JPEG -n02206856/ILSVRC2012_val_00044010.JPEG -n03000134/ILSVRC2012_val_00014448.JPEG -n01917289/ILSVRC2012_val_00008692.JPEG -n04136333/ILSVRC2012_val_00042756.JPEG -n02107312/ILSVRC2012_val_00024305.JPEG -n01924916/ILSVRC2012_val_00039672.JPEG -n07802026/ILSVRC2012_val_00017729.JPEG -n03673027/ILSVRC2012_val_00005111.JPEG -n02129604/ILSVRC2012_val_00034686.JPEG -n03920288/ILSVRC2012_val_00010434.JPEG -n04200800/ILSVRC2012_val_00003880.JPEG -n02281787/ILSVRC2012_val_00040785.JPEG -n03134739/ILSVRC2012_val_00045455.JPEG -n13054560/ILSVRC2012_val_00042631.JPEG -n02097130/ILSVRC2012_val_00048493.JPEG -n01491361/ILSVRC2012_val_00024854.JPEG -n07716906/ILSVRC2012_val_00013755.JPEG -n02100735/ILSVRC2012_val_00030271.JPEG -n02110341/ILSVRC2012_val_00027840.JPEG -n02236044/ILSVRC2012_val_00004255.JPEG -n03347037/ILSVRC2012_val_00023050.JPEG -n02906734/ILSVRC2012_val_00028772.JPEG -n02099267/ILSVRC2012_val_00032490.JPEG -n02009912/ILSVRC2012_val_00034776.JPEG -n01664065/ILSVRC2012_val_00026634.JPEG -n03384352/ILSVRC2012_val_00025503.JPEG -n02484975/ILSVRC2012_val_00017290.JPEG -n04557648/ILSVRC2012_val_00003318.JPEG -n03792972/ILSVRC2012_val_00042646.JPEG -n02808440/ILSVRC2012_val_00002563.JPEG -n03843555/ILSVRC2012_val_00032354.JPEG -n01641577/ILSVRC2012_val_00019578.JPEG -n03594734/ILSVRC2012_val_00028663.JPEG -n02279972/ILSVRC2012_val_00019145.JPEG -n02101006/ILSVRC2012_val_00016520.JPEG -n01986214/ILSVRC2012_val_00012143.JPEG -n04376876/ILSVRC2012_val_00037242.JPEG -n02403003/ILSVRC2012_val_00027362.JPEG -n01592084/ILSVRC2012_val_00007912.JPEG -n02013706/ILSVRC2012_val_00031582.JPEG -n01677366/ILSVRC2012_val_00033698.JPEG -n02488291/ILSVRC2012_val_00003438.JPEG -n03742115/ILSVRC2012_val_00018986.JPEG -n02361337/ILSVRC2012_val_00008030.JPEG -n04235860/ILSVRC2012_val_00048416.JPEG -n02509815/ILSVRC2012_val_00025768.JPEG -n03197337/ILSVRC2012_val_00035747.JPEG -n03777568/ILSVRC2012_val_00026352.JPEG -n02088632/ILSVRC2012_val_00019481.JPEG -n02088094/ILSVRC2012_val_00042972.JPEG -n03527444/ILSVRC2012_val_00035955.JPEG -n03887697/ILSVRC2012_val_00043048.JPEG -n02219486/ILSVRC2012_val_00044296.JPEG -n01824575/ILSVRC2012_val_00019511.JPEG -n04277352/ILSVRC2012_val_00006016.JPEG -n02445715/ILSVRC2012_val_00005312.JPEG -n02091134/ILSVRC2012_val_00007933.JPEG -n02089973/ILSVRC2012_val_00033611.JPEG -n02480855/ILSVRC2012_val_00005826.JPEG -n02012849/ILSVRC2012_val_00037379.JPEG -n02093256/ILSVRC2012_val_00035952.JPEG -n02099712/ILSVRC2012_val_00000375.JPEG -n02177972/ILSVRC2012_val_00041117.JPEG -n01775062/ILSVRC2012_val_00049134.JPEG -n01855672/ILSVRC2012_val_00008834.JPEG -n02097658/ILSVRC2012_val_00049653.JPEG -n04251144/ILSVRC2012_val_00027861.JPEG -n02105641/ILSVRC2012_val_00041865.JPEG -n03272010/ILSVRC2012_val_00000374.JPEG -n13044778/ILSVRC2012_val_00037473.JPEG -n02328150/ILSVRC2012_val_00002723.JPEG -n07932039/ILSVRC2012_val_00029971.JPEG -n03062245/ILSVRC2012_val_00038514.JPEG -n03000247/ILSVRC2012_val_00023046.JPEG -n01735189/ILSVRC2012_val_00018284.JPEG -n09229709/ILSVRC2012_val_00014026.JPEG -n07584110/ILSVRC2012_val_00028538.JPEG -n02749479/ILSVRC2012_val_00034057.JPEG -n01734418/ILSVRC2012_val_00000941.JPEG -n01797886/ILSVRC2012_val_00042038.JPEG -n02233338/ILSVRC2012_val_00021514.JPEG -n06785654/ILSVRC2012_val_00011264.JPEG -n04277352/ILSVRC2012_val_00011226.JPEG -n02085782/ILSVRC2012_val_00048123.JPEG -n04591157/ILSVRC2012_val_00031551.JPEG -n02009912/ILSVRC2012_val_00011369.JPEG -n02091032/ILSVRC2012_val_00045382.JPEG -n03770439/ILSVRC2012_val_00011367.JPEG -n03534580/ILSVRC2012_val_00028977.JPEG -n03697007/ILSVRC2012_val_00040424.JPEG -n04371774/ILSVRC2012_val_00007021.JPEG -n03877472/ILSVRC2012_val_00045869.JPEG -n04525305/ILSVRC2012_val_00049713.JPEG -n04317175/ILSVRC2012_val_00046901.JPEG -n02364673/ILSVRC2012_val_00005591.JPEG -n01514859/ILSVRC2012_val_00032930.JPEG -n02226429/ILSVRC2012_val_00025061.JPEG -n04517823/ILSVRC2012_val_00036519.JPEG -n02102480/ILSVRC2012_val_00000539.JPEG -n09468604/ILSVRC2012_val_00037143.JPEG -n02104365/ILSVRC2012_val_00032923.JPEG -n04266014/ILSVRC2012_val_00039273.JPEG -n04465501/ILSVRC2012_val_00004854.JPEG -n03179701/ILSVRC2012_val_00019796.JPEG -n02643566/ILSVRC2012_val_00003610.JPEG -n02106382/ILSVRC2012_val_00032822.JPEG -n02087394/ILSVRC2012_val_00000077.JPEG -n03481172/ILSVRC2012_val_00021070.JPEG -n03787032/ILSVRC2012_val_00031999.JPEG -n02356798/ILSVRC2012_val_00007315.JPEG -n04389033/ILSVRC2012_val_00048799.JPEG -n01630670/ILSVRC2012_val_00020393.JPEG -n04008634/ILSVRC2012_val_00007055.JPEG -n01631663/ILSVRC2012_val_00023450.JPEG -n03476684/ILSVRC2012_val_00037950.JPEG -n03337140/ILSVRC2012_val_00021258.JPEG -n02791270/ILSVRC2012_val_00022409.JPEG -n03954731/ILSVRC2012_val_00001966.JPEG -n07579787/ILSVRC2012_val_00031247.JPEG -n03250847/ILSVRC2012_val_00000542.JPEG -n04229816/ILSVRC2012_val_00049768.JPEG -n02486261/ILSVRC2012_val_00025698.JPEG -n04465501/ILSVRC2012_val_00020987.JPEG -n02086240/ILSVRC2012_val_00005898.JPEG -n03272562/ILSVRC2012_val_00048678.JPEG -n03857828/ILSVRC2012_val_00013357.JPEG -n02865351/ILSVRC2012_val_00009940.JPEG -n04596742/ILSVRC2012_val_00049551.JPEG -n07747607/ILSVRC2012_val_00023626.JPEG -n02917067/ILSVRC2012_val_00033558.JPEG -n03920288/ILSVRC2012_val_00030325.JPEG -n04366367/ILSVRC2012_val_00027827.JPEG -n04149813/ILSVRC2012_val_00044442.JPEG -n04033901/ILSVRC2012_val_00038824.JPEG -n02786058/ILSVRC2012_val_00038219.JPEG -n01824575/ILSVRC2012_val_00038638.JPEG -n07753592/ILSVRC2012_val_00022037.JPEG -n03355925/ILSVRC2012_val_00046366.JPEG -n02101556/ILSVRC2012_val_00018827.JPEG -n02119022/ILSVRC2012_val_00000157.JPEG -n04525305/ILSVRC2012_val_00039221.JPEG -n04596742/ILSVRC2012_val_00024149.JPEG -n12985857/ILSVRC2012_val_00010464.JPEG -n07873807/ILSVRC2012_val_00048050.JPEG -n03495258/ILSVRC2012_val_00010855.JPEG -n02071294/ILSVRC2012_val_00024324.JPEG -n07615774/ILSVRC2012_val_00049964.JPEG -n02443114/ILSVRC2012_val_00008891.JPEG -n01669191/ILSVRC2012_val_00010145.JPEG -n01692333/ILSVRC2012_val_00046106.JPEG -n02492660/ILSVRC2012_val_00048140.JPEG -n02120505/ILSVRC2012_val_00039937.JPEG -n03775546/ILSVRC2012_val_00029594.JPEG -n02364673/ILSVRC2012_val_00027337.JPEG -n13044778/ILSVRC2012_val_00042840.JPEG -n03992509/ILSVRC2012_val_00025632.JPEG -n02793495/ILSVRC2012_val_00011475.JPEG -n02281787/ILSVRC2012_val_00011979.JPEG -n04179913/ILSVRC2012_val_00001127.JPEG -n04179913/ILSVRC2012_val_00034658.JPEG -n07717410/ILSVRC2012_val_00027719.JPEG -n04507155/ILSVRC2012_val_00019266.JPEG -n02870880/ILSVRC2012_val_00021309.JPEG -n07716906/ILSVRC2012_val_00031469.JPEG -n02096051/ILSVRC2012_val_00011394.JPEG -n01739381/ILSVRC2012_val_00003095.JPEG -n07614500/ILSVRC2012_val_00039323.JPEG -n02871525/ILSVRC2012_val_00047726.JPEG -n02088632/ILSVRC2012_val_00044875.JPEG -n02098413/ILSVRC2012_val_00048235.JPEG -n03958227/ILSVRC2012_val_00004566.JPEG -n02025239/ILSVRC2012_val_00004791.JPEG -n03866082/ILSVRC2012_val_00003476.JPEG -n04515003/ILSVRC2012_val_00007908.JPEG -n02256656/ILSVRC2012_val_00010562.JPEG -n01877812/ILSVRC2012_val_00026606.JPEG -n03742115/ILSVRC2012_val_00004111.JPEG -n02002724/ILSVRC2012_val_00014949.JPEG -n01616318/ILSVRC2012_val_00022088.JPEG -n03781244/ILSVRC2012_val_00049439.JPEG -n06596364/ILSVRC2012_val_00003830.JPEG -n03938244/ILSVRC2012_val_00021635.JPEG -n02606052/ILSVRC2012_val_00045650.JPEG -n02087046/ILSVRC2012_val_00000170.JPEG -n03673027/ILSVRC2012_val_00034477.JPEG -n02328150/ILSVRC2012_val_00015683.JPEG -n02422106/ILSVRC2012_val_00032603.JPEG -n03680355/ILSVRC2012_val_00043555.JPEG -n04200800/ILSVRC2012_val_00024574.JPEG -n04118776/ILSVRC2012_val_00013176.JPEG -n03891251/ILSVRC2012_val_00034458.JPEG -n04350905/ILSVRC2012_val_00031712.JPEG -n02206856/ILSVRC2012_val_00017073.JPEG -n02835271/ILSVRC2012_val_00040248.JPEG -n03775071/ILSVRC2012_val_00019606.JPEG -n03250847/ILSVRC2012_val_00008089.JPEG -n07749582/ILSVRC2012_val_00018769.JPEG -n02279972/ILSVRC2012_val_00049460.JPEG -n02892767/ILSVRC2012_val_00012129.JPEG -n02930766/ILSVRC2012_val_00034269.JPEG -n02011460/ILSVRC2012_val_00008929.JPEG -n02102480/ILSVRC2012_val_00045044.JPEG -n04330267/ILSVRC2012_val_00046914.JPEG -n04367480/ILSVRC2012_val_00039160.JPEG -n01629819/ILSVRC2012_val_00016747.JPEG -n02980441/ILSVRC2012_val_00033614.JPEG -n03133878/ILSVRC2012_val_00011276.JPEG -n02840245/ILSVRC2012_val_00042130.JPEG -n01440764/ILSVRC2012_val_00029930.JPEG -n03623198/ILSVRC2012_val_00039114.JPEG -n04404412/ILSVRC2012_val_00037285.JPEG -n02125311/ILSVRC2012_val_00021045.JPEG -n03857828/ILSVRC2012_val_00039136.JPEG -n09472597/ILSVRC2012_val_00019071.JPEG -n02098413/ILSVRC2012_val_00005255.JPEG -n02487347/ILSVRC2012_val_00039563.JPEG -n02692877/ILSVRC2012_val_00037584.JPEG -n03388043/ILSVRC2012_val_00049852.JPEG -n04347754/ILSVRC2012_val_00029680.JPEG -n02504013/ILSVRC2012_val_00022872.JPEG -n02408429/ILSVRC2012_val_00025011.JPEG -n01860187/ILSVRC2012_val_00015899.JPEG -n09472597/ILSVRC2012_val_00031323.JPEG -n03950228/ILSVRC2012_val_00009107.JPEG -n03657121/ILSVRC2012_val_00007448.JPEG -n03461385/ILSVRC2012_val_00027864.JPEG -n03888605/ILSVRC2012_val_00033964.JPEG -n04392985/ILSVRC2012_val_00017709.JPEG -n02094258/ILSVRC2012_val_00038911.JPEG -n11939491/ILSVRC2012_val_00018189.JPEG -n04409515/ILSVRC2012_val_00046228.JPEG -n02128757/ILSVRC2012_val_00000186.JPEG -n03544143/ILSVRC2012_val_00011019.JPEG -n02190166/ILSVRC2012_val_00014356.JPEG -n02865351/ILSVRC2012_val_00031993.JPEG -n02486410/ILSVRC2012_val_00039253.JPEG -n03991062/ILSVRC2012_val_00043626.JPEG -n01491361/ILSVRC2012_val_00012328.JPEG -n07920052/ILSVRC2012_val_00029463.JPEG -n12985857/ILSVRC2012_val_00044046.JPEG -n03710721/ILSVRC2012_val_00016043.JPEG -n03535780/ILSVRC2012_val_00030209.JPEG -n04141076/ILSVRC2012_val_00017679.JPEG -n03041632/ILSVRC2012_val_00004815.JPEG -n01644373/ILSVRC2012_val_00003985.JPEG -n03796401/ILSVRC2012_val_00008394.JPEG -n03590841/ILSVRC2012_val_00009971.JPEG -n04264628/ILSVRC2012_val_00044098.JPEG -n02325366/ILSVRC2012_val_00033548.JPEG -n02442845/ILSVRC2012_val_00005621.JPEG -n03014705/ILSVRC2012_val_00017258.JPEG -n03388549/ILSVRC2012_val_00016167.JPEG -n04336792/ILSVRC2012_val_00006542.JPEG -n03291819/ILSVRC2012_val_00034362.JPEG -n02102177/ILSVRC2012_val_00010758.JPEG -n03133878/ILSVRC2012_val_00044396.JPEG -n04033901/ILSVRC2012_val_00017841.JPEG -n02536864/ILSVRC2012_val_00004905.JPEG -n07880968/ILSVRC2012_val_00010112.JPEG -n02317335/ILSVRC2012_val_00007644.JPEG -n03775071/ILSVRC2012_val_00030946.JPEG -n04275548/ILSVRC2012_val_00043164.JPEG -n01990800/ILSVRC2012_val_00005954.JPEG -n03891332/ILSVRC2012_val_00044621.JPEG -n03657121/ILSVRC2012_val_00000359.JPEG -n01537544/ILSVRC2012_val_00047272.JPEG -n04263257/ILSVRC2012_val_00045097.JPEG -n01924916/ILSVRC2012_val_00024354.JPEG -n03961711/ILSVRC2012_val_00037441.JPEG -n01665541/ILSVRC2012_val_00015568.JPEG -n04070727/ILSVRC2012_val_00046187.JPEG -n01944390/ILSVRC2012_val_00045031.JPEG -n04591713/ILSVRC2012_val_00027648.JPEG -n03180011/ILSVRC2012_val_00035357.JPEG -n03857828/ILSVRC2012_val_00048946.JPEG -n02092339/ILSVRC2012_val_00015798.JPEG -n01773157/ILSVRC2012_val_00001370.JPEG -n03929855/ILSVRC2012_val_00025618.JPEG -n02437616/ILSVRC2012_val_00020158.JPEG -n03467068/ILSVRC2012_val_00049842.JPEG -n03498962/ILSVRC2012_val_00031055.JPEG -n03534580/ILSVRC2012_val_00039018.JPEG -n04049303/ILSVRC2012_val_00046223.JPEG -n02110341/ILSVRC2012_val_00019824.JPEG -n03417042/ILSVRC2012_val_00037947.JPEG -n02669723/ILSVRC2012_val_00019360.JPEG -n03089624/ILSVRC2012_val_00026113.JPEG -n03658185/ILSVRC2012_val_00037859.JPEG -n07892512/ILSVRC2012_val_00037863.JPEG -n02747177/ILSVRC2012_val_00024193.JPEG -n01950731/ILSVRC2012_val_00031822.JPEG -n02871525/ILSVRC2012_val_00000879.JPEG -n02992529/ILSVRC2012_val_00008314.JPEG -n02111889/ILSVRC2012_val_00041377.JPEG -n03271574/ILSVRC2012_val_00009217.JPEG -n02101556/ILSVRC2012_val_00008944.JPEG -n02091635/ILSVRC2012_val_00041912.JPEG -n02110806/ILSVRC2012_val_00046197.JPEG -n02088238/ILSVRC2012_val_00040185.JPEG -n04550184/ILSVRC2012_val_00043210.JPEG -n04554684/ILSVRC2012_val_00013010.JPEG -n04040759/ILSVRC2012_val_00034869.JPEG -n01677366/ILSVRC2012_val_00027206.JPEG -n03188531/ILSVRC2012_val_00039269.JPEG -n01496331/ILSVRC2012_val_00008749.JPEG -n01818515/ILSVRC2012_val_00007576.JPEG -n01685808/ILSVRC2012_val_00004757.JPEG -n03476684/ILSVRC2012_val_00043418.JPEG -n03388183/ILSVRC2012_val_00035197.JPEG -n01983481/ILSVRC2012_val_00038154.JPEG -n04209239/ILSVRC2012_val_00026264.JPEG -n04517823/ILSVRC2012_val_00039615.JPEG -n03877472/ILSVRC2012_val_00001424.JPEG -n02091635/ILSVRC2012_val_00000079.JPEG -n01514859/ILSVRC2012_val_00039943.JPEG -n02747177/ILSVRC2012_val_00014213.JPEG -n02109047/ILSVRC2012_val_00006257.JPEG -n03297495/ILSVRC2012_val_00004741.JPEG -n03924679/ILSVRC2012_val_00016983.JPEG -n03776460/ILSVRC2012_val_00020024.JPEG -n04258138/ILSVRC2012_val_00032454.JPEG -n03961711/ILSVRC2012_val_00016375.JPEG -n02093647/ILSVRC2012_val_00033466.JPEG -n04355338/ILSVRC2012_val_00036305.JPEG -n02071294/ILSVRC2012_val_00002166.JPEG -n04037443/ILSVRC2012_val_00008457.JPEG -n01749939/ILSVRC2012_val_00046847.JPEG -n01843383/ILSVRC2012_val_00049553.JPEG -n02807133/ILSVRC2012_val_00005145.JPEG -n03544143/ILSVRC2012_val_00022005.JPEG -n02028035/ILSVRC2012_val_00002170.JPEG -n01669191/ILSVRC2012_val_00043444.JPEG -n02799071/ILSVRC2012_val_00006073.JPEG -n02111277/ILSVRC2012_val_00008111.JPEG -n04540053/ILSVRC2012_val_00026071.JPEG -n02894605/ILSVRC2012_val_00025334.JPEG -n03908714/ILSVRC2012_val_00046578.JPEG -n01729322/ILSVRC2012_val_00016392.JPEG -n01945685/ILSVRC2012_val_00041593.JPEG -n03085013/ILSVRC2012_val_00042686.JPEG -n02423022/ILSVRC2012_val_00021795.JPEG -n03717622/ILSVRC2012_val_00015092.JPEG -n01806143/ILSVRC2012_val_00049584.JPEG -n02098286/ILSVRC2012_val_00027017.JPEG -n03065424/ILSVRC2012_val_00047502.JPEG -n03877845/ILSVRC2012_val_00003919.JPEG -n02110341/ILSVRC2012_val_00003355.JPEG -n03690938/ILSVRC2012_val_00037512.JPEG -n13037406/ILSVRC2012_val_00012112.JPEG -n09332890/ILSVRC2012_val_00011436.JPEG -n03814906/ILSVRC2012_val_00040617.JPEG -n02087046/ILSVRC2012_val_00015701.JPEG -n02094114/ILSVRC2012_val_00007392.JPEG -n03841143/ILSVRC2012_val_00014722.JPEG -n03400231/ILSVRC2012_val_00009196.JPEG -n02398521/ILSVRC2012_val_00036081.JPEG -n02096294/ILSVRC2012_val_00011152.JPEG -n03240683/ILSVRC2012_val_00023790.JPEG -n03720891/ILSVRC2012_val_00023160.JPEG -n07860988/ILSVRC2012_val_00011386.JPEG -n03544143/ILSVRC2012_val_00017010.JPEG -n01818515/ILSVRC2012_val_00001163.JPEG -n02454379/ILSVRC2012_val_00038784.JPEG -n02814533/ILSVRC2012_val_00040706.JPEG -n02025239/ILSVRC2012_val_00003526.JPEG -n03733131/ILSVRC2012_val_00024037.JPEG -n07614500/ILSVRC2012_val_00049702.JPEG -n04404412/ILSVRC2012_val_00019755.JPEG -n03775546/ILSVRC2012_val_00035190.JPEG -n01978287/ILSVRC2012_val_00017751.JPEG -n02017213/ILSVRC2012_val_00043456.JPEG -n03042490/ILSVRC2012_val_00009357.JPEG -n01843065/ILSVRC2012_val_00014942.JPEG -n04367480/ILSVRC2012_val_00039395.JPEG -n03400231/ILSVRC2012_val_00022472.JPEG -n02486261/ILSVRC2012_val_00014940.JPEG -n01753488/ILSVRC2012_val_00027790.JPEG -n03131574/ILSVRC2012_val_00036262.JPEG -n04325704/ILSVRC2012_val_00047994.JPEG -n04501370/ILSVRC2012_val_00035595.JPEG -n03590841/ILSVRC2012_val_00039391.JPEG -n03018349/ILSVRC2012_val_00038333.JPEG -n02165105/ILSVRC2012_val_00040348.JPEG -n02488702/ILSVRC2012_val_00043067.JPEG -n07695742/ILSVRC2012_val_00033249.JPEG -n03843555/ILSVRC2012_val_00039328.JPEG -n03970156/ILSVRC2012_val_00033787.JPEG -n07932039/ILSVRC2012_val_00006934.JPEG -n03445924/ILSVRC2012_val_00011173.JPEG -n09835506/ILSVRC2012_val_00025002.JPEG -n02939185/ILSVRC2012_val_00003247.JPEG -n04141076/ILSVRC2012_val_00039518.JPEG -n02097474/ILSVRC2012_val_00025055.JPEG -n01692333/ILSVRC2012_val_00007579.JPEG -n02017213/ILSVRC2012_val_00023024.JPEG -n01729322/ILSVRC2012_val_00032012.JPEG -n02091244/ILSVRC2012_val_00012608.JPEG -n07716906/ILSVRC2012_val_00019550.JPEG -n03781244/ILSVRC2012_val_00014077.JPEG -n04270147/ILSVRC2012_val_00003369.JPEG -n02110627/ILSVRC2012_val_00006933.JPEG -n03498962/ILSVRC2012_val_00041471.JPEG -n03649909/ILSVRC2012_val_00020327.JPEG -n02095889/ILSVRC2012_val_00047487.JPEG -n01739381/ILSVRC2012_val_00013397.JPEG -n12267677/ILSVRC2012_val_00024975.JPEG -n01749939/ILSVRC2012_val_00029525.JPEG -n02105412/ILSVRC2012_val_00003545.JPEG -n04209239/ILSVRC2012_val_00013948.JPEG -n07745940/ILSVRC2012_val_00012216.JPEG -n03733805/ILSVRC2012_val_00023829.JPEG -n02112018/ILSVRC2012_val_00042213.JPEG -n02669723/ILSVRC2012_val_00020475.JPEG -n02094433/ILSVRC2012_val_00028993.JPEG -n03126707/ILSVRC2012_val_00026461.JPEG -n04152593/ILSVRC2012_val_00013135.JPEG -n03124043/ILSVRC2012_val_00032549.JPEG -n01630670/ILSVRC2012_val_00049350.JPEG -n02643566/ILSVRC2012_val_00025355.JPEG -n03837869/ILSVRC2012_val_00026256.JPEG -n04273569/ILSVRC2012_val_00007310.JPEG -n02100735/ILSVRC2012_val_00034436.JPEG -n04467665/ILSVRC2012_val_00032742.JPEG -n03188531/ILSVRC2012_val_00044826.JPEG -n03376595/ILSVRC2012_val_00002914.JPEG -n03459775/ILSVRC2012_val_00010573.JPEG -n03494278/ILSVRC2012_val_00018926.JPEG -n02110063/ILSVRC2012_val_00017722.JPEG -n02481823/ILSVRC2012_val_00038902.JPEG -n01770393/ILSVRC2012_val_00028814.JPEG -n02777292/ILSVRC2012_val_00015457.JPEG -n02095314/ILSVRC2012_val_00037195.JPEG -n02808304/ILSVRC2012_val_00018964.JPEG -n03775546/ILSVRC2012_val_00037888.JPEG -n02090379/ILSVRC2012_val_00024043.JPEG -n03888257/ILSVRC2012_val_00042262.JPEG -n02123597/ILSVRC2012_val_00039931.JPEG -n01644373/ILSVRC2012_val_00048995.JPEG -n03584829/ILSVRC2012_val_00019833.JPEG -n02233338/ILSVRC2012_val_00024767.JPEG -n02174001/ILSVRC2012_val_00038181.JPEG -n03773504/ILSVRC2012_val_00001525.JPEG -n07697313/ILSVRC2012_val_00040728.JPEG -n01820546/ILSVRC2012_val_00013739.JPEG -n03649909/ILSVRC2012_val_00025989.JPEG -n04548280/ILSVRC2012_val_00049764.JPEG -n01558993/ILSVRC2012_val_00028224.JPEG -n02276258/ILSVRC2012_val_00026877.JPEG -n02120079/ILSVRC2012_val_00034065.JPEG -n02033041/ILSVRC2012_val_00045006.JPEG -n09468604/ILSVRC2012_val_00008318.JPEG -n04522168/ILSVRC2012_val_00042947.JPEG -n02643566/ILSVRC2012_val_00021303.JPEG -n02114712/ILSVRC2012_val_00025845.JPEG -n01855672/ILSVRC2012_val_00026155.JPEG -n09229709/ILSVRC2012_val_00023304.JPEG -n02963159/ILSVRC2012_val_00019310.JPEG -n01756291/ILSVRC2012_val_00030623.JPEG -n03530642/ILSVRC2012_val_00007383.JPEG -n03041632/ILSVRC2012_val_00039665.JPEG -n03825788/ILSVRC2012_val_00038613.JPEG -n02102177/ILSVRC2012_val_00013127.JPEG -n02110958/ILSVRC2012_val_00028408.JPEG -n02395406/ILSVRC2012_val_00032734.JPEG -n03791053/ILSVRC2012_val_00045153.JPEG -n04522168/ILSVRC2012_val_00018977.JPEG -n01833805/ILSVRC2012_val_00041200.JPEG -n09399592/ILSVRC2012_val_00041801.JPEG -n04590129/ILSVRC2012_val_00025427.JPEG -n07930864/ILSVRC2012_val_00037747.JPEG -n03124170/ILSVRC2012_val_00035920.JPEG -n03666591/ILSVRC2012_val_00020197.JPEG -n01558993/ILSVRC2012_val_00041944.JPEG -n02441942/ILSVRC2012_val_00021127.JPEG -n03733131/ILSVRC2012_val_00012236.JPEG -n02106166/ILSVRC2012_val_00034069.JPEG -n02106662/ILSVRC2012_val_00031135.JPEG -n04259630/ILSVRC2012_val_00013276.JPEG -n04346328/ILSVRC2012_val_00042051.JPEG -n03187595/ILSVRC2012_val_00020238.JPEG -n02879718/ILSVRC2012_val_00015639.JPEG -n07565083/ILSVRC2012_val_00019149.JPEG -n02526121/ILSVRC2012_val_00041211.JPEG -n03445777/ILSVRC2012_val_00023215.JPEG -n04033901/ILSVRC2012_val_00008657.JPEG -n02120079/ILSVRC2012_val_00023960.JPEG -n01755581/ILSVRC2012_val_00016287.JPEG -n03944341/ILSVRC2012_val_00024974.JPEG -n04154565/ILSVRC2012_val_00031862.JPEG -n02823750/ILSVRC2012_val_00011370.JPEG -n03000134/ILSVRC2012_val_00010835.JPEG -n01914609/ILSVRC2012_val_00049977.JPEG -n02099712/ILSVRC2012_val_00011507.JPEG -n02099601/ILSVRC2012_val_00019212.JPEG -n02747177/ILSVRC2012_val_00021981.JPEG -n02102177/ILSVRC2012_val_00027168.JPEG -n02233338/ILSVRC2012_val_00013951.JPEG -n04591713/ILSVRC2012_val_00007320.JPEG -n04037443/ILSVRC2012_val_00038994.JPEG -n03544143/ILSVRC2012_val_00043070.JPEG -n02966193/ILSVRC2012_val_00007637.JPEG -n02488291/ILSVRC2012_val_00001458.JPEG -n02666196/ILSVRC2012_val_00009486.JPEG -n07248320/ILSVRC2012_val_00041686.JPEG -n02443484/ILSVRC2012_val_00036439.JPEG -n03062245/ILSVRC2012_val_00042505.JPEG -n01924916/ILSVRC2012_val_00027357.JPEG -n03180011/ILSVRC2012_val_00016800.JPEG -n01943899/ILSVRC2012_val_00006713.JPEG -n03216828/ILSVRC2012_val_00037556.JPEG -n02090622/ILSVRC2012_val_00013399.JPEG -n04554684/ILSVRC2012_val_00028059.JPEG -n03991062/ILSVRC2012_val_00040601.JPEG -n03954731/ILSVRC2012_val_00004803.JPEG -n03908714/ILSVRC2012_val_00010000.JPEG -n03868242/ILSVRC2012_val_00001773.JPEG -n04505470/ILSVRC2012_val_00036727.JPEG -n04399382/ILSVRC2012_val_00039275.JPEG -n02840245/ILSVRC2012_val_00021609.JPEG -n02493509/ILSVRC2012_val_00035042.JPEG -n09256479/ILSVRC2012_val_00021175.JPEG -n04081281/ILSVRC2012_val_00007483.JPEG -n02167151/ILSVRC2012_val_00039157.JPEG -n02056570/ILSVRC2012_val_00015660.JPEG -n04153751/ILSVRC2012_val_00013898.JPEG -n02112018/ILSVRC2012_val_00034667.JPEG -n13133613/ILSVRC2012_val_00008281.JPEG -n03804744/ILSVRC2012_val_00035232.JPEG -n04509417/ILSVRC2012_val_00004318.JPEG -n03742115/ILSVRC2012_val_00006806.JPEG -n02795169/ILSVRC2012_val_00031883.JPEG -n03733281/ILSVRC2012_val_00022141.JPEG -n03197337/ILSVRC2012_val_00012383.JPEG -n03417042/ILSVRC2012_val_00028914.JPEG -n03314780/ILSVRC2012_val_00021313.JPEG -n01744401/ILSVRC2012_val_00024851.JPEG -n01601694/ILSVRC2012_val_00045581.JPEG -n01560419/ILSVRC2012_val_00048178.JPEG -n03983396/ILSVRC2012_val_00044297.JPEG -n03877472/ILSVRC2012_val_00022197.JPEG -n02974003/ILSVRC2012_val_00044439.JPEG -n03179701/ILSVRC2012_val_00036160.JPEG -n02096294/ILSVRC2012_val_00013756.JPEG -n03483316/ILSVRC2012_val_00043535.JPEG -n07584110/ILSVRC2012_val_00025228.JPEG -n03495258/ILSVRC2012_val_00002209.JPEG -n01983481/ILSVRC2012_val_00033104.JPEG -n04542943/ILSVRC2012_val_00003642.JPEG -n03814639/ILSVRC2012_val_00011068.JPEG -n07831146/ILSVRC2012_val_00033336.JPEG -n02018795/ILSVRC2012_val_00004001.JPEG -n02174001/ILSVRC2012_val_00047820.JPEG -n03903868/ILSVRC2012_val_00044198.JPEG -n04004767/ILSVRC2012_val_00043636.JPEG -n02276258/ILSVRC2012_val_00014187.JPEG -n03857828/ILSVRC2012_val_00026686.JPEG -n04286575/ILSVRC2012_val_00014051.JPEG -n07714571/ILSVRC2012_val_00027632.JPEG -n07880968/ILSVRC2012_val_00011471.JPEG -n02090379/ILSVRC2012_val_00023101.JPEG -n02112137/ILSVRC2012_val_00020728.JPEG -n01494475/ILSVRC2012_val_00034115.JPEG -n03085013/ILSVRC2012_val_00018846.JPEG -n02978881/ILSVRC2012_val_00005321.JPEG -n07892512/ILSVRC2012_val_00020261.JPEG -n02091032/ILSVRC2012_val_00031302.JPEG -n01443537/ILSVRC2012_val_00002241.JPEG -n03877845/ILSVRC2012_val_00004952.JPEG -n02106030/ILSVRC2012_val_00027164.JPEG -n07697313/ILSVRC2012_val_00047577.JPEG -n01855672/ILSVRC2012_val_00043968.JPEG -n03447721/ILSVRC2012_val_00035792.JPEG -n03630383/ILSVRC2012_val_00021598.JPEG -n03345487/ILSVRC2012_val_00036198.JPEG -n04019541/ILSVRC2012_val_00000682.JPEG -n03710193/ILSVRC2012_val_00028280.JPEG -n04579432/ILSVRC2012_val_00000497.JPEG -n02493793/ILSVRC2012_val_00033780.JPEG -n03089624/ILSVRC2012_val_00025322.JPEG -n03494278/ILSVRC2012_val_00023742.JPEG -n04204238/ILSVRC2012_val_00002519.JPEG -n03637318/ILSVRC2012_val_00012801.JPEG -n02640242/ILSVRC2012_val_00000050.JPEG -n02978881/ILSVRC2012_val_00016154.JPEG -n02108551/ILSVRC2012_val_00006486.JPEG -n04039381/ILSVRC2012_val_00028105.JPEG -n02444819/ILSVRC2012_val_00046184.JPEG -n03447721/ILSVRC2012_val_00030989.JPEG -n01742172/ILSVRC2012_val_00021032.JPEG -n01914609/ILSVRC2012_val_00012332.JPEG -n01917289/ILSVRC2012_val_00037660.JPEG -n03255030/ILSVRC2012_val_00046545.JPEG -n04270147/ILSVRC2012_val_00015262.JPEG -n02840245/ILSVRC2012_val_00036216.JPEG -n03197337/ILSVRC2012_val_00006034.JPEG -n03724870/ILSVRC2012_val_00020896.JPEG -n02226429/ILSVRC2012_val_00009090.JPEG -n01694178/ILSVRC2012_val_00004555.JPEG -n01496331/ILSVRC2012_val_00047589.JPEG -n02236044/ILSVRC2012_val_00011079.JPEG -n01692333/ILSVRC2012_val_00048104.JPEG -n02085782/ILSVRC2012_val_00011749.JPEG -n04235860/ILSVRC2012_val_00049290.JPEG -n02701002/ILSVRC2012_val_00020319.JPEG -n02110185/ILSVRC2012_val_00002283.JPEG -n07920052/ILSVRC2012_val_00049872.JPEG -n02130308/ILSVRC2012_val_00017208.JPEG -n03924679/ILSVRC2012_val_00003366.JPEG -n04483307/ILSVRC2012_val_00005581.JPEG -n03633091/ILSVRC2012_val_00022704.JPEG -n02105162/ILSVRC2012_val_00030239.JPEG -n02492035/ILSVRC2012_val_00039459.JPEG -n06794110/ILSVRC2012_val_00035300.JPEG -n03804744/ILSVRC2012_val_00024466.JPEG -n02514041/ILSVRC2012_val_00019293.JPEG -n04590129/ILSVRC2012_val_00032011.JPEG -n12985857/ILSVRC2012_val_00035903.JPEG -n01530575/ILSVRC2012_val_00037116.JPEG -n02128925/ILSVRC2012_val_00045488.JPEG -n02895154/ILSVRC2012_val_00008968.JPEG -n02963159/ILSVRC2012_val_00013156.JPEG -n02808440/ILSVRC2012_val_00006019.JPEG -n04344873/ILSVRC2012_val_00016616.JPEG -n02167151/ILSVRC2012_val_00018839.JPEG -n02106030/ILSVRC2012_val_00031799.JPEG -n04461696/ILSVRC2012_val_00022323.JPEG -n02361337/ILSVRC2012_val_00037480.JPEG -n11879895/ILSVRC2012_val_00029679.JPEG -n01494475/ILSVRC2012_val_00012546.JPEG -n09421951/ILSVRC2012_val_00001767.JPEG -n02128385/ILSVRC2012_val_00038812.JPEG -n01944390/ILSVRC2012_val_00034957.JPEG -n02268853/ILSVRC2012_val_00036639.JPEG -n04049303/ILSVRC2012_val_00019554.JPEG -n04311174/ILSVRC2012_val_00011156.JPEG -n02835271/ILSVRC2012_val_00034037.JPEG -n02013706/ILSVRC2012_val_00029329.JPEG -n07873807/ILSVRC2012_val_00005438.JPEG -n04311174/ILSVRC2012_val_00006311.JPEG -n02108089/ILSVRC2012_val_00017602.JPEG -n02108089/ILSVRC2012_val_00023231.JPEG -n02423022/ILSVRC2012_val_00028513.JPEG -n01843383/ILSVRC2012_val_00032492.JPEG -n03127925/ILSVRC2012_val_00008681.JPEG -n02091244/ILSVRC2012_val_00036181.JPEG -n02443484/ILSVRC2012_val_00025162.JPEG -n03642806/ILSVRC2012_val_00022093.JPEG -n03443371/ILSVRC2012_val_00045766.JPEG -n03314780/ILSVRC2012_val_00028520.JPEG -n02927161/ILSVRC2012_val_00014566.JPEG -n03180011/ILSVRC2012_val_00001246.JPEG -n02091134/ILSVRC2012_val_00022753.JPEG -n02894605/ILSVRC2012_val_00006881.JPEG -n02667093/ILSVRC2012_val_00008447.JPEG -n03935335/ILSVRC2012_val_00022849.JPEG -n04116512/ILSVRC2012_val_00004247.JPEG -n03146219/ILSVRC2012_val_00048085.JPEG -n01843065/ILSVRC2012_val_00044330.JPEG -n03045698/ILSVRC2012_val_00043446.JPEG -n02134084/ILSVRC2012_val_00048086.JPEG -n01537544/ILSVRC2012_val_00023438.JPEG -n01677366/ILSVRC2012_val_00041096.JPEG -n02099267/ILSVRC2012_val_00030810.JPEG -n01631663/ILSVRC2012_val_00029622.JPEG -n01677366/ILSVRC2012_val_00028571.JPEG -n02437616/ILSVRC2012_val_00022237.JPEG -n03259280/ILSVRC2012_val_00044281.JPEG -n03661043/ILSVRC2012_val_00047573.JPEG -n03109150/ILSVRC2012_val_00026171.JPEG -n01776313/ILSVRC2012_val_00014587.JPEG -n01756291/ILSVRC2012_val_00006690.JPEG -n02114367/ILSVRC2012_val_00011188.JPEG -n03271574/ILSVRC2012_val_00048524.JPEG -n02804610/ILSVRC2012_val_00012025.JPEG -n01978287/ILSVRC2012_val_00030907.JPEG -n03344393/ILSVRC2012_val_00001859.JPEG -n02442845/ILSVRC2012_val_00033047.JPEG -n03710721/ILSVRC2012_val_00044100.JPEG -n03884397/ILSVRC2012_val_00041007.JPEG -n02892767/ILSVRC2012_val_00009816.JPEG -n01855032/ILSVRC2012_val_00041850.JPEG -n02704792/ILSVRC2012_val_00033902.JPEG -n07753592/ILSVRC2012_val_00049606.JPEG -n01829413/ILSVRC2012_val_00026845.JPEG -n07871810/ILSVRC2012_val_00047989.JPEG -n02096585/ILSVRC2012_val_00045047.JPEG -n03837869/ILSVRC2012_val_00036145.JPEG -n04554684/ILSVRC2012_val_00019346.JPEG -n03775071/ILSVRC2012_val_00024697.JPEG -n03697007/ILSVRC2012_val_00024323.JPEG -n02007558/ILSVRC2012_val_00039372.JPEG -n07583066/ILSVRC2012_val_00001378.JPEG -n03776460/ILSVRC2012_val_00009641.JPEG -n04243546/ILSVRC2012_val_00011959.JPEG -n02096294/ILSVRC2012_val_00004354.JPEG -n04525305/ILSVRC2012_val_00034568.JPEG -n02871525/ILSVRC2012_val_00045104.JPEG -n12998815/ILSVRC2012_val_00016330.JPEG -n02114548/ILSVRC2012_val_00006174.JPEG -n03063689/ILSVRC2012_val_00045675.JPEG -n02793495/ILSVRC2012_val_00003727.JPEG -n13037406/ILSVRC2012_val_00043934.JPEG -n01698640/ILSVRC2012_val_00049732.JPEG -n03785016/ILSVRC2012_val_00048082.JPEG -n02894605/ILSVRC2012_val_00001941.JPEG -n03742115/ILSVRC2012_val_00040111.JPEG -n03485407/ILSVRC2012_val_00008016.JPEG -n02457408/ILSVRC2012_val_00044120.JPEG -n01582220/ILSVRC2012_val_00040439.JPEG -n03584254/ILSVRC2012_val_00017141.JPEG -n02011460/ILSVRC2012_val_00008135.JPEG -n02281787/ILSVRC2012_val_00019795.JPEG -n03874599/ILSVRC2012_val_00013842.JPEG -n09332890/ILSVRC2012_val_00043027.JPEG -n03761084/ILSVRC2012_val_00003754.JPEG -n02012849/ILSVRC2012_val_00038886.JPEG -n04429376/ILSVRC2012_val_00042822.JPEG -n02071294/ILSVRC2012_val_00010966.JPEG -n03467068/ILSVRC2012_val_00044502.JPEG -n03062245/ILSVRC2012_val_00041455.JPEG -n07749582/ILSVRC2012_val_00007458.JPEG -n07734744/ILSVRC2012_val_00011179.JPEG -n04208210/ILSVRC2012_val_00002999.JPEG -n03594945/ILSVRC2012_val_00041895.JPEG -n03032252/ILSVRC2012_val_00016574.JPEG -n15075141/ILSVRC2012_val_00041824.JPEG -n03594734/ILSVRC2012_val_00039574.JPEG -n02804414/ILSVRC2012_val_00016280.JPEG -n03837869/ILSVRC2012_val_00037528.JPEG -n02102973/ILSVRC2012_val_00034804.JPEG -n02526121/ILSVRC2012_val_00016239.JPEG -n04208210/ILSVRC2012_val_00030129.JPEG -n04429376/ILSVRC2012_val_00037306.JPEG -n03494278/ILSVRC2012_val_00029729.JPEG -n01735189/ILSVRC2012_val_00032161.JPEG -n02116738/ILSVRC2012_val_00049136.JPEG -n03018349/ILSVRC2012_val_00049207.JPEG -n03272562/ILSVRC2012_val_00018483.JPEG -n03207743/ILSVRC2012_val_00021115.JPEG -n01749939/ILSVRC2012_val_00008829.JPEG -n01675722/ILSVRC2012_val_00036899.JPEG -n02112137/ILSVRC2012_val_00007654.JPEG -n03637318/ILSVRC2012_val_00038494.JPEG -n02389026/ILSVRC2012_val_00039474.JPEG -n01980166/ILSVRC2012_val_00042567.JPEG -n01688243/ILSVRC2012_val_00025581.JPEG -n03843555/ILSVRC2012_val_00044997.JPEG -n02102177/ILSVRC2012_val_00020837.JPEG -n03710193/ILSVRC2012_val_00036617.JPEG -n02116738/ILSVRC2012_val_00039573.JPEG -n03874293/ILSVRC2012_val_00015870.JPEG -n03207941/ILSVRC2012_val_00033207.JPEG -n03791053/ILSVRC2012_val_00009285.JPEG -n02492660/ILSVRC2012_val_00021147.JPEG -n03495258/ILSVRC2012_val_00000448.JPEG -n02107312/ILSVRC2012_val_00048723.JPEG -n01498041/ILSVRC2012_val_00013655.JPEG -n07718747/ILSVRC2012_val_00011408.JPEG -n01494475/ILSVRC2012_val_00023074.JPEG -n02281406/ILSVRC2012_val_00037876.JPEG -n02669723/ILSVRC2012_val_00039353.JPEG -n03240683/ILSVRC2012_val_00040675.JPEG -n04487081/ILSVRC2012_val_00040823.JPEG -n07717556/ILSVRC2012_val_00017014.JPEG -n02782093/ILSVRC2012_val_00003640.JPEG -n02110806/ILSVRC2012_val_00009093.JPEG -n03041632/ILSVRC2012_val_00014528.JPEG -n02417914/ILSVRC2012_val_00038004.JPEG -n12267677/ILSVRC2012_val_00012754.JPEG -n04483307/ILSVRC2012_val_00019354.JPEG -n04372370/ILSVRC2012_val_00026447.JPEG -n02837789/ILSVRC2012_val_00029050.JPEG -n02408429/ILSVRC2012_val_00037147.JPEG -n02037110/ILSVRC2012_val_00049227.JPEG -n02396427/ILSVRC2012_val_00009353.JPEG -n02130308/ILSVRC2012_val_00008094.JPEG -n03459775/ILSVRC2012_val_00034367.JPEG -n01930112/ILSVRC2012_val_00043302.JPEG -n03527444/ILSVRC2012_val_00047608.JPEG -n02009229/ILSVRC2012_val_00018939.JPEG -n04591713/ILSVRC2012_val_00042920.JPEG -n02108089/ILSVRC2012_val_00011921.JPEG -n02457408/ILSVRC2012_val_00041441.JPEG -n07836838/ILSVRC2012_val_00007307.JPEG -n02002724/ILSVRC2012_val_00043307.JPEG -n04487081/ILSVRC2012_val_00043686.JPEG -n09468604/ILSVRC2012_val_00020172.JPEG -n03134739/ILSVRC2012_val_00034751.JPEG -n02916936/ILSVRC2012_val_00040683.JPEG -n02895154/ILSVRC2012_val_00013879.JPEG -n03866082/ILSVRC2012_val_00027238.JPEG -n03777754/ILSVRC2012_val_00007543.JPEG -n02281406/ILSVRC2012_val_00046852.JPEG -n02690373/ILSVRC2012_val_00003960.JPEG -n03026506/ILSVRC2012_val_00009400.JPEG -n01983481/ILSVRC2012_val_00038454.JPEG -n02113624/ILSVRC2012_val_00003698.JPEG -n03657121/ILSVRC2012_val_00011439.JPEG -n01580077/ILSVRC2012_val_00017024.JPEG -n02837789/ILSVRC2012_val_00038813.JPEG -n04404412/ILSVRC2012_val_00042957.JPEG -n01773549/ILSVRC2012_val_00018347.JPEG -n03535780/ILSVRC2012_val_00013706.JPEG -n03866082/ILSVRC2012_val_00024930.JPEG -n09421951/ILSVRC2012_val_00041063.JPEG -n01985128/ILSVRC2012_val_00022720.JPEG -n02231487/ILSVRC2012_val_00001585.JPEG -n02119789/ILSVRC2012_val_00007406.JPEG -n03868863/ILSVRC2012_val_00026092.JPEG -n03216828/ILSVRC2012_val_00024765.JPEG -n03743016/ILSVRC2012_val_00017896.JPEG -n07614500/ILSVRC2012_val_00004495.JPEG -n04462240/ILSVRC2012_val_00006988.JPEG -n03388043/ILSVRC2012_val_00006101.JPEG -n03775071/ILSVRC2012_val_00044497.JPEG -n02106382/ILSVRC2012_val_00046672.JPEG -n02894605/ILSVRC2012_val_00021166.JPEG -n03887697/ILSVRC2012_val_00027867.JPEG -n03485794/ILSVRC2012_val_00022649.JPEG -n04040759/ILSVRC2012_val_00011655.JPEG -n01770393/ILSVRC2012_val_00031977.JPEG -n03598930/ILSVRC2012_val_00024475.JPEG -n02692877/ILSVRC2012_val_00018959.JPEG -n01753488/ILSVRC2012_val_00034461.JPEG -n02749479/ILSVRC2012_val_00044979.JPEG -n03085013/ILSVRC2012_val_00020212.JPEG -n03452741/ILSVRC2012_val_00047672.JPEG -n02777292/ILSVRC2012_val_00021216.JPEG -n04040759/ILSVRC2012_val_00007508.JPEG -n04004767/ILSVRC2012_val_00045332.JPEG -n04147183/ILSVRC2012_val_00027291.JPEG -n02100877/ILSVRC2012_val_00015518.JPEG -n03207941/ILSVRC2012_val_00038108.JPEG -n02666196/ILSVRC2012_val_00008091.JPEG -n02169497/ILSVRC2012_val_00025729.JPEG -n03187595/ILSVRC2012_val_00025136.JPEG -n02391049/ILSVRC2012_val_00035356.JPEG -n03404251/ILSVRC2012_val_00031173.JPEG -n01560419/ILSVRC2012_val_00026929.JPEG -n04540053/ILSVRC2012_val_00021825.JPEG -n02117135/ILSVRC2012_val_00035999.JPEG -n02704792/ILSVRC2012_val_00028986.JPEG -n03938244/ILSVRC2012_val_00014936.JPEG -n04270147/ILSVRC2012_val_00041113.JPEG -n02174001/ILSVRC2012_val_00004339.JPEG -n07613480/ILSVRC2012_val_00010091.JPEG -n02397096/ILSVRC2012_val_00043023.JPEG -n03141823/ILSVRC2012_val_00037313.JPEG -n01768244/ILSVRC2012_val_00023700.JPEG -n04039381/ILSVRC2012_val_00033869.JPEG -n01873310/ILSVRC2012_val_00004678.JPEG -n03792972/ILSVRC2012_val_00034853.JPEG -n03388549/ILSVRC2012_val_00025229.JPEG -n07615774/ILSVRC2012_val_00003471.JPEG -n12144580/ILSVRC2012_val_00007467.JPEG -n02676566/ILSVRC2012_val_00028353.JPEG -n02097658/ILSVRC2012_val_00026153.JPEG -n02488291/ILSVRC2012_val_00041180.JPEG -n04252225/ILSVRC2012_val_00025745.JPEG -n04590129/ILSVRC2012_val_00023491.JPEG -n03804744/ILSVRC2012_val_00037696.JPEG -n02489166/ILSVRC2012_val_00014275.JPEG -n02113186/ILSVRC2012_val_00013557.JPEG -n02177972/ILSVRC2012_val_00012825.JPEG -n02992529/ILSVRC2012_val_00028116.JPEG -n02231487/ILSVRC2012_val_00043867.JPEG -n03291819/ILSVRC2012_val_00037625.JPEG -n03125729/ILSVRC2012_val_00025177.JPEG -n03250847/ILSVRC2012_val_00030142.JPEG -n04523525/ILSVRC2012_val_00001798.JPEG -n02123394/ILSVRC2012_val_00027906.JPEG -n01978455/ILSVRC2012_val_00038694.JPEG -n04266014/ILSVRC2012_val_00023257.JPEG -n02101556/ILSVRC2012_val_00008368.JPEG -n03709823/ILSVRC2012_val_00022303.JPEG -n03680355/ILSVRC2012_val_00000385.JPEG -n04355933/ILSVRC2012_val_00015733.JPEG -n03461385/ILSVRC2012_val_00042117.JPEG -n02276258/ILSVRC2012_val_00042337.JPEG -n02086240/ILSVRC2012_val_00002701.JPEG -n02871525/ILSVRC2012_val_00034004.JPEG -n02091831/ILSVRC2012_val_00025170.JPEG -n13052670/ILSVRC2012_val_00022641.JPEG -n03887697/ILSVRC2012_val_00020299.JPEG -n02799071/ILSVRC2012_val_00033880.JPEG -n04204238/ILSVRC2012_val_00037482.JPEG -n04536866/ILSVRC2012_val_00008158.JPEG -n02277742/ILSVRC2012_val_00016811.JPEG -n03854065/ILSVRC2012_val_00031272.JPEG -n02487347/ILSVRC2012_val_00029161.JPEG -n03680355/ILSVRC2012_val_00024652.JPEG -n02106382/ILSVRC2012_val_00030086.JPEG -n07615774/ILSVRC2012_val_00032703.JPEG -n04270147/ILSVRC2012_val_00031848.JPEG -n03710721/ILSVRC2012_val_00031620.JPEG -n02730930/ILSVRC2012_val_00028783.JPEG -n03804744/ILSVRC2012_val_00022621.JPEG -n02113186/ILSVRC2012_val_00003910.JPEG -n03837869/ILSVRC2012_val_00013406.JPEG -n01871265/ILSVRC2012_val_00008041.JPEG -n03424325/ILSVRC2012_val_00015369.JPEG -n03637318/ILSVRC2012_val_00029645.JPEG -n04332243/ILSVRC2012_val_00004551.JPEG -n02749479/ILSVRC2012_val_00028527.JPEG -n03884397/ILSVRC2012_val_00006638.JPEG -n04209239/ILSVRC2012_val_00010492.JPEG -n04310018/ILSVRC2012_val_00024627.JPEG -n01734418/ILSVRC2012_val_00006218.JPEG -n03532672/ILSVRC2012_val_00040536.JPEG -n02094258/ILSVRC2012_val_00007906.JPEG -n02445715/ILSVRC2012_val_00033947.JPEG -n02097474/ILSVRC2012_val_00038294.JPEG -n02500267/ILSVRC2012_val_00047142.JPEG -n02879718/ILSVRC2012_val_00042569.JPEG -n02815834/ILSVRC2012_val_00036779.JPEG -n03297495/ILSVRC2012_val_00033095.JPEG -n07932039/ILSVRC2012_val_00011790.JPEG -n03729826/ILSVRC2012_val_00028678.JPEG -n02113186/ILSVRC2012_val_00011165.JPEG -n04435653/ILSVRC2012_val_00003912.JPEG -n03627232/ILSVRC2012_val_00021018.JPEG -n04346328/ILSVRC2012_val_00014319.JPEG -n03337140/ILSVRC2012_val_00017828.JPEG -n01917289/ILSVRC2012_val_00000011.JPEG -n01630670/ILSVRC2012_val_00007257.JPEG -n01749939/ILSVRC2012_val_00014252.JPEG -n01737021/ILSVRC2012_val_00010870.JPEG -n03937543/ILSVRC2012_val_00040537.JPEG -n02999410/ILSVRC2012_val_00000376.JPEG -n02174001/ILSVRC2012_val_00016150.JPEG -n02099429/ILSVRC2012_val_00041503.JPEG -n02276258/ILSVRC2012_val_00044711.JPEG -n02097047/ILSVRC2012_val_00032445.JPEG -n01930112/ILSVRC2012_val_00029033.JPEG -n02500267/ILSVRC2012_val_00023461.JPEG -n02667093/ILSVRC2012_val_00035215.JPEG -n04065272/ILSVRC2012_val_00002745.JPEG -n04069434/ILSVRC2012_val_00012086.JPEG -n03026506/ILSVRC2012_val_00040138.JPEG -n02979186/ILSVRC2012_val_00047974.JPEG -n04404412/ILSVRC2012_val_00016605.JPEG -n02028035/ILSVRC2012_val_00040471.JPEG -n07248320/ILSVRC2012_val_00027350.JPEG -n02607072/ILSVRC2012_val_00023406.JPEG -n03534580/ILSVRC2012_val_00015156.JPEG -n04120489/ILSVRC2012_val_00038922.JPEG -n02483362/ILSVRC2012_val_00042637.JPEG -n02009912/ILSVRC2012_val_00049286.JPEG -n01496331/ILSVRC2012_val_00027301.JPEG -n02100583/ILSVRC2012_val_00010808.JPEG -n02415577/ILSVRC2012_val_00016135.JPEG -n07745940/ILSVRC2012_val_00019754.JPEG -n02017213/ILSVRC2012_val_00032538.JPEG -n02105641/ILSVRC2012_val_00013007.JPEG -n02236044/ILSVRC2012_val_00012127.JPEG -n02009229/ILSVRC2012_val_00025421.JPEG -n04209239/ILSVRC2012_val_00028999.JPEG -n09421951/ILSVRC2012_val_00038275.JPEG -n01860187/ILSVRC2012_val_00019022.JPEG -n03000247/ILSVRC2012_val_00047839.JPEG -n02127052/ILSVRC2012_val_00028583.JPEG -n02797295/ILSVRC2012_val_00016849.JPEG -n02088238/ILSVRC2012_val_00026622.JPEG -n01697457/ILSVRC2012_val_00038647.JPEG -n03388549/ILSVRC2012_val_00026165.JPEG -n03670208/ILSVRC2012_val_00007113.JPEG -n04118538/ILSVRC2012_val_00004829.JPEG -n03584254/ILSVRC2012_val_00021482.JPEG -n03983396/ILSVRC2012_val_00035504.JPEG -n06359193/ILSVRC2012_val_00022255.JPEG -n01944390/ILSVRC2012_val_00023958.JPEG -n01744401/ILSVRC2012_val_00016821.JPEG -n02017213/ILSVRC2012_val_00006192.JPEG -n03871628/ILSVRC2012_val_00002169.JPEG -n02056570/ILSVRC2012_val_00046966.JPEG -n01945685/ILSVRC2012_val_00030258.JPEG -n02640242/ILSVRC2012_val_00029790.JPEG -n01770081/ILSVRC2012_val_00045563.JPEG -n02090379/ILSVRC2012_val_00035689.JPEG -n01807496/ILSVRC2012_val_00047397.JPEG -n01622779/ILSVRC2012_val_00004345.JPEG -n02480495/ILSVRC2012_val_00011425.JPEG -n01882714/ILSVRC2012_val_00000970.JPEG -n09472597/ILSVRC2012_val_00029608.JPEG -n03124043/ILSVRC2012_val_00007970.JPEG -n03908714/ILSVRC2012_val_00008499.JPEG -n02098105/ILSVRC2012_val_00036959.JPEG -n04579432/ILSVRC2012_val_00035948.JPEG -n03794056/ILSVRC2012_val_00034073.JPEG -n07745940/ILSVRC2012_val_00035170.JPEG -n02917067/ILSVRC2012_val_00023503.JPEG -n01855032/ILSVRC2012_val_00042824.JPEG -n02007558/ILSVRC2012_val_00034548.JPEG -n03633091/ILSVRC2012_val_00009892.JPEG -n02093991/ILSVRC2012_val_00019544.JPEG -n02071294/ILSVRC2012_val_00006253.JPEG -n03775546/ILSVRC2012_val_00045421.JPEG -n02094433/ILSVRC2012_val_00016973.JPEG -n07747607/ILSVRC2012_val_00006756.JPEG -n03450230/ILSVRC2012_val_00021132.JPEG -n02108089/ILSVRC2012_val_00009553.JPEG -n04579432/ILSVRC2012_val_00021960.JPEG -n03498962/ILSVRC2012_val_00023600.JPEG -n02317335/ILSVRC2012_val_00004153.JPEG -n03141823/ILSVRC2012_val_00008131.JPEG -n02412080/ILSVRC2012_val_00042513.JPEG -n03393912/ILSVRC2012_val_00043311.JPEG -n02814533/ILSVRC2012_val_00041979.JPEG -n03642806/ILSVRC2012_val_00000151.JPEG -n02112350/ILSVRC2012_val_00047515.JPEG -n01694178/ILSVRC2012_val_00015667.JPEG -n04154565/ILSVRC2012_val_00020251.JPEG -n03180011/ILSVRC2012_val_00037573.JPEG -n03630383/ILSVRC2012_val_00034841.JPEG -n02109525/ILSVRC2012_val_00010245.JPEG -n01950731/ILSVRC2012_val_00019684.JPEG -n02268853/ILSVRC2012_val_00028579.JPEG -n03017168/ILSVRC2012_val_00011842.JPEG -n02115913/ILSVRC2012_val_00004746.JPEG -n03250847/ILSVRC2012_val_00048628.JPEG -n07749582/ILSVRC2012_val_00014929.JPEG -n04204347/ILSVRC2012_val_00041108.JPEG -n03255030/ILSVRC2012_val_00011290.JPEG -n02666196/ILSVRC2012_val_00047691.JPEG -n04228054/ILSVRC2012_val_00027692.JPEG -n02028035/ILSVRC2012_val_00031922.JPEG -n01675722/ILSVRC2012_val_00033640.JPEG -n03775071/ILSVRC2012_val_00014685.JPEG -n01773797/ILSVRC2012_val_00003353.JPEG -n09835506/ILSVRC2012_val_00002751.JPEG -n02797295/ILSVRC2012_val_00005347.JPEG -n07697313/ILSVRC2012_val_00002753.JPEG -n04254680/ILSVRC2012_val_00024905.JPEG -n04258138/ILSVRC2012_val_00007822.JPEG -n02100735/ILSVRC2012_val_00016108.JPEG -n15075141/ILSVRC2012_val_00029967.JPEG -n03710721/ILSVRC2012_val_00041966.JPEG -n03124043/ILSVRC2012_val_00004432.JPEG -n03841143/ILSVRC2012_val_00046264.JPEG -n04254777/ILSVRC2012_val_00002806.JPEG -n03642806/ILSVRC2012_val_00031574.JPEG -n02115641/ILSVRC2012_val_00015821.JPEG -n02085782/ILSVRC2012_val_00016603.JPEG -n02916936/ILSVRC2012_val_00047218.JPEG -n02397096/ILSVRC2012_val_00026512.JPEG -n01748264/ILSVRC2012_val_00028813.JPEG -n02206856/ILSVRC2012_val_00012294.JPEG -n03478589/ILSVRC2012_val_00001644.JPEG -n03873416/ILSVRC2012_val_00042777.JPEG -n01985128/ILSVRC2012_val_00010640.JPEG -n02423022/ILSVRC2012_val_00047318.JPEG -n04265275/ILSVRC2012_val_00045434.JPEG -n02090721/ILSVRC2012_val_00017787.JPEG -n03180011/ILSVRC2012_val_00034411.JPEG -n03016953/ILSVRC2012_val_00033266.JPEG -n03983396/ILSVRC2012_val_00039941.JPEG -n02096051/ILSVRC2012_val_00024400.JPEG -n03201208/ILSVRC2012_val_00018913.JPEG -n06785654/ILSVRC2012_val_00011536.JPEG -n01871265/ILSVRC2012_val_00031675.JPEG -n02492660/ILSVRC2012_val_00036365.JPEG -n04239074/ILSVRC2012_val_00041234.JPEG -n03976657/ILSVRC2012_val_00019249.JPEG -n02437616/ILSVRC2012_val_00004793.JPEG -n03347037/ILSVRC2012_val_00028906.JPEG -n03337140/ILSVRC2012_val_00010321.JPEG -n02950826/ILSVRC2012_val_00031280.JPEG -n02708093/ILSVRC2012_val_00028769.JPEG -n07716358/ILSVRC2012_val_00009543.JPEG -n07720875/ILSVRC2012_val_00028969.JPEG -n04458633/ILSVRC2012_val_00006560.JPEG -n02105505/ILSVRC2012_val_00046836.JPEG -n02840245/ILSVRC2012_val_00011302.JPEG -n03347037/ILSVRC2012_val_00002272.JPEG -n04019541/ILSVRC2012_val_00044487.JPEG -n12985857/ILSVRC2012_val_00026794.JPEG -n02607072/ILSVRC2012_val_00024315.JPEG -n03658185/ILSVRC2012_val_00034287.JPEG -n02672831/ILSVRC2012_val_00044448.JPEG -n02930766/ILSVRC2012_val_00036204.JPEG -n02328150/ILSVRC2012_val_00016059.JPEG -n02107574/ILSVRC2012_val_00026426.JPEG -n01592084/ILSVRC2012_val_00010128.JPEG -n02483708/ILSVRC2012_val_00031916.JPEG -n03450230/ILSVRC2012_val_00035428.JPEG -n01537544/ILSVRC2012_val_00008420.JPEG -n02111129/ILSVRC2012_val_00037325.JPEG -n01986214/ILSVRC2012_val_00021819.JPEG -n02676566/ILSVRC2012_val_00037509.JPEG -n04443257/ILSVRC2012_val_00045194.JPEG -n01820546/ILSVRC2012_val_00005925.JPEG -n01944390/ILSVRC2012_val_00002602.JPEG -n02814533/ILSVRC2012_val_00031953.JPEG -n02108422/ILSVRC2012_val_00023294.JPEG -n02105412/ILSVRC2012_val_00010357.JPEG -n02492660/ILSVRC2012_val_00027333.JPEG -n01806143/ILSVRC2012_val_00024040.JPEG -n02980441/ILSVRC2012_val_00023472.JPEG -n04254120/ILSVRC2012_val_00042528.JPEG -n01990800/ILSVRC2012_val_00041213.JPEG -n04596742/ILSVRC2012_val_00027468.JPEG -n04147183/ILSVRC2012_val_00032731.JPEG -n02894605/ILSVRC2012_val_00016396.JPEG -n04542943/ILSVRC2012_val_00047664.JPEG -n03485794/ILSVRC2012_val_00048409.JPEG -n03218198/ILSVRC2012_val_00023915.JPEG -n02871525/ILSVRC2012_val_00005712.JPEG -n03899768/ILSVRC2012_val_00015476.JPEG -n02109047/ILSVRC2012_val_00023076.JPEG -n03498962/ILSVRC2012_val_00034911.JPEG -n04252077/ILSVRC2012_val_00043595.JPEG -n03457902/ILSVRC2012_val_00041313.JPEG -n03933933/ILSVRC2012_val_00001062.JPEG -n03803284/ILSVRC2012_val_00044666.JPEG -n07714990/ILSVRC2012_val_00027503.JPEG -n03447721/ILSVRC2012_val_00024238.JPEG -n02422699/ILSVRC2012_val_00024665.JPEG -n01560419/ILSVRC2012_val_00007263.JPEG -n02814860/ILSVRC2012_val_00005382.JPEG -n02692877/ILSVRC2012_val_00011499.JPEG -n03970156/ILSVRC2012_val_00048130.JPEG -n02097130/ILSVRC2012_val_00029609.JPEG -n02408429/ILSVRC2012_val_00031319.JPEG -n02190166/ILSVRC2012_val_00031033.JPEG -n02268443/ILSVRC2012_val_00035582.JPEG -n04485082/ILSVRC2012_val_00011069.JPEG -n03133878/ILSVRC2012_val_00006347.JPEG -n02992211/ILSVRC2012_val_00042488.JPEG -n03459775/ILSVRC2012_val_00000852.JPEG -n07768694/ILSVRC2012_val_00031058.JPEG -n04141975/ILSVRC2012_val_00016073.JPEG -n01692333/ILSVRC2012_val_00032682.JPEG -n02804610/ILSVRC2012_val_00037157.JPEG -n03868863/ILSVRC2012_val_00021700.JPEG -n02356798/ILSVRC2012_val_00038993.JPEG -n03595614/ILSVRC2012_val_00022047.JPEG -n03271574/ILSVRC2012_val_00045715.JPEG -n03450230/ILSVRC2012_val_00026729.JPEG -n02002556/ILSVRC2012_val_00035528.JPEG -n01496331/ILSVRC2012_val_00042245.JPEG -n07715103/ILSVRC2012_val_00049310.JPEG -n01806143/ILSVRC2012_val_00005599.JPEG -n02782093/ILSVRC2012_val_00004812.JPEG -n03888605/ILSVRC2012_val_00010060.JPEG -n03658185/ILSVRC2012_val_00019407.JPEG -n04209133/ILSVRC2012_val_00016376.JPEG -n01770081/ILSVRC2012_val_00048708.JPEG -n07749582/ILSVRC2012_val_00045149.JPEG -n04099969/ILSVRC2012_val_00016203.JPEG -n01797886/ILSVRC2012_val_00028604.JPEG -n03384352/ILSVRC2012_val_00041656.JPEG -n03662601/ILSVRC2012_val_00031200.JPEG -n03938244/ILSVRC2012_val_00013547.JPEG -n02102480/ILSVRC2012_val_00030059.JPEG -n02096294/ILSVRC2012_val_00016374.JPEG -n02113712/ILSVRC2012_val_00007261.JPEG -n03874293/ILSVRC2012_val_00006741.JPEG -n03967562/ILSVRC2012_val_00023747.JPEG -n04162706/ILSVRC2012_val_00009975.JPEG -n03937543/ILSVRC2012_val_00030685.JPEG -n02837789/ILSVRC2012_val_00013449.JPEG -n02526121/ILSVRC2012_val_00001747.JPEG -n04392985/ILSVRC2012_val_00045645.JPEG -n04591157/ILSVRC2012_val_00020606.JPEG -n01751748/ILSVRC2012_val_00025096.JPEG -n03895866/ILSVRC2012_val_00018523.JPEG -n02093859/ILSVRC2012_val_00041776.JPEG -n02107574/ILSVRC2012_val_00014816.JPEG -n01739381/ILSVRC2012_val_00023108.JPEG -n02280649/ILSVRC2012_val_00011175.JPEG -n04542943/ILSVRC2012_val_00049214.JPEG -n03496892/ILSVRC2012_val_00013970.JPEG -n01873310/ILSVRC2012_val_00016323.JPEG -n01735189/ILSVRC2012_val_00026122.JPEG -n01855672/ILSVRC2012_val_00018602.JPEG -n02114712/ILSVRC2012_val_00032720.JPEG -n04483307/ILSVRC2012_val_00028262.JPEG -n01770393/ILSVRC2012_val_00016030.JPEG -n02091134/ILSVRC2012_val_00004956.JPEG -n02108551/ILSVRC2012_val_00017006.JPEG -n02128925/ILSVRC2012_val_00013107.JPEG -n04380533/ILSVRC2012_val_00031497.JPEG -n03388043/ILSVRC2012_val_00030286.JPEG -n03032252/ILSVRC2012_val_00007133.JPEG -n02105641/ILSVRC2012_val_00012930.JPEG -n01820546/ILSVRC2012_val_00042679.JPEG -n03967562/ILSVRC2012_val_00000840.JPEG -n02116738/ILSVRC2012_val_00013584.JPEG -n04479046/ILSVRC2012_val_00041995.JPEG -n02389026/ILSVRC2012_val_00049605.JPEG -n02417914/ILSVRC2012_val_00034207.JPEG -n02951358/ILSVRC2012_val_00048773.JPEG -n01818515/ILSVRC2012_val_00025872.JPEG -n03598930/ILSVRC2012_val_00004589.JPEG -n01494475/ILSVRC2012_val_00013847.JPEG -n02966687/ILSVRC2012_val_00022860.JPEG -n02797295/ILSVRC2012_val_00007717.JPEG -n03110669/ILSVRC2012_val_00013454.JPEG -n02804414/ILSVRC2012_val_00005935.JPEG -n02356798/ILSVRC2012_val_00016105.JPEG -n02398521/ILSVRC2012_val_00014656.JPEG -n02437312/ILSVRC2012_val_00027819.JPEG -n03759954/ILSVRC2012_val_00008381.JPEG -n03729826/ILSVRC2012_val_00010707.JPEG -n03100240/ILSVRC2012_val_00007348.JPEG -n04355338/ILSVRC2012_val_00002015.JPEG -n04596742/ILSVRC2012_val_00001534.JPEG -n02606052/ILSVRC2012_val_00046237.JPEG -n04133789/ILSVRC2012_val_00029095.JPEG -n02676566/ILSVRC2012_val_00038042.JPEG -n03908618/ILSVRC2012_val_00006607.JPEG -n03532672/ILSVRC2012_val_00024123.JPEG -n02417914/ILSVRC2012_val_00047806.JPEG -n02108422/ILSVRC2012_val_00002317.JPEG -n04118538/ILSVRC2012_val_00038323.JPEG -n02120505/ILSVRC2012_val_00018714.JPEG -n03661043/ILSVRC2012_val_00041823.JPEG -n03633091/ILSVRC2012_val_00046392.JPEG -n01694178/ILSVRC2012_val_00001375.JPEG -n07753275/ILSVRC2012_val_00012451.JPEG -n01855032/ILSVRC2012_val_00021341.JPEG -n09399592/ILSVRC2012_val_00045881.JPEG -n03109150/ILSVRC2012_val_00038897.JPEG -n02105056/ILSVRC2012_val_00036843.JPEG -n02397096/ILSVRC2012_val_00028546.JPEG -n02229544/ILSVRC2012_val_00049638.JPEG -n04118538/ILSVRC2012_val_00041014.JPEG -n02168699/ILSVRC2012_val_00013153.JPEG -n01630670/ILSVRC2012_val_00032171.JPEG -n03877845/ILSVRC2012_val_00045327.JPEG -n04325704/ILSVRC2012_val_00027609.JPEG -n03179701/ILSVRC2012_val_00047051.JPEG -n03776460/ILSVRC2012_val_00035097.JPEG -n07615774/ILSVRC2012_val_00047975.JPEG -n01968897/ILSVRC2012_val_00033183.JPEG -n02823750/ILSVRC2012_val_00006552.JPEG -n03045698/ILSVRC2012_val_00000750.JPEG -n02123045/ILSVRC2012_val_00033420.JPEG -n04328186/ILSVRC2012_val_00045012.JPEG -n01532829/ILSVRC2012_val_00016198.JPEG -n02111500/ILSVRC2012_val_00049200.JPEG -n04404412/ILSVRC2012_val_00021932.JPEG -n02869837/ILSVRC2012_val_00009041.JPEG -n01776313/ILSVRC2012_val_00001760.JPEG -n02797295/ILSVRC2012_val_00034874.JPEG -n02093754/ILSVRC2012_val_00045978.JPEG -n03379051/ILSVRC2012_val_00021745.JPEG -n04273569/ILSVRC2012_val_00008080.JPEG -n02480495/ILSVRC2012_val_00004003.JPEG -n03825788/ILSVRC2012_val_00007804.JPEG -n02389026/ILSVRC2012_val_00000892.JPEG -n02787622/ILSVRC2012_val_00014733.JPEG -n02089867/ILSVRC2012_val_00016255.JPEG -n02101556/ILSVRC2012_val_00000643.JPEG -n03710721/ILSVRC2012_val_00018600.JPEG -n02090622/ILSVRC2012_val_00040878.JPEG -n04591713/ILSVRC2012_val_00043192.JPEG -n02093754/ILSVRC2012_val_00010785.JPEG -n03733281/ILSVRC2012_val_00045736.JPEG -n02279972/ILSVRC2012_val_00047733.JPEG -n07742313/ILSVRC2012_val_00019453.JPEG -n03814639/ILSVRC2012_val_00034327.JPEG -n04525305/ILSVRC2012_val_00049957.JPEG -n04335435/ILSVRC2012_val_00014433.JPEG -n02447366/ILSVRC2012_val_00023870.JPEG -n01582220/ILSVRC2012_val_00044507.JPEG -n01950731/ILSVRC2012_val_00040586.JPEG -n09332890/ILSVRC2012_val_00015764.JPEG -n02494079/ILSVRC2012_val_00014206.JPEG -n03404251/ILSVRC2012_val_00033667.JPEG -n02114855/ILSVRC2012_val_00039555.JPEG -n03089624/ILSVRC2012_val_00041701.JPEG -n04258138/ILSVRC2012_val_00017908.JPEG -n03063689/ILSVRC2012_val_00002022.JPEG -n04553703/ILSVRC2012_val_00013941.JPEG -n01697457/ILSVRC2012_val_00041980.JPEG -n03495258/ILSVRC2012_val_00044541.JPEG -n03967562/ILSVRC2012_val_00019675.JPEG -n04523525/ILSVRC2012_val_00025290.JPEG -n01773797/ILSVRC2012_val_00047551.JPEG -n02500267/ILSVRC2012_val_00040160.JPEG -n02730930/ILSVRC2012_val_00034506.JPEG -n03345487/ILSVRC2012_val_00004238.JPEG -n02999410/ILSVRC2012_val_00025216.JPEG -n03131574/ILSVRC2012_val_00043169.JPEG -n02123597/ILSVRC2012_val_00045347.JPEG -n02676566/ILSVRC2012_val_00003813.JPEG -n02177972/ILSVRC2012_val_00001737.JPEG -n02174001/ILSVRC2012_val_00020026.JPEG -n03379051/ILSVRC2012_val_00035645.JPEG -n01514859/ILSVRC2012_val_00009615.JPEG -n02107683/ILSVRC2012_val_00040427.JPEG -n02939185/ILSVRC2012_val_00034904.JPEG -n02788148/ILSVRC2012_val_00029432.JPEG -n04523525/ILSVRC2012_val_00033029.JPEG -n01582220/ILSVRC2012_val_00041051.JPEG -n01914609/ILSVRC2012_val_00040958.JPEG -n04599235/ILSVRC2012_val_00037061.JPEG -n01784675/ILSVRC2012_val_00028449.JPEG -n04592741/ILSVRC2012_val_00019965.JPEG -n02124075/ILSVRC2012_val_00032637.JPEG -n01773157/ILSVRC2012_val_00022618.JPEG -n02091831/ILSVRC2012_val_00007002.JPEG -n02172182/ILSVRC2012_val_00006883.JPEG -n02643566/ILSVRC2012_val_00006279.JPEG -n03792782/ILSVRC2012_val_00042457.JPEG -n02128925/ILSVRC2012_val_00010173.JPEG -n02102177/ILSVRC2012_val_00018942.JPEG -n01882714/ILSVRC2012_val_00007389.JPEG -n02328150/ILSVRC2012_val_00027898.JPEG -n02219486/ILSVRC2012_val_00003814.JPEG -n02110341/ILSVRC2012_val_00023913.JPEG -n02113978/ILSVRC2012_val_00026094.JPEG -n01817953/ILSVRC2012_val_00028862.JPEG -n02105412/ILSVRC2012_val_00041115.JPEG -n04552348/ILSVRC2012_val_00028193.JPEG -n03617480/ILSVRC2012_val_00006657.JPEG -n02917067/ILSVRC2012_val_00034240.JPEG -n02102040/ILSVRC2012_val_00001968.JPEG -n01601694/ILSVRC2012_val_00040903.JPEG -n03742115/ILSVRC2012_val_00009627.JPEG -n02747177/ILSVRC2012_val_00044878.JPEG -n04418357/ILSVRC2012_val_00028155.JPEG -n03047690/ILSVRC2012_val_00029828.JPEG -n02437312/ILSVRC2012_val_00016367.JPEG -n02174001/ILSVRC2012_val_00034738.JPEG -n04542943/ILSVRC2012_val_00035062.JPEG -n01704323/ILSVRC2012_val_00017962.JPEG -n07717556/ILSVRC2012_val_00029658.JPEG -n04596742/ILSVRC2012_val_00017344.JPEG -n03384352/ILSVRC2012_val_00001579.JPEG -n01924916/ILSVRC2012_val_00033248.JPEG -n02102480/ILSVRC2012_val_00023012.JPEG -n03483316/ILSVRC2012_val_00044558.JPEG -n03982430/ILSVRC2012_val_00029955.JPEG -n01687978/ILSVRC2012_val_00049072.JPEG -n02002724/ILSVRC2012_val_00042532.JPEG -n04004767/ILSVRC2012_val_00042574.JPEG -n03063689/ILSVRC2012_val_00025249.JPEG -n03014705/ILSVRC2012_val_00002535.JPEG -n01537544/ILSVRC2012_val_00022076.JPEG -n02747177/ILSVRC2012_val_00043970.JPEG -n02871525/ILSVRC2012_val_00012973.JPEG -n03617480/ILSVRC2012_val_00000900.JPEG -n01775062/ILSVRC2012_val_00046017.JPEG -n02007558/ILSVRC2012_val_00030095.JPEG -n02363005/ILSVRC2012_val_00017230.JPEG -n07614500/ILSVRC2012_val_00024640.JPEG -n03942813/ILSVRC2012_val_00038827.JPEG -n04417672/ILSVRC2012_val_00004491.JPEG -n01753488/ILSVRC2012_val_00001582.JPEG -n02033041/ILSVRC2012_val_00034186.JPEG -n03854065/ILSVRC2012_val_00048283.JPEG -n04229816/ILSVRC2012_val_00014358.JPEG -n02481823/ILSVRC2012_val_00038769.JPEG -n02488291/ILSVRC2012_val_00039225.JPEG -n04367480/ILSVRC2012_val_00049863.JPEG -n02988304/ILSVRC2012_val_00004114.JPEG -n04239074/ILSVRC2012_val_00017133.JPEG -n03633091/ILSVRC2012_val_00009885.JPEG -n02094258/ILSVRC2012_val_00020681.JPEG -n02808304/ILSVRC2012_val_00014072.JPEG -n01729322/ILSVRC2012_val_00003071.JPEG -n02504013/ILSVRC2012_val_00037615.JPEG -n02116738/ILSVRC2012_val_00044707.JPEG -n02321529/ILSVRC2012_val_00034757.JPEG -n02110627/ILSVRC2012_val_00032948.JPEG -n02109047/ILSVRC2012_val_00041514.JPEG -n04389033/ILSVRC2012_val_00015638.JPEG -n03710193/ILSVRC2012_val_00048316.JPEG -n01877812/ILSVRC2012_val_00042917.JPEG -n01630670/ILSVRC2012_val_00021083.JPEG -n04579145/ILSVRC2012_val_00006747.JPEG -n07717410/ILSVRC2012_val_00025236.JPEG -n07720875/ILSVRC2012_val_00001463.JPEG -n02002724/ILSVRC2012_val_00022140.JPEG -n03223299/ILSVRC2012_val_00040733.JPEG -n04125021/ILSVRC2012_val_00019656.JPEG -n02279972/ILSVRC2012_val_00047039.JPEG -n03498962/ILSVRC2012_val_00044873.JPEG -n03937543/ILSVRC2012_val_00017943.JPEG -n07614500/ILSVRC2012_val_00031184.JPEG -n04584207/ILSVRC2012_val_00005990.JPEG -n04606251/ILSVRC2012_val_00038015.JPEG -n02795169/ILSVRC2012_val_00046794.JPEG -n01843065/ILSVRC2012_val_00024916.JPEG -n03868863/ILSVRC2012_val_00015601.JPEG -n02783161/ILSVRC2012_val_00005048.JPEG -n02906734/ILSVRC2012_val_00002937.JPEG -n02500267/ILSVRC2012_val_00040939.JPEG -n04192698/ILSVRC2012_val_00031758.JPEG -n02104029/ILSVRC2012_val_00024981.JPEG -n03930630/ILSVRC2012_val_00001156.JPEG -n04335435/ILSVRC2012_val_00017135.JPEG -n03930630/ILSVRC2012_val_00000049.JPEG -n02128925/ILSVRC2012_val_00021585.JPEG -n02701002/ILSVRC2012_val_00013686.JPEG -n03485794/ILSVRC2012_val_00047363.JPEG -n02102177/ILSVRC2012_val_00047744.JPEG -n04070727/ILSVRC2012_val_00029720.JPEG -n01978455/ILSVRC2012_val_00024557.JPEG -n04509417/ILSVRC2012_val_00011781.JPEG -n03947888/ILSVRC2012_val_00031484.JPEG -n03444034/ILSVRC2012_val_00042238.JPEG -n07747607/ILSVRC2012_val_00040188.JPEG -n01773549/ILSVRC2012_val_00035063.JPEG -n02877765/ILSVRC2012_val_00030430.JPEG -n02389026/ILSVRC2012_val_00022302.JPEG -n01582220/ILSVRC2012_val_00034437.JPEG -n02190166/ILSVRC2012_val_00034895.JPEG -n02979186/ILSVRC2012_val_00034563.JPEG -n03063599/ILSVRC2012_val_00038548.JPEG -n02114367/ILSVRC2012_val_00029659.JPEG -n02124075/ILSVRC2012_val_00000709.JPEG -n04200800/ILSVRC2012_val_00047526.JPEG -n02791124/ILSVRC2012_val_00006275.JPEG -n01734418/ILSVRC2012_val_00019148.JPEG -n03207743/ILSVRC2012_val_00041412.JPEG -n01737021/ILSVRC2012_val_00027228.JPEG -n02174001/ILSVRC2012_val_00040044.JPEG -n01675722/ILSVRC2012_val_00048134.JPEG -n02279972/ILSVRC2012_val_00033695.JPEG -n04523525/ILSVRC2012_val_00032984.JPEG -n03452741/ILSVRC2012_val_00026127.JPEG -n02229544/ILSVRC2012_val_00009275.JPEG -n02110063/ILSVRC2012_val_00039359.JPEG -n01532829/ILSVRC2012_val_00017577.JPEG -n02012849/ILSVRC2012_val_00009564.JPEG -n02749479/ILSVRC2012_val_00035791.JPEG -n04133789/ILSVRC2012_val_00021228.JPEG -n04074963/ILSVRC2012_val_00013950.JPEG -n04553703/ILSVRC2012_val_00038528.JPEG -n03871628/ILSVRC2012_val_00015005.JPEG -n02027492/ILSVRC2012_val_00002488.JPEG -n04532106/ILSVRC2012_val_00011984.JPEG -n04208210/ILSVRC2012_val_00010415.JPEG -n04597913/ILSVRC2012_val_00032311.JPEG -n04467665/ILSVRC2012_val_00032726.JPEG -n01819313/ILSVRC2012_val_00019123.JPEG -n02097130/ILSVRC2012_val_00038174.JPEG -n03764736/ILSVRC2012_val_00020505.JPEG -n03891332/ILSVRC2012_val_00038260.JPEG -n02097047/ILSVRC2012_val_00035480.JPEG -n07613480/ILSVRC2012_val_00031778.JPEG -n01514859/ILSVRC2012_val_00049833.JPEG -n03000134/ILSVRC2012_val_00008513.JPEG -n04461696/ILSVRC2012_val_00004512.JPEG -n04204238/ILSVRC2012_val_00043309.JPEG -n02892767/ILSVRC2012_val_00033714.JPEG -n03063689/ILSVRC2012_val_00014552.JPEG -n02086079/ILSVRC2012_val_00027838.JPEG -n04336792/ILSVRC2012_val_00009356.JPEG -n03950228/ILSVRC2012_val_00049774.JPEG -n02927161/ILSVRC2012_val_00002526.JPEG -n02102040/ILSVRC2012_val_00012689.JPEG -n03207941/ILSVRC2012_val_00018647.JPEG -n01986214/ILSVRC2012_val_00027404.JPEG -n03160309/ILSVRC2012_val_00017090.JPEG -n03179701/ILSVRC2012_val_00006423.JPEG -n03594734/ILSVRC2012_val_00018771.JPEG -n03016953/ILSVRC2012_val_00037394.JPEG -n03388183/ILSVRC2012_val_00005179.JPEG -n13133613/ILSVRC2012_val_00034855.JPEG -n03792782/ILSVRC2012_val_00042393.JPEG -n02999410/ILSVRC2012_val_00043064.JPEG -n02999410/ILSVRC2012_val_00013634.JPEG -n03676483/ILSVRC2012_val_00027437.JPEG -n04317175/ILSVRC2012_val_00030733.JPEG -n03085013/ILSVRC2012_val_00045039.JPEG -n02807133/ILSVRC2012_val_00039882.JPEG -n02105505/ILSVRC2012_val_00043538.JPEG -n02090721/ILSVRC2012_val_00013507.JPEG -n04465501/ILSVRC2012_val_00035513.JPEG -n04310018/ILSVRC2012_val_00025265.JPEG -n01632458/ILSVRC2012_val_00003350.JPEG -n02980441/ILSVRC2012_val_00021425.JPEG -n02669723/ILSVRC2012_val_00047602.JPEG -n02137549/ILSVRC2012_val_00023825.JPEG -n07715103/ILSVRC2012_val_00028394.JPEG -n02092339/ILSVRC2012_val_00047244.JPEG -n02114367/ILSVRC2012_val_00006666.JPEG -n03220513/ILSVRC2012_val_00002674.JPEG -n03868863/ILSVRC2012_val_00035346.JPEG -n02939185/ILSVRC2012_val_00031283.JPEG -n01806567/ILSVRC2012_val_00018648.JPEG -n02769748/ILSVRC2012_val_00016935.JPEG -n04479046/ILSVRC2012_val_00018547.JPEG -n02167151/ILSVRC2012_val_00023624.JPEG -n02097209/ILSVRC2012_val_00028152.JPEG -n02110806/ILSVRC2012_val_00046623.JPEG -n02787622/ILSVRC2012_val_00000315.JPEG -n03388549/ILSVRC2012_val_00023839.JPEG -n03935335/ILSVRC2012_val_00012052.JPEG -n04579432/ILSVRC2012_val_00001245.JPEG -n03903868/ILSVRC2012_val_00013811.JPEG -n03498962/ILSVRC2012_val_00042039.JPEG -n02640242/ILSVRC2012_val_00036484.JPEG -n07565083/ILSVRC2012_val_00047347.JPEG -n01729322/ILSVRC2012_val_00018453.JPEG -n04487081/ILSVRC2012_val_00014513.JPEG -n04154565/ILSVRC2012_val_00002631.JPEG -n02916936/ILSVRC2012_val_00047056.JPEG -n02342885/ILSVRC2012_val_00039968.JPEG -n03384352/ILSVRC2012_val_00005128.JPEG -n02096294/ILSVRC2012_val_00042133.JPEG -n03208938/ILSVRC2012_val_00018891.JPEG -n02268853/ILSVRC2012_val_00012681.JPEG -n02437616/ILSVRC2012_val_00010441.JPEG -n02965783/ILSVRC2012_val_00012763.JPEG -n04590129/ILSVRC2012_val_00041532.JPEG -n03100240/ILSVRC2012_val_00036091.JPEG -n04023962/ILSVRC2012_val_00017642.JPEG -n03995372/ILSVRC2012_val_00009245.JPEG -n02859443/ILSVRC2012_val_00042176.JPEG -n04009552/ILSVRC2012_val_00017302.JPEG -n04127249/ILSVRC2012_val_00020871.JPEG -n02174001/ILSVRC2012_val_00023013.JPEG -n03393912/ILSVRC2012_val_00003909.JPEG -n02167151/ILSVRC2012_val_00042127.JPEG -n02870880/ILSVRC2012_val_00046060.JPEG -n02966193/ILSVRC2012_val_00044864.JPEG -n01981276/ILSVRC2012_val_00011453.JPEG -n02088238/ILSVRC2012_val_00020468.JPEG -n02102177/ILSVRC2012_val_00007927.JPEG -n02443114/ILSVRC2012_val_00016220.JPEG -n03459775/ILSVRC2012_val_00028470.JPEG -n03443371/ILSVRC2012_val_00034156.JPEG -n07873807/ILSVRC2012_val_00012243.JPEG -n02112350/ILSVRC2012_val_00016303.JPEG -n02823428/ILSVRC2012_val_00040545.JPEG -n03690938/ILSVRC2012_val_00036956.JPEG -n07880968/ILSVRC2012_val_00006293.JPEG -n01530575/ILSVRC2012_val_00023441.JPEG -n04554684/ILSVRC2012_val_00014530.JPEG -n03218198/ILSVRC2012_val_00007393.JPEG -n03461385/ILSVRC2012_val_00019818.JPEG -n03794056/ILSVRC2012_val_00024361.JPEG -n01945685/ILSVRC2012_val_00014762.JPEG -n04372370/ILSVRC2012_val_00035679.JPEG -n04613696/ILSVRC2012_val_00016339.JPEG -n04548362/ILSVRC2012_val_00028532.JPEG -n02808304/ILSVRC2012_val_00001384.JPEG -n02098286/ILSVRC2012_val_00013573.JPEG -n01698640/ILSVRC2012_val_00045338.JPEG -n02098286/ILSVRC2012_val_00041358.JPEG -n03495258/ILSVRC2012_val_00003669.JPEG -n03954731/ILSVRC2012_val_00015496.JPEG -n09193705/ILSVRC2012_val_00003282.JPEG -n02007558/ILSVRC2012_val_00047052.JPEG -n03216828/ILSVRC2012_val_00005968.JPEG -n03141823/ILSVRC2012_val_00014253.JPEG -n04486054/ILSVRC2012_val_00043357.JPEG -n01824575/ILSVRC2012_val_00014903.JPEG -n01737021/ILSVRC2012_val_00004040.JPEG -n01532829/ILSVRC2012_val_00031171.JPEG -n02124075/ILSVRC2012_val_00046964.JPEG -n02100877/ILSVRC2012_val_00004223.JPEG -n03916031/ILSVRC2012_val_00048412.JPEG -n03793489/ILSVRC2012_val_00018677.JPEG -n02113624/ILSVRC2012_val_00037437.JPEG -n03127925/ILSVRC2012_val_00005893.JPEG -n03208938/ILSVRC2012_val_00044142.JPEG -n03127925/ILSVRC2012_val_00045564.JPEG -n02086646/ILSVRC2012_val_00022521.JPEG -n03271574/ILSVRC2012_val_00022306.JPEG -n03877845/ILSVRC2012_val_00046055.JPEG -n01855032/ILSVRC2012_val_00007973.JPEG -n03063599/ILSVRC2012_val_00023112.JPEG -n03998194/ILSVRC2012_val_00022178.JPEG -n02091134/ILSVRC2012_val_00045708.JPEG -n03109150/ILSVRC2012_val_00000678.JPEG -n04204238/ILSVRC2012_val_00001633.JPEG -n03902125/ILSVRC2012_val_00021715.JPEG -n02699494/ILSVRC2012_val_00005723.JPEG -n02123597/ILSVRC2012_val_00015401.JPEG -n04344873/ILSVRC2012_val_00015043.JPEG -n07932039/ILSVRC2012_val_00043644.JPEG -n03742115/ILSVRC2012_val_00026974.JPEG -n02097130/ILSVRC2012_val_00014157.JPEG -n04208210/ILSVRC2012_val_00023586.JPEG -n03041632/ILSVRC2012_val_00002212.JPEG -n02487347/ILSVRC2012_val_00001656.JPEG -n03314780/ILSVRC2012_val_00038233.JPEG -n02110958/ILSVRC2012_val_00024249.JPEG -n03459775/ILSVRC2012_val_00023023.JPEG -n03188531/ILSVRC2012_val_00010735.JPEG -n04447861/ILSVRC2012_val_00018712.JPEG -n04505470/ILSVRC2012_val_00000666.JPEG -n02077923/ILSVRC2012_val_00027866.JPEG -n02808440/ILSVRC2012_val_00048839.JPEG -n03534580/ILSVRC2012_val_00002592.JPEG -n04229816/ILSVRC2012_val_00011305.JPEG -n02326432/ILSVRC2012_val_00022000.JPEG -n03947888/ILSVRC2012_val_00023104.JPEG -n02087046/ILSVRC2012_val_00037669.JPEG -n02129165/ILSVRC2012_val_00040607.JPEG -n03131574/ILSVRC2012_val_00004602.JPEG -n04590129/ILSVRC2012_val_00023987.JPEG -n02009229/ILSVRC2012_val_00016684.JPEG -n02093859/ILSVRC2012_val_00036491.JPEG -n02058221/ILSVRC2012_val_00031023.JPEG -n03637318/ILSVRC2012_val_00012139.JPEG -n02087394/ILSVRC2012_val_00019209.JPEG -n01798484/ILSVRC2012_val_00041832.JPEG -n02119789/ILSVRC2012_val_00002881.JPEG -n02441942/ILSVRC2012_val_00042355.JPEG -n03532672/ILSVRC2012_val_00019398.JPEG -n07579787/ILSVRC2012_val_00032565.JPEG -n03271574/ILSVRC2012_val_00031563.JPEG -n02423022/ILSVRC2012_val_00018436.JPEG -n04009552/ILSVRC2012_val_00020745.JPEG -n02110063/ILSVRC2012_val_00029783.JPEG -n02095889/ILSVRC2012_val_00015325.JPEG -n02489166/ILSVRC2012_val_00029721.JPEG -n02111129/ILSVRC2012_val_00041337.JPEG -n03769881/ILSVRC2012_val_00007633.JPEG -n04532670/ILSVRC2012_val_00024004.JPEG -n07768694/ILSVRC2012_val_00001514.JPEG -n01914609/ILSVRC2012_val_00048292.JPEG -n02978881/ILSVRC2012_val_00019144.JPEG -n02094433/ILSVRC2012_val_00019702.JPEG -n07768694/ILSVRC2012_val_00018186.JPEG -n02927161/ILSVRC2012_val_00039437.JPEG -n04074963/ILSVRC2012_val_00038370.JPEG -n01843383/ILSVRC2012_val_00020909.JPEG -n01677366/ILSVRC2012_val_00046346.JPEG -n03983396/ILSVRC2012_val_00001970.JPEG -n01882714/ILSVRC2012_val_00040233.JPEG -n01704323/ILSVRC2012_val_00040206.JPEG -n02114855/ILSVRC2012_val_00044732.JPEG -n02730930/ILSVRC2012_val_00016687.JPEG -n01580077/ILSVRC2012_val_00003386.JPEG -n02910353/ILSVRC2012_val_00041070.JPEG -n04458633/ILSVRC2012_val_00024934.JPEG -n03793489/ILSVRC2012_val_00021458.JPEG -n02667093/ILSVRC2012_val_00030369.JPEG -n01669191/ILSVRC2012_val_00048480.JPEG -n04404412/ILSVRC2012_val_00042660.JPEG -n02134084/ILSVRC2012_val_00030136.JPEG -n04131690/ILSVRC2012_val_00032630.JPEG -n02510455/ILSVRC2012_val_00026307.JPEG -n03930630/ILSVRC2012_val_00036504.JPEG -n03476991/ILSVRC2012_val_00037362.JPEG -n03697007/ILSVRC2012_val_00047600.JPEG -n03649909/ILSVRC2012_val_00038906.JPEG -n02883205/ILSVRC2012_val_00006456.JPEG -n03063689/ILSVRC2012_val_00033518.JPEG -n13044778/ILSVRC2012_val_00034526.JPEG -n02790996/ILSVRC2012_val_00010468.JPEG -n09468604/ILSVRC2012_val_00045923.JPEG -n01955084/ILSVRC2012_val_00042771.JPEG -n04116512/ILSVRC2012_val_00035956.JPEG -n04507155/ILSVRC2012_val_00022988.JPEG -n02123394/ILSVRC2012_val_00015372.JPEG -n04548362/ILSVRC2012_val_00006169.JPEG -n02106030/ILSVRC2012_val_00024292.JPEG -n02109961/ILSVRC2012_val_00049654.JPEG -n02916936/ILSVRC2012_val_00000366.JPEG -n01817953/ILSVRC2012_val_00015324.JPEG -n03000134/ILSVRC2012_val_00046648.JPEG -n01688243/ILSVRC2012_val_00043288.JPEG -n01930112/ILSVRC2012_val_00027709.JPEG -n04447861/ILSVRC2012_val_00025192.JPEG -n03344393/ILSVRC2012_val_00041736.JPEG -n03782006/ILSVRC2012_val_00036107.JPEG -n03623198/ILSVRC2012_val_00029975.JPEG -n02104029/ILSVRC2012_val_00028605.JPEG -n02892767/ILSVRC2012_val_00013681.JPEG -n03637318/ILSVRC2012_val_00002980.JPEG -n04200800/ILSVRC2012_val_00004257.JPEG -n03250847/ILSVRC2012_val_00026731.JPEG -n01734418/ILSVRC2012_val_00015762.JPEG -n03692522/ILSVRC2012_val_00003231.JPEG -n04604644/ILSVRC2012_val_00015677.JPEG -n07590611/ILSVRC2012_val_00031682.JPEG -n02088094/ILSVRC2012_val_00043993.JPEG -n02906734/ILSVRC2012_val_00016839.JPEG -n03617480/ILSVRC2012_val_00021792.JPEG -n02977058/ILSVRC2012_val_00037604.JPEG -n03825788/ILSVRC2012_val_00021292.JPEG -n03761084/ILSVRC2012_val_00005324.JPEG -n01491361/ILSVRC2012_val_00002922.JPEG -n04074963/ILSVRC2012_val_00010727.JPEG -n02128925/ILSVRC2012_val_00018859.JPEG -n03347037/ILSVRC2012_val_00028252.JPEG -n09399592/ILSVRC2012_val_00034546.JPEG -n02493793/ILSVRC2012_val_00032964.JPEG -n03935335/ILSVRC2012_val_00025716.JPEG -n01665541/ILSVRC2012_val_00000930.JPEG -n03991062/ILSVRC2012_val_00037312.JPEG -n02113624/ILSVRC2012_val_00020474.JPEG -n03459775/ILSVRC2012_val_00035643.JPEG -n02092339/ILSVRC2012_val_00020010.JPEG -n02102973/ILSVRC2012_val_00018530.JPEG -n02480855/ILSVRC2012_val_00005565.JPEG -n03773504/ILSVRC2012_val_00016588.JPEG -n03929855/ILSVRC2012_val_00018444.JPEG -n02099429/ILSVRC2012_val_00025714.JPEG -n04476259/ILSVRC2012_val_00040453.JPEG -n03938244/ILSVRC2012_val_00046170.JPEG -n02484975/ILSVRC2012_val_00000695.JPEG -n02490219/ILSVRC2012_val_00029804.JPEG -n03709823/ILSVRC2012_val_00025557.JPEG -n03617480/ILSVRC2012_val_00002198.JPEG -n01667114/ILSVRC2012_val_00032700.JPEG -n02655020/ILSVRC2012_val_00034545.JPEG -n01558993/ILSVRC2012_val_00019208.JPEG -n03290653/ILSVRC2012_val_00038152.JPEG -n02111277/ILSVRC2012_val_00034681.JPEG -n07749582/ILSVRC2012_val_00038212.JPEG -n02089867/ILSVRC2012_val_00002373.JPEG -n07614500/ILSVRC2012_val_00014743.JPEG -n02086646/ILSVRC2012_val_00030642.JPEG -n02102480/ILSVRC2012_val_00033198.JPEG -n03000134/ILSVRC2012_val_00047124.JPEG -n02123597/ILSVRC2012_val_00029839.JPEG -n02169497/ILSVRC2012_val_00019171.JPEG -n02480495/ILSVRC2012_val_00028784.JPEG -n04350905/ILSVRC2012_val_00033166.JPEG -n01877812/ILSVRC2012_val_00024264.JPEG -n03272010/ILSVRC2012_val_00048443.JPEG -n01665541/ILSVRC2012_val_00005919.JPEG -n02277742/ILSVRC2012_val_00000810.JPEG -n04604644/ILSVRC2012_val_00042739.JPEG -n03063689/ILSVRC2012_val_00043146.JPEG -n01775062/ILSVRC2012_val_00037348.JPEG -n02108422/ILSVRC2012_val_00020989.JPEG -n01664065/ILSVRC2012_val_00024221.JPEG -n03584829/ILSVRC2012_val_00026533.JPEG -n03450230/ILSVRC2012_val_00026368.JPEG -n02123159/ILSVRC2012_val_00047925.JPEG -n07871810/ILSVRC2012_val_00024893.JPEG -n04265275/ILSVRC2012_val_00047522.JPEG -n01984695/ILSVRC2012_val_00008182.JPEG -n01734418/ILSVRC2012_val_00028503.JPEG -n01818515/ILSVRC2012_val_00017029.JPEG -n04456115/ILSVRC2012_val_00049142.JPEG -n02437312/ILSVRC2012_val_00039249.JPEG -n03016953/ILSVRC2012_val_00043073.JPEG -n04286575/ILSVRC2012_val_00020426.JPEG -n01704323/ILSVRC2012_val_00024993.JPEG -n02165456/ILSVRC2012_val_00030697.JPEG -n01629819/ILSVRC2012_val_00011661.JPEG -n03706229/ILSVRC2012_val_00048067.JPEG -n02092002/ILSVRC2012_val_00000405.JPEG -n07711569/ILSVRC2012_val_00012622.JPEG -n03063689/ILSVRC2012_val_00012589.JPEG -n04258138/ILSVRC2012_val_00036418.JPEG -n02133161/ILSVRC2012_val_00039819.JPEG -n11879895/ILSVRC2012_val_00021912.JPEG -n02676566/ILSVRC2012_val_00037301.JPEG -n03417042/ILSVRC2012_val_00009345.JPEG -n02974003/ILSVRC2012_val_00048672.JPEG -n09472597/ILSVRC2012_val_00048040.JPEG -n01843383/ILSVRC2012_val_00040051.JPEG -n04251144/ILSVRC2012_val_00005230.JPEG -n02804610/ILSVRC2012_val_00018419.JPEG -n03594945/ILSVRC2012_val_00036304.JPEG -n03769881/ILSVRC2012_val_00010881.JPEG -n03786901/ILSVRC2012_val_00006394.JPEG -n03888257/ILSVRC2012_val_00024340.JPEG -n03026506/ILSVRC2012_val_00044713.JPEG -n03325584/ILSVRC2012_val_00030089.JPEG -n01667778/ILSVRC2012_val_00021982.JPEG -n03207743/ILSVRC2012_val_00034844.JPEG -n03899768/ILSVRC2012_val_00021364.JPEG -n02091244/ILSVRC2012_val_00020049.JPEG -n02871525/ILSVRC2012_val_00013599.JPEG -n01582220/ILSVRC2012_val_00030017.JPEG -n01882714/ILSVRC2012_val_00049217.JPEG -n04562935/ILSVRC2012_val_00016930.JPEG -n03961711/ILSVRC2012_val_00002850.JPEG -n04557648/ILSVRC2012_val_00036165.JPEG -n01735189/ILSVRC2012_val_00035429.JPEG -n02028035/ILSVRC2012_val_00006580.JPEG -n01631663/ILSVRC2012_val_00006737.JPEG -n01616318/ILSVRC2012_val_00035953.JPEG -n03249569/ILSVRC2012_val_00041425.JPEG -n04116512/ILSVRC2012_val_00001905.JPEG -n03461385/ILSVRC2012_val_00009965.JPEG -n03733281/ILSVRC2012_val_00038862.JPEG -n03908618/ILSVRC2012_val_00007771.JPEG -n03131574/ILSVRC2012_val_00016407.JPEG -n03982430/ILSVRC2012_val_00017419.JPEG -n01697457/ILSVRC2012_val_00034085.JPEG -n02101388/ILSVRC2012_val_00013927.JPEG -n02099849/ILSVRC2012_val_00044036.JPEG -n01669191/ILSVRC2012_val_00028162.JPEG -n02106550/ILSVRC2012_val_00026214.JPEG -n02098286/ILSVRC2012_val_00033033.JPEG -n02108422/ILSVRC2012_val_00008003.JPEG -n11879895/ILSVRC2012_val_00010754.JPEG -n04344873/ILSVRC2012_val_00027749.JPEG -n02107574/ILSVRC2012_val_00031343.JPEG -n03110669/ILSVRC2012_val_00038682.JPEG -n04346328/ILSVRC2012_val_00016110.JPEG -n02412080/ILSVRC2012_val_00023019.JPEG -n02992211/ILSVRC2012_val_00000736.JPEG -n03443371/ILSVRC2012_val_00003629.JPEG -n02138441/ILSVRC2012_val_00046136.JPEG -n02699494/ILSVRC2012_val_00016758.JPEG -n02190166/ILSVRC2012_val_00019460.JPEG -n02444819/ILSVRC2012_val_00017559.JPEG -n02415577/ILSVRC2012_val_00017917.JPEG -n02134084/ILSVRC2012_val_00042273.JPEG -n07695742/ILSVRC2012_val_00025057.JPEG -n02977058/ILSVRC2012_val_00005655.JPEG -n03467068/ILSVRC2012_val_00000169.JPEG -n03042490/ILSVRC2012_val_00047100.JPEG -n04548362/ILSVRC2012_val_00000866.JPEG -n01644900/ILSVRC2012_val_00004427.JPEG -n02133161/ILSVRC2012_val_00026962.JPEG -n04328186/ILSVRC2012_val_00044825.JPEG -n04517823/ILSVRC2012_val_00025993.JPEG -n11939491/ILSVRC2012_val_00034010.JPEG -n02110627/ILSVRC2012_val_00044949.JPEG -n02096585/ILSVRC2012_val_00015204.JPEG -n02321529/ILSVRC2012_val_00042639.JPEG -n03929660/ILSVRC2012_val_00018730.JPEG -n04049303/ILSVRC2012_val_00004440.JPEG -n03857828/ILSVRC2012_val_00022267.JPEG -n02123159/ILSVRC2012_val_00009964.JPEG -n01695060/ILSVRC2012_val_00039578.JPEG -n03776460/ILSVRC2012_val_00009264.JPEG -n01980166/ILSVRC2012_val_00000818.JPEG -n04239074/ILSVRC2012_val_00026618.JPEG -n03249569/ILSVRC2012_val_00016762.JPEG -n04141975/ILSVRC2012_val_00039660.JPEG -n02487347/ILSVRC2012_val_00029789.JPEG -n02110806/ILSVRC2012_val_00018009.JPEG -n03742115/ILSVRC2012_val_00038646.JPEG -n01756291/ILSVRC2012_val_00028598.JPEG -n03782006/ILSVRC2012_val_00032019.JPEG -n07590611/ILSVRC2012_val_00022369.JPEG -n02281787/ILSVRC2012_val_00028202.JPEG -n02134084/ILSVRC2012_val_00040753.JPEG -n03259280/ILSVRC2012_val_00006376.JPEG -n03424325/ILSVRC2012_val_00005566.JPEG -n04239074/ILSVRC2012_val_00021286.JPEG -n07753275/ILSVRC2012_val_00031427.JPEG -n02099429/ILSVRC2012_val_00027322.JPEG -n03047690/ILSVRC2012_val_00035313.JPEG -n03344393/ILSVRC2012_val_00018727.JPEG -n04462240/ILSVRC2012_val_00014334.JPEG -n02256656/ILSVRC2012_val_00003876.JPEG -n03042490/ILSVRC2012_val_00043697.JPEG -n01704323/ILSVRC2012_val_00003603.JPEG -n02480495/ILSVRC2012_val_00033841.JPEG -n03721384/ILSVRC2012_val_00022555.JPEG -n03445924/ILSVRC2012_val_00012668.JPEG -n04285008/ILSVRC2012_val_00022246.JPEG -n02437312/ILSVRC2012_val_00027955.JPEG -n01740131/ILSVRC2012_val_00021093.JPEG -n04357314/ILSVRC2012_val_00010438.JPEG -n02817516/ILSVRC2012_val_00015849.JPEG -n07892512/ILSVRC2012_val_00004421.JPEG -n04136333/ILSVRC2012_val_00000442.JPEG -n02480495/ILSVRC2012_val_00042045.JPEG -n04409515/ILSVRC2012_val_00010812.JPEG -n04266014/ILSVRC2012_val_00014262.JPEG -n04009552/ILSVRC2012_val_00045112.JPEG -n01537544/ILSVRC2012_val_00018676.JPEG -n01518878/ILSVRC2012_val_00010594.JPEG -n02098105/ILSVRC2012_val_00042628.JPEG -n02109961/ILSVRC2012_val_00040410.JPEG -n02268443/ILSVRC2012_val_00034192.JPEG -n03379051/ILSVRC2012_val_00012164.JPEG -n03840681/ILSVRC2012_val_00002324.JPEG -n02097047/ILSVRC2012_val_00022713.JPEG -n07615774/ILSVRC2012_val_00020614.JPEG -n02484975/ILSVRC2012_val_00022174.JPEG -n03649909/ILSVRC2012_val_00012348.JPEG -n03018349/ILSVRC2012_val_00018538.JPEG -n03954731/ILSVRC2012_val_00021761.JPEG -n03187595/ILSVRC2012_val_00021110.JPEG -n04069434/ILSVRC2012_val_00016109.JPEG -n03018349/ILSVRC2012_val_00028952.JPEG -n04209133/ILSVRC2012_val_00041604.JPEG -n04335435/ILSVRC2012_val_00012076.JPEG -n04392985/ILSVRC2012_val_00040920.JPEG -n04209133/ILSVRC2012_val_00046639.JPEG -n02279972/ILSVRC2012_val_00033925.JPEG -n02086240/ILSVRC2012_val_00014174.JPEG -n07802026/ILSVRC2012_val_00018320.JPEG -n03134739/ILSVRC2012_val_00024198.JPEG -n02325366/ILSVRC2012_val_00001670.JPEG -n07714571/ILSVRC2012_val_00040469.JPEG -n04252225/ILSVRC2012_val_00017609.JPEG -n01728572/ILSVRC2012_val_00032556.JPEG -n01833805/ILSVRC2012_val_00026090.JPEG -n02017213/ILSVRC2012_val_00014599.JPEG -n03876231/ILSVRC2012_val_00010790.JPEG -n04243546/ILSVRC2012_val_00046861.JPEG -n02794156/ILSVRC2012_val_00023345.JPEG -n03961711/ILSVRC2012_val_00010631.JPEG -n02487347/ILSVRC2012_val_00025312.JPEG -n01608432/ILSVRC2012_val_00025636.JPEG -n03041632/ILSVRC2012_val_00015679.JPEG -n07711569/ILSVRC2012_val_00044645.JPEG -n03223299/ILSVRC2012_val_00030296.JPEG -n07836838/ILSVRC2012_val_00008271.JPEG -n10565667/ILSVRC2012_val_00046317.JPEG -n02782093/ILSVRC2012_val_00047373.JPEG -n02687172/ILSVRC2012_val_00047290.JPEG -n01608432/ILSVRC2012_val_00000328.JPEG -n02640242/ILSVRC2012_val_00046943.JPEG -n12998815/ILSVRC2012_val_00000731.JPEG -n04311004/ILSVRC2012_val_00044125.JPEG -n02403003/ILSVRC2012_val_00001513.JPEG -n03662601/ILSVRC2012_val_00031941.JPEG -n03937543/ILSVRC2012_val_00017685.JPEG -n04344873/ILSVRC2012_val_00038296.JPEG -n02454379/ILSVRC2012_val_00024257.JPEG -n01629819/ILSVRC2012_val_00009616.JPEG -n03394916/ILSVRC2012_val_00005548.JPEG -n02056570/ILSVRC2012_val_00002352.JPEG -n02835271/ILSVRC2012_val_00037658.JPEG -n03866082/ILSVRC2012_val_00031324.JPEG -n01729322/ILSVRC2012_val_00012188.JPEG -n04033995/ILSVRC2012_val_00030906.JPEG -n04153751/ILSVRC2012_val_00001878.JPEG -n03141823/ILSVRC2012_val_00034736.JPEG -n04153751/ILSVRC2012_val_00031988.JPEG -n02487347/ILSVRC2012_val_00000095.JPEG -n09246464/ILSVRC2012_val_00031523.JPEG -n02102040/ILSVRC2012_val_00019414.JPEG -n02487347/ILSVRC2012_val_00022040.JPEG -n02106550/ILSVRC2012_val_00018972.JPEG -n01667114/ILSVRC2012_val_00046053.JPEG -n03804744/ILSVRC2012_val_00039068.JPEG -n03877845/ILSVRC2012_val_00034639.JPEG -n01558993/ILSVRC2012_val_00001598.JPEG -n04501370/ILSVRC2012_val_00032392.JPEG -n07747607/ILSVRC2012_val_00045093.JPEG -n02109961/ILSVRC2012_val_00008320.JPEG -n02109047/ILSVRC2012_val_00004731.JPEG -n03633091/ILSVRC2012_val_00037904.JPEG -n02917067/ILSVRC2012_val_00022610.JPEG -n01740131/ILSVRC2012_val_00030229.JPEG -n03272562/ILSVRC2012_val_00030381.JPEG -n02793495/ILSVRC2012_val_00009958.JPEG -n04074963/ILSVRC2012_val_00000275.JPEG -n03445924/ILSVRC2012_val_00026881.JPEG -n03690938/ILSVRC2012_val_00020840.JPEG -n02963159/ILSVRC2012_val_00046157.JPEG -n03478589/ILSVRC2012_val_00035615.JPEG -n02108000/ILSVRC2012_val_00008114.JPEG -n02988304/ILSVRC2012_val_00013693.JPEG -n02110341/ILSVRC2012_val_00030476.JPEG -n04350905/ILSVRC2012_val_00040799.JPEG -n03394916/ILSVRC2012_val_00034826.JPEG -n02786058/ILSVRC2012_val_00013543.JPEG -n02102973/ILSVRC2012_val_00044387.JPEG -n02120079/ILSVRC2012_val_00001335.JPEG -n02536864/ILSVRC2012_val_00019489.JPEG -n04270147/ILSVRC2012_val_00045450.JPEG -n02107683/ILSVRC2012_val_00011322.JPEG -n04200800/ILSVRC2012_val_00036014.JPEG -n03388043/ILSVRC2012_val_00019659.JPEG -n04005630/ILSVRC2012_val_00040234.JPEG -n02939185/ILSVRC2012_val_00011050.JPEG -n03724870/ILSVRC2012_val_00008913.JPEG -n01692333/ILSVRC2012_val_00037417.JPEG -n03710637/ILSVRC2012_val_00007816.JPEG -n03393912/ILSVRC2012_val_00020706.JPEG -n02799071/ILSVRC2012_val_00040705.JPEG -n03961711/ILSVRC2012_val_00041237.JPEG -n03873416/ILSVRC2012_val_00012609.JPEG -n03494278/ILSVRC2012_val_00022564.JPEG -n03649909/ILSVRC2012_val_00029405.JPEG -n02087046/ILSVRC2012_val_00042289.JPEG -n02791124/ILSVRC2012_val_00044551.JPEG -n07734744/ILSVRC2012_val_00046024.JPEG -n03016953/ILSVRC2012_val_00010764.JPEG -n07718747/ILSVRC2012_val_00019457.JPEG -n02802426/ILSVRC2012_val_00022998.JPEG -n03196217/ILSVRC2012_val_00023369.JPEG -n01665541/ILSVRC2012_val_00009639.JPEG -n09428293/ILSVRC2012_val_00028242.JPEG -n06874185/ILSVRC2012_val_00025516.JPEG -n04507155/ILSVRC2012_val_00046429.JPEG -n01806143/ILSVRC2012_val_00040811.JPEG -n01817953/ILSVRC2012_val_00024592.JPEG -n02110958/ILSVRC2012_val_00029246.JPEG -n01735189/ILSVRC2012_val_00045256.JPEG -n02749479/ILSVRC2012_val_00002719.JPEG -n04049303/ILSVRC2012_val_00046327.JPEG -n01980166/ILSVRC2012_val_00005836.JPEG -n04548280/ILSVRC2012_val_00041547.JPEG -n02951585/ILSVRC2012_val_00044537.JPEG -n07718472/ILSVRC2012_val_00001920.JPEG -n04131690/ILSVRC2012_val_00048078.JPEG -n03976467/ILSVRC2012_val_00017894.JPEG -n03825788/ILSVRC2012_val_00038428.JPEG -n03874293/ILSVRC2012_val_00008933.JPEG -n04592741/ILSVRC2012_val_00028274.JPEG -n02667093/ILSVRC2012_val_00008504.JPEG -n02088632/ILSVRC2012_val_00031347.JPEG -n02412080/ILSVRC2012_val_00022880.JPEG -n02117135/ILSVRC2012_val_00023004.JPEG -n02219486/ILSVRC2012_val_00032504.JPEG -n06785654/ILSVRC2012_val_00041680.JPEG -n03109150/ILSVRC2012_val_00019143.JPEG -n01614925/ILSVRC2012_val_00042380.JPEG -n02037110/ILSVRC2012_val_00014991.JPEG -n04325704/ILSVRC2012_val_00047302.JPEG -n03666591/ILSVRC2012_val_00047097.JPEG -n02110958/ILSVRC2012_val_00030877.JPEG -n01440764/ILSVRC2012_val_00046252.JPEG -n01534433/ILSVRC2012_val_00014139.JPEG -n03445924/ILSVRC2012_val_00041371.JPEG -n02965783/ILSVRC2012_val_00000215.JPEG -n03538406/ILSVRC2012_val_00044450.JPEG -n02009229/ILSVRC2012_val_00048814.JPEG -n02939185/ILSVRC2012_val_00006325.JPEG -n04270147/ILSVRC2012_val_00049955.JPEG -n02797295/ILSVRC2012_val_00039627.JPEG -n02108551/ILSVRC2012_val_00039687.JPEG -n01632458/ILSVRC2012_val_00009779.JPEG -n02992211/ILSVRC2012_val_00014603.JPEG -n03042490/ILSVRC2012_val_00018709.JPEG -n02097298/ILSVRC2012_val_00025649.JPEG -n03970156/ILSVRC2012_val_00041302.JPEG -n02099429/ILSVRC2012_val_00044629.JPEG -n04251144/ILSVRC2012_val_00023981.JPEG -n01631663/ILSVRC2012_val_00019116.JPEG -n03871628/ILSVRC2012_val_00008492.JPEG -n01669191/ILSVRC2012_val_00002355.JPEG -n04509417/ILSVRC2012_val_00011412.JPEG -n01440764/ILSVRC2012_val_00009396.JPEG -n04326547/ILSVRC2012_val_00038127.JPEG -n01871265/ILSVRC2012_val_00038632.JPEG -n02422699/ILSVRC2012_val_00015081.JPEG -n02172182/ILSVRC2012_val_00037889.JPEG -n01735189/ILSVRC2012_val_00034054.JPEG -n03724870/ILSVRC2012_val_00035216.JPEG -n07716358/ILSVRC2012_val_00018255.JPEG -n07714571/ILSVRC2012_val_00029321.JPEG -n03042490/ILSVRC2012_val_00021502.JPEG -n01740131/ILSVRC2012_val_00033547.JPEG -n01847000/ILSVRC2012_val_00047009.JPEG -n03793489/ILSVRC2012_val_00032713.JPEG -n07753113/ILSVRC2012_val_00002124.JPEG -n01532829/ILSVRC2012_val_00013456.JPEG -n02317335/ILSVRC2012_val_00029527.JPEG -n04081281/ILSVRC2012_val_00020263.JPEG -n03089624/ILSVRC2012_val_00027919.JPEG -n02096177/ILSVRC2012_val_00044615.JPEG -n02277742/ILSVRC2012_val_00047996.JPEG -n02089973/ILSVRC2012_val_00020403.JPEG -n03467068/ILSVRC2012_val_00042555.JPEG -n02085620/ILSVRC2012_val_00045358.JPEG -n03884397/ILSVRC2012_val_00041523.JPEG -n02927161/ILSVRC2012_val_00021162.JPEG -n03443371/ILSVRC2012_val_00018786.JPEG -n06794110/ILSVRC2012_val_00019897.JPEG -n03384352/ILSVRC2012_val_00021815.JPEG -n02488291/ILSVRC2012_val_00002477.JPEG -n01824575/ILSVRC2012_val_00000929.JPEG -n04330267/ILSVRC2012_val_00033191.JPEG -n03642806/ILSVRC2012_val_00017736.JPEG -n03877472/ILSVRC2012_val_00012917.JPEG -n04356056/ILSVRC2012_val_00023754.JPEG -n02094114/ILSVRC2012_val_00042473.JPEG -n07836838/ILSVRC2012_val_00038954.JPEG -n02457408/ILSVRC2012_val_00027033.JPEG -n03180011/ILSVRC2012_val_00001757.JPEG -n03710721/ILSVRC2012_val_00010762.JPEG -n04081281/ILSVRC2012_val_00043267.JPEG -n02113978/ILSVRC2012_val_00021475.JPEG -n04023962/ILSVRC2012_val_00040578.JPEG -n03089624/ILSVRC2012_val_00008801.JPEG -n03908618/ILSVRC2012_val_00037720.JPEG -n03840681/ILSVRC2012_val_00029902.JPEG -n03124043/ILSVRC2012_val_00007876.JPEG -n04376876/ILSVRC2012_val_00002160.JPEG -n02793495/ILSVRC2012_val_00032510.JPEG -n03272562/ILSVRC2012_val_00040246.JPEG -n04479046/ILSVRC2012_val_00026891.JPEG -n04376876/ILSVRC2012_val_00046671.JPEG -n02788148/ILSVRC2012_val_00048426.JPEG -n04239074/ILSVRC2012_val_00006471.JPEG -n07614500/ILSVRC2012_val_00044555.JPEG -n04146614/ILSVRC2012_val_00031870.JPEG -n01737021/ILSVRC2012_val_00004143.JPEG -n06785654/ILSVRC2012_val_00042236.JPEG -n04335435/ILSVRC2012_val_00046946.JPEG -n01601694/ILSVRC2012_val_00033770.JPEG -n03761084/ILSVRC2012_val_00008779.JPEG -n03930313/ILSVRC2012_val_00019384.JPEG -n07714571/ILSVRC2012_val_00039258.JPEG -n04270147/ILSVRC2012_val_00031162.JPEG -n03724870/ILSVRC2012_val_00040925.JPEG -n02097047/ILSVRC2012_val_00045674.JPEG -n02481823/ILSVRC2012_val_00027941.JPEG -n02444819/ILSVRC2012_val_00032447.JPEG -n02108915/ILSVRC2012_val_00049895.JPEG -n02326432/ILSVRC2012_val_00049897.JPEG -n07873807/ILSVRC2012_val_00035995.JPEG -n01917289/ILSVRC2012_val_00001057.JPEG -n03496892/ILSVRC2012_val_00032094.JPEG -n04548280/ILSVRC2012_val_00004662.JPEG -n02363005/ILSVRC2012_val_00026915.JPEG -n04317175/ILSVRC2012_val_00041030.JPEG -n01883070/ILSVRC2012_val_00042485.JPEG -n04417672/ILSVRC2012_val_00037701.JPEG -n02977058/ILSVRC2012_val_00034150.JPEG -n03788195/ILSVRC2012_val_00029390.JPEG -n04033995/ILSVRC2012_val_00031404.JPEG -n02906734/ILSVRC2012_val_00038800.JPEG -n01817953/ILSVRC2012_val_00038169.JPEG -n02105855/ILSVRC2012_val_00019155.JPEG -n02109525/ILSVRC2012_val_00042657.JPEG -n03967562/ILSVRC2012_val_00030925.JPEG -n04254680/ILSVRC2012_val_00005713.JPEG -n04026417/ILSVRC2012_val_00031129.JPEG -n07684084/ILSVRC2012_val_00003329.JPEG -n03662601/ILSVRC2012_val_00015311.JPEG -n07745940/ILSVRC2012_val_00013499.JPEG -n04127249/ILSVRC2012_val_00040574.JPEG -n03950228/ILSVRC2012_val_00047381.JPEG -n02093647/ILSVRC2012_val_00035940.JPEG -n04179913/ILSVRC2012_val_00037519.JPEG -n15075141/ILSVRC2012_val_00006953.JPEG -n04275548/ILSVRC2012_val_00012305.JPEG -n01945685/ILSVRC2012_val_00031116.JPEG -n02988304/ILSVRC2012_val_00036782.JPEG -n02091635/ILSVRC2012_val_00014357.JPEG -n07745940/ILSVRC2012_val_00012645.JPEG -n02514041/ILSVRC2012_val_00020803.JPEG -n02102480/ILSVRC2012_val_00041335.JPEG -n03443371/ILSVRC2012_val_00022560.JPEG -n07768694/ILSVRC2012_val_00010111.JPEG -n03485794/ILSVRC2012_val_00033870.JPEG -n01698640/ILSVRC2012_val_00043473.JPEG -n02865351/ILSVRC2012_val_00000763.JPEG -n03452741/ILSVRC2012_val_00039717.JPEG -n02133161/ILSVRC2012_val_00033328.JPEG -n04355933/ILSVRC2012_val_00047580.JPEG -n04239074/ILSVRC2012_val_00008251.JPEG -n03599486/ILSVRC2012_val_00013642.JPEG -n15075141/ILSVRC2012_val_00021201.JPEG -n04141327/ILSVRC2012_val_00013147.JPEG -n02480855/ILSVRC2012_val_00017200.JPEG -n07873807/ILSVRC2012_val_00043781.JPEG -n04285008/ILSVRC2012_val_00025465.JPEG -n04005630/ILSVRC2012_val_00004865.JPEG -n02939185/ILSVRC2012_val_00031452.JPEG -n02101556/ILSVRC2012_val_00045011.JPEG -n03950228/ILSVRC2012_val_00032633.JPEG -n03773504/ILSVRC2012_val_00045982.JPEG -n01978287/ILSVRC2012_val_00040223.JPEG -n03877472/ILSVRC2012_val_00010560.JPEG -n03874293/ILSVRC2012_val_00020720.JPEG -n04252225/ILSVRC2012_val_00025743.JPEG -n02488702/ILSVRC2012_val_00048491.JPEG -n02877765/ILSVRC2012_val_00030650.JPEG -n01494475/ILSVRC2012_val_00005339.JPEG -n04152593/ILSVRC2012_val_00034850.JPEG -n02799071/ILSVRC2012_val_00026506.JPEG -n13044778/ILSVRC2012_val_00041738.JPEG -n02129165/ILSVRC2012_val_00012353.JPEG -n03908618/ILSVRC2012_val_00020480.JPEG -n01518878/ILSVRC2012_val_00023778.JPEG -n03065424/ILSVRC2012_val_00025594.JPEG -n03018349/ILSVRC2012_val_00028820.JPEG -n01756291/ILSVRC2012_val_00042384.JPEG -n02497673/ILSVRC2012_val_00027093.JPEG -n04523525/ILSVRC2012_val_00029686.JPEG -n03146219/ILSVRC2012_val_00023457.JPEG -n03467068/ILSVRC2012_val_00016636.JPEG -n02097658/ILSVRC2012_val_00005174.JPEG -n01753488/ILSVRC2012_val_00036910.JPEG -n02437312/ILSVRC2012_val_00011354.JPEG -n04591713/ILSVRC2012_val_00031837.JPEG -n01537544/ILSVRC2012_val_00029310.JPEG -n01824575/ILSVRC2012_val_00027555.JPEG -n02102973/ILSVRC2012_val_00036140.JPEG -n01728572/ILSVRC2012_val_00049017.JPEG -n03532672/ILSVRC2012_val_00017687.JPEG -n02104029/ILSVRC2012_val_00033303.JPEG -n01753488/ILSVRC2012_val_00016066.JPEG -n02110063/ILSVRC2012_val_00008270.JPEG -n02655020/ILSVRC2012_val_00019369.JPEG -n04409515/ILSVRC2012_val_00010370.JPEG -n09468604/ILSVRC2012_val_00007658.JPEG -n03770679/ILSVRC2012_val_00015295.JPEG -n02486261/ILSVRC2012_val_00020727.JPEG -n02132136/ILSVRC2012_val_00006703.JPEG -n02799071/ILSVRC2012_val_00037792.JPEG -n03496892/ILSVRC2012_val_00022171.JPEG -n03871628/ILSVRC2012_val_00027103.JPEG -n04404412/ILSVRC2012_val_00049406.JPEG -n03291819/ILSVRC2012_val_00018067.JPEG -n07753113/ILSVRC2012_val_00002179.JPEG -n01697457/ILSVRC2012_val_00006286.JPEG -n07749582/ILSVRC2012_val_00048264.JPEG -n02028035/ILSVRC2012_val_00024808.JPEG -n06794110/ILSVRC2012_val_00000995.JPEG -n03791053/ILSVRC2012_val_00004619.JPEG -n03670208/ILSVRC2012_val_00044032.JPEG -n03291819/ILSVRC2012_val_00017382.JPEG -n02410509/ILSVRC2012_val_00040511.JPEG -n04067472/ILSVRC2012_val_00043864.JPEG -n04009552/ILSVRC2012_val_00038006.JPEG -n02169497/ILSVRC2012_val_00024558.JPEG -n04243546/ILSVRC2012_val_00014313.JPEG -n04004767/ILSVRC2012_val_00032509.JPEG -n04606251/ILSVRC2012_val_00028404.JPEG -n03796401/ILSVRC2012_val_00022413.JPEG -n03124170/ILSVRC2012_val_00024379.JPEG -n02091244/ILSVRC2012_val_00009065.JPEG -n07684084/ILSVRC2012_val_00000820.JPEG -n03337140/ILSVRC2012_val_00048016.JPEG -n01682714/ILSVRC2012_val_00020154.JPEG -n01697457/ILSVRC2012_val_00001573.JPEG -n03710637/ILSVRC2012_val_00013356.JPEG -n07730033/ILSVRC2012_val_00022010.JPEG -n01773549/ILSVRC2012_val_00002688.JPEG -n02106662/ILSVRC2012_val_00045684.JPEG -n03584254/ILSVRC2012_val_00027569.JPEG -n03786901/ILSVRC2012_val_00048332.JPEG -n02676566/ILSVRC2012_val_00029538.JPEG -n03782006/ILSVRC2012_val_00041449.JPEG -n03777754/ILSVRC2012_val_00048137.JPEG -n01484850/ILSVRC2012_val_00037135.JPEG -n02917067/ILSVRC2012_val_00009056.JPEG -n01688243/ILSVRC2012_val_00047224.JPEG -n04525305/ILSVRC2012_val_00043189.JPEG -n04133789/ILSVRC2012_val_00030250.JPEG -n04252077/ILSVRC2012_val_00009906.JPEG -n04465501/ILSVRC2012_val_00035906.JPEG -n03692522/ILSVRC2012_val_00035877.JPEG -n01873310/ILSVRC2012_val_00000940.JPEG -n01910747/ILSVRC2012_val_00039314.JPEG -n02484975/ILSVRC2012_val_00008539.JPEG -n02094114/ILSVRC2012_val_00049926.JPEG -n03709823/ILSVRC2012_val_00034532.JPEG -n02916936/ILSVRC2012_val_00006943.JPEG -n02091032/ILSVRC2012_val_00022684.JPEG -n03929660/ILSVRC2012_val_00046594.JPEG -n02231487/ILSVRC2012_val_00047704.JPEG -n04493381/ILSVRC2012_val_00038337.JPEG -n01537544/ILSVRC2012_val_00037454.JPEG -n04328186/ILSVRC2012_val_00045064.JPEG -n02422699/ILSVRC2012_val_00007395.JPEG -n02074367/ILSVRC2012_val_00028337.JPEG -n03627232/ILSVRC2012_val_00039484.JPEG -n07753275/ILSVRC2012_val_00045126.JPEG -n13054560/ILSVRC2012_val_00040478.JPEG -n02097130/ILSVRC2012_val_00045760.JPEG -n04152593/ILSVRC2012_val_00031893.JPEG -n03544143/ILSVRC2012_val_00035185.JPEG -n04254680/ILSVRC2012_val_00014154.JPEG -n04356056/ILSVRC2012_val_00042996.JPEG -n02804610/ILSVRC2012_val_00014404.JPEG -n03690938/ILSVRC2012_val_00032514.JPEG -n06794110/ILSVRC2012_val_00034107.JPEG -n02093754/ILSVRC2012_val_00004053.JPEG -n03041632/ILSVRC2012_val_00022700.JPEG -n01689811/ILSVRC2012_val_00008561.JPEG -n03775546/ILSVRC2012_val_00011309.JPEG -n01630670/ILSVRC2012_val_00003702.JPEG -n03983396/ILSVRC2012_val_00030143.JPEG -n03787032/ILSVRC2012_val_00026799.JPEG -n02769748/ILSVRC2012_val_00009722.JPEG -n02280649/ILSVRC2012_val_00000031.JPEG -n02093859/ILSVRC2012_val_00031752.JPEG -n02096585/ILSVRC2012_val_00043564.JPEG -n03775071/ILSVRC2012_val_00036862.JPEG -n03888605/ILSVRC2012_val_00028863.JPEG -n04552348/ILSVRC2012_val_00047921.JPEG -n09229709/ILSVRC2012_val_00018097.JPEG -n02104029/ILSVRC2012_val_00025299.JPEG -n02100735/ILSVRC2012_val_00043577.JPEG -n03877472/ILSVRC2012_val_00010822.JPEG -n02951358/ILSVRC2012_val_00021156.JPEG -n03920288/ILSVRC2012_val_00012663.JPEG -n02497673/ILSVRC2012_val_00002652.JPEG -n03201208/ILSVRC2012_val_00029073.JPEG -n01667114/ILSVRC2012_val_00000229.JPEG -n02112137/ILSVRC2012_val_00028936.JPEG -n03697007/ILSVRC2012_val_00001874.JPEG -n02102973/ILSVRC2012_val_00013752.JPEG -n03874293/ILSVRC2012_val_00002995.JPEG -n03814639/ILSVRC2012_val_00015284.JPEG -n02095570/ILSVRC2012_val_00023143.JPEG -n02124075/ILSVRC2012_val_00040600.JPEG -n04204238/ILSVRC2012_val_00048590.JPEG -n02110958/ILSVRC2012_val_00047583.JPEG -n04443257/ILSVRC2012_val_00001510.JPEG -n03028079/ILSVRC2012_val_00019331.JPEG -n01692333/ILSVRC2012_val_00041887.JPEG -n03345487/ILSVRC2012_val_00019954.JPEG -n02113712/ILSVRC2012_val_00002460.JPEG -n03535780/ILSVRC2012_val_00012133.JPEG -n02106662/ILSVRC2012_val_00032995.JPEG -n01682714/ILSVRC2012_val_00001942.JPEG -n07695742/ILSVRC2012_val_00013493.JPEG -n01644373/ILSVRC2012_val_00032917.JPEG -n03063689/ILSVRC2012_val_00044838.JPEG -n02088364/ILSVRC2012_val_00008675.JPEG -n03483316/ILSVRC2012_val_00000876.JPEG -n06874185/ILSVRC2012_val_00004646.JPEG -n02782093/ILSVRC2012_val_00012050.JPEG -n02277742/ILSVRC2012_val_00022979.JPEG -n02104029/ILSVRC2012_val_00036264.JPEG -n04515003/ILSVRC2012_val_00032863.JPEG -n01855032/ILSVRC2012_val_00024273.JPEG -n01532829/ILSVRC2012_val_00042370.JPEG -n02992529/ILSVRC2012_val_00000089.JPEG -n03976657/ILSVRC2012_val_00018564.JPEG -n03832673/ILSVRC2012_val_00031166.JPEG -n02823428/ILSVRC2012_val_00008121.JPEG -n04040759/ILSVRC2012_val_00040259.JPEG -n03763968/ILSVRC2012_val_00013572.JPEG -n03837869/ILSVRC2012_val_00044841.JPEG -n03325584/ILSVRC2012_val_00027255.JPEG -n01689811/ILSVRC2012_val_00009980.JPEG -n01798484/ILSVRC2012_val_00031938.JPEG -n12768682/ILSVRC2012_val_00027020.JPEG -n03662601/ILSVRC2012_val_00006294.JPEG -n02342885/ILSVRC2012_val_00020285.JPEG -n02441942/ILSVRC2012_val_00019059.JPEG -n01693334/ILSVRC2012_val_00026626.JPEG -n09421951/ILSVRC2012_val_00006564.JPEG -n01968897/ILSVRC2012_val_00040877.JPEG -n02825657/ILSVRC2012_val_00015948.JPEG -n03840681/ILSVRC2012_val_00046678.JPEG -n01644373/ILSVRC2012_val_00043860.JPEG -n02672831/ILSVRC2012_val_00020900.JPEG -n02190166/ILSVRC2012_val_00011393.JPEG -n03982430/ILSVRC2012_val_00046837.JPEG -n01608432/ILSVRC2012_val_00028139.JPEG -n03179701/ILSVRC2012_val_00003559.JPEG -n02361337/ILSVRC2012_val_00047875.JPEG -n02104029/ILSVRC2012_val_00019060.JPEG -n04553703/ILSVRC2012_val_00034191.JPEG -n07760859/ILSVRC2012_val_00011505.JPEG -n03014705/ILSVRC2012_val_00037817.JPEG -n02096051/ILSVRC2012_val_00022763.JPEG -n03388043/ILSVRC2012_val_00004673.JPEG -n01796340/ILSVRC2012_val_00013641.JPEG -n04131690/ILSVRC2012_val_00019434.JPEG -n02447366/ILSVRC2012_val_00041647.JPEG -n03710193/ILSVRC2012_val_00019006.JPEG -n01641577/ILSVRC2012_val_00003748.JPEG -n04525038/ILSVRC2012_val_00006115.JPEG -n02086240/ILSVRC2012_val_00042523.JPEG -n03666591/ILSVRC2012_val_00014322.JPEG -n02094114/ILSVRC2012_val_00026313.JPEG -n04033995/ILSVRC2012_val_00026389.JPEG -n01685808/ILSVRC2012_val_00024684.JPEG -n01729977/ILSVRC2012_val_00029550.JPEG -n02978881/ILSVRC2012_val_00017819.JPEG -n02916936/ILSVRC2012_val_00018897.JPEG -n02910353/ILSVRC2012_val_00036043.JPEG -n02606052/ILSVRC2012_val_00005062.JPEG -n02687172/ILSVRC2012_val_00011510.JPEG -n04311174/ILSVRC2012_val_00048484.JPEG -n01689811/ILSVRC2012_val_00017174.JPEG -n02109525/ILSVRC2012_val_00003390.JPEG -n02096437/ILSVRC2012_val_00009036.JPEG -n02777292/ILSVRC2012_val_00007181.JPEG -n04065272/ILSVRC2012_val_00030525.JPEG -n03733281/ILSVRC2012_val_00034428.JPEG -n02096437/ILSVRC2012_val_00026267.JPEG -n02091635/ILSVRC2012_val_00022982.JPEG -n04254120/ILSVRC2012_val_00040688.JPEG -n02415577/ILSVRC2012_val_00022966.JPEG -n03594734/ILSVRC2012_val_00008023.JPEG -n02859443/ILSVRC2012_val_00011660.JPEG -n04037443/ILSVRC2012_val_00020181.JPEG -n03933933/ILSVRC2012_val_00020648.JPEG -n01518878/ILSVRC2012_val_00001398.JPEG -n01843383/ILSVRC2012_val_00024423.JPEG -n02094258/ILSVRC2012_val_00019094.JPEG -n02106030/ILSVRC2012_val_00011698.JPEG -n03721384/ILSVRC2012_val_00019690.JPEG -n03710721/ILSVRC2012_val_00047553.JPEG -n03372029/ILSVRC2012_val_00014407.JPEG -n01704323/ILSVRC2012_val_00010061.JPEG -n03530642/ILSVRC2012_val_00020609.JPEG -n04486054/ILSVRC2012_val_00044806.JPEG -n01770081/ILSVRC2012_val_00039298.JPEG -n02099601/ILSVRC2012_val_00035252.JPEG -n02101556/ILSVRC2012_val_00030540.JPEG -n03902125/ILSVRC2012_val_00022006.JPEG -n03255030/ILSVRC2012_val_00027650.JPEG -n03887697/ILSVRC2012_val_00021774.JPEG -n02028035/ILSVRC2012_val_00030102.JPEG -n02169497/ILSVRC2012_val_00021794.JPEG -n02098413/ILSVRC2012_val_00011063.JPEG -n03376595/ILSVRC2012_val_00012771.JPEG -n03788365/ILSVRC2012_val_00008171.JPEG -n01986214/ILSVRC2012_val_00005125.JPEG -n01498041/ILSVRC2012_val_00046497.JPEG -n03476684/ILSVRC2012_val_00044415.JPEG -n02102177/ILSVRC2012_val_00048513.JPEG -n03796401/ILSVRC2012_val_00025827.JPEG -n01883070/ILSVRC2012_val_00043018.JPEG -n02105162/ILSVRC2012_val_00030061.JPEG -n02457408/ILSVRC2012_val_00015126.JPEG -n01734418/ILSVRC2012_val_00017967.JPEG -n04099969/ILSVRC2012_val_00001953.JPEG -n01981276/ILSVRC2012_val_00021080.JPEG -n03877845/ILSVRC2012_val_00003804.JPEG -n01601694/ILSVRC2012_val_00021656.JPEG -n02095570/ILSVRC2012_val_00002714.JPEG -n01818515/ILSVRC2012_val_00030397.JPEG -n04525305/ILSVRC2012_val_00046926.JPEG -n03794056/ILSVRC2012_val_00003270.JPEG -n07871810/ILSVRC2012_val_00042577.JPEG -n02454379/ILSVRC2012_val_00025235.JPEG -n12998815/ILSVRC2012_val_00015658.JPEG -n07590611/ILSVRC2012_val_00010005.JPEG -n02815834/ILSVRC2012_val_00019648.JPEG -n03417042/ILSVRC2012_val_00043909.JPEG -n04554684/ILSVRC2012_val_00006551.JPEG -n02113023/ILSVRC2012_val_00045068.JPEG -n03775546/ILSVRC2012_val_00005835.JPEG -n02403003/ILSVRC2012_val_00023387.JPEG -n03873416/ILSVRC2012_val_00032858.JPEG -n02823428/ILSVRC2012_val_00031840.JPEG -n03857828/ILSVRC2012_val_00027075.JPEG -n02526121/ILSVRC2012_val_00025800.JPEG -n02129604/ILSVRC2012_val_00027576.JPEG -n02108551/ILSVRC2012_val_00039969.JPEG -n02892201/ILSVRC2012_val_00017464.JPEG -n02098105/ILSVRC2012_val_00032049.JPEG -n02231487/ILSVRC2012_val_00020649.JPEG -n02096177/ILSVRC2012_val_00009789.JPEG -n04118776/ILSVRC2012_val_00035583.JPEG -n02687172/ILSVRC2012_val_00032182.JPEG -n03843555/ILSVRC2012_val_00007698.JPEG -n02840245/ILSVRC2012_val_00037762.JPEG -n04239074/ILSVRC2012_val_00041869.JPEG -n02488702/ILSVRC2012_val_00045991.JPEG -n06596364/ILSVRC2012_val_00004468.JPEG -n03417042/ILSVRC2012_val_00004119.JPEG -n02113023/ILSVRC2012_val_00021908.JPEG -n02091244/ILSVRC2012_val_00029555.JPEG -n04548362/ILSVRC2012_val_00021957.JPEG -n02794156/ILSVRC2012_val_00009861.JPEG -n02113799/ILSVRC2012_val_00038393.JPEG -n04192698/ILSVRC2012_val_00024634.JPEG -n04476259/ILSVRC2012_val_00037691.JPEG -n02963159/ILSVRC2012_val_00016246.JPEG -n03461385/ILSVRC2012_val_00043522.JPEG -n04548280/ILSVRC2012_val_00015199.JPEG -n01773549/ILSVRC2012_val_00028677.JPEG -n07873807/ILSVRC2012_val_00014607.JPEG -n01843383/ILSVRC2012_val_00018780.JPEG -n01687978/ILSVRC2012_val_00042029.JPEG -n04192698/ILSVRC2012_val_00026513.JPEG -n02116738/ILSVRC2012_val_00016666.JPEG -n01608432/ILSVRC2012_val_00001813.JPEG -n02965783/ILSVRC2012_val_00003090.JPEG -n02492035/ILSVRC2012_val_00043766.JPEG -n01751748/ILSVRC2012_val_00007885.JPEG -n03769881/ILSVRC2012_val_00030789.JPEG -n02497673/ILSVRC2012_val_00044774.JPEG -n02124075/ILSVRC2012_val_00026403.JPEG -n04550184/ILSVRC2012_val_00024717.JPEG -n02114712/ILSVRC2012_val_00018735.JPEG -n04120489/ILSVRC2012_val_00035596.JPEG -n04033901/ILSVRC2012_val_00036076.JPEG -n07718747/ILSVRC2012_val_00011881.JPEG -n03188531/ILSVRC2012_val_00033272.JPEG -n02018207/ILSVRC2012_val_00048163.JPEG -n03018349/ILSVRC2012_val_00010121.JPEG -n04522168/ILSVRC2012_val_00035219.JPEG -n04462240/ILSVRC2012_val_00017863.JPEG -n04536866/ILSVRC2012_val_00027587.JPEG -n09288635/ILSVRC2012_val_00011567.JPEG -n02793495/ILSVRC2012_val_00016366.JPEG -n02096437/ILSVRC2012_val_00047757.JPEG -n02093647/ILSVRC2012_val_00013323.JPEG -n02124075/ILSVRC2012_val_00014773.JPEG -n03016953/ILSVRC2012_val_00004641.JPEG -n02168699/ILSVRC2012_val_00006867.JPEG -n02086646/ILSVRC2012_val_00025911.JPEG -n03297495/ILSVRC2012_val_00010821.JPEG -n03676483/ILSVRC2012_val_00048584.JPEG -n12985857/ILSVRC2012_val_00047107.JPEG -n02098105/ILSVRC2012_val_00008926.JPEG -n09229709/ILSVRC2012_val_00027662.JPEG -n04517823/ILSVRC2012_val_00031756.JPEG -n02100877/ILSVRC2012_val_00021079.JPEG -n03729826/ILSVRC2012_val_00042218.JPEG -n01872401/ILSVRC2012_val_00025946.JPEG -n03544143/ILSVRC2012_val_00048860.JPEG -n04251144/ILSVRC2012_val_00043728.JPEG -n01910747/ILSVRC2012_val_00028644.JPEG -n07734744/ILSVRC2012_val_00028371.JPEG -n02892767/ILSVRC2012_val_00040902.JPEG -n03776460/ILSVRC2012_val_00001631.JPEG -n12267677/ILSVRC2012_val_00031998.JPEG -n07871810/ILSVRC2012_val_00018608.JPEG -n04179913/ILSVRC2012_val_00042808.JPEG -n04606251/ILSVRC2012_val_00040924.JPEG -n01608432/ILSVRC2012_val_00046733.JPEG -n02492660/ILSVRC2012_val_00041501.JPEG -n04252077/ILSVRC2012_val_00044879.JPEG -n03584254/ILSVRC2012_val_00014368.JPEG -n03249569/ILSVRC2012_val_00010381.JPEG -n03016953/ILSVRC2012_val_00004544.JPEG -n04285008/ILSVRC2012_val_00009607.JPEG -n02259212/ILSVRC2012_val_00046801.JPEG -n01833805/ILSVRC2012_val_00016746.JPEG -n02123045/ILSVRC2012_val_00020136.JPEG -n02111129/ILSVRC2012_val_00021370.JPEG -n03903868/ILSVRC2012_val_00004595.JPEG -n01806143/ILSVRC2012_val_00005751.JPEG -n01631663/ILSVRC2012_val_00033866.JPEG -n01829413/ILSVRC2012_val_00029283.JPEG -n02687172/ILSVRC2012_val_00026474.JPEG -n04285008/ILSVRC2012_val_00029021.JPEG -n02666196/ILSVRC2012_val_00005970.JPEG -n02097047/ILSVRC2012_val_00037277.JPEG -n02093859/ILSVRC2012_val_00010356.JPEG -n01968897/ILSVRC2012_val_00037882.JPEG -n04589890/ILSVRC2012_val_00006743.JPEG -n01693334/ILSVRC2012_val_00025809.JPEG -n02526121/ILSVRC2012_val_00020757.JPEG -n02107312/ILSVRC2012_val_00004386.JPEG -n07880968/ILSVRC2012_val_00024099.JPEG -n03627232/ILSVRC2012_val_00049916.JPEG -n02834397/ILSVRC2012_val_00036586.JPEG -n02172182/ILSVRC2012_val_00049259.JPEG -n04423845/ILSVRC2012_val_00041828.JPEG -n03355925/ILSVRC2012_val_00049608.JPEG -n03297495/ILSVRC2012_val_00046108.JPEG -n03840681/ILSVRC2012_val_00022877.JPEG -n03873416/ILSVRC2012_val_00045082.JPEG -n03857828/ILSVRC2012_val_00022455.JPEG -n02704792/ILSVRC2012_val_00049628.JPEG -n02356798/ILSVRC2012_val_00042187.JPEG -n02397096/ILSVRC2012_val_00044182.JPEG -n04118538/ILSVRC2012_val_00019086.JPEG -n04019541/ILSVRC2012_val_00038669.JPEG -n02951358/ILSVRC2012_val_00037707.JPEG -n02493509/ILSVRC2012_val_00024398.JPEG -n03393912/ILSVRC2012_val_00009717.JPEG -n04423845/ILSVRC2012_val_00043115.JPEG -n03676483/ILSVRC2012_val_00018920.JPEG -n03804744/ILSVRC2012_val_00043239.JPEG -n02177972/ILSVRC2012_val_00031097.JPEG -n04252225/ILSVRC2012_val_00019444.JPEG -n09399592/ILSVRC2012_val_00013304.JPEG -n03967562/ILSVRC2012_val_00010642.JPEG -n01910747/ILSVRC2012_val_00027727.JPEG -n04192698/ILSVRC2012_val_00041324.JPEG -n09835506/ILSVRC2012_val_00032275.JPEG -n02110627/ILSVRC2012_val_00040979.JPEG -n03047690/ILSVRC2012_val_00030064.JPEG -n04465501/ILSVRC2012_val_00024074.JPEG -n03908618/ILSVRC2012_val_00007562.JPEG -n02086240/ILSVRC2012_val_00015738.JPEG -n01828970/ILSVRC2012_val_00000051.JPEG -n02101006/ILSVRC2012_val_00016085.JPEG -n04467665/ILSVRC2012_val_00030748.JPEG -n02099712/ILSVRC2012_val_00023471.JPEG -n01829413/ILSVRC2012_val_00014256.JPEG -n03623198/ILSVRC2012_val_00019441.JPEG -n01950731/ILSVRC2012_val_00029111.JPEG -n02098413/ILSVRC2012_val_00008487.JPEG -n02101388/ILSVRC2012_val_00024686.JPEG -n03691459/ILSVRC2012_val_00004050.JPEG -n02389026/ILSVRC2012_val_00045409.JPEG -n02481823/ILSVRC2012_val_00048741.JPEG -n03791053/ILSVRC2012_val_00018448.JPEG -n02690373/ILSVRC2012_val_00018616.JPEG -n03388043/ILSVRC2012_val_00034662.JPEG -n02110627/ILSVRC2012_val_00038448.JPEG -n02799071/ILSVRC2012_val_00022766.JPEG -n01537544/ILSVRC2012_val_00002071.JPEG -n04209133/ILSVRC2012_val_00037601.JPEG -n07697537/ILSVRC2012_val_00034152.JPEG -n01728572/ILSVRC2012_val_00036166.JPEG -n07920052/ILSVRC2012_val_00005624.JPEG -n01644373/ILSVRC2012_val_00016177.JPEG -n04458633/ILSVRC2012_val_00040845.JPEG -n01644900/ILSVRC2012_val_00012231.JPEG -n02487347/ILSVRC2012_val_00036612.JPEG -n03977966/ILSVRC2012_val_00001143.JPEG -n04317175/ILSVRC2012_val_00017086.JPEG -n02389026/ILSVRC2012_val_00014304.JPEG -n01910747/ILSVRC2012_val_00016200.JPEG -n03494278/ILSVRC2012_val_00034610.JPEG -n03240683/ILSVRC2012_val_00035038.JPEG -n04208210/ILSVRC2012_val_00031032.JPEG -n03223299/ILSVRC2012_val_00032622.JPEG -n04398044/ILSVRC2012_val_00005162.JPEG -n04525305/ILSVRC2012_val_00009844.JPEG -n04019541/ILSVRC2012_val_00028690.JPEG -n02769748/ILSVRC2012_val_00042327.JPEG -n03291819/ILSVRC2012_val_00042559.JPEG -n02454379/ILSVRC2012_val_00018994.JPEG -n03417042/ILSVRC2012_val_00021253.JPEG -n04367480/ILSVRC2012_val_00015088.JPEG -n02107683/ILSVRC2012_val_00030297.JPEG -n02114548/ILSVRC2012_val_00012800.JPEG -n02939185/ILSVRC2012_val_00024512.JPEG -n03376595/ILSVRC2012_val_00001685.JPEG -n02107574/ILSVRC2012_val_00007071.JPEG -n02415577/ILSVRC2012_val_00033844.JPEG -n04517823/ILSVRC2012_val_00000712.JPEG -n12768682/ILSVRC2012_val_00014129.JPEG -n06596364/ILSVRC2012_val_00014214.JPEG -n03877472/ILSVRC2012_val_00020630.JPEG -n02871525/ILSVRC2012_val_00003028.JPEG -n02120079/ILSVRC2012_val_00015412.JPEG -n02814533/ILSVRC2012_val_00047473.JPEG -n03825788/ILSVRC2012_val_00000540.JPEG -n07697537/ILSVRC2012_val_00003930.JPEG -n04335435/ILSVRC2012_val_00024071.JPEG -n03676483/ILSVRC2012_val_00049924.JPEG -n02236044/ILSVRC2012_val_00001948.JPEG -n02086910/ILSVRC2012_val_00019603.JPEG -n03777754/ILSVRC2012_val_00029230.JPEG -n07871810/ILSVRC2012_val_00014713.JPEG -n01530575/ILSVRC2012_val_00033092.JPEG -n07760859/ILSVRC2012_val_00031970.JPEG -n07613480/ILSVRC2012_val_00019023.JPEG -n02088632/ILSVRC2012_val_00030757.JPEG -n04554684/ILSVRC2012_val_00038965.JPEG -n07730033/ILSVRC2012_val_00046230.JPEG -n01667114/ILSVRC2012_val_00046803.JPEG -n02454379/ILSVRC2012_val_00009806.JPEG -n02841315/ILSVRC2012_val_00003215.JPEG -n04041544/ILSVRC2012_val_00007349.JPEG -n01768244/ILSVRC2012_val_00019564.JPEG -n02105251/ILSVRC2012_val_00049272.JPEG -n02802426/ILSVRC2012_val_00038836.JPEG -n02091032/ILSVRC2012_val_00002024.JPEG -n02127052/ILSVRC2012_val_00035366.JPEG -n07718747/ILSVRC2012_val_00004031.JPEG -n02025239/ILSVRC2012_val_00038431.JPEG -n12985857/ILSVRC2012_val_00002047.JPEG -n02641379/ILSVRC2012_val_00033541.JPEG -n02013706/ILSVRC2012_val_00005203.JPEG -n02794156/ILSVRC2012_val_00045655.JPEG -n03794056/ILSVRC2012_val_00020103.JPEG -n02106382/ILSVRC2012_val_00020216.JPEG -n07613480/ILSVRC2012_val_00011413.JPEG -n12057211/ILSVRC2012_val_00036925.JPEG -n02134084/ILSVRC2012_val_00001498.JPEG -n02086240/ILSVRC2012_val_00038882.JPEG -n03026506/ILSVRC2012_val_00029270.JPEG -n03933933/ILSVRC2012_val_00045013.JPEG -n02169497/ILSVRC2012_val_00020159.JPEG -n03950228/ILSVRC2012_val_00034717.JPEG -n02892767/ILSVRC2012_val_00004599.JPEG -n04254680/ILSVRC2012_val_00017782.JPEG -n07831146/ILSVRC2012_val_00016446.JPEG -n03709823/ILSVRC2012_val_00033950.JPEG -n02011460/ILSVRC2012_val_00004572.JPEG -n02356798/ILSVRC2012_val_00006392.JPEG -n02115913/ILSVRC2012_val_00041976.JPEG -n02342885/ILSVRC2012_val_00027868.JPEG -n02018207/ILSVRC2012_val_00009270.JPEG -n03649909/ILSVRC2012_val_00038677.JPEG -n03495258/ILSVRC2012_val_00030189.JPEG -n02117135/ILSVRC2012_val_00043079.JPEG -n03337140/ILSVRC2012_val_00009094.JPEG -n03201208/ILSVRC2012_val_00002772.JPEG -n03085013/ILSVRC2012_val_00035954.JPEG -n03770679/ILSVRC2012_val_00048610.JPEG -n04204347/ILSVRC2012_val_00008219.JPEG -n02077923/ILSVRC2012_val_00007193.JPEG -n02791124/ILSVRC2012_val_00007789.JPEG -n04065272/ILSVRC2012_val_00028467.JPEG -n04146614/ILSVRC2012_val_00038209.JPEG -n02980441/ILSVRC2012_val_00011145.JPEG -n02089078/ILSVRC2012_val_00018840.JPEG -n04070727/ILSVRC2012_val_00023711.JPEG -n03196217/ILSVRC2012_val_00041483.JPEG -n02971356/ILSVRC2012_val_00030546.JPEG -n02102973/ILSVRC2012_val_00032521.JPEG -n03961711/ILSVRC2012_val_00006124.JPEG -n04355338/ILSVRC2012_val_00046496.JPEG -n02006656/ILSVRC2012_val_00005186.JPEG -n03376595/ILSVRC2012_val_00036783.JPEG -n04458633/ILSVRC2012_val_00016593.JPEG -n07583066/ILSVRC2012_val_00041753.JPEG -n02165105/ILSVRC2012_val_00048477.JPEG -n02445715/ILSVRC2012_val_00026215.JPEG -n02113978/ILSVRC2012_val_00044187.JPEG -n02319095/ILSVRC2012_val_00008674.JPEG -n02006656/ILSVRC2012_val_00014266.JPEG -n07753592/ILSVRC2012_val_00023220.JPEG -n07717556/ILSVRC2012_val_00029151.JPEG -n03759954/ILSVRC2012_val_00025450.JPEG -n04509417/ILSVRC2012_val_00031602.JPEG -n03188531/ILSVRC2012_val_00002075.JPEG -n07730033/ILSVRC2012_val_00000881.JPEG -n02410509/ILSVRC2012_val_00044556.JPEG -n02165105/ILSVRC2012_val_00012894.JPEG -n04252077/ILSVRC2012_val_00011313.JPEG -n02321529/ILSVRC2012_val_00011857.JPEG -n04125021/ILSVRC2012_val_00037377.JPEG -n01667114/ILSVRC2012_val_00023686.JPEG -n02111129/ILSVRC2012_val_00038704.JPEG -n02109961/ILSVRC2012_val_00006059.JPEG -n04428191/ILSVRC2012_val_00003800.JPEG -n02116738/ILSVRC2012_val_00037354.JPEG -n03379051/ILSVRC2012_val_00007649.JPEG -n02110627/ILSVRC2012_val_00027947.JPEG -n03884397/ILSVRC2012_val_00030107.JPEG -n04251144/ILSVRC2012_val_00044813.JPEG -n02808304/ILSVRC2012_val_00005585.JPEG -n04204238/ILSVRC2012_val_00012046.JPEG -n04009552/ILSVRC2012_val_00029794.JPEG -n04404412/ILSVRC2012_val_00024928.JPEG -n01882714/ILSVRC2012_val_00045070.JPEG -n02963159/ILSVRC2012_val_00038207.JPEG -n03062245/ILSVRC2012_val_00000344.JPEG -n06596364/ILSVRC2012_val_00000985.JPEG -n03476991/ILSVRC2012_val_00004493.JPEG -n02119789/ILSVRC2012_val_00046145.JPEG -n02115641/ILSVRC2012_val_00033743.JPEG -n01847000/ILSVRC2012_val_00016631.JPEG -n02480495/ILSVRC2012_val_00010576.JPEG -n02088238/ILSVRC2012_val_00027114.JPEG -n02108915/ILSVRC2012_val_00007318.JPEG -n04592741/ILSVRC2012_val_00001901.JPEG -n03220513/ILSVRC2012_val_00003991.JPEG -n02480855/ILSVRC2012_val_00044145.JPEG -n04005630/ILSVRC2012_val_00004526.JPEG -n03388549/ILSVRC2012_val_00048243.JPEG -n04328186/ILSVRC2012_val_00020153.JPEG -n04275548/ILSVRC2012_val_00039054.JPEG -n04252225/ILSVRC2012_val_00034323.JPEG -n07715103/ILSVRC2012_val_00046030.JPEG -n04525038/ILSVRC2012_val_00025160.JPEG -n01694178/ILSVRC2012_val_00021092.JPEG -n07718747/ILSVRC2012_val_00037649.JPEG -n04557648/ILSVRC2012_val_00008077.JPEG -n07565083/ILSVRC2012_val_00010286.JPEG -n04086273/ILSVRC2012_val_00012865.JPEG -n04442312/ILSVRC2012_val_00023688.JPEG -n02823750/ILSVRC2012_val_00045499.JPEG -n03124170/ILSVRC2012_val_00008522.JPEG -n01978455/ILSVRC2012_val_00020970.JPEG -n02110185/ILSVRC2012_val_00036248.JPEG -n02825657/ILSVRC2012_val_00018459.JPEG -n02410509/ILSVRC2012_val_00020346.JPEG -n04133789/ILSVRC2012_val_00032308.JPEG -n02028035/ILSVRC2012_val_00013808.JPEG -n01773157/ILSVRC2012_val_00032657.JPEG -n02840245/ILSVRC2012_val_00008578.JPEG -n02177972/ILSVRC2012_val_00047065.JPEG -n13052670/ILSVRC2012_val_00020929.JPEG -n03223299/ILSVRC2012_val_00004775.JPEG -n03532672/ILSVRC2012_val_00007783.JPEG -n01755581/ILSVRC2012_val_00029615.JPEG -n03018349/ILSVRC2012_val_00015123.JPEG -n04355933/ILSVRC2012_val_00005414.JPEG -n01828970/ILSVRC2012_val_00048824.JPEG -n03063599/ILSVRC2012_val_00028575.JPEG -n02115641/ILSVRC2012_val_00042731.JPEG -n03770679/ILSVRC2012_val_00007374.JPEG -n04612504/ILSVRC2012_val_00013403.JPEG -n03447447/ILSVRC2012_val_00016814.JPEG -n02128385/ILSVRC2012_val_00046032.JPEG -n03599486/ILSVRC2012_val_00008961.JPEG -n03146219/ILSVRC2012_val_00034509.JPEG -n02804610/ILSVRC2012_val_00030862.JPEG -n02977058/ILSVRC2012_val_00033182.JPEG -n02398521/ILSVRC2012_val_00009331.JPEG -n03866082/ILSVRC2012_val_00041091.JPEG -n02415577/ILSVRC2012_val_00024683.JPEG -n04389033/ILSVRC2012_val_00021989.JPEG -n04509417/ILSVRC2012_val_00045666.JPEG -n01773157/ILSVRC2012_val_00001618.JPEG -n02113186/ILSVRC2012_val_00040018.JPEG -n02487347/ILSVRC2012_val_00032067.JPEG -n03599486/ILSVRC2012_val_00022242.JPEG -n01985128/ILSVRC2012_val_00028073.JPEG -n04254120/ILSVRC2012_val_00009872.JPEG -n02277742/ILSVRC2012_val_00004850.JPEG -n02843684/ILSVRC2012_val_00041742.JPEG -n07754684/ILSVRC2012_val_00007036.JPEG -n04074963/ILSVRC2012_val_00008267.JPEG -n03594945/ILSVRC2012_val_00019410.JPEG -n03424325/ILSVRC2012_val_00011700.JPEG -n01693334/ILSVRC2012_val_00041069.JPEG -n02206856/ILSVRC2012_val_00016575.JPEG -n04147183/ILSVRC2012_val_00021861.JPEG -n03160309/ILSVRC2012_val_00031658.JPEG -n03344393/ILSVRC2012_val_00020486.JPEG -n02804610/ILSVRC2012_val_00001369.JPEG -n02011460/ILSVRC2012_val_00028920.JPEG -n01592084/ILSVRC2012_val_00036534.JPEG -n02058221/ILSVRC2012_val_00017197.JPEG -n03838899/ILSVRC2012_val_00021771.JPEG -n02123045/ILSVRC2012_val_00033837.JPEG -n03384352/ILSVRC2012_val_00038817.JPEG -n01773549/ILSVRC2012_val_00004271.JPEG -n02172182/ILSVRC2012_val_00045513.JPEG -n03958227/ILSVRC2012_val_00008350.JPEG -n04517823/ILSVRC2012_val_00021506.JPEG -n03496892/ILSVRC2012_val_00048862.JPEG -n02977058/ILSVRC2012_val_00034597.JPEG -n03792782/ILSVRC2012_val_00041265.JPEG -n03109150/ILSVRC2012_val_00020055.JPEG -n04332243/ILSVRC2012_val_00009360.JPEG -n02100877/ILSVRC2012_val_00031882.JPEG -n03759954/ILSVRC2012_val_00049128.JPEG -n04136333/ILSVRC2012_val_00001348.JPEG -n02843684/ILSVRC2012_val_00014364.JPEG -n03032252/ILSVRC2012_val_00040566.JPEG -n04005630/ILSVRC2012_val_00033648.JPEG -n02509815/ILSVRC2012_val_00018974.JPEG -n07714571/ILSVRC2012_val_00003020.JPEG -n01494475/ILSVRC2012_val_00016395.JPEG -n03776460/ILSVRC2012_val_00012900.JPEG -n03017168/ILSVRC2012_val_00018301.JPEG -n01558993/ILSVRC2012_val_00030475.JPEG -n02791270/ILSVRC2012_val_00010017.JPEG -n03930313/ILSVRC2012_val_00045178.JPEG -n03733131/ILSVRC2012_val_00038830.JPEG -n03496892/ILSVRC2012_val_00027389.JPEG -n02028035/ILSVRC2012_val_00006499.JPEG -n02971356/ILSVRC2012_val_00013098.JPEG -n03425413/ILSVRC2012_val_00042375.JPEG -n02977058/ILSVRC2012_val_00021001.JPEG -n03534580/ILSVRC2012_val_00040896.JPEG -n01828970/ILSVRC2012_val_00047229.JPEG -n07734744/ILSVRC2012_val_00041837.JPEG -n02894605/ILSVRC2012_val_00016237.JPEG -n03417042/ILSVRC2012_val_00019147.JPEG -n01873310/ILSVRC2012_val_00038330.JPEG -n03792782/ILSVRC2012_val_00027449.JPEG -n02814533/ILSVRC2012_val_00026029.JPEG -n01872401/ILSVRC2012_val_00000319.JPEG -n03937543/ILSVRC2012_val_00005519.JPEG -n02489166/ILSVRC2012_val_00018017.JPEG -n03089624/ILSVRC2012_val_00001190.JPEG -n03775546/ILSVRC2012_val_00042695.JPEG -n02992529/ILSVRC2012_val_00031438.JPEG -n04154565/ILSVRC2012_val_00034399.JPEG -n01806567/ILSVRC2012_val_00028529.JPEG -n03670208/ILSVRC2012_val_00039264.JPEG -n03187595/ILSVRC2012_val_00031617.JPEG -n02441942/ILSVRC2012_val_00037395.JPEG -n03803284/ILSVRC2012_val_00014884.JPEG -n02410509/ILSVRC2012_val_00039117.JPEG -n03495258/ILSVRC2012_val_00019556.JPEG -n04483307/ILSVRC2012_val_00033339.JPEG -n02892201/ILSVRC2012_val_00023195.JPEG -n02025239/ILSVRC2012_val_00048735.JPEG -n03188531/ILSVRC2012_val_00027528.JPEG -n03976467/ILSVRC2012_val_00028460.JPEG -n02096051/ILSVRC2012_val_00014738.JPEG -n01530575/ILSVRC2012_val_00000747.JPEG -n02113186/ILSVRC2012_val_00001494.JPEG -n02112137/ILSVRC2012_val_00028420.JPEG -n03124170/ILSVRC2012_val_00005346.JPEG -n03899768/ILSVRC2012_val_00045144.JPEG -n02074367/ILSVRC2012_val_00017048.JPEG -n03125729/ILSVRC2012_val_00036063.JPEG -n02815834/ILSVRC2012_val_00027061.JPEG -n03201208/ILSVRC2012_val_00034560.JPEG -n02326432/ILSVRC2012_val_00028827.JPEG -n01843383/ILSVRC2012_val_00046755.JPEG -n03877472/ILSVRC2012_val_00001870.JPEG -n03271574/ILSVRC2012_val_00022054.JPEG -n03770679/ILSVRC2012_val_00047910.JPEG -n02484975/ILSVRC2012_val_00038869.JPEG -n03461385/ILSVRC2012_val_00020954.JPEG -n02992211/ILSVRC2012_val_00006782.JPEG -n01532829/ILSVRC2012_val_00007237.JPEG -n04552348/ILSVRC2012_val_00023701.JPEG -n03832673/ILSVRC2012_val_00038350.JPEG -n02114712/ILSVRC2012_val_00017363.JPEG -n03188531/ILSVRC2012_val_00039191.JPEG -n03201208/ILSVRC2012_val_00021136.JPEG -n02100236/ILSVRC2012_val_00039507.JPEG -n01631663/ILSVRC2012_val_00037037.JPEG -n02747177/ILSVRC2012_val_00048583.JPEG -n01807496/ILSVRC2012_val_00034512.JPEG -n03709823/ILSVRC2012_val_00023792.JPEG -n04311174/ILSVRC2012_val_00013300.JPEG -n07716358/ILSVRC2012_val_00013114.JPEG -n02086240/ILSVRC2012_val_00036888.JPEG -n03100240/ILSVRC2012_val_00008981.JPEG -n03733281/ILSVRC2012_val_00024860.JPEG -n02319095/ILSVRC2012_val_00039456.JPEG -n03249569/ILSVRC2012_val_00041539.JPEG -n04553703/ILSVRC2012_val_00026584.JPEG -n03803284/ILSVRC2012_val_00012683.JPEG -n03840681/ILSVRC2012_val_00014053.JPEG -n03781244/ILSVRC2012_val_00028195.JPEG -n04487394/ILSVRC2012_val_00020259.JPEG -n02823428/ILSVRC2012_val_00031867.JPEG -n02025239/ILSVRC2012_val_00046677.JPEG -n04127249/ILSVRC2012_val_00016436.JPEG -n03089624/ILSVRC2012_val_00036006.JPEG -n03891251/ILSVRC2012_val_00036218.JPEG -n03787032/ILSVRC2012_val_00041446.JPEG -n13040303/ILSVRC2012_val_00021986.JPEG -n04350905/ILSVRC2012_val_00000380.JPEG -n01667778/ILSVRC2012_val_00013301.JPEG -n04540053/ILSVRC2012_val_00048430.JPEG -n03063599/ILSVRC2012_val_00017456.JPEG -n03220513/ILSVRC2012_val_00032479.JPEG -n02825657/ILSVRC2012_val_00018151.JPEG -n01742172/ILSVRC2012_val_00043349.JPEG -n03529860/ILSVRC2012_val_00025117.JPEG -n02056570/ILSVRC2012_val_00028670.JPEG -n01795545/ILSVRC2012_val_00002991.JPEG -n03297495/ILSVRC2012_val_00006536.JPEG -n03538406/ILSVRC2012_val_00049729.JPEG -n02123159/ILSVRC2012_val_00019303.JPEG -n02110341/ILSVRC2012_val_00036389.JPEG -n02825657/ILSVRC2012_val_00027229.JPEG -n04120489/ILSVRC2012_val_00004532.JPEG -n04026417/ILSVRC2012_val_00037263.JPEG -n02443484/ILSVRC2012_val_00043098.JPEG -n03372029/ILSVRC2012_val_00031149.JPEG -n02111277/ILSVRC2012_val_00005626.JPEG -n03345487/ILSVRC2012_val_00034340.JPEG -n01819313/ILSVRC2012_val_00013822.JPEG -n03095699/ILSVRC2012_val_00003200.JPEG -n02437616/ILSVRC2012_val_00005960.JPEG -n04428191/ILSVRC2012_val_00036974.JPEG -n02494079/ILSVRC2012_val_00014065.JPEG -n02672831/ILSVRC2012_val_00042605.JPEG -n07760859/ILSVRC2012_val_00039541.JPEG -n04204238/ILSVRC2012_val_00048927.JPEG -n01688243/ILSVRC2012_val_00008011.JPEG -n02091032/ILSVRC2012_val_00002325.JPEG -n04418357/ILSVRC2012_val_00016570.JPEG -n04325704/ILSVRC2012_val_00026365.JPEG -n02098286/ILSVRC2012_val_00013050.JPEG -n01530575/ILSVRC2012_val_00006763.JPEG -n02802426/ILSVRC2012_val_00026413.JPEG -n02690373/ILSVRC2012_val_00030411.JPEG -n03325584/ILSVRC2012_val_00007663.JPEG -n03000684/ILSVRC2012_val_00026061.JPEG -n01833805/ILSVRC2012_val_00016325.JPEG -n07716906/ILSVRC2012_val_00021629.JPEG -n04542943/ILSVRC2012_val_00000797.JPEG -n03032252/ILSVRC2012_val_00046675.JPEG -n04204238/ILSVRC2012_val_00011252.JPEG -n03459775/ILSVRC2012_val_00046453.JPEG -n01944390/ILSVRC2012_val_00010996.JPEG -n02483362/ILSVRC2012_val_00009925.JPEG -n09472597/ILSVRC2012_val_00021559.JPEG -n03272010/ILSVRC2012_val_00008655.JPEG -n02417914/ILSVRC2012_val_00029315.JPEG -n03873416/ILSVRC2012_val_00008700.JPEG -n03494278/ILSVRC2012_val_00001367.JPEG -n02105412/ILSVRC2012_val_00037290.JPEG -n04553703/ILSVRC2012_val_00007994.JPEG -n01560419/ILSVRC2012_val_00018680.JPEG -n03786901/ILSVRC2012_val_00043703.JPEG -n03908714/ILSVRC2012_val_00021673.JPEG -n02268853/ILSVRC2012_val_00004215.JPEG -n01689811/ILSVRC2012_val_00007482.JPEG -n03347037/ILSVRC2012_val_00008269.JPEG -n03888257/ILSVRC2012_val_00015689.JPEG -n13054560/ILSVRC2012_val_00007572.JPEG -n02666196/ILSVRC2012_val_00037785.JPEG -n03314780/ILSVRC2012_val_00017163.JPEG -n02793495/ILSVRC2012_val_00024047.JPEG -n02325366/ILSVRC2012_val_00020917.JPEG -n04273569/ILSVRC2012_val_00042203.JPEG -n01742172/ILSVRC2012_val_00028810.JPEG -n04325704/ILSVRC2012_val_00044116.JPEG -n02276258/ILSVRC2012_val_00037056.JPEG -n03188531/ILSVRC2012_val_00023348.JPEG -n02088364/ILSVRC2012_val_00047309.JPEG -n04476259/ILSVRC2012_val_00002432.JPEG -n04239074/ILSVRC2012_val_00033585.JPEG -n04008634/ILSVRC2012_val_00020190.JPEG -n09256479/ILSVRC2012_val_00007613.JPEG -n04275548/ILSVRC2012_val_00045949.JPEG -n02817516/ILSVRC2012_val_00041077.JPEG -n03775071/ILSVRC2012_val_00023240.JPEG -n04525038/ILSVRC2012_val_00010262.JPEG -n01632777/ILSVRC2012_val_00025348.JPEG -n02101388/ILSVRC2012_val_00041460.JPEG -n02120079/ILSVRC2012_val_00034932.JPEG -n04336792/ILSVRC2012_val_00023176.JPEG -n01677366/ILSVRC2012_val_00028764.JPEG -n03538406/ILSVRC2012_val_00037435.JPEG -n03478589/ILSVRC2012_val_00017907.JPEG -n02107312/ILSVRC2012_val_00004733.JPEG -n03944341/ILSVRC2012_val_00012260.JPEG -n02097209/ILSVRC2012_val_00010718.JPEG -n09288635/ILSVRC2012_val_00043251.JPEG -n03929855/ILSVRC2012_val_00043840.JPEG -n04389033/ILSVRC2012_val_00047296.JPEG -n03937543/ILSVRC2012_val_00005967.JPEG -n04591713/ILSVRC2012_val_00012594.JPEG -n03584254/ILSVRC2012_val_00044047.JPEG -n07880968/ILSVRC2012_val_00013959.JPEG -n03160309/ILSVRC2012_val_00018927.JPEG -n04208210/ILSVRC2012_val_00012030.JPEG -n03662601/ILSVRC2012_val_00028180.JPEG -n02190166/ILSVRC2012_val_00043104.JPEG -n04423845/ILSVRC2012_val_00046188.JPEG -n03630383/ILSVRC2012_val_00016123.JPEG -n02090721/ILSVRC2012_val_00006382.JPEG -n01824575/ILSVRC2012_val_00036702.JPEG -n02090721/ILSVRC2012_val_00014293.JPEG -n09246464/ILSVRC2012_val_00046022.JPEG -n01990800/ILSVRC2012_val_00036289.JPEG -n03594734/ILSVRC2012_val_00045877.JPEG -n03930630/ILSVRC2012_val_00027997.JPEG -n02443484/ILSVRC2012_val_00011477.JPEG -n13044778/ILSVRC2012_val_00040976.JPEG -n02727426/ILSVRC2012_val_00039440.JPEG -n02106030/ILSVRC2012_val_00018046.JPEG -n10148035/ILSVRC2012_val_00001410.JPEG -n03743016/ILSVRC2012_val_00012483.JPEG -n02108551/ILSVRC2012_val_00013964.JPEG -n03016953/ILSVRC2012_val_00039301.JPEG -n03710193/ILSVRC2012_val_00013096.JPEG -n02105505/ILSVRC2012_val_00007534.JPEG -n02099849/ILSVRC2012_val_00017377.JPEG -n02002724/ILSVRC2012_val_00029692.JPEG -n03014705/ILSVRC2012_val_00008418.JPEG -n04392985/ILSVRC2012_val_00001249.JPEG -n04532106/ILSVRC2012_val_00048503.JPEG -n04423845/ILSVRC2012_val_00026002.JPEG -n02102480/ILSVRC2012_val_00041060.JPEG -n02096177/ILSVRC2012_val_00048476.JPEG -n01534433/ILSVRC2012_val_00030657.JPEG -n03876231/ILSVRC2012_val_00016070.JPEG -n02799071/ILSVRC2012_val_00021134.JPEG -n04429376/ILSVRC2012_val_00031730.JPEG -n03483316/ILSVRC2012_val_00016875.JPEG -n04560804/ILSVRC2012_val_00012572.JPEG -n02708093/ILSVRC2012_val_00017378.JPEG -n02120505/ILSVRC2012_val_00044957.JPEG -n03630383/ILSVRC2012_val_00004189.JPEG -n01698640/ILSVRC2012_val_00039475.JPEG -n09256479/ILSVRC2012_val_00004126.JPEG -n03314780/ILSVRC2012_val_00043430.JPEG -n03908714/ILSVRC2012_val_00034235.JPEG -n03998194/ILSVRC2012_val_00035541.JPEG -n01985128/ILSVRC2012_val_00049686.JPEG -n02110627/ILSVRC2012_val_00029382.JPEG -n03895866/ILSVRC2012_val_00038339.JPEG -n04584207/ILSVRC2012_val_00012244.JPEG -n04584207/ILSVRC2012_val_00027984.JPEG -n04162706/ILSVRC2012_val_00002091.JPEG -n03291819/ILSVRC2012_val_00024925.JPEG -n02130308/ILSVRC2012_val_00014974.JPEG -n07716358/ILSVRC2012_val_00023848.JPEG -n04493381/ILSVRC2012_val_00012077.JPEG -n01694178/ILSVRC2012_val_00015831.JPEG -n02094258/ILSVRC2012_val_00025947.JPEG -n02927161/ILSVRC2012_val_00012448.JPEG -n02128757/ILSVRC2012_val_00036622.JPEG -n02488702/ILSVRC2012_val_00036767.JPEG -n13054560/ILSVRC2012_val_00048790.JPEG -n04355338/ILSVRC2012_val_00030338.JPEG -n02107908/ILSVRC2012_val_00000758.JPEG -n07730033/ILSVRC2012_val_00041488.JPEG -n02802426/ILSVRC2012_val_00031833.JPEG -n02128757/ILSVRC2012_val_00022170.JPEG -n02281787/ILSVRC2012_val_00021806.JPEG -n03590841/ILSVRC2012_val_00032642.JPEG -n02106662/ILSVRC2012_val_00007922.JPEG -n04536866/ILSVRC2012_val_00039680.JPEG -n04552348/ILSVRC2012_val_00015306.JPEG -n02791124/ILSVRC2012_val_00009064.JPEG -n04370456/ILSVRC2012_val_00002955.JPEG -n02825657/ILSVRC2012_val_00046779.JPEG -n09332890/ILSVRC2012_val_00022291.JPEG -n04517823/ILSVRC2012_val_00012535.JPEG -n02457408/ILSVRC2012_val_00033283.JPEG -n07880968/ILSVRC2012_val_00000606.JPEG -n01641577/ILSVRC2012_val_00018339.JPEG -n03146219/ILSVRC2012_val_00036276.JPEG -n03887697/ILSVRC2012_val_00029545.JPEG -n03126707/ILSVRC2012_val_00037953.JPEG -n02410509/ILSVRC2012_val_00034061.JPEG -n04398044/ILSVRC2012_val_00048377.JPEG -n02841315/ILSVRC2012_val_00025790.JPEG -n02087394/ILSVRC2012_val_00017792.JPEG -n07930864/ILSVRC2012_val_00045302.JPEG -n03530642/ILSVRC2012_val_00028432.JPEG -n03874599/ILSVRC2012_val_00002115.JPEG -n04008634/ILSVRC2012_val_00018494.JPEG -n02356798/ILSVRC2012_val_00036257.JPEG -n02090622/ILSVRC2012_val_00001626.JPEG -n03764736/ILSVRC2012_val_00021965.JPEG -n02504013/ILSVRC2012_val_00010713.JPEG -n02676566/ILSVRC2012_val_00041930.JPEG -n02999410/ILSVRC2012_val_00009591.JPEG -n03014705/ILSVRC2012_val_00029640.JPEG -n03344393/ILSVRC2012_val_00011769.JPEG -n02093256/ILSVRC2012_val_00031334.JPEG -n04462240/ILSVRC2012_val_00042489.JPEG -n04209133/ILSVRC2012_val_00002465.JPEG -n02823428/ILSVRC2012_val_00032921.JPEG -n03018349/ILSVRC2012_val_00034866.JPEG -n04265275/ILSVRC2012_val_00028998.JPEG -n02106030/ILSVRC2012_val_00014086.JPEG -n02090622/ILSVRC2012_val_00006495.JPEG -n02085620/ILSVRC2012_val_00028943.JPEG -n03891332/ILSVRC2012_val_00007481.JPEG -n07584110/ILSVRC2012_val_00031813.JPEG -n12985857/ILSVRC2012_val_00033845.JPEG -n07614500/ILSVRC2012_val_00003592.JPEG -n04136333/ILSVRC2012_val_00003658.JPEG -n02090622/ILSVRC2012_val_00005718.JPEG -n02089078/ILSVRC2012_val_00023203.JPEG -n03535780/ILSVRC2012_val_00014669.JPEG -n03770439/ILSVRC2012_val_00016482.JPEG -n04399382/ILSVRC2012_val_00044173.JPEG -n01818515/ILSVRC2012_val_00041556.JPEG -n02963159/ILSVRC2012_val_00014297.JPEG -n01860187/ILSVRC2012_val_00009998.JPEG -n02096051/ILSVRC2012_val_00023011.JPEG -n01773797/ILSVRC2012_val_00015751.JPEG -n02110185/ILSVRC2012_val_00024669.JPEG -n02974003/ILSVRC2012_val_00033710.JPEG -n03089624/ILSVRC2012_val_00010089.JPEG -n02128925/ILSVRC2012_val_00037004.JPEG -n07749582/ILSVRC2012_val_00047221.JPEG -n03916031/ILSVRC2012_val_00026846.JPEG -n04204238/ILSVRC2012_val_00009223.JPEG -n07716358/ILSVRC2012_val_00006290.JPEG -n02108915/ILSVRC2012_val_00041623.JPEG -n04325704/ILSVRC2012_val_00016515.JPEG -n03208938/ILSVRC2012_val_00004668.JPEG -n07753113/ILSVRC2012_val_00042121.JPEG -n01558993/ILSVRC2012_val_00005594.JPEG -n03908618/ILSVRC2012_val_00014429.JPEG -n03417042/ILSVRC2012_val_00033842.JPEG -n01924916/ILSVRC2012_val_00041450.JPEG -n02108551/ILSVRC2012_val_00039115.JPEG -n02797295/ILSVRC2012_val_00044612.JPEG -n04259630/ILSVRC2012_val_00020420.JPEG -n02085782/ILSVRC2012_val_00015730.JPEG -n02091635/ILSVRC2012_val_00042053.JPEG -n04418357/ILSVRC2012_val_00036685.JPEG -n03884397/ILSVRC2012_val_00039417.JPEG -n02006656/ILSVRC2012_val_00005107.JPEG -n02281787/ILSVRC2012_val_00017393.JPEG -n04428191/ILSVRC2012_val_00025547.JPEG -n04592741/ILSVRC2012_val_00022734.JPEG -n02007558/ILSVRC2012_val_00022414.JPEG -n02268853/ILSVRC2012_val_00049961.JPEG -n02808440/ILSVRC2012_val_00043669.JPEG -n02109525/ILSVRC2012_val_00025620.JPEG -n01930112/ILSVRC2012_val_00004923.JPEG -n04399382/ILSVRC2012_val_00004523.JPEG -n04597913/ILSVRC2012_val_00006044.JPEG -n07716358/ILSVRC2012_val_00010580.JPEG -n04296562/ILSVRC2012_val_00039881.JPEG -n03424325/ILSVRC2012_val_00018874.JPEG -n04081281/ILSVRC2012_val_00003807.JPEG -n04228054/ILSVRC2012_val_00029671.JPEG -n03657121/ILSVRC2012_val_00046352.JPEG -n02692877/ILSVRC2012_val_00020020.JPEG -n03125729/ILSVRC2012_val_00000210.JPEG -n04251144/ILSVRC2012_val_00039532.JPEG -n03481172/ILSVRC2012_val_00044928.JPEG -n02917067/ILSVRC2012_val_00005622.JPEG -n02105855/ILSVRC2012_val_00049871.JPEG -n02093754/ILSVRC2012_val_00031172.JPEG -n02087046/ILSVRC2012_val_00019566.JPEG -n03089624/ILSVRC2012_val_00037663.JPEG -n01697457/ILSVRC2012_val_00043428.JPEG -n11879895/ILSVRC2012_val_00000216.JPEG -n02980441/ILSVRC2012_val_00006522.JPEG -n03291819/ILSVRC2012_val_00025489.JPEG -n04591713/ILSVRC2012_val_00034842.JPEG -n04579145/ILSVRC2012_val_00047046.JPEG -n06596364/ILSVRC2012_val_00007624.JPEG -n02843684/ILSVRC2012_val_00013736.JPEG -n03425413/ILSVRC2012_val_00004860.JPEG -n03201208/ILSVRC2012_val_00048762.JPEG -n03657121/ILSVRC2012_val_00006803.JPEG -n02091134/ILSVRC2012_val_00012888.JPEG -n02100735/ILSVRC2012_val_00031785.JPEG -n03920288/ILSVRC2012_val_00034145.JPEG -n03259280/ILSVRC2012_val_00027418.JPEG -n03372029/ILSVRC2012_val_00016025.JPEG -n01560419/ILSVRC2012_val_00020788.JPEG -n02408429/ILSVRC2012_val_00005454.JPEG -n09246464/ILSVRC2012_val_00017728.JPEG -n02672831/ILSVRC2012_val_00047423.JPEG -n01698640/ILSVRC2012_val_00037268.JPEG -n02788148/ILSVRC2012_val_00000584.JPEG -n07613480/ILSVRC2012_val_00038412.JPEG -n02667093/ILSVRC2012_val_00001774.JPEG -n03733805/ILSVRC2012_val_00009255.JPEG -n03976467/ILSVRC2012_val_00000765.JPEG -n03887697/ILSVRC2012_val_00046711.JPEG -n11939491/ILSVRC2012_val_00010143.JPEG -n04263257/ILSVRC2012_val_00037205.JPEG -n07892512/ILSVRC2012_val_00049928.JPEG -n02879718/ILSVRC2012_val_00031070.JPEG -n02442845/ILSVRC2012_val_00014437.JPEG -n02093256/ILSVRC2012_val_00019931.JPEG -n03961711/ILSVRC2012_val_00039193.JPEG -n03494278/ILSVRC2012_val_00019269.JPEG -n02823750/ILSVRC2012_val_00036113.JPEG -n01774384/ILSVRC2012_val_00016936.JPEG -n02112706/ILSVRC2012_val_00046390.JPEG -n04131690/ILSVRC2012_val_00042814.JPEG -n02676566/ILSVRC2012_val_00036762.JPEG -n01873310/ILSVRC2012_val_00007800.JPEG -n01667778/ILSVRC2012_val_00024736.JPEG -n03832673/ILSVRC2012_val_00026011.JPEG -n02088238/ILSVRC2012_val_00024881.JPEG -n02107908/ILSVRC2012_val_00045080.JPEG -n04235860/ILSVRC2012_val_00017165.JPEG -n04554684/ILSVRC2012_val_00048391.JPEG -n03124170/ILSVRC2012_val_00001875.JPEG -n02087394/ILSVRC2012_val_00042252.JPEG -n03337140/ILSVRC2012_val_00018172.JPEG -n07873807/ILSVRC2012_val_00008506.JPEG -n04350905/ILSVRC2012_val_00002541.JPEG -n02727426/ILSVRC2012_val_00012219.JPEG -n04540053/ILSVRC2012_val_00022449.JPEG -n02865351/ILSVRC2012_val_00012866.JPEG -n02108422/ILSVRC2012_val_00003022.JPEG -n02672831/ILSVRC2012_val_00003432.JPEG -n02791270/ILSVRC2012_val_00035205.JPEG -n03637318/ILSVRC2012_val_00027525.JPEG -n02109047/ILSVRC2012_val_00017333.JPEG -n03207941/ILSVRC2012_val_00006788.JPEG -n02749479/ILSVRC2012_val_00003684.JPEG -n02892201/ILSVRC2012_val_00027425.JPEG -n03062245/ILSVRC2012_val_00040099.JPEG -n03617480/ILSVRC2012_val_00027188.JPEG -n02504013/ILSVRC2012_val_00020120.JPEG -n03250847/ILSVRC2012_val_00045452.JPEG -n01667778/ILSVRC2012_val_00013738.JPEG -n03534580/ILSVRC2012_val_00042217.JPEG -n03417042/ILSVRC2012_val_00026865.JPEG -n01688243/ILSVRC2012_val_00014251.JPEG -n03125729/ILSVRC2012_val_00028808.JPEG -n07693725/ILSVRC2012_val_00019949.JPEG -n01669191/ILSVRC2012_val_00018362.JPEG -n02457408/ILSVRC2012_val_00029890.JPEG -n03534580/ILSVRC2012_val_00033004.JPEG -n01774384/ILSVRC2012_val_00006437.JPEG -n03187595/ILSVRC2012_val_00009824.JPEG -n02112137/ILSVRC2012_val_00028855.JPEG -n04409515/ILSVRC2012_val_00020170.JPEG -n04243546/ILSVRC2012_val_00017216.JPEG -n04355338/ILSVRC2012_val_00020611.JPEG -n04429376/ILSVRC2012_val_00049487.JPEG -n01665541/ILSVRC2012_val_00025441.JPEG -n02927161/ILSVRC2012_val_00031010.JPEG -n11939491/ILSVRC2012_val_00049436.JPEG -n04429376/ILSVRC2012_val_00001160.JPEG -n01582220/ILSVRC2012_val_00033150.JPEG -n13040303/ILSVRC2012_val_00008821.JPEG -n01978287/ILSVRC2012_val_00013891.JPEG -n01855032/ILSVRC2012_val_00031847.JPEG -n07892512/ILSVRC2012_val_00029480.JPEG -n02089078/ILSVRC2012_val_00009405.JPEG -n04479046/ILSVRC2012_val_00035477.JPEG -n03980874/ILSVRC2012_val_00039869.JPEG -n03866082/ILSVRC2012_val_00046877.JPEG -n02093754/ILSVRC2012_val_00044977.JPEG -n02860847/ILSVRC2012_val_00025196.JPEG -n02769748/ILSVRC2012_val_00002183.JPEG -n02727426/ILSVRC2012_val_00041842.JPEG -n03014705/ILSVRC2012_val_00044754.JPEG -n02408429/ILSVRC2012_val_00029776.JPEG -n04081281/ILSVRC2012_val_00018924.JPEG -n11879895/ILSVRC2012_val_00043005.JPEG -n07753113/ILSVRC2012_val_00038857.JPEG -n02787622/ILSVRC2012_val_00010207.JPEG -n02317335/ILSVRC2012_val_00045028.JPEG -n02110063/ILSVRC2012_val_00021051.JPEG -n06874185/ILSVRC2012_val_00041744.JPEG -n02088094/ILSVRC2012_val_00029140.JPEG -n03788365/ILSVRC2012_val_00014303.JPEG -n02105162/ILSVRC2012_val_00038991.JPEG -n03017168/ILSVRC2012_val_00031125.JPEG -n01883070/ILSVRC2012_val_00036590.JPEG -n02112350/ILSVRC2012_val_00040313.JPEG -n02091244/ILSVRC2012_val_00009334.JPEG -n02105162/ILSVRC2012_val_00004754.JPEG -n01742172/ILSVRC2012_val_00009271.JPEG -n04120489/ILSVRC2012_val_00027118.JPEG -n03109150/ILSVRC2012_val_00009323.JPEG -n03770439/ILSVRC2012_val_00016619.JPEG -n02128757/ILSVRC2012_val_00035192.JPEG -n04162706/ILSVRC2012_val_00029687.JPEG -n04507155/ILSVRC2012_val_00004816.JPEG -n03216828/ILSVRC2012_val_00012195.JPEG -n04275548/ILSVRC2012_val_00046830.JPEG -n03967562/ILSVRC2012_val_00024846.JPEG -n01534433/ILSVRC2012_val_00032025.JPEG -n04418357/ILSVRC2012_val_00041170.JPEG -n04127249/ILSVRC2012_val_00027502.JPEG -n03481172/ILSVRC2012_val_00032972.JPEG -n04118538/ILSVRC2012_val_00017229.JPEG -n01693334/ILSVRC2012_val_00017730.JPEG -n02488291/ILSVRC2012_val_00028982.JPEG -n02106166/ILSVRC2012_val_00001900.JPEG -n02132136/ILSVRC2012_val_00018369.JPEG -n04346328/ILSVRC2012_val_00011106.JPEG -n01980166/ILSVRC2012_val_00025960.JPEG -n01751748/ILSVRC2012_val_00000324.JPEG -n03658185/ILSVRC2012_val_00014001.JPEG -n03534580/ILSVRC2012_val_00022084.JPEG -n02092339/ILSVRC2012_val_00025680.JPEG -n02492660/ILSVRC2012_val_00015033.JPEG -n02117135/ILSVRC2012_val_00029816.JPEG -n04070727/ILSVRC2012_val_00004133.JPEG -n02815834/ILSVRC2012_val_00018885.JPEG -n04479046/ILSVRC2012_val_00006196.JPEG -n02281406/ILSVRC2012_val_00039971.JPEG -n02108551/ILSVRC2012_val_00020152.JPEG -n02233338/ILSVRC2012_val_00042811.JPEG -n02504013/ILSVRC2012_val_00021237.JPEG -n02114548/ILSVRC2012_val_00030270.JPEG -n02009229/ILSVRC2012_val_00034814.JPEG -n13044778/ILSVRC2012_val_00027611.JPEG -n03814639/ILSVRC2012_val_00034119.JPEG -n07749582/ILSVRC2012_val_00000711.JPEG -n02113978/ILSVRC2012_val_00001917.JPEG -n03938244/ILSVRC2012_val_00048945.JPEG -n04465501/ILSVRC2012_val_00027978.JPEG -n04162706/ILSVRC2012_val_00021842.JPEG -n03930630/ILSVRC2012_val_00017360.JPEG -n03717622/ILSVRC2012_val_00023787.JPEG -n04228054/ILSVRC2012_val_00000568.JPEG -n02011460/ILSVRC2012_val_00049711.JPEG -n03496892/ILSVRC2012_val_00012088.JPEG -n01580077/ILSVRC2012_val_00026136.JPEG -n04136333/ILSVRC2012_val_00022316.JPEG -n02006656/ILSVRC2012_val_00022418.JPEG -n03290653/ILSVRC2012_val_00010260.JPEG -n02974003/ILSVRC2012_val_00047912.JPEG -n07749582/ILSVRC2012_val_00002098.JPEG -n03788195/ILSVRC2012_val_00031002.JPEG -n02445715/ILSVRC2012_val_00015239.JPEG -n02797295/ILSVRC2012_val_00017834.JPEG -n04532106/ILSVRC2012_val_00034337.JPEG -n04507155/ILSVRC2012_val_00036058.JPEG -n04162706/ILSVRC2012_val_00032515.JPEG -n02363005/ILSVRC2012_val_00015977.JPEG -n01828970/ILSVRC2012_val_00025758.JPEG -n03075370/ILSVRC2012_val_00046514.JPEG -n07880968/ILSVRC2012_val_00047263.JPEG -n04418357/ILSVRC2012_val_00046075.JPEG -n02927161/ILSVRC2012_val_00045329.JPEG -n03447447/ILSVRC2012_val_00045000.JPEG -n09288635/ILSVRC2012_val_00014161.JPEG -n03095699/ILSVRC2012_val_00007161.JPEG -n02815834/ILSVRC2012_val_00028875.JPEG -n09193705/ILSVRC2012_val_00028907.JPEG -n02120079/ILSVRC2012_val_00020481.JPEG -n04370456/ILSVRC2012_val_00038408.JPEG -n02120505/ILSVRC2012_val_00020859.JPEG -n02105162/ILSVRC2012_val_00027807.JPEG -n03179701/ILSVRC2012_val_00021078.JPEG -n01728572/ILSVRC2012_val_00042100.JPEG -n02097130/ILSVRC2012_val_00044820.JPEG -n02105412/ILSVRC2012_val_00008397.JPEG -n02504013/ILSVRC2012_val_00048547.JPEG -n06359193/ILSVRC2012_val_00036311.JPEG -n04179913/ILSVRC2012_val_00006735.JPEG -n04070727/ILSVRC2012_val_00036831.JPEG -n02701002/ILSVRC2012_val_00018957.JPEG -n04417672/ILSVRC2012_val_00025280.JPEG -n02093859/ILSVRC2012_val_00008502.JPEG -n02086646/ILSVRC2012_val_00037252.JPEG -n02494079/ILSVRC2012_val_00017776.JPEG -n03347037/ILSVRC2012_val_00026726.JPEG -n02281787/ILSVRC2012_val_00010102.JPEG -n02799071/ILSVRC2012_val_00022312.JPEG -n02481823/ILSVRC2012_val_00000974.JPEG -n07720875/ILSVRC2012_val_00041116.JPEG -n04517823/ILSVRC2012_val_00004240.JPEG -n03637318/ILSVRC2012_val_00026798.JPEG -n04376876/ILSVRC2012_val_00041555.JPEG -n02963159/ILSVRC2012_val_00019491.JPEG -n01737021/ILSVRC2012_val_00018334.JPEG -n07753275/ILSVRC2012_val_00001479.JPEG -n02089973/ILSVRC2012_val_00011586.JPEG -n02108422/ILSVRC2012_val_00041144.JPEG -n02328150/ILSVRC2012_val_00027686.JPEG -n03208938/ILSVRC2012_val_00015384.JPEG -n09421951/ILSVRC2012_val_00005296.JPEG -n13052670/ILSVRC2012_val_00048466.JPEG -n03706229/ILSVRC2012_val_00049101.JPEG -n03529860/ILSVRC2012_val_00024939.JPEG -n04487394/ILSVRC2012_val_00027975.JPEG -n09332890/ILSVRC2012_val_00038916.JPEG -n03494278/ILSVRC2012_val_00007924.JPEG -n03891332/ILSVRC2012_val_00029440.JPEG -n04266014/ILSVRC2012_val_00045801.JPEG -n04599235/ILSVRC2012_val_00035088.JPEG -n03125729/ILSVRC2012_val_00037113.JPEG -n04310018/ILSVRC2012_val_00045560.JPEG -n01735189/ILSVRC2012_val_00029712.JPEG -n02133161/ILSVRC2012_val_00016621.JPEG -n02690373/ILSVRC2012_val_00024747.JPEG -n03920288/ILSVRC2012_val_00029067.JPEG -n03733805/ILSVRC2012_val_00027209.JPEG -n02112018/ILSVRC2012_val_00018688.JPEG -n04550184/ILSVRC2012_val_00048746.JPEG -n02490219/ILSVRC2012_val_00001504.JPEG -n01945685/ILSVRC2012_val_00029558.JPEG -n02096585/ILSVRC2012_val_00014729.JPEG -n02095889/ILSVRC2012_val_00009347.JPEG -n02100877/ILSVRC2012_val_00041739.JPEG -n01917289/ILSVRC2012_val_00045307.JPEG -n04252225/ILSVRC2012_val_00045557.JPEG -n02906734/ILSVRC2012_val_00014004.JPEG -n04418357/ILSVRC2012_val_00022941.JPEG -n02092339/ILSVRC2012_val_00001485.JPEG -n12768682/ILSVRC2012_val_00008329.JPEG -n03291819/ILSVRC2012_val_00018094.JPEG -n04370456/ILSVRC2012_val_00020241.JPEG -n02281787/ILSVRC2012_val_00027787.JPEG -n04065272/ILSVRC2012_val_00022419.JPEG -n03690938/ILSVRC2012_val_00011599.JPEG -n04041544/ILSVRC2012_val_00033549.JPEG -n04346328/ILSVRC2012_val_00041257.JPEG -n03272010/ILSVRC2012_val_00014896.JPEG -n04562935/ILSVRC2012_val_00018833.JPEG -n03673027/ILSVRC2012_val_00049165.JPEG -n03982430/ILSVRC2012_val_00042092.JPEG -n02090379/ILSVRC2012_val_00042442.JPEG -n04277352/ILSVRC2012_val_00032999.JPEG -n03207941/ILSVRC2012_val_00048287.JPEG -n02687172/ILSVRC2012_val_00009104.JPEG -n01978455/ILSVRC2012_val_00049440.JPEG -n02281787/ILSVRC2012_val_00002787.JPEG -n03297495/ILSVRC2012_val_00037578.JPEG -n04149813/ILSVRC2012_val_00000563.JPEG -n02105251/ILSVRC2012_val_00035781.JPEG -n01833805/ILSVRC2012_val_00003259.JPEG -n04487394/ILSVRC2012_val_00045716.JPEG -n02797295/ILSVRC2012_val_00021614.JPEG -n03452741/ILSVRC2012_val_00010946.JPEG -n02097298/ILSVRC2012_val_00049757.JPEG -n03014705/ILSVRC2012_val_00012144.JPEG -n02747177/ILSVRC2012_val_00042198.JPEG -n02490219/ILSVRC2012_val_00047085.JPEG -n01632458/ILSVRC2012_val_00029732.JPEG -n03998194/ILSVRC2012_val_00042119.JPEG -n04009552/ILSVRC2012_val_00001253.JPEG -n03891251/ILSVRC2012_val_00025864.JPEG -n02123597/ILSVRC2012_val_00011738.JPEG -n02281787/ILSVRC2012_val_00043218.JPEG -n04266014/ILSVRC2012_val_00046484.JPEG -n03781244/ILSVRC2012_val_00047206.JPEG -n12985857/ILSVRC2012_val_00040193.JPEG -n02097474/ILSVRC2012_val_00038580.JPEG -n02101556/ILSVRC2012_val_00021360.JPEG -n04335435/ILSVRC2012_val_00045852.JPEG -n03100240/ILSVRC2012_val_00025647.JPEG -n02692877/ILSVRC2012_val_00016891.JPEG -n04599235/ILSVRC2012_val_00049409.JPEG -n03944341/ILSVRC2012_val_00028761.JPEG -n03325584/ILSVRC2012_val_00046949.JPEG -n03000134/ILSVRC2012_val_00014205.JPEG -n04209239/ILSVRC2012_val_00034535.JPEG -n04041544/ILSVRC2012_val_00015604.JPEG -n02013706/ILSVRC2012_val_00012475.JPEG -n03775546/ILSVRC2012_val_00013468.JPEG -n01739381/ILSVRC2012_val_00033552.JPEG -n07745940/ILSVRC2012_val_00012497.JPEG -n09246464/ILSVRC2012_val_00042182.JPEG -n01697457/ILSVRC2012_val_00038287.JPEG -n01756291/ILSVRC2012_val_00019270.JPEG -n04584207/ILSVRC2012_val_00026020.JPEG -n02086910/ILSVRC2012_val_00035935.JPEG -n03376595/ILSVRC2012_val_00044181.JPEG -n01829413/ILSVRC2012_val_00020284.JPEG -n03938244/ILSVRC2012_val_00011231.JPEG -n01843383/ILSVRC2012_val_00034194.JPEG -n02965783/ILSVRC2012_val_00018623.JPEG -n03457902/ILSVRC2012_val_00005071.JPEG -n03000247/ILSVRC2012_val_00002834.JPEG -n01883070/ILSVRC2012_val_00013845.JPEG -n01847000/ILSVRC2012_val_00036040.JPEG -n02093256/ILSVRC2012_val_00028332.JPEG -n02074367/ILSVRC2012_val_00022669.JPEG -n01910747/ILSVRC2012_val_00026530.JPEG -n03769881/ILSVRC2012_val_00017651.JPEG -n03529860/ILSVRC2012_val_00004596.JPEG -n02105056/ILSVRC2012_val_00016092.JPEG -n03017168/ILSVRC2012_val_00007510.JPEG -n04008634/ILSVRC2012_val_00033210.JPEG -n02879718/ILSVRC2012_val_00002777.JPEG -n03891332/ILSVRC2012_val_00018258.JPEG -n02410509/ILSVRC2012_val_00008629.JPEG -n02094258/ILSVRC2012_val_00042642.JPEG -n04008634/ILSVRC2012_val_00008246.JPEG -n06596364/ILSVRC2012_val_00040030.JPEG -n03461385/ILSVRC2012_val_00025959.JPEG -n01829413/ILSVRC2012_val_00004697.JPEG -n07693725/ILSVRC2012_val_00035111.JPEG -n04548362/ILSVRC2012_val_00021967.JPEG -n12144580/ILSVRC2012_val_00020295.JPEG -n01914609/ILSVRC2012_val_00042456.JPEG -n03344393/ILSVRC2012_val_00007736.JPEG -n04049303/ILSVRC2012_val_00040866.JPEG -n12998815/ILSVRC2012_val_00002153.JPEG -n01978287/ILSVRC2012_val_00018052.JPEG -n04613696/ILSVRC2012_val_00045840.JPEG -n02114712/ILSVRC2012_val_00037617.JPEG -n03485794/ILSVRC2012_val_00049825.JPEG -n02110806/ILSVRC2012_val_00035890.JPEG -n09288635/ILSVRC2012_val_00016478.JPEG -n01644373/ILSVRC2012_val_00030352.JPEG -n02095570/ILSVRC2012_val_00026286.JPEG -n03908618/ILSVRC2012_val_00012016.JPEG -n02129165/ILSVRC2012_val_00014967.JPEG -n02493509/ILSVRC2012_val_00037781.JPEG -n07684084/ILSVRC2012_val_00006466.JPEG -n03259280/ILSVRC2012_val_00000182.JPEG -n01514668/ILSVRC2012_val_00005505.JPEG -n03444034/ILSVRC2012_val_00030549.JPEG -n01630670/ILSVRC2012_val_00000498.JPEG -n04429376/ILSVRC2012_val_00003519.JPEG -n03127925/ILSVRC2012_val_00015057.JPEG -n02483362/ILSVRC2012_val_00004753.JPEG -n02978881/ILSVRC2012_val_00027299.JPEG -n02108000/ILSVRC2012_val_00002277.JPEG -n02108551/ILSVRC2012_val_00034181.JPEG -n03874599/ILSVRC2012_val_00047645.JPEG -n03888257/ILSVRC2012_val_00010336.JPEG -n02058221/ILSVRC2012_val_00038725.JPEG -n01592084/ILSVRC2012_val_00045568.JPEG -n01677366/ILSVRC2012_val_00033961.JPEG -n04505470/ILSVRC2012_val_00008530.JPEG -n02090379/ILSVRC2012_val_00021575.JPEG -n01980166/ILSVRC2012_val_00012779.JPEG -n07565083/ILSVRC2012_val_00042087.JPEG -n02319095/ILSVRC2012_val_00035430.JPEG -n03983396/ILSVRC2012_val_00012540.JPEG -n04005630/ILSVRC2012_val_00024281.JPEG -n03530642/ILSVRC2012_val_00018635.JPEG -n02113978/ILSVRC2012_val_00033581.JPEG -n01748264/ILSVRC2012_val_00013245.JPEG -n03085013/ILSVRC2012_val_00021913.JPEG -n03733131/ILSVRC2012_val_00024970.JPEG -n03733805/ILSVRC2012_val_00033697.JPEG -n04479046/ILSVRC2012_val_00038352.JPEG -n04317175/ILSVRC2012_val_00048424.JPEG -n02172182/ILSVRC2012_val_00031062.JPEG -n01828970/ILSVRC2012_val_00009536.JPEG -n02096437/ILSVRC2012_val_00035260.JPEG -n02037110/ILSVRC2012_val_00010080.JPEG -n03218198/ILSVRC2012_val_00043332.JPEG -n07716358/ILSVRC2012_val_00039647.JPEG -n02090721/ILSVRC2012_val_00027629.JPEG -n02102318/ILSVRC2012_val_00004995.JPEG -n02500267/ILSVRC2012_val_00036531.JPEG -n02107312/ILSVRC2012_val_00010372.JPEG -n03857828/ILSVRC2012_val_00015720.JPEG -n03141823/ILSVRC2012_val_00044432.JPEG -n02326432/ILSVRC2012_val_00033403.JPEG -n02114855/ILSVRC2012_val_00019680.JPEG -n04090263/ILSVRC2012_val_00030031.JPEG -n04371430/ILSVRC2012_val_00023575.JPEG -n03982430/ILSVRC2012_val_00023993.JPEG -n04487394/ILSVRC2012_val_00036346.JPEG -n09835506/ILSVRC2012_val_00039644.JPEG -n02492660/ILSVRC2012_val_00034985.JPEG -n01978455/ILSVRC2012_val_00023601.JPEG -n07697537/ILSVRC2012_val_00024032.JPEG -n04486054/ILSVRC2012_val_00010914.JPEG -n03179701/ILSVRC2012_val_00046652.JPEG -n03127747/ILSVRC2012_val_00022689.JPEG -n02860847/ILSVRC2012_val_00036614.JPEG -n04548362/ILSVRC2012_val_00046827.JPEG -n01883070/ILSVRC2012_val_00014545.JPEG -n02264363/ILSVRC2012_val_00022155.JPEG -n01986214/ILSVRC2012_val_00036320.JPEG -n02979186/ILSVRC2012_val_00018559.JPEG -n02113023/ILSVRC2012_val_00049215.JPEG -n02108915/ILSVRC2012_val_00026976.JPEG -n03476991/ILSVRC2012_val_00012370.JPEG -n02277742/ILSVRC2012_val_00033639.JPEG -n07613480/ILSVRC2012_val_00000928.JPEG -n04442312/ILSVRC2012_val_00004478.JPEG -n02134084/ILSVRC2012_val_00021883.JPEG -n04118776/ILSVRC2012_val_00027784.JPEG -n02097298/ILSVRC2012_val_00031084.JPEG -n04285008/ILSVRC2012_val_00028336.JPEG -n04065272/ILSVRC2012_val_00043339.JPEG -n03045698/ILSVRC2012_val_00023447.JPEG -n01537544/ILSVRC2012_val_00035565.JPEG -n02111500/ILSVRC2012_val_00003794.JPEG -n07583066/ILSVRC2012_val_00000728.JPEG -n01601694/ILSVRC2012_val_00030011.JPEG -n02980441/ILSVRC2012_val_00019377.JPEG -n04399382/ILSVRC2012_val_00001186.JPEG -n03874599/ILSVRC2012_val_00048559.JPEG -n02129165/ILSVRC2012_val_00011018.JPEG -n04371430/ILSVRC2012_val_00019658.JPEG -n07720875/ILSVRC2012_val_00005981.JPEG -n02500267/ILSVRC2012_val_00011603.JPEG -n04154565/ILSVRC2012_val_00020366.JPEG -n06596364/ILSVRC2012_val_00017365.JPEG -n03877472/ILSVRC2012_val_00016590.JPEG -n03868863/ILSVRC2012_val_00007585.JPEG -n07930864/ILSVRC2012_val_00005759.JPEG -n02097209/ILSVRC2012_val_00006178.JPEG -n01688243/ILSVRC2012_val_00012820.JPEG -n02090379/ILSVRC2012_val_00033094.JPEG -n02229544/ILSVRC2012_val_00042010.JPEG -n04487081/ILSVRC2012_val_00034724.JPEG -n04456115/ILSVRC2012_val_00036237.JPEG -n02493793/ILSVRC2012_val_00036994.JPEG -n02018795/ILSVRC2012_val_00034725.JPEG -n07831146/ILSVRC2012_val_00024518.JPEG -n02492660/ILSVRC2012_val_00013754.JPEG -n03047690/ILSVRC2012_val_00014595.JPEG -n01819313/ILSVRC2012_val_00018156.JPEG -n01774384/ILSVRC2012_val_00014659.JPEG -n03838899/ILSVRC2012_val_00032257.JPEG -n04310018/ILSVRC2012_val_00037088.JPEG -n02489166/ILSVRC2012_val_00031562.JPEG -n01737021/ILSVRC2012_val_00000084.JPEG -n01797886/ILSVRC2012_val_00005315.JPEG -n01534433/ILSVRC2012_val_00026687.JPEG -n02123394/ILSVRC2012_val_00049996.JPEG -n03018349/ILSVRC2012_val_00021844.JPEG -n03720891/ILSVRC2012_val_00017528.JPEG -n04147183/ILSVRC2012_val_00037184.JPEG -n03223299/ILSVRC2012_val_00005057.JPEG -n01860187/ILSVRC2012_val_00030887.JPEG -n01871265/ILSVRC2012_val_00002625.JPEG -n04599235/ILSVRC2012_val_00015111.JPEG -n04041544/ILSVRC2012_val_00038785.JPEG -n02484975/ILSVRC2012_val_00002767.JPEG -n04592741/ILSVRC2012_val_00027706.JPEG -n03933933/ILSVRC2012_val_00046001.JPEG -n04067472/ILSVRC2012_val_00032026.JPEG -n04606251/ILSVRC2012_val_00009289.JPEG -n02687172/ILSVRC2012_val_00023116.JPEG -n02093256/ILSVRC2012_val_00000248.JPEG -n03876231/ILSVRC2012_val_00031812.JPEG -n02950826/ILSVRC2012_val_00034326.JPEG -n01776313/ILSVRC2012_val_00001283.JPEG -n09399592/ILSVRC2012_val_00033587.JPEG -n03691459/ILSVRC2012_val_00041210.JPEG -n02013706/ILSVRC2012_val_00039389.JPEG -n04208210/ILSVRC2012_val_00049240.JPEG -n01740131/ILSVRC2012_val_00043694.JPEG -n03794056/ILSVRC2012_val_00002087.JPEG -n03956157/ILSVRC2012_val_00019543.JPEG -n02091244/ILSVRC2012_val_00012149.JPEG -n02028035/ILSVRC2012_val_00038002.JPEG -n01685808/ILSVRC2012_val_00042202.JPEG -n02422106/ILSVRC2012_val_00010317.JPEG -n02090721/ILSVRC2012_val_00022071.JPEG -n02988304/ILSVRC2012_val_00013108.JPEG -n03769881/ILSVRC2012_val_00042007.JPEG -n04372370/ILSVRC2012_val_00036463.JPEG -n13133613/ILSVRC2012_val_00015477.JPEG -n04486054/ILSVRC2012_val_00035081.JPEG -n01491361/ILSVRC2012_val_00041635.JPEG -n02123045/ILSVRC2012_val_00002823.JPEG -n02086910/ILSVRC2012_val_00001389.JPEG -n03447447/ILSVRC2012_val_00010200.JPEG -n02403003/ILSVRC2012_val_00041495.JPEG -n04311174/ILSVRC2012_val_00000842.JPEG -n04228054/ILSVRC2012_val_00036197.JPEG -n04209239/ILSVRC2012_val_00035393.JPEG -n02091467/ILSVRC2012_val_00033843.JPEG -n01695060/ILSVRC2012_val_00003865.JPEG -n02113186/ILSVRC2012_val_00020270.JPEG -n02497673/ILSVRC2012_val_00014914.JPEG -n09193705/ILSVRC2012_val_00029172.JPEG -n02687172/ILSVRC2012_val_00013281.JPEG -n03633091/ILSVRC2012_val_00025920.JPEG -n07711569/ILSVRC2012_val_00006569.JPEG -n02410509/ILSVRC2012_val_00037054.JPEG -n03942813/ILSVRC2012_val_00025976.JPEG -n02006656/ILSVRC2012_val_00005379.JPEG -n02100877/ILSVRC2012_val_00041364.JPEG -n04376876/ILSVRC2012_val_00027991.JPEG -n02783161/ILSVRC2012_val_00015458.JPEG -n03954731/ILSVRC2012_val_00000645.JPEG -n03888605/ILSVRC2012_val_00031613.JPEG -n09246464/ILSVRC2012_val_00024359.JPEG -n02099849/ILSVRC2012_val_00006078.JPEG -n03201208/ILSVRC2012_val_00030447.JPEG -n07720875/ILSVRC2012_val_00009973.JPEG -n02206856/ILSVRC2012_val_00035503.JPEG -n04409515/ILSVRC2012_val_00000397.JPEG -n12057211/ILSVRC2012_val_00015582.JPEG -n02056570/ILSVRC2012_val_00027559.JPEG -n01872401/ILSVRC2012_val_00044584.JPEG -n02480495/ILSVRC2012_val_00035901.JPEG -n03788365/ILSVRC2012_val_00024288.JPEG -n01871265/ILSVRC2012_val_00033331.JPEG -n04179913/ILSVRC2012_val_00037030.JPEG -n02099601/ILSVRC2012_val_00029751.JPEG -n02793495/ILSVRC2012_val_00044751.JPEG -n03207941/ILSVRC2012_val_00022852.JPEG -n02168699/ILSVRC2012_val_00006595.JPEG -n02786058/ILSVRC2012_val_00034958.JPEG -n02094258/ILSVRC2012_val_00040263.JPEG -n02397096/ILSVRC2012_val_00006975.JPEG -n07697537/ILSVRC2012_val_00043773.JPEG -n04118538/ILSVRC2012_val_00032103.JPEG -n03623198/ILSVRC2012_val_00020292.JPEG -n02111500/ILSVRC2012_val_00000174.JPEG -n07615774/ILSVRC2012_val_00030790.JPEG -n04251144/ILSVRC2012_val_00010908.JPEG -n03627232/ILSVRC2012_val_00023148.JPEG -n03967562/ILSVRC2012_val_00024414.JPEG -n03496892/ILSVRC2012_val_00037531.JPEG -n01945685/ILSVRC2012_val_00034015.JPEG -n04389033/ILSVRC2012_val_00030415.JPEG -n02423022/ILSVRC2012_val_00044771.JPEG -n02093991/ILSVRC2012_val_00025690.JPEG -n04153751/ILSVRC2012_val_00004966.JPEG -n09399592/ILSVRC2012_val_00003321.JPEG -n02093754/ILSVRC2012_val_00014730.JPEG -n03388183/ILSVRC2012_val_00047917.JPEG -n03791053/ILSVRC2012_val_00004465.JPEG -n04461696/ILSVRC2012_val_00025748.JPEG -n02006656/ILSVRC2012_val_00043081.JPEG -n03443371/ILSVRC2012_val_00044868.JPEG -n02398521/ILSVRC2012_val_00026103.JPEG -n01985128/ILSVRC2012_val_00025521.JPEG -n04141975/ILSVRC2012_val_00047207.JPEG -n03769881/ILSVRC2012_val_00012837.JPEG -n02783161/ILSVRC2012_val_00006972.JPEG -n03759954/ILSVRC2012_val_00036985.JPEG -n04039381/ILSVRC2012_val_00044918.JPEG -n04344873/ILSVRC2012_val_00049662.JPEG -n03100240/ILSVRC2012_val_00038876.JPEG -n02009912/ILSVRC2012_val_00026861.JPEG -n03095699/ILSVRC2012_val_00012534.JPEG -n04099969/ILSVRC2012_val_00047789.JPEG -n04265275/ILSVRC2012_val_00008244.JPEG -n01774750/ILSVRC2012_val_00005326.JPEG -n02086910/ILSVRC2012_val_00030141.JPEG -n02095314/ILSVRC2012_val_00010099.JPEG -n09288635/ILSVRC2012_val_00025861.JPEG -n04008634/ILSVRC2012_val_00018586.JPEG -n02782093/ILSVRC2012_val_00038687.JPEG -n04589890/ILSVRC2012_val_00017272.JPEG -n02086910/ILSVRC2012_val_00024887.JPEG -n03187595/ILSVRC2012_val_00037784.JPEG -n04162706/ILSVRC2012_val_00001792.JPEG -n03240683/ILSVRC2012_val_00019875.JPEG -n02093647/ILSVRC2012_val_00009033.JPEG -n04429376/ILSVRC2012_val_00048901.JPEG -n03255030/ILSVRC2012_val_00046720.JPEG -n01843065/ILSVRC2012_val_00033469.JPEG -n02098105/ILSVRC2012_val_00025970.JPEG -n03599486/ILSVRC2012_val_00029537.JPEG -n03958227/ILSVRC2012_val_00042740.JPEG -n02108089/ILSVRC2012_val_00025897.JPEG -n02113799/ILSVRC2012_val_00047597.JPEG -n02169497/ILSVRC2012_val_00017264.JPEG -n03770439/ILSVRC2012_val_00045468.JPEG -n02894605/ILSVRC2012_val_00043379.JPEG -n01687978/ILSVRC2012_val_00011448.JPEG -n02910353/ILSVRC2012_val_00049838.JPEG -n02895154/ILSVRC2012_val_00011775.JPEG -n04613696/ILSVRC2012_val_00009584.JPEG -n02165456/ILSVRC2012_val_00030921.JPEG -n02749479/ILSVRC2012_val_00039737.JPEG -n02107683/ILSVRC2012_val_00011491.JPEG -n02497673/ILSVRC2012_val_00036690.JPEG -n03785016/ILSVRC2012_val_00006596.JPEG -n04208210/ILSVRC2012_val_00040253.JPEG -n03775546/ILSVRC2012_val_00046554.JPEG -n02342885/ILSVRC2012_val_00015181.JPEG -n02074367/ILSVRC2012_val_00011012.JPEG -n01734418/ILSVRC2012_val_00012578.JPEG -n02086079/ILSVRC2012_val_00009403.JPEG -n03272010/ILSVRC2012_val_00021556.JPEG -n02666196/ILSVRC2012_val_00024965.JPEG -n02091467/ILSVRC2012_val_00046530.JPEG -n12620546/ILSVRC2012_val_00008895.JPEG -n02486261/ILSVRC2012_val_00035493.JPEG -n07579787/ILSVRC2012_val_00028525.JPEG -n03929855/ILSVRC2012_val_00010891.JPEG -n03220513/ILSVRC2012_val_00011990.JPEG -n02279972/ILSVRC2012_val_00024165.JPEG -n03661043/ILSVRC2012_val_00028061.JPEG -n03530642/ILSVRC2012_val_00042804.JPEG -n02917067/ILSVRC2012_val_00017843.JPEG -n02134084/ILSVRC2012_val_00012893.JPEG -n02097047/ILSVRC2012_val_00015136.JPEG -n02397096/ILSVRC2012_val_00020202.JPEG -n02097209/ILSVRC2012_val_00023893.JPEG -n03793489/ILSVRC2012_val_00014424.JPEG -n01514668/ILSVRC2012_val_00024957.JPEG -n03584254/ILSVRC2012_val_00028974.JPEG -n02111889/ILSVRC2012_val_00039709.JPEG -n02486261/ILSVRC2012_val_00001236.JPEG -n02129165/ILSVRC2012_val_00047193.JPEG -n03871628/ILSVRC2012_val_00022943.JPEG -n03759954/ILSVRC2012_val_00035587.JPEG -n02807133/ILSVRC2012_val_00037533.JPEG -n01532829/ILSVRC2012_val_00038091.JPEG -n01496331/ILSVRC2012_val_00047080.JPEG -n02526121/ILSVRC2012_val_00030260.JPEG -n04548362/ILSVRC2012_val_00047367.JPEG -n04238763/ILSVRC2012_val_00043419.JPEG -n02870880/ILSVRC2012_val_00015895.JPEG -n02795169/ILSVRC2012_val_00037322.JPEG -n04201297/ILSVRC2012_val_00039751.JPEG -n04505470/ILSVRC2012_val_00013404.JPEG -n02749479/ILSVRC2012_val_00016457.JPEG -n12620546/ILSVRC2012_val_00011901.JPEG -n02417914/ILSVRC2012_val_00034916.JPEG -n03125729/ILSVRC2012_val_00015525.JPEG -n02113186/ILSVRC2012_val_00003368.JPEG -n02483708/ILSVRC2012_val_00022740.JPEG -n02930766/ILSVRC2012_val_00040571.JPEG -n02071294/ILSVRC2012_val_00042590.JPEG -n01871265/ILSVRC2012_val_00007884.JPEG -n02493793/ILSVRC2012_val_00047641.JPEG -n02107142/ILSVRC2012_val_00040464.JPEG -n03680355/ILSVRC2012_val_00036064.JPEG -n02165105/ILSVRC2012_val_00005482.JPEG -n02100735/ILSVRC2012_val_00021191.JPEG -n03930313/ILSVRC2012_val_00015192.JPEG -n02097298/ILSVRC2012_val_00032291.JPEG -n04560804/ILSVRC2012_val_00029830.JPEG -n02105641/ILSVRC2012_val_00028477.JPEG -n02111500/ILSVRC2012_val_00043940.JPEG -n04487081/ILSVRC2012_val_00011148.JPEG -n03424325/ILSVRC2012_val_00009315.JPEG -n02500267/ILSVRC2012_val_00010932.JPEG -n04263257/ILSVRC2012_val_00024365.JPEG -n03492542/ILSVRC2012_val_00025010.JPEG -n02834397/ILSVRC2012_val_00011752.JPEG -n01981276/ILSVRC2012_val_00006315.JPEG -n04467665/ILSVRC2012_val_00008391.JPEG -n02088632/ILSVRC2012_val_00029334.JPEG -n03126707/ILSVRC2012_val_00030723.JPEG -n03954731/ILSVRC2012_val_00034134.JPEG -n01534433/ILSVRC2012_val_00003878.JPEG -n01944390/ILSVRC2012_val_00001959.JPEG -n03379051/ILSVRC2012_val_00044073.JPEG -n02268443/ILSVRC2012_val_00018415.JPEG -n03908618/ILSVRC2012_val_00016007.JPEG -n04550184/ILSVRC2012_val_00018341.JPEG -n03947888/ILSVRC2012_val_00036816.JPEG -n02074367/ILSVRC2012_val_00001608.JPEG -n07753113/ILSVRC2012_val_00037431.JPEG -n02012849/ILSVRC2012_val_00034198.JPEG -n02488291/ILSVRC2012_val_00003539.JPEG -n03240683/ILSVRC2012_val_00026560.JPEG -n03476991/ILSVRC2012_val_00003174.JPEG -n02966687/ILSVRC2012_val_00003360.JPEG -n01847000/ILSVRC2012_val_00045692.JPEG -n02099849/ILSVRC2012_val_00027174.JPEG -n04370456/ILSVRC2012_val_00011887.JPEG -n03188531/ILSVRC2012_val_00024563.JPEG -n02093991/ILSVRC2012_val_00016668.JPEG -n02280649/ILSVRC2012_val_00039030.JPEG -n01531178/ILSVRC2012_val_00029581.JPEG -n03733131/ILSVRC2012_val_00027769.JPEG -n01833805/ILSVRC2012_val_00023796.JPEG -n02177972/ILSVRC2012_val_00044574.JPEG -n03980874/ILSVRC2012_val_00028456.JPEG -n02090721/ILSVRC2012_val_00001808.JPEG -n01484850/ILSVRC2012_val_00002752.JPEG -n03832673/ILSVRC2012_val_00016121.JPEG -n02445715/ILSVRC2012_val_00022864.JPEG -n02101388/ILSVRC2012_val_00027846.JPEG -n02879718/ILSVRC2012_val_00041962.JPEG -n02977058/ILSVRC2012_val_00041758.JPEG -n04273569/ILSVRC2012_val_00036660.JPEG -n03776460/ILSVRC2012_val_00028000.JPEG -n02981792/ILSVRC2012_val_00006558.JPEG -n02114548/ILSVRC2012_val_00045961.JPEG -n02002724/ILSVRC2012_val_00031399.JPEG -n02791270/ILSVRC2012_val_00045504.JPEG -n02088364/ILSVRC2012_val_00019187.JPEG -n03188531/ILSVRC2012_val_00009299.JPEG -n02860847/ILSVRC2012_val_00018212.JPEG -n02727426/ILSVRC2012_val_00007765.JPEG -n04532106/ILSVRC2012_val_00043620.JPEG -n02797295/ILSVRC2012_val_00019109.JPEG -n02526121/ILSVRC2012_val_00037407.JPEG -n02860847/ILSVRC2012_val_00013695.JPEG -n09193705/ILSVRC2012_val_00025916.JPEG -n04243546/ILSVRC2012_val_00019474.JPEG -n01829413/ILSVRC2012_val_00044317.JPEG -n03976467/ILSVRC2012_val_00042635.JPEG -n04328186/ILSVRC2012_val_00010517.JPEG -n04133789/ILSVRC2012_val_00015569.JPEG -n03026506/ILSVRC2012_val_00032846.JPEG -n02110958/ILSVRC2012_val_00013653.JPEG -n02672831/ILSVRC2012_val_00028627.JPEG -n02869837/ILSVRC2012_val_00015589.JPEG -n03240683/ILSVRC2012_val_00009606.JPEG -n03045698/ILSVRC2012_val_00040125.JPEG -n02090721/ILSVRC2012_val_00014913.JPEG -n02107908/ILSVRC2012_val_00044380.JPEG -n02814533/ILSVRC2012_val_00027081.JPEG -n04310018/ILSVRC2012_val_00043296.JPEG -n04204347/ILSVRC2012_val_00040297.JPEG -n03617480/ILSVRC2012_val_00026587.JPEG -n02089973/ILSVRC2012_val_00038944.JPEG -n09193705/ILSVRC2012_val_00011422.JPEG -n01855672/ILSVRC2012_val_00028438.JPEG -n03000247/ILSVRC2012_val_00027729.JPEG -n01514668/ILSVRC2012_val_00011403.JPEG -n04179913/ILSVRC2012_val_00042253.JPEG -n03657121/ILSVRC2012_val_00043055.JPEG -n03874599/ILSVRC2012_val_00023222.JPEG -n02939185/ILSVRC2012_val_00035031.JPEG -n02099429/ILSVRC2012_val_00018862.JPEG -n02177972/ILSVRC2012_val_00034203.JPEG -n04536866/ILSVRC2012_val_00026957.JPEG -n02138441/ILSVRC2012_val_00001742.JPEG -n03485794/ILSVRC2012_val_00044989.JPEG -n11879895/ILSVRC2012_val_00003367.JPEG -n03697007/ILSVRC2012_val_00032905.JPEG -n03483316/ILSVRC2012_val_00031410.JPEG -n02168699/ILSVRC2012_val_00028427.JPEG -n02125311/ILSVRC2012_val_00015558.JPEG -n02667093/ILSVRC2012_val_00035834.JPEG -n02965783/ILSVRC2012_val_00048270.JPEG -n07579787/ILSVRC2012_val_00039992.JPEG -n02110958/ILSVRC2012_val_00013670.JPEG -n02128757/ILSVRC2012_val_00010472.JPEG -n07747607/ILSVRC2012_val_00043565.JPEG -n02074367/ILSVRC2012_val_00039158.JPEG -n03259280/ILSVRC2012_val_00007721.JPEG -n03494278/ILSVRC2012_val_00024537.JPEG -n04579145/ILSVRC2012_val_00021588.JPEG -n02226429/ILSVRC2012_val_00033360.JPEG -n04259630/ILSVRC2012_val_00044965.JPEG -n02669723/ILSVRC2012_val_00016186.JPEG -n07753592/ILSVRC2012_val_00038819.JPEG -n03924679/ILSVRC2012_val_00026874.JPEG -n03777754/ILSVRC2012_val_00025706.JPEG -n04548280/ILSVRC2012_val_00018976.JPEG -n02085782/ILSVRC2012_val_00046629.JPEG -n01641577/ILSVRC2012_val_00011664.JPEG -n04548362/ILSVRC2012_val_00046586.JPEG -n02115641/ILSVRC2012_val_00012401.JPEG -n03240683/ILSVRC2012_val_00042101.JPEG -n02102973/ILSVRC2012_val_00038848.JPEG -n13044778/ILSVRC2012_val_00011025.JPEG -n01608432/ILSVRC2012_val_00015746.JPEG -n02098286/ILSVRC2012_val_00024637.JPEG -n03777568/ILSVRC2012_val_00003977.JPEG -n03617480/ILSVRC2012_val_00019061.JPEG -n03047690/ILSVRC2012_val_00040449.JPEG -n01824575/ILSVRC2012_val_00046710.JPEG -n01984695/ILSVRC2012_val_00002243.JPEG -n01728920/ILSVRC2012_val_00013215.JPEG -n07753275/ILSVRC2012_val_00013450.JPEG -n01704323/ILSVRC2012_val_00027700.JPEG -n09288635/ILSVRC2012_val_00035380.JPEG -n01632777/ILSVRC2012_val_00039922.JPEG -n02110063/ILSVRC2012_val_00024218.JPEG -n02172182/ILSVRC2012_val_00012986.JPEG -n02497673/ILSVRC2012_val_00005026.JPEG -n02100877/ILSVRC2012_val_00003516.JPEG -n13037406/ILSVRC2012_val_00006266.JPEG -n07802026/ILSVRC2012_val_00029143.JPEG -n02326432/ILSVRC2012_val_00003697.JPEG -n01734418/ILSVRC2012_val_00026589.JPEG -n02056570/ILSVRC2012_val_00020888.JPEG -n01677366/ILSVRC2012_val_00012156.JPEG -n04120489/ILSVRC2012_val_00026868.JPEG -n03290653/ILSVRC2012_val_00021387.JPEG -n02013706/ILSVRC2012_val_00025929.JPEG -n03062245/ILSVRC2012_val_00023646.JPEG -n03485407/ILSVRC2012_val_00028176.JPEG -n01682714/ILSVRC2012_val_00035598.JPEG -n03467068/ILSVRC2012_val_00014231.JPEG -n03891251/ILSVRC2012_val_00029949.JPEG -n02834397/ILSVRC2012_val_00027813.JPEG -n09288635/ILSVRC2012_val_00005608.JPEG -n02165456/ILSVRC2012_val_00029086.JPEG -n01806143/ILSVRC2012_val_00035287.JPEG -n01491361/ILSVRC2012_val_00023935.JPEG -n02108551/ILSVRC2012_val_00044714.JPEG -n02085936/ILSVRC2012_val_00040150.JPEG -n04613696/ILSVRC2012_val_00013677.JPEG -n02817516/ILSVRC2012_val_00027214.JPEG -n09399592/ILSVRC2012_val_00016654.JPEG -n03372029/ILSVRC2012_val_00014577.JPEG -n01484850/ILSVRC2012_val_00037998.JPEG -n02113799/ILSVRC2012_val_00026858.JPEG -n09399592/ILSVRC2012_val_00025481.JPEG -n07892512/ILSVRC2012_val_00024477.JPEG -n01498041/ILSVRC2012_val_00042547.JPEG -n03131574/ILSVRC2012_val_00016481.JPEG -n01687978/ILSVRC2012_val_00033596.JPEG -n02167151/ILSVRC2012_val_00022273.JPEG -n04238763/ILSVRC2012_val_00007304.JPEG -n03868863/ILSVRC2012_val_00029486.JPEG -n10148035/ILSVRC2012_val_00000341.JPEG -n07715103/ILSVRC2012_val_00042805.JPEG -n03724870/ILSVRC2012_val_00048196.JPEG -n03388183/ILSVRC2012_val_00005686.JPEG -n02870880/ILSVRC2012_val_00020066.JPEG -n04273569/ILSVRC2012_val_00038450.JPEG -n04525305/ILSVRC2012_val_00014061.JPEG -n01798484/ILSVRC2012_val_00004256.JPEG -n02793495/ILSVRC2012_val_00020925.JPEG -n01981276/ILSVRC2012_val_00016333.JPEG -n01729977/ILSVRC2012_val_00005357.JPEG -n03127747/ILSVRC2012_val_00047186.JPEG -n02102973/ILSVRC2012_val_00030527.JPEG -n03100240/ILSVRC2012_val_00022130.JPEG -n03961711/ILSVRC2012_val_00016064.JPEG -n03657121/ILSVRC2012_val_00027626.JPEG -n04557648/ILSVRC2012_val_00046459.JPEG -n02113023/ILSVRC2012_val_00027856.JPEG -n04133789/ILSVRC2012_val_00045999.JPEG -n04435653/ILSVRC2012_val_00020316.JPEG -n02687172/ILSVRC2012_val_00004089.JPEG -n02834397/ILSVRC2012_val_00027477.JPEG -n06596364/ILSVRC2012_val_00001725.JPEG -n03075370/ILSVRC2012_val_00020842.JPEG -n03775546/ILSVRC2012_val_00028647.JPEG -n01698640/ILSVRC2012_val_00020290.JPEG -n10565667/ILSVRC2012_val_00013619.JPEG -n04141975/ILSVRC2012_val_00048812.JPEG -n02106550/ILSVRC2012_val_00043222.JPEG -n02096294/ILSVRC2012_val_00026427.JPEG -n02486261/ILSVRC2012_val_00024107.JPEG -n02091635/ILSVRC2012_val_00022137.JPEG -n03930630/ILSVRC2012_val_00023096.JPEG -n04428191/ILSVRC2012_val_00015883.JPEG -n02454379/ILSVRC2012_val_00045646.JPEG -n04141076/ILSVRC2012_val_00036869.JPEG -n03773504/ILSVRC2012_val_00031503.JPEG -n04285008/ILSVRC2012_val_00003896.JPEG -n04277352/ILSVRC2012_val_00001240.JPEG -n03792782/ILSVRC2012_val_00018304.JPEG -n04548280/ILSVRC2012_val_00007256.JPEG -n01739381/ILSVRC2012_val_00046490.JPEG -n04350905/ILSVRC2012_val_00025482.JPEG -n03218198/ILSVRC2012_val_00002266.JPEG -n02536864/ILSVRC2012_val_00006292.JPEG -n04310018/ILSVRC2012_val_00038885.JPEG -n02123045/ILSVRC2012_val_00004422.JPEG -n04579145/ILSVRC2012_val_00021950.JPEG -n07720875/ILSVRC2012_val_00048089.JPEG -n02788148/ILSVRC2012_val_00037788.JPEG -n04552348/ILSVRC2012_val_00012932.JPEG -n01728920/ILSVRC2012_val_00018041.JPEG -n01667778/ILSVRC2012_val_00031793.JPEG -n06596364/ILSVRC2012_val_00012581.JPEG -n02363005/ILSVRC2012_val_00029016.JPEG -n04591157/ILSVRC2012_val_00006154.JPEG -n04041544/ILSVRC2012_val_00026300.JPEG -n03733281/ILSVRC2012_val_00012842.JPEG -n03627232/ILSVRC2012_val_00037616.JPEG -n07717556/ILSVRC2012_val_00020363.JPEG -n02422699/ILSVRC2012_val_00032385.JPEG -n02791124/ILSVRC2012_val_00004679.JPEG -n01824575/ILSVRC2012_val_00033888.JPEG -n02807133/ILSVRC2012_val_00048300.JPEG -n02510455/ILSVRC2012_val_00008673.JPEG -n12620546/ILSVRC2012_val_00024830.JPEG -n03372029/ILSVRC2012_val_00003736.JPEG -n03207743/ILSVRC2012_val_00030446.JPEG -n04461696/ILSVRC2012_val_00039723.JPEG -n15075141/ILSVRC2012_val_00041217.JPEG -n03691459/ILSVRC2012_val_00002577.JPEG -n07747607/ILSVRC2012_val_00049490.JPEG -n03759954/ILSVRC2012_val_00021431.JPEG -n02281787/ILSVRC2012_val_00043381.JPEG -n03400231/ILSVRC2012_val_00014537.JPEG -n02939185/ILSVRC2012_val_00014420.JPEG -n13133613/ILSVRC2012_val_00025432.JPEG \ No newline at end of file diff --git a/robustbench/leaderboard/__init__.py b/robustbench/leaderboard/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/robustbench/leaderboard/leaderboard.html.j2 b/robustbench/leaderboard/leaderboard.html.j2 deleted file mode 100644 index 817897ce..00000000 --- a/robustbench/leaderboard/leaderboard.html.j2 +++ /dev/null @@ -1,82 +0,0 @@ - - - - - - - {% if threat_model != "corruptions" %} - - - - {% endif %} - {% if threat_model == "corruptions" %} - - {% endif %} - - - - - - - {% for model in models %} - - - - - - {% if threat_model != "corruptions" %} - - - {% endif %} - - - - - {% endfor %} - -
RankMethod - Standard
- accuracy - 		- AutoAttack
- robust
- accuracy - 		- Best known
- robust
- accuracy - 		- AA eval.
- potentially
- unreliable - 		- Robust
- accuracy - 		Extra
data		ArchitectureVenue
{{ loop.index }} - {{ model.name }} - {% if model.footnote is defined and model.footnote != None %} -
- - {{ model.footnote }} - - {% endif %} - 		{{ model.clean_acc }}%{{ model[acc_field] }}%{{ model.external if model.external is defined and model.external else model[acc_field]}}%{{ "Unknown" if model.unreliable is not defined else ("
☑
" if model.unreliable else "
×
") }}		{{ "☑" if model.additional_data else "×" }}{{ model.architecture }}{{ model.venue }}
- \ No newline at end of file diff --git a/robustbench/leaderboard/template.py b/robustbench/leaderboard/template.py deleted file mode 100644 index ba72642d..00000000 --- a/robustbench/leaderboard/template.py +++ /dev/null @@ -1,103 +0,0 @@ -import json -from argparse import ArgumentParser -from pathlib import Path -from typing import Union - -from jinja2 import Environment, PackageLoader, select_autoescape - -from robustbench.model_zoo.enums import BenchmarkDataset, ThreatModel -from robustbench.utils import ACC_FIELDS - - -def generate_leaderboard(dataset: Union[str, BenchmarkDataset], - threat_model: Union[str, ThreatModel], - models_folder: str = "model_info") -> str: - """Prints the HTML leaderboard starting from the .json results. - - The result is a that can be put directly into the RobustBench index.html page, - and looks the same as the tables that are already existing. - - The .json results must have the same structure as the following: - `` - { - "link": "https://arxiv.org/abs/2003.09461", - "name": "Adversarial Robustness on In- and Out-Distribution Improves Explainability", - "authors": "Maximilian Augustin, Alexander Meinke, Matthias Hein", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ECCV 2020", - "architecture": "ResNet-50", - "eps": "0.5", - "clean_acc": "91.08", - "reported": "73.27", - "autoattack_acc": "72.91" - } - `` - - If the model is robust to common corruptions, then the "autoattack_acc" field should be - "corruptions_acc". - - :param dataset: The dataset of the wanted leaderboard. - :param threat_model: The threat model of the wanted leaderboard. - :param models_folder: The base folder of the model jsons (e.g. our "model_info" folder). - - :return: The resulting HTML table. - """ - dataset_: BenchmarkDataset = BenchmarkDataset(dataset) - threat_model_: ThreatModel = ThreatModel(threat_model) - - folder = Path(models_folder) / dataset_.value / threat_model_.value - - acc_field = ACC_FIELDS[threat_model_] - - models = [] - for model_path in folder.glob("*.json"): - with open(model_path) as fp: - model = json.load(fp) - - models.append(model) - - #models.sort(key=lambda x: x[acc_field], reverse=True) - def get_key(x): - if isinstance(acc_field, str): - return float(x[acc_field]) - else: - for k in acc_field: - if k in x.keys(): - return float(x[k]) - models.sort(key=get_key, reverse=True) - - env = Environment(loader=PackageLoader('robustbench', 'leaderboard'), - autoescape=select_autoescape(['html', 'xml'])) - - template = env.get_template('leaderboard.html.j2') - - result = template.render(threat_model=threat_model, dataset=dataset, - models=models, acc_field=acc_field if isinstance(acc_field, str) else acc_field[-1]) - print(result) - return result - - -if __name__ == "__main__": - parser = ArgumentParser() - parser.add_argument( - "--dataset", - type=str, - default="cifar10", - help="The dataset of the desired leaderboard." - ) - parser.add_argument( - "--threat_model", - type=str, - help="The threat model of the desired leaderboard." - ) - parser.add_argument( - "--models_folder", - type=str, - default="model_info", - help="The base folder of the model jsons (e.g. our 'model_info' folder)" - ) - args = parser.parse_args() - - generate_leaderboard(args.dataset, args.threat_model, args.models_folder) diff --git a/robustbench/loaders.py b/robustbench/loaders.py deleted file mode 100644 index e050b14d..00000000 --- a/robustbench/loaders.py +++ /dev/null @@ -1,215 +0,0 @@ -""" -This file is based on the code from https://github.com/pytorch/vision/blob/master/torchvision/datasets/folder.py. -""" -import pkg_resources - -from torchvision.datasets.vision import VisionDataset - -import torch -import torch.utils.data as data -import torchvision.transforms as transforms - -from PIL import Image - -import os -import os.path -import sys - - -def make_custom_dataset(root, path_imgs, class_to_idx): - with open(pkg_resources.resource_filename(__name__, path_imgs), 'r') as f: - fnames = f.readlines() - images = [(os.path.join(root, - c.split('\n')[0]), class_to_idx[c.split('/')[0]]) - for c in fnames] - - return images - - -class CustomDatasetFolder(VisionDataset): - """A generic data loader where the samples are arranged in this way: :: - root/class_x/xxx.ext - root/class_x/xxy.ext - root/class_x/xxz.ext - root/class_y/123.ext - root/class_y/nsdf3.ext - root/class_y/asd932_.ext - Args: - root (string): Root directory path. - loader (callable): A function to load a sample given its path. - extensions (tuple[string]): A list of allowed extensions. - both extensions and is_valid_file should not be passed. - transform (callable, optional): A function/transform that takes in - a sample and returns a transformed version. - E.g, ``transforms.RandomCrop`` for images. - target_transform (callable, optional): A function/transform that takes - in the target and transforms it. - is_valid_file (callable, optional): A function that takes path of an Image file - and check if the file is a valid_file (used to check of corrupt files) - both extensions and is_valid_file should not be passed. - Attributes: - classes (list): List of the class names. - class_to_idx (dict): Dict with items (class_name, class_index). - samples (list): List of (sample path, class_index) tuples - targets (list): The class_index value for each image in the dataset - """ - - def __init__(self, - root, - loader, - extensions=None, - transform=None, - target_transform=None, - is_valid_file=None): - super(CustomDatasetFolder, self).__init__(root) - self.transform = transform - self.target_transform = target_transform - classes, class_to_idx = self._find_classes(self.root) - samples = make_custom_dataset( - self.root, 'helper_files/imagenet_test_image_ids.txt', - class_to_idx) - if len(samples) == 0: - raise (RuntimeError("Found 0 files in subfolders of: " + - self.root + "\n" - "Supported extensions are: " + - ",".join(extensions))) - - self.loader = loader - self.extensions = extensions - - self.classes = classes - self.class_to_idx = class_to_idx - self.samples = samples - self.targets = [s[1] for s in samples] - - def _find_classes(self, dir): - """ - Finds the class folders in a dataset. - Args: - dir (string): Root directory path. - Returns: - tuple: (classes, class_to_idx) where classes are relative to (dir), and class_to_idx is a dictionary. - Ensures: - No class is a subdirectory of another. - """ - if sys.version_info >= (3, 5): - # Faster and available in Python 3.5 and above - classes = [d.name for d in os.scandir(dir) if d.is_dir()] - else: - classes = [ - d for d in os.listdir(dir) - if os.path.isdir(os.path.join(dir, d)) - ] - classes.sort() - class_to_idx = {classes[i]: i for i in range(len(classes))} - return classes, class_to_idx - - def __getitem__(self, index): - """ - Args: - index (int): Index - Returns: - tuple: (sample, target) where target is class_index of the target class. - """ - path, target = self.samples[index] - sample = self.loader(path) - if self.transform is not None: - sample = self.transform(sample) - if self.target_transform is not None: - target = self.target_transform(target) - return sample, target, path - - def __len__(self): - return len(self.samples) - - -IMG_EXTENSIONS = ('.jpg', '.jpeg', '.png', '.ppm', '.bmp', '.pgm', '.tif', - '.tiff', '.webp') - - -def pil_loader(path): - # open path as file to avoid ResourceWarning (https://github.com/python-pillow/Pillow/issues/835) - with open(path, 'rb') as f: - img = Image.open(f) - return img.convert('RGB') - - -def accimage_loader(path): - import accimage - try: - return accimage.Image(path) - except IOError: - # Potentially a decoding problem, fall back to PIL.Image - return pil_loader(path) - - -def default_loader(path): - from torchvision import get_image_backend - if get_image_backend() == 'accimage': - return accimage_loader(path) - else: - return pil_loader(path) - - -class CustomImageFolder(CustomDatasetFolder): - """A generic data loader where the images are arranged in this way: :: - root/dog/xxx.png - root/dog/xxy.png - root/dog/xxz.png - root/cat/123.png - root/cat/nsdf3.png - root/cat/asd932_.png - Args: - root (string): Root directory path. - transform (callable, optional): A function/transform that takes in an PIL image - and returns a transformed version. E.g, ``transforms.RandomCrop`` - target_transform (callable, optional): A function/transform that takes in the - target and transforms it. - loader (callable, optional): A function to load an image given its path. - is_valid_file (callable, optional): A function that takes path of an Image file - and check if the file is a valid_file (used to check of corrupt files) - Attributes: - classes (list): List of the class names. - class_to_idx (dict): Dict with items (class_name, class_index). - imgs (list): List of (image path, class_index) tuples - """ - - def __init__(self, - root, - transform=None, - target_transform=None, - loader=default_loader, - is_valid_file=None): - super(CustomImageFolder, - self).__init__(root, - loader, - IMG_EXTENSIONS if is_valid_file is None else None, - transform=transform, - target_transform=target_transform, - is_valid_file=is_valid_file) - - self.imgs = self.samples - - -if __name__ == '__main__': - data_dir = '~/imagenet/val' - imagenet = CustomImageFolder( - data_dir, - transforms.Compose([ - transforms.Resize(256), - transforms.CenterCrop(224), - transforms.ToTensor() - ])) - - torch.manual_seed(0) - - test_loader = data.DataLoader(imagenet, - batch_size=5000, - shuffle=True, - num_workers=30) - - x, y, path = next(iter(test_loader)) - - with open('path_imgs_2.txt', 'w') as f: - f.write('\n'.join(path)) - f.flush() diff --git a/robustbench/model_info/cifar10/L2/Augustin2020Adversarial.json b/robustbench/model_info/cifar10/L2/Augustin2020Adversarial.json deleted file mode 100644 index fb4d8553..00000000 --- a/robustbench/model_info/cifar10/L2/Augustin2020Adversarial.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2003.09461", - "name": "Adversarial Robustness on In- and Out-Distribution Improves Explainability", - "authors": "Maximilian Augustin, Alexander Meinke, Matthias Hein", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ECCV 2020", - "architecture": "ResNet-50", - "eps": "0.5", - "clean_acc": "91.08", - "reported": "73.27", - "autoattack_acc": "72.91", - "footnote": "Extra data used only as OOD dataset." -} diff --git a/robustbench/model_info/cifar10/L2/Augustin2020Adversarial_34_10.json b/robustbench/model_info/cifar10/L2/Augustin2020Adversarial_34_10.json deleted file mode 100644 index 03eca715..00000000 --- a/robustbench/model_info/cifar10/L2/Augustin2020Adversarial_34_10.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2003.09461", - "name": "Adversarial Robustness on In- and Out-Distribution Improves Explainability", - "authors": "Maximilian Augustin, Alexander Meinke, Matthias Hein", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ECCV 2020", - "architecture": "WideResNet-34-10", - "eps": "0.5", - "clean_acc": "92.23", - "reported": "76.25", - "autoattack_acc": "76.25", - "footnote": "Extra data used only as OOD dataset.", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/L2/Augustin2020Adversarial_34_10_extra.json b/robustbench/model_info/cifar10/L2/Augustin2020Adversarial_34_10_extra.json deleted file mode 100644 index 265e8ec7..00000000 --- a/robustbench/model_info/cifar10/L2/Augustin2020Adversarial_34_10_extra.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2003.09461", - "name": "Adversarial Robustness on In- and Out-Distribution Improves Explainability", - "authors": "Maximilian Augustin, Alexander Meinke, Matthias Hein", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ECCV 2020", - "architecture": "WideResNet-34-10", - "eps": "0.5", - "clean_acc": "93.96", - "reported": "78.79", - "autoattack_acc": "78.79", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/L2/Ding2020MMA.json b/robustbench/model_info/cifar10/L2/Ding2020MMA.json deleted file mode 100644 index c0fe401c..00000000 --- a/robustbench/model_info/cifar10/L2/Ding2020MMA.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://openreview.net/forum?id=HkeryxBtPB", - "name": "MMA Training: Direct Input Space Margin Maximization through Adversarial Training", - "authors": "Gavin Weiguang Ding, Yash Sharma, Kry Yik Chau Lui, Ruitong Huang", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ICLR 2020", - "architecture": "WideResNet-28-4", - "eps": "0.5", - "clean_acc": "88.02", - "reported": "66.18", - "autoattack_acc": "66.09", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/L2/Engstrom2019Robustness.json b/robustbench/model_info/cifar10/L2/Engstrom2019Robustness.json deleted file mode 100644 index 899a598d..00000000 --- a/robustbench/model_info/cifar10/L2/Engstrom2019Robustness.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://github.com/MadryLab/robustness", - "name": "Robustness library", - "authors": "Logan Engstrom, Andrew Ilyas, Hadi Salman, Shibani Santurkar, Dimitris Tsipras", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "GitHub,
Sep 2019", - "architecture": "ResNet-50", - "eps": "0.5", - "clean_acc": "90.83", - "reported": "70.11", - "autoattack_acc": "69.24", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/L2/Gowal2020Uncovering.json b/robustbench/model_info/cifar10/L2/Gowal2020Uncovering.json deleted file mode 100644 index 755b77d1..00000000 --- a/robustbench/model_info/cifar10/L2/Gowal2020Uncovering.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2010.03593", - "name": "Uncovering the Limits of Adversarial Training against Norm-Bounded Adversarial Examples", - "authors": "Sven Gowal, Chongli Qin, Jonathan Uesato, Timothy Mann, Pushmeet Kohli", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Oct 2020", - "architecture": "WideResNet-70-16", - "eps": "0.5", - "clean_acc": "90.90", - "reported": "74.50", - "autoattack_acc": "74.50", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/L2/Gowal2020Uncovering_extra.json b/robustbench/model_info/cifar10/L2/Gowal2020Uncovering_extra.json deleted file mode 100644 index b517f83d..00000000 --- a/robustbench/model_info/cifar10/L2/Gowal2020Uncovering_extra.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2010.03593", - "name": "Uncovering the Limits of Adversarial Training against Norm-Bounded Adversarial Examples", - "authors": "Sven Gowal, Chongli Qin, Jonathan Uesato, Timothy Mann, Pushmeet Kohli", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Oct 2020", - "architecture": "WideResNet-70-16", - "eps": "0.5", - "clean_acc": "94.74", - "reported": "80.53", - "autoattack_acc": "80.53", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/L2/Rade2021Helper_R18_ddpm.json b/robustbench/model_info/cifar10/L2/Rade2021Helper_R18_ddpm.json deleted file mode 100644 index 6cb0db77..00000000 --- a/robustbench/model_info/cifar10/L2/Rade2021Helper_R18_ddpm.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://openreview.net/forum?id=BuD2LmNaU3a", - "name": "Helper-based Adversarial Training: Reducing Excessive Margin to Achieve a Better Accuracy vs. Robustness Trade-off", - "authors": "Rahul Rade and Seyed-Mohsen Moosavi-Dezfooli", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "OpenReview, Jun 2021", - "architecture": "PreActResNet-18", - "eps": "0.5", - "clean_acc": "90.57", - "reported": "76.15", - "footnote": "It uses additional 1M synthetic images in training.", - "autoattack_acc": "76.15", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/L2/Rebuffi2021Fixing_28_10_cutmix_ddpm.json b/robustbench/model_info/cifar10/L2/Rebuffi2021Fixing_28_10_cutmix_ddpm.json deleted file mode 100644 index 1dce6285..00000000 --- a/robustbench/model_info/cifar10/L2/Rebuffi2021Fixing_28_10_cutmix_ddpm.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2103.01946", - "name": "Fixing Data Augmentation to Improve Adversarial Robustness", - "authors": "Sylvestre-Alvise Rebuffi, Sven Gowal, Dan A. Calian, Florian Stimberg, Olivia Wiles, Timothy Mann", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Mar 2021", - "architecture": "WideResNet-28-10", - "eps": "0.5", - "clean_acc": "91.79", - "reported": "78.80", - "footnote": "It uses additional 1M synthetic images in training.", - "autoattack_acc": "78.80", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/L2/Rebuffi2021Fixing_70_16_cutmix_ddpm.json b/robustbench/model_info/cifar10/L2/Rebuffi2021Fixing_70_16_cutmix_ddpm.json deleted file mode 100644 index a42dcbf7..00000000 --- a/robustbench/model_info/cifar10/L2/Rebuffi2021Fixing_70_16_cutmix_ddpm.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2103.01946", - "name": "Fixing Data Augmentation to Improve Adversarial Robustness", - "authors": "Sylvestre-Alvise Rebuffi, Sven Gowal, Dan A. Calian, Florian Stimberg, Olivia Wiles, Timothy Mann", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Mar 2021", - "architecture": "WideResNet-70-16", - "eps": "0.5", - "clean_acc": "92.41", - "reported": "80.42", - "footnote": "It uses additional 1M synthetic images in training.", - "autoattack_acc": "80.42", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/L2/Rebuffi2021Fixing_70_16_cutmix_extra.json b/robustbench/model_info/cifar10/L2/Rebuffi2021Fixing_70_16_cutmix_extra.json deleted file mode 100644 index 92dcb985..00000000 --- a/robustbench/model_info/cifar10/L2/Rebuffi2021Fixing_70_16_cutmix_extra.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2103.01946", - "name": "Fixing Data Augmentation to Improve Adversarial Robustness", - "authors": "Sylvestre-Alvise Rebuffi, Sven Gowal, Dan A. Calian, Florian Stimberg, Olivia Wiles, Timothy Mann", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Mar 2021", - "architecture": "WideResNet-70-16", - "eps": "0.5", - "clean_acc": "95.74", - "reported": "82.32", - "footnote": "", - "autoattack_acc": "82.32", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/L2/Rebuffi2021Fixing_R18_cutmix_ddpm.json b/robustbench/model_info/cifar10/L2/Rebuffi2021Fixing_R18_cutmix_ddpm.json deleted file mode 100644 index 5e461cc8..00000000 --- a/robustbench/model_info/cifar10/L2/Rebuffi2021Fixing_R18_cutmix_ddpm.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2103.01946", - "name": "Fixing Data Augmentation to Improve Adversarial Robustness", - "authors": "Sylvestre-Alvise Rebuffi, Sven Gowal, Dan A. Calian, Florian Stimberg, Olivia Wiles, Timothy Mann", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Mar 2021", - "architecture": "PreActResNet-18", - "eps": "0.5", - "clean_acc": "90.33", - "reported": "75.86", - "footnote": "It uses additional 1M synthetic images in training.", - "autoattack_acc": "75.86", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/L2/Rice2020Overfitting.json b/robustbench/model_info/cifar10/L2/Rice2020Overfitting.json deleted file mode 100644 index 93b41489..00000000 --- a/robustbench/model_info/cifar10/L2/Rice2020Overfitting.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2002.11569", - "name": "Overfitting in adversarially robust deep learning", - "authors": "Leslie Rice, Eric Wong, J. Zico Kolter", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ICML 2020", - "architecture": "PreActResNet-18", - "eps": "0.5", - "clean_acc": "88.67", - "reported": "71.6", - "autoattack_acc": "67.68", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/L2/Rony2019Decoupling.json b/robustbench/model_info/cifar10/L2/Rony2019Decoupling.json deleted file mode 100644 index 03a7f12d..00000000 --- a/robustbench/model_info/cifar10/L2/Rony2019Decoupling.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "https://arxiv.org/abs/1811.09600", - "name": "Decoupling Direction and Norm for Efficient Gradient-Based L2 Adversarial Attacks and Defenses", - "authors": "J\u00e9r\u00f4me Rony, Luiz G. Hafemann, Luiz S. Oliveira, Ismail Ben Ayed, Robert Sabourin, Eric Granger", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "CVPR 2019", - "architecture": "WideResNet-28-10", - "eps": "0.5", - "clean_acc": "89.05", - "reported": "67.6", - "autoattack_acc": "66.44" -} diff --git a/robustbench/model_info/cifar10/L2/Sehwag2021Proxy.json b/robustbench/model_info/cifar10/L2/Sehwag2021Proxy.json deleted file mode 100644 index 9a888fc6..00000000 --- a/robustbench/model_info/cifar10/L2/Sehwag2021Proxy.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2104.09425", - "name": "Robust Learning Meets Generative Models: Can Proxy Distributions Improve Adversarial Robustness?", - "authors": "Vikash Sehwag, Saeed Mahloujifar, Tinashe Handina, Sihui Dai, Chong Xiang, Mung Chiang, Prateek Mittal", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ICLR 2022", - "architecture": "WideResNet-34-10", - "eps": "0.5", - "clean_acc": "90.93", - "reported": "77.24", - "autoattack_acc": "77.24", - "footnote": "It uses additional 10M synthetic images in training.", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/L2/Sehwag2021Proxy_R18.json b/robustbench/model_info/cifar10/L2/Sehwag2021Proxy_R18.json deleted file mode 100644 index fae40335..00000000 --- a/robustbench/model_info/cifar10/L2/Sehwag2021Proxy_R18.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2104.09425", - "name": "Robust Learning Meets Generative Models: Can Proxy Distributions Improve Adversarial Robustness?", - "authors": "Vikash Sehwag, Saeed Mahloujifar, Tinashe Handina, Sihui Dai, Chong Xiang, Mung Chiang, Prateek Mittal", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ICLR 2022", - "architecture": "ResNet-18", - "eps": "0.5", - "clean_acc": "89.76", - "reported": "74.41", - "autoattack_acc": "74.41", - "footnote": "It uses additional 10M synthetic images in training.", - "unreliable": false -} \ No newline at end of file diff --git a/robustbench/model_info/cifar10/L2/Standard.json b/robustbench/model_info/cifar10/L2/Standard.json deleted file mode 100644 index 0fd1bd12..00000000 --- a/robustbench/model_info/cifar10/L2/Standard.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "https://github.com/RobustBench/robustbench/", - "name": "Standardly trained model", - "authors": "", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "N/A", - "architecture": "WideResNet-28-10", - "eps": "0.5", - "clean_acc": "94.78", - "reported": "0.0", - "autoattack_acc": "0.0" -} diff --git a/robustbench/model_info/cifar10/L2/Wu2020Adversarial.json b/robustbench/model_info/cifar10/L2/Wu2020Adversarial.json deleted file mode 100644 index c3a6f9ae..00000000 --- a/robustbench/model_info/cifar10/L2/Wu2020Adversarial.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2004.05884", - "name": "Adversarial Weight Perturbation Helps Robust Generalization", - "authors": "Dongxian Wu, Shu-tao Xia, Yisen Wang", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "NeurIPS 2020", - "architecture": "WideResNet-34-10", - "eps": "0.5", - "clean_acc": "88.51", - "reported": "73.66", - "autoattack_acc": "73.66", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Addepalli2021Towards_RN18.json b/robustbench/model_info/cifar10/Linf/Addepalli2021Towards_RN18.json deleted file mode 100644 index 732ecdb2..00000000 --- a/robustbench/model_info/cifar10/Linf/Addepalli2021Towards_RN18.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://openreview.net/forum?id=SHB_znlW5G7", - "name": "Towards Achieving Adversarial Robustness Beyond Perceptual Limits", - "authors": "Sravanti Addepalli, Samyak Jain, Gaurang Sriramanan, Shivangi Khare, Venkatesh Babu Radhakrishnan", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "OpenReview, Jun 2021", - "architecture": "ResNet-18", - "eps": "8/255", - "clean_acc": "80.24", - "reported": "51.06", - "autoattack_acc": "51.06", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Addepalli2021Towards_WRN34.json b/robustbench/model_info/cifar10/Linf/Addepalli2021Towards_WRN34.json deleted file mode 100644 index 522c50f3..00000000 --- a/robustbench/model_info/cifar10/Linf/Addepalli2021Towards_WRN34.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://openreview.net/forum?id=SHB_znlW5G7", - "name": "Towards Achieving Adversarial Robustness Beyond Perceptual Limits", - "authors": "Sravanti Addepalli, Samyak Jain, Gaurang Sriramanan, Shivangi Khare, Venkatesh Babu Radhakrishnan", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "OpenReview, Jun 2021", - "architecture": "WideResNet-34-10", - "eps": "8/255", - "clean_acc": "85.32", - "reported": "58.04", - "autoattack_acc": "58.04", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Addepalli2022Efficient_RN18.json b/robustbench/model_info/cifar10/Linf/Addepalli2022Efficient_RN18.json deleted file mode 100644 index 88156856..00000000 --- a/robustbench/model_info/cifar10/Linf/Addepalli2022Efficient_RN18.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://artofrobust.github.io/short_paper/31.pdf", - "name": "Efficient and Effective Augmentation Strategy for Adversarial Training", - "authors": "Sravanti Addepalli, Samyak Jain, Venkatesh Babu Radhakrishnan", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "CVPRW 2022", - "architecture": "ResNet-18", - "eps": "8/255", - "clean_acc": "85.71", - "reported": "52.50", - "autoattack_acc": "52.48", - "unreliable": false -} \ No newline at end of file diff --git a/robustbench/model_info/cifar10/Linf/Addepalli2022Efficient_WRN_34_10.json b/robustbench/model_info/cifar10/Linf/Addepalli2022Efficient_WRN_34_10.json deleted file mode 100644 index 32b3df61..00000000 --- a/robustbench/model_info/cifar10/Linf/Addepalli2022Efficient_WRN_34_10.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://artofrobust.github.io/short_paper/31.pdf", - "name": "Efficient and Effective Augmentation Strategy for Adversarial Training", - "authors": "Sravanti Addepalli, Samyak Jain, Venkatesh Babu Radhakrishnan", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "CVPRW 2022", - "architecture": "WideResNet-34-10", - "eps": "8/255", - "clean_acc": "88.71", - "reported": "57.81", - "autoattack_acc": "57.81", - "unreliable": false -} \ No newline at end of file diff --git a/robustbench/model_info/cifar10/Linf/Alayrac2019Labels.json b/robustbench/model_info/cifar10/Linf/Alayrac2019Labels.json deleted file mode 100644 index d2d63534..00000000 --- a/robustbench/model_info/cifar10/Linf/Alayrac2019Labels.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "https://arxiv.org/abs/1905.13725", - "name": "Are Labels Required for Improving Adversarial Robustness?", - "authors": "Jonathan Uesato, Jean-Baptiste Alayrac, Po-Sen Huang, Robert Stanforth, Alhussein Fawzi, Pushmeet Kohli", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "NeurIPS 2019", - "architecture": "WideResNet-28-10", - "eps": "8/255", - "clean_acc": "86.46", - "reported": "56.30", - "autoattack_acc": "56.03" -} diff --git a/robustbench/model_info/cifar10/Linf/Alfarra2020ClusTR.json b/robustbench/model_info/cifar10/Linf/Alfarra2020ClusTR.json deleted file mode 100644 index 97a7f070..00000000 --- a/robustbench/model_info/cifar10/Linf/Alfarra2020ClusTR.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2006.07682", - "name": "ClusTR: Clustering Training for Robustness\n", - "authors": "Motasem Alfarra, Juan C. Perez, Adel Bibi, Ali Thabet, Pablo Arbelaez, Bernard Ghanem", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Jun 2020", - "architecture": "WideResNet-28-10", - "eps": "8/255", - "clean_acc": "91.03", - "reported": "74.04", - "autoattack_acc": "0.00" -} diff --git a/robustbench/model_info/cifar10/Linf/Andriushchenko2020Understanding.json b/robustbench/model_info/cifar10/Linf/Andriushchenko2020Understanding.json deleted file mode 100644 index 352c0340..00000000 --- a/robustbench/model_info/cifar10/Linf/Andriushchenko2020Understanding.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2007.02617", - "name": "Understanding and Improving Fast Adversarial Training", - "authors": "Maksym Andriushchenko, Nicolas Flammarion", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "NeurIPS 2020", - "architecture": "PreActResNet-18", - "eps": "8/255", - "clean_acc": "79.84", - "reported": "44.54", - "footnote": "Focuses on fast adversarial training.", - "autoattack_acc": "43.93" -} diff --git a/robustbench/model_info/cifar10/Linf/Atzmon2019Controlling.json b/robustbench/model_info/cifar10/Linf/Atzmon2019Controlling.json deleted file mode 100644 index 312a8374..00000000 --- a/robustbench/model_info/cifar10/Linf/Atzmon2019Controlling.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/1905.11911", - "name": "Controlling Neural Level Sets", - "authors": "Matan Atzmon, Niv Haim, Lior Yariv, Ofer Israelov, Haggai Maron, Yaron Lipman", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "NeurIPS 2019", - "architecture": "ResNet-18", - "eps": "0.031", - "clean_acc": "81.30", - "reported": "43.17", - "footnote": "Uses \\(\\ell_{\\infty} \\) = 0.031 \u2248 7.9/255 instead of 8/255.", - "autoattack_acc": "40.22" -} diff --git a/robustbench/model_info/cifar10/Linf/Carmon2019Unlabeled.json b/robustbench/model_info/cifar10/Linf/Carmon2019Unlabeled.json deleted file mode 100644 index 253adaff..00000000 --- a/robustbench/model_info/cifar10/Linf/Carmon2019Unlabeled.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/1905.13736", - "name": "Unlabeled Data Improves Adversarial Robustness", - "authors": "Yair Carmon, Aditi Raghunathan, Ludwig Schmidt, Percy Liang, John C. Duchi", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "NeurIPS 2019", - "architecture": "WideResNet-28-10", - "eps": "8/255", - "clean_acc": "89.69", - "reported": "62.5", - "autoattack_acc": "59.53", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Chan2020Jacobian.json b/robustbench/model_info/cifar10/Linf/Chan2020Jacobian.json deleted file mode 100644 index 02c9a2a9..00000000 --- a/robustbench/model_info/cifar10/Linf/Chan2020Jacobian.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "https://arxiv.org/abs/1912.10185", - "name": "Jacobian Adversarially Regularized Networks for Robustness", - "authors": "Alvin Chan, Yi Tay, Yew Soon Ong, Jie Fu", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ICLR 2020", - "architecture": "WideResNet-34-10", - "eps": "8/255", - "clean_acc": "93.79", - "reported": "15.5", - "autoattack_acc": "0.26" -} diff --git a/robustbench/model_info/cifar10/Linf/Chen2020Adversarial.json b/robustbench/model_info/cifar10/Linf/Chen2020Adversarial.json deleted file mode 100644 index 87b59f2b..00000000 --- a/robustbench/model_info/cifar10/Linf/Chen2020Adversarial.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2003.12862", - "name": "Adversarial Robustness: From Self-Supervised Pre-Training to Fine-Tuning", - "authors": "Tianlong Chen, Sijia Liu, Shiyu Chang, Yu Cheng, Lisa Amini, Zhangyang Wang", - "additional_data": false, - "number_forward_passes": 3, - "dataset": "cifar10", - "venue": "CVPR 2020", - "architecture": "ResNet-50", - "eps": "8/255", - "clean_acc": "86.04", - "reported": "54.64", - "footnote": "Uses ensembles of 3 models.", - "autoattack_acc": "51.56" -} diff --git a/robustbench/model_info/cifar10/Linf/Chen2020Efficient.json b/robustbench/model_info/cifar10/Linf/Chen2020Efficient.json deleted file mode 100644 index 0e34af09..00000000 --- a/robustbench/model_info/cifar10/Linf/Chen2020Efficient.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2010.01278", - "name": "Efficient Robust Training via Backward Smoothing", - "authors": "Jinghui Chen and Yu Cheng and Zhe Gan and Quanquan Gu and Jingjing Liu", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Oct 2020", - "architecture": "WideResNet-34-10", - "eps": "8/255", - "clean_acc": "85.32", - "reported": "51.13", - "footnote": "", - "autoattack_acc": "51.12" -} diff --git a/robustbench/model_info/cifar10/Linf/Chen2021LTD_WRN34_10.json b/robustbench/model_info/cifar10/Linf/Chen2021LTD_WRN34_10.json deleted file mode 100644 index 23acb167..00000000 --- a/robustbench/model_info/cifar10/Linf/Chen2021LTD_WRN34_10.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2111.02331", - "name": "LTD: Low Temperature Distillation for Robust Adversarial Training", - "authors": "Erh-Chung Chen, Che-Rung Lee", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Nov 2021", - "architecture": "WideResNet-34-10", - "eps": "8/255", - "clean_acc": "85.21", - "reported": "56.94", - "footnote": "", - "autoattack_acc": "56.94", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Chen2021LTD_WRN34_20.json b/robustbench/model_info/cifar10/Linf/Chen2021LTD_WRN34_20.json deleted file mode 100644 index 3ef99562..00000000 --- a/robustbench/model_info/cifar10/Linf/Chen2021LTD_WRN34_20.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2111.02331", - "name": "LTD: Low Temperature Distillation for Robust Adversarial Training", - "authors": "Erh-Chung Chen, Che-Rung Lee", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Nov 2021", - "architecture": "WideResNet-34-20", - "eps": "8/255", - "clean_acc": "86.03", - "reported": "57.71", - "footnote": "", - "autoattack_acc": "57.71", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Cui2020Learnable_34_10.json b/robustbench/model_info/cifar10/Linf/Cui2020Learnable_34_10.json deleted file mode 100644 index ecedc570..00000000 --- a/robustbench/model_info/cifar10/Linf/Cui2020Learnable_34_10.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2011.11164", - "name": "Learnable Boundary Guided Adversarial Training", - "authors": "Jiequan Cui and Shu Liu and Liwei Wang and Jiaya Jia", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ICCV 2021", - "architecture": "WideResNet-34-10", - "eps": "0.031", - "clean_acc": "88.22", - "reported": "52.86", - "footnote": "Uses \\(\\ell_{\\infty} \\) = 0.031 \u2248 7.9/255 instead of 8/255", - "autoattack_acc": "52.86", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Cui2020Learnable_34_20.json b/robustbench/model_info/cifar10/Linf/Cui2020Learnable_34_20.json deleted file mode 100644 index c77bb28e..00000000 --- a/robustbench/model_info/cifar10/Linf/Cui2020Learnable_34_20.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2011.11164", - "name": "Learnable Boundary Guided Adversarial Training", - "authors": "Jiequan Cui and Shu Liu and Liwei Wang and Jiaya Jia", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ICCV 2021", - "architecture": "WideResNet-34-20", - "eps": "0.031", - "clean_acc": "88.70", - "reported": "53.57", - "footnote": "Uses \\(\\ell_{\\infty} \\) = 0.031 \u2248 7.9/255 instead of 8/255", - "autoattack_acc": "53.57", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Dai2021Parameterizing.json b/robustbench/model_info/cifar10/Linf/Dai2021Parameterizing.json deleted file mode 100644 index 158cc772..00000000 --- a/robustbench/model_info/cifar10/Linf/Dai2021Parameterizing.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2110.05626", - "name": "Parameterizing Activation Functions for Adversarial Robustness", - "authors": "Sihui Dai, Saeed Mahloujifar, Prateek Mittal", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Oct 2021", - "architecture": "WideResNet-28-10-PSSiLU", - "eps": "8/255", - "clean_acc": "87.02", - "reported": "61.55", - "autoattack_acc": "61.55", - "footnote": "It uses additional ~6M synthetic images in training.", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Debenedetti2022Light_XCiT-L12.json b/robustbench/model_info/cifar10/Linf/Debenedetti2022Light_XCiT-L12.json deleted file mode 100644 index 43aa00e6..00000000 --- a/robustbench/model_info/cifar10/Linf/Debenedetti2022Light_XCiT-L12.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2209.07399", - "name": "A Light Recipe to Train Robust Vision Transformers", - "authors": "Edoardo Debenedetti, Vikash Sehwag, Prateek Mittal", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Sep 2022", - "architecture": "XCiT-L12", - "eps": "8/255", - "clean_acc": "91.73", - "reported": "57.58", - "autoattack_acc": "57.58", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Debenedetti2022Light_XCiT-M12.json b/robustbench/model_info/cifar10/Linf/Debenedetti2022Light_XCiT-M12.json deleted file mode 100644 index 79d193a8..00000000 --- a/robustbench/model_info/cifar10/Linf/Debenedetti2022Light_XCiT-M12.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2209.07399", - "name": "A Light Recipe to Train Robust Vision Transformers", - "authors": "Edoardo Debenedetti, Vikash Sehwag, Prateek Mittal", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Sep 2022", - "architecture": "XCiT-M12", - "eps": "8/255", - "clean_acc": "91.30", - "reported": "57.27", - "autoattack_acc": "57.27", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Debenedetti2022Light_XCiT-S12.json b/robustbench/model_info/cifar10/Linf/Debenedetti2022Light_XCiT-S12.json deleted file mode 100644 index e955c230..00000000 --- a/robustbench/model_info/cifar10/Linf/Debenedetti2022Light_XCiT-S12.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2209.07399", - "name": "A Light Recipe to Train Robust Vision Transformers", - "authors": "Edoardo Debenedetti, Vikash Sehwag, Prateek Mittal", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Sep 2022", - "architecture": "XCiT-S12", - "eps": "8/255", - "clean_acc": "90.06", - "reported": "56.14", - "autoattack_acc": "56.14", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Ding2020MMA.json b/robustbench/model_info/cifar10/Linf/Ding2020MMA.json deleted file mode 100644 index eb67c083..00000000 --- a/robustbench/model_info/cifar10/Linf/Ding2020MMA.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "https://openreview.net/forum?id=HkeryxBtPB", - "name": "MMA Training: Direct Input Space Margin Maximization through Adversarial Training", - "authors": "Gavin Weiguang Ding, Yash Sharma, Kry Yik Chau Lui, Ruitong Huang", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ICLR 2020", - "architecture": "WideResNet-28-4", - "eps": "8/255", - "clean_acc": "84.36", - "reported": "47.18", - "autoattack_acc": "41.44" -} diff --git a/robustbench/model_info/cifar10/Linf/Engstrom2019Robustness.json b/robustbench/model_info/cifar10/Linf/Engstrom2019Robustness.json deleted file mode 100644 index aed35ddd..00000000 --- a/robustbench/model_info/cifar10/Linf/Engstrom2019Robustness.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "https://github.com/MadryLab/robustness", - "name": "Robustness library", - "authors": "Logan Engstrom, Andrew Ilyas, Hadi Salman, Shibani Santurkar, Dimitris Tsipras", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "GitHub,
Oct 2019", - "architecture": "ResNet-50", - "eps": "8/255", - "clean_acc": "87.03", - "reported": "53.29", - "autoattack_acc": "49.25" -} diff --git a/robustbench/model_info/cifar10/Linf/Gowal2020Uncovering_28_10_extra.json b/robustbench/model_info/cifar10/Linf/Gowal2020Uncovering_28_10_extra.json deleted file mode 100644 index f99d5279..00000000 --- a/robustbench/model_info/cifar10/Linf/Gowal2020Uncovering_28_10_extra.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2010.03593", - "name": "Uncovering the Limits of Adversarial Training against Norm-Bounded Adversarial Examples", - "authors": "Sven Gowal, Chongli Qin, Jonathan Uesato, Timothy Mann, Pushmeet Kohli", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Oct 2020", - "architecture": "WideResNet-28-10", - "eps": "8/255", - "clean_acc": "89.48", - "reported": "62.76", - "autoattack_acc": "62.80", - "external": "62.76", - "footnote": "62.76% robust accuracy is due to the original evaluation (AutoAttack + MultiTargeted)", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Gowal2020Uncovering_34_20.json b/robustbench/model_info/cifar10/Linf/Gowal2020Uncovering_34_20.json deleted file mode 100644 index 285669ab..00000000 --- a/robustbench/model_info/cifar10/Linf/Gowal2020Uncovering_34_20.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2010.03593", - "name": "Uncovering the Limits of Adversarial Training against Norm-Bounded Adversarial Examples", - "authors": "Sven Gowal, Chongli Qin, Jonathan Uesato, Timothy Mann, Pushmeet Kohli", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Oct 2020", - "architecture": "WideResNet-34-20", - "eps": "8/255", - "clean_acc": "85.64", - "reported": "56.82", - "autoattack_acc": "56.86", - "external": "56.82", - "footnote": "56.82% robust accuracy is due to the original evaluation (AutoAttack + MultiTargeted)", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Gowal2020Uncovering_70_16.json b/robustbench/model_info/cifar10/Linf/Gowal2020Uncovering_70_16.json deleted file mode 100644 index fbc686e5..00000000 --- a/robustbench/model_info/cifar10/Linf/Gowal2020Uncovering_70_16.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2010.03593", - "name": "Uncovering the Limits of Adversarial Training against Norm-Bounded Adversarial Examples", - "authors": "Sven Gowal, Chongli Qin, Jonathan Uesato, Timothy Mann, Pushmeet Kohli", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Oct 2020", - "architecture": "WideResNet-70-16", - "eps": "8/255", - "clean_acc": "85.29", - "reported": "57.14", - "autoattack_acc": "57.20", - "external": "57.14", - "footnote": "57.14% robust accuracy is due to the original evaluation (AutoAttack + MultiTargeted)", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Gowal2020Uncovering_70_16_extra.json b/robustbench/model_info/cifar10/Linf/Gowal2020Uncovering_70_16_extra.json deleted file mode 100644 index c300a31f..00000000 --- a/robustbench/model_info/cifar10/Linf/Gowal2020Uncovering_70_16_extra.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2010.03593", - "name": "Uncovering the Limits of Adversarial Training against Norm-Bounded Adversarial Examples", - "authors": "Sven Gowal, Chongli Qin, Jonathan Uesato, Timothy Mann, Pushmeet Kohli", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Oct 2020", - "architecture": "WideResNet-70-16", - "eps": "8/255", - "clean_acc": "91.10", - "reported": "65.87", - "autoattack_acc": "65.88", - "external": "65.87", - "footnote": "65.87% robust accuracy is due to the original evaluation (AutoAttack + MultiTargeted)", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Gowal2021Improving_28_10_ddpm_100m.json b/robustbench/model_info/cifar10/Linf/Gowal2021Improving_28_10_ddpm_100m.json deleted file mode 100644 index 0ff112bb..00000000 --- a/robustbench/model_info/cifar10/Linf/Gowal2021Improving_28_10_ddpm_100m.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2110.09468", - "name": "Improving Robustness using Generated Data", - "authors": "Sven Gowal, Sylvestre-Alvise Rebuffi, Olivia Wiles, Florian Stimberg, Dan Andrei Calian, Timothy Mann", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "NeurIPS 2021", - "architecture": "WideResNet-28-10", - "eps": "8/255", - "clean_acc": "87.50", - "reported": "63.38", - "autoattack_acc": "63.44", - "external": "63.38", - "footnote": "It uses additional 100M synthetic images in training. 63.38% robust accuracy is due to the original evaluation (AutoAttack + MultiTargeted)", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Gowal2021Improving_70_16_ddpm_100m.json b/robustbench/model_info/cifar10/Linf/Gowal2021Improving_70_16_ddpm_100m.json deleted file mode 100644 index 04971785..00000000 --- a/robustbench/model_info/cifar10/Linf/Gowal2021Improving_70_16_ddpm_100m.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2110.09468", - "name": "Improving Robustness using Generated Data", - "authors": "Sven Gowal, Sylvestre-Alvise Rebuffi, Olivia Wiles, Florian Stimberg, Dan Andrei Calian, Timothy Mann", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "NeurIPS 2021", - "architecture": "WideResNet-70-16", - "eps": "8/255", - "clean_acc": "88.74", - "reported": "66.10", - "autoattack_acc": "66.11", - "external": "66.10", - "footnote": "It uses additional 100M synthetic images in training. 66.10% robust accuracy is due to the original evaluation (AutoAttack + MultiTargeted)", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Gowal2021Improving_R18_ddpm_100m.json b/robustbench/model_info/cifar10/Linf/Gowal2021Improving_R18_ddpm_100m.json deleted file mode 100644 index 2d37d4cf..00000000 --- a/robustbench/model_info/cifar10/Linf/Gowal2021Improving_R18_ddpm_100m.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2110.09468", - "name": "Improving Robustness using Generated Data", - "authors": "Sven Gowal, Sylvestre-Alvise Rebuffi, Olivia Wiles, Florian Stimberg, Dan Andrei Calian, Timothy Mann", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "NeurIPS 2021", - "architecture": "PreActResNet-18", - "eps": "8/255", - "clean_acc": "87.35", - "reported": "58.50", - "autoattack_acc": "58.63", - "external": "58.50", - "footnote": "It uses additional 100M synthetic images in training. 58.50% robust accuracy is due to the original evaluation (AutoAttack + MultiTargeted)", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Hendrycks2019Using.json b/robustbench/model_info/cifar10/Linf/Hendrycks2019Using.json deleted file mode 100644 index 376fbd14..00000000 --- a/robustbench/model_info/cifar10/Linf/Hendrycks2019Using.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/1901.09960", - "name": "Using Pre-Training Can Improve Model Robustness and Uncertainty", - "authors": "Dan Hendrycks, Kimin Lee, Mantas Mazeika", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ICML 2019", - "architecture": "WideResNet-28-10", - "eps": "8/255", - "clean_acc": "87.11", - "reported": "57.4", - "autoattack_acc": "54.92", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Huang2020Self.json b/robustbench/model_info/cifar10/Linf/Huang2020Self.json deleted file mode 100644 index 17a0bd11..00000000 --- a/robustbench/model_info/cifar10/Linf/Huang2020Self.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2002.10319", - "name": "Self-Adaptive Training: beyond Empirical Risk Minimization", - "authors": "Lang Huang, Chao Zhang, Hongyang Zhang", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "NeurIPS 2020", - "architecture": "WideResNet-34-10", - "eps": "0.031", - "clean_acc": "83.48", - "reported": "58.03", - "footnote": "Uses \\(\\ell_{\\infty} \\) = 0.031 \u2248 7.9/255 instead of 8/255.", - "autoattack_acc": "53.34" -} diff --git a/robustbench/model_info/cifar10/Linf/Huang2021Exploring.json b/robustbench/model_info/cifar10/Linf/Huang2021Exploring.json deleted file mode 100644 index 8b90df10..00000000 --- a/robustbench/model_info/cifar10/Linf/Huang2021Exploring.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2110.03825", - "name": "Exploring Architectural Ingredients of Adversarially Robust Deep Neural Networks", - "authors": "Hanxun Huang, Yisen Wang, Sarah Monazam Erfani, Quanquan Gu, James Bailey, Xingjun Ma", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "NeurIPS 2021", - "architecture": "WideResNet-34-R", - "eps": "8/255", - "clean_acc": "90.56", - "reported": "61.56", - "autoattack_acc": "61.56", - "unreliable": false - -} diff --git a/robustbench/model_info/cifar10/Linf/Huang2021Exploring_ema.json b/robustbench/model_info/cifar10/Linf/Huang2021Exploring_ema.json deleted file mode 100644 index 8e9064fa..00000000 --- a/robustbench/model_info/cifar10/Linf/Huang2021Exploring_ema.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2110.03825", - "name": "Exploring Architectural Ingredients of Adversarially Robust Deep Neural Networks", - "authors": "Hanxun Huang, Yisen Wang, Sarah Monazam Erfani, Quanquan Gu, James Bailey, Xingjun Ma", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "NeurIPS 2021", - "architecture": "WideResNet-34-R", - "eps": "8/255", - "clean_acc": "91.23", - "reported": "62.54", - "autoattack_acc": "62.54", - "footnote": "Uses exponential moving average (EMA)", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Jang2019Adversarial.json b/robustbench/model_info/cifar10/Linf/Jang2019Adversarial.json deleted file mode 100644 index 4805fe62..00000000 --- a/robustbench/model_info/cifar10/Linf/Jang2019Adversarial.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "http://openaccess.thecvf.com/content_ICCV_2019/html/Jang_Adversarial_Defense_via_Learning_to_Generate_Diverse_Attacks_ICCV_2019_paper.html", - "name": "Adversarial Defense via Learning to Generate Diverse Attacks", - "authors": "Yunseok Jang, Tianchen Zhao, Seunghoon Hong, Honglak Lee", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ICCV 2019", - "architecture": "ResNet-20", - "eps": "8/255", - "clean_acc": "78.91", - "reported": "37.40", - "autoattack_acc": "34.95" -} diff --git a/robustbench/model_info/cifar10/Linf/Jia2022LAS-AT_34_10.json b/robustbench/model_info/cifar10/Linf/Jia2022LAS-AT_34_10.json deleted file mode 100644 index fc3a6f1b..00000000 --- a/robustbench/model_info/cifar10/Linf/Jia2022LAS-AT_34_10.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2203.06616", - "name": "LAS-AT: Adversarial Training with Learnable Attack Strategy", - "authors": "Xiaojun Jia, Yong Zhang, Baoyuan Wu, Ke Ma, Jue Wang, Xiaochun Cao", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Mar 2022", - "architecture": "WideResNet-34-10", - "eps": "8/255", - "clean_acc": "84.98", - "reported": "56.26", - "autoattack_acc": "56.26", - "footnote": "", - "unreliable": false -} \ No newline at end of file diff --git a/robustbench/model_info/cifar10/Linf/Jia2022LAS-AT_70_16.json b/robustbench/model_info/cifar10/Linf/Jia2022LAS-AT_70_16.json deleted file mode 100644 index baaf1db2..00000000 --- a/robustbench/model_info/cifar10/Linf/Jia2022LAS-AT_70_16.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2203.06616", - "name": "LAS-AT: Adversarial Training with Learnable Attack Strategy", - "authors": "Xiaojun Jia, Yong Zhang, Baoyuan Wu, Ke Ma, Jue Wang, Xiaochun Cao", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Mar 2022", - "architecture": "WideResNet-70-16", - "eps": "8/255", - "clean_acc": "85.66", - "reported": "57.61", - "autoattack_acc": "57.61", - "footnote": "", - "unreliable": false -} \ No newline at end of file diff --git a/robustbench/model_info/cifar10/Linf/JinRinard2020Manifold.json b/robustbench/model_info/cifar10/Linf/JinRinard2020Manifold.json deleted file mode 100644 index 9340e1e7..00000000 --- a/robustbench/model_info/cifar10/Linf/JinRinard2020Manifold.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2003.04286", - "name": "Manifold Regularization for Adversarial Robustness", - "authors": "Charles Jin, Martin Rinard", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Mar 2020", - "architecture": "ResNet-18", - "eps": "8/255", - "clean_acc": "90.84", - "reported": "71.22", - "autoattack_acc": "1.35" -} diff --git a/robustbench/model_info/cifar10/Linf/Kang2021Stable.json b/robustbench/model_info/cifar10/Linf/Kang2021Stable.json deleted file mode 100644 index 651df459..00000000 --- a/robustbench/model_info/cifar10/Linf/Kang2021Stable.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2110.12976", - "name": "Stable Neural ODE with Lyapunov-Stable Equilibrium Points for Defending Against Adversarial Attacks", - "authors": "Qiyu Kang, Yang Song, Qinxu Ding, Wee Peng Tay", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "NeurIPS 2021", - "architecture": "WideResNet-70-16, Neural ODE block", - "eps": "8/255", - "clean_acc": "93.73", - "reported": "71.28", - "autoattack_acc": "71.28", - "external": "64.20", - "footnote": "Based on the model Rebuffi2021Fixing_70_16_cutmix_extra. 64.20% robust accuracy is due to AutoAttack + transfer APGD from Rebuffi2021Fixing_70_16_cutmix_extra", - "unreliable": true -} diff --git a/robustbench/model_info/cifar10/Linf/KimWang2020Sensible.json b/robustbench/model_info/cifar10/Linf/KimWang2020Sensible.json deleted file mode 100644 index 4f3e9d6c..00000000 --- a/robustbench/model_info/cifar10/Linf/KimWang2020Sensible.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "https://openreview.net/forum?id=rJlf_RVKwr", - "name": "Sensible adversarial learning", - "authors": "Jungeum Kim, Xiao Wang", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "OpenReview, Sep 2019", - "architecture": "WideResNet-34-10", - "eps": "8/255", - "clean_acc": "91.51", - "reported": "57.23", - "autoattack_acc": "34.22" -} diff --git a/robustbench/model_info/cifar10/Linf/Kumari2019Harnessing.json b/robustbench/model_info/cifar10/Linf/Kumari2019Harnessing.json deleted file mode 100644 index 9efa5da4..00000000 --- a/robustbench/model_info/cifar10/Linf/Kumari2019Harnessing.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "https://arxiv.org/abs/1905.05186", - "name": "Harnessing the Vulnerability of Latent Layers in Adversarially Trained Models", - "authors": "Mayank Singh, Abhishek Sinha, Nupur Kumari, Harshitha Machiraju, Balaji Krishnamurthy, Vineeth N Balasubramanian", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "IJCAI 2019", - "architecture": "WideResNet-34-10", - "eps": "8/255", - "clean_acc": "87.80", - "reported": "53.04", - "autoattack_acc": "49.12" -} diff --git a/robustbench/model_info/cifar10/Linf/Kundu2020Tunable.json b/robustbench/model_info/cifar10/Linf/Kundu2020Tunable.json deleted file mode 100644 index 76fc4b9c..00000000 --- a/robustbench/model_info/cifar10/Linf/Kundu2020Tunable.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2011.03083", - "name": "A Tunable Robust Pruning Framework Through Dynamic Network Rewiring of DNNs", - "authors": "Kundu, Souvik and Nazemi, Mahdi and Beerel, Peter A and Pedram, Massoud", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ASP-DAC 2021", - "architecture": "ResNet-18", - "eps": "8/255", - "clean_acc": "87.32", - "reported": "47.35", - "footnote": "Compressed model", - "autoattack_acc": "40.41", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Madry2018Towards.json b/robustbench/model_info/cifar10/Linf/Madry2018Towards.json deleted file mode 100644 index 58ab2924..00000000 --- a/robustbench/model_info/cifar10/Linf/Madry2018Towards.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "https://arxiv.org/abs/1706.06083", - "name": "Towards Deep Learning Models Resistant to Adversarial Attacks", - "authors": "Aleksander Madry, Aleksandar Makelov, Ludwig Schmidt, Dimitris Tsipras, Adrian Vladu", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ICLR 2018", - "architecture": "WideResNet-34-10", - "eps": "8/255", - "clean_acc": "87.14", - "reported": "47.04", - "autoattack_acc": "44.04" -} diff --git a/robustbench/model_info/cifar10/Linf/Mao2019Metric.json b/robustbench/model_info/cifar10/Linf/Mao2019Metric.json deleted file mode 100644 index da86ec48..00000000 --- a/robustbench/model_info/cifar10/Linf/Mao2019Metric.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "http://papers.nips.cc/paper/8339-metric-learning-for-adversarial-robustness", - "name": "Metric Learning for Adversarial Robustness", - "authors": "Chengzhi Mao, Ziyuan Zhong, Junfeng Yang, Carl Vondrick, Baishakhi Ray", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "NeurIPS 2019", - "architecture": "WideResNet-34-10", - "eps": "8/255", - "clean_acc": "86.21", - "reported": "50.03", - "autoattack_acc": "47.41" -} diff --git a/robustbench/model_info/cifar10/Linf/Moosavi-Dezfooli2019Robustness.json b/robustbench/model_info/cifar10/Linf/Moosavi-Dezfooli2019Robustness.json deleted file mode 100644 index 04a5c286..00000000 --- a/robustbench/model_info/cifar10/Linf/Moosavi-Dezfooli2019Robustness.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "http://openaccess.thecvf.com/content_CVPR_2019/html/Moosavi-Dezfooli_Robustness_via_Curvature_Regularization_and_Vice_Versa_CVPR_2019_paper", - "name": "Robustness via Curvature Regularization, and Vice Versa", - "authors": "Seyed-Mohsen Moosavi-Dezfooli, Alhussein Fawzi, Jonathan Uesato, Pascal Frossard", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "CVPR 2019", - "architecture": "ResNet-18", - "eps": "8/255", - "clean_acc": "83.11", - "reported": "41.4", - "autoattack_acc": "38.50" -} diff --git a/robustbench/model_info/cifar10/Linf/Mustafa2019Adversarial.json b/robustbench/model_info/cifar10/Linf/Mustafa2019Adversarial.json deleted file mode 100644 index afd850c7..00000000 --- a/robustbench/model_info/cifar10/Linf/Mustafa2019Adversarial.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "https://arxiv.org/abs/1904.00887", - "name": "Adversarial Defense by Restricting the Hidden Space of Deep Neural Networks", - "authors": "Aamir Mustafa, Salman Khan, Munawar Hayat, Roland Goecke, Jianbing Shen, Ling Shao", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ICCV 2019", - "architecture": "ResNet-110", - "eps": "8/255", - "clean_acc": "89.16", - "reported": "32.32", - "autoattack_acc": "0.28" -} diff --git a/robustbench/model_info/cifar10/Linf/Pang2020Bag.json b/robustbench/model_info/cifar10/Linf/Pang2020Bag.json deleted file mode 100644 index 39164729..00000000 --- a/robustbench/model_info/cifar10/Linf/Pang2020Bag.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2010.00467", - "name": "Bag of Tricks for Adversarial Training", - "authors": "Tianyu Pang and Xiao Yang and Yinpeng Dong and Hang Su and Jun Zhu", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ICLR 2021", - "architecture": "WideResNet-34-20", - "eps": "8/255", - "clean_acc": "86.43", - "reported": "54.39", - "footnote": "", - "autoattack_acc": "54.39" -} diff --git a/robustbench/model_info/cifar10/Linf/Pang2020Boosting.json b/robustbench/model_info/cifar10/Linf/Pang2020Boosting.json deleted file mode 100644 index b60df9ab..00000000 --- a/robustbench/model_info/cifar10/Linf/Pang2020Boosting.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2002.08619", - "name": "Boosting Adversarial Training with Hypersphere Embedding", - "authors": "Tianyu Pang, Xiao Yang, Yinpeng Dong, Kun Xu, Hang Su, Jun Zhu", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "NeurIPS 2020", - "architecture": "WideResNet-34-20", - "eps": "8/255", - "clean_acc": "85.14", - "reported": "53.74", - "autoattack_acc": "53.74", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Pang2020Rethinking.json b/robustbench/model_info/cifar10/Linf/Pang2020Rethinking.json deleted file mode 100644 index 54ac5ac8..00000000 --- a/robustbench/model_info/cifar10/Linf/Pang2020Rethinking.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "https://arxiv.org/abs/1905.10626", - "name": "Rethinking Softmax Cross-Entropy Loss for Adversarial Robustness", - "authors": "Tianyu Pang, Kun Xu, Yinpeng Dong, Chao Du, Ning Chen, Jun Zhu", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ICLR 2020", - "architecture": "ResNet-32", - "eps": "8/255", - "clean_acc": "80.89", - "reported": "55.00", - "autoattack_acc": "43.48" -} diff --git a/robustbench/model_info/cifar10/Linf/Pang2022Robustness_WRN28_10.json b/robustbench/model_info/cifar10/Linf/Pang2022Robustness_WRN28_10.json deleted file mode 100644 index 3cf28eaf..00000000 --- a/robustbench/model_info/cifar10/Linf/Pang2022Robustness_WRN28_10.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/pdf/2202.10103.pdf", - "name": " Robustness and Accuracy Could Be Reconcilable by (Proper) Definition", - "authors": "Tianyu Pang, Min Lin, Xiao Yang, Jun Zhu, Shuicheng Yan", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ICML 2022", - "architecture": "WideResNet-28-10", - "eps": "8/255", - "clean_acc": "88.61", - "reported": "61.04", - "autoattack_acc": "61.04", - "footnote": "It uses additional 1M synthetic images in training.", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Pang2022Robustness_WRN70_16.json b/robustbench/model_info/cifar10/Linf/Pang2022Robustness_WRN70_16.json deleted file mode 100644 index 56071f7e..00000000 --- a/robustbench/model_info/cifar10/Linf/Pang2022Robustness_WRN70_16.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/pdf/2202.10103.pdf", - "name": " Robustness and Accuracy Could Be Reconcilable by (Proper) Definition", - "authors": "Tianyu Pang, Min Lin, Xiao Yang, Jun Zhu, Shuicheng Yan", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ICML 2022", - "architecture": "WideResNet-70-16", - "eps": "8/255", - "clean_acc": "89.01", - "reported": "63.35", - "autoattack_acc": "63.35", - "footnote": "It uses additional 1M synthetic images in training.", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Qin2019Adversarial.json b/robustbench/model_info/cifar10/Linf/Qin2019Adversarial.json deleted file mode 100644 index 1d4a0c02..00000000 --- a/robustbench/model_info/cifar10/Linf/Qin2019Adversarial.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/1907.02610v2", - "name": "Adversarial Robustness through Local Linearization", - "authors": "Chongli Qin, James Martens, Sven Gowal, Dilip Krishnan, Krishnamurthy Dvijotham, Alhussein Fawzi, Soham De, Robert Stanforth, Pushmeet Kohli", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "NeurIPS 2019", - "architecture": "WideResNet-40-8", - "eps": "8/255", - "clean_acc": "86.28", - "reported": "52.81", - "autoattack_acc": "52.84", - "external": "52.84" -} diff --git a/robustbench/model_info/cifar10/Linf/Rade2021Helper_R18_ddpm.json b/robustbench/model_info/cifar10/Linf/Rade2021Helper_R18_ddpm.json deleted file mode 100644 index 5c3ae278..00000000 --- a/robustbench/model_info/cifar10/Linf/Rade2021Helper_R18_ddpm.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://openreview.net/forum?id=BuD2LmNaU3a", - "name": "Helper-based Adversarial Training: Reducing Excessive Margin to Achieve a Better Accuracy vs. Robustness Trade-off", - "authors": "Rahul Rade and Seyed-Mohsen Moosavi-Dezfooli", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "OpenReview, Jun 2021", - "architecture": "PreActResNet-18", - "eps": "8/255", - "clean_acc": "86.86", - "reported": "57.09", - "footnote": "It uses additional 1M synthetic images in training.", - "autoattack_acc": "57.09", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Rade2021Helper_R18_extra.json b/robustbench/model_info/cifar10/Linf/Rade2021Helper_R18_extra.json deleted file mode 100644 index 8fe79605..00000000 --- a/robustbench/model_info/cifar10/Linf/Rade2021Helper_R18_extra.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://openreview.net/forum?id=BuD2LmNaU3a", - "name": "Helper-based Adversarial Training: Reducing Excessive Margin to Achieve a Better Accuracy vs. Robustness Trade-off", - "authors": "Rahul Rade and Seyed-Mohsen Moosavi-Dezfooli", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "OpenReview, Jun 2021", - "architecture": "PreActResNet-18", - "eps": "8/255", - "clean_acc": "89.02", - "reported": "57.67", - "autoattack_acc": "57.67", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Rade2021Helper_ddpm.json b/robustbench/model_info/cifar10/Linf/Rade2021Helper_ddpm.json deleted file mode 100644 index 4ea707bc..00000000 --- a/robustbench/model_info/cifar10/Linf/Rade2021Helper_ddpm.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://openreview.net/forum?id=BuD2LmNaU3a", - "name": "Helper-based Adversarial Training: Reducing Excessive Margin to Achieve a Better Accuracy vs. Robustness Trade-off", - "authors": "Rahul Rade and Seyed-Mohsen Moosavi-Dezfooli", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "OpenReview, Jun 2021", - "architecture": "WideResNet-28-10", - "eps": "8/255", - "clean_acc": "88.16", - "reported": "60.97", - "footnote": "It uses additional 1M synthetic images in training.", - "autoattack_acc": "60.97", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Rade2021Helper_extra.json b/robustbench/model_info/cifar10/Linf/Rade2021Helper_extra.json deleted file mode 100644 index 4dc9901c..00000000 --- a/robustbench/model_info/cifar10/Linf/Rade2021Helper_extra.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://openreview.net/forum?id=BuD2LmNaU3a", - "name": "Helper-based Adversarial Training: Reducing Excessive Margin to Achieve a Better Accuracy vs. Robustness Trade-off", - "authors": "Rahul Rade and Seyed-Mohsen Moosavi-Dezfooli", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "OpenReview, Jun 2021", - "architecture": "WideResNet-34-10", - "eps": "8/255", - "clean_acc": "91.47", - "reported": "62.83", - "autoattack_acc": "62.83", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Rebuffi2021Fixing_106_16_cutmix_ddpm.json b/robustbench/model_info/cifar10/Linf/Rebuffi2021Fixing_106_16_cutmix_ddpm.json deleted file mode 100644 index f12ebc12..00000000 --- a/robustbench/model_info/cifar10/Linf/Rebuffi2021Fixing_106_16_cutmix_ddpm.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2103.01946", - "name": "Fixing Data Augmentation to Improve Adversarial Robustness", - "authors": "Sylvestre-Alvise Rebuffi, Sven Gowal, Dan A. Calian, Florian Stimberg, Olivia Wiles, Timothy Mann", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Mar 2021", - "architecture": "WideResNet-106-16", - "eps": "8/255", - "clean_acc": "88.50", - "reported": "64.58", - "autoattack_acc": "64.64", - "external": "64.58", - "footnote": "It uses additional 1M synthetic images in training. 64.58% robust accuracy is due to the original evaluation (AutoAttack + MultiTargeted)", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Rebuffi2021Fixing_28_10_cutmix_ddpm.json b/robustbench/model_info/cifar10/Linf/Rebuffi2021Fixing_28_10_cutmix_ddpm.json deleted file mode 100644 index d5ad7ba7..00000000 --- a/robustbench/model_info/cifar10/Linf/Rebuffi2021Fixing_28_10_cutmix_ddpm.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2103.01946", - "name": "Fixing Data Augmentation to Improve Adversarial Robustness", - "authors": "Sylvestre-Alvise Rebuffi, Sven Gowal, Dan A. Calian, Florian Stimberg, Olivia Wiles, Timothy Mann", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Mar 2021", - "architecture": "WideResNet-28-10", - "eps": "8/255", - "clean_acc": "87.33", - "reported": "60.73", - "autoattack_acc": "60.75", - "external": "60.73", - "footnote": "It uses additional 1M synthetic images in training. 60.73% robust accuracy is due to the original evaluation (AutoAttack + MultiTargeted)", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Rebuffi2021Fixing_70_16_cutmix_ddpm.json b/robustbench/model_info/cifar10/Linf/Rebuffi2021Fixing_70_16_cutmix_ddpm.json deleted file mode 100644 index 11490482..00000000 --- a/robustbench/model_info/cifar10/Linf/Rebuffi2021Fixing_70_16_cutmix_ddpm.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2103.01946", - "name": "Fixing Data Augmentation to Improve Adversarial Robustness", - "authors": "Sylvestre-Alvise Rebuffi, Sven Gowal, Dan A. Calian, Florian Stimberg, Olivia Wiles, Timothy Mann", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Mar 2021", - "architecture": "WideResNet-70-16", - "eps": "8/255", - "clean_acc": "88.54", - "reported": "64.20", - "autoattack_acc": "64.25", - "external": "64.20", - "footnote": "It uses additional 1M synthetic images in training. 64.20% robust accuracy is due to the original evaluation (AutoAttack + MultiTargeted)", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Rebuffi2021Fixing_70_16_cutmix_extra.json b/robustbench/model_info/cifar10/Linf/Rebuffi2021Fixing_70_16_cutmix_extra.json deleted file mode 100644 index 0d88a5be..00000000 --- a/robustbench/model_info/cifar10/Linf/Rebuffi2021Fixing_70_16_cutmix_extra.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2103.01946", - "name": "Fixing Data Augmentation to Improve Adversarial Robustness", - "authors": "Sylvestre-Alvise Rebuffi, Sven Gowal, Dan A. Calian, Florian Stimberg, Olivia Wiles, Timothy Mann", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Mar 2021", - "architecture": "WideResNet-70-16", - "eps": "8/255", - "clean_acc": "92.23", - "reported": "66.56", - "autoattack_acc": "66.58", - "external": "66.56", - "footnote": "66.56% robust accuracy is due to the original evaluation (AutoAttack + MultiTargeted)", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Rebuffi2021Fixing_R18_ddpm.json b/robustbench/model_info/cifar10/Linf/Rebuffi2021Fixing_R18_ddpm.json deleted file mode 100644 index d714526a..00000000 --- a/robustbench/model_info/cifar10/Linf/Rebuffi2021Fixing_R18_ddpm.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2103.01946", - "name": "Fixing Data Augmentation to Improve Adversarial Robustness", - "authors": "Sylvestre-Alvise Rebuffi, Sven Gowal, Dan A. Calian, Florian Stimberg, Olivia Wiles, Timothy Mann", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Mar 2021", - "architecture": "PreActResNet-18", - "eps": "8/255", - "clean_acc": "83.53", - "reported": "56.66", - "footnote": "It uses additional 1M synthetic images in training.", - "autoattack_acc": "56.66", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Rice2020Overfitting.json b/robustbench/model_info/cifar10/Linf/Rice2020Overfitting.json deleted file mode 100644 index 1a02239d..00000000 --- a/robustbench/model_info/cifar10/Linf/Rice2020Overfitting.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2002.11569", - "name": "Overfitting in adversarially robust deep learning", - "authors": "Leslie Rice, Eric Wong, J. Zico Kolter", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ICML 2020", - "architecture": "WideResNet-34-20", - "eps": "8/255", - "clean_acc": "85.34", - "reported": "58", - "autoattack_acc": "53.42", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Sehwag2020Hydra.json b/robustbench/model_info/cifar10/Linf/Sehwag2020Hydra.json deleted file mode 100644 index 99802b04..00000000 --- a/robustbench/model_info/cifar10/Linf/Sehwag2020Hydra.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2002.10509", - "name": "HYDRA: Pruning Adversarially Robust Neural Networks", - "authors": "Vikash Sehwag, Shiqi Wang, Prateek Mittal, Suman Jana", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "NeurIPS 2020", - "architecture": "WideResNet-28-10", - "eps": "8/255", - "clean_acc": "88.98", - "reported": "62.24", - "footnote": "Compressed model", - "autoattack_acc": "57.14", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Sehwag2021Proxy.json b/robustbench/model_info/cifar10/Linf/Sehwag2021Proxy.json deleted file mode 100644 index 3a440b58..00000000 --- a/robustbench/model_info/cifar10/Linf/Sehwag2021Proxy.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2104.09425", - "name": "Robust Learning Meets Generative Models: Can Proxy Distributions Improve Adversarial Robustness?", - "authors": "Vikash Sehwag, Saeed Mahloujifar, Tinashe Handina, Sihui Dai, Chong Xiang, Mung Chiang, Prateek Mittal", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ICLR 2022", - "architecture": "WideResNet-34-10", - "eps": "8/255", - "clean_acc": "86.68", - "reported": "60.30", - "autoattack_acc": "60.27", - "footnote": "It uses additional 10M synthetic images in training.", - "unreliable": false -} \ No newline at end of file diff --git a/robustbench/model_info/cifar10/Linf/Sehwag2021Proxy_R18.json b/robustbench/model_info/cifar10/Linf/Sehwag2021Proxy_R18.json deleted file mode 100644 index da0a15de..00000000 --- a/robustbench/model_info/cifar10/Linf/Sehwag2021Proxy_R18.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2104.09425", - "name": "Robust Learning Meets Generative Models: Can Proxy Distributions Improve Adversarial Robustness?", - "authors": "Vikash Sehwag, Saeed Mahloujifar, Tinashe Handina, Sihui Dai, Chong Xiang, Mung Chiang, Prateek Mittal", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ICLR 2022", - "architecture": "ResNet-18", - "eps": "8/255", - "clean_acc": "84.59", - "reported": "55.54", - "autoattack_acc": "55.54", - "footnote": "It uses additional 10M synthetic images in training.", - "unreliable": false -} \ No newline at end of file diff --git a/robustbench/model_info/cifar10/Linf/Sehwag2021Proxy_ResNest152.json b/robustbench/model_info/cifar10/Linf/Sehwag2021Proxy_ResNest152.json deleted file mode 100644 index a0e3cff1..00000000 --- a/robustbench/model_info/cifar10/Linf/Sehwag2021Proxy_ResNest152.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2104.09425", - "name": "Robust Learning Meets Generative Models: Can Proxy Distributions Improve Adversarial Robustness?", - "authors": "Vikash Sehwag, Saeed Mahloujifar, Tinashe Handina, Sihui Dai, Chong Xiang, Mung Chiang, Prateek Mittal", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ICLR 2022", - "architecture": "ResNest152", - "eps": "8/255", - "clean_acc": "87.30", - "reported": "62.79", - "autoattack_acc": "62.79", - "footnote": "It uses additional 10M synthetic images in training.", - "unreliable": false -} \ No newline at end of file diff --git a/robustbench/model_info/cifar10/Linf/Shafahi2019Adversarial.json b/robustbench/model_info/cifar10/Linf/Shafahi2019Adversarial.json deleted file mode 100644 index 15becba5..00000000 --- a/robustbench/model_info/cifar10/Linf/Shafahi2019Adversarial.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "https://arxiv.org/abs/1904.12843", - "name": "Adversarial Training for Free!", - "authors": "Ali Shafahi, Mahyar Najibi, Amin Ghiasi, Zheng Xu, John Dickerson, Christoph Studer, Larry S. Davis, Gavin Taylor, Tom Goldstein", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "NeurIPS 2019", - "architecture": "WideResNet-34-10", - "eps": "8/255", - "clean_acc": "86.11", - "reported": "46.19", - "autoattack_acc": "41.47" -} diff --git a/robustbench/model_info/cifar10/Linf/Sitawarin2020Improving.json b/robustbench/model_info/cifar10/Linf/Sitawarin2020Improving.json deleted file mode 100644 index bfcf8711..00000000 --- a/robustbench/model_info/cifar10/Linf/Sitawarin2020Improving.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2003.09347", - "name": "Improving Adversarial Robustness Through Progressive Hardening", - "authors": "Chawin Sitawarin and Supriyo Chakraborty and David Wagner", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Mar 2020", - "architecture": "WideResNet-34-10", - "eps": "8/255", - "clean_acc": "86.84", - "reported": "50.72", - "footnote": "", - "autoattack_acc": "50.72" -} diff --git a/robustbench/model_info/cifar10/Linf/Sridhar2021Robust.json b/robustbench/model_info/cifar10/Linf/Sridhar2021Robust.json deleted file mode 100644 index e6675737..00000000 --- a/robustbench/model_info/cifar10/Linf/Sridhar2021Robust.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2106.02078", - "name": "Improving Neural Network Robustness via Persistency of Excitation", - "authors": "Kaustubh Sridhar and Oleg Sokolsky and Insup Lee and James Weimer", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ACC 2022", - "architecture": "WideResNet-28-10", - "eps": "8/255", - "clean_acc": "89.46", - "reported": "59.66", - "autoattack_acc": "59.66", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Sridhar2021Robust_34_15.json b/robustbench/model_info/cifar10/Linf/Sridhar2021Robust_34_15.json deleted file mode 100644 index d7b5dffa..00000000 --- a/robustbench/model_info/cifar10/Linf/Sridhar2021Robust_34_15.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2106.02078", - "name": "Improving Neural Network Robustness via Persistency of Excitation", - "authors": "Kaustubh Sridhar and Oleg Sokolsky and Insup Lee and James Weimer", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ACC 2022", - "architecture": "WideResNet-34-15", - "eps": "8/255", - "clean_acc": "86.53", - "reported": "60.41", - "autoattack_acc": "60.41", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Standard.json b/robustbench/model_info/cifar10/Linf/Standard.json deleted file mode 100644 index c28ef588..00000000 --- a/robustbench/model_info/cifar10/Linf/Standard.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "https://github.com/RobustBench/robustbench/", - "name": "Standardly trained model", - "authors": "", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "N/A", - "architecture": "WideResNet-28-10", - "eps": "8/255", - "clean_acc": "94.78", - "reported": "0.0", - "autoattack_acc": "0.0" -} diff --git a/robustbench/model_info/cifar10/Linf/Wang2020Improving.json b/robustbench/model_info/cifar10/Linf/Wang2020Improving.json deleted file mode 100644 index d348145a..00000000 --- a/robustbench/model_info/cifar10/Linf/Wang2020Improving.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://openreview.net/forum?id=rklOg6EFwS", - "name": "Improving Adversarial Robustness Requires Revisiting Misclassified Examples", - "authors": "Yisen Wang, Difan Zou, Jinfeng Yi, James Bailey, Xingjun Ma, Quanquan Gu", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ICLR 2020", - "architecture": "WideResNet-28-10", - "eps": "8/255", - "clean_acc": "87.50", - "reported": "65.04", - "autoattack_acc": "56.29", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/WangZhang2019Bilateral.json b/robustbench/model_info/cifar10/Linf/WangZhang2019Bilateral.json deleted file mode 100644 index 2a8c4ef9..00000000 --- a/robustbench/model_info/cifar10/Linf/WangZhang2019Bilateral.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "http://openaccess.thecvf.com/content_ICCV_2019/html/Wang_Bilateral_Adversarial_Training_Towards_Fast_Training_of_More_Robust_Models_ICCV_2019_paper.html", - "name": "Bilateral Adversarial Training: Towards Fast Training of More Robust Models Against Adversarial Attacks", - "authors": "Jianyu Wang, Haichao Zhang", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ICCV 2019", - "architecture": "WideResNet-28-10", - "eps": "8/255", - "clean_acc": "92.80", - "reported": "58.6", - "autoattack_acc": "29.35" -} diff --git a/robustbench/model_info/cifar10/Linf/Wong2020Fast.json b/robustbench/model_info/cifar10/Linf/Wong2020Fast.json deleted file mode 100644 index bef8f04a..00000000 --- a/robustbench/model_info/cifar10/Linf/Wong2020Fast.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2001.03994", - "name": "Fast is better than free: Revisiting adversarial training", - "authors": "Eric Wong, Leslie Rice, J. Zico Kolter", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ICLR 2020", - "architecture": "PreActResNet-18", - "eps": "8/255", - "clean_acc": "83.34", - "reported": "46.06", - "footnote": "Focuses on fast adversarial training.", - "autoattack_acc": "43.21" -} diff --git a/robustbench/model_info/cifar10/Linf/Wu2020Adversarial.json b/robustbench/model_info/cifar10/Linf/Wu2020Adversarial.json deleted file mode 100644 index 50a481b0..00000000 --- a/robustbench/model_info/cifar10/Linf/Wu2020Adversarial.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2004.05884", - "name": "Adversarial Weight Perturbation Helps Robust Generalization", - "authors": "Dongxian Wu, Shu-tao Xia, Yisen Wang", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "NeurIPS 2020", - "architecture": "WideResNet-34-10", - "eps": "8/255", - "clean_acc": "85.36", - "reported": "56.17", - "autoattack_acc": "56.17", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Wu2020Adversarial_extra.json b/robustbench/model_info/cifar10/Linf/Wu2020Adversarial_extra.json deleted file mode 100644 index 4686e3f9..00000000 --- a/robustbench/model_info/cifar10/Linf/Wu2020Adversarial_extra.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2004.05884", - "name": "Adversarial Weight Perturbation Helps Robust Generalization", - "authors": "Dongxian Wu, Shu-tao Xia, Yisen Wang", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "NeurIPS 2020", - "architecture": "WideResNet-28-10", - "eps": "8/255", - "clean_acc": "88.25", - "reported": "60.04", - "autoattack_acc": "60.04", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Wu2020Do.json b/robustbench/model_info/cifar10/Linf/Wu2020Do.json deleted file mode 100644 index 36796fc0..00000000 --- a/robustbench/model_info/cifar10/Linf/Wu2020Do.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2010.01279", - "name": "Do Wider Neural Networks Really Help Adversarial Robustness?", - "authors": "Boxi Wu and Jinghui Chen and Deng Cai and Xiaofei He and Quanquan Gu", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Oct 2020", - "architecture": "WideResNet-34-15", - "eps": "8/255", - "clean_acc": "87.67", - "reported": "60.65", - "footnote": "", - "autoattack_acc": "60.65" -} diff --git a/robustbench/model_info/cifar10/Linf/Xiao2020Enhancing.json b/robustbench/model_info/cifar10/Linf/Xiao2020Enhancing.json deleted file mode 100644 index d5500bb2..00000000 --- a/robustbench/model_info/cifar10/Linf/Xiao2020Enhancing.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "link": "https://arxiv.org/abs/1905.10510", - "name": "Enhancing Adversarial Defense by k-Winners-Take-All", - "authors": "Chang Xiao, Peilin Zhong, Changxi Zheng", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ICLR 2020", - "architecture": "DenseNet-121", - "eps": "0.031", - "clean_acc": "79.28", - "reported": "52.4", - "footnote": "Uses \\(\\ell_{\\infty} \\) = 0.031 \u2248 7.9/255 instead of 8/255.
7.40% robust accuracy is due to 1 restart of APGD-CE and 30 restarts of Square Attack
Note: this adaptive evaluation (Section 5) reports 0.16% robust accuracy on a different model (adversarially trained ResNet-18).", - "autoattack_acc": "18.50", - "external": "7.40", - "unreliable": true -} diff --git a/robustbench/model_info/cifar10/Linf/Zhang2019Theoretically.json b/robustbench/model_info/cifar10/Linf/Zhang2019Theoretically.json deleted file mode 100644 index 37bb6fb6..00000000 --- a/robustbench/model_info/cifar10/Linf/Zhang2019Theoretically.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/1901.08573", - "name": "Theoretically Principled Trade-off between Robustness and Accuracy", - "authors": "Hongyang Zhang, Yaodong Yu, Jiantao Jiao, Eric P. Xing, Laurent El Ghaoui, Michael I. Jordan", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ICML 2019", - "architecture": "WideResNet-34-10", - "eps": "0.031", - "clean_acc": "84.92", - "reported": "56.43", - "footnote": "Uses \\(\\ell_{\\infty} \\) = 0.031 \u2248 7.9/255 instead of 8/255.", - "autoattack_acc": "53.08" -} diff --git a/robustbench/model_info/cifar10/Linf/Zhang2019You.json b/robustbench/model_info/cifar10/Linf/Zhang2019You.json deleted file mode 100644 index 488203c7..00000000 --- a/robustbench/model_info/cifar10/Linf/Zhang2019You.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/1905.00877", - "name": "You Only Propagate Once: Accelerating Adversarial Training via Maximal Principle", - "authors": "Dinghuai Zhang, Tianyuan Zhang, Yiping Lu, Zhanxing Zhu, Bin Dong", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "NeurIPS 2019", - "architecture": "WideResNet-34-10", - "eps": "8/255", - "clean_acc": "87.20", - "reported": "47.98", - "footnote": "Focuses on fast adversarial training.", - "autoattack_acc": "44.83" -} diff --git a/robustbench/model_info/cifar10/Linf/Zhang2020Attacks.json b/robustbench/model_info/cifar10/Linf/Zhang2020Attacks.json deleted file mode 100644 index bdfab976..00000000 --- a/robustbench/model_info/cifar10/Linf/Zhang2020Attacks.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2002.11242", - "name": "Attacks Which Do Not Kill Training Make Adversarial Learning Stronger", - "authors": "Jingfeng Zhang, Xilie Xu, Bo Han, Gang Niu, Lizhen Cui, Masashi Sugiyama, Mohan Kankanhalli", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ICML 2020", - "architecture": "WideResNet-34-10", - "eps": "8/255", - "clean_acc": "84.52", - "reported": "54.36", - "autoattack_acc": "53.51", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Zhang2020Geometry.json b/robustbench/model_info/cifar10/Linf/Zhang2020Geometry.json deleted file mode 100644 index e27fe426..00000000 --- a/robustbench/model_info/cifar10/Linf/Zhang2020Geometry.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2010.01736", - "name": "Geometry-aware Instance-reweighted Adversarial Training", - "authors": "Jingfeng Zhang, Jianing Zhu, Gang Niu, Bo Han, Masashi Sugiyama, Mohan Kankanhalli", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ICLR 2021", - "architecture": "WideResNet-28-10", - "eps": "8/255", - "clean_acc": "89.36", - "reported": "59.64", - "footnote": "Uses \\(\\ell_{\\infty} \\) = 0.031 \u2248 7.9/255 instead of 8/255.", - "autoattack_acc": "59.64", - "unreliable": false -} diff --git a/robustbench/model_info/cifar10/Linf/Zhang2020Towards.json b/robustbench/model_info/cifar10/Linf/Zhang2020Towards.json deleted file mode 100644 index f2992f7b..00000000 --- a/robustbench/model_info/cifar10/Linf/Zhang2020Towards.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/1906.06316", - "name": "Towards Stable and Efficient Training of Verifiably Robust Neural Networks", - "authors": "Huan Zhang, Hongge Chen, Chaowei Xiao, Sven Gowal, Robert Stanforth, Bo Li, Duane Boning, Cho-Jui Hsieh", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ICLR 2020", - "architecture": "5-layer-CNN", - "eps": "8/255", - "clean_acc": "44.73", - "reported": "34.29", - "autoattack_acc": "32.64", - "footnote": "Verifiably robust model with 32.24% provable robust accuracy" -} diff --git a/robustbench/model_info/cifar10/Linf/ZhangWang2019Defense.json b/robustbench/model_info/cifar10/Linf/ZhangWang2019Defense.json deleted file mode 100644 index a144a271..00000000 --- a/robustbench/model_info/cifar10/Linf/ZhangWang2019Defense.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "http://papers.nips.cc/paper/8459-defense-against-adversarial-attacks-using-feature-scattering-based-adversarial-training", - "name": "Defense Against Adversarial Attacks Using Feature Scattering-based Adversarial Training", - "authors": "Haichao Zhang, Jianyu Wang", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "NeurIPS 2019", - "architecture": "WideResNet-28-10", - "eps": "8/255", - "clean_acc": "89.98", - "reported": "60.6", - "autoattack_acc": "36.64" -} diff --git a/robustbench/model_info/cifar10/Linf/ZhangXu2020Adversarial.json b/robustbench/model_info/cifar10/Linf/ZhangXu2020Adversarial.json deleted file mode 100644 index d3725967..00000000 --- a/robustbench/model_info/cifar10/Linf/ZhangXu2020Adversarial.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "https://openreview.net/forum?id=Syejj0NYvr¬eId=Syejj0NYvr", - "name": "Adversarial Interpolation Training: A Simple Approach for Improving Model Robustness", - "authors": "Haichao Zhang, Wei Xu", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "OpenReview, Sep 2019", - "architecture": "WideResNet-28-10", - "eps": "8/255", - "clean_acc": "90.25", - "reported": "68.7", - "autoattack_acc": "36.45" -} diff --git a/robustbench/model_info/cifar10/corruptions/Addepalli2021Towards_WRN34.json b/robustbench/model_info/cifar10/corruptions/Addepalli2021Towards_WRN34.json deleted file mode 100644 index 76777796..00000000 --- a/robustbench/model_info/cifar10/corruptions/Addepalli2021Towards_WRN34.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "https://openreview.net/forum?id=SHB_znlW5G7", - "name": "Towards Achieving Adversarial Robustness Beyond Perceptual Limits", - "authors": "Sravanti Addepalli, Samyak Jain, Gaurang Sriramanan, Shivangi Khare, Venkatesh Babu Radhakrishnan", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Apr 2021", - "architecture": "WideResNet-34-10", - "eps": null, - "clean_acc": "85.32", - "reported": "76.78", - "corruptions_acc": "76.78" - } \ No newline at end of file diff --git a/robustbench/model_info/cifar10/corruptions/Addepalli2022Efficient_WRN_34_10.json b/robustbench/model_info/cifar10/corruptions/Addepalli2022Efficient_WRN_34_10.json deleted file mode 100644 index 174d2e7f..00000000 --- a/robustbench/model_info/cifar10/corruptions/Addepalli2022Efficient_WRN_34_10.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "https://artofrobust.github.io/short_paper/31.pdf", - "name": "Efficient and Effective Augmentation Strategy for Adversarial Training", - "authors": "Sravanti Addepalli, Samyak Jain, Venkatesh Babu Radhakrishnan", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "CVPRW 2022", - "architecture": "WideResNet-34-10", - "eps": null, - "clean_acc": "88.71", - "reported": "80.12", - "corruptions_acc": "80.12" -} \ No newline at end of file diff --git a/robustbench/model_info/cifar10/corruptions/Calian2021Defending.json b/robustbench/model_info/cifar10/corruptions/Calian2021Defending.json deleted file mode 100644 index 0396eeb4..00000000 --- a/robustbench/model_info/cifar10/corruptions/Calian2021Defending.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2104.01086", - "name": "Defending Against Image Corruptions Through Adversarial Augmentations", - "authors": "Dan A. Calian, Florian Stimberg, Olivia Wiles, Sylvestre-Alvise Rebuffi, Andras Gyorgy, Timothy Mann, Sven Gowal", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Apr 2021", - "architecture": "ResNet-50", - "eps": null, - "clean_acc": "94.93", - "reported": "92.17", - "corruptions_acc": "92.17", - "footnote": "Uses extra data indirectly via a super resolution and autoencoder networks that were pre-trained on other datasets." -} \ No newline at end of file diff --git a/robustbench/model_info/cifar10/corruptions/Diffenderfer2021Winning_Binary.json b/robustbench/model_info/cifar10/corruptions/Diffenderfer2021Winning_Binary.json deleted file mode 100644 index 124fac62..00000000 --- a/robustbench/model_info/cifar10/corruptions/Diffenderfer2021Winning_Binary.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2106.09129", - "name": "A Winning Hand: Compressing Deep Networks Can Improve Out-Of-Distribution Robustness", - "authors": "James Diffenderfer, Brian R. Bartoldson, Shreya Chaganti, Jize Zhang, Bhavya Kailkhura", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "NeurIPS 2021", - "architecture": "WideResNet-18-2", - "eps": null, - "clean_acc": "94.87", - "reported": "88.32", - "corruptions_acc": "88.32", - "footnote": "Binary weight network trained with AugMix and pruned to 95% sparsity" -} \ No newline at end of file diff --git a/robustbench/model_info/cifar10/corruptions/Diffenderfer2021Winning_Binary_CARD_Deck.json b/robustbench/model_info/cifar10/corruptions/Diffenderfer2021Winning_Binary_CARD_Deck.json deleted file mode 100644 index dfb61ce6..00000000 --- a/robustbench/model_info/cifar10/corruptions/Diffenderfer2021Winning_Binary_CARD_Deck.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2106.09129", - "name": "A Winning Hand: Compressing Deep Networks Can Improve Out-Of-Distribution Robustness", - "authors": "James Diffenderfer, Brian R. Bartoldson, Shreya Chaganti, Jize Zhang, Bhavya Kailkhura", - "additional_data": false, - "number_forward_passes": 6, - "dataset": "cifar10", - "venue": "NeurIPS 2021", - "architecture": "WideResNet-18-2", - "eps": null, - "clean_acc": "95.09", - "reported": "90.15", - "corruptions_acc": "90.15", - "footnote": "Ensemble of binary weight networks each of which are pruned to 95% sparsity" -} \ No newline at end of file diff --git a/robustbench/model_info/cifar10/corruptions/Diffenderfer2021Winning_LRR.json b/robustbench/model_info/cifar10/corruptions/Diffenderfer2021Winning_LRR.json deleted file mode 100644 index fce9e50a..00000000 --- a/robustbench/model_info/cifar10/corruptions/Diffenderfer2021Winning_LRR.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2106.09129", - "name": "A Winning Hand: Compressing Deep Networks Can Improve Out-Of-Distribution Robustness", - "authors": "James Diffenderfer, Brian R. Bartoldson, Shreya Chaganti, Jize Zhang, Bhavya Kailkhura", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "NeurIPS 2021", - "architecture": "WideResNet-18-2", - "eps": null, - "clean_acc": "96.66", - "reported": "90.94", - "corruptions_acc": "90.94", - "footnote": "Trained with AugMix and pruned to 95% sparsity" -} \ No newline at end of file diff --git a/robustbench/model_info/cifar10/corruptions/Diffenderfer2021Winning_LRR_CARD_Deck.json b/robustbench/model_info/cifar10/corruptions/Diffenderfer2021Winning_LRR_CARD_Deck.json deleted file mode 100644 index a68986f7..00000000 --- a/robustbench/model_info/cifar10/corruptions/Diffenderfer2021Winning_LRR_CARD_Deck.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2106.09129", - "name": "A Winning Hand: Compressing Deep Networks Can Improve Out-Of-Distribution Robustness", - "authors": "James Diffenderfer, Brian R. Bartoldson, Shreya Chaganti, Jize Zhang, Bhavya Kailkhura", - "additional_data": false, - "number_forward_passes": 6, - "dataset": "cifar10", - "venue": "NeurIPS 2021", - "architecture": "WideResNet-18-2", - "eps": null, - "clean_acc": "96.56", - "reported": "null", - "corruptions_acc": "92.78", - "corruptions_acc": "92.78", - "footnote": "Ensemble of networks each of which are pruned to 95% sparsity" -} \ No newline at end of file diff --git a/robustbench/model_info/cifar10/corruptions/Erichson2022NoisyMix.json b/robustbench/model_info/cifar10/corruptions/Erichson2022NoisyMix.json deleted file mode 100644 index 474bdcf8..00000000 --- a/robustbench/model_info/cifar10/corruptions/Erichson2022NoisyMix.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "https://arxiv.org/pdf/2202.01263.pdf", - "name": "NoisyMix: Boosting Robustness by Combining Data Augmentations, Stability Training, and Noise Injections", - "authors": "N. Benjamin Erichson, Soon Hoe Lim, Francisco Utrera, Winnie Xu, Ziang Cao, and Michael W. Mahoney", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Feb 2022", - "architecture": "WideResNet-28-4", - "eps": null, - "clean_acc": "96.73", - "reported": "92.78", - "corruptions_acc": "92.78" -} \ No newline at end of file diff --git a/robustbench/model_info/cifar10/corruptions/Gowal2020Uncovering_70_16_L2.json b/robustbench/model_info/cifar10/corruptions/Gowal2020Uncovering_70_16_L2.json deleted file mode 100644 index 907ccb5e..00000000 --- a/robustbench/model_info/cifar10/corruptions/Gowal2020Uncovering_70_16_L2.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2010.03593", - "name": "Uncovering the Limits of Adversarial Training against Norm-Bounded Adversarial Examples", - "authors": "Sven Gowal, Chongli Qin, Jonathan Uesato, Timothy Mann, Pushmeet Kohli", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Oct 2020", - "architecture": "WideResNet-70-16", - "eps": "8/255", - "clean_acc": "90.90", - "reported": "", - "corruptions_acc": "84.90", - "footnote": "Trained for \\(\\ell_2 \\) robustness with \\(\\varepsilon = 0.5\\)." -} diff --git a/robustbench/model_info/cifar10/corruptions/Gowal2020Uncovering_70_16_Linf.json b/robustbench/model_info/cifar10/corruptions/Gowal2020Uncovering_70_16_Linf.json deleted file mode 100644 index 00b9784c..00000000 --- a/robustbench/model_info/cifar10/corruptions/Gowal2020Uncovering_70_16_Linf.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2010.03593", - "name": "Uncovering the Limits of Adversarial Training against Norm-Bounded Adversarial Examples", - "authors": "Sven Gowal, Chongli Qin, Jonathan Uesato, Timothy Mann, Pushmeet Kohli", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Oct 2020", - "architecture": "WideResNet-70-16", - "eps": "8/255", - "clean_acc": "85.29", - "reported": "", - "corruptions_acc": "76.37", - "footnote": "Trained for \\(\\ell_{\\infty} \\) robustness with \\(\\varepsilon = 8/255\\)." -} diff --git a/robustbench/model_info/cifar10/corruptions/Gowal2020Uncovering_70_16_extra_L2.json b/robustbench/model_info/cifar10/corruptions/Gowal2020Uncovering_70_16_extra_L2.json deleted file mode 100644 index 08970ebe..00000000 --- a/robustbench/model_info/cifar10/corruptions/Gowal2020Uncovering_70_16_extra_L2.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2010.03593", - "name": "Uncovering the Limits of Adversarial Training against Norm-Bounded Adversarial Examples", - "authors": "Sven Gowal, Chongli Qin, Jonathan Uesato, Timothy Mann, Pushmeet Kohli", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Oct 2020", - "architecture": "WideResNet-70-16", - "eps": "8/255", - "clean_acc": "94.74", - "reported": "", - "corruptions_acc": "87.68", - "footnote": "Trained for \\(\\ell_2 \\) robustness with \\(\\varepsilon = 0.5\\)." -} diff --git a/robustbench/model_info/cifar10/corruptions/Gowal2020Uncovering_70_16_extra_Linf.json b/robustbench/model_info/cifar10/corruptions/Gowal2020Uncovering_70_16_extra_Linf.json deleted file mode 100644 index efdc6071..00000000 --- a/robustbench/model_info/cifar10/corruptions/Gowal2020Uncovering_70_16_extra_Linf.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2010.03593", - "name": "Uncovering the Limits of Adversarial Training against Norm-Bounded Adversarial Examples", - "authors": "Sven Gowal, Chongli Qin, Jonathan Uesato, Timothy Mann, Pushmeet Kohli", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Oct 2020", - "architecture": "WideResNet-70-16", - "eps": "8/255", - "clean_acc": "91.10", - "reported": "", - "corruptions_acc": "81.84", - "footnote": "Trained for \\(\\ell_{\\infty} \\) robustness with \\(\\varepsilon = 8/255\\)." -} diff --git a/robustbench/model_info/cifar10/corruptions/Hendrycks2020AugMix_ResNeXt.json b/robustbench/model_info/cifar10/corruptions/Hendrycks2020AugMix_ResNeXt.json deleted file mode 100644 index da66460e..00000000 --- a/robustbench/model_info/cifar10/corruptions/Hendrycks2020AugMix_ResNeXt.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "https://arxiv.org/abs/1912.02781", - "name": "AugMix: A Simple Data Processing Method to Improve Robustness and Uncertainty", - "authors": "Dan Hendrycks, Norman Mu, Ekin D. Cubuk, Barret Zoph, Justin Gilmer, Balaji Lakshminarayanan", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ICLR 2020", - "architecture": "ResNeXt29_32x4d", - "eps": null, - "clean_acc": "95.83", - "reported": "89.1", - "corruptions_acc": "89.09" -} diff --git a/robustbench/model_info/cifar10/corruptions/Hendrycks2020AugMix_WRN.json b/robustbench/model_info/cifar10/corruptions/Hendrycks2020AugMix_WRN.json deleted file mode 100644 index af249968..00000000 --- a/robustbench/model_info/cifar10/corruptions/Hendrycks2020AugMix_WRN.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "https://arxiv.org/abs/1912.02781", - "name": "AugMix: A Simple Data Processing Method to Improve Robustness and Uncertainty", - "authors": "Dan Hendrycks, Norman Mu, Ekin D. Cubuk, Barret Zoph, Justin Gilmer, Balaji Lakshminarayanan", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ICLR 2020", - "architecture": "WideResNet-40-2", - "eps": null, - "clean_acc": "95.08", - "reported": "88.8", - "corruptions_acc": "88.82" -} diff --git a/robustbench/model_info/cifar10/corruptions/Kireev2021Effectiveness_AugMixNoJSD.json b/robustbench/model_info/cifar10/corruptions/Kireev2021Effectiveness_AugMixNoJSD.json deleted file mode 100644 index b51b7939..00000000 --- a/robustbench/model_info/cifar10/corruptions/Kireev2021Effectiveness_AugMixNoJSD.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2103.02325", - "name": "On the effectiveness of adversarial training against common corruptions", - "authors": "Klim Kireev, Maksym Andriushchenko, Nicolas Flammarion", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Mar 2021", - "architecture": "PreActResNet-18", - "eps": null, - "clean_acc": "94.97", - "reported": "86.6", - "corruptions_acc": "86.60", - "footnote": "Training with AugMix without the JSD term." -} diff --git a/robustbench/model_info/cifar10/corruptions/Kireev2021Effectiveness_Gauss50percent.json b/robustbench/model_info/cifar10/corruptions/Kireev2021Effectiveness_Gauss50percent.json deleted file mode 100644 index cee64388..00000000 --- a/robustbench/model_info/cifar10/corruptions/Kireev2021Effectiveness_Gauss50percent.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2103.02325", - "name": "On the effectiveness of adversarial training against common corruptions", - "authors": "Klim Kireev, Maksym Andriushchenko, Nicolas Flammarion", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Mar 2021", - "architecture": "PreActResNet-18", - "eps": null, - "clean_acc": "93.24", - "reported": "85.0", - "corruptions_acc": "85.04", - "footnote": "Trained with 50% Gaussian noise per batch. Note: Gaussian noise is contained in CIFAR-10-C." -} diff --git a/robustbench/model_info/cifar10/corruptions/Kireev2021Effectiveness_RLAT.json b/robustbench/model_info/cifar10/corruptions/Kireev2021Effectiveness_RLAT.json deleted file mode 100644 index 9207f2c4..00000000 --- a/robustbench/model_info/cifar10/corruptions/Kireev2021Effectiveness_RLAT.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2103.02325", - "name": "On the effectiveness of adversarial training against common corruptions", - "authors": "Klim Kireev, Maksym Andriushchenko, Nicolas Flammarion", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Mar 2021", - "architecture": "PreActResNet-18", - "eps": null, - "clean_acc": "93.10", - "reported": "84.1", - "corruptions_acc": "84.10", - "footnote": "Trained with RLAT." -} diff --git a/robustbench/model_info/cifar10/corruptions/Kireev2021Effectiveness_RLATAugMix.json b/robustbench/model_info/cifar10/corruptions/Kireev2021Effectiveness_RLATAugMix.json deleted file mode 100644 index 2c60abff..00000000 --- a/robustbench/model_info/cifar10/corruptions/Kireev2021Effectiveness_RLATAugMix.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2103.02325", - "name": "On the effectiveness of adversarial training against common corruptions", - "authors": "Klim Kireev, Maksym Andriushchenko, Nicolas Flammarion", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Mar 2021", - "architecture": "ResNet-18", - "eps": null, - "clean_acc": "94.75", - "reported": "88.5", - "corruptions_acc": "89.60", - "footnote": "Trained with RLAT and AugMix." -} diff --git a/robustbench/model_info/cifar10/corruptions/Kireev2021Effectiveness_RLATAugMixNoJSD.json b/robustbench/model_info/cifar10/corruptions/Kireev2021Effectiveness_RLATAugMixNoJSD.json deleted file mode 100644 index 237a0d66..00000000 --- a/robustbench/model_info/cifar10/corruptions/Kireev2021Effectiveness_RLATAugMixNoJSD.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2103.02325", - "name": "On the effectiveness of adversarial training against common corruptions", - "authors": "Klim Kireev, Maksym Andriushchenko, Nicolas Flammarion", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Mar 2021", - "architecture": "PreActResNet-18", - "eps": null, - "clean_acc": "94.77", - "reported": "88.5", - "corruptions_acc": "88.53", - "footnote": "Trained with RLAT and AugMix without the JSD term." -} diff --git a/robustbench/model_info/cifar10/corruptions/Modas2021PRIMEResNet18.json b/robustbench/model_info/cifar10/corruptions/Modas2021PRIMEResNet18.json deleted file mode 100644 index 03ddb2be..00000000 --- a/robustbench/model_info/cifar10/corruptions/Modas2021PRIMEResNet18.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2112.13547", - "name": "PRIME: A Few Primitives Can Boost Robustness to Common Corruptions", - "authors": "Apostolos Modas, Rahul Rade, Guillermo Ortiz-Jiménez, Seyed-Mohsen Moosavi-Dezfooli, Pascal Frossard", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Dec 2021", - "architecture": "ResNet-18", - "eps": null, - "clean_acc": "93.06", - "reported": "89.05", - "corruptions_acc": "89.05" -} \ No newline at end of file diff --git a/robustbench/model_info/cifar10/corruptions/Rebuffi2021Fixing_70_16_cutmix_extra_L2.json b/robustbench/model_info/cifar10/corruptions/Rebuffi2021Fixing_70_16_cutmix_extra_L2.json deleted file mode 100644 index 44a1fb5e..00000000 --- a/robustbench/model_info/cifar10/corruptions/Rebuffi2021Fixing_70_16_cutmix_extra_L2.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2103.01946", - "name": "Fixing Data Augmentation to Improve Adversarial Robustness", - "authors": "Sylvestre-Alvise Rebuffi, Sven Gowal, Dan A. Calian, Florian Stimberg, Olivia Wiles, Timothy Mann", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Mar 2021", - "architecture": "WideResNet-70-16", - "eps": "8/255", - "clean_acc": "95.74", - "reported": "", - "corruptions_acc": "88.23", - "footnote": "Trained for \\(\\ell_2 \\) robustness with \\(\\varepsilon = 0.5\\)." -} diff --git a/robustbench/model_info/cifar10/corruptions/Rebuffi2021Fixing_70_16_cutmix_extra_Linf.json b/robustbench/model_info/cifar10/corruptions/Rebuffi2021Fixing_70_16_cutmix_extra_Linf.json deleted file mode 100644 index dee4ba24..00000000 --- a/robustbench/model_info/cifar10/corruptions/Rebuffi2021Fixing_70_16_cutmix_extra_Linf.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2103.01946", - "name": "Fixing Data Augmentation to Improve Adversarial Robustness", - "authors": "Sylvestre-Alvise Rebuffi, Sven Gowal, Dan A. Calian, Florian Stimberg, Olivia Wiles, Timothy Mann", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Mar 2021", - "architecture": "WideResNet-70-16", - "eps": "8/255", - "clean_acc": "92.23", - "reported": "", - "corruptions_acc": "82.82", - "footnote": "Trained for \\(\\ell_{\\infty} \\) robustness with \\(\\varepsilon = 8/255\\)." -} diff --git a/robustbench/model_info/cifar10/corruptions/Standard.json b/robustbench/model_info/cifar10/corruptions/Standard.json deleted file mode 100644 index 681958d9..00000000 --- a/robustbench/model_info/cifar10/corruptions/Standard.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "https://github.com/RobustBench/robustbench/", - "name": "Standardly trained model", - "authors": "", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "N/A", - "architecture": "WideResNet-28-10", - "eps": "0.5", - "clean_acc": "94.78", - "reported": "73.46", - "corruptions_acc": "73.46" -} diff --git a/robustbench/model_info/cifar10/corruptions/unaggregated_results.csv b/robustbench/model_info/cifar10/corruptions/unaggregated_results.csv deleted file mode 100644 index 672546a9..00000000 --- a/robustbench/model_info/cifar10/corruptions/unaggregated_results.csv +++ /dev/null @@ -1,25 +0,0 @@ -,shot_noise,shot_noise,shot_noise,shot_noise,shot_noise,motion_blur,motion_blur,motion_blur,motion_blur,motion_blur,snow,snow,snow,snow,snow,pixelate,pixelate,pixelate,pixelate,pixelate,gaussian_noise,gaussian_noise,gaussian_noise,gaussian_noise,gaussian_noise,defocus_blur,defocus_blur,defocus_blur,defocus_blur,defocus_blur,brightness,brightness,brightness,brightness,brightness,fog,fog,fog,fog,fog,zoom_blur,zoom_blur,zoom_blur,zoom_blur,zoom_blur,frost,frost,frost,frost,frost,glass_blur,glass_blur,glass_blur,glass_blur,glass_blur,impulse_noise,impulse_noise,impulse_noise,impulse_noise,impulse_noise,contrast,contrast,contrast,contrast,contrast,jpeg_compression,jpeg_compression,jpeg_compression,jpeg_compression,jpeg_compression,elastic_transform,elastic_transform,elastic_transform,elastic_transform,elastic_transform -,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5 -Calian2021Defending,0.9421,0.9383,0.9217,0.9137,0.8959,0.9443,0.9356,0.9232,0.9234,0.9081,0.9390000000000001,0.9279999999999999,0.9129999999999999,0.8917,0.9036,0.9443,0.9418,0.9386,0.9258,0.8682,0.9359999999999999,0.9276,0.9064,0.9001,0.8899,0.9490000000000001,0.9470000000000001,0.9439,0.9385,0.9253,0.9493,0.9493,0.9489,0.9485,0.9440000000000001,0.9498,0.9413,0.9302,0.9113,0.8344,0.9395,0.9395,0.9366,0.9344,0.9254,0.9447,0.9392,0.9309,0.9288,0.9209,0.9099,0.9097,0.9118,0.8655,0.8685,0.9318,0.9151,0.8998,0.8646,0.828,0.9497,0.9457,0.9432,0.9385,0.9143,0.9365,0.9281,0.9222,0.9166,0.9052,0.9286,0.9324,0.9265,0.9068,0.8781 -Gowal2020Uncovering_70_16_Linf,0.8478,0.8416,0.8211,0.8085,0.7803,0.8238,0.7982,0.7611,0.7636,0.7255,0.8454,0.8296,0.8086,0.7558,0.7211,0.8473,0.8389,0.8365,0.825,0.8066,0.8426,0.8325,0.8089,0.7933,0.7794,0.8471,0.8334,0.8161,0.7977,0.7483,0.8546,0.8524,0.8396,0.8163,0.7345,0.8306,0.7678,0.6732,0.5437,0.3283,0.8139,0.8119,0.8004,0.7888,0.7718,0.8289,0.7747,0.7157,0.7267,0.6976,0.8088,0.8061,0.8015,0.7536,0.7558,0.8222,0.7909,0.7647,0.7102,0.6647,0.8138,0.6183,0.4366,0.266,0.1895,0.8406,0.8325,0.8282,0.826,0.8216,0.8036,0.8058,0.8004,0.7843,0.778 -Gowal2020Uncovering_70_16_extra_Linf,0.9043,0.8952,0.8595,0.8368,0.7922,0.8866,0.8593,0.8206,0.8243,0.7788,0.9014,0.8906,0.8714,0.8423,0.8243,0.9042,0.8991,0.8944,0.8817,0.8613,0.8972,0.8752,0.8361,0.8099,0.7836,0.9064,0.8937,0.8759,0.8579,0.805,0.9142,0.9131,0.907,0.8949,0.8479,0.892,0.8212,0.7232,0.5942,0.3582,0.8747,0.8714,0.8605,0.8508,0.8326,0.8993,0.8732,0.8320000000000001,0.8327,0.8003,0.8529,0.8549,0.8529,0.7883,0.7965,0.8689,0.8299,0.794,0.725,0.6584,0.8709,0.6793,0.4836,0.2764,0.1812,0.8959999999999999,0.8871,0.8852,0.8804,0.8768,0.8672,0.8687,0.8607,0.8476,0.8311 -Gowal2020Uncovering_70_16_L2,0.919,0.917,0.889,0.8809999999999999,0.861,0.8959999999999999,0.878,0.851,0.847,0.818,0.916,0.9,0.88,0.851,0.858,0.915,0.909,0.906,0.897,0.877,0.913,0.902,0.875,0.868,0.8590000000000001,0.914,0.902,0.8859999999999999,0.871,0.835,0.921,0.915,0.913,0.91,0.889,0.902,0.865,0.785,0.652,0.386,0.8859999999999999,0.882,0.873,0.8690000000000001,0.843,0.917,0.907,0.884,0.8809999999999999,0.8490000000000001,0.87,0.871,0.8640000000000001,0.818,0.8290000000000001,0.913,0.889,0.868,0.828,0.754,0.897,0.7879999999999999,0.632,0.416,0.208,0.909,0.902,0.903,0.901,0.894,0.875,0.866,0.866,0.856,0.841 -Gowal2020Uncovering_70_16_extra_L2,0.9390000000000001,0.9329999999999999,0.898,0.887,0.851,0.9309999999999999,0.915,0.885,0.883,0.838,0.935,0.912,0.9109999999999999,0.8859999999999999,0.88,0.9420000000000001,0.9420000000000001,0.941,0.9309999999999999,0.91,0.9329999999999999,0.912,0.889,0.866,0.8420000000000001,0.941,0.935,0.9259999999999999,0.915,0.866,0.9470000000000001,0.946,0.941,0.941,0.919,0.94,0.9009999999999999,0.8320000000000001,0.711,0.442,0.93,0.922,0.912,0.9109999999999999,0.889,0.948,0.9309999999999999,0.902,0.9,0.8640000000000001,0.887,0.8959999999999999,0.904,0.8240000000000001,0.85,0.935,0.915,0.884,0.805,0.684,0.93,0.8370000000000001,0.722,0.486,0.19399999999999998,0.934,0.93,0.93,0.93,0.929,0.9279999999999999,0.919,0.9109999999999999,0.907,0.884 -Hendrycks2020AugMixWRN,0.9352,0.9119,0.8536,0.8289,0.7704,0.9448,0.9349,0.9143,0.9156,0.8943,0.9356,0.908,0.9014,0.8779,0.8585,0.9419,0.9216,0.9041,0.8057,0.5870000000000001,0.9135,0.8655,0.7955,0.7587,0.7122,0.9506,0.9487,0.9458,0.9337,0.9053,0.9495,0.9469,0.9447,0.9385,0.9238,0.951,0.947,0.9367,0.9173,0.8251,0.9406,0.9418,0.9349,0.9264,0.9074,0.9347,0.915,0.8859999999999999,0.8846,0.8474,0.8588,0.8582,0.8707,0.7652,0.7939,0.9378,0.9204,0.903,0.8363,0.738,0.9501,0.939,0.9282,0.9082,0.7906,0.9134,0.8908,0.8787,0.8687,0.8533,0.9306,0.935,0.9262,0.8938,0.8526 -Hendrycks2020AugMixResNeXt,0.9344,0.9091,0.8281,0.7928,0.7116,0.9505,0.9387,0.9238,0.9237,0.8977,0.9448,0.9126,0.9079,0.8822,0.8674,0.9492,0.9322,0.9151,0.8345,0.6499,0.909,0.8426,0.7577,0.7045,0.6518,0.9575,0.9546,0.9508,0.9438,0.9177,0.9561,0.9547,0.9502,0.9466,0.9273,0.9561,0.9503,0.9359,0.917,0.8170000000000001,0.9468,0.946,0.9398,0.9339,0.9147,0.9403,0.9218,0.8923,0.8901,0.8551,0.8442,0.8567,0.8712,0.7485,0.7832,0.9497,0.9371,0.9226,0.8711,0.7878,0.9553,0.9435,0.9322,0.9109,0.8059,0.9211,0.9009,0.8875,0.8763,0.8572,0.9372,0.9401,0.934,0.8976,0.8579 -Standard,0.8504,0.7573,0.5349,0.4535,0.3429,0.9029,0.8385,0.7440000000000001,0.7483,0.6525,0.8991,0.8043,0.8365,0.8051,0.7493,0.9195,0.8604,0.8002,0.6035,0.4155,0.7778,0.5788,0.3919,0.3261,0.2767,0.9458,0.9338,0.8898,0.7744,0.5306,0.9475,0.9438,0.9378,0.9281,0.907,0.9451,0.9356,0.9229,0.8959,0.7399,0.8771,0.8455,0.7748,0.7029,0.5798,0.8955,0.835,0.7261,0.7086,0.5870000000000001,0.5343,0.5575,0.5695,0.4319,0.4568,0.8289,0.6905,0.5737,0.4011,0.2708,0.9427,0.9216,0.8972,0.8359,0.5331,0.8636,0.7965,0.7796,0.7415,0.6970000000000001,0.9047,0.9022,0.8586,0.7894,0.7341 -Kireev2021Effectiveness_Gauss50percent,0.9232,0.9172,0.9113,0.9079,0.9008,0.903,0.86,0.8045,0.8047,0.7414,0.9129,0.8805,0.8561,0.8205,0.8233,0.923,0.9098,0.9045,0.8689,0.7818,0.9197,0.9093,0.9064,0.904,0.902,0.9288,0.9162,0.8902,0.8504,0.7374,0.9309,0.9247,0.9173,0.908,0.8799,0.9232,0.8947,0.8402,0.7558,0.5231,0.8887,0.8796,0.8575,0.8370000000000001,0.7963,0.913,0.8911,0.8631,0.8629,0.8282,0.8315,0.8311,0.8417,0.7181,0.7415,0.921,0.9102,0.8947,0.8519,0.7869,0.917,0.8447,0.7559999999999999,0.5717,0.2279,0.9166,0.9044,0.8975,0.8897,0.88,0.893,0.8941,0.8717,0.8412,0.8139 -Kireev2021Effectiveness_AugMixNoJSD,0.9249,0.9051,0.8295,0.7844,0.7,0.9395,0.9239,0.8996,0.8971,0.8711,0.9249,0.8829,0.8785,0.8549,0.8412,0.9405,0.9269,0.9117,0.8437,0.7066,0.9064,0.8417,0.7522,0.6976,0.6424,0.9504,0.9482,0.9444,0.9317,0.8973,0.9487,0.9462,0.9424,0.9359999999999999,0.9236,0.9474,0.9412,0.9306,0.9128,0.8191,0.9332,0.9328,0.9215,0.9077,0.8794,0.9317,0.9069,0.8628,0.8658,0.8152,0.7681,0.7686,0.787,0.6464,0.6777,0.9317,0.9077,0.8746,0.7645,0.5991,0.9473,0.934,0.922,0.8935,0.6832,0.9051,0.8699,0.863,0.8461,0.8151,0.9262,0.9279999999999999,0.9202,0.8625,0.8012 -Kireev2021Effectiveness_RLATAugMixNoJSD,0.9398,0.9312,0.8907,0.8658,0.8109999999999999,0.9366,0.9223,0.9044,0.9024,0.8759999999999999,0.9304,0.8948,0.8866,0.858,0.8543,0.9409,0.9309999999999999,0.9259999999999999,0.8897,0.7975,0.9317,0.9013,0.8505,0.8123,0.7782,0.9473,0.9454,0.9388,0.9302,0.8986,0.9481,0.9462,0.9415,0.9347,0.9178,0.9478,0.938,0.9209,0.892,0.7755,0.9341,0.9344,0.9267,0.9193,0.9025,0.9382,0.9159,0.8892,0.8879,0.8551,0.8345,0.8365,0.852,0.7298,0.7595,0.9346,0.9132,0.8872,0.8005,0.6669,0.9443,0.9188,0.8984,0.8378,0.4563,0.9276,0.9096,0.9047,0.8934,0.882,0.9255,0.9283,0.9214,0.8915,0.8548 -Kireev2021Effectiveness_RLAT,0.927,0.9206,0.8808,0.8626,0.8074,0.9045,0.863,0.8024,0.8042,0.7422,0.9125,0.8825,0.8648,0.8245,0.8251,0.9266,0.919,0.9148,0.8986,0.851,0.9242,0.8965,0.8571,0.8293,0.7926,0.9272,0.9172,0.8962,0.8625,0.7557,0.9312,0.9285,0.9215,0.9124,0.8804,0.9214,0.8851,0.828,0.7242,0.4763,0.8944,0.8876,0.8676,0.8464,0.8086,0.9186,0.8944,0.8583,0.8607,0.8185,0.8370000000000001,0.8445,0.8531,0.7299,0.7561,0.9136,0.8835,0.8461,0.736,0.5995,0.9134,0.8251,0.7116,0.4903,0.1918,0.9195,0.9087,0.9048,0.9009999999999999,0.8923,0.8959999999999999,0.8981,0.8796,0.8553,0.8314 -Kireev2021Effectiveness_RLATAugMix,0.9394,0.9322,0.8923,0.8717,0.8249,0.94,0.9295,0.9138,0.9147,0.8953,0.9321,0.9061,0.8967,0.8702,0.8645,0.9418,0.9345,0.9281,0.9009,0.8293,0.9315,0.9068,0.8631,0.8343,0.7979,0.9469,0.9440000000000001,0.94,0.932,0.9121,0.9487,0.9481,0.9447,0.9398,0.9277,0.9464,0.9397,0.9232,0.8959,0.784,0.9377,0.9382,0.9325,0.9253,0.9133,0.9389,0.9224,0.8977,0.8976,0.8646,0.8521,0.8573,0.8694,0.7558,0.7829999999999999,0.937,0.9182,0.8883,0.8187,0.7070000000000001,0.9455,0.9296,0.9159,0.8873,0.6509999999999999,0.9259,0.9102,0.9031,0.8977,0.8817,0.9262,0.9304,0.9211,0.8957,0.8574 -Rebuffi2021Fixing_70_16_cutmix_extra_L2,0.9511,0.9405,0.8915,0.8595,0.7835,0.9435,0.9224,0.8979,0.8969,0.8589,0.9451,0.9278,0.9156,0.8879,0.8832,0.9524,0.9491,0.946,0.9357,0.9151,0.9444,0.9121,0.8573,0.8175,0.7667,0.9557,0.9484,0.9378,0.9249,0.8788,0.9583,0.9579,0.955,0.9488,0.9276,0.9467,0.911,0.8527,0.7519,0.5136,0.9385,0.9361,0.9291,0.9189,0.9016,0.9499,0.9353,0.9099,0.9076,0.8754,0.8942,0.8984,0.9086,0.8143,0.8462,0.9425,0.9188,0.8887,0.7737,0.6045,0.9394,0.8571,0.7524,0.5383,0.215,0.9494,0.9439,0.9414,0.9362,0.9332,0.9326,0.9339,0.9257,0.9136,0.9011 -Rebuffi2021Fixing_70_16_cutmix_extra_Linf,0.9136,0.9025,0.8590000000000001,0.8321,0.7813,0.8981,0.8736,0.8356,0.8369,0.7918,0.9113,0.9022,0.8859,0.8571,0.8484,0.9155,0.9098,0.9053,0.8934,0.8719,0.9056,0.877,0.8302,0.8006,0.7694,0.9176,0.9057,0.8898,0.8717,0.82,0.9262,0.9247,0.9207,0.9119,0.8732,0.9023,0.8364,0.7361,0.6069,0.3637,0.8904,0.8864,0.8757,0.8672,0.8472,0.9133,0.8899,0.8523,0.8509,0.8169,0.8617,0.8624,0.8643,0.7943,0.8064,0.8831,0.84,0.7935,0.7048,0.6112,0.8861,0.6968,0.4959,0.2794,0.1886,0.9082,0.8988,0.8976,0.8941,0.8871,0.8845,0.8841,0.8752,0.8622,0.8472 -Diffenderfer2021Winning_LRR,0.9494,0.9312,0.8661,0.8259,0.7391,0.9594,0.9513,0.9403,0.942,0.9303,0.9525,0.933,0.9265,0.909,0.9025,0.9606,0.9493,0.9424,0.8973,0.7847,0.9319,0.8805,0.7967,0.7424,0.6869,0.9658,0.9645,0.9601,0.9535,0.9375,0.9661,0.9633,0.961,0.9579,0.9481,0.9647,0.9608,0.9548,0.9425,0.8815,0.9559,0.9576,0.9525,0.9489,0.9327,0.9541,0.9372,0.916,0.9128,0.8805,0.8578,0.8666,0.8744,0.7571,0.7844,0.9569,0.944,0.9271,0.8642,0.7626,0.9647,0.9588,0.9539,0.939,0.8474,0.9258,0.9062,0.897,0.8796,0.8557,0.947,0.9524,0.9478,0.9121,0.8641 -Diffenderfer2021Winning_Binary,0.934,0.9226,0.8737,0.8495,0.8019,0.9390000000000001,0.9277,0.9066,0.909,0.8899,0.9299,0.8919,0.8843,0.8625,0.8429,0.9399,0.9277,0.9179999999999999,0.8745,0.7773,0.9208,0.8841,0.8244,0.7925,0.7514,0.948,0.9457,0.9406,0.9341,0.9079,0.9466,0.9432,0.9374,0.9333,0.9196,0.9441,0.9406,0.9285,0.9067,0.8105,0.9353,0.9347,0.9279,0.9194,0.8994,0.9327,0.9086,0.8744,0.8742,0.828,0.8128,0.8181,0.8282,0.7103,0.7316,0.9281,0.9054,0.873,0.7781,0.652,0.9444,0.9308,0.9165,0.8888,0.6984,0.9161,0.8957,0.8891,0.8753,0.8531,0.9266,0.9279999999999999,0.9224,0.8853,0.8340000000000001 -Diffenderfer2021Winning_LRR_CARD_Deck,0.9572,0.9515,0.9304,0.9229,0.9042,0.9585,0.9456,0.9304,0.9351,0.9147,0.956,0.9417,0.9305,0.9092,0.9055,0.961,0.9553,0.9506,0.9251,0.8495,0.9504,0.9348,0.9172,0.9054,0.8914,0.9653,0.9634,0.9601,0.9506,0.9224,0.9646,0.9636,0.9615,0.9574,0.9457,0.9642,0.9609,0.9453,0.9281,0.8340000000000001,0.9551,0.9542,0.9475,0.9389,0.9219,0.9569,0.9444,0.9269,0.9273,0.9006,0.8925,0.8958,0.8996,0.8039,0.8228,0.9604,0.9528,0.9462,0.912,0.8640000000000001,0.9647,0.952,0.9375,0.9132,0.7956,0.9445,0.9334,0.929,0.9235,0.9088,0.9512,0.9511,0.9464,0.9154,0.8783 -Diffenderfer2021Winning_Binary_CARD_Deck,0.945,0.9388,0.9229,0.9149,0.9036,0.9376,0.9214,0.8957,0.8954,0.8701,0.9385,0.9134,0.8982,0.8668,0.8621,0.9441,0.9362,0.9307,0.9015,0.8225,0.9388,0.9276,0.9112,0.9004,0.8925,0.9499,0.9447,0.9369,0.9243,0.8857,0.9503,0.9474,0.9418,0.9356,0.9172,0.9481,0.9366,0.9171,0.884,0.746,0.9339,0.9309,0.9229999999999999,0.9134,0.8909999999999999,0.9381,0.9195,0.8985,0.8986,0.8692,0.8724,0.8713,0.8741,0.778,0.7933,0.9431,0.9291,0.9186,0.8843,0.8426,0.9444,0.9192,0.8918,0.8402,0.6042,0.9329999999999999,0.9203,0.9161,0.9096,0.8966,0.9295,0.9321,0.9211,0.8895,0.851 -Addepalli2021Towards_WRN34,0.8466,0.8389,0.7994,0.7758,0.7151,0.8236,0.7945,0.7601,0.7581,0.724,0.8451,0.8379,0.8177,0.8048,0.7985,0.8448,0.8405,0.8333,0.8212,0.8016,0.8394,0.8172,0.7719,0.7423,0.7037,0.8459,0.8326,0.815,0.7972,0.7467,0.8587,0.8588,0.8568,0.8526,0.8317,0.8303,0.7687,0.684,0.5667,0.3517,0.816,0.8127,0.8043,0.7947,0.7759,0.8483,0.8285,0.7935,0.7843,0.7467,0.7998,0.8012,0.7935,0.7425,0.7465,0.8248,0.7948,0.7542,0.6549,0.551,0.8118,0.6633,0.5305,0.3584,0.1953,0.8379,0.8307,0.8279,0.8231,0.8195,0.8062,0.8101,0.7952,0.781,0.7728 -Modas2021PRIMEResNet18,0.9203,0.9157,0.8993,0.8975,0.8848,0.92,0.906,0.8883,0.8885,0.8677,0.918,0.899,0.8857,0.856,0.856,0.9222,0.9118,0.9073,0.8814,0.8356,0.9145,0.901,0.8902,0.8767,0.8686,0.928,0.9236,0.9165,0.907,0.8845,0.929,0.9275,0.925,0.9238,0.9148,0.9288,0.9173,0.898,0.8575,0.7277,0.9156,0.9163,0.9112,0.9063,0.8931,0.9211,0.9097,0.8953,0.894,0.8807,0.8737,0.8744,0.8802,0.8246,0.8343,0.9148,0.8877,0.8668,0.8048,0.743,0.9291,0.9172,0.9096,0.8936,0.8389,0.914,0.8997,0.8954,0.8903,0.8804,0.9055,0.908,0.8996,0.8799,0.8594 -Erichson2022NoisyMix,0.9611,0.9529,0.9339,0.9198,0.8991,0.9588,0.9463,0.9318,0.9327,0.9118,0.9581,0.9396,0.935,0.9166,0.9065,0.9594,0.95,0.9427,0.8862,0.7533,0.9548,0.9395,0.912,0.9003,0.8818,0.9668,0.9637,0.959,0.9513,0.9269,0.9682,0.9659,0.9636,0.9601,0.9493,0.9666,0.961,0.9509,0.9338,0.8563,0.9545,0.9536,0.9469,0.9417,0.9276,0.9621,0.9497,0.9329,0.9292,0.9076,0.8973,0.8964,0.9038,0.8305,0.8412,0.9662,0.9658,0.9647,0.9614,0.9618,0.9666,0.957,0.946,0.9136,0.6425,0.9429,0.9272,0.9201,0.9098,0.8909,0.9512,0.9506,0.945,0.9175,0.8805 -Addepalli2022Efficient_WRN_34_10,0.8819,0.8752,0.8474,0.8345,0.8038,0.8565,0.8319,0.7952,0.7941,0.7508,0.8752,0.8623,0.8473,0.8132,0.797,0.8798,0.8738,0.8701,0.8565,0.838,0.8769,0.8596,0.8286,0.8123,0.7974,0.8811,0.8657,0.8483,0.8315,0.7803,0.888,0.8859,0.8799,0.8693,0.8309,0.8674,0.7994,0.7051,0.5727,0.3483,0.8485,0.8429,0.8306,0.8208,0.8012,0.87,0.8425,0.8033,0.805,0.7798,0.8378,0.8365,0.8345,0.7796,0.7841,0.8618,0.8301,0.8075,0.7472,0.703,0.8498,0.6766,0.5126,0.3187,0.1941,0.873,0.866,0.8609,0.8576,0.8571,0.8401,0.84,0.8353,0.8175,0.8074 diff --git a/robustbench/model_info/cifar100/Linf/Addepalli2021Towards_PARN18.json b/robustbench/model_info/cifar100/Linf/Addepalli2021Towards_PARN18.json deleted file mode 100644 index e0cbe9e1..00000000 --- a/robustbench/model_info/cifar100/Linf/Addepalli2021Towards_PARN18.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://openreview.net/forum?id=SHB_znlW5G7", - "name": "Towards Achieving Adversarial Robustness Beyond Perceptual Limits", - "authors": "Sravanti Addepalli, Samyak Jain, Gaurang Sriramanan, Shivangi Khare, Venkatesh Babu Radhakrishnan", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar100", - "venue": "OpenReview, Jun 2021", - "architecture": "PreActResNet-18", - "eps": "8/255", - "clean_acc": "62.02", - "reported": "27.14", - "autoattack_acc": "27.14", - "unreliable": false -} \ No newline at end of file diff --git a/robustbench/model_info/cifar100/Linf/Addepalli2021Towards_WRN34.json b/robustbench/model_info/cifar100/Linf/Addepalli2021Towards_WRN34.json deleted file mode 100644 index 2f82a135..00000000 --- a/robustbench/model_info/cifar100/Linf/Addepalli2021Towards_WRN34.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://openreview.net/forum?id=SHB_znlW5G7", - "name": "Towards Achieving Adversarial Robustness Beyond Perceptual Limits", - "authors": "Sravanti Addepalli, Samyak Jain, Gaurang Sriramanan, Shivangi Khare, Venkatesh Babu Radhakrishnan", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar100", - "venue": "OpenReview, Jun 2021", - "architecture": "WideResNet-34-10", - "eps": "8/255", - "clean_acc": "65.73", - "reported": "30.35", - "autoattack_acc": "30.35", - "unreliable": false -} \ No newline at end of file diff --git a/robustbench/model_info/cifar100/Linf/Addepalli2022Efficient_RN18.json b/robustbench/model_info/cifar100/Linf/Addepalli2022Efficient_RN18.json deleted file mode 100644 index 0aae4cf4..00000000 --- a/robustbench/model_info/cifar100/Linf/Addepalli2022Efficient_RN18.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://artofrobust.github.io/short_paper/31.pdf", - "name": "Efficient and Effective Augmentation Strategy for Adversarial Training", - "authors": "Sravanti Addepalli, Samyak Jain, Venkatesh Babu Radhakrishnan", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar100", - "venue": "CVPRW 2022", - "architecture": "ResNet-18", - "eps": "8/255", - "clean_acc": "65.45", - "reported": "27.69", - "autoattack_acc": "27.67", - "unreliable": false -} \ No newline at end of file diff --git a/robustbench/model_info/cifar100/Linf/Addepalli2022Efficient_WRN_34_10.json b/robustbench/model_info/cifar100/Linf/Addepalli2022Efficient_WRN_34_10.json deleted file mode 100644 index 85984d36..00000000 --- a/robustbench/model_info/cifar100/Linf/Addepalli2022Efficient_WRN_34_10.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://artofrobust.github.io/short_paper/31.pdf", - "name": "Efficient and Effective Augmentation Strategy for Adversarial Training", - "authors": "Sravanti Addepalli, Samyak Jain, Venkatesh Babu Radhakrishnan", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar100", - "venue": "CVPRW 2022", - "architecture": "WideResNet-34-10", - "eps": "8/255", - "clean_acc": "68.75", - "reported": "31.85", - "autoattack_acc": "31.85", - "unreliable": false -} \ No newline at end of file diff --git a/robustbench/model_info/cifar100/Linf/Chen2020Efficient.json b/robustbench/model_info/cifar100/Linf/Chen2020Efficient.json deleted file mode 100644 index e35c0a1f..00000000 --- a/robustbench/model_info/cifar100/Linf/Chen2020Efficient.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2010.01278", - "name": "Efficient Robust Training via Backward Smoothing", - "authors": "Jinghui Chen, Yu Cheng, Zhe Gan, Quanquan Gu, Jingjing Liu", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar100", - "venue": "arXiv, Oct 2020", - "architecture": "WideResNet-34-10", - "eps": "8/255", - "clean_acc": "62.15", - "reported": "26.94", - "autoattack_acc": "26.94", - "footnote": null -} diff --git a/robustbench/model_info/cifar100/Linf/Chen2021LTD_WRN34_10.json b/robustbench/model_info/cifar100/Linf/Chen2021LTD_WRN34_10.json deleted file mode 100644 index 940befcc..00000000 --- a/robustbench/model_info/cifar100/Linf/Chen2021LTD_WRN34_10.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2111.02331", - "name": "LTD: Low Temperature Distillation for Robust Adversarial Training", - "authors": "Erh-Chung Chen, Che-Rung Lee", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar100", - "venue": "arXiv, Nov 2021", - "architecture": "WideResNet-34-10", - "eps": "8/255", - "clean_acc": "64.07", - "reported": "30.59", - "footnote": "", - "autoattack_acc": "30.59", - "unreliable": false -} diff --git a/robustbench/model_info/cifar100/Linf/Cui2020Learnable_34_10_LBGAT0.json b/robustbench/model_info/cifar100/Linf/Cui2020Learnable_34_10_LBGAT0.json deleted file mode 100644 index 0c9a1fef..00000000 --- a/robustbench/model_info/cifar100/Linf/Cui2020Learnable_34_10_LBGAT0.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2011.11164", - "name": "Learnable Boundary Guided Adversarial Training", - "authors": "Jiequan Cui, Shu Liu, Liwei Wang, Jiaya Jia", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar100", - "venue": "ICCV 2021", - "architecture": "WideResNet-34-10", - "eps": "8/255", - "clean_acc": "70.25", - "reported": "27.16", - "autoattack_acc": "27.16", - "footnote": "Uses \\(\\ell_{\\infty} \\) = 0.031 \u2248 7.9/255 instead of 8/255", - "unreliable": false -} \ No newline at end of file diff --git a/robustbench/model_info/cifar100/Linf/Cui2020Learnable_34_10_LBGAT6.json b/robustbench/model_info/cifar100/Linf/Cui2020Learnable_34_10_LBGAT6.json deleted file mode 100644 index aa3f5e97..00000000 --- a/robustbench/model_info/cifar100/Linf/Cui2020Learnable_34_10_LBGAT6.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2011.11164", - "name": "Learnable Boundary Guided Adversarial Training", - "authors": "Jiequan Cui, Shu Liu, Liwei Wang, Jiaya Jia", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar100", - "venue": "ICCV 2021", - "architecture": "WideResNet-34-10", - "eps": "8/255", - "clean_acc": "60.64", - "reported": "29.33", - "autoattack_acc": "29.33", - "footnote": "Uses \\(\\ell_{\\infty} \\) = 0.031 \u2248 7.9/255 instead of 8/255" -} \ No newline at end of file diff --git a/robustbench/model_info/cifar100/Linf/Cui2020Learnable_34_20_LBGAT6.json b/robustbench/model_info/cifar100/Linf/Cui2020Learnable_34_20_LBGAT6.json deleted file mode 100644 index bd7e65aa..00000000 --- a/robustbench/model_info/cifar100/Linf/Cui2020Learnable_34_20_LBGAT6.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2011.11164", - "name": "Learnable Boundary Guided Adversarial Training", - "authors": "Jiequan Cui, Shu Liu, Liwei Wang, Jiaya Jia", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar100", - "venue": "ICCV 2021", - "architecture": "WideResNet-34-20", - "eps": "8/255", - "clean_acc": "62.55", - "reported": "30.20", - "autoattack_acc": "30.20", - "footnote": "Uses \\(\\ell_{\\infty} \\) = 0.031 \u2248 7.9/255 instead of 8/255" -} \ No newline at end of file diff --git a/robustbench/model_info/cifar100/Linf/Debenedetti2022Light_XCiT-L12.json b/robustbench/model_info/cifar100/Linf/Debenedetti2022Light_XCiT-L12.json deleted file mode 100644 index 7fe1dd3a..00000000 --- a/robustbench/model_info/cifar100/Linf/Debenedetti2022Light_XCiT-L12.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2209.07399", - "name": "A Light Recipe to Train Robust Vision Transformers", - "authors": "Edoardo Debenedetti, Vikash Sehwag, Prateek Mittal", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar100", - "venue": "arXiv, Sep 2022", - "architecture": "XCiT-L12", - "eps": "8/255", - "clean_acc": "70.76", - "reported": "35.08", - "autoattack_acc": "35.08", - "unreliable": false -} diff --git a/robustbench/model_info/cifar100/Linf/Debenedetti2022Light_XCiT-M12.json b/robustbench/model_info/cifar100/Linf/Debenedetti2022Light_XCiT-M12.json deleted file mode 100644 index 7eeaea60..00000000 --- a/robustbench/model_info/cifar100/Linf/Debenedetti2022Light_XCiT-M12.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2209.07399", - "name": "A Light Recipe to Train Robust Vision Transformers", - "authors": "Edoardo Debenedetti, Vikash Sehwag, Prateek Mittal", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar100", - "venue": "arXiv, Sep 2022", - "architecture": "XCiT-M12", - "eps": "8/255", - "clean_acc": "69.21", - "reported": "34.21", - "autoattack_acc": "34.21", - "unreliable": false -} diff --git a/robustbench/model_info/cifar100/Linf/Debenedetti2022Light_XCiT-S12.json b/robustbench/model_info/cifar100/Linf/Debenedetti2022Light_XCiT-S12.json deleted file mode 100644 index dfb9f7f5..00000000 --- a/robustbench/model_info/cifar100/Linf/Debenedetti2022Light_XCiT-S12.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2209.07399", - "name": "A Light Recipe to Train Robust Vision Transformers", - "authors": "Edoardo Debenedetti, Vikash Sehwag, Prateek Mittal", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar100", - "venue": "arXiv, Sep 2022", - "architecture": "XCiT-S12", - "eps": "8/255", - "clean_acc": "67.34", - "reported": "32.19", - "autoattack_acc": "32.19", - "unreliable": false -} diff --git a/robustbench/model_info/cifar100/Linf/Gowal2020Uncovering.json b/robustbench/model_info/cifar100/Linf/Gowal2020Uncovering.json deleted file mode 100644 index e981980d..00000000 --- a/robustbench/model_info/cifar100/Linf/Gowal2020Uncovering.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2010.03593", - "name": "Uncovering the Limits of Adversarial Training against Norm-Bounded Adversarial Examples", - "authors": "Sven Gowal, Chongli Qin, Jonathan Uesato, Timothy Mann, Pushmeet Kohli", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar100", - "venue": "arXiv, Oct 2020", - "architecture": "WideResNet-70-16", - "eps": "8/255", - "clean_acc": "60.86", - "reported": "30.67", - "autoattack_acc": "30.03", - "footnote": null, - "unreliable": false -} diff --git a/robustbench/model_info/cifar100/Linf/Gowal2020Uncovering_extra.json b/robustbench/model_info/cifar100/Linf/Gowal2020Uncovering_extra.json deleted file mode 100644 index ac3e7590..00000000 --- a/robustbench/model_info/cifar100/Linf/Gowal2020Uncovering_extra.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2010.03593", - "name": "Uncovering the Limits of Adversarial Training against Norm-Bounded Adversarial Examples", - "authors": "Sven Gowal, Chongli Qin, Jonathan Uesato, Timothy Mann, Pushmeet Kohli", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar100", - "venue": "arXiv, Oct 2020", - "architecture": "WideResNet-70-16", - "eps": "8/255", - "clean_acc": "69.15", - "reported": "37.70", - "autoattack_acc": "36.88", - "footnote": null, - "unreliable": false -} diff --git a/robustbench/model_info/cifar100/Linf/Hendrycks2019Using.json b/robustbench/model_info/cifar100/Linf/Hendrycks2019Using.json deleted file mode 100644 index d9418e96..00000000 --- a/robustbench/model_info/cifar100/Linf/Hendrycks2019Using.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "https://arxiv.org/abs/1901.09960", - "name": "Using Pre-Training Can Improve Model Robustness and Uncertainty", - "authors": "Dan Hendrycks, Kimin Lee, Mantas Mazeika", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ICML 2019", - "architecture": "WideResNet-28-10", - "eps": "8/255", - "clean_acc": "59.23", - "reported": "33.5", - "autoattack_acc": "28.42" -} diff --git a/robustbench/model_info/cifar100/Linf/Jia2022LAS-AT_34_10.json b/robustbench/model_info/cifar100/Linf/Jia2022LAS-AT_34_10.json deleted file mode 100644 index ada80a53..00000000 --- a/robustbench/model_info/cifar100/Linf/Jia2022LAS-AT_34_10.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2203.06616", - "name": "LAS-AT: Adversarial Training with Learnable Attack Strategy", - "authors": "Xiaojun Jia, Yong Zhang, Baoyuan Wu, Ke Ma, Jue Wang, Xiaochun Cao", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar100", - "venue": "arXiv, Mar 2022", - "architecture": "WideResNet-34-10", - "eps": "8/255", - "clean_acc": "64.89", - "reported": "30.77", - "autoattack_acc": "30.77", - "footnote": "", - "unreliable": false -} \ No newline at end of file diff --git a/robustbench/model_info/cifar100/Linf/Jia2022LAS-AT_34_20.json b/robustbench/model_info/cifar100/Linf/Jia2022LAS-AT_34_20.json deleted file mode 100644 index 74385c2e..00000000 --- a/robustbench/model_info/cifar100/Linf/Jia2022LAS-AT_34_20.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2203.06616", - "name": "LAS-AT: Adversarial Training with Learnable Attack Strategy", - "authors": "Xiaojun Jia, Yong Zhang, Baoyuan Wu, Ke Ma, Jue Wang, Xiaochun Cao", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar100", - "venue": "arXiv, Mar 2022", - "architecture": "WideResNet-34-20", - "eps": "8/255", - "clean_acc": "67.31", - "reported": "31.91", - "autoattack_acc": "31.91", - "footnote": "", - "unreliable": false -} \ No newline at end of file diff --git a/robustbench/model_info/cifar100/Linf/Pang2022Robustness_WRN28_10.json b/robustbench/model_info/cifar100/Linf/Pang2022Robustness_WRN28_10.json deleted file mode 100644 index 6f8b62ff..00000000 --- a/robustbench/model_info/cifar100/Linf/Pang2022Robustness_WRN28_10.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/pdf/2202.10103.pdf", - "name": " Robustness and Accuracy Could Be Reconcilable by (Proper) Definition", - "authors": "Tianyu Pang, Min Lin, Xiao Yang, Jun Zhu, Shuicheng Yan", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar100", - "venue": "ICML 2022", - "architecture": "WideResNet-28-10", - "eps": "8/255", - "clean_acc": "63.66", - "reported": "31.08", - "autoattack_acc": "31.08", - "footnote": "It uses additional 1M synthetic images in training.", - "unreliable": false -} diff --git a/robustbench/model_info/cifar100/Linf/Pang2022Robustness_WRN70_16.json b/robustbench/model_info/cifar100/Linf/Pang2022Robustness_WRN70_16.json deleted file mode 100644 index 564dc06b..00000000 --- a/robustbench/model_info/cifar100/Linf/Pang2022Robustness_WRN70_16.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/pdf/2202.10103.pdf", - "name": " Robustness and Accuracy Could Be Reconcilable by (Proper) Definition", - "authors": "Tianyu Pang, Min Lin, Xiao Yang, Jun Zhu, Shuicheng Yan", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar100", - "venue": "ICML 2022", - "architecture": "WideResNet-70-16", - "eps": "8/255", - "clean_acc": "65.56", - "reported": "33.05", - "autoattack_acc": "33.05", - "footnote": "It uses additional 1M synthetic images in training.", - "unreliable": false -} diff --git a/robustbench/model_info/cifar100/Linf/Rade2021Helper_R18_ddpm.json b/robustbench/model_info/cifar100/Linf/Rade2021Helper_R18_ddpm.json deleted file mode 100644 index 03451595..00000000 --- a/robustbench/model_info/cifar100/Linf/Rade2021Helper_R18_ddpm.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://openreview.net/forum?id=BuD2LmNaU3a", - "name": "Helper-based Adversarial Training: Reducing Excessive Margin to Achieve a Better Accuracy vs. Robustness Trade-off", - "authors": "Rahul Rade and Seyed-Mohsen Moosavi-Dezfooli", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar100", - "venue": "OpenReview, Jun 2021", - "architecture": "PreActResNet-18", - "eps": "8/255", - "clean_acc": "61.50", - "reported": "28.88", - "footnote": "It uses additional 1M synthetic images in training.", - "autoattack_acc": "28.88", - "unreliable": false -} diff --git a/robustbench/model_info/cifar100/Linf/Rebuffi2021Fixing_28_10_cutmix_ddpm.json b/robustbench/model_info/cifar100/Linf/Rebuffi2021Fixing_28_10_cutmix_ddpm.json deleted file mode 100644 index 2cf25e89..00000000 --- a/robustbench/model_info/cifar100/Linf/Rebuffi2021Fixing_28_10_cutmix_ddpm.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2103.01946", - "name": "Fixing Data Augmentation to Improve Adversarial Robustness", - "authors": "Sylvestre-Alvise Rebuffi, Sven Gowal, Dan A. Calian, Florian Stimberg, Olivia Wiles, Timothy Mann", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar100", - "venue": "arXiv, Mar 2021", - "architecture": "WideResNet-28-10", - "eps": "8/255", - "clean_acc": "62.41", - "reported": "32.06", - "footnote": "It uses additional 1M synthetic images in training.", - "autoattack_acc": "32.06", - "unreliable": false -} diff --git a/robustbench/model_info/cifar100/Linf/Rebuffi2021Fixing_70_16_cutmix_ddpm.json b/robustbench/model_info/cifar100/Linf/Rebuffi2021Fixing_70_16_cutmix_ddpm.json deleted file mode 100644 index 224792ab..00000000 --- a/robustbench/model_info/cifar100/Linf/Rebuffi2021Fixing_70_16_cutmix_ddpm.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2103.01946", - "name": "Fixing Data Augmentation to Improve Adversarial Robustness", - "authors": "Sylvestre-Alvise Rebuffi, Sven Gowal, Dan A. Calian, Florian Stimberg, Olivia Wiles, Timothy Mann", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar100", - "venue": "arXiv, Mar 2021", - "architecture": "WideResNet-70-16", - "eps": "8/255", - "clean_acc": "63.56", - "reported": "34.64", - "footnote": "It uses additional 1M synthetic images in training.", - "autoattack_acc": "34.64", - "unreliable": false -} diff --git a/robustbench/model_info/cifar100/Linf/Rebuffi2021Fixing_R18_ddpm.json b/robustbench/model_info/cifar100/Linf/Rebuffi2021Fixing_R18_ddpm.json deleted file mode 100644 index 51f4233f..00000000 --- a/robustbench/model_info/cifar100/Linf/Rebuffi2021Fixing_R18_ddpm.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2103.01946", - "name": "Fixing Data Augmentation to Improve Adversarial Robustness", - "authors": "Sylvestre-Alvise Rebuffi, Sven Gowal, Dan A. Calian, Florian Stimberg, Olivia Wiles, Timothy Mann", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar100", - "venue": "arXiv, Mar 2021", - "architecture": "PreActResNet-18", - "eps": "8/255", - "clean_acc": "56.87", - "reported": "28.50", - "footnote": "It uses additional 1M synthetic images in training.", - "autoattack_acc": "28.50", - "unreliable": false -} diff --git a/robustbench/model_info/cifar100/Linf/Rice2020Overfitting.json b/robustbench/model_info/cifar100/Linf/Rice2020Overfitting.json deleted file mode 100644 index 2ed76f9b..00000000 --- a/robustbench/model_info/cifar100/Linf/Rice2020Overfitting.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2002.11569", - "name": "Overfitting in adversarially robust deep learning", - "authors": "Leslie Rice, Eric Wong, J. Zico Kolter", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar100", - "venue": "ICML 2020", - "architecture": "PreActResNet-18", - "eps": "8/255", - "clean_acc": "53.83", - "reported": "28.1", - "autoattack_acc": "18.95", - "footnote": null -} \ No newline at end of file diff --git a/robustbench/model_info/cifar100/Linf/Sehwag2021Proxy.json b/robustbench/model_info/cifar100/Linf/Sehwag2021Proxy.json deleted file mode 100644 index 799b40b8..00000000 --- a/robustbench/model_info/cifar100/Linf/Sehwag2021Proxy.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2104.09425", - "name": "Robust Learning Meets Generative Models: Can Proxy Distributions Improve Adversarial Robustness?", - "authors": "Vikash Sehwag, Saeed Mahloujifar, Tinashe Handina, Sihui Dai, Chong Xiang, Mung Chiang, Prateek Mittal", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar100", - "venue": "ICLR 2022", - "architecture": "WideResNet-34-10", - "eps": "8/255", - "clean_acc": "65.93", - "reported": "31.15", - "autoattack_acc": "31.15", - "footnote": "It uses additional 1M synthetic images in training.", - "unreliable": false -} \ No newline at end of file diff --git a/robustbench/model_info/cifar100/Linf/Sitawarin2020Improving.json b/robustbench/model_info/cifar100/Linf/Sitawarin2020Improving.json deleted file mode 100644 index 2f56a056..00000000 --- a/robustbench/model_info/cifar100/Linf/Sitawarin2020Improving.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2003.09347", - "name": "Improving Adversarial Robustness Through Progressive Hardening", - "authors": "Chawin Sitawarin, Supriyo Chakraborty, David Wagner", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar100", - "venue": "arXiv, Mar 2020", - "architecture": "WideResNet-34-10", - "eps": "8/255", - "clean_acc": "62.82", - "reported": "24.57", - "autoattack_acc": "24.57", - "footnote": null -} \ No newline at end of file diff --git a/robustbench/model_info/cifar100/Linf/Wu2020Adversarial.json b/robustbench/model_info/cifar100/Linf/Wu2020Adversarial.json deleted file mode 100644 index 675044a2..00000000 --- a/robustbench/model_info/cifar100/Linf/Wu2020Adversarial.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2004.05884", - "name": "Adversarial Weight Perturbation Helps Robust Generalization", - "authors": "Dongxian Wu, Shu-tao Xia, Yisen Wang", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar100", - "venue": "NeurIPS 2020", - "architecture": "WideResNet-34-10", - "eps": "8/255", - "clean_acc": "60.38", - "reported": "28.86", - "autoattack_acc": "28.86", - "footnote": null, - "unreliable": false -} diff --git a/robustbench/model_info/cifar100/corruptions/Addepalli2021Towards_PARN18.json b/robustbench/model_info/cifar100/corruptions/Addepalli2021Towards_PARN18.json deleted file mode 100644 index 7bb2960d..00000000 --- a/robustbench/model_info/cifar100/corruptions/Addepalli2021Towards_PARN18.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "https://openreview.net/forum?id=SHB_znlW5G7", - "name": "Towards Achieving Adversarial Robustness Beyond Perceptual Limits", - "authors": "Sravanti Addepalli, Samyak Jain, Gaurang Sriramanan, Shivangi Khare, Venkatesh Babu Radhakrishnan", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar100", - "venue": "OpenReview, Jun 2021", - "architecture": "PreActResNet-18", - "eps": null, - "clean_acc": "62.02", - "reported": "51.77", - "corruptions_acc": "51.77" -} \ No newline at end of file diff --git a/robustbench/model_info/cifar100/corruptions/Addepalli2021Towards_WRN34.json b/robustbench/model_info/cifar100/corruptions/Addepalli2021Towards_WRN34.json deleted file mode 100644 index 9d56c162..00000000 --- a/robustbench/model_info/cifar100/corruptions/Addepalli2021Towards_WRN34.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "https://openreview.net/forum?id=SHB_znlW5G7", - "name": "Towards Achieving Adversarial Robustness Beyond Perceptual Limits", - "authors": "Sravanti Addepalli, Samyak Jain, Gaurang Sriramanan, Shivangi Khare, Venkatesh Babu Radhakrishnan", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar100", - "venue": "OpenReview, Jun 2021", - "architecture": "WideResNet-34-10", - "eps": null, - "clean_acc": "65.73", - "reported": "54.88", - "corruptions_acc": "54.88" -} \ No newline at end of file diff --git a/robustbench/model_info/cifar100/corruptions/Addepalli2022Efficient_WRN_34_10.json b/robustbench/model_info/cifar100/corruptions/Addepalli2022Efficient_WRN_34_10.json deleted file mode 100644 index 641522f2..00000000 --- a/robustbench/model_info/cifar100/corruptions/Addepalli2022Efficient_WRN_34_10.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "https://artofrobust.github.io/short_paper/31.pdf", - "name": "Efficient and Effective Augmentation Strategy for Adversarial Training", - "authors": "Sravanti Addepalli, Samyak Jain, Venkatesh Babu Radhakrishnan", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar100", - "venue": "CVPRW 2022", - "architecture": "WideResNet-34-10", - "eps": null, - "clean_acc": "68.75", - "reported": "56.95", - "corruptions_acc": "56.95" -} \ No newline at end of file diff --git a/robustbench/model_info/cifar100/corruptions/Diffenderfer2021Winning_Binary.json b/robustbench/model_info/cifar100/corruptions/Diffenderfer2021Winning_Binary.json deleted file mode 100644 index 6bf80689..00000000 --- a/robustbench/model_info/cifar100/corruptions/Diffenderfer2021Winning_Binary.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2106.09129", - "name": "A Winning Hand: Compressing Deep Networks Can Improve Out-Of-Distribution Robustness", - "authors": "James Diffenderfer, Brian R. Bartoldson, Shreya Chaganti, Jize Zhang, Bhavya Kailkhura", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar100", - "venue": "NeurIPS 2021", - "architecture": "WideResNet-18-2", - "eps": null, - "clean_acc": "77.69", - "reported": "65.26", - "corruptions_acc": "65.26", - "footnote": "Binary weight network trained with AugMix and pruned to 95% sparsity" -} \ No newline at end of file diff --git a/robustbench/model_info/cifar100/corruptions/Diffenderfer2021Winning_Binary_CARD_Deck.json b/robustbench/model_info/cifar100/corruptions/Diffenderfer2021Winning_Binary_CARD_Deck.json deleted file mode 100644 index 8602c8ac..00000000 --- a/robustbench/model_info/cifar100/corruptions/Diffenderfer2021Winning_Binary_CARD_Deck.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2106.09129", - "name": "A Winning Hand: Compressing Deep Networks Can Improve Out-Of-Distribution Robustness", - "authors": "James Diffenderfer, Brian R. Bartoldson, Shreya Chaganti, Jize Zhang, Bhavya Kailkhura", - "additional_data": false, - "number_forward_passes": 6, - "dataset": "cifar100", - "venue": "NeurIPS 2021", - "architecture": "WideResNet-18-2", - "eps": null, - "clean_acc": "78.50", - "reported": "69.09", - "corruptions_acc": "69.09", - "footnote": "Ensemble of binary weight networks each of which are pruned to 95% sparsity" -} \ No newline at end of file diff --git a/robustbench/model_info/cifar100/corruptions/Diffenderfer2021Winning_LRR.json b/robustbench/model_info/cifar100/corruptions/Diffenderfer2021Winning_LRR.json deleted file mode 100644 index 71b6d114..00000000 --- a/robustbench/model_info/cifar100/corruptions/Diffenderfer2021Winning_LRR.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2106.09129", - "name": "A Winning Hand: Compressing Deep Networks Can Improve Out-Of-Distribution Robustness", - "authors": "James Diffenderfer, Brian R. Bartoldson, Shreya Chaganti, Jize Zhang, Bhavya Kailkhura", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar100", - "venue": "NeurIPS 2021", - "architecture": "WideResNet-18-2", - "eps": null, - "clean_acc": "78.41", - "reported": "66.45", - "corruptions_acc": "66.45", - "footnote": "Trained with AugMix and pruned to 95% sparsity" -} \ No newline at end of file diff --git a/robustbench/model_info/cifar100/corruptions/Diffenderfer2021Winning_LRR_CARD_Deck.json b/robustbench/model_info/cifar100/corruptions/Diffenderfer2021Winning_LRR_CARD_Deck.json deleted file mode 100644 index a8a5e3b5..00000000 --- a/robustbench/model_info/cifar100/corruptions/Diffenderfer2021Winning_LRR_CARD_Deck.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2106.09129", - "name": "A Winning Hand: Compressing Deep Networks Can Improve Out-Of-Distribution Robustness", - "authors": "James Diffenderfer, Brian R. Bartoldson, Shreya Chaganti, Jize Zhang, Bhavya Kailkhura", - "additional_data": false, - "number_forward_passes": 6, - "dataset": "cifar100", - "venue": "NeurIPS 2021", - "architecture": "WideResNet-18-2", - "eps": null, - "clean_acc": "79.93", - "reported": "71.08", - "corruptions_acc": "71.08", - "footnote": "Ensemble of networks each of which are pruned to 95% sparsity" -} \ No newline at end of file diff --git a/robustbench/model_info/cifar100/corruptions/Erichson2022NoisyMix.json b/robustbench/model_info/cifar100/corruptions/Erichson2022NoisyMix.json deleted file mode 100644 index c6341298..00000000 --- a/robustbench/model_info/cifar100/corruptions/Erichson2022NoisyMix.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "https://arxiv.org/pdf/2202.01263.pdf", - "name": "NoisyMix: Boosting Robustness by Combining Data Augmentations, Stability Training, and Noise Injections", - "authors": "N. Benjamin Erichson, Soon Hoe Lim, Francisco Utrera, Winnie Xu, Ziang Cao, and Michael W. Mahoney", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar100", - "venue": "arXiv, Feb 2022", - "architecture": "WideResNet-28-4", - "eps": null, - "clean_acc": "81.16", - "reported": "72.06", - "corruptions_acc": "72.06" -} \ No newline at end of file diff --git a/robustbench/model_info/cifar100/corruptions/Gowal2020Uncovering_Linf.json b/robustbench/model_info/cifar100/corruptions/Gowal2020Uncovering_Linf.json deleted file mode 100644 index 5f5f3203..00000000 --- a/robustbench/model_info/cifar100/corruptions/Gowal2020Uncovering_Linf.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2010.03593", - "name": "Uncovering the Limits of Adversarial Training against Norm-Bounded Adversarial Examples", - "authors": "Sven Gowal, Chongli Qin, Jonathan Uesato, Timothy Mann, Pushmeet Kohli", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Oct 2020", - "architecture": "WideResNet-70-16", - "eps": "8/255", - "clean_acc": "60.86", - "reported": "", - "corruptions_acc": "49.46", - "footnote": "Trained for \\(\\ell_{\\infty} \\) robustness with \\(\\varepsilon = 8/255\\)." -} diff --git a/robustbench/model_info/cifar100/corruptions/Gowal2020Uncovering_extra_Linf.json b/robustbench/model_info/cifar100/corruptions/Gowal2020Uncovering_extra_Linf.json deleted file mode 100644 index a4e97917..00000000 --- a/robustbench/model_info/cifar100/corruptions/Gowal2020Uncovering_extra_Linf.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2010.03593", - "name": "Uncovering the Limits of Adversarial Training against Norm-Bounded Adversarial Examples", - "authors": "Sven Gowal, Chongli Qin, Jonathan Uesato, Timothy Mann, Pushmeet Kohli", - "additional_data": true, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "arXiv, Oct 2020", - "architecture": "WideResNet-70-16", - "eps": "8/255", - "clean_acc": "69.15", - "reported": "", - "corruptions_acc": "56.00", - "footnote": "Trained for \\(\\ell_{\\infty} \\) robustness with \\(\\varepsilon = 8/255\\)." -} diff --git a/robustbench/model_info/cifar100/corruptions/Hendrycks2020AugMix_ResNeXt.json b/robustbench/model_info/cifar100/corruptions/Hendrycks2020AugMix_ResNeXt.json deleted file mode 100644 index d10a6886..00000000 --- a/robustbench/model_info/cifar100/corruptions/Hendrycks2020AugMix_ResNeXt.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "https://arxiv.org/abs/1912.02781", - "name": "AugMix: A Simple Data Processing Method to Improve Robustness and Uncertainty", - "authors": "Dan Hendrycks, Norman Mu, Ekin D. Cubuk, Barret Zoph, Justin Gilmer, Balaji Lakshminarayanan", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ICLR 2020", - "architecture": "ResNeXt29_32x4d", - "eps": null, - "clean_acc": "78.90", - "reported": "65.1", - "corruptions_acc": "65.14" -} diff --git a/robustbench/model_info/cifar100/corruptions/Hendrycks2020AugMix_WRN.json b/robustbench/model_info/cifar100/corruptions/Hendrycks2020AugMix_WRN.json deleted file mode 100644 index 404e1b72..00000000 --- a/robustbench/model_info/cifar100/corruptions/Hendrycks2020AugMix_WRN.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "https://arxiv.org/abs/1912.02781", - "name": "AugMix: A Simple Data Processing Method to Improve Robustness and Uncertainty", - "authors": "Dan Hendrycks, Norman Mu, Ekin D. Cubuk, Barret Zoph, Justin Gilmer, Balaji Lakshminarayanan", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar10", - "venue": "ICLR 2020", - "architecture": "WideResNet-40-2", - "eps": null, - "clean_acc": "76.28", - "reported": "64.1", - "corruptions_acc": "64.11" -} diff --git a/robustbench/model_info/cifar100/corruptions/Modas2021PRIMEResNet18.json b/robustbench/model_info/cifar100/corruptions/Modas2021PRIMEResNet18.json deleted file mode 100644 index 63e32e2d..00000000 --- a/robustbench/model_info/cifar100/corruptions/Modas2021PRIMEResNet18.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2112.13547", - "name": "PRIME: A Few Primitives Can Boost Robustness to Common Corruptions", - "authors": "Apostolos Modas, Rahul Rade, Guillermo Ortiz-Jiménez, Seyed-Mohsen Moosavi-Dezfooli, Pascal Frossard", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "cifar100", - "venue": "arXiv, Dec 2021", - "architecture": "ResNet-18", - "eps": null, - "clean_acc": "77.60", - "reported": "68.28", - "corruptions_acc": "68.28" -} \ No newline at end of file diff --git a/robustbench/model_info/cifar100/corruptions/unaggregated_results.csv b/robustbench/model_info/cifar100/corruptions/unaggregated_results.csv deleted file mode 100644 index 1e392794..00000000 --- a/robustbench/model_info/cifar100/corruptions/unaggregated_results.csv +++ /dev/null @@ -1,15 +0,0 @@ -,brightness,brightness,brightness,brightness,brightness,contrast,contrast,contrast,contrast,contrast,defocus_blur,defocus_blur,defocus_blur,defocus_blur,defocus_blur,elastic_transform,elastic_transform,elastic_transform,elastic_transform,elastic_transform,fog,fog,fog,fog,fog,frost,frost,frost,frost,frost,gaussian_noise,gaussian_noise,gaussian_noise,gaussian_noise,gaussian_noise,glass_blur,glass_blur,glass_blur,glass_blur,glass_blur,impulse_noise,impulse_noise,impulse_noise,impulse_noise,impulse_noise,jpeg_compression,jpeg_compression,jpeg_compression,jpeg_compression,jpeg_compression,motion_blur,motion_blur,motion_blur,motion_blur,motion_blur,pixelate,pixelate,pixelate,pixelate,pixelate,shot_noise,shot_noise,shot_noise,shot_noise,shot_noise,snow,snow,snow,snow,snow,zoom_blur,zoom_blur,zoom_blur,zoom_blur,zoom_blur -,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5 -Hendrycks2020AugMix_WRN,0.7621,0.7564,0.7446,0.7282,0.6836,0.7568,0.7246,0.6946,0.6414,0.4451,0.7625,0.7568,0.7432,0.7288,0.6796,0.7156,0.7212,0.7138,0.6565,0.5975,0.7598,0.7321,0.7048,0.6464,0.4863,0.7259,0.6818,0.6187,0.6097,0.5483,0.6563,0.5539999999999999,0.4482,0.3929,0.34299999999999997,0.5712,0.5851,0.6013,0.4561,0.4904,0.7306,0.6927,0.6308,0.5211,0.409,0.6764,0.635,0.6195,0.5988,0.5758,0.7441,0.7228,0.6949,0.6968,0.6642,0.7476,0.7198,0.6974,0.6095,0.4027,0.7045,0.6555,0.5406,0.4842,0.3995,0.7392,0.6732,0.6672,0.6256,0.5862,0.7390000000000001,0.7383,0.7261,0.7115,0.6766 -Hendrycks2020AugMix_ResNeXt,0.7865,0.7766,0.7664,0.7488,0.7047,0.7819,0.7521,0.7192,0.6573,0.449,0.7894,0.7806,0.7677,0.7516,0.7068,0.7441,0.7499,0.7368,0.6895,0.6277,0.7845,0.7612,0.7225,0.6584,0.4970000000000001,0.7419,0.6973,0.6214,0.6108,0.5419,0.642,0.4926,0.3653,0.312,0.27,0.5439,0.5658,0.6061,0.3951,0.4589,0.7758,0.7531,0.7312,0.6804,0.6063,0.7049,0.6579,0.6412,0.6245,0.5875,0.7699,0.7509,0.7229,0.7240000000000001,0.6919,0.7654,0.723,0.6908,0.5225,0.2531,0.7091,0.6357,0.4715,0.4062,0.3199,0.7653,0.6992,0.6962,0.6519,0.6051,0.7676,0.7658,0.7545,0.7396,0.7124 -Gowal2020Uncovering_extra_Linf,0.6953,0.6889,0.6682,0.6373,0.5502,0.6424,0.439,0.2767,0.1292,0.028999999999999998,0.6819,0.66,0.6338,0.6076,0.5464,0.6214,0.626,0.6084,0.5987,0.5896,0.6636,0.5579999999999999,0.4284,0.2858,0.1115,0.6586,0.6052,0.5436,0.544,0.4944,0.6761,0.6391,0.5585,0.5118,0.4611,0.6298,0.6331,0.6224,0.5577,0.5706,0.5605,0.4392,0.3503,0.2347,0.1585,0.6708,0.6592,0.6537,0.649,0.6431,0.648,0.61,0.5652,0.5658,0.5228,0.6833,0.6719,0.6696,0.6543,0.6298,0.6835,0.6713,0.6052,0.5551,0.4641,0.6787,0.6488,0.6194,0.5672,0.5231,0.6366,0.6337,0.6144,0.5987,0.5751 -Gowal2020Uncovering_Linf,0.6072,0.5961,0.5709,0.5365,0.4255,0.5491,0.3384,0.199,0.0852,0.024,0.5977,0.5749,0.5474,0.5238,0.4732,0.5427,0.5443,0.5305,0.522,0.5202,0.5768,0.474,0.3517,0.2371,0.0972,0.5629,0.4944,0.4274,0.4364,0.3866,0.6021,0.5870000000000001,0.5416,0.5133,0.477,0.5702,0.5702,0.5537,0.5239,0.5195,0.5467,0.4708,0.4059,0.29100000000000004,0.2061,0.5879,0.5798,0.573,0.5716,0.564,0.5657,0.5263,0.4922,0.4868,0.4481,0.6013,0.5924,0.5893,0.5784,0.5553,0.6051,0.5982,0.5646,0.5361,0.4741,0.5961,0.5708,0.5441,0.491,0.4353,0.5512,0.5448,0.5291,0.5167,0.4939 -Diffenderfer2021Winning_LRR,0.7839,0.7777,0.7698,0.7561,0.7190000000000001,0.7795,0.755,0.7365,0.6956,0.5132,0.7831,0.7792,0.7676,0.7543,0.7166,0.7433,0.7465,0.7389,0.6867,0.6159,0.7809,0.7563,0.7239,0.6629,0.4926,0.7489,0.7036,0.6499,0.6393,0.5849,0.6789,0.5512,0.4253,0.369,0.3066,0.5685,0.584,0.6018,0.4554,0.4816,0.7621,0.7275,0.684,0.5707,0.4413,0.7048,0.6574,0.6406,0.6212,0.599,0.7696,0.7497,0.7237,0.7272,0.6970000000000001,0.7703,0.7509999999999999,0.7337,0.6732,0.5564,0.7244,0.6729999999999999,0.5276,0.4674,0.3699,0.7568,0.7040000000000001,0.6920000000000001,0.6499,0.6164,0.7672,0.7613,0.7509999999999999,0.7341,0.7027 -Diffenderfer2021Winning_LRR_CARD_Deck,0.7987,0.7916,0.7847,0.7677,0.7263,0.7903,0.7485,0.7095,0.6468,0.4467,0.7961,0.7871,0.7747,0.7536,0.7083,0.7511,0.7581,0.7512,0.7091,0.6592,0.7915,0.764,0.7144,0.6391,0.4421,0.7724,0.7357,0.6893,0.6816,0.6316,0.7501,0.7138,0.6822,0.6723,0.6593,0.6286,0.6407,0.6605,0.5044,0.5435,0.7853,0.765,0.7406,0.6707,0.5678,0.7518,0.7286,0.7143,0.7048,0.6869,0.7722,0.7532,0.7182,0.7207,0.6834,0.7885,0.775,0.7657,0.7238,0.6348,0.768,0.7501,0.7108,0.6977,0.6689,0.7807,0.7314,0.7159,0.6697,0.6512,0.7712,0.769,0.7557,0.7388,0.7082 -Diffenderfer2021Winning_Binary,0.775,0.7663,0.7552,0.7393,0.6926,0.7676,0.731,0.6997,0.6415,0.4229,0.7757,0.769,0.7559,0.7367,0.7016,0.7273,0.7297,0.7202,0.6669,0.5902,0.7713,0.7455,0.7085,0.6476,0.4861,0.7302,0.6761,0.6069,0.5944,0.5288,0.6941,0.621,0.5355,0.4901,0.449,0.5237,0.5343,0.5565,0.4216,0.4399,0.7272,0.6747,0.6146,0.4724,0.3391,0.6998,0.6542,0.6374,0.6172,0.5921,0.7531,0.7305,0.7061,0.7051,0.6709,0.7603,0.7390000000000001,0.7208,0.6758,0.5785,0.73,0.6946,0.6028,0.5584,0.4863,0.7399,0.6605,0.6583,0.6162,0.5749,0.7507,0.7466,0.7340000000000001,0.7159,0.6831 -Diffenderfer2021Winning_Binary_CARD_Deck,0.7855,0.7767,0.7644,0.7451,0.6893,0.7759,0.7198,0.6681,0.5857,0.3261,0.7827,0.7705,0.7541,0.7287,0.6759999999999999,0.7357,0.7414,0.7289,0.6844,0.6317,0.7836,0.7472,0.7023,0.6265,0.4314,0.7548,0.7126,0.6598,0.6597,0.6001,0.7553,0.7304,0.7058,0.691,0.6814,0.6048,0.6125,0.6329,0.4737,0.5094,0.7633,0.7335,0.7047,0.6216,0.5325,0.7478,0.7191,0.7049,0.6915,0.6774,0.7572,0.7303,0.6945,0.6936,0.655,0.7728,0.7573,0.7429,0.7002,0.5959,0.7677,0.7533,0.7239,0.716,0.6867,0.7592,0.7016,0.6799,0.6409,0.6184,0.753,0.7471,0.733,0.7139,0.6824 -Addepalli2021Towards_PARN18,0.6211,0.6115,0.5906,0.5595,0.4787,0.5629,0.3907,0.2686,0.14300000000000002,0.0373,0.6104,0.5829,0.5523,0.5304,0.4789,0.5529999999999999,0.5536,0.5365,0.5294,0.5209,0.5897,0.4889,0.3733,0.2549,0.1087,0.5859,0.5368,0.4695,0.4705,0.4164,0.6182,0.6038,0.5716,0.5417,0.5124,0.5705,0.5737,0.5565,0.5156,0.5163,0.5998,0.5624,0.5147,0.4023,0.294,0.6041,0.5953,0.5895,0.5868,0.5796,0.5761,0.5382,0.4999,0.4975,0.4624,0.6132,0.603,0.5997,0.5869,0.5629,0.6205,0.6168,0.5877,0.5649,0.5113,0.6122,0.5904,0.564,0.5242,0.4817,0.564,0.557,0.541,0.5303,0.5081 -Addepalli2021Towards_WRN34,0.6612,0.6483,0.629,0.601,0.5244,0.601,0.413,0.2831,0.1501,0.0367,0.6479,0.6225,0.5948,0.5679,0.5046,0.5824,0.5841,0.5723,0.5581,0.5497,0.6252,0.521,0.3998,0.2813,0.1164,0.6244,0.5759,0.508,0.507,0.4457,0.6502,0.6324,0.5878,0.5586,0.5251,0.5992,0.5995,0.5866,0.5378,0.5375,0.6286,0.592,0.551,0.4678,0.3738,0.6358,0.6241,0.6194,0.6153,0.6076,0.6104,0.5701,0.5238,0.5243,0.4813,0.6477,0.6413,0.6321,0.6204,0.5883,0.6534,0.65,0.6141,0.5872,0.5317,0.6522,0.6274,0.6021,0.5634,0.5248,0.5955,0.5896,0.5718,0.5582,0.5327 -Modas2021PRIMEResNet18,0.7739,0.7698,0.7601,0.7442,0.7095,0.7709,0.7508,0.7316,0.7032,0.604,0.7733,0.7593,0.7371,0.707,0.639,0.7184,0.7261,0.7147,0.6767,0.6272,0.7717,0.7484,0.7074,0.6518,0.4926,0.7423,0.7127,0.6769,0.6801,0.6395,0.7315,0.7068,0.6725,0.6565,0.6407,0.623,0.6319,0.6361,0.5417,0.5471,0.7361,0.68,0.6207,0.5216,0.4336,0.7184,0.677,0.6553,0.6399,0.6178,0.7458,0.7153,0.6769,0.6785,0.6433,0.7563,0.7286,0.7057,0.6267,0.5212,0.7488,0.7284,0.7001,0.6911,0.6648,0.7527,0.7045,0.6824,0.6457,0.6284,0.7428,0.7341,0.718,0.697,0.6658 -Erichson2022NoisyMix,0.7887,0.7659,0.6938,0.664,0.597,0.7888,0.7653,0.7323,0.7305,0.6957,0.7925,0.74,0.7366,0.7,0.656,0.7977,0.7792,0.7606,0.6877,0.5489,0.7666,0.7109,0.6427,0.6046,0.5732,0.8101,0.8017,0.7868,0.7634,0.7187,0.8097,0.8034,0.7896,0.7744,0.7241,0.81,0.7935,0.758,0.7166,0.5775,0.7809,0.7793,0.766,0.7496,0.721,0.7835,0.7417,0.6895,0.6884,0.6334,0.6286,0.6355,0.6528,0.5229,0.5414,0.8111,0.8104,0.8087,0.8033,0.8018,0.8071,0.7716,0.7376,0.6672,0.3828,0.7563,0.7151,0.6997,0.6826,0.6571,0.7624,0.7724,0.7612,0.7121,0.6538 -Addepalli2022Efficient_WRN_34_10,0.6816,0.6689,0.6102,0.5803,0.5064,0.6409,0.5995,0.5611,0.5587,0.5168,0.6695,0.6484,0.6201,0.5767,0.5478,0.6785,0.6701,0.6641,0.649,0.6196,0.6762,0.6414,0.578,0.5358,0.497,0.6749,0.652,0.6244,0.6013,0.5432,0.6855,0.6786,0.663,0.6389,0.5722,0.6583,0.569,0.4518,0.3217,0.1393,0.6286,0.62,0.6027,0.5904,0.5669,0.6558,0.6155,0.5626,0.5606,0.5112,0.6163,0.6185,0.6118,0.5383,0.552,0.617,0.5381,0.4629,0.3326,0.2355,0.6424,0.4941,0.3669,0.2089,0.0425,0.6658,0.6531,0.6487,0.6427,0.637,0.6138,0.6131,0.6048,0.589,0.5807 diff --git a/robustbench/model_info/imagenet/Linf/Debenedetti2022Light_XCiT-L12.json b/robustbench/model_info/imagenet/Linf/Debenedetti2022Light_XCiT-L12.json deleted file mode 100644 index c990ffca..00000000 --- a/robustbench/model_info/imagenet/Linf/Debenedetti2022Light_XCiT-L12.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2209.07399", - "name": "A Light Recipe to Train Robust Vision Transformers", - "authors": "Edoardo Debenedetti, Vikash Sehwag, Prateek Mittal", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "imagenet", - "venue": "arXiv, Sep 2022", - "architecture": "XCiT-L12", - "eps": "4/255", - "clean_acc": "73.76", - "reported": "47.60", - "autoattack_acc": "47.60", - "unreliable": false -} diff --git a/robustbench/model_info/imagenet/Linf/Debenedetti2022Light_XCiT-M12.json b/robustbench/model_info/imagenet/Linf/Debenedetti2022Light_XCiT-M12.json deleted file mode 100644 index 0d91c4c7..00000000 --- a/robustbench/model_info/imagenet/Linf/Debenedetti2022Light_XCiT-M12.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2209.07399", - "name": "A Light Recipe to Train Robust Vision Transformers", - "authors": "Edoardo Debenedetti, Vikash Sehwag, Prateek Mittal", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "imagenet", - "venue": "arXiv, Sep 2022", - "architecture": "XCiT-M12", - "eps": "4/255", - "clean_acc": "74.04", - "reported": "45.24", - "autoattack_acc": "45.24", - "unreliable": false -} diff --git a/robustbench/model_info/imagenet/Linf/Debenedetti2022Light_XCiT-S12.json b/robustbench/model_info/imagenet/Linf/Debenedetti2022Light_XCiT-S12.json deleted file mode 100644 index 1bb4b367..00000000 --- a/robustbench/model_info/imagenet/Linf/Debenedetti2022Light_XCiT-S12.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2209.07399", - "name": "A Light Recipe to Train Robust Vision Transformers", - "authors": "Edoardo Debenedetti, Vikash Sehwag, Prateek Mittal", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "imagenet", - "venue": "arXiv, Sep 2022", - "architecture": "XCiT-S12", - "eps": "4/255", - "clean_acc": "72.34", - "reported": "41.78", - "autoattack_acc": "41.78", - "unreliable": false -} diff --git a/robustbench/model_info/imagenet/Linf/Engstrom2019Robustness.json b/robustbench/model_info/imagenet/Linf/Engstrom2019Robustness.json deleted file mode 100644 index 7e12e08d..00000000 --- a/robustbench/model_info/imagenet/Linf/Engstrom2019Robustness.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://github.com/MadryLab/robustness", - "name": "Robustness library", - "authors": "Logan Engstrom, Andrew Ilyas, Hadi Salman, Shibani Santurkar, Dimitris Tsipras", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "imagenet", - "venue": "GitHub,
Oct 2019", - "architecture": "ResNet-50", - "eps": "4/255", - "clean_acc": "62.56", - "reported": "33.38", - "autoattack_acc": "29.22", - "unreliable": false -} diff --git a/robustbench/model_info/imagenet/Linf/Salman2020Do_50_2.json b/robustbench/model_info/imagenet/Linf/Salman2020Do_50_2.json deleted file mode 100644 index 953f4421..00000000 --- a/robustbench/model_info/imagenet/Linf/Salman2020Do_50_2.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2007.08489", - "name": "Do Adversarially Robust ImageNet Models Transfer Better?", - "authors": "Hadi Salman, Andrew Ilyas, Logan Engstrom, Ashish Kapoor, Aleksander Madry", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "imagenet", - "venue": "NeurIPS 2020", - "architecture": "WideResNet-50-2", - "eps": "4/255", - "clean_acc": "68.46", - "reported": "", - "autoattack_acc": "38.14", - "unreliable": false -} diff --git a/robustbench/model_info/imagenet/Linf/Salman2020Do_R18.json b/robustbench/model_info/imagenet/Linf/Salman2020Do_R18.json deleted file mode 100644 index cf065fae..00000000 --- a/robustbench/model_info/imagenet/Linf/Salman2020Do_R18.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2007.08489", - "name": "Do Adversarially Robust ImageNet Models Transfer Better?", - "authors": "Hadi Salman, Andrew Ilyas, Logan Engstrom, Ashish Kapoor, Aleksander Madry", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "imagenet", - "venue": "NeurIPS 2020", - "architecture": "ResNet-18", - "eps": "4/255", - "clean_acc": "52.92", - "reported": "", - "autoattack_acc": "25.32", - "unreliable": false -} diff --git a/robustbench/model_info/imagenet/Linf/Salman2020Do_R50.json b/robustbench/model_info/imagenet/Linf/Salman2020Do_R50.json deleted file mode 100644 index 2233637e..00000000 --- a/robustbench/model_info/imagenet/Linf/Salman2020Do_R50.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2007.08489", - "name": "Do Adversarially Robust ImageNet Models Transfer Better?", - "authors": "Hadi Salman, Andrew Ilyas, Logan Engstrom, Ashish Kapoor, Aleksander Madry", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "imagenet", - "venue": "NeurIPS 2020", - "architecture": "ResNet-50", - "eps": "4/255", - "clean_acc": "64.02", - "reported": "", - "autoattack_acc": "34.96", - "unreliable": false -} diff --git a/robustbench/model_info/imagenet/Linf/Standard_R50.json b/robustbench/model_info/imagenet/Linf/Standard_R50.json deleted file mode 100644 index 6764060b..00000000 --- a/robustbench/model_info/imagenet/Linf/Standard_R50.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "link": "https://github.com/RobustBench/robustbench/", - "name": "Standardly trained model", - "authors": "Torchvision", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "imagenet", - "venue": "N/A", - "architecture": "ResNet-50", - "eps": "4/255", - "clean_acc": "76.52", - "reported": "0.0", - "autoattack_acc": "0.0", - "unreliable": false -} diff --git a/robustbench/model_info/imagenet/Linf/Wong2020Fast.json b/robustbench/model_info/imagenet/Linf/Wong2020Fast.json deleted file mode 100644 index 88423baa..00000000 --- a/robustbench/model_info/imagenet/Linf/Wong2020Fast.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2001.03994", - "name": "Fast is better than free: Revisiting adversarial training", - "authors": "Eric Wong, Leslie Rice, J. Zico Kolter", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "imagenet", - "venue": "ICLR 2020", - "architecture": "ResNet-50", - "eps": "4/255", - "clean_acc": "55.62", - "reported": "30.18", - "footnote": "Focuses on fast adversarial training.", - "autoattack_acc": "26.24", - "unreliable": false -} diff --git a/robustbench/model_info/imagenet/corruptions/Erichson2022NoisyMix.json b/robustbench/model_info/imagenet/corruptions/Erichson2022NoisyMix.json deleted file mode 100644 index 7c63e990..00000000 --- a/robustbench/model_info/imagenet/corruptions/Erichson2022NoisyMix.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "link": "https://arxiv.org/pdf/2202.01263.pdf", - "name": "NoisyMix: Boosting Robustness by Combining Data Augmentations, Stability Training, and Noise Injections", - "authors": "N. Benjamin Erichson, Soon Hoe Lim, Francisco Utrera, Winnie Xu, Ziang Cao, and Michael W. Mahoney", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "imagenet", - "venue": "arXiv, Feb 2022", - "architecture": "ResNet-50", - "eps": null, - "clean_acc": "77.14", - "reported": "52.25", - "corruptions_acc_2d": "52.25", - "corruptions_acc_3d": "53.10", - "corruptions_mce_2d": "60.70", - "corruptions_mce_3d": "62.33" -} diff --git a/robustbench/model_info/imagenet/corruptions/Geirhos2018_SIN.json b/robustbench/model_info/imagenet/corruptions/Geirhos2018_SIN.json deleted file mode 100644 index 1c6e05bc..00000000 --- a/robustbench/model_info/imagenet/corruptions/Geirhos2018_SIN.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "link": "https://arxiv.org/abs/1811.12231", - "name": "ImageNet-trained CNNs are biased towards texture; increasing shape bias improves accuracy and robustness", - "authors": "Robert Geirhos, Patricia Rubisch, Claudio Michaelis, Matthias Bethge, Felix A. Wichmann, Wieland Brendel", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "imagenet", - "venue": "ICLR 2019", - "architecture": "ResNet-50", - "eps": null, - "clean_acc": "60.24", - "reported": "", - "corruptions_acc_2d": "39.53", - "corruptions_acc_3d": "37.80", - "corruptions_mce_2d": "77.14", - "corruptions_mce_3d": "83.33", - "footnote": "Model A: trained on Stylized ImageNet." -} diff --git a/robustbench/model_info/imagenet/corruptions/Geirhos2018_SIN_IN.json b/robustbench/model_info/imagenet/corruptions/Geirhos2018_SIN_IN.json deleted file mode 100644 index 67677359..00000000 --- a/robustbench/model_info/imagenet/corruptions/Geirhos2018_SIN_IN.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "link": "https://arxiv.org/abs/1811.12231", - "name": "ImageNet-trained CNNs are biased towards texture; increasing shape bias improves accuracy and robustness", - "authors": "Robert Geirhos, Patricia Rubisch, Claudio Michaelis, Matthias Bethge, Felix A. Wichmann, Wieland Brendel", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "imagenet", - "venue": "ICLR 2019", - "architecture": "ResNet-50", - "eps": null, - "clean_acc": "74.88", - "reported": "", - "corruptions_acc_2d": "45.52", - "corruptions_acc_3d": "48.09", - "corruptions_mce_2d": "68.95", - "corruptions_mce_3d": "68.89", - "footnote": "Model B: trained on Stylized ImageNet and standard ImageNet." -} diff --git a/robustbench/model_info/imagenet/corruptions/Geirhos2018_SIN_IN_IN.json b/robustbench/model_info/imagenet/corruptions/Geirhos2018_SIN_IN_IN.json deleted file mode 100644 index 1368aa6f..00000000 --- a/robustbench/model_info/imagenet/corruptions/Geirhos2018_SIN_IN_IN.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "link": "https://arxiv.org/abs/1811.12231", - "name": "ImageNet-trained CNNs are biased towards texture; increasing shape bias improves accuracy and robustness", - "authors": "Robert Geirhos, Patricia Rubisch, Claudio Michaelis, Matthias Bethge, Felix A. Wichmann, Wieland Brendel", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "imagenet", - "venue": "ICLR 2019", - "architecture": "ResNet-50", - "eps": null, - "clean_acc": "77.44", - "reported": "", - "corruptions_acc_2d": "41.72", - "corruptions_acc_3d": "46.92", - "corruptions_mce_2d": "73.52", - "corruptions_mce_3d": "70.11", - "footnote": "Model C: trained on Stylized ImageNet and standard ImageNet, then fine-tuned on standard ImageNet." -} diff --git a/robustbench/model_info/imagenet/corruptions/Hendrycks2020AugMix.json b/robustbench/model_info/imagenet/corruptions/Hendrycks2020AugMix.json deleted file mode 100644 index 2c353543..00000000 --- a/robustbench/model_info/imagenet/corruptions/Hendrycks2020AugMix.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "link": "https://arxiv.org/abs/1912.02781", - "name": "AugMix: A Simple Data Processing Method to Improve Robustness and Uncertainty", - "authors": "Dan Hendrycks, Norman Mu, Ekin D. Cubuk, Barret Zoph, Justin Gilmer, Balaji Lakshminarayanan", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "imagenet", - "venue": "ICLR 2020", - "architecture": "ResNet-50", - "eps": null, - "clean_acc": "76.98", - "reported": "", - "corruptions_acc_2d": "48.31", - "corruptions_acc_2d": "51.35", - "corruptions_acc_2d": "65.33", - "corruptions_acc_2d": "64.51" -} diff --git a/robustbench/model_info/imagenet/corruptions/Hendrycks2020Many.json b/robustbench/model_info/imagenet/corruptions/Hendrycks2020Many.json deleted file mode 100644 index 4be0c7e9..00000000 --- a/robustbench/model_info/imagenet/corruptions/Hendrycks2020Many.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2006.16241", - "name": "The Many Faces of Robustness: A Critical Analysis of Out-of-Distribution Generalization", - "authors": "Dan Hendrycks, Steven Basart, Norman Mu, Saurav Kadavath, Frank Wang, Evan Dorundo, Rahul Desai, Tyler Zhu, Samyak Parajuli, Mike Guo, Dawn Song, Jacob Steinhardt, Justin Gilmer", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "imagenet", - "venue": "ICCV 2021", - "architecture": "ResNet-50", - "eps": null, - "clean_acc": "76.88", - "reported": "", - "corruptions_acc_2d": "52.65", - "corruptions_acc_3d": "54.13", - "corruptions_mce_2d": "60.32", - "corruptions_mce_3d": "61.02" -} diff --git a/robustbench/model_info/imagenet/corruptions/Salman2020Do_50_2_Linf.json b/robustbench/model_info/imagenet/corruptions/Salman2020Do_50_2_Linf.json deleted file mode 100644 index f4c49af3..00000000 --- a/robustbench/model_info/imagenet/corruptions/Salman2020Do_50_2_Linf.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "link": "https://arxiv.org/abs/2007.08489", - "name": "Do Adversarially Robust ImageNet Models Transfer Better?", - "authors": "Hadi Salman, Andrew Ilyas, Logan Engstrom, Ashish Kapoor, Aleksander Madry", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "imagenet", - "venue": "NeurIPS 2020", - "architecture": "WideResNet-50-2", - "eps": "4/255", - "clean_acc": "68.46", - "reported": "", - "corruptions_acc_2d": "35.69", - "corruptions_acc_3d": "39.71", - "corruptions_mce_2d": "80.42", - "corruptions_mce_3d": "79.65" -} diff --git a/robustbench/model_info/imagenet/corruptions/Standard_R50.json b/robustbench/model_info/imagenet/corruptions/Standard_R50.json deleted file mode 100644 index d49c9b0b..00000000 --- a/robustbench/model_info/imagenet/corruptions/Standard_R50.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "link": "https://github.com/RobustBench/robustbench/", - "name": "Standardly trained model", - "authors": "Torchvision", - "additional_data": false, - "number_forward_passes": 1, - "dataset": "imagenet", - "venue": "N/A", - "architecture": "ResNet-50", - "eps": null, - "clean_acc": "76.52", - "reported": "", - "corruptions_acc_2d": "39.23", - "corruptions_acc_3d": "44.77", - "corruptions_mce_2d": "76.64", - "corruptions_mce_3d": "72.95" -} diff --git a/robustbench/model_info/imagenet/corruptions/unaggregated_results_2d.csv b/robustbench/model_info/imagenet/corruptions/unaggregated_results_2d.csv deleted file mode 100644 index 672791cb..00000000 --- a/robustbench/model_info/imagenet/corruptions/unaggregated_results_2d.csv +++ /dev/null @@ -1,10 +0,0 @@ -,shot_noise,shot_noise,shot_noise,shot_noise,shot_noise,impulse_noise,impulse_noise,impulse_noise,impulse_noise,impulse_noise,defocus_blur,defocus_blur,defocus_blur,defocus_blur,defocus_blur,contrast,contrast,contrast,contrast,contrast,brightness,brightness,brightness,brightness,brightness,jpeg_compression,jpeg_compression,jpeg_compression,jpeg_compression,jpeg_compression,pixelate,pixelate,pixelate,pixelate,pixelate,glass_blur,glass_blur,glass_blur,glass_blur,glass_blur,motion_blur,motion_blur,motion_blur,motion_blur,motion_blur,zoom_blur,zoom_blur,zoom_blur,zoom_blur,zoom_blur,gaussian_noise,gaussian_noise,gaussian_noise,gaussian_noise,gaussian_noise,fog,fog,fog,fog,fog,frost,frost,frost,frost,frost,snow,snow,snow,snow,snow,elastic_transform,elastic_transform,elastic_transform,elastic_transform,elastic_transform,average -,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5, -Erichson2022NoisyMix,0.6872,0.6376,0.5588,0.4222,0.308,0.6658,0.6166,0.5714,0.4416,0.2832,0.638,0.5848,0.4716,0.3758,0.268,0.7098,0.687,0.6226,0.4306,0.1764,0.7552,0.7426,0.7196,0.6846,0.6312,0.6978,0.6728,0.6584,0.6088,0.529,0.6932,0.6658,0.5732,0.4146,0.315,0.617,0.5126,0.2944,0.2498,0.2014,0.705,0.65,0.5494,0.4054,0.3048,0.6004,0.528,0.4906,0.4174,0.3396,0.6946,0.653,0.569,0.459,0.276,0.663,0.618,0.538,0.4884,0.3322,0.687,0.5814,0.4858,0.4718,0.4004,0.6376,0.4614,0.4868,0.3742,0.2884,0.6932,0.4926,0.6238,0.5218,0.3092,0.522549333 -Hendrycks2020Many,0.7234,0.6726,0.5924,0.4374,0.3192,0.706,0.658,0.6136,0.4968,0.37,0.6898,0.6476,0.5304,0.4154,0.3016,0.7004,0.6562,0.5742,0.3546,0.1268,0.7524,0.741,0.7214,0.694,0.6498,0.7042,0.6786,0.6568,0.5476,0.3696,0.7522,0.7498,0.7214,0.636,0.4246,0.673,0.5734,0.3132,0.2594,0.1868,0.705,0.6482,0.5168,0.3326,0.2188,0.5778,0.4734,0.3922,0.316,0.2378,0.726,0.6896,0.6116,0.48,0.33,0.647,0.59,0.5222,0.4832,0.3472,0.667,0.5548,0.4638,0.436,0.374,0.6086,0.4618,0.4654,0.361,0.3024,0.6984,0.4714,0.623,0.5094,0.2512,0.526469333 -Hendrycks2020AugMix,0.677,0.5828,0.4524,0.228,0.1152,0.6442,0.5388,0.4414,0.2134,0.0536,0.6428,0.6042,0.5028,0.3836,0.2566,0.7066,0.6772,0.6096,0.4158,0.1572,0.7476,0.7334,0.7048,0.6672,0.6064,0.6814,0.6492,0.6314,0.5632,0.4782,0.7006,0.6842,0.5934,0.4744,0.4078,0.6048,0.4828,0.2678,0.223,0.1706,0.7176,0.674,0.5762,0.4134,0.2904,0.6168,0.5532,0.516,0.4308,0.3534,0.6776,0.6068,0.4534,0.2412,0.057,0.6328,0.5792,0.4816,0.4138,0.2386,0.6466,0.4888,0.3672,0.3492,0.2784,0.6096,0.414,0.4278,0.3062,0.2318,0.6914,0.4802,0.6274,0.5262,0.2862,0.483069333 -Geirhos2018_SIN_IN,0.6352,0.554,0.4464,0.2598,0.1546,0.5696,0.5004,0.43,0.2652,0.1204,0.588,0.5392,0.4436,0.3396,0.2508,0.65,0.6094,0.5144,0.3162,0.1022,0.7286,0.714,0.6904,0.6586,0.6018,0.662,0.6364,0.6162,0.5426,0.4288,0.6788,0.6724,0.5768,0.4692,0.404,0.5712,0.4676,0.2438,0.2016,0.1538,0.6512,0.577,0.4696,0.3336,0.2504,0.5164,0.4244,0.367,0.303,0.2312,0.6402,0.5796,0.4504,0.2972,0.1302,0.6142,0.5648,0.474,0.4164,0.2776,0.6078,0.463,0.3522,0.3342,0.278,0.5812,0.3948,0.447,0.342,0.2774,0.653,0.4488,0.605,0.5114,0.2684,0.455202667 -Geirhos2018_SIN_IN_IN,0.6082,0.4768,0.3058,0.1228,0.0562,0.5394,0.4158,0.3126,0.1322,0.0462,0.6022,0.5414,0.418,0.3026,0.211,0.6602,0.6136,0.508,0.2694,0.078,0.75,0.7302,0.7072,0.658,0.5984,0.6682,0.6358,0.599,0.5122,0.378,0.6476,0.641,0.4782,0.3258,0.2376,0.5654,0.4232,0.2044,0.1544,0.1138,0.6634,0.5678,0.4158,0.26,0.1766,0.5294,0.4332,0.3636,0.2854,0.225,0.6236,0.504,0.3198,0.1508,0.0464,0.6282,0.5606,0.4698,0.3978,0.231,0.6292,0.453,0.3354,0.3192,0.2582,0.5714,0.3382,0.3864,0.2716,0.2012,0.6826,0.4662,0.5878,0.4698,0.2186,0.417197333 -Standard_R50,0.5744,0.4294,0.2528,0.0792,0.029,0.4844,0.358,0.2546,0.083,0.0182,0.5956,0.5152,0.382,0.2748,0.183,0.6494,0.5852,0.469,0.2122,0.0546,0.7392,0.7244,0.6956,0.6498,0.5866,0.6664,0.6244,0.5982,0.4728,0.3144,0.6402,0.6426,0.4598,0.2898,0.2068,0.54,0.4012,0.1688,0.131,0.1024,0.6382,0.537,0.3758,0.2154,0.1482,0.529,0.4324,0.358,0.2878,0.2206,0.5978,0.4686,0.2748,0.1128,0.0218,0.6138,0.5612,0.4704,0.4088,0.241,0.615,0.4346,0.309,0.2896,0.2294,0.5498,0.3154,0.3518,0.2382,0.1654,0.6704,0.4556,0.5526,0.4196,0.1746,0.392304 -Geirhos2018_SIN,0.4924,0.4196,0.3604,0.254,0.1954,0.4508,0.3948,0.353,0.2732,0.1936,0.4132,0.3352,0.2638,0.201,0.1452,0.5636,0.5436,0.5056,0.4114,0.2626,0.5956,0.573,0.558,0.5376,0.5074,0.5522,0.5178,0.488,0.404,0.3178,0.5704,0.5736,0.4844,0.3974,0.3624,0.4922,0.4036,0.247,0.2042,0.157,0.5246,0.455,0.3704,0.2854,0.2402,0.3854,0.3188,0.303,0.2408,0.187,0.5222,0.4638,0.39,0.299,0.1854,0.5386,0.5066,0.4564,0.4292,0.3638,0.5044,0.4158,0.3564,0.3378,0.298,0.488,0.3844,0.3978,0.331,0.3126,0.5314,0.3522,0.5598,0.53,0.4056,0.395290667 -Salman2020Do_50_2_Linf,0.5622,0.3802,0.203,0.0608,0.0222,0.3818,0.2122,0.12,0.033,0.0098,0.4628,0.377,0.24,0.1568,0.102,0.3508,0.1528,0.0256,0.0054,0.0054,0.6784,0.6588,0.6168,0.5496,0.4518,0.6634,0.6546,0.6528,0.6384,0.616,0.6618,0.6546,0.6274,0.5772,0.5408,0.56,0.4712,0.3832,0.3036,0.1898,0.584,0.49,0.3614,0.2318,0.1692,0.5238,0.4578,0.3818,0.3362,0.2744,0.5898,0.424,0.2038,0.0678,0.0142,0.242,0.1026,0.0396,0.0304,0.0108,0.5912,0.4304,0.2818,0.2566,0.1836,0.5512,0.3892,0.3752,0.2484,0.2356,0.5886,0.4088,0.6244,0.5886,0.4642,0.356896 \ No newline at end of file diff --git a/robustbench/model_info/imagenet/corruptions/unaggregated_results_3d.csv b/robustbench/model_info/imagenet/corruptions/unaggregated_results_3d.csv deleted file mode 100644 index 2a0a13e1..00000000 --- a/robustbench/model_info/imagenet/corruptions/unaggregated_results_3d.csv +++ /dev/null @@ -1,10 +0,0 @@ -,bit_error,bit_error,bit_error,bit_error,bit_error,color_quant,color_quant,color_quant,color_quant,color_quant,near_focus,near_focus,near_focus,near_focus,near_focus,far_focus,far_focus,far_focus,far_focus,far_focus,flash,flash,flash,flash,flash,fog_3d,fog_3d,fog_3d,fog_3d,fog_3d,h265_abr,h265_abr,h265_abr,h265_abr,h265_abr,h265_crf,h265_crf,h265_crf,h265_crf,h265_crf,iso_noise,iso_noise,iso_noise,iso_noise,iso_noise,low_light,low_light,low_light,low_light,low_light,xy_motion_blur,xy_motion_blur,xy_motion_blur,xy_motion_blur,xy_motion_blur,z_motion_blur,z_motion_blur,z_motion_blur,z_motion_blur,z_motion_blur,average -,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5,1,2,3,4,5, -Erichson2022NoisyMix,0.6174,0.5534,0.463,0.3836,0.2198,0.7484,0.7442,0.7226,0.6396,0.3932,0.6982,0.6724,0.6428,0.6128,0.586,0.7018,0.6492,0.5938,0.5466,0.5118,0.5562,0.5202,0.471,0.3998,0.292,0.7082,0.5966,0.4794,0.388,0.3136,0.674,0.6414,0.541,0.3772,0.3788,0.675,0.6734,0.6424,0.5424,0.5044,0.6178,0.5986,0.5612,0.498,0.39,0.6356,0.612,0.589,0.5338,0.4316,0.6056,0.5154,0.425,0.3456,0.286,0.547,0.4808,0.4152,0.3622,0.3336,0.530993333 -Hendrycks2020Many,0.6362,0.5672,0.464,0.3768,0.2022,0.7484,0.7428,0.7182,0.628,0.37,0.7082,0.6932,0.6664,0.6342,0.6054,0.7124,0.6548,0.6014,0.543,0.503,0.5888,0.5522,0.515,0.451,0.3578,0.7038,0.5988,0.4794,0.386,0.3194,0.7164,0.6642,0.559,0.3726,0.374,0.7178,0.712,0.6772,0.573,0.524,0.6624,0.652,0.6234,0.5582,0.448,0.6602,0.6438,0.6086,0.552,0.4494,0.6138,0.5096,0.402,0.3148,0.2472,0.531,0.4424,0.3646,0.301,0.2728,0.541256667 -Hendrycks2020AugMix,0.6298,0.5526,0.4618,0.377,0.2032,0.7434,0.7372,0.7076,0.5942,0.3336,0.6908,0.6656,0.6386,0.6052,0.5808,0.6918,0.6334,0.574,0.5206,0.4752,0.5234,0.4878,0.4322,0.3622,0.2648,0.6888,0.5698,0.459,0.3698,0.2944,0.6958,0.666,0.5646,0.3822,0.3818,0.7038,0.696,0.6696,0.565,0.5242,0.5686,0.5454,0.4958,0.41,0.2574,0.603,0.5734,0.5358,0.4742,0.3584,0.611,0.5232,0.4248,0.3404,0.269,0.5494,0.478,0.4094,0.3482,0.3146,0.51346 -Geirhos2018_SIN_IN,0.5792,0.5104,0.4078,0.3266,0.1702,0.714,0.7102,0.6796,0.5668,0.2618,0.6658,0.6372,0.6048,0.5742,0.5464,0.664,0.6078,0.5442,0.4928,0.4512,0.5266,0.4858,0.4372,0.3632,0.262,0.6556,0.529,0.423,0.329,0.2668,0.6626,0.6396,0.5502,0.3666,0.3718,0.6654,0.662,0.6386,0.5564,0.5086,0.531,0.5076,0.4568,0.375,0.244,0.5948,0.5674,0.5298,0.4742,0.373,0.5464,0.4496,0.3606,0.2946,0.2382,0.4654,0.3866,0.3254,0.273,0.2434,0.480863333 -Geirhos2018_SIN_IN_IN,0.5902,0.5176,0.4272,0.3382,0.1862,0.7386,0.7278,0.6852,0.5374,0.232,0.6774,0.6444,0.6146,0.5822,0.5548,0.6814,0.6204,0.5572,0.5004,0.4558,0.5124,0.4718,0.4194,0.3496,0.2466,0.677,0.5386,0.4212,0.3324,0.257,0.6758,0.6464,0.5396,0.3616,0.3698,0.6764,0.6712,0.6478,0.5464,0.5046,0.4486,0.4062,0.3286,0.2314,0.109,0.5744,0.5438,0.5012,0.4432,0.328,0.54,0.423,0.329,0.2544,0.2016,0.4738,0.404,0.3394,0.2806,0.2556,0.469173333 -Standard_R50,0.5702,0.5048,0.4122,0.3224,0.1732,0.7298,0.7178,0.6774,0.5072,0.1856,0.667,0.63,0.5892,0.5588,0.523,0.6706,0.6066,0.5374,0.4886,0.4458,0.4924,0.453,0.3978,0.3312,0.2196,0.6542,0.5174,0.4022,0.313,0.2526,0.6558,0.6262,0.5174,0.3322,0.3408,0.6586,0.6534,0.6284,0.5278,0.4768,0.406,0.3696,0.2898,0.19,0.0804,0.555,0.5188,0.4648,0.399,0.2832,0.511,0.3932,0.2972,0.2294,0.179,0.4764,0.398,0.3278,0.276,0.2492,0.447703333 -Geirhos2018_SIN,0.4518,0.3902,0.2976,0.237,0.116,0.5996,0.5942,0.566,0.4694,0.2262,0.5082,0.4598,0.422,0.3914,0.368,0.5306,0.44,0.3756,0.328,0.295,0.4176,0.3918,0.3472,0.2896,0.2056,0.568,0.4902,0.4008,0.327,0.28,0.535,0.493,0.3694,0.215,0.21,0.5336,0.5302,0.4928,0.3804,0.3324,0.4384,0.4294,0.4,0.354,0.2756,0.5108,0.4964,0.4706,0.4284,0.376,0.429,0.3498,0.2794,0.223,0.184,0.3314,0.2684,0.2162,0.1814,0.164,0.37804 -Salman2020Do_50_2_Linf,0.5056,0.456,0.38,0.3122,0.1794,0.6768,0.6724,0.6678,0.6104,0.2868,0.5952,0.5584,0.5216,0.4836,0.457,0.623,0.5462,0.4804,0.4302,0.3894,0.4746,0.44,0.3866,0.3166,0.2284,0.4906,0.2892,0.1794,0.1256,0.0898,0.5594,0.5386,0.4822,0.383,0.3992,0.563,0.5598,0.5448,0.4914,0.4632,0.3858,0.3404,0.2534,0.1518,0.0558,0.4396,0.3546,0.2422,0.1128,0.017,0.476,0.376,0.3016,0.2422,0.1948,0.4978,0.45,0.4074,0.3578,0.334,0.397146667 \ No newline at end of file diff --git a/robustbench/model_zoo/__init__.py b/robustbench/model_zoo/__init__.py deleted file mode 100644 index 3eafc941..00000000 --- a/robustbench/model_zoo/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .models import model_dicts - diff --git a/robustbench/model_zoo/architectures/CARD_resnet.py b/robustbench/model_zoo/architectures/CARD_resnet.py deleted file mode 100644 index 58f65a6f..00000000 --- a/robustbench/model_zoo/architectures/CARD_resnet.py +++ /dev/null @@ -1,122 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. - -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import torch -import torchvision - - -class LRR_ResNet(torchvision.models.ResNet): - expansion = 1 - - def __init__(self, block=torchvision.models.resnet.BasicBlock, layers=[2, 2, 2, 2], num_classes=10, width=64): - """To make it possible to vary the width, we need to override the constructor of the torchvision resnet.""" - - torch.nn.Module.__init__(self) # Skip the parent constructor. This replaces it. - self._norm_layer = torch.nn.BatchNorm2d - self.inplanes = width - self.dilation = 1 - self.groups = 1 - self.base_width = 64 - - # The initial convolutional layer. - self.conv1 = torch.nn.Conv2d(3, self.inplanes, kernel_size=3, stride=1, padding=1, bias=False) - self.bn1 = self._norm_layer(self.inplanes) - self.relu = torch.nn.ReLU(inplace=True) - - # The subsequent blocks. - self.layer1 = self._make_layer(block, width, layers[0]) - self.layer2 = self._make_layer(block, width*2, layers[1], stride=2, dilate=False) - self.layer3 = self._make_layer(block, width*4, layers[2], stride=2, dilate=False) - self.layer4 = self._make_layer(block, width*8, layers[3], stride=2, dilate=False) - - # The last layers. - self.avgpool = torch.nn.AvgPool2d(4) - self.fc = torch.nn.Linear(width*8*block.expansion, num_classes) - - # Default init. - for m in self.modules(): - if isinstance(m, torch.nn.Conv2d): - torch.nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') - elif isinstance(m, torch.nn.BatchNorm2d): - torch.nn.init.constant_(m.weight, 1) - torch.nn.init.constant_(m.bias, 0) - - def forward(self, x): - x = self.conv1(x) - x = self.bn1(x) - x = self.relu(x) - - x = self.layer1(x) - x = self.layer2(x) - x = self.layer3(x) - x = self.layer4(x) - - x = self.avgpool(x) - x = torch.flatten(x, 1) - x = self.fc(x) - - return x - - -# edgepopup -class PreActBasicBlock(torch.nn.Module): - expansion = 1 - - def __init__(self, in_planes, planes, stride=1): - super(PreActBasicBlock, self).__init__() - self.conv1 = torch.nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) - self.bn1 = torch.nn.BatchNorm2d(in_planes, affine=False) - self.conv2 = torch.nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) - self.bn2 = torch.nn.BatchNorm2d(planes, affine=False) - - self.shortcut = torch.nn.Sequential() - if stride != 1 or in_planes != self.expansion * planes: - self.shortcut = torch.nn.Sequential( - torch.nn.Conv2d(in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=False), - torch.nn.BatchNorm2d(self.expansion * planes, affine=False), - ) - - def forward(self, x): - out = torch.nn.functional.relu(self.bn1(x)) - shortcut = self.shortcut(out) if hasattr(self, 'shortcut') else x # Important: using out instead of x - out = self.conv1(out) - out = self.conv2(torch.nn.functional.relu(self.bn2(out))) - out += shortcut - return out - - -class WidePreActResNet(torch.nn.Module): - def __init__(self, block=PreActBasicBlock, num_blocks=[2, 2, 2, 2], num_classes=10, widen_factor=2): - super(WidePreActResNet, self).__init__() - self.in_planes = 64 - - self.conv1 = torch.nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False) - self.bn1 = torch.nn.BatchNorm2d(256*(widen_factor+1) * block.expansion, affine=False) - self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1) - self.layer2 = self._make_layer(block, 64*(widen_factor+1), num_blocks[1], stride=2) - self.layer3 = self._make_layer(block, 128*(widen_factor+1), num_blocks[2], stride=2) - self.layer4 = self._make_layer(block, 256*(widen_factor+1), num_blocks[3], stride=2) - self.avgpool = torch.nn.AdaptiveAvgPool2d(1) - self.fc = torch.nn.Conv2d(256*(widen_factor+1) * block.expansion, num_classes, kernel_size=1, bias=False) - - def _make_layer(self, block, planes, num_blocks, stride): - strides = [stride] + [1] * (num_blocks - 1) - layers = [] - for stride in strides: - layers.append(block(self.in_planes, planes, stride)) - self.in_planes = planes * block.expansion - - return torch.nn.Sequential(*layers) - - def forward(self, x): - out = self.conv1(x) - out = self.layer1(out) - out = self.layer2(out) - out = self.layer3(out) - out = self.layer4(out) - out = torch.nn.functional.relu(self.bn1(out)) - out = torch.nn.functional.avg_pool2d(out, 4) - out = self.fc(out) - return out.flatten(1) \ No newline at end of file diff --git a/robustbench/model_zoo/architectures/__init__.py b/robustbench/model_zoo/architectures/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/robustbench/model_zoo/architectures/boosting_wide_resnet.py b/robustbench/model_zoo/architectures/boosting_wide_resnet.py deleted file mode 100644 index 6a8f385a..00000000 --- a/robustbench/model_zoo/architectures/boosting_wide_resnet.py +++ /dev/null @@ -1,34 +0,0 @@ -import torch -import math -import torch.nn.functional as F -from torch import nn -from .wide_resnet import WideResNet - -class BoostingWideResNet(WideResNet): - - def __init__(self, depth=34, widen_factor=20): - super(BoostingWideResNet, self).__init__(depth=depth, - widen_factor=widen_factor, - sub_block1=True, - bias_last=False) - self.register_buffer( - 'mu', - torch.tensor([0.4914, 0.4822, 0.4465]).view(1, 3, 1, 1)) - self.register_buffer( - 'sigma', - torch.tensor([0.2471, 0.2435, 0.2616]).view(1, 3, 1, 1)) - - def forward(self, x): - x = (x - self.mu) / self.sigma - out = self.conv1(x) - out = self.block1(out) - out = self.block2(out) - out = self.block3(out) - out = self.relu(self.bn1(out)) - out = F.avg_pool2d(out, 8) - out = out.view(-1, self.nChannels) - out = F.normalize(out, p=2, dim=1) - for _, module in self.fc.named_modules(): - if isinstance(module, nn.Linear): - module.weight.data = F.normalize(module.weight, p=2, dim=1) - return self.fc(out) \ No newline at end of file diff --git a/robustbench/model_zoo/architectures/dm_wide_resnet.py b/robustbench/model_zoo/architectures/dm_wide_resnet.py deleted file mode 100644 index 46fee2c6..00000000 --- a/robustbench/model_zoo/architectures/dm_wide_resnet.py +++ /dev/null @@ -1,299 +0,0 @@ -# Copyright 2020 Deepmind Technologies Limited. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""WideResNet implementation in PyTorch. From: -https://github.com/deepmind/deepmind-research/blob/master/adversarial_robustness/pytorch/model_zoo.py -""" - -from typing import Tuple, Type, Union - -import torch -import torch.nn as nn -import torch.nn.functional as F - -CIFAR10_MEAN = (0.4914, 0.4822, 0.4465) -CIFAR10_STD = (0.2471, 0.2435, 0.2616) -CIFAR100_MEAN = (0.5071, 0.4865, 0.4409) -CIFAR100_STD = (0.2673, 0.2564, 0.2762) - - -class _Swish(torch.autograd.Function): - """Custom implementation of swish.""" - - @staticmethod - def forward(ctx, i): - result = i * torch.sigmoid(i) - ctx.save_for_backward(i) - return result - - @staticmethod - def backward(ctx, grad_output): - i = ctx.saved_variables[0] - sigmoid_i = torch.sigmoid(i) - return grad_output * (sigmoid_i * (1 + i * (1 - sigmoid_i))) - - -class Swish(nn.Module): - """Module using custom implementation.""" - - def forward(self, input_tensor): - return _Swish.apply(input_tensor) - - -class _Block(nn.Module): - """WideResNet Block.""" - - def __init__(self, - in_planes, - out_planes, - stride, - activation_fn: Type[nn.Module] = nn.ReLU): - super().__init__() - self.batchnorm_0 = nn.BatchNorm2d(in_planes) - self.relu_0 = activation_fn() - # We manually pad to obtain the same effect as `SAME` (necessary when - # `stride` is different than 1). - self.conv_0 = nn.Conv2d(in_planes, - out_planes, - kernel_size=3, - stride=stride, - padding=0, - bias=False) - self.batchnorm_1 = nn.BatchNorm2d(out_planes) - self.relu_1 = activation_fn() - self.conv_1 = nn.Conv2d(out_planes, - out_planes, - kernel_size=3, - stride=1, - padding=1, - bias=False) - self.has_shortcut = in_planes != out_planes - if self.has_shortcut: - self.shortcut = nn.Conv2d(in_planes, - out_planes, - kernel_size=1, - stride=stride, - padding=0, - bias=False) - else: - self.shortcut = None - self._stride = stride - - def forward(self, x): - if self.has_shortcut: - x = self.relu_0(self.batchnorm_0(x)) - else: - out = self.relu_0(self.batchnorm_0(x)) - v = x if self.has_shortcut else out - if self._stride == 1: - v = F.pad(v, (1, 1, 1, 1)) - elif self._stride == 2: - v = F.pad(v, (0, 1, 0, 1)) - else: - raise ValueError('Unsupported `stride`.') - out = self.conv_0(v) - out = self.relu_1(self.batchnorm_1(out)) - out = self.conv_1(out) - out = torch.add(self.shortcut(x) if self.has_shortcut else x, out) - return out - - -class _BlockGroup(nn.Module): - """WideResNet block group.""" - - def __init__(self, - num_blocks, - in_planes, - out_planes, - stride, - activation_fn: Type[nn.Module] = nn.ReLU): - super().__init__() - block = [] - for i in range(num_blocks): - block.append( - _Block(i == 0 and in_planes or out_planes, - out_planes, - i == 0 and stride or 1, - activation_fn=activation_fn)) - self.block = nn.Sequential(*block) - - def forward(self, x): - return self.block(x) - - -class DMWideResNet(nn.Module): - """WideResNet.""" - - def __init__(self, - num_classes: int = 10, - depth: int = 28, - width: int = 10, - activation_fn: Type[nn.Module] = nn.ReLU, - mean: Union[Tuple[float, ...], float] = CIFAR10_MEAN, - std: Union[Tuple[float, ...], float] = CIFAR10_STD, - padding: int = 0, - num_input_channels: int = 3): - super().__init__() - # persistent=False to not put these tensors in the module's state_dict and not try to - # load it from the checkpoint - self.register_buffer('mean', torch.tensor(mean).view(num_input_channels, 1, 1), - persistent=False) - self.register_buffer('std', torch.tensor(std).view(num_input_channels, 1, 1), - persistent=False) - self.padding = padding - num_channels = [16, 16 * width, 32 * width, 64 * width] - assert (depth - 4) % 6 == 0 - num_blocks = (depth - 4) // 6 - self.init_conv = nn.Conv2d(num_input_channels, - num_channels[0], - kernel_size=3, - stride=1, - padding=1, - bias=False) - self.layer = nn.Sequential( - _BlockGroup(num_blocks, - num_channels[0], - num_channels[1], - 1, - activation_fn=activation_fn), - _BlockGroup(num_blocks, - num_channels[1], - num_channels[2], - 2, - activation_fn=activation_fn), - _BlockGroup(num_blocks, - num_channels[2], - num_channels[3], - 2, - activation_fn=activation_fn)) - self.batchnorm = nn.BatchNorm2d(num_channels[3]) - self.relu = activation_fn() - self.logits = nn.Linear(num_channels[3], num_classes) - self.num_channels = num_channels[3] - - def forward(self, x): - if self.padding > 0: - x = F.pad(x, (self.padding,) * 4) - out = (x - self.mean) / self.std - out = self.init_conv(out) - out = self.layer(out) - out = self.relu(self.batchnorm(out)) - out = F.avg_pool2d(out, 8) - out = out.view(-1, self.num_channels) - return self.logits(out) - - -class _PreActBlock(nn.Module): - """Pre-activation ResNet Block.""" - - def __init__(self, in_planes, out_planes, stride, activation_fn=nn.ReLU): - super().__init__() - self._stride = stride - self.batchnorm_0 = nn.BatchNorm2d(in_planes) - self.relu_0 = activation_fn() - # We manually pad to obtain the same effect as `SAME` (necessary when - # `stride` is different than 1). - self.conv_2d_1 = nn.Conv2d(in_planes, out_planes, kernel_size=3, - stride=stride, padding=0, bias=False) - self.batchnorm_1 = nn.BatchNorm2d(out_planes) - self.relu_1 = activation_fn() - self.conv_2d_2 = nn.Conv2d(out_planes, out_planes, kernel_size=3, stride=1, - padding=1, bias=False) - self.has_shortcut = stride != 1 or in_planes != out_planes - if self.has_shortcut: - self.shortcut = nn.Conv2d(in_planes, out_planes, kernel_size=3, - stride=stride, padding=0, bias=False) - - def _pad(self, x): - if self._stride == 1: - x = F.pad(x, (1, 1, 1, 1)) - elif self._stride == 2: - x = F.pad(x, (0, 1, 0, 1)) - else: - raise ValueError('Unsupported `stride`.') - return x - - def forward(self, x): - out = self.relu_0(self.batchnorm_0(x)) - shortcut = self.shortcut(self._pad(x)) if self.has_shortcut else x - out = self.conv_2d_1(self._pad(out)) - out = self.conv_2d_2(self.relu_1(self.batchnorm_1(out))) - return out + shortcut - - -class DMPreActResNet(nn.Module): - """Pre-activation ResNet.""" - - def __init__(self, - num_classes: int = 10, - depth: int = 18, - width: int = 0, # Used to make the constructor consistent. - activation_fn: Type[nn.Module] = nn.ReLU, - mean: Union[Tuple[float, ...], float] = CIFAR10_MEAN, - std: Union[Tuple[float, ...], float] = CIFAR10_STD, - padding: int = 0, - num_input_channels: int = 3, - use_cuda: bool = True): - super().__init__() - if width != 0: - raise ValueError('Unsupported `width`.') - # persistent=False to not put these tensors in the module's state_dict and not try to - # load it from the checkpoint - self.register_buffer('mean', torch.tensor(mean).view(num_input_channels, 1, 1), - persistent=False) - self.register_buffer('std', torch.tensor(std).view(num_input_channels, 1, 1), - persistent=False) - self.mean_cuda = None - self.std_cuda = None - self.padding = padding - self.conv_2d = nn.Conv2d(num_input_channels, 64, kernel_size=3, stride=1, - padding=1, bias=False) - if depth == 18: - num_blocks = (2, 2, 2, 2) - elif depth == 34: - num_blocks = (3, 4, 6, 3) - else: - raise ValueError('Unsupported `depth`.') - self.layer_0 = self._make_layer(64, 64, num_blocks[0], 1, activation_fn) - self.layer_1 = self._make_layer(64, 128, num_blocks[1], 2, activation_fn) - self.layer_2 = self._make_layer(128, 256, num_blocks[2], 2, activation_fn) - self.layer_3 = self._make_layer(256, 512, num_blocks[3], 2, activation_fn) - self.batchnorm = nn.BatchNorm2d(512) - self.relu = activation_fn() - self.logits = nn.Linear(512, num_classes) - - def _make_layer(self, in_planes, out_planes, num_blocks, stride, - activation_fn): - layers = [] - for i, stride in enumerate([stride] + [1] * (num_blocks - 1)): - layers.append( - _PreActBlock(i == 0 and in_planes or out_planes, - out_planes, - stride, - activation_fn)) - return nn.Sequential(*layers) - - def forward(self, x): - if self.padding > 0: - x = F.pad(x, (self.padding,) * 4) - out = (x - self.mean) / self.std - out = self.conv_2d(out) - out = self.layer_0(out) - out = self.layer_1(out) - out = self.layer_2(out) - out = self.layer_3(out) - out = self.relu(self.batchnorm(out)) - out = F.avg_pool2d(out, 4) - out = out.view(out.size(0), -1) - return self.logits(out) diff --git a/robustbench/model_zoo/architectures/paf_wide_resnet.py b/robustbench/model_zoo/architectures/paf_wide_resnet.py deleted file mode 100644 index cdaeb1e9..00000000 --- a/robustbench/model_zoo/architectures/paf_wide_resnet.py +++ /dev/null @@ -1,125 +0,0 @@ -import math -import torch -import torch.nn as nn -import torch.nn.functional as F - -class PSSiLU(nn.Module): - def __init__(self): - super(PSSiLU, self).__init__() - self.beta = nn.Parameter(torch.tensor([1e-8])) - self.alpha = nn.Parameter(torch.tensor([1.0])) - def forward(self, x): - return x * (F.sigmoid(torch.abs(self.alpha) * x) - torch.abs(self.beta)) / (1 - torch.abs(self.beta)) - -class PAF_BasicBlock(nn.Module): - def __init__(self, activation, in_planes, out_planes, stride, dropRate=0.0): - super(PAF_BasicBlock, self).__init__() - self.bn1 = nn.BatchNorm2d(in_planes) - self.activation = activation - self.conv1 = nn.Conv2d( - in_planes, out_planes, kernel_size=3, stride=stride, padding=1, bias=False - ) - self.bn2 = nn.BatchNorm2d(out_planes) - - self.conv2 = nn.Conv2d( - out_planes, out_planes, kernel_size=3, stride=1, padding=1, bias=False - ) - self.droprate = dropRate - self.equalInOut = in_planes == out_planes - self.convShortcut = ( - (not self.equalInOut) - and nn.Conv2d( - in_planes, - out_planes, - kernel_size=1, - stride=stride, - padding=0, - bias=False, - ) - or None - ) - - def forward(self, x): - if not self.equalInOut: - x = self.activation(self.bn1(x)) - else: - out = self.activation(self.bn1(x)) - out = self.activation(self.bn2(self.conv1(out if self.equalInOut else x))) - if self.droprate > 0: - out = F.dropout(out, p=self.droprate, training=self.training) - out = self.conv2(out) - return torch.add(x if self.equalInOut else self.convShortcut(x), out) - - -class PAF_NetworkBlock(nn.Module): - def __init__(self, activation, nb_layers, in_planes, out_planes, block, stride, dropRate=0.0): - super(PAF_NetworkBlock, self).__init__() - self.layer = self._make_layer( - activation, block, in_planes, out_planes, nb_layers, stride, dropRate - ) - - def _make_layer(self, activation, block, in_planes, out_planes, nb_layers, stride, dropRate): - layers = [] - for i in range(int(nb_layers)): - layers.append( - block( activation, - i == 0 and in_planes or out_planes, - out_planes, - i == 0 and stride or 1, - dropRate - ) - ) - return nn.Sequential(*layers) - - def forward(self, x): - return self.layer(x) - - -class PAF_WideResNet(nn.Module): - def __init__(self, activation, depth=34, num_classes=10, widen_factor=10, dropRate=0.0, **kwargs): - super(PAF_WideResNet, self).__init__() - nChannels = [16, 16 * widen_factor, 32 * widen_factor, 64 * widen_factor] - assert (depth - 4) % 6 == 0 - n = (depth - 4) / 6 - block = PAF_BasicBlock - # 1st conv before any network block - self.conv1 = nn.Conv2d( - 3, nChannels[0], kernel_size=3, stride=1, padding=1, bias=False - ) - # 1st block - self.block1 = PAF_NetworkBlock(activation, n, nChannels[0], nChannels[1], block, 1, dropRate) - # 2nd block - self.block2 = PAF_NetworkBlock(activation, n, nChannels[1], nChannels[2], block, 2, dropRate) - # 3rd block - self.block3 = PAF_NetworkBlock(activation, n, nChannels[2], nChannels[3], block, 2, dropRate) - # global average pooling and classifier - self.bn1 = nn.BatchNorm2d(nChannels[3]) - self.activation = activation - self.fc = nn.Linear(nChannels[3], num_classes) - self.nChannels = nChannels[3] - - for m in self.modules(): - if isinstance(m, nn.Conv2d): - n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels - m.weight.data.normal_(0, math.sqrt(2.0 / n)) - elif isinstance(m, nn.BatchNorm2d): - m.weight.data.fill_(1) - m.bias.data.zero_() - elif isinstance(m, nn.Linear): - m.bias.data.zero_() - - def forward(self, x): - out = self.conv1(x) - out = self.block1(out) - out = self.block2(out) - out = self.block3(out) - out = self.activation(self.bn1(out)) - out = F.avg_pool2d(out, 8) - out = out.view(-1, self.nChannels) - return self.fc(out) - - -def pssilu_wrn_28_10(**kwargs): - act = PSSiLU() - return PAF_WideResNet(act, depth=28, widen_factor=10, **kwargs) - diff --git a/robustbench/model_zoo/architectures/resnest.py b/robustbench/model_zoo/architectures/resnest.py deleted file mode 100644 index 8a1b8f32..00000000 --- a/robustbench/model_zoo/architectures/resnest.py +++ /dev/null @@ -1,584 +0,0 @@ -"""ResNet implementation in PyTorch. -Adapted from https://github.com/zhanghang1989/ResNeSt/blob/master/resnest/torch/models/resnest.py -""" - -import math -import torch -from torch import nn -import torch.nn.functional as F -from torch.nn import Conv2d, Module, Linear, BatchNorm2d, ReLU -from torch.nn.modules.utils import _pair - - -class DropBlock2D(object): - def __init__(self, *args, **kwargs): - raise NotImplementedError - - -class SplAtConv2d(Module): - """Split-Attention Conv2d""" - - def __init__( - self, - in_channels, - channels, - kernel_size, - stride=(1, 1), - padding=(0, 0), - dilation=(1, 1), - groups=1, - bias=True, - radix=2, - reduction_factor=4, - norm_layer=None, - dropblock_prob=0.0, - swish=False, - **kwargs - ): - super(SplAtConv2d, self).__init__() - padding = _pair(padding) - inter_channels = max(in_channels * radix // reduction_factor, 32) - self.radix = radix - self.cardinality = groups - self.channels = channels - self.dropblock_prob = dropblock_prob - self.conv = Conv2d( - in_channels, - channels * radix, - kernel_size, - stride, - padding, - dilation, - groups=groups * radix, - bias=bias, - **kwargs - ) - self.use_bn = norm_layer is not None - if self.use_bn: - self.bn0 = norm_layer(channels * radix) - self.relu = nn.SiLU() if swish else nn.ReLU(inplace=True) - self.fc1 = Conv2d(channels, inter_channels, 1, groups=self.cardinality) - if self.use_bn: - self.bn1 = norm_layer(inter_channels) - self.fc2 = Conv2d(inter_channels, channels * radix, 1, groups=self.cardinality) - if dropblock_prob > 0.0: - self.dropblock = DropBlock2D(dropblock_prob, 3) - self.rsoftmax = rSoftMax(radix, groups) - - def forward(self, x): - x = self.conv(x) - if self.use_bn: - x = self.bn0(x) - if self.dropblock_prob > 0.0: - x = self.dropblock(x) - x = self.relu(x) - - batch, rchannel = x.shape[:2] - if self.radix > 1: - if torch.__version__ < "1.5": - splited = torch.split(x, int(rchannel // self.radix), dim=1) - else: - splited = torch.split(x, rchannel // self.radix, dim=1) - gap = sum(splited) - else: - gap = x - gap = F.adaptive_avg_pool2d(gap, 1) - gap = self.fc1(gap) - - if self.use_bn: - gap = self.bn1(gap) - gap = self.relu(gap) - - atten = self.fc2(gap) - atten = self.rsoftmax(atten).view(batch, -1, 1, 1) - - if self.radix > 1: - if torch.__version__ < "1.5": - attens = torch.split(atten, int(rchannel // self.radix), dim=1) - else: - attens = torch.split(atten, rchannel // self.radix, dim=1) - out = sum([att * split for (att, split) in zip(attens, splited)]) - else: - out = atten * x - return out.contiguous() - - -class rSoftMax(nn.Module): - def __init__(self, radix, cardinality): - super().__init__() - self.radix = radix - self.cardinality = cardinality - - def forward(self, x): - batch = x.size(0) - if self.radix > 1: - x = x.view(batch, self.cardinality, self.radix, -1).transpose(1, 2) - x = F.softmax(x, dim=1) - x = x.reshape(batch, -1) - else: - x = torch.sigmoid(x) - return x - - -class GlobalAvgPool2d(nn.Module): - def __init__(self): - """Global average pooling over the input's spatial dimensions""" - super(GlobalAvgPool2d, self).__init__() - - def forward(self, inputs): - return nn.functional.adaptive_avg_pool2d(inputs, 1).view(inputs.size(0), -1) - - -class Bottleneck(nn.Module): - """ResNet Bottleneck""" - - # pylint: disable=unused-argument - expansion = 4 - - def __init__( - self, - inplanes, - planes, - stride=1, - downsample=None, - radix=1, - cardinality=1, - bottleneck_width=64, - avd=False, - avd_first=False, - dilation=1, - is_first=False, - norm_layer=None, - dropblock_prob=0.0, - last_gamma=False, - swish=False, - ): - super(Bottleneck, self).__init__() - group_width = int(planes * (bottleneck_width / 64.0)) * cardinality - self.conv1 = nn.Conv2d(inplanes, group_width, kernel_size=1, bias=False) - self.bn1 = norm_layer(group_width) - self.dropblock_prob = dropblock_prob - self.radix = radix - self.avd = avd and (stride > 1 or is_first) - self.avd_first = avd_first - - if self.avd: - self.avd_layer = nn.AvgPool2d(3, stride, padding=1) - stride = 1 - - if dropblock_prob > 0.0: - self.dropblock1 = DropBlock2D(dropblock_prob, 3) - if radix == 1: - self.dropblock2 = DropBlock2D(dropblock_prob, 3) - self.dropblock3 = DropBlock2D(dropblock_prob, 3) - - if radix >= 1: - self.conv2 = SplAtConv2d( - group_width, - group_width, - kernel_size=3, - stride=stride, - padding=dilation, - dilation=dilation, - groups=cardinality, - bias=False, - radix=radix, - norm_layer=norm_layer, - dropblock_prob=dropblock_prob, - swish=swish, - ) - else: - self.conv2 = nn.Conv2d( - group_width, - group_width, - kernel_size=3, - stride=stride, - padding=dilation, - dilation=dilation, - groups=cardinality, - bias=False, - ) - self.bn2 = norm_layer(group_width) - - self.conv3 = nn.Conv2d(group_width, planes * 4, kernel_size=1, bias=False) - self.bn3 = norm_layer(planes * 4) - - if last_gamma: - from torch.nn.init import zeros_ - - zeros_(self.bn3.weight) - self.relu = nn.SiLU() if swish else nn.ReLU(inplace=True) - self.downsample = downsample - self.dilation = dilation - self.stride = stride - - def forward(self, x): - residual = x - - out = self.conv1(x) - out = self.bn1(out) - if self.dropblock_prob > 0.0: - out = self.dropblock1(out) - out = self.relu(out) - - if self.avd and self.avd_first: - out = self.avd_layer(out) - - out = self.conv2(out) - if self.radix == 0: - out = self.bn2(out) - if self.dropblock_prob > 0.0: - out = self.dropblock2(out) - out = self.relu(out) - - if self.avd and not self.avd_first: - out = self.avd_layer(out) - - out = self.conv3(out) - out = self.bn3(out) - if self.dropblock_prob > 0.0: - out = self.dropblock3(out) - - if self.downsample is not None: - residual = self.downsample(x) - - out += residual - out = self.relu(out) - - return out - - -class ResNest(nn.Module): - """ResNet Variants - - Parameters - ---------- - block : Block - Class for the residual block. Options are BasicBlockV1, BottleneckV1. - layers : list of int - Numbers of layers in each block - classes : int, default 1000 - Number of classification classes. - dilated : bool, default False - Applying dilation strategy to pretrained ResNet yielding a stride-8 model, - typically used in Semantic Segmentation. - norm_layer : object - Normalization layer used in backbone network (default: :class:`mxnet.gluon.nn.BatchNorm`; - for Synchronized Cross-GPU BachNormalization). - - Reference: - - - He, Kaiming, et al. "Deep residual learning for image recognition." Proceedings of the IEEE conference on computer vision and pattern recognition. 2016. - - - Yu, Fisher, and Vladlen Koltun. "Multi-scale context aggregation by dilated convolutions." - """ - - # pylint: disable=unused-variable - def __init__( - self, - block, - layers, - radix=1, - groups=1, - bottleneck_width=64, - num_classes=10, - dilated=False, - dilation=1, - deep_stem=False, - stem_width=64, - avg_down=False, - avd=False, - avd_first=False, - final_drop=0.0, - dropblock_prob=0, - last_gamma=False, - norm_layer=nn.BatchNorm2d, - swish=False, - ): - self.cardinality = groups - self.bottleneck_width = bottleneck_width - # ResNet-D params - self.inplanes = stem_width * 2 if deep_stem else 64 - self.avg_down = avg_down - self.last_gamma = last_gamma - self.swish = swish - # ResNeSt params - self.radix = radix - self.avd = avd - self.avd_first = avd_first - self.relu = nn.SiLU if swish else nn.ReLU - super(ResNest, self).__init__() - conv_layer = nn.Conv2d - conv_kwargs = {} - if deep_stem: - self.conv1 = nn.Sequential( - conv_layer( - 3, - stem_width, - kernel_size=3, - stride=1, - padding=1, - bias=False, - **conv_kwargs - ), - norm_layer(stem_width), - self.relu(), - conv_layer( - stem_width, - stem_width, - kernel_size=3, - stride=1, - padding=1, - bias=False, - **conv_kwargs - ), - norm_layer(stem_width), - self.relu(), - conv_layer( - stem_width, - stem_width * 2, - kernel_size=3, - stride=1, - padding=1, - bias=False, - **conv_kwargs - ), - ) - else: - self.conv1 = conv_layer( - 3, 64, kernel_size=3, stride=1, padding=3, bias=False, **conv_kwargs - ) - self.bn1 = norm_layer(self.inplanes) - self.relu = self.relu() - # self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) - self.maxpool = nn.Identity() - self.layer1 = self._make_layer( - block, 64, layers[0], norm_layer=norm_layer, is_first=False - ) - self.layer2 = self._make_layer( - block, 128, layers[1], stride=2, norm_layer=norm_layer - ) - if dilated or dilation == 4: - self.layer3 = self._make_layer( - block, - 256, - layers[2], - stride=1, - dilation=2, - norm_layer=norm_layer, - dropblock_prob=dropblock_prob, - ) - self.layer4 = self._make_layer( - block, - 512, - layers[3], - stride=1, - dilation=4, - norm_layer=norm_layer, - dropblock_prob=dropblock_prob, - ) - elif dilation == 2: - self.layer3 = self._make_layer( - block, - 256, - layers[2], - stride=2, - dilation=1, - norm_layer=norm_layer, - dropblock_prob=dropblock_prob, - ) - self.layer4 = self._make_layer( - block, - 512, - layers[3], - stride=1, - dilation=2, - norm_layer=norm_layer, - dropblock_prob=dropblock_prob, - ) - else: - self.layer3 = self._make_layer( - block, - 256, - layers[2], - stride=2, - norm_layer=norm_layer, - dropblock_prob=dropblock_prob, - ) - self.layer4 = self._make_layer( - block, - 512, - layers[3], - stride=2, - norm_layer=norm_layer, - dropblock_prob=dropblock_prob, - ) - self.avgpool = GlobalAvgPool2d() - self.drop = nn.Dropout(final_drop) if final_drop > 0.0 else None - self.fc = nn.Linear(512 * block.expansion, num_classes) - - for m in self.modules(): - if isinstance(m, nn.Conv2d): - n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels - m.weight.data.normal_(0, math.sqrt(2.0 / n)) - elif isinstance(m, norm_layer): - m.weight.data.fill_(1) - m.bias.data.zero_() - - def _make_layer( - self, - block, - planes, - blocks, - stride=1, - dilation=1, - norm_layer=None, - dropblock_prob=0.0, - is_first=True, - ): - downsample = None - if stride != 1 or self.inplanes != planes * block.expansion: - down_layers = [] - if self.avg_down: - if dilation == 1: - down_layers.append( - nn.AvgPool2d( - kernel_size=stride, - stride=stride, - ceil_mode=True, - count_include_pad=False, - ) - ) - else: - down_layers.append( - nn.AvgPool2d( - kernel_size=1, - stride=1, - ceil_mode=True, - count_include_pad=False, - ) - ) - down_layers.append( - nn.Conv2d( - self.inplanes, - planes * block.expansion, - kernel_size=1, - stride=1, - bias=False, - ) - ) - else: - down_layers.append( - nn.Conv2d( - self.inplanes, - planes * block.expansion, - kernel_size=1, - stride=stride, - bias=False, - ) - ) - down_layers.append(norm_layer(planes * block.expansion)) - downsample = nn.Sequential(*down_layers) - - layers = [] - if dilation == 1 or dilation == 2: - layers.append( - block( - self.inplanes, - planes, - stride, - downsample=downsample, - radix=self.radix, - cardinality=self.cardinality, - bottleneck_width=self.bottleneck_width, - avd=self.avd, - avd_first=self.avd_first, - dilation=1, - is_first=is_first, - norm_layer=norm_layer, - dropblock_prob=dropblock_prob, - last_gamma=self.last_gamma, - swish=self.swish, - ) - ) - elif dilation == 4: - layers.append( - block( - self.inplanes, - planes, - stride, - downsample=downsample, - radix=self.radix, - cardinality=self.cardinality, - bottleneck_width=self.bottleneck_width, - avd=self.avd, - avd_first=self.avd_first, - dilation=2, - is_first=is_first, - norm_layer=norm_layer, - dropblock_prob=dropblock_prob, - last_gamma=self.last_gamma, - swish=self.swish, - ) - ) - else: - raise RuntimeError("=> unknown dilation size: {}".format(dilation)) - - self.inplanes = planes * block.expansion - for i in range(1, blocks): - layers.append( - block( - self.inplanes, - planes, - radix=self.radix, - cardinality=self.cardinality, - bottleneck_width=self.bottleneck_width, - avd=self.avd, - avd_first=self.avd_first, - dilation=dilation, - norm_layer=norm_layer, - dropblock_prob=dropblock_prob, - last_gamma=self.last_gamma, - swish=self.swish, - ) - ) - - return nn.Sequential(*layers) - - def forward(self, x): - x = self.conv1(x) - x = self.bn1(x) - x = self.relu(x) - x = self.maxpool(x) - - x = self.layer1(x) - x = self.layer2(x) - x = self.layer3(x) - x = self.layer4(x) - - x = self.avgpool(x) - x = torch.flatten(x, 1) - if self.drop: - x = self.drop(x) - x = self.fc(x) - - return x - - -def ResNest152(num_classes=10, **kwargs): - model = ResNest( - Bottleneck, - [3, 8, 36, 3], - radix=4, - groups=1, - bottleneck_width=64, - num_classes=num_classes, - deep_stem=True, - stem_width=64, - avg_down=True, - avd=True, - avd_first=False, - swish=True, - **kwargs - ) - return model diff --git a/robustbench/model_zoo/architectures/resnet.py b/robustbench/model_zoo/architectures/resnet.py deleted file mode 100644 index e4bfa0be..00000000 --- a/robustbench/model_zoo/architectures/resnet.py +++ /dev/null @@ -1,260 +0,0 @@ -import torch.nn as nn -import torch.nn.functional as F - - -class BasicBlock(nn.Module): - expansion = 1 - - def __init__(self, in_planes, planes, stride=1): - super(BasicBlock, self).__init__() - self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) - self.bn1 = nn.BatchNorm2d(planes) - self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) - self.bn2 = nn.BatchNorm2d(planes) - - self.shortcut = nn.Sequential() - if stride != 1 or in_planes != self.expansion * planes: - self.shortcut = nn.Sequential( - nn.Conv2d(in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=False), - nn.BatchNorm2d(self.expansion * planes) - ) - - def forward(self, x): - out = F.relu(self.bn1(self.conv1(x))) - out = self.bn2(self.conv2(out)) - out += self.shortcut(x) - out = F.relu(out) - return out - - -class Bottleneck(nn.Module): - expansion = 4 - - def __init__(self, in_planes, planes, stride=1): - super(Bottleneck, self).__init__() - self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False) - self.bn1 = nn.BatchNorm2d(planes) - self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) - self.bn2 = nn.BatchNorm2d(planes) - self.conv3 = nn.Conv2d(planes, self.expansion * planes, kernel_size=1, bias=False) - self.bn3 = nn.BatchNorm2d(self.expansion * planes) - - self.shortcut = nn.Sequential() - if stride != 1 or in_planes != self.expansion * planes: - self.shortcut = nn.Sequential( - nn.Conv2d(in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=False), - nn.BatchNorm2d(self.expansion * planes) - ) - - def forward(self, x): - out = F.relu(self.bn1(self.conv1(x))) - out = F.relu(self.bn2(self.conv2(out))) - out = self.bn3(self.conv3(out)) - out += self.shortcut(x) - out = F.relu(out) - return out - - -class BottleneckChen2020AdversarialNet(nn.Module): - expansion = 4 - - def __init__(self, in_planes, planes, stride=1): - super(BottleneckChen2020AdversarialNet, self).__init__() - self.bn0 = nn.BatchNorm2d(in_planes) - self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False) - self.bn1 = nn.BatchNorm2d(planes) - self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) - self.bn2 = nn.BatchNorm2d(planes) - self.conv3 = nn.Conv2d(planes, self.expansion * planes, kernel_size=1, bias=False) - self.shortcut = nn.Sequential() - if stride != 1 or in_planes != self.expansion * planes: - self.shortcut = nn.Sequential( - nn.Conv2d(in_planes, self.expansion * planes, kernel_size=1, stride=stride, bias=False), - nn.BatchNorm2d(self.expansion * planes) - ) - - def forward(self, x): - pre = F.relu(self.bn0(x)) - out = F.relu(self.bn1(self.conv1(pre))) - out = F.relu(self.bn2(self.conv2(out))) - out = self.conv3(out) - if len(self.shortcut) == 0: - out += self.shortcut(x) - else: - out += self.shortcut(pre) - return out - - -class ResNet(nn.Module): - def __init__(self, block, num_blocks, num_classes=10): - super(ResNet, self).__init__() - self.in_planes = 64 - - self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False) - self.bn1 = nn.BatchNorm2d(64) - self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1) - self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2) - self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2) - self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2) - self.linear = nn.Linear(512 * block.expansion, num_classes) - - def _make_layer(self, block, planes, num_blocks, stride): - strides = [stride] + [1] * (num_blocks - 1) - layers = [] - for stride in strides: - layers.append(block(self.in_planes, planes, stride)) - self.in_planes = planes * block.expansion - return nn.Sequential(*layers) - - def forward(self, x): - out = F.relu(self.bn1(self.conv1(x))) - out = self.layer1(out) - out = self.layer2(out) - out = self.layer3(out) - out = self.layer4(out) - out = F.avg_pool2d(out, 4) - out = out.view(out.size(0), -1) - out = self.linear(out) - return out - - -class PreActBlock(nn.Module): - '''Pre-activation version of the BasicBlock.''' - expansion = 1 - - def __init__(self, in_planes, planes, stride=1, out_shortcut=False): - super(PreActBlock, self).__init__() - self.out_shortcut = out_shortcut - self.bn1 = nn.BatchNorm2d(in_planes) - self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) - self.bn2 = nn.BatchNorm2d(planes) - self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) - - if stride != 1 or in_planes != self.expansion*planes: - self.shortcut = nn.Sequential( - nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False) - ) - - def forward(self, x): - out = F.relu(self.bn1(x)) - shortcut = self.shortcut(out if self.out_shortcut else x) if hasattr(self, 'shortcut') else x - out = self.conv1(out) - out = self.conv2(F.relu(self.bn2(out))) - out += shortcut - return out - - -class PreActBlockV2(nn.Module): - '''Pre-activation version of the BasicBlock (slightly different forward pass)''' - expansion = 1 - - def __init__(self, in_planes, planes, stride=1, out_shortcut=False): - super(PreActBlockV2, self).__init__() - self.bn1 = nn.BatchNorm2d(in_planes) - self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) - self.bn2 = nn.BatchNorm2d(planes) - self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False) - - if stride != 1 or in_planes != self.expansion*planes: - self.shortcut = nn.Sequential( - nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False) - ) - - def forward(self, x): - out = F.relu(self.bn1(x)) - shortcut = self.shortcut(out) if hasattr(self, 'shortcut') else x - out = self.conv1(out) - out = self.conv2(F.relu(self.bn2(out))) - out += shortcut - return out - - -class PreActBottleneck(nn.Module): - '''Pre-activation version of the original Bottleneck module.''' - expansion = 4 - - def __init__(self, in_planes, planes, stride=1): - super(PreActBottleneck, self).__init__() - self.bn1 = nn.BatchNorm2d(in_planes) - self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, bias=False) - self.bn2 = nn.BatchNorm2d(planes) - self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) - self.bn3 = nn.BatchNorm2d(planes) - self.conv3 = nn.Conv2d(planes, self.expansion*planes, kernel_size=1, bias=False) - - if stride != 1 or in_planes != self.expansion*planes: - self.shortcut = nn.Sequential( - nn.Conv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False) - ) - - def forward(self, x): - out = F.relu(self.bn1(x)) - shortcut = self.shortcut(out) if hasattr(self, 'shortcut') else x - out = self.conv1(out) - out = self.conv2(F.relu(self.bn2(out))) - out = self.conv3(F.relu(self.bn3(out))) - out += shortcut - return out - - -class PreActResNet(nn.Module): - def __init__(self, block, num_blocks, num_classes=10, bn_before_fc=False, out_shortcut=False): - super(PreActResNet, self).__init__() - self.in_planes = 64 - self.bn_before_fc = bn_before_fc - self.out_shortcut = out_shortcut - - self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=1, padding=1, bias=False) - self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1) - self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2) - self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2) - self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2) - if bn_before_fc: - self.bn = nn.BatchNorm2d(512 * block.expansion) - self.linear = nn.Linear(512*block.expansion, num_classes) - - def _make_layer(self, block, planes, num_blocks, stride): - strides = [stride] + [1]*(num_blocks-1) - layers = [] - for stride in strides: - layers.append(block(self.in_planes, planes, stride, out_shortcut=self.out_shortcut)) - self.in_planes = planes * block.expansion - return nn.Sequential(*layers) - - def forward(self, x): - out = self.conv1(x) - out = self.layer1(out) - out = self.layer2(out) - out = self.layer3(out) - out = self.layer4(out) - if self.bn_before_fc: - out = F.relu(self.bn(out)) - out = F.avg_pool2d(out, 4) - out = out.view(out.size(0), -1) - out = self.linear(out) - return out - - -def ResNet18(): - return ResNet(BasicBlock, [2, 2, 2, 2]) - - -def ResNet34(): - return ResNet(BasicBlock, [3, 4, 6, 3]) - - -def ResNet50(): - return ResNet(Bottleneck, [3, 4, 6, 3]) - - -def ResNet101(): - return ResNet(Bottleneck, [3, 4, 23, 3]) - - -def ResNet152(): - return ResNet(Bottleneck, [3, 8, 36, 3]) - - -def PreActResNet18(): - return PreActResNet(PreActBlock, [2, 2, 2, 2]) - diff --git a/robustbench/model_zoo/architectures/resnext.py b/robustbench/model_zoo/architectures/resnext.py deleted file mode 100644 index bad94548..00000000 --- a/robustbench/model_zoo/architectures/resnext.py +++ /dev/null @@ -1,170 +0,0 @@ -"""ResNeXt implementation (https://arxiv.org/abs/1611.05431). - -MIT License - -Copyright (c) 2017 Xuanyi Dong - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - -From: -https://github.com/google-research/augmix/blob/master/third_party/WideResNet_pytorch/wideresnet.py - -""" - -import math - -import torch.nn as nn -import torch.nn.functional as F -from torch.nn import init - - -class ResNeXtBottleneck(nn.Module): - """ - ResNeXt Bottleneck Block type C (https://github.com/facebookresearch/ResNeXt/blob/master/models/resnext.lua). - """ - expansion = 4 - - def __init__(self, - inplanes, - planes, - cardinality, - base_width, - stride=1, - downsample=None): - super(ResNeXtBottleneck, self).__init__() - - dim = int(math.floor(planes * (base_width / 64.0))) - - self.conv_reduce = nn.Conv2d( - inplanes, - dim * cardinality, - kernel_size=1, - stride=1, - padding=0, - bias=False) - self.bn_reduce = nn.BatchNorm2d(dim * cardinality) - - self.conv_conv = nn.Conv2d( - dim * cardinality, - dim * cardinality, - kernel_size=3, - stride=stride, - padding=1, - groups=cardinality, - bias=False) - self.bn = nn.BatchNorm2d(dim * cardinality) - - self.conv_expand = nn.Conv2d( - dim * cardinality, - planes * 4, - kernel_size=1, - stride=1, - padding=0, - bias=False) - self.bn_expand = nn.BatchNorm2d(planes * 4) - - self.downsample = downsample - - def forward(self, x): - residual = x - - bottleneck = self.conv_reduce(x) - bottleneck = F.relu(self.bn_reduce(bottleneck), inplace=True) - - bottleneck = self.conv_conv(bottleneck) - bottleneck = F.relu(self.bn(bottleneck), inplace=True) - - bottleneck = self.conv_expand(bottleneck) - bottleneck = self.bn_expand(bottleneck) - - if self.downsample is not None: - residual = self.downsample(x) - - return F.relu(residual + bottleneck, inplace=True) - - -class CifarResNeXt(nn.Module): - """ResNext optimized for the Cifar dataset, as specified in - https://arxiv.org/pdf/1611.05431.pdf.""" - - def __init__(self, block, depth, cardinality, base_width, num_classes): - super(CifarResNeXt, self).__init__() - - # Model type specifies number of layers for CIFAR-10 and CIFAR-100 model - assert (depth - 2) % 9 == 0, 'depth should be one of 29, 38, 47, 56, 101' - layer_blocks = (depth - 2) // 9 - - self.cardinality = cardinality - self.base_width = base_width - self.num_classes = num_classes - - self.conv_1_3x3 = nn.Conv2d(3, 64, 3, 1, 1, bias=False) - self.bn_1 = nn.BatchNorm2d(64) - - self.inplanes = 64 - self.stage_1 = self._make_layer(block, 64, layer_blocks, 1) - self.stage_2 = self._make_layer(block, 128, layer_blocks, 2) - self.stage_3 = self._make_layer(block, 256, layer_blocks, 2) - self.avgpool = nn.AvgPool2d(8) - self.classifier = nn.Linear(256 * block.expansion, num_classes) - - for m in self.modules(): - if isinstance(m, nn.Conv2d): - n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels - m.weight.data.normal_(0, math.sqrt(2. / n)) - elif isinstance(m, nn.BatchNorm2d): - m.weight.data.fill_(1) - m.bias.data.zero_() - elif isinstance(m, nn.Linear): - init.kaiming_normal_(m.weight) - m.bias.data.zero_() - - def _make_layer(self, block, planes, blocks, stride=1): - downsample = None - if stride != 1 or self.inplanes != planes * block.expansion: - downsample = nn.Sequential( - nn.Conv2d( - self.inplanes, - planes * block.expansion, - kernel_size=1, - stride=stride, - bias=False), - nn.BatchNorm2d(planes * block.expansion), - ) - - layers = [] - layers.append( - block(self.inplanes, planes, self.cardinality, self.base_width, stride, - downsample)) - self.inplanes = planes * block.expansion - for _ in range(1, blocks): - layers.append( - block(self.inplanes, planes, self.cardinality, self.base_width)) - - return nn.Sequential(*layers) - - def forward(self, x): - x = self.conv_1_3x3(x) - x = F.relu(self.bn_1(x), inplace=True) - x = self.stage_1(x) - x = self.stage_2(x) - x = self.stage_3(x) - x = self.avgpool(x) - x = x.view(x.size(0), -1) - return self.classifier(x) diff --git a/robustbench/model_zoo/architectures/robust_wide_resnet.py b/robustbench/model_zoo/architectures/robust_wide_resnet.py deleted file mode 100644 index bffa5db7..00000000 --- a/robustbench/model_zoo/architectures/robust_wide_resnet.py +++ /dev/null @@ -1,54 +0,0 @@ -import torch -import math -import torch.nn.functional as F -from torch import nn -from .wide_resnet import WideResNet, NetworkBlock, BasicBlock - - -class RobustWideResNet(nn.Module): - def __init__(self, num_classes=10, channel_configs=[16, 160, 320, 640], - depth_configs=[5, 5, 5], stride_config=[1, 2, 2], - drop_rate_config=[0.0, 0.0, 0.0]): - super(RobustWideResNet, self).__init__() - assert len(channel_configs) - 1 == len(depth_configs) == len(stride_config) == len(drop_rate_config) - self.channel_configs = channel_configs - self.depth_configs = depth_configs - self.stride_config = stride_config - - self.stem_conv = nn.Conv2d(3, channel_configs[0], kernel_size=3, - stride=1, padding=1, bias=False) - self.blocks = nn.ModuleList([]) - for i, stride in enumerate(stride_config): - self.blocks.append(NetworkBlock(block=BasicBlock, - nb_layers=depth_configs[i], - in_planes=channel_configs[i], - out_planes=channel_configs[i+1], - stride=stride, - dropRate=drop_rate_config[i],)) - - # global average pooling and classifier - self.bn1 = nn.BatchNorm2d(channel_configs[-1]) - self.relu = nn.ReLU(inplace=True) - self.global_pooling = nn.AdaptiveAvgPool2d(1) - self.fc = nn.Linear(channel_configs[-1], num_classes) - self.fc_size = channel_configs[-1] - - for m in self.modules(): - if isinstance(m, nn.Conv2d): - n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels - m.weight.data.normal_(0, math.sqrt(2. / n)) - elif isinstance(m, nn.BatchNorm2d) or isinstance(m, nn.GroupNorm): - m.weight.data.fill_(1) - m.bias.data.zero_() - elif isinstance(m, nn.Linear): - m.bias.data.zero_() - - def forward(self, x): - out = self.stem_conv(x) - for i, block in enumerate(self.blocks): - out = block(out) - out = self.relu(self.bn1(out)) - out = self.global_pooling(out) - out = out.view(-1, self.fc_size) - out = self.fc(out) - return out diff --git a/robustbench/model_zoo/architectures/utils_architectures.py b/robustbench/model_zoo/architectures/utils_architectures.py deleted file mode 100644 index f64307fd..00000000 --- a/robustbench/model_zoo/architectures/utils_architectures.py +++ /dev/null @@ -1,38 +0,0 @@ -import torch -import torch.nn as nn -from collections import OrderedDict -from typing import Tuple, TypeVar -from torch import Tensor - - -class ImageNormalizer(nn.Module): - - def __init__(self, mean: Tuple[float, float, float], - std: Tuple[float, float, float]) -> None: - super(ImageNormalizer, self).__init__() - - self.register_buffer('mean', torch.as_tensor(mean).view(1, 3, 1, 1)) - self.register_buffer('std', torch.as_tensor(std).view(1, 3, 1, 1)) - - def forward(self, input: Tensor) -> Tensor: - return (input - self.mean) / self.std - - def __repr__(self): - return f'ImageNormalizer(mean={self.mean.squeeze()}, std={self.std.squeeze()})' # type: ignore - - -def normalize_model(model: nn.Module, mean: Tuple[float, float, float], - std: Tuple[float, float, float]) -> nn.Module: - layers = OrderedDict([('normalize', ImageNormalizer(mean, std)), - ('model', model)]) - return nn.Sequential(layers) - - -M = TypeVar('M', bound=nn.Module) - - -# def normalize_timm_model(model: M) -> M: -# return normalize_model( -# model, -# model.default_cfg['mean'], # type: ignore -# model.default_cfg['std']) # type: ignore diff --git a/robustbench/model_zoo/architectures/wide_resnet.py b/robustbench/model_zoo/architectures/wide_resnet.py deleted file mode 100644 index b38308bc..00000000 --- a/robustbench/model_zoo/architectures/wide_resnet.py +++ /dev/null @@ -1,95 +0,0 @@ -import math -import torch -import torch.nn as nn -import torch.nn.functional as F - - -class BasicBlock(nn.Module): - def __init__(self, in_planes, out_planes, stride, dropRate=0.0): - super(BasicBlock, self).__init__() - self.bn1 = nn.BatchNorm2d(in_planes) - self.relu1 = nn.ReLU(inplace=True) - self.conv1 = nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride, - padding=1, bias=False) - self.bn2 = nn.BatchNorm2d(out_planes) - self.relu2 = nn.ReLU(inplace=True) - self.conv2 = nn.Conv2d(out_planes, out_planes, kernel_size=3, stride=1, - padding=1, bias=False) - self.droprate = dropRate - self.equalInOut = (in_planes == out_planes) - self.convShortcut = (not self.equalInOut) and nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride, - padding=0, bias=False) or None - - def forward(self, x): - if not self.equalInOut: - x = self.relu1(self.bn1(x)) - else: - out = self.relu1(self.bn1(x)) - out = self.relu2(self.bn2(self.conv1(out if self.equalInOut else x))) - if self.droprate > 0: - out = F.dropout(out, p=self.droprate, training=self.training) - out = self.conv2(out) - return torch.add(x if self.equalInOut else self.convShortcut(x), out) - - -class NetworkBlock(nn.Module): - def __init__(self, nb_layers, in_planes, out_planes, block, stride, dropRate=0.0): - super(NetworkBlock, self).__init__() - self.layer = self._make_layer(block, in_planes, out_planes, nb_layers, stride, dropRate) - - def _make_layer(self, block, in_planes, out_planes, nb_layers, stride, dropRate): - layers = [] - for i in range(int(nb_layers)): - layers.append(block(i == 0 and in_planes or out_planes, out_planes, i == 0 and stride or 1, dropRate)) - return nn.Sequential(*layers) - - def forward(self, x): - return self.layer(x) - - -class WideResNet(nn.Module): - """ Based on code from https://github.com/yaodongyu/TRADES """ - def __init__(self, depth=28, num_classes=10, widen_factor=10, sub_block1=False, dropRate=0.0, bias_last=True): - super(WideResNet, self).__init__() - nChannels = [16, 16 * widen_factor, 32 * widen_factor, 64 * widen_factor] - assert ((depth - 4) % 6 == 0) - n = (depth - 4) / 6 - block = BasicBlock - # 1st conv before any network block - self.conv1 = nn.Conv2d(3, nChannels[0], kernel_size=3, stride=1, - padding=1, bias=False) - # 1st block - self.block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate) - if sub_block1: - # 1st sub-block - self.sub_block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate) - # 2nd block - self.block2 = NetworkBlock(n, nChannels[1], nChannels[2], block, 2, dropRate) - # 3rd block - self.block3 = NetworkBlock(n, nChannels[2], nChannels[3], block, 2, dropRate) - # global average pooling and classifier - self.bn1 = nn.BatchNorm2d(nChannels[3]) - self.relu = nn.ReLU(inplace=True) - self.fc = nn.Linear(nChannels[3], num_classes, bias=bias_last) - self.nChannels = nChannels[3] - - for m in self.modules(): - if isinstance(m, nn.Conv2d): - n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels - m.weight.data.normal_(0, math.sqrt(2. / n)) - elif isinstance(m, nn.BatchNorm2d): - m.weight.data.fill_(1) - m.bias.data.zero_() - elif isinstance(m, nn.Linear) and not m.bias is None: - m.bias.data.zero_() - - def forward(self, x): - out = self.conv1(x) - out = self.block1(out) - out = self.block2(out) - out = self.block3(out) - out = self.relu(self.bn1(out)) - out = F.avg_pool2d(out, 8) - out = out.view(-1, self.nChannels) - return self.fc(out) - diff --git a/robustbench/model_zoo/cifar10.py b/robustbench/model_zoo/cifar10.py deleted file mode 100644 index a82bc3bf..00000000 --- a/robustbench/model_zoo/cifar10.py +++ /dev/null @@ -1,1048 +0,0 @@ -from collections import OrderedDict - -# import timm -import torch -from torch import nn - -from robustbench.model_zoo.architectures.dm_wide_resnet import CIFAR10_MEAN, CIFAR10_STD, \ - DMWideResNet, Swish, DMPreActResNet -from robustbench.model_zoo.architectures.resnet import Bottleneck, BottleneckChen2020AdversarialNet, \ - PreActBlock, PreActBlockV2, PreActResNet, ResNet, ResNet18, BasicBlock -from robustbench.model_zoo.architectures.resnext import CifarResNeXt, \ - ResNeXtBottleneck -from robustbench.model_zoo.architectures.resnest import ResNest152 -from robustbench.model_zoo.architectures.wide_resnet import WideResNet -from robustbench.model_zoo.architectures.robust_wide_resnet import RobustWideResNet -from robustbench.model_zoo.architectures.boosting_wide_resnet import BoostingWideResNet -from robustbench.model_zoo.enums import ThreatModel -from robustbench.model_zoo.architectures.CARD_resnet import LRR_ResNet, WidePreActResNet -from robustbench.model_zoo.architectures.paf_wide_resnet import pssilu_wrn_28_10 - - -class Hendrycks2020AugMixResNeXtNet(CifarResNeXt): - - def __init__(self, depth=29, num_classes=10, cardinality=4, base_width=32): - super().__init__(ResNeXtBottleneck, - depth=depth, - num_classes=num_classes, - cardinality=cardinality, - base_width=base_width) - self.register_buffer('mu', torch.tensor([0.5] * 3).view(1, 3, 1, 1)) - self.register_buffer('sigma', torch.tensor([0.5] * 3).view(1, 3, 1, 1)) - - def forward(self, x): - x = (x - self.mu) / self.sigma - return super().forward(x) - - -class Hendrycks2020AugMixWRNNet(WideResNet): - - def __init__(self, depth=40, widen_factor=2): - super().__init__(depth=depth, - widen_factor=widen_factor, - sub_block1=False) - self.register_buffer('mu', torch.tensor([0.5] * 3).view(1, 3, 1, 1)) - self.register_buffer('sigma', torch.tensor([0.5] * 3).view(1, 3, 1, 1)) - - def forward(self, x): - x = (x - self.mu) / self.sigma - return super().forward(x) - - -class Hendrycks2019UsingNet(WideResNet): - - def __init__(self, depth=28, widen_factor=10): - super(Hendrycks2019UsingNet, self).__init__(depth=depth, - widen_factor=widen_factor, - sub_block1=False) - - def forward(self, x): - x = 2. * x - 1. - return super(Hendrycks2019UsingNet, self).forward(x) - - -class Rice2020OverfittingNet(WideResNet): - - def __init__(self, depth=34, widen_factor=20): - super(Rice2020OverfittingNet, self).__init__(depth=depth, - widen_factor=widen_factor, - sub_block1=False) - self.register_buffer( - 'mu', - torch.tensor([0.4914, 0.4822, 0.4465]).view(1, 3, 1, 1)) - self.register_buffer( - 'sigma', - torch.tensor([0.2471, 0.2435, 0.2616]).view(1, 3, 1, 1)) - - def forward(self, x): - x = (x - self.mu) / self.sigma - return super(Rice2020OverfittingNet, self).forward(x) - - -class Engstrom2019RobustnessNet(ResNet): - - def __init__(self): - super(Engstrom2019RobustnessNet, - self).__init__(Bottleneck, [3, 4, 6, 3]) - self.register_buffer( - 'mu', - torch.tensor([0.4914, 0.4822, 0.4465]).view(1, 3, 1, 1)) - self.register_buffer( - 'sigma', - torch.tensor([0.2023, 0.1994, 0.2010]).view(1, 3, 1, 1)) - - def forward(self, x): - x = (x - self.mu) / self.sigma - return super(Engstrom2019RobustnessNet, self).forward(x) - - -class Chen2020AdversarialNet(nn.Module): - - def __init__(self): - super(Chen2020AdversarialNet, self).__init__() - self.branch1 = ResNet(BottleneckChen2020AdversarialNet, [3, 4, 6, 3]) - self.branch2 = ResNet(BottleneckChen2020AdversarialNet, [3, 4, 6, 3]) - self.branch3 = ResNet(BottleneckChen2020AdversarialNet, [3, 4, 6, 3]) - - self.models = nn.ModuleList([self.branch1, self.branch2, self.branch3]) - - self.register_buffer( - 'mu', - torch.tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1)) - self.register_buffer( - 'sigma', - torch.tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1)) - - def forward(self, x): - out = (x - self.mu) / self.sigma - - out1 = self.branch1(out) - out2 = self.branch2(out) - out3 = self.branch3(out) - - prob1 = torch.softmax(out1, dim=1) - prob2 = torch.softmax(out2, dim=1) - prob3 = torch.softmax(out3, dim=1) - - return (prob1 + prob2 + prob3) / 3 - - -class Wong2020FastNet(PreActResNet): - - def __init__(self): - super(Wong2020FastNet, self).__init__(PreActBlock, [2, 2, 2, 2]) - self.register_buffer( - 'mu', - torch.tensor([0.4914, 0.4822, 0.4465]).view(1, 3, 1, 1)) - self.register_buffer( - 'sigma', - torch.tensor([0.2471, 0.2435, 0.2616]).view(1, 3, 1, 1)) - - def forward(self, x): - x = (x - self.mu) / self.sigma - return super(Wong2020FastNet, self).forward(x) - - -class Ding2020MMANet(WideResNet): - """ - See the appendix of the LICENSE file specifically for this model. - """ - - def __init__(self, depth=28, widen_factor=4): - super(Ding2020MMANet, self).__init__(depth=depth, - widen_factor=widen_factor, - sub_block1=False) - - def forward(self, x): - mu = x.mean(dim=(1, 2, 3), keepdim=True) - std = x.std(dim=(1, 2, 3), keepdim=True) - std_min = torch.ones_like(std) / (x.shape[1] * x.shape[2] * - x.shape[3])**.5 - x = (x - mu) / torch.max(std, std_min) - return super(Ding2020MMANet, self).forward(x) - - -class Augustin2020AdversarialNet(ResNet): - - def __init__(self): - super(Augustin2020AdversarialNet, - self).__init__(Bottleneck, [3, 4, 6, 3]) - self.register_buffer( - 'mu', - torch.tensor( - [0.4913997551666284, 0.48215855929893703, - 0.4465309133731618]).view(1, 3, 1, 1)) - self.register_buffer( - 'sigma', - torch.tensor( - [0.24703225141799082, 0.24348516474564, - 0.26158783926049628]).view(1, 3, 1, 1)) - - def forward(self, x): - x = (x - self.mu) / self.sigma - return super(Augustin2020AdversarialNet, self).forward(x) - - -class Augustin2020AdversarialWideNet(WideResNet): - - def __init__(self, depth=34, widen_factor=10): - super(Augustin2020AdversarialWideNet, - self).__init__(depth=depth, - widen_factor=widen_factor, - sub_block1=False) - self.register_buffer( - 'mu', - torch.tensor( - [0.4913997551666284, 0.48215855929893703, - 0.4465309133731618]).view(1, 3, 1, 1)) - self.register_buffer( - 'sigma', - torch.tensor( - [0.24703225141799082, 0.24348516474564, - 0.26158783926049628]).view(1, 3, 1, 1)) - - def forward(self, x): - x = (x - self.mu) / self.sigma - return super(Augustin2020AdversarialWideNet, self).forward(x) - - -class Rice2020OverfittingNetL2(PreActResNet): - - def __init__(self): - super(Rice2020OverfittingNetL2, self).__init__(PreActBlockV2, - [2, 2, 2, 2], - bn_before_fc=True) - self.register_buffer( - 'mu', - torch.tensor([0.4914, 0.4822, 0.4465]).view(1, 3, 1, 1)) - self.register_buffer( - 'sigma', - torch.tensor([0.2471, 0.2435, 0.2616]).view(1, 3, 1, 1)) - - def forward(self, x): - x = (x - self.mu) / self.sigma - return super(Rice2020OverfittingNetL2, self).forward(x) - - -class Rony2019DecouplingNet(WideResNet): - - def __init__(self, depth=28, widen_factor=10): - super(Rony2019DecouplingNet, self).__init__(depth=depth, - widen_factor=widen_factor, - sub_block1=False) - self.register_buffer( - 'mu', - torch.tensor([0.491, 0.482, 0.447]).view(1, 3, 1, 1)) - self.register_buffer( - 'sigma', - torch.tensor([0.247, 0.243, 0.262]).view(1, 3, 1, 1)) - - def forward(self, x): - x = (x - self.mu) / self.sigma - return super(Rony2019DecouplingNet, self).forward(x) - - -class Kireev2021EffectivenessNet(PreActResNet): - - def __init__(self): - super(Kireev2021EffectivenessNet, self).__init__(PreActBlockV2, - [2, 2, 2, 2], - bn_before_fc=True) - self.register_buffer( - 'mu', - torch.tensor([0.4914, 0.4822, 0.4465]).view(1, 3, 1, 1)) - self.register_buffer( - 'sigma', - torch.tensor([0.2471, 0.2435, 0.2616]).view(1, 3, 1, 1)) - - def forward(self, x): - x = (x - self.mu) / self.sigma - return super(Kireev2021EffectivenessNet, self).forward(x) - - -class Chen2020EfficientNet(WideResNet): - - def __init__(self, depth=34, widen_factor=10): - super().__init__(depth=depth, - widen_factor=widen_factor, - sub_block1=True) - self.register_buffer( - 'mu', - torch.tensor([0.4914, 0.4822, 0.4465]).view(1, 3, 1, 1)) - self.register_buffer( - 'sigma', - torch.tensor([0.2471, 0.2435, 0.2616]).view(1, 3, 1, 1)) - - def forward(self, x): - x = (x - self.mu) / self.sigma - return super().forward(x) - - -class Diffenderfer2021CARD(LRR_ResNet): - - def __init__(self, width=128): - super(Diffenderfer2021CARD, self).__init__(width=width) - self.register_buffer( - 'mu', - torch.tensor([0.491, 0.482, 0.447]).view(1, 3, 1, 1)) - self.register_buffer( - 'sigma', - torch.tensor([0.247, 0.243, 0.262]).view(1, 3, 1, 1)) - - def forward(self, x): - x = (x - self.mu) / self.sigma - return super().forward(x) - - -class Diffenderfer2021CARD_Deck(nn.Module): - - def __init__(self, width=128): - super(Diffenderfer2021CARD_Deck, self).__init__() - self.num_cards = 6 - self.models = nn.ModuleList() - - for i in range(self.num_cards): - self.models.append(LRR_ResNet(width=width)) - - self.register_buffer( - 'mu', - torch.tensor([0.491, 0.482, 0.447]).view(1, 3, 1, 1)) - self.register_buffer( - 'sigma', - torch.tensor([0.247, 0.243, 0.262]).view(1, 3, 1, 1)) - - def forward(self, x): - x = (x - self.mu) / self.sigma - - x_cl = x.clone( - ) # clone to make sure x is not changed by inplace methods - out_list = [] - for i in range(self.num_cards): - # Evaluate model i at input - out = self.models[i](x_cl) - # Compute softmax - out = torch.softmax(out, dim=1) - # Append output to list of logits - out_list.append(out) - - return torch.mean(torch.stack(out_list), dim=0) - - -class Diffenderfer2021CARD_Binary(WidePreActResNet): - - def __init__(self, num_classes=10): - super(Diffenderfer2021CARD_Binary, - self).__init__(num_classes=num_classes) - self.register_buffer( - 'mu', - torch.tensor([0.491, 0.482, 0.447]).view(1, 3, 1, 1)) - self.register_buffer( - 'sigma', - torch.tensor([0.247, 0.243, 0.262]).view(1, 3, 1, 1)) - - def forward(self, x): - x = (x - self.mu) / self.sigma - return super().forward(x) - - -class Diffenderfer2021CARD_Deck_Binary(torch.nn.Module): - - def __init__(self, num_classes=10): - super(Diffenderfer2021CARD_Deck_Binary, self).__init__() - self.num_cards = 6 - self.models = nn.ModuleList() - - for i in range(self.num_cards): - self.models.append(WidePreActResNet(num_classes=num_classes)) - - self.register_buffer( - 'mu', - torch.tensor([0.491, 0.482, 0.447]).view(1, 3, 1, 1)) - self.register_buffer( - 'sigma', - torch.tensor([0.247, 0.243, 0.262]).view(1, 3, 1, 1)) - - def forward(self, x): - x = (x - self.mu) / self.sigma - - x_cl = x.clone( - ) # clone to make sure x is not changed by inplace methods - out_list = [] - for i in range(self.num_cards): - # Evaluate model i at input - out = self.models[i](x_cl) - # Compute softmax - out = torch.softmax(out, dim=1) - # Append output to list of logits - out_list.append(out) - - return torch.mean(torch.stack(out_list), dim=0) - - -class Modas2021PRIMEResNet18(ResNet): - - def __init__(self, num_classes=10): - super().__init__(BasicBlock, [2, 2, 2, 2], num_classes=num_classes) - - # mu & sigma are updated from weights - self.register_buffer('mu', torch.tensor([0.5] * 3).view(1, 3, 1, 1)) - self.register_buffer('sigma', torch.tensor([0.5] * 3).view(1, 3, 1, 1)) - - def forward(self, x): - x = (x - self.mu) / self.sigma - return super().forward(x) - - -linf = OrderedDict( - [ - ('Andriushchenko2020Understanding', { - 'model': lambda: PreActResNet(PreActBlock, [2, 2, 2, 2]), - 'gdrive_id': '1Uyvprd98bIyxfMjLdCZwm-NEJ-6GMVis', - }), - ('Carmon2019Unlabeled', { - 'model': - lambda: WideResNet(depth=28, widen_factor=10, sub_block1=True), - 'gdrive_id': - '15tUx-gkZMYx7BfEOw1GY5OKC-jECIsPQ', - }), - ('Sehwag2020Hydra', { - 'model': - lambda: WideResNet(depth=28, widen_factor=10, sub_block1=True), - 'gdrive_id': - '1pi8GHwAVkxVH41hEnf0IAJb_7y-Q8a2Y', - }), - ('Wang2020Improving', { - 'model': - lambda: WideResNet(depth=28, widen_factor=10, sub_block1=True), - 'gdrive_id': - '1T939mU4kXYt5bbvM55aT4fLBvRhyzjiQ', - }), - ('Hendrycks2019Using', { - 'model': Hendrycks2019UsingNet, - 'gdrive_id': '1-DcJsYw2dNEOyF9epks2QS7r9nqBDEsw', - }), - ('Rice2020Overfitting', { - 'model': Rice2020OverfittingNet, - 'gdrive_id': '1vC_Twazji7lBjeMQvAD9uEQxi9Nx2oG-', - }), - ('Zhang2019Theoretically', { - 'model': - lambda: WideResNet(depth=34, widen_factor=10, sub_block1=True), - 'gdrive_id': - '1hPz9QQwyM7QSuWu-ANG_uXR-29xtL8t_', - }), - ('Engstrom2019Robustness', { - 'model': Engstrom2019RobustnessNet, - 'gdrive_id': '1etqmQsksNIWBvBQ4r8ZFk_3FJlLWr8Rr', - }), - ('Chen2020Adversarial', { - 'model': - Chen2020AdversarialNet, - 'gdrive_id': [ - '1HrG22y_A9F0hKHhh2cLLvKxsQTJTLE_y', - '1DB2ymt0rMnsMk5hTuUzoMTpMKEKWpExd', - '1GfgzNZcC190-IrT7056IZFDB6LfMUL9m' - ], - }), - ('Huang2020Self', { - 'model': - lambda: WideResNet(depth=34, widen_factor=10, sub_block1=True), - 'gdrive_id': - '1nInDeIyZe2G-mJFxQJ3UoclQNomWjMgm', - }), - ('Pang2020Boosting', { - 'model': BoostingWideResNet, - 'gdrive_id': '1iNWOj3MP7kGe8yTAS4XnDaDXDLt0mwqw', - }), - ('Wong2020Fast', { - 'model': Wong2020FastNet, - 'gdrive_id': '1Re--_lf3jCEw9bnQqGkjw3J7v2tSZKrv', - }), - ('Ding2020MMA', { - 'model': Ding2020MMANet, - 'gdrive_id': '19Q_rIIHXsYzxZ0WcZdqT-N2OD7MfgoZ0', - }), - ('Zhang2019You', { - 'model': - lambda: WideResNet(depth=34, widen_factor=10, sub_block1=True), - 'gdrive_id': - '1kB2qqPQ8qUNmK8VKuTOhT1X4GT46kAoA', - }), - ('Zhang2020Attacks', { - 'model': - lambda: WideResNet(depth=34, widen_factor=10, sub_block1=True), - 'gdrive_id': - '1lBVvLG6JLXJgQP2gbsTxNHl6s3YAopqk', - }), - ('Wu2020Adversarial_extra', { - 'model': - lambda: WideResNet(depth=28, widen_factor=10, sub_block1=True), - 'gdrive_id': - '1-WJWpAZLlmc4gJ8XXNf7IETjnSZzaCNp', - }), - ('Wu2020Adversarial', { - 'model': lambda: WideResNet(depth=34, widen_factor=10), - 'gdrive_id': '13LBcgNvhFppCFG22i1xATrahFPfMgXGf', - }), - ('Gowal2020Uncovering_70_16', { - 'model': - lambda: DMWideResNet(num_classes=10, - depth=70, - width=16, - activation_fn=Swish, - mean=CIFAR10_MEAN, - std=CIFAR10_STD), - 'gdrive_id': - "1DVwKclibqzniE2Ss5_g6BY77ChG8QKzl" - }), - ('Gowal2020Uncovering_70_16_extra', { - 'model': - lambda: DMWideResNet(num_classes=10, - depth=70, - width=16, - activation_fn=Swish, - mean=CIFAR10_MEAN, - std=CIFAR10_STD), - 'gdrive_id': - "1GxryYj_Or-VCDca0wgiFLz4ssXSZXQoJ" - }), - ('Gowal2020Uncovering_34_20', { - 'model': - lambda: DMWideResNet(num_classes=10, - depth=34, - width=20, - activation_fn=Swish, - mean=CIFAR10_MEAN, - std=CIFAR10_STD), - 'gdrive_id': - "1YWvZO1u9_yNLFNC3JYd_TVkvrRSMER1O" - }), - ('Gowal2020Uncovering_28_10_extra', { - 'model': - lambda: DMWideResNet(num_classes=10, - depth=28, - width=10, - activation_fn=Swish, - mean=CIFAR10_MEAN, - std=CIFAR10_STD), - 'gdrive_id': - "1MBAWGxiZxKt-GfqEqtLcXcd3tAxPhvV2" - }), - ('Sehwag2021Proxy', { - 'model': lambda: WideResNet(34, 10, sub_block1=False), - 'gdrive_id': '1QFA5fPMj2Qw4aYNG33PkFqiv_RTDWvzm', - }), - ('Sehwag2021Proxy_R18', { - 'model': ResNet18, - 'gdrive_id': '1-ZgoSlD_AMhtXdnUElilxVXnzK2DcHuu', - }), - ('Sehwag2021Proxy_ResNest152', { - 'model': ResNest152, - 'gdrive_id': '1XSjtJZAvDlua6wTM6WRLvW_jON-DqLgT', - }), - ('Sitawarin2020Improving', { - 'model': - lambda: WideResNet(depth=34, widen_factor=10, sub_block1=True), - 'gdrive_id': - '12teknvo6dQGSWBaGnbNFwFO3-Y8j2eB6', - }), - ('Chen2020Efficient', { - 'model': Chen2020EfficientNet, - 'gdrive_id': '1c5EXpd3Kn_s6qQIbkLX3tTOOPC8VslHg', - }), - ('Cui2020Learnable_34_20', { - 'model': - lambda: WideResNet(depth=34, widen_factor=20, sub_block1=True), - 'gdrive_id': - '1y7BUxPhQjNlb4w4BUlDyYJIS4w4fsGiS' - }), - ('Cui2020Learnable_34_10', { - 'model': - lambda: WideResNet(depth=34, widen_factor=10, sub_block1=True), - 'gdrive_id': - '16s9pi_1QgMbFLISVvaVUiNfCzah6g2YV' - }), - ('Zhang2020Geometry', { - 'model': - lambda: WideResNet(depth=28, widen_factor=10, sub_block1=True), - 'gdrive_id': - '1UoG1JhbAps1MdMc6PEFiZ2yVXl_Ii5Jk' - }), - ('Rebuffi2021Fixing_28_10_cutmix_ddpm', { - 'model': - lambda: DMWideResNet(num_classes=10, - depth=28, - width=10, - activation_fn=Swish, - mean=CIFAR10_MEAN, - std=CIFAR10_STD), - 'gdrive_id': - '1-0EChXbc6pOvx26O17av263bCeqIAz6s' - }), - ('Rebuffi2021Fixing_106_16_cutmix_ddpm', { - 'model': - lambda: DMWideResNet(num_classes=10, - depth=106, - width=16, - activation_fn=Swish, - mean=CIFAR10_MEAN, - std=CIFAR10_STD), - 'gdrive_id': - '1-4qnkveIkeWoGdF72kpEFHETiY3y4_tF' - }), - ('Rebuffi2021Fixing_70_16_cutmix_ddpm', { - 'model': - lambda: DMWideResNet(num_classes=10, - depth=70, - width=16, - activation_fn=Swish, - mean=CIFAR10_MEAN, - std=CIFAR10_STD), - 'gdrive_id': - '1-8CWRT-OFWyrz4T4s0I2mbFjPg8K_MUi' - }), - ('Rebuffi2021Fixing_70_16_cutmix_extra', { - 'model': - lambda: DMWideResNet(num_classes=10, - depth=70, - width=16, - activation_fn=Swish, - mean=CIFAR10_MEAN, - std=CIFAR10_STD), - 'gdrive_id': - '1qKDTp6IJ1BUXZaRtbYuo_t0tuDl_4mLg' - }), - ('Sridhar2021Robust', { - 'model': - lambda: WideResNet(depth=28, widen_factor=10, sub_block1=True), - 'gdrive_id': - '1muDMpOyRlgJ7n2rhS2NpfFGp3rzjuIu0' - }), - ('Sridhar2021Robust_34_15', { - 'model': - lambda: WideResNet(depth=34, widen_factor=15, sub_block1=True), - 'gdrive_id': - '1-3ii3GX93YqIcmJ3VNsOgYA7ecdnSZ0Z', - }), - ('Rebuffi2021Fixing_R18_ddpm', { - 'model': - lambda: DMPreActResNet(num_classes=10, - depth=18, - width=0, - activation_fn=Swish, - mean=CIFAR10_MEAN, - std=CIFAR10_STD), - 'gdrive_id': - '1--dxE66AsgBSUsuK2sXCTrsYUV9B5f95' - }), - ('Rade2021Helper_R18_extra', { - 'model': - lambda: DMPreActResNet(num_classes=10, - depth=18, - width=0, - activation_fn=Swish, - mean=CIFAR10_MEAN, - std=CIFAR10_STD), - 'gdrive_id': - '1hdXk1rPJql2Oa84Kky64fMTQzng5UcTL' - }), - ('Rade2021Helper_R18_ddpm', { - 'model': - lambda: DMPreActResNet(num_classes=10, - depth=18, - width=0, - activation_fn=Swish, - mean=CIFAR10_MEAN, - std=CIFAR10_STD), - 'gdrive_id': - '1f2yJUo-jxCQNk589frzriv6wPyrQEZdX' - }), - ('Rade2021Helper_extra', { - 'model': - lambda: DMWideResNet(num_classes=10, - depth=34, - width=10, - activation_fn=Swish, - mean=CIFAR10_MEAN, - std=CIFAR10_STD), - 'gdrive_id': - '1GhAp-0C3ONRy9BxIe0J9vKc082vHvR7t' - }), - ('Rade2021Helper_ddpm', { - 'model': - lambda: DMWideResNet(num_classes=10, - depth=28, - width=10, - activation_fn=Swish, - mean=CIFAR10_MEAN, - std=CIFAR10_STD), - 'gdrive_id': - '1AOF6LxnwgS5fCz_lVLYqs_wnUYuv6O7z' - }), - ('Huang2021Exploring', { - 'model': - lambda: RobustWideResNet(num_classes=10, - channel_configs=[16, 320, 640, 512], - depth_configs=[5, 5, 5]), - 'gdrive_id': - '1-2ram-xtoEidOh1SCYY5KTiyKieINkZe' - }), - ('Huang2021Exploring_ema', { - 'model': - lambda: RobustWideResNet(num_classes=10, - channel_configs=[16, 320, 640, 512], - depth_configs=[5, 5, 5]), - 'gdrive_id': - '1-GRwO5t9HxOS2y6RFK8QEsDXjdcgmVu6' - }), - ('Addepalli2021Towards_RN18', { - 'model': lambda: ResNet18(), - 'gdrive_id': '1-1DxecXz5U_xZ54DVdE-GVm71Tiox-Ri' - }), - ('Addepalli2021Towards_WRN34', { - 'model': - lambda: WideResNet(num_classes=10, depth=34, sub_block1=True), - 'gdrive_id': '1-3vgjTNfSq7LSMKuayEQ-jLflAP196dB' - }), - ('Gowal2021Improving_70_16_ddpm_100m', { - 'model': - lambda: DMWideResNet(num_classes=10, - depth=70, - width=16, - activation_fn=Swish, - mean=CIFAR10_MEAN, - std=CIFAR10_STD), - 'gdrive_id': - '10qBoDG_NomJBrRFeTsSwEcJF1GA-sjC4' - }), - ('Dai2021Parameterizing', { - 'model': lambda: pssilu_wrn_28_10(num_classes=10), - 'gdrive_id': '1eO-MNXQSAoCuNFjIbdCheprT4Beqo9Zv' - }), - ('Gowal2021Improving_28_10_ddpm_100m', { - 'model': - lambda: DMWideResNet(num_classes=10, - depth=28, - width=10, - activation_fn=Swish, - mean=CIFAR10_MEAN, - std=CIFAR10_STD), - 'gdrive_id': - '12L8YE6VBgUDKyW6GMSNefSYj2gg4LEKx' - }), - ('Gowal2021Improving_R18_ddpm_100m', { - 'model': - lambda: DMPreActResNet(num_classes=10, - depth=18, - width=0, - activation_fn=Swish, - mean=CIFAR10_MEAN, - std=CIFAR10_STD), - 'gdrive_id': - '1-0EuCJashqSXEkkd1DOzFA4tH8KL2kim' - }), - ('Chen2021LTD_WRN34_10', { - 'model': - lambda: WideResNet(depth=34, widen_factor=10, sub_block1=False), - 'gdrive_id': - '1-0RoQKYvHLNh7hZ71wJjSit1XtrJQo9D' - }), - ('Chen2021LTD_WRN34_20', { - 'model': - lambda: WideResNet(depth=34, widen_factor=20, sub_block1=False), - 'gdrive_id': - '1-5JmY9rapuGwCUc2nPvy7Rb_sn_dHhFm' - }), - ('Standard', { - 'model': lambda: WideResNet(depth=28, widen_factor=10), - 'gdrive_id': '1t98aEuzeTL8P7Kpd5DIrCoCL21BNZUhC', - }), - # ('Kang2021Stable', { - # 'model': rebuffi_sodef, - # 'gdrive_id': '1-HjG9f7wJDnNRdMQSiz8dlCI3sq5mfqj', - # }), - ('Jia2022LAS-AT_34_10', { - 'model': lambda: WideResNet(depth=34, widen_factor=10), - 'gdrive_id': '1-3l7xKhIPyes3O4QSz0HU6L-hfOoS0xD', - }), - ('Jia2022LAS-AT_70_16', { - 'model': lambda: WideResNet(depth=70, widen_factor=16), - 'gdrive_id': '1-4p-Gr0hjl8wq6qvvTza4x4a5Rmu-Bfr', - }), - ('Pang2022Robustness_WRN28_10', { - 'model': - lambda: DMWideResNet(num_classes=10, - depth=28, - width=10, - activation_fn=Swish, - mean=CIFAR10_MEAN, - std=CIFAR10_STD), - 'gdrive_id': - '16ChNkterCp17BXv-xxqpfedb4u2_CjjS' - }), - ('Pang2022Robustness_WRN70_16', { - 'model': - lambda: DMWideResNet(num_classes=10, - depth=70, - width=16, - activation_fn=Swish, - mean=CIFAR10_MEAN, - std=CIFAR10_STD), - 'gdrive_id': - '1uQZYUuUiL9BzaQUeXLhjr_RhoyFRrHe3' - }), - ('Addepalli2022Efficient_RN18', { - 'model': ResNet18, - 'gdrive_id': '1m5vhdzIUUKhDbsZdOG9z76Eyp6f4xe_f', - }), - ('Addepalli2022Efficient_WRN_34_10', { - 'model': lambda: WideResNet(depth=34, widen_factor=10), - 'gdrive_id': '1--dVDtZhAk4D2zMtTDwIGnImuCGxTcBA', - }), - # ('Debenedetti2022Light_XCiT-S12', { - # 'model': (lambda: timm.create_model( - # 'debenedetti2020light_xcit_s_cifar10_linf', pretrained=True)), - # 'gdrive_id': - # None - # }), - # ('Debenedetti2022Light_XCiT-M12', { - # 'model': (lambda: timm.create_model( - # 'debenedetti2020light_xcit_m_cifar10_linf', pretrained=True)), - # 'gdrive_id': - # None - # }), - # ('Debenedetti2022Light_XCiT-L12', { - # 'model': (lambda: timm.create_model( - # 'debenedetti2020light_xcit_l_cifar10_linf', pretrained=True)), - # 'gdrive_id': - # None - # }), - ]) - -l2 = OrderedDict([ - ('Augustin2020Adversarial', { - 'model': Augustin2020AdversarialNet, - 'gdrive_id': '1oDghrzNfkStC2wr5Fq8T896yNV4wVG4d', - }), - ('Engstrom2019Robustness', { - 'model': Engstrom2019RobustnessNet, - 'gdrive_id': '1O8rGa6xOUIRwQ-M4ESrCjzknby8TM2ZE', - }), - ('Rice2020Overfitting', { - 'model': Rice2020OverfittingNetL2, - 'gdrive_id': '1jo-31utiYNBVzLM0NxUEWz0teo3Z0xa7', - }), - ('Rony2019Decoupling', { - 'model': Rony2019DecouplingNet, - 'gdrive_id': '1Oua2ZYSxNvoDrtlY9vTtRzyBWHziE4Uy', - }), - ('Standard', { - 'model': lambda: WideResNet(depth=28, widen_factor=10), - 'gdrive_id': '1t98aEuzeTL8P7Kpd5DIrCoCL21BNZUhC', - }), - ('Ding2020MMA', { - 'model': Ding2020MMANet, - 'gdrive_id': '13wgY0Q_eor52ltZ0PkfJx5BCZ8cLM52E', - }), - ('Wu2020Adversarial', { - 'model': lambda: WideResNet(depth=34, widen_factor=10), - 'gdrive_id': '1M5AZ0EZQt7d2AlTmsnqZcfx91-x7YEAV', - }), - ('Gowal2020Uncovering', { - 'model': - lambda: DMWideResNet(num_classes=10, - depth=70, - width=16, - activation_fn=Swish, - mean=CIFAR10_MEAN, - std=CIFAR10_STD), - 'gdrive_id': - "1QL4SNvYydjIg1uI3VP9SyNt-2kTXRisG" - }), - ('Gowal2020Uncovering_extra', { - 'model': - lambda: DMWideResNet(num_classes=10, - depth=70, - width=16, - activation_fn=Swish, - mean=CIFAR10_MEAN, - std=CIFAR10_STD), - 'gdrive_id': - "1pkZDCpCBShpAnx92n8PUeNOY1fSiTi0s" - }), - ('Sehwag2021Proxy', { - 'model': lambda: WideResNet(34, 10, sub_block1=False), - 'gdrive_id': '1UviikNzpltVFsgMuqQ8YhpmvGczGRS4S', - }), - ('Sehwag2021Proxy_R18', { - 'model': ResNet18, - 'gdrive_id': '1zPjjZj9wujBNkAmHHHIikem6_aIjMhXG', - }), - ('Rebuffi2021Fixing_70_16_cutmix_ddpm', { - 'model': - lambda: DMWideResNet(num_classes=10, - depth=70, - width=16, - activation_fn=Swish, - mean=CIFAR10_MEAN, - std=CIFAR10_STD), - 'gdrive_id': - '1-8ECIOYF4JB0ywxJOmhkefnv4TW-KuXp' - }), - ('Rebuffi2021Fixing_28_10_cutmix_ddpm', { - 'model': - lambda: DMWideResNet(num_classes=10, - depth=28, - width=10, - activation_fn=Swish, - mean=CIFAR10_MEAN, - std=CIFAR10_STD), - 'gdrive_id': - '1-DUKcvfDzeWwt0NK7q2XvU-dIi8up8B0' - }), - ('Rebuffi2021Fixing_70_16_cutmix_extra', { - 'model': - lambda: DMWideResNet(num_classes=10, - depth=70, - width=16, - activation_fn=Swish, - mean=CIFAR10_MEAN, - std=CIFAR10_STD), - 'gdrive_id': - '1JX82BDVBNO-Ffa2J37EuB8C-aFCbz708' - }), - ('Augustin2020Adversarial_34_10', { - 'model': Augustin2020AdversarialWideNet, - 'gdrive_id': '1qPsKS546mKcs71IEhzOS-kLpQFSFhaKL' - }), - ('Augustin2020Adversarial_34_10_extra', { - 'model': Augustin2020AdversarialWideNet, - 'gdrive_id': '1--1MFZja6C2iVWi9MgetYjnSIenRBLT-' - }), - ('Rebuffi2021Fixing_R18_cutmix_ddpm', { - 'model': - lambda: DMPreActResNet(num_classes=10, - depth=18, - width=0, - activation_fn=Swish, - mean=CIFAR10_MEAN, - std=CIFAR10_STD), - 'gdrive_id': - '1-AlwHsXU28tCOJsf9RKAZxVzbinzzQU3' - }), - ('Rade2021Helper_R18_ddpm', { - 'model': - lambda: DMPreActResNet(num_classes=10, - depth=18, - width=0, - activation_fn=Swish, - mean=CIFAR10_MEAN, - std=CIFAR10_STD), - 'gdrive_id': - '1VWrStAYy5CrUR18sjcpq_LKLpeqgUaoQ' - }), -]) - -common_corruptions = OrderedDict([ - ('Diffenderfer2021Winning_LRR', { - 'model': Diffenderfer2021CARD, - 'gdrive_id': '1-NFL1OxfXgC0peeAV7G60ohhESO-8G0c' - }), - ('Diffenderfer2021Winning_LRR_CARD_Deck', { - 'model': - Diffenderfer2021CARD_Deck, - 'gdrive_id': [ - '1-R56enDUZ3oY74zfmj8M5ogBsV2SEHFR', - '1-_3eqvtBxTn-Afvg4fB5d4lx1cDi_AqC', - '1-cY0IYzLQrXzTa3LQvl6d26KnocsSurs', - '1-phHz-8gnNjN-m08pT_dagsW7Fa5-aTQ', - '1-ryE4owBqQ1HiWS6L60eW5YTPII_88MH', - '1-vka-h7krlJOjN7SRKMW5aDY73_WuZI2' - ], - }), - ('Diffenderfer2021Winning_Binary', { - 'model': Diffenderfer2021CARD_Binary, - 'gdrive_id': '101VSovRuFA0M7idR9ioeVn-UxROdJUYu' - }), - ('Diffenderfer2021Winning_Binary_CARD_Deck', { - 'model': - Diffenderfer2021CARD_Deck_Binary, - 'gdrive_id': [ - '10BOwYD-JdPguAp3dc4owWcyTI_MRWzmG', - '10IuU66vHBNXVmzeY3VCfpRjGoMOpsGkl', - '10WGSGQ0EHaJ0QZcoS0fAPqzlqwzGY-o2', - '10X1mo5I6fKhlZzQQeLFbjnHvOHJupHeC', - '10d8-swRiQutUSbSKksGwKeZfn8poYF3y', - '10ilDm_fojXiQve_LaHApbNalBpk6v53Z' - ], - }), - ('Rebuffi2021Fixing_70_16_cutmix_extra_Linf', { - 'model': - lambda: DMWideResNet(num_classes=10, - depth=70, - width=16, - activation_fn=Swish, - mean=CIFAR10_MEAN, - std=CIFAR10_STD), - 'gdrive_id': - '1qKDTp6IJ1BUXZaRtbYuo_t0tuDl_4mLg' - }), - ('Rebuffi2021Fixing_70_16_cutmix_extra_L2', { - 'model': - lambda: DMWideResNet(num_classes=10, - depth=70, - width=16, - activation_fn=Swish, - mean=CIFAR10_MEAN, - std=CIFAR10_STD), - 'gdrive_id': - '1JX82BDVBNO-Ffa2J37EuB8C-aFCbz708' - }), - ('Hendrycks2020AugMix_WRN', { - 'model': Hendrycks2020AugMixWRNNet, - 'gdrive_id': "1wy7gSRsUZzCzj8QhmTbcnwmES_2kkNph" - }), - ('Hendrycks2020AugMix_ResNeXt', { - 'model': Hendrycks2020AugMixResNeXtNet, - 'gdrive_id': "1uGP3nZbL3LC160kOsxwkkt6tDd4qbZT1" - }), - ('Kireev2021Effectiveness_Gauss50percent', { - 'model': Kireev2021EffectivenessNet, - 'gdrive_id': '1zR6lwYLkO3TFSgeqvu_CMYTq_IS-eicQ', - }), - ('Kireev2021Effectiveness_AugMixNoJSD', { - 'model': Kireev2021EffectivenessNet, - 'gdrive_id': '1p_1v1Oa-FSrjHTAq63QX4WtLYETkcbdH', - }), - ('Kireev2021Effectiveness_RLAT', { - 'model': Kireev2021EffectivenessNet, - 'gdrive_id': '16bCDA_5Rhr6qMKHRAO5W-4nu9_10kFyF', - }), - ('Kireev2021Effectiveness_RLATAugMixNoJSD', { - 'model': Kireev2021EffectivenessNet, - 'gdrive_id': '1hgJuvLPSVQMbUczn8qnIphONlJePsWgU', - }), - ('Kireev2021Effectiveness_RLATAugMix', { - 'model': Kireev2021EffectivenessNet, - 'gdrive_id': '19HNTdqJiuNyqFqIarPejniJEjZ3RQ_nj', - }), - ('Standard', { - 'model': lambda: WideResNet(depth=28, widen_factor=10), - 'gdrive_id': '1t98aEuzeTL8P7Kpd5DIrCoCL21BNZUhC', - }), - ('Addepalli2021Towards_WRN34', { - 'model': lambda: WideResNet(num_classes=10, depth=34, sub_block1=True), - 'gdrive_id': '1-3vgjTNfSq7LSMKuayEQ-jLflAP196dB' - }), - ('Modas2021PRIMEResNet18', { - 'model': Modas2021PRIMEResNet18, - 'gdrive_id': '13oDyqi16FeXy5j4Vm6IghnjTVqp_XF5U' - }), - ('Addepalli2022Efficient_WRN_34_10', { - 'model': lambda: WideResNet(depth=34, widen_factor=10), - 'gdrive_id': '1--dVDtZhAk4D2zMtTDwIGnImuCGxTcBA', - }) -]) - -cifar_10_models = OrderedDict([(ThreatModel.Linf, linf), (ThreatModel.L2, l2), - (ThreatModel.corruptions, common_corruptions)]) diff --git a/robustbench/model_zoo/cifar100.py b/robustbench/model_zoo/cifar100.py deleted file mode 100644 index b0f98f5a..00000000 --- a/robustbench/model_zoo/cifar100.py +++ /dev/null @@ -1,533 +0,0 @@ -from collections import OrderedDict - -# import timm -import torch -from torch import nn - -from robustbench.model_zoo.architectures.dm_wide_resnet import CIFAR100_MEAN, CIFAR100_STD, \ - DMWideResNet, Swish, DMPreActResNet -from robustbench.model_zoo.architectures.resnet import PreActBlock, PreActResNet,PreActBlockV2, \ - ResNet, BasicBlock -from robustbench.model_zoo.architectures.resnext import CifarResNeXt, ResNeXtBottleneck -from robustbench.model_zoo.architectures.wide_resnet import WideResNet -from robustbench.model_zoo.enums import ThreatModel -from robustbench.model_zoo.architectures.CARD_resnet import LRR_ResNet, WidePreActResNet -# from robustbench.model_zoo.architectures import xcit - - -class Chen2020EfficientNet(WideResNet): - - def __init__(self, depth=34, widen_factor=10): - super().__init__(depth=depth, - widen_factor=widen_factor, - sub_block1=True, - num_classes=100) - self.register_buffer( - 'mu', - torch.tensor([0.5071, 0.4867, 0.4408]).view(1, 3, 1, 1)) - self.register_buffer( - 'sigma', - torch.tensor([0.2675, 0.2565, 0.2761]).view(1, 3, 1, 1)) - - def forward(self, x): - x = (x - self.mu) / self.sigma - return super().forward(x) - - -class Wu2020AdversarialNet(WideResNet): - - def __init__(self, depth=34, widen_factor=10): - super().__init__(depth=depth, - widen_factor=widen_factor, - sub_block1=False, - num_classes=100) - self.register_buffer( - 'mu', - torch.tensor( - [0.5070751592371323, 0.48654887331495095, - 0.4409178433670343]).view(1, 3, 1, 1)) - self.register_buffer( - 'sigma', - torch.tensor( - [0.2673342858792401, 0.2564384629170883, - 0.27615047132568404]).view(1, 3, 1, 1)) - - def forward(self, x): - x = (x - self.mu) / self.sigma - return super().forward(x) - - -class Rice2020OverfittingNet(PreActResNet): - - def __init__(self): - super(Rice2020OverfittingNet, self).__init__(PreActBlock, [2, 2, 2, 2], - num_classes=100, - bn_before_fc=True, - out_shortcut=True) - self.register_buffer( - 'mu', - torch.tensor( - [0.5070751592371323, 0.48654887331495095, - 0.4409178433670343]).view(1, 3, 1, 1)) - self.register_buffer( - 'sigma', - torch.tensor( - [0.2673342858792401, 0.2564384629170883, - 0.27615047132568404]).view(1, 3, 1, 1)) - - def forward(self, x): - x = (x - self.mu) / self.sigma - return super(Rice2020OverfittingNet, self).forward(x) - - -class Hendrycks2019UsingNet(WideResNet): - - def __init__(self, depth=28, widen_factor=10): - super(Hendrycks2019UsingNet, self).__init__(depth=depth, - widen_factor=widen_factor, - num_classes=100, - sub_block1=False) - - def forward(self, x): - x = 2. * x - 1. - return super(Hendrycks2019UsingNet, self).forward(x) - - -class Hendrycks2020AugMixResNeXtNet(CifarResNeXt): - - def __init__(self, depth=29, cardinality=4, base_width=32): - super().__init__(ResNeXtBottleneck, - depth=depth, - num_classes=100, - cardinality=cardinality, - base_width=base_width) - self.register_buffer('mu', torch.tensor([0.5] * 3).view(1, 3, 1, 1)) - self.register_buffer('sigma', torch.tensor([0.5] * 3).view(1, 3, 1, 1)) - - def forward(self, x): - x = (x - self.mu) / self.sigma - return super().forward(x) - - -class Hendrycks2020AugMixWRNNet(WideResNet): - - def __init__(self, depth=40, widen_factor=2): - super().__init__(depth=depth, - widen_factor=widen_factor, - sub_block1=False, - num_classes=100) - self.register_buffer('mu', torch.tensor([0.5] * 3).view(1, 3, 1, 1)) - self.register_buffer('sigma', torch.tensor([0.5] * 3).view(1, 3, 1, 1)) - - def forward(self, x): - x = (x - self.mu) / self.sigma - return super().forward(x) - - -class Diffenderfer2021CARD(LRR_ResNet): - - def __init__(self, width=128, num_classes=100): - super(Diffenderfer2021CARD, self).__init__(width=width, - num_classes=num_classes) - self.register_buffer( - 'mu', - torch.tensor([0.5071, 0.4865, 0.4409]).view(1, 3, 1, 1)) - self.register_buffer( - 'sigma', - torch.tensor([0.2673, 0.2564, 0.2762]).view(1, 3, 1, 1)) - - def forward(self, x): - x = (x - self.mu) / self.sigma - return super().forward(x) - - -class Diffenderfer2021CARD_Deck(torch.nn.Module): - - def __init__(self, width=128, num_classes=100): - super(Diffenderfer2021CARD_Deck, self).__init__() - self.num_cards = 6 - self.models = nn.ModuleList() - - for i in range(self.num_cards): - self.models.append(LRR_ResNet(width=width, - num_classes=num_classes)) - - self.register_buffer( - 'mu', - torch.tensor([0.5071, 0.4865, 0.4409]).view(1, 3, 1, 1)) - self.register_buffer( - 'sigma', - torch.tensor([0.2673, 0.2564, 0.2762]).view(1, 3, 1, 1)) - - def forward(self, x): - x = (x - self.mu) / self.sigma - - x_cl = x.clone( - ) # clone to make sure x is not changed by inplace methods - out_list = [] - for i in range(self.num_cards): - # Evaluate model i at input - out = self.models[i](x_cl) - # Compute softmax - out = torch.softmax(out, dim=1) - # Append output to list of logits - out_list.append(out) - - return torch.mean(torch.stack(out_list), dim=0) - - -class Diffenderfer2021CARD_Binary(WidePreActResNet): - - def __init__(self, num_classes=100): - super(Diffenderfer2021CARD_Binary, - self).__init__(num_classes=num_classes) - self.register_buffer( - 'mu', - torch.tensor([0.5071, 0.4865, 0.4409]).view(1, 3, 1, 1)) - self.register_buffer( - 'sigma', - torch.tensor([0.2673, 0.2564, 0.2762]).view(1, 3, 1, 1)) - - def forward(self, x): - x = (x - self.mu) / self.sigma - return super().forward(x) - - -class Diffenderfer2021CARD_Deck_Binary(torch.nn.Module): - - def __init__(self, num_classes=100): - super(Diffenderfer2021CARD_Deck_Binary, self).__init__() - self.num_cards = 6 - self.models = nn.ModuleList() - - for i in range(self.num_cards): - self.models.append(WidePreActResNet(num_classes=num_classes)) - - self.register_buffer( - 'mu', - torch.tensor([0.5071, 0.4865, 0.4409]).view(1, 3, 1, 1)) - self.register_buffer( - 'sigma', - torch.tensor([0.2673, 0.2564, 0.2762]).view(1, 3, 1, 1)) - - def forward(self, x): - x = (x - self.mu) / self.sigma - - x_cl = x.clone( - ) # clone to make sure x is not changed by inplace methods - out_list = [] - for i in range(self.num_cards): - # Evaluate model i at input - out = self.models[i](x_cl) - # Compute softmax - out = torch.softmax(out, dim=1) - # Append output to list of logits - out_list.append(out) - - return torch.mean(torch.stack(out_list), dim=0) - - -class Modas2021PRIMEResNet18(ResNet): - - def __init__(self, num_classes=100): - super().__init__(BasicBlock, [2, 2, 2, 2], num_classes=num_classes) - # mu & sigma are updated from weights - self.register_buffer('mu', torch.tensor([0.5] * 3).view(1, 3, 1, 1)) - self.register_buffer('sigma', torch.tensor([0.5] * 3).view(1, 3, 1, 1)) - - def forward(self, x): - x = (x - self.mu) / self.sigma - return super().forward(x) - - -linf = OrderedDict([ - ('Gowal2020Uncovering', { - 'model': - lambda: DMWideResNet(num_classes=100, - depth=70, - width=16, - activation_fn=Swish, - mean=CIFAR100_MEAN, - std=CIFAR100_STD), - 'gdrive_id': - "16I86x2Vv_HCRKROC86G4dQKgO3Po5mT3" - }), - ('Gowal2020Uncovering_extra', { - 'model': - lambda: DMWideResNet(num_classes=100, - depth=70, - width=16, - activation_fn=Swish, - mean=CIFAR100_MEAN, - std=CIFAR100_STD), - 'gdrive_id': - "1LQBdwO2b391mg7VKcP6I0HIOpC6O83gn" - }), - ('Cui2020Learnable_34_20_LBGAT6', { - 'model': - lambda: WideResNet( - depth=34, widen_factor=20, num_classes=100, sub_block1=True), - 'gdrive_id': - '1rN76st8q_32j6Uo8DI5XhcC2cwVhXBwK' - }), - ('Cui2020Learnable_34_10_LBGAT0', { - 'model': - lambda: WideResNet( - depth=34, widen_factor=10, num_classes=100, sub_block1=True), - 'gdrive_id': - '1RnWbGxN-A-ltsfOvulr68U6i2L8ohAJi' - }), - ('Cui2020Learnable_34_10_LBGAT6', { - 'model': - lambda: WideResNet( - depth=34, widen_factor=10, num_classes=100, sub_block1=True), - 'gdrive_id': - '1TfIgvW3BAkL8jL9J7AAWFSLW3SSzJ2AE' - }), - ('Chen2020Efficient', { - 'model': Chen2020EfficientNet, - 'gdrive_id': '1JEh95fvsfKireoELoVCBxOi12IPGFDUT' - }), - ('Wu2020Adversarial', { - 'model': Wu2020AdversarialNet, - 'gdrive_id': '1yWGvHmrgjtd9vOpV5zVDqZmeGhCgVYq7' - }), - ('Sehwag2021Proxy', { - 'model': - lambda: WideResNet( - depth=34, widen_factor=10, num_classes=100, sub_block1=False), - 'gdrive_id': - '1ejMNF2O4xkSdrjtZt2UXUeim-y9F7Req', - }), - ('Sitawarin2020Improving', { - 'model': - lambda: WideResNet( - depth=34, widen_factor=10, num_classes=100, sub_block1=True), - 'gdrive_id': - '1hbpwans776KM1SMbOxISkDx0KR0DW8EN' - }), - ('Hendrycks2019Using', { - 'model': Hendrycks2019UsingNet, - 'gdrive_id': '1If3tppQsCe5dN8Vbo9ff0tjlKQTTrShd' - }), - ('Rice2020Overfitting', { - 'model': Rice2020OverfittingNet, - 'gdrive_id': '1XXNZn3fZBOkD1aqNL1cvcD8zZDccyAZ6' - }), - ('Rebuffi2021Fixing_70_16_cutmix_ddpm', { - 'model': - lambda: DMWideResNet(num_classes=100, - depth=70, - width=16, - activation_fn=Swish, - mean=CIFAR100_MEAN, - std=CIFAR100_STD), - 'gdrive_id': - '1-GkVLo9QaRjCJl-by67xda1ySVhYxsLV' - }), - ('Rebuffi2021Fixing_28_10_cutmix_ddpm', { - 'model': - lambda: DMWideResNet(num_classes=100, - depth=28, - width=10, - activation_fn=Swish, - mean=CIFAR100_MEAN, - std=CIFAR100_STD), - 'gdrive_id': - '1-P7cs82Tj6UVx7Coin3tVurVKYwXWA9p' - }), - ('Rebuffi2021Fixing_R18_ddpm', { - 'model': - lambda: DMPreActResNet(num_classes=100, - depth=18, - width=0, - activation_fn=Swish, - mean=CIFAR100_MEAN, - std=CIFAR100_STD), - 'gdrive_id': - '1-Qcph_EXw1SCYhDIl8cwqTQQy0sJKO8N' - }), - ('Rade2021Helper_R18_ddpm', { - 'model': - lambda: DMPreActResNet(num_classes=100, - depth=18, - width=0, - activation_fn=Swish, - mean=CIFAR100_MEAN, - std=CIFAR100_STD), - 'gdrive_id': - '1-qUvfOjq6x4I8mZynfGtzzCH_nvqS_VQ' - }), - ('Addepalli2021Towards_PARN18', { - 'model': - lambda: PreActResNet( - PreActBlockV2, [2, 2, 2, 2], num_classes=100, bn_before_fc=True), - 'gdrive_id': - '1-FwVya1sDvdFXr0_ZBoXEJW9ukGC7hPK', - }), - ('Addepalli2021Towards_WRN34', { - 'model': - lambda: WideResNet(num_classes=100, depth=34, sub_block1=True), - 'gdrive_id': '1-9GAld_105-jWBLXL73btmfOCwAqvz7Y', - }), - ('Chen2021LTD_WRN34_10', { - 'model': - lambda: WideResNet( - depth=34, widen_factor=10, num_classes=100, sub_block1=True), - 'gdrive_id': - '1-I4NZyULdEWH46b4EaCTxuuRo4eFXsg_' - }), - ('Pang2022Robustness_WRN28_10', { - 'model': - lambda: DMWideResNet(num_classes=100, - depth=28, - width=10, - activation_fn=Swish, - mean=CIFAR100_MEAN, - std=CIFAR100_STD), - 'gdrive_id': - "1VDDM_j5M4b6sZpt1Nnhkr8FER3kjE33M" - }), - ('Pang2022Robustness_WRN70_16', { - 'model': - lambda: DMWideResNet(num_classes=100, - depth=70, - width=16, - activation_fn=Swish, - mean=CIFAR100_MEAN, - std=CIFAR100_STD), - 'gdrive_id': - "1F3kn8KIdBVls8QuTWc3BbB83htkQeVQD", - }), - ('Jia2022LAS-AT_34_10', { - 'model': - lambda: WideResNet( - depth=34, widen_factor=10, num_classes=100, sub_block1=True), - 'gdrive_id': - '1-338K2PUf5FTwk4cbUUeTNz247GrXaMG', - }), - ('Jia2022LAS-AT_34_20', { - 'model': - lambda: WideResNet(depth=34, widen_factor=20, num_classes=100), - 'gdrive_id': '1WhRq01Yl1v8O3skkrGUBuySlptidc5a6', - }), - ('Addepalli2022Efficient_RN18', { - 'model': lambda: ResNet(BasicBlock, [2, 2, 2, 2], num_classes=100), - 'gdrive_id': '1-2hnxC7lZOQDqQbum4yPbtRtTND86I5N', - }), - ('Addepalli2022Efficient_WRN_34_10', { - 'model': - lambda: WideResNet(depth=34, widen_factor=10, num_classes=100), - 'gdrive_id': '1-3c-iniqNfiwGoGPHC3nSostnG6J9fDt', - }), - # ('Debenedetti2022Light_XCiT-S12', { - # 'model': - # (lambda: timm.create_model('debenedetti2020light_xcit_s_cifar100_linf', - # pretrained=True)), - # 'gdrive_id': - # None - # }), - # ('Debenedetti2022Light_XCiT-M12', { - # 'model': - # (lambda: timm.create_model('debenedetti2020light_xcit_m_cifar100_linf', - # pretrained=True)), - # 'gdrive_id': - # None - # }), - # ('Debenedetti2022Light_XCiT-L12', { - # 'model': - # (lambda: timm.create_model('debenedetti2020light_xcit_l_cifar100_linf', - # pretrained=True)), - # 'gdrive_id': - # None - # }), -]) - -common_corruptions = OrderedDict([ - ('Diffenderfer2021Winning_LRR', { - 'model': Diffenderfer2021CARD, - 'gdrive_id': '1-2egZ5WrO22A2pixw_UxOpENy7zwah8j' - }), - ('Diffenderfer2021Winning_LRR_CARD_Deck', { - 'model': - Diffenderfer2021CARD_Deck, - 'gdrive_id': [ - '1-9-O8k6FZO0k-WhcIZCXvMBQLutxwF0I', - '1-H_kInicE70twnsOaK3axVtHBV7WTalI', - '1-MQjiJy01rc0Wt-dpgEx94pBYIPeXD6F', - '1-VpIloQl8GePLSYbUjh_Sc0ehZgfiWny', - '1-i6HADuWHZ8s598mvUL8dIYpL1mxM94f', - '1-jRg4TpyIYcf-9SeG8vptu4X98VK1ZwE' - ], - }), - ('Diffenderfer2021Winning_Binary', { - 'model': Diffenderfer2021CARD_Binary, - 'gdrive_id': '1-vFzi6uF6hgORX6sgJt1sKDPcr3SXUxB' - }), - ('Diffenderfer2021Winning_Binary_CARD_Deck', { - 'model': - Diffenderfer2021CARD_Deck_Binary, - 'gdrive_id': [ - '107TKzt9Nd1ZBx5u-Lc2lgkiqCeiUChw_', - '10EbQ3BxVQJ0-FyDV42fZL6DEVy5wT7D_', - '10IRU_otxEVWNRLeG2D4UI5s6O97APCYH', - '10PyjvWTTyziwpAUxyohkJZZrVHBTwABz', - '10Skhbub7Uu6_WqQiyzBka4T91-5pOR-K', - '10_thReUp-ia8Gxq1xdOAFelIHyoMWdV5' - ], - }), - ('Gowal2020Uncovering_Linf', { - 'model': - lambda: DMWideResNet(num_classes=100, - depth=70, - width=16, - activation_fn=Swish, - mean=CIFAR100_MEAN, - std=CIFAR100_STD), - 'gdrive_id': - "16I86x2Vv_HCRKROC86G4dQKgO3Po5mT3" - }), - ('Gowal2020Uncovering_extra_Linf', { - 'model': - lambda: DMWideResNet(num_classes=100, - depth=70, - width=16, - activation_fn=Swish, - mean=CIFAR100_MEAN, - std=CIFAR100_STD), - 'gdrive_id': - "1LQBdwO2b391mg7VKcP6I0HIOpC6O83gn" - }), - ('Hendrycks2020AugMix_WRN', { - 'model': Hendrycks2020AugMixWRNNet, - 'gdrive_id': '1XpFFdCdU9LcDtcyNfo6_BV1RZHKKkBVE' - }), - ('Hendrycks2020AugMix_ResNeXt', { - 'model': Hendrycks2020AugMixResNeXtNet, - 'gdrive_id': '1ocnHbvDdOBLvgNr6K7vEYL08hUdkD1Rv' - }), - ('Addepalli2021Towards_PARN18', { - 'model': - lambda: PreActResNet( - PreActBlockV2, [2, 2, 2, 2], num_classes=100, bn_before_fc=True), - 'gdrive_id': - '1-FwVya1sDvdFXr0_ZBoXEJW9ukGC7hPK', - }), - ('Addepalli2021Towards_WRN34', { - 'model': - lambda: WideResNet(num_classes=100, depth=34, sub_block1=True), - 'gdrive_id': '1-9GAld_105-jWBLXL73btmfOCwAqvz7Y' - }), - ('Modas2021PRIMEResNet18', { - 'model': Modas2021PRIMEResNet18, - 'gdrive_id': '1kcohb2tBuJHa5pGSi4nAkvK-hXPSI6Hr' - }), - ('Addepalli2022Efficient_WRN_34_10', { - 'model': - lambda: WideResNet(depth=34, widen_factor=10, num_classes=100), - 'gdrive_id': '1-3c-iniqNfiwGoGPHC3nSostnG6J9fDt', - }), -]) - -cifar_100_models = OrderedDict([(ThreatModel.Linf, linf), - (ThreatModel.corruptions, common_corruptions)]) diff --git a/robustbench/model_zoo/enums.py b/robustbench/model_zoo/enums.py deleted file mode 100644 index 720b268f..00000000 --- a/robustbench/model_zoo/enums.py +++ /dev/null @@ -1,14 +0,0 @@ -from enum import Enum - - -class BenchmarkDataset(Enum): - cifar_10 = 'cifar10' - cifar_100 = 'cifar100' - imagenet = 'imagenet' - imagenet_3d = 'imagenet_3d' - - -class ThreatModel(Enum): - Linf = "Linf" - L2 = "L2" - corruptions = "corruptions" diff --git a/robustbench/model_zoo/imagenet.py b/robustbench/model_zoo/imagenet.py deleted file mode 100644 index 89d86d77..00000000 --- a/robustbench/model_zoo/imagenet.py +++ /dev/null @@ -1,109 +0,0 @@ -from collections import OrderedDict - -# import timm -from torchvision import models as pt_models - -from robustbench.model_zoo.enums import ThreatModel -from robustbench.model_zoo.architectures.utils_architectures import normalize_model -# from robustbench.model_zoo.architectures import xcit - - -mu = (0.485, 0.456, 0.406) -sigma = (0.229, 0.224, 0.225) - - -linf = OrderedDict( - [ - ('Wong2020Fast', { # requires resolution 288 x 288 - 'model': lambda: normalize_model(pt_models.resnet50(), mu, sigma), - 'gdrive_id': '1deM2ZNS5tf3S_-eRURJi-IlvUL8WJQ_w', - 'preprocessing': 'Crop288' - }), - ('Engstrom2019Robustness', { - 'model': lambda: normalize_model(pt_models.resnet50(), mu, sigma), - 'gdrive_id': '1T2Fvi1eCJTeAOEzrH_4TAIwO8HTOYVyn', - 'preprocessing': 'Res256Crop224', - }), - ('Salman2020Do_R50', { - 'model': lambda: normalize_model(pt_models.resnet50(), mu, sigma), - 'gdrive_id': '1TmT5oGa1UvVjM3d-XeSj_XmKqBNRUg8r', - 'preprocessing': 'Res256Crop224' - }), - ('Salman2020Do_R18', { - 'model': lambda: normalize_model(pt_models.resnet18(), mu, sigma), - 'gdrive_id': '1OThCOQCOxY6lAgxZxgiK3YuZDD7PPfPx', - 'preprocessing': 'Res256Crop224' - }), - ('Salman2020Do_50_2', { - 'model': lambda: normalize_model(pt_models.wide_resnet50_2(), mu, sigma), - 'gdrive_id': '1OT7xaQYljrTr3vGbM37xK9SPoPJvbSKB', - 'preprocessing': 'Res256Crop224' - }), - ('Standard_R50', { - 'model': lambda: normalize_model(pt_models.resnet50(pretrained=True), mu, sigma), - 'gdrive_id': '', - 'preprocessing': 'Res256Crop224' - }), - # ('Debenedetti2022Light_XCiT-S12', { - # 'model': (lambda: timm.create_model( - # 'debenedetti2020light_xcit_s_imagenet_linf', pretrained=True)), - # 'gdrive_id': - # None - # }), - # ('Debenedetti2022Light_XCiT-M12', { - # 'model': (lambda: timm.create_model( - # 'debenedetti2020light_xcit_m_imagenet_linf', pretrained=True)), - # 'gdrive_id': - # None - # }), - # ('Debenedetti2022Light_XCiT-L12', { - # 'model': (lambda: timm.create_model( - # 'debenedetti2020light_xcit_l_imagenet_linf', pretrained=True)), - # 'gdrive_id': - # None - # }), - ]) - -common_corruptions = OrderedDict( - [ - ('Geirhos2018_SIN', { - 'model': lambda: normalize_model(pt_models.resnet50(), mu, sigma), - 'gdrive_id': '1hLgeY_rQIaOT4R-t_KyOqPNkczfaedgs', - 'preprocessing': 'Res256Crop224' - }), - ('Geirhos2018_SIN_IN', { - 'model': lambda: normalize_model(pt_models.resnet50(), mu, sigma), - 'gdrive_id': '139pWopDnNERObZeLsXUysRcLg6N1iZHK', - 'preprocessing': 'Res256Crop224' - }), - ('Geirhos2018_SIN_IN_IN', { - 'model': lambda: normalize_model(pt_models.resnet50(), mu, sigma), - 'gdrive_id': '1xOvyuxpOZ8I5CZOi0EGYG_R6tu3ZaJdO', - 'preprocessing': 'Res256Crop224' - }), - ('Hendrycks2020Many', { - 'model': lambda: normalize_model(pt_models.resnet50(), mu, sigma), - 'gdrive_id': '1kylueoLtYtxkpVzoOA1B6tqdbRl2xt9X', - 'preprocessing': 'Res256Crop224' - }), - ('Hendrycks2020AugMix', { - 'model': lambda: normalize_model(pt_models.resnet50(), mu, sigma), - 'gdrive_id': '1xRMj1GlO93tLoCMm0e5wEvZwqhIjxhoJ', - 'preprocessing': 'Res256Crop224' - }), - ('Salman2020Do_50_2_Linf', { - 'model': lambda: normalize_model(pt_models.wide_resnet50_2(), mu, sigma), - 'gdrive_id': '1OT7xaQYljrTr3vGbM37xK9SPoPJvbSKB', - 'preprocessing': 'Res256Crop224' - }), - ('Standard_R50', { - 'model': lambda: normalize_model(pt_models.resnet50(pretrained=True), mu, sigma), - 'gdrive_id': '', - 'preprocessing': 'Res256Crop224' - }), - ]) - -imagenet_models = OrderedDict([(ThreatModel.Linf, linf), - (ThreatModel.corruptions, common_corruptions)]) - - diff --git a/robustbench/model_zoo/models.py b/robustbench/model_zoo/models.py deleted file mode 100644 index 8d5bf962..00000000 --- a/robustbench/model_zoo/models.py +++ /dev/null @@ -1,17 +0,0 @@ -from collections import OrderedDict -from typing import Any, Dict, Dict as OrderedDictType - -from robustbench.model_zoo.cifar10 import cifar_10_models -from robustbench.model_zoo.cifar100 import cifar_100_models -from robustbench.model_zoo.imagenet import imagenet_models -from robustbench.model_zoo.enums import BenchmarkDataset, ThreatModel - -ModelsDict = OrderedDictType[str, Dict[str, Any]] -ThreatModelsDict = OrderedDictType[ThreatModel, ModelsDict] -BenchmarkDict = OrderedDictType[BenchmarkDataset, ThreatModelsDict] - -model_dicts: BenchmarkDict = OrderedDict([ - (BenchmarkDataset.cifar_10, cifar_10_models), - (BenchmarkDataset.cifar_100, cifar_100_models), - (BenchmarkDataset.imagenet, imagenet_models) -]) diff --git a/robustbench/utils.py b/robustbench/utils.py deleted file mode 100644 index 255f4d30..00000000 --- a/robustbench/utils.py +++ /dev/null @@ -1,489 +0,0 @@ -import argparse -import dataclasses -import json -import math -import os -import warnings -from collections import OrderedDict -from pathlib import Path -from typing import Dict, Optional, Union - -import requests -# import timm -import torch -from torch import nn - -from robustbench.model_zoo import model_dicts as all_models -from robustbench.model_zoo.enums import BenchmarkDataset, ThreatModel - - -ACC_FIELDS = { - ThreatModel.corruptions: "corruptions_acc", - ThreatModel.L2: ("external", "autoattack_acc"), - ThreatModel.Linf: ("external", "autoattack_acc") -} - - -def download_gdrive(gdrive_id, fname_save): - """ source: https://stackoverflow.com/questions/38511444/python-download-files-from-google-drive-using-url """ - def get_confirm_token(response): - for key, value in response.cookies.items(): - if key.startswith('download_warning'): - return value - - return None - - def save_response_content(response, fname_save): - CHUNK_SIZE = 32768 - - with open(fname_save, "wb") as f: - for chunk in response.iter_content(CHUNK_SIZE): - if chunk: # filter out keep-alive new chunks - f.write(chunk) - - print('Download started: path={} (gdrive_id={})'.format( - fname_save, gdrive_id)) - - url_base = "https://docs.google.com/uc?export=download&confirm=t" - session = requests.Session() - - response = session.get(url_base, params={'id': gdrive_id}, stream=True) - token = get_confirm_token(response) - - if token: - params = {'id': gdrive_id, 'confirm': token} - response = session.get(url_base, params=params, stream=True) - - save_response_content(response, fname_save) - session.close() - print('Download finished: path={} (gdrive_id={})'.format( - fname_save, gdrive_id)) - - -def rm_substr_from_state_dict(state_dict, substr): - new_state_dict = OrderedDict() - for key in state_dict.keys(): - if substr in key: # to delete prefix 'module.' if it exists - new_key = key[len(substr):] - new_state_dict[new_key] = state_dict[key] - else: - new_state_dict[key] = state_dict[key] - return new_state_dict - - -def add_substr_to_state_dict(state_dict, substr): - new_state_dict = OrderedDict() - for k, v in state_dict.items(): - new_state_dict[substr + k] = v - return new_state_dict - - -def load_model(model_name: str, - model_dir: Union[str, Path] = './models', - dataset: Union[str, - BenchmarkDataset] = BenchmarkDataset.cifar_10, - threat_model: Union[str, ThreatModel] = ThreatModel.Linf, - norm: Optional[str] = None) -> nn.Module: - """Loads a model from the model_zoo. - - The model is trained on the given ``dataset``, for the given ``threat_model``. - - :param model_name: The name used in the model zoo. - :param model_dir: The base directory where the models are saved. - :param dataset: The dataset on which the model is trained. - :param threat_model: The threat model for which the model is trained. - :param norm: Deprecated argument that can be used in place of ``threat_model``. If specified, it - overrides ``threat_model`` - - :return: A ready-to-used trained model. - """ - - # if model_name in timm.list_models(): - # return timm.create_model(model_name, pretrained=True).eval() - - dataset_: BenchmarkDataset = BenchmarkDataset(dataset) - if norm is None: - threat_model_: ThreatModel = ThreatModel(threat_model) - else: - threat_model_ = ThreatModel(norm) - warnings.warn( - "`norm` has been deprecated and will be removed in a future version.", - DeprecationWarning) - - model_dir_ = Path(model_dir) / dataset_.value / threat_model_.value - model_path = model_dir_ / f'{model_name}.pt' - - models = all_models[dataset_][threat_model_] - - # if models[model_name]['gdrive_id'] is None: - # raise ValueError(f"Model `{model_name}` is not a timm model and has no `gdrive_id` specified.") - - if not isinstance(models[model_name]['gdrive_id'], list): - model = models[model_name]['model']() - if dataset_ == BenchmarkDataset.imagenet and 'Standard' in model_name: - return model.eval() - - if not os.path.exists(model_dir_): - os.makedirs(model_dir_) - if not os.path.isfile(model_path): - download_gdrive(models[model_name]['gdrive_id'], model_path) - checkpoint = torch.load(model_path, map_location=torch.device('cpu')) - - if 'Kireev2021Effectiveness' in model_name or model_name == 'Andriushchenko2020Understanding': - checkpoint = checkpoint['last'] # we take the last model (choices: 'last', 'best') - try: - # needed for the model of `Carmon2019Unlabeled` - state_dict = rm_substr_from_state_dict(checkpoint['state_dict'], - 'module.') - # needed for the model of `Chen2020Efficient` - state_dict = rm_substr_from_state_dict(state_dict, - 'model.') - except: - state_dict = rm_substr_from_state_dict(checkpoint, 'module.') - state_dict = rm_substr_from_state_dict(state_dict, 'model.') - - if dataset_ == BenchmarkDataset.imagenet: - # so far all models need input normalization, which is added as extra layer - state_dict = add_substr_to_state_dict(state_dict, 'model.') - - model = _safe_load_state_dict(model, model_name, state_dict, dataset_) - - return model.eval() - - # If we have an ensemble of models (e.g., Chen2020Adversarial, Diffenderfer2021Winning_LRR_CARD_Deck) - else: - model = models[model_name]['model']() - if not os.path.exists(model_dir_): - os.makedirs(model_dir_) - for i, gid in enumerate(models[model_name]['gdrive_id']): - if not os.path.isfile('{}_m{}.pt'.format(model_path, i)): - download_gdrive(gid, '{}_m{}.pt'.format(model_path, i)) - checkpoint = torch.load('{}_m{}.pt'.format(model_path, i), - map_location=torch.device('cpu')) - try: - state_dict = rm_substr_from_state_dict( - checkpoint['state_dict'], 'module.') - except KeyError: - state_dict = rm_substr_from_state_dict(checkpoint, 'module.') - - model.models[i] = _safe_load_state_dict(model.models[i], - model_name, state_dict, - dataset_) - model.models[i].eval() - - return model.eval() - - -def _safe_load_state_dict(model: nn.Module, model_name: str, - state_dict: Dict[str, torch.Tensor], - dataset_: BenchmarkDataset) -> nn.Module: - known_failing_models = { - "Andriushchenko2020Understanding", "Augustin2020Adversarial", - "Engstrom2019Robustness", "Pang2020Boosting", "Rice2020Overfitting", - "Rony2019Decoupling", "Wong2020Fast", "Hendrycks2020AugMix_WRN", - "Hendrycks2020AugMix_ResNeXt", "Kireev2021Effectiveness_Gauss50percent", - "Kireev2021Effectiveness_AugMixNoJSD", "Kireev2021Effectiveness_RLAT", - "Kireev2021Effectiveness_RLATAugMixNoJSD", "Kireev2021Effectiveness_RLATAugMixNoJSD", - "Kireev2021Effectiveness_RLATAugMix", "Chen2020Efficient", - "Wu2020Adversarial", "Augustin2020Adversarial_34_10", - "Augustin2020Adversarial_34_10_extra", "Diffenderfer2021Winning_LRR", - "Diffenderfer2021Winning_LRR_CARD_Deck", "Diffenderfer2021Winning_Binary", - "Diffenderfer2021Winning_Binary_CARD_Deck" - } - - failure_messages = ['Missing key(s) in state_dict: "mu", "sigma".', - 'Unexpected key(s) in state_dict: "model_preact_hl1.1.weight"', - 'Missing key(s) in state_dict: "normalize.mean", "normalize.std"', - 'Unexpected key(s) in state_dict: "conv1.scores"'] - - try: - model.load_state_dict(state_dict, strict=True) - except RuntimeError as e: - if (model_name in known_failing_models or dataset_ == BenchmarkDataset.imagenet - ) and any([msg in str(e) for msg in failure_messages]): - model.load_state_dict(state_dict, strict=False) - else: - raise e - - return model - - -def clean_accuracy(model: nn.Module, - x: torch.Tensor, - y: torch.Tensor, - batch_size: int = 100, - device: torch.device = None): - if device is None: - device = x.device - acc = 0. - n_batches = math.ceil(x.shape[0] / batch_size) - with torch.no_grad(): - for counter in range(n_batches): - x_curr = x[counter * batch_size:(counter + 1) * - batch_size].to(device) - y_curr = y[counter * batch_size:(counter + 1) * - batch_size].to(device) - - output = model(x_curr) - acc += (output.max(1)[1] == y_curr).float().sum() - - return acc.item() / x.shape[0] - - -def get_key(x, keys): - if isinstance(keys, str): - return float(x[keys]) - else: - for k in keys: - if k in x.keys(): - return float(x[k]) - - -def list_available_models( - dataset: Union[str, BenchmarkDataset] = BenchmarkDataset.cifar_10, - threat_model: Union[str, ThreatModel] = ThreatModel.Linf, - norm: Optional[str] = None): - dataset_: BenchmarkDataset = BenchmarkDataset(dataset) - - if norm is None: - threat_model_: ThreatModel = ThreatModel(threat_model) - else: - threat_model_ = ThreatModel(norm) - warnings.warn( - "`norm` has been deprecated and will be removed in a future version.", - DeprecationWarning) - - models = all_models[dataset_][threat_model_].keys() - - acc_field = ACC_FIELDS[threat_model_] - - json_dicts = [] - - jsons_dir = Path("./model_info") / dataset_.value / threat_model_.value - - for model_name in models: - json_path = jsons_dir / f"{model_name}.json" - - # Some models might not yet be in model_info - if not json_path.exists(): - continue - - with open(json_path, 'r') as model_info: - json_dict = json.load(model_info) - - json_dict['model_name'] = model_name - json_dict['venue'] = 'Unpublished' if json_dict[ - 'venue'] == '' else json_dict['venue'] - if isinstance(acc_field, str): - json_dict[acc_field] = float(json_dict[acc_field]) / 100 - else: - for k in acc_field: - if k in json_dict.keys(): - json_dict[k] = float(json_dict[k]) / 100 - json_dict['clean_acc'] = float(json_dict['clean_acc']) / 100 - json_dicts.append(json_dict) - - json_dicts = sorted(json_dicts, key=lambda d: -get_key(d, acc_field)) - print('| # | Model ID | Paper | Clean accuracy | Robust accuracy | Architecture | Venue |') - print('|:---:|---|---|:---:|:---:|:---:|:---:|') - for i, json_dict in enumerate(json_dicts): - if json_dict['model_name'] == 'Chen2020Adversarial': - json_dict['architecture'] = json_dict[ - 'architecture'] + '
(3x ensemble)' - if json_dict['model_name'] != 'Natural': - print( - '| **{}** | **{}** | *[{}]({})* | {:.2%} | {:.2%} | {} | {} |' - .format(i + 1, json_dict['model_name'], json_dict['name'], - json_dict['link'], json_dict['clean_acc'], - get_key(json_dict, acc_field), json_dict['architecture'], - json_dict['venue'])) - else: - print( - '| **{}** | **{}** | *{}* | {:.2%} | {:.2%} | {} | {} |' - .format(i + 1, json_dict['model_name'], json_dict['name'], - json_dict['clean_acc'], get_key(json_dict, acc_field), - json_dict['architecture'], json_dict['venue'])) - - -def _get_bibtex_entry(model_name: str, title: str, authors: str, venue: str, year: int): - authors = authors.replace(', ', ' and ') - return (f"@article{{{model_name},\n" - f"\ttitle\t= {{{title}}},\n" - f"\tauthor\t= {{{authors}}},\n" - f"\tjournal\t= {{{venue}}},\n" - f"\tyear\t= {{{year}}}\n" - "}\n") - - -def get_leaderboard_bibtex(dataset: Union[str, BenchmarkDataset], threat_model: Union[str, ThreatModel]): - dataset_: BenchmarkDataset = BenchmarkDataset(dataset) - threat_model_: ThreatModel = ThreatModel(threat_model) - - jsons_dir = Path("./model_info") / dataset_.value / threat_model_.value - - bibtex_entries = set() - - for json_path in jsons_dir.glob("*.json"): - - model_name = json_path.stem.split("_")[0] - - with open(json_path, 'r') as model_info: - model_dict = json.load(model_info) - title = model_dict["name"] - authors = model_dict["authors"] - full_venue = model_dict["venue"] - if full_venue == "N/A": - continue - venue = full_venue.split(" ")[0] - venue = venue.split(",")[0] - - year = model_dict["venue"].split(" ")[-1] - - bibtex_entry = _get_bibtex_entry( - model_name, title, authors, venue, year) - bibtex_entries.add(bibtex_entry) - - str_entries = '' - for entry in bibtex_entries: - print(entry) - str_entries += entry - - return bibtex_entries, str_entries - - -def get_leaderboard_latex(dataset: Union[str, BenchmarkDataset], - threat_model: Union[str, ThreatModel], - l_keys=['clean_acc', 'external', #'autoattack_acc', - 'additional_data', 'architecture', 'venue', - 'modelzoo_id'], - sort_by='external' #'autoattack_acc' - ): - dataset_: BenchmarkDataset = BenchmarkDataset(dataset) - threat_model_: ThreatModel = ThreatModel(threat_model) - - models = all_models[dataset_][threat_model_] - print(models.keys()) - - jsons_dir = Path("./model_info") / dataset_.value / threat_model_.value - entries = [] - - for json_path in jsons_dir.glob("*.json"): - if not json_path.stem.startswith('Standard'): - model_name = json_path.stem.split("_")[0] - else: - model_name = json_path.stem - - with open(json_path, 'r') as model_info: - model_dict = json.load(model_info) - - str_curr = '\\citet{{{}}}'.format(model_name) if not model_name in ['Standard', 'Standard_R50'] \ - else model_name.replace('_', '\\_') - - for k in l_keys: - if k == 'external' and not 'external' in model_dict.keys(): - model_dict[k] = model_dict['autoattack_acc'] - if k == 'additional_data': - v = 'Y' if model_dict[k] else 'N' - elif k == 'architecture': - v = model_dict[k].replace('WideResNet', 'WRN') - v = v.replace('ResNet', 'RN') - elif k == 'modelzoo_id': - # print(json_path.stem) - v = json_path.stem.split('.json')[0] - if not v in models.keys(): - v = 'N/A' - else: - v = v.replace('_', '\\_') - else: - v = model_dict[k] - str_curr += ' & {}'.format(v) - str_curr += '\\\\' - entries.append((str_curr, float(model_dict[sort_by]))) - - entries = sorted(entries, key=lambda k: k[1], reverse=True) - entries = ['{} &'.format(i + 1) + a for i, (a, b) in enumerate(entries)] - entries = '\n'.join(entries).replace('
', ' ') - - return entries - - -def update_json(dataset: BenchmarkDataset, threat_model: ThreatModel, - model_name: str, accuracy: float, adv_accuracy: float, - eps: Optional[float]) -> None: - json_path = Path( - "model_info" - ) / dataset.value / threat_model.value / f"{model_name}.json" - if not json_path.parent.exists(): - json_path.parent.mkdir(parents=True, exist_ok=True) - - acc_field = ACC_FIELDS[threat_model] - if isinstance(acc_field, tuple): - acc_field = acc_field[-1] - - acc_field_kwarg = {acc_field: adv_accuracy} - - model_info = ModelInfo(dataset=dataset.value, eps=eps, clean_acc=accuracy, **acc_field_kwarg) - - with open(json_path, "w") as f: - f.write(json.dumps(dataclasses.asdict(model_info), indent=2)) - - -@dataclasses.dataclass -class ModelInfo: - link: Optional[str] = None - name: Optional[str] = None - authors: Optional[str] = None - additional_data: Optional[bool] = None - number_forward_passes: Optional[int] = None - dataset: Optional[str] = None - venue: Optional[str] = None - architecture: Optional[str] = None - eps: Optional[float] = None - clean_acc: Optional[float] = None - reported: Optional[float] = None - corruptions_acc: Optional[str] = None - autoattack_acc: Optional[str] = None - footnote: Optional[str] = None - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument('--model_name', - type=str, - default='Carmon2019Unlabeled') - parser.add_argument('--threat_model', - type=str, - default='Linf', - choices=[x.value for x in ThreatModel]) - parser.add_argument('--dataset', - type=str, - default='cifar10', - choices=[x.value for x in BenchmarkDataset]) - parser.add_argument('--eps', type=float, default=8 / 255) - parser.add_argument('--n_ex', - type=int, - default=100, - help='number of examples to evaluate on') - parser.add_argument('--batch_size', - type=int, - default=500, - help='batch size for evaluation') - parser.add_argument('--data_dir', - type=str, - default='./data', - help='where to store downloaded datasets') - parser.add_argument('--model_dir', - type=str, - default='./models', - help='where to store downloaded models') - parser.add_argument('--seed', - type=int, - default=0, - help='random seed') - parser.add_argument('--device', - type=str, - default='cuda:0', - help='device to use for computations') - parser.add_argument('--to_disk', type=bool, default=True) - args = parser.parse_args() - return args diff --git a/robustbench/zenodo_download.py b/robustbench/zenodo_download.py deleted file mode 100644 index 2429813b..00000000 --- a/robustbench/zenodo_download.py +++ /dev/null @@ -1,83 +0,0 @@ -import hashlib -import shutil -from pathlib import Path -from typing import Set - -import requests -from tqdm import tqdm - -ZENODO_ENTRY_POINT = "https://zenodo.org/api" -RECORDS_ENTRY_POINT = f"{ZENODO_ENTRY_POINT}/records/" - -CHUNK_SIZE = 65536 - - -class DownloadError(Exception): - pass - - -def download_file(url: str, save_dir: Path, total_bytes: int) -> Path: - """Downloads large files from the given URL. - - From: https://stackoverflow.com/a/16696317 - - :param url: The URL of the file. - :param save_dir: The directory where the file should be saved. - :param total_bytes: The total bytes of the file. - :return: The path to the downloaded file. - """ - local_filename = save_dir / url.split('/')[-1] - print(f"Starting download from {url}") - with requests.get(url, stream=True) as r: - r.raise_for_status() - with open(local_filename, 'wb') as f: - iters = total_bytes // CHUNK_SIZE - for chunk in tqdm(r.iter_content(chunk_size=CHUNK_SIZE), - total=iters): - f.write(chunk) - - return local_filename - - -def file_md5(filename: Path) -> str: - """Computes the MD5 hash of a given file""" - hash_md5 = hashlib.md5() - with open(filename, "rb") as f: - for chunk in iter(lambda: f.read(32768), b""): - hash_md5.update(chunk) - - return hash_md5.hexdigest() - - -def zenodo_download(record_id: str, filenames_to_download: Set[str], - save_dir: Path) -> None: - """Downloads the given files from the given Zenodo record. - - :param record_id: The ID of the record. - :param filenames_to_download: The files to download from the record. - :param save_dir: The directory where the files should be saved. - """ - if not save_dir.exists(): - save_dir.mkdir(parents=True, exist_ok=True) - - url = f"{RECORDS_ENTRY_POINT}/{record_id}" - res = requests.get(url) - files = res.json()["files"] - files_to_download = list( - filter(lambda file: file["key"] in filenames_to_download, files)) - - for file in files_to_download: - if (save_dir / file["key"]).exists(): - continue - file_url = file["links"]["self"] - file_checksum = file["checksum"].split(":")[-1] - filename = download_file(file_url, save_dir, file["size"]) - if file_md5(filename) != file_checksum: - raise DownloadError( - "The hash of the downloaded file does not match" - " the expected one.") - print("Download finished, extracting...") - shutil.unpack_archive(filename, - extract_dir=save_dir, - format=file["type"]) - print("Downloaded and extracted.") diff --git a/torchattacks/__init__.py b/torchattacks/__init__.py index 48f18a74..1dddc0dc 100644 --- a/torchattacks/__init__.py +++ b/torchattacks/__init__.py @@ -7,6 +7,7 @@ from .attacks.bim import BIM from .attacks.rfgsm import RFGSM from .attacks.pgd import PGD +from .attacks.espgd import ESPGD from .attacks.eotpgd import EOTPGD from .attacks.ffgsm import FFGSM from .attacks.tpgd import TPGD @@ -25,16 +26,24 @@ from .attacks.spsa import SPSA from .attacks.pifgsm import PIFGSM from .attacks.pifgsmpp import PIFGSMPP +from .attacks.fab import FAB # L2 attacks from .attacks.cw import CW +from .attacks.cwl0 import CWL0 +from .attacks.cwlinf import CWLinf +from .attacks.cwbs import CWBS +from .attacks.cwbsl0 import CWBSL0 +from .attacks.cwbslinf import CWBSLinf from .attacks.pgdl2 import PGDL2 from .attacks.pgdrsl2 import PGDRSL2 from .attacks.deepfool import DeepFool from .attacks.eaden import EADEN +from .attacks.fabl2 import FABL2 # L1 attacks from .attacks.eadl1 import EADL1 +from .attacks.fabl1 import FABL1 # L0 attacks from .attacks.sparsefool import SparseFool @@ -43,9 +52,9 @@ from .attacks.jsma import JSMA # Linf, L2 attacks -from .attacks.fab import FAB from .attacks.autoattack import AutoAttack from .attacks.square import Square +from .attacks.afab import AFAB # Wrapper Class from .wrappers.multiattack import MultiAttack @@ -59,6 +68,7 @@ "BIM", "RFGSM", "PGD", + "ESPGD", "EOTPGD", "FFGSM", "TPGD", @@ -81,6 +91,11 @@ "PIFGSM", "PIFGSMPP", "CW", + "CWL0", + "CWLinf", + "CWBS", + "CWBSL0", + "CWBSLinf", "PGDL2", "DeepFool", "PGDRSL2", @@ -88,6 +103,9 @@ "OnePixel", "Pixle", "FAB", + "FABL1", + "FABL2", + "AFAB", "AutoAttack", "Square", "MultiAttack", diff --git a/torchattacks/attack.py b/torchattacks/attack.py index 4661272d..d91cc070 100644 --- a/torchattacks/attack.py +++ b/torchattacks/attack.py @@ -1,6 +1,8 @@ import time +from typing import Optional, Union, List, Tuple from collections import OrderedDict +import numpy as np import torch from torch.utils.data import DataLoader, TensorDataset @@ -25,7 +27,7 @@ class Attack(object): To change this, please see `set_model_training_mode`. """ - def __init__(self, name, model): + def __init__(self, name: str, model: torch.nn.Module) -> None: r""" Initializes internal attack state. @@ -48,6 +50,7 @@ def __init__(self, name, model): self.attack_mode = "default" self.supported_mode = ["default"] self.targeted = False + self.target_labels = None self._target_map_function = None # Controls when normalization is used. @@ -61,7 +64,7 @@ def __init__(self, name, model): self._batchnorm_training = False self._dropout_training = False - def forward(self, inputs, labels=None, *args, **kwargs): + def forward(self, inputs: torch.nn.Module, labels: Optional[torch.nn.Module], *args, **kwargs) -> None: r""" It defines the computation performed at every call. Should be overridden by all subclasses. @@ -69,26 +72,26 @@ def forward(self, inputs, labels=None, *args, **kwargs): raise NotImplementedError @wrapper_method - def set_model(self, model): + def set_model(self, model: torch.nn.Module) -> None: self.model = model self.model_name = model.__class__.__name__ - def get_logits(self, inputs, labels=None, *args, **kwargs): + def get_logits(self, inputs: torch.Tensor, *args, **kwargs) -> torch.tensor: if self._normalization_applied is False: inputs = self.normalize(inputs) logits = self.model(inputs) return logits @wrapper_method - def _set_normalization_applied(self, flag): + def _set_normalization_applied(self, flag: bool) -> None: self._normalization_applied = flag @wrapper_method - def set_device(self, device): + def set_device(self, device: Union[str, torch.device]) -> None: self.device = device @wrapper_method - def _set_rmodel_normalization_used(self, model): + def _set_rmodel_normalization_used(self, model: torch.nn.Module) -> None: r""" Set attack normalization for MAIR [https://github.com/Harry24k/MAIR]. @@ -104,7 +107,7 @@ def _set_rmodel_normalization_used(self, model): self.set_normalization_used(mean, std) @wrapper_method - def set_normalization_used(self, mean, std): + def set_normalization_used(self, mean: Union[np.ndarray, torch.Tensor, List, Tuple], std: Union[np.ndarray, torch.Tensor, List, Tuple]) -> None: self.normalization_used = {} n_channels = len(mean) mean = torch.tensor(mean).reshape(1, n_channels, 1, 1) @@ -113,17 +116,17 @@ def set_normalization_used(self, mean, std): self.normalization_used["std"] = std self._set_normalization_applied(True) - def normalize(self, inputs): + def normalize(self, inputs: torch.Tensor) -> torch.Tensor: mean = self.normalization_used["mean"].to(inputs.device) std = self.normalization_used["std"].to(inputs.device) return (inputs - mean) / std - def inverse_normalize(self, inputs): + def inverse_normalize(self, inputs: torch.Tensor) -> torch.Tensor: mean = self.normalization_used["mean"].to(inputs.device) std = self.normalization_used["std"].to(inputs.device) return inputs * std + mean - def get_mode(self): + def get_mode(self) -> str: r""" Get attack mode. @@ -131,7 +134,7 @@ def get_mode(self): return self.attack_mode @wrapper_method - def set_mode_default(self): + def set_mode_default(self) -> None: r""" Set attack mode as default mode. @@ -141,7 +144,7 @@ def set_mode_default(self): print("Attack mode is changed to 'default.'") @wrapper_method - def _set_mode_targeted(self, mode, quiet): + def _set_mode_targeted(self, mode: str, quiet: bool) -> None: if "targeted" not in self.supported_mode: raise ValueError("Targeted mode is not supported.") self.targeted = True @@ -150,7 +153,7 @@ def _set_mode_targeted(self, mode, quiet): print("Attack mode is changed to '%s'." % mode) @wrapper_method - def set_mode_targeted_by_function(self, target_map_function, quiet=False): + def set_mode_targeted_by_function(self, target_map_function, quiet: bool = False) -> None: r""" Set attack mode as targeted. @@ -165,7 +168,7 @@ def set_mode_targeted_by_function(self, target_map_function, quiet=False): self._target_map_function = target_map_function @wrapper_method - def set_mode_targeted_random(self, quiet=False): + def set_mode_targeted_random(self, quiet: bool = False) -> None: r""" Set attack mode as targeted with random labels. @@ -177,13 +180,12 @@ def set_mode_targeted_random(self, quiet=False): self._target_map_function = self.get_random_target_label @wrapper_method - def set_mode_targeted_least_likely(self, kth_min=1, quiet=False): + def set_mode_targeted_least_likely(self, kth_min: int = 1, quiet: bool = False) -> None: r""" Set attack mode as targeted with least likely labels. Arguments: - kth_min (str): label with the k-th smallest probability used as target labels. (Default: 1) - num_classses (str): number of classes. (Default: False) + kth_min (int): label with the k-th smallest probability used as target labels. (Default: 1) """ self._set_mode_targeted("targeted(least-likely)", quiet) @@ -192,23 +194,32 @@ def set_mode_targeted_least_likely(self, kth_min=1, quiet=False): self._target_map_function = self.get_least_likely_label @wrapper_method - def set_mode_targeted_by_label(self, quiet=False): + def set_mode_targeted_by_label(self, target_labels: torch.Tensor, quiet: bool = False) -> None: r""" Set attack mode as targeted. Arguments: + target_label (torch.Tensor): Target labels of the targeted attack. quiet (bool): Display information message or not. (Default: False) .. note:: Use user-supplied labels as target labels. """ - self._set_mode_targeted("targeted(label)", quiet) - self._target_map_function = "function is a string" + if isinstance(target_labels, torch.Tensor): + self.target_labels = target_labels + self._set_mode_targeted("targeted(label)", quiet) + self._target_map_function = "function is a string" + else: + raise ValueError( + 'Target labels types not supported: {}'.format(type(target_labels))) @wrapper_method def set_model_training_mode( - self, model_training=False, batchnorm_training=False, dropout_training=False - ): + self, + model_training: bool = False, + batchnorm_training: bool = False, + dropout_training: bool = False + ) -> None: r""" Set training mode during attack process. @@ -247,13 +258,13 @@ def _recover_model_mode(self, given_training): def save( self, data_loader, - save_path=None, - verbose=True, - return_verbose=False, - save_predictions=False, - save_clean_inputs=False, + save_path: Union[str, None] = None, + verbose: bool = True, + return_verbose: bool = False, + save_predictions: bool = False, + save_clean_inputs: bool = False, save_type="float", - ): + ) -> Tuple[float, torch.tensor, float]: r""" Save adversarial inputs as torch.tensor from given torch.utils.data.DataLoader. @@ -370,7 +381,7 @@ def save( return rob_acc, l2, elapsed_time @staticmethod - def to_type(inputs, type): + def to_type(inputs, type: str) -> torch.Tensor: r""" Return inputs as int if float is given. """ @@ -385,11 +396,12 @@ def to_type(inputs, type): ): return inputs.float() / 255 else: - raise ValueError(type + " is not a valid type. [Options: float, int]") + raise ValueError( + type + " is not a valid type. [Options: float, int]") return inputs @staticmethod - def _save_print(progress, rob_acc, l2, elapsed_time, end): + def _save_print(progress: float, rob_acc: float, l2: float, elapsed_time: float, end: Union[str, None]) -> None: print( "- Save progress: %2.2f %% / Robust accuracy: %2.2f %% / L2: %1.5f (%2.3f it/s) \t" % (progress, rob_acc, l2, elapsed_time), @@ -398,13 +410,13 @@ def _save_print(progress, rob_acc, l2, elapsed_time, end): @staticmethod def load( - load_path, - batch_size=128, - shuffle=False, + load_path: str, + batch_size: int = 128, + shuffle: bool = False, normalize=None, - load_predictions=False, - load_clean_inputs=False, - ): + load_predictions: bool = False, + load_clean_inputs: bool = False, + ) -> DataLoader: save_dict = torch.load(load_path) keys = ["adv_inputs", "labels"] @@ -431,14 +443,15 @@ def load( ) / std # nopep8 adv_data = TensorDataset(*[save_dict[key] for key in keys]) - adv_loader = DataLoader(adv_data, batch_size=batch_size, shuffle=shuffle) + adv_loader = DataLoader( + adv_data, batch_size=batch_size, shuffle=shuffle) print( "Data is loaded in the following order: [%s]" % (", ".join(keys)) ) # nopep8 return adv_loader @torch.no_grad() - def get_output_with_eval_nograd(self, inputs): + def get_output_with_eval_nograd(self, inputs: torch.Tensor) -> torch.Tensor: given_training = self.model.training if given_training: self.model.eval() @@ -447,7 +460,7 @@ def get_output_with_eval_nograd(self, inputs): self.model.train() return outputs - def get_target_label(self, inputs, labels=None): + def get_target_label(self, inputs: torch.Tensor, labels: Union[torch.Tensor, None] = None) -> torch.Tensor: r""" Function for changing the attack mode. Return input labels. @@ -457,13 +470,13 @@ def get_target_label(self, inputs, labels=None): "target_map_function is not initialized by set_mode_targeted." ) if self.attack_mode == "targeted(label)": - target_labels = labels + target_labels = self.target_labels else: target_labels = self._target_map_function(inputs, labels) return target_labels @torch.no_grad() - def get_least_likely_label(self, inputs, labels=None): + def get_least_likely_label(self, inputs: torch.Tensor, labels: Union[torch.Tensor, None] = None) -> torch.Tensor: outputs = self.get_output_with_eval_nograd(inputs) if labels is None: _, labels = torch.max(outputs, dim=1) @@ -479,7 +492,7 @@ def get_least_likely_label(self, inputs, labels=None): return target_labels.long().to(self.device) @torch.no_grad() - def get_random_target_label(self, inputs, labels=None): + def get_random_target_label(self, inputs: torch.Tensor, labels: Union[torch.Tensor, None] = None) -> torch.Tensor: outputs = self.get_output_with_eval_nograd(inputs) if labels is None: _, labels = torch.max(outputs, dim=1) @@ -494,7 +507,7 @@ def get_random_target_label(self, inputs, labels=None): return target_labels.long().to(self.device) - def __call__(self, inputs, labels=None, *args, **kwargs): + def __call__(self, inputs: torch.Tensor, labels: Union[torch.Tensor, None] = None, *args, **kwargs) -> torch.Tensor: given_training = self.model.training self._change_model_mode(given_training) diff --git a/torchattacks/attacks/afab.py b/torchattacks/attacks/afab.py new file mode 100644 index 00000000..15b29edc --- /dev/null +++ b/torchattacks/attacks/afab.py @@ -0,0 +1,909 @@ +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +import time +import math + +import torch +import torch.nn.functional as F + +# zero_gradients deprecated in torch >= 1.9. +# zero_gradients is re-defined in the bottom of the code. +# from torch.autograd.gradcheck import zero_gradients +from collections import abc as container_abcs + +from ..attack import Attack + + +class AFAB(Attack): + r""" + Fast Adaptive Boundary Attack in the paper 'Minimally distorted Adversarial Examples with a Fast Adaptive Boundary Attack' + [https://arxiv.org/abs/1907.02044] + [https://github.com/fra31/auto-attack] + + Distance Measure : Linf, L2, L1 + + Arguments: + model (nn.Module): model to attack. + norm (str) : Lp-norm to minimize. ['Linf', 'L2', 'L1'] (Default: 'Linf') + eps (float): maximum perturbation. (Default: 8/255) + steps (int): number of steps. (Default: 10) + n_restarts (int): number of random restarts. (Default: 1) + alpha_max (float): alpha_max. (Default: 0.1) + eta (float): overshooting. (Default: 1.05) + beta (float): backward step. (Default: 0.9) + verbose (bool): print progress. (Default: False) + seed (int): random seed for the starting point. (Default: 0) + targeted (bool): targeted attack for every wrong classes. (Default: False) + n_classes (int): number of classes. (Default: 10) + + Shape: + - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. + - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. + - output: :math:`(N, C, H, W)`. + + Examples:: + >>> attack = torchattacks.AFAB(model, norm='Linf', steps=10, eps=8/255, n_restarts=1, alpha_max=0.1, eta=1.05, beta=0.9, loss_fn=None, verbose=False, seed=0, targeted=False, n_classes=10) + >>> adv_images = attack(images, labels) + + """ + + def __init__( + self, + model, + norm="Linf", + eps=8 / 255, + steps=10, + n_restarts=1, + alpha_max=0.1, + eta=1.05, + beta=0.9, + verbose=False, + seed=0, + multi_targeted=False, + n_classes=10, + ): + super().__init__("AFAB", model) + self.norm = norm + self.n_restarts = n_restarts + Default_EPS_DICT_BY_NORM = {"Linf": 0.3, "L2": 1.0, "L1": 5.0} + self.eps = eps if eps is not None else Default_EPS_DICT_BY_NORM[norm] + self.alpha_max = alpha_max + self.eta = eta + self.beta = beta + self.steps = steps + self.verbose = verbose + self.seed = seed + self.target_class = None + self.multi_targeted = multi_targeted + self.n_target_classes = n_classes - 1 + self.supported_mode = ["default", "targeted"] + + def forward(self, images, labels): + r""" + Overridden. + """ + + images = images.clone().detach().to(self.device) + labels = labels.clone().detach().to(self.device) + adv_images = self.perturb(images, labels) + + return adv_images + + def _get_predicted_label(self, x): + with torch.no_grad(): + outputs = self.get_logits(x) + _, y = torch.max(outputs, dim=1) + return y + + def check_shape(self, x): + return x if len(x.shape) > 0 else x.unsqueeze(0) + + def get_diff_logits_grads_batch(self, imgs, la): + im = imgs.clone().requires_grad_() + with torch.enable_grad(): + y = self.get_logits(im) + + g2 = torch.zeros([y.shape[-1], *imgs.size()]).to(self.device) + grad_mask = torch.zeros_like(y) + for counter in range(y.shape[-1]): + zero_gradients(im) + grad_mask[:, counter] = 1.0 + y.backward(grad_mask, retain_graph=True) + grad_mask[:, counter] = 0.0 + g2[counter] = im.grad.data + + g2 = torch.transpose(g2, 0, 1).detach() + # y2 = self.get_logits(imgs).detach() + y2 = y.detach() + df = y2 - y2[torch.arange(imgs.shape[0]), la].unsqueeze(1) + dg = g2 - g2[torch.arange(imgs.shape[0]), la].unsqueeze(1) + df[torch.arange(imgs.shape[0]), la] = 1e10 + + return df, dg + + def get_diff_logits_grads_batch_targeted(self, imgs, la, la_target): + u = torch.arange(imgs.shape[0]) + im = imgs.clone().requires_grad_() + with torch.enable_grad(): + y = self.get_logits(im) + diffy = -(y[u, la] - y[u, la_target]) + sumdiffy = diffy.sum() + + zero_gradients(im) + sumdiffy.backward() + graddiffy = im.grad.data + df = diffy.detach().unsqueeze(1) + dg = graddiffy.unsqueeze(1) + + return df, dg + + def attack_single_run(self, x, y=None, use_rand_start=False): + """ + :param x: clean images + :param y: clean labels, if None we use the predicted labels + """ + + # self.device = x.device + self.orig_dim = list(x.shape[1:]) + self.ndims = len(self.orig_dim) + + x = x.detach().clone().float().to(self.device) + # assert next(self.model.parameters()).device == x.device + + y_pred = self._get_predicted_label(x) + if y is None: + y = y_pred.detach().clone().long().to(self.device) + else: + y = y.detach().clone().long().to(self.device) + pred = y_pred == y + corr_classified = pred.float().sum() + if self.verbose: + print("Clean accuracy: {:.2%}".format(pred.float().mean())) + if pred.sum() == 0: + return x + pred = self.check_shape(pred.nonzero().squeeze()) + + startt = time.time() + # runs the attack only on correctly classified points + im2 = x[pred].detach().clone() + la2 = y[pred].detach().clone() + if len(im2.shape) == self.ndims: + im2 = im2.unsqueeze(0) + bs = im2.shape[0] + u1 = torch.arange(bs) + adv = im2.clone() + adv_c = x.clone() + res2 = 1e10 * torch.ones([bs]).to(self.device) + res_c = torch.zeros([x.shape[0]]).to(self.device) + x1 = im2.clone() + x0 = im2.clone().reshape([bs, -1]) + counter_restarts = 0 + + while counter_restarts < 1: + if use_rand_start: + if self.norm == "Linf": + t = 2 * torch.rand(x1.shape).to(self.device) - 1 + x1 = ( + im2 + + ( + torch.min( + res2, self.eps * torch.ones(res2.shape).to(self.device) + ).reshape([-1, *[1] * self.ndims]) + ) + * t + / ( + t.reshape([t.shape[0], -1]) + .abs() + .max(dim=1, keepdim=True)[0] + .reshape([-1, *[1] * self.ndims]) + ) + * 0.5 + ) + elif self.norm == "L2": + t = torch.randn(x1.shape).to(self.device) + x1 = ( + im2 + + ( + torch.min( + res2, self.eps * torch.ones(res2.shape).to(self.device) + ).reshape([-1, *[1] * self.ndims]) + ) + * t + / ( + (t ** 2) + .view(t.shape[0], -1) + .sum(dim=-1) + .sqrt() + .view(t.shape[0], *[1] * self.ndims) + ) + * 0.5 + ) + elif self.norm == "L1": + t = torch.randn(x1.shape).to(self.device) + x1 = ( + im2 + + ( + torch.min( + res2, self.eps * torch.ones(res2.shape).to(self.device) + ).reshape([-1, *[1] * self.ndims]) + ) + * t + / ( + t.abs() + .view(t.shape[0], -1) + .sum(dim=-1) + .view(t.shape[0], *[1] * self.ndims) + ) + / 2 + ) + + x1 = x1.clamp(0.0, 1.0) + + counter_iter = 0 + while counter_iter < self.steps: + with torch.no_grad(): + df, dg = self.get_diff_logits_grads_batch(x1, la2) + if self.norm == "Linf": + dist1 = df.abs() / ( + 1e-12 + + dg.abs().view(dg.shape[0], dg.shape[1], -1).sum(dim=-1) + ) + elif self.norm == "L2": + dist1 = df.abs() / ( + 1e-12 + + (dg ** 2) + .view(dg.shape[0], dg.shape[1], -1) + .sum(dim=-1) + .sqrt() + ) + elif self.norm == "L1": + dist1 = df.abs() / ( + 1e-12 + + dg.abs() + .reshape([df.shape[0], df.shape[1], -1]) + .max(dim=2)[0] + ) + else: + raise ValueError("norm not supported") + ind = dist1.min(dim=1)[1] + dg2 = dg[u1, ind] + b = -df[u1, ind] + (dg2 * x1).view(x1.shape[0], -1).sum(dim=-1) + w = dg2.reshape([bs, -1]) + + if self.norm == "Linf": + d3 = projection_linf( + torch.cat((x1.reshape([bs, -1]), x0), 0), + torch.cat((w, w), 0), + torch.cat((b, b), 0), + ) + elif self.norm == "L2": + d3 = projection_l2( + torch.cat((x1.reshape([bs, -1]), x0), 0), + torch.cat((w, w), 0), + torch.cat((b, b), 0), + ) + elif self.norm == "L1": + d3 = projection_l1( + torch.cat((x1.reshape([bs, -1]), x0), 0), + torch.cat((w, w), 0), + torch.cat((b, b), 0), + ) + d1 = torch.reshape(d3[:bs], x1.shape) + d2 = torch.reshape(d3[-bs:], x1.shape) + if self.norm == "Linf": + a0 = ( + d3.abs() + .max(dim=1, keepdim=True)[0] + .view(-1, *[1] * self.ndims) + ) + elif self.norm == "L2": + a0 = ( + (d3 ** 2) + .sum(dim=1, keepdim=True) + .sqrt() + .view(-1, *[1] * self.ndims) + ) + elif self.norm == "L1": + a0 = ( + d3.abs() + .sum(dim=1, keepdim=True) + .view(-1, *[1] * self.ndims) + ) + a0 = torch.max(a0, 1e-8 * torch.ones(a0.shape).to(self.device)) + a1 = a0[:bs] + a2 = a0[-bs:] + alpha = torch.min( + torch.max( + a1 / (a1 + a2), torch.zeros(a1.shape).to(self.device) + ), + self.alpha_max * torch.ones(a1.shape).to(self.device), + ) + x1 = ( + (x1 + self.eta * d1) * (1 - alpha) + + (im2 + d2 * self.eta) * alpha + ).clamp(0.0, 1.0) + + is_adv = self._get_predicted_label(x1) != la2 + + if is_adv.sum() > 0: + ind_adv = is_adv.nonzero().squeeze() + ind_adv = self.check_shape(ind_adv) + if self.norm == "Linf": + t = ( + (x1[ind_adv] - im2[ind_adv]) + .reshape([ind_adv.shape[0], -1]) + .abs() + .max(dim=1)[0] + ) + elif self.norm == "L2": + t = ( + ((x1[ind_adv] - im2[ind_adv]) ** 2) + .view(ind_adv.shape[0], -1) + .sum(dim=-1) + .sqrt() + ) + elif self.norm == "L1": + t = ( + (x1[ind_adv] - im2[ind_adv]) + .abs() + .view(ind_adv.shape[0], -1) + .sum(dim=-1) + ) + adv[ind_adv] = x1[ind_adv] * ( + t < res2[ind_adv] + ).float().reshape([-1, *[1] * self.ndims]) + adv[ind_adv] * ( + t >= res2[ind_adv] + ).float().reshape( + [-1, *[1] * self.ndims] + ) + res2[ind_adv] = ( + t * (t < res2[ind_adv]).float() + + res2[ind_adv] * (t >= res2[ind_adv]).float() + ) + x1[ind_adv] = ( + im2[ind_adv] + (x1[ind_adv] - im2[ind_adv]) * self.beta + ) + + counter_iter += 1 + + counter_restarts += 1 + + ind_succ = res2 < 1e10 + if self.verbose: + print( + "success rate: {:.0f}/{:.0f}".format( + ind_succ.float().sum(), corr_classified + ) + + " (on correctly classified points) in {:.1f} s".format( + time.time() - startt + ) + ) + + res_c[pred] = res2 * ind_succ.float() + 1e10 * (1 - ind_succ.float()) + ind_succ = self.check_shape(ind_succ.nonzero().squeeze()) + adv_c[pred[ind_succ]] = adv[ind_succ].clone() + + return adv_c + + def attack_single_run_targeted(self, x, y=None, use_rand_start=False): + """ + :param x: clean images + :param y: clean labels, if None we use the predicted labels + """ + + if self.device is None: + self.device = x.device + self.orig_dim = list(x.shape[1:]) + self.ndims = len(self.orig_dim) + + x = x.detach().clone().float().to(self.device) + # assert next(self.model.parameters()).device == x.device + + y_pred = self._get_predicted_label(x) + if y is None: + y = y_pred.detach().clone().long().to(self.device) + else: + y = y.detach().clone().long().to(self.device) + pred = y_pred == y + corr_classified = pred.float().sum() + if self.verbose: + print("Clean accuracy: {:.2%}".format(pred.float().mean())) + if pred.sum() == 0: + return x + pred = self.check_shape(pred.nonzero().squeeze()) + + output = self.get_logits(x) + if self.multi_targeted: + la_target = output.sort(dim=-1)[1][:, -self.target_class] + else: + la_target = self.target_class + + startt = time.time() + # runs the attack only on correctly classified points + im2 = x[pred].detach().clone() + la2 = y[pred].detach().clone() + la_target2 = la_target[pred].detach().clone() + if len(im2.shape) == self.ndims: + im2 = im2.unsqueeze(0) + bs = im2.shape[0] + u1 = torch.arange(bs) + adv = im2.clone() + adv_c = x.clone() + res2 = 1e10 * torch.ones([bs]).to(self.device) + res_c = torch.zeros([x.shape[0]]).to(self.device) + x1 = im2.clone() + x0 = im2.clone().reshape([bs, -1]) + counter_restarts = 0 + + while counter_restarts < 1: + if use_rand_start: + if self.norm == "Linf": + t = 2 * torch.rand(x1.shape).to(self.device) - 1 + x1 = ( + im2 + + ( + torch.min( + res2, self.eps * torch.ones(res2.shape).to(self.device) + ).reshape([-1, *[1] * self.ndims]) + ) + * t + / ( + t.reshape([t.shape[0], -1]) + .abs() + .max(dim=1, keepdim=True)[0] + .reshape([-1, *[1] * self.ndims]) + ) + * 0.5 + ) + elif self.norm == "L2": + t = torch.randn(x1.shape).to(self.device) + x1 = ( + im2 + + ( + torch.min( + res2, self.eps * torch.ones(res2.shape).to(self.device) + ).reshape([-1, *[1] * self.ndims]) + ) + * t + / ( + (t ** 2) + .view(t.shape[0], -1) + .sum(dim=-1) + .sqrt() + .view(t.shape[0], *[1] * self.ndims) + ) + * 0.5 + ) + elif self.norm == "L1": + t = torch.randn(x1.shape).to(self.device) + x1 = ( + im2 + + ( + torch.min( + res2, self.eps * torch.ones(res2.shape).to(self.device) + ).reshape([-1, *[1] * self.ndims]) + ) + * t + / ( + t.abs() + .view(t.shape[0], -1) + .sum(dim=-1) + .view(t.shape[0], *[1] * self.ndims) + ) + / 2 + ) + + x1 = x1.clamp(0.0, 1.0) + + counter_iter = 0 + while counter_iter < self.steps: + with torch.no_grad(): + df, dg = self.get_diff_logits_grads_batch_targeted( + x1, la2, la_target2 + ) + if self.norm == "Linf": + dist1 = df.abs() / ( + 1e-12 + + dg.abs().view(dg.shape[0], dg.shape[1], -1).sum(dim=-1) + ) + elif self.norm == "L2": + dist1 = df.abs() / ( + 1e-12 + + (dg ** 2) + .view(dg.shape[0], dg.shape[1], -1) + .sum(dim=-1) + .sqrt() + ) + elif self.norm == "L1": + dist1 = df.abs() / ( + 1e-12 + + dg.abs() + .reshape([df.shape[0], df.shape[1], -1]) + .max(dim=2)[0] + ) + else: + raise ValueError("norm not supported") + ind = dist1.min(dim=1)[1] + + dg2 = dg[u1, ind] + b = -df[u1, ind] + (dg2 * x1).view(x1.shape[0], -1).sum(dim=-1) + w = dg2.reshape([bs, -1]) + + if self.norm == "Linf": + d3 = projection_linf( + torch.cat((x1.reshape([bs, -1]), x0), 0), + torch.cat((w, w), 0), + torch.cat((b, b), 0), + ) + elif self.norm == "L2": + d3 = projection_l2( + torch.cat((x1.reshape([bs, -1]), x0), 0), + torch.cat((w, w), 0), + torch.cat((b, b), 0), + ) + elif self.norm == "L1": + d3 = projection_l1( + torch.cat((x1.reshape([bs, -1]), x0), 0), + torch.cat((w, w), 0), + torch.cat((b, b), 0), + ) + d1 = torch.reshape(d3[:bs], x1.shape) + d2 = torch.reshape(d3[-bs:], x1.shape) + if self.norm == "Linf": + a0 = ( + d3.abs() + .max(dim=1, keepdim=True)[0] + .view(-1, *[1] * self.ndims) + ) + elif self.norm == "L2": + a0 = ( + (d3 ** 2) + .sum(dim=1, keepdim=True) + .sqrt() + .view(-1, *[1] * self.ndims) + ) + elif self.norm == "L1": + a0 = ( + d3.abs() + .sum(dim=1, keepdim=True) + .view(-1, *[1] * self.ndims) + ) + a0 = torch.max(a0, 1e-8 * torch.ones(a0.shape).to(self.device)) + a1 = a0[:bs] + a2 = a0[-bs:] + alpha = torch.min( + torch.max( + a1 / (a1 + a2), torch.zeros(a1.shape).to(self.device) + ), + self.alpha_max * torch.ones(a1.shape).to(self.device), + ) + x1 = ( + (x1 + self.eta * d1) * (1 - alpha) + + (im2 + d2 * self.eta) * alpha + ).clamp(0.0, 1.0) + + is_adv = self._get_predicted_label(x1) != la2 + + if is_adv.sum() > 0: + ind_adv = is_adv.nonzero().squeeze() + ind_adv = self.check_shape(ind_adv) + if self.norm == "Linf": + t = ( + (x1[ind_adv] - im2[ind_adv]) + .reshape([ind_adv.shape[0], -1]) + .abs() + .max(dim=1)[0] + ) + elif self.norm == "L2": + t = ( + ((x1[ind_adv] - im2[ind_adv]) ** 2) + .view(ind_adv.shape[0], -1) + .sum(dim=-1) + .sqrt() + ) + elif self.norm == "L1": + t = ( + (x1[ind_adv] - im2[ind_adv]) + .abs() + .view(ind_adv.shape[0], -1) + .sum(dim=-1) + ) + adv[ind_adv] = x1[ind_adv] * ( + t < res2[ind_adv] + ).float().reshape([-1, *[1] * self.ndims]) + adv[ind_adv] * ( + t >= res2[ind_adv] + ).float().reshape( + [-1, *[1] * self.ndims] + ) + res2[ind_adv] = ( + t * (t < res2[ind_adv]).float() + + res2[ind_adv] * (t >= res2[ind_adv]).float() + ) + x1[ind_adv] = ( + im2[ind_adv] + (x1[ind_adv] - im2[ind_adv]) * self.beta + ) + + counter_iter += 1 + + counter_restarts += 1 + + ind_succ = res2 < 1e10 + if self.verbose: + print( + "success rate: {:.0f}/{:.0f}".format( + ind_succ.float().sum(), corr_classified + ) + + " (on correctly classified points) in {:.1f} s".format( + time.time() - startt + ) + ) + + res_c[pred] = res2 * ind_succ.float() + 1e10 * (1 - ind_succ.float()) + ind_succ = self.check_shape(ind_succ.nonzero().squeeze()) + adv_c[pred[ind_succ]] = adv[ind_succ].clone() + + return adv_c + + def perturb(self, x, y): + adv = x.clone() + with torch.no_grad(): + acc = self.get_logits(x).max(1)[1] == y + + startt = time.time() + + torch.random.manual_seed(self.seed) + torch.cuda.random.manual_seed(self.seed) + + def inner_perturb(targeted): + for counter in range(self.n_restarts): + ind_to_fool = acc.nonzero().squeeze() + if len(ind_to_fool.shape) == 0: + ind_to_fool = ind_to_fool.unsqueeze(0) + if ind_to_fool.numel() != 0: + x_to_fool, y_to_fool = ( + x[ind_to_fool].clone(), + y[ind_to_fool].clone(), + ) # nopep8 + + if targeted: + adv_curr = self.attack_single_run_targeted( + x_to_fool, y_to_fool, use_rand_start=(counter > 0) + ) + else: + adv_curr = self.attack_single_run( + x_to_fool, y_to_fool, use_rand_start=(counter > 0) + ) + + acc_curr = self.get_logits(adv_curr).max(1)[1] == y_to_fool + if self.norm == "Linf": + res = ( + (x_to_fool - adv_curr) + .abs() + .view(x_to_fool.shape[0], -1) + .max(1)[0] + ) # nopep8 + elif self.norm == "L2": + res = ( + ((x_to_fool - adv_curr) ** 2) + .view(x_to_fool.shape[0], -1) + .sum(dim=-1) + .sqrt() + ) # nopep8 + acc_curr = torch.max(acc_curr, res > self.eps) + + ind_curr = (acc_curr == 0).nonzero().squeeze() + acc[ind_to_fool[ind_curr]] = 0 + adv[ind_to_fool[ind_curr]] = adv_curr[ind_curr].clone() + + if self.verbose: + if targeted: + print( + "restart {} - target_class {} - robust accuracy: {:.2%} at eps = {:.5f} - cum. time: {:.1f} s".format( + counter, + self.target_class, + acc.float().mean(), + self.eps, + time.time() - startt, + ) + ) + else: + print( + "restart {} - robust accuracy: {:.2%} at eps = {:.5f} - cum. time: {:.1f} s".format( + counter, + acc.float().mean(), + self.eps, + time.time() - startt, + ) + ) + + if self.multi_targeted: + for target_class in range(2, self.n_target_classes + 2): + self.target_class = target_class + inner_perturb(targeted=True) + elif self.targeted: + self.target_class = self.get_target_label(x, y) + inner_perturb(targeted=True) + else: + inner_perturb(targeted=False) + return adv + + +def projection_linf(points_to_project, w_hyperplane, b_hyperplane): + device = points_to_project.device + t, w, b = points_to_project, w_hyperplane.clone(), b_hyperplane.clone() + + sign = 2 * ((w * t).sum(1) - b >= 0) - 1 + w.mul_(sign.unsqueeze(1)) + b.mul_(sign) + + a = (w < 0).float() + d = (a - t) * (w != 0).float() + + p = a - t * (2 * a - 1) + indp = torch.argsort(p, dim=1) + + b = b - (w * t).sum(1) + b0 = (w * d).sum(1) + + indp2 = indp.flip((1,)) + ws = w.gather(1, indp2) + bs2 = -ws * d.gather(1, indp2) + + s = torch.cumsum(ws.abs(), dim=1) + sb = torch.cumsum(bs2, dim=1) + b0.unsqueeze(1) + + b2 = sb[:, -1] - s[:, -1] * p.gather(1, indp[:, 0:1]).squeeze(1) + c_l = b - b2 > 0 + c2 = (b - b0 > 0) & (~c_l) + lb = torch.zeros(c2.sum(), device=device) + ub = torch.full_like(lb, w.shape[1] - 1) + nitermax = math.ceil(math.log2(w.shape[1])) + + indp_, sb_, s_, p_, b_ = indp[c2], sb[c2], s[c2], p[c2], b[c2] + for counter in range(nitermax): + counter4 = torch.floor((lb + ub) / 2) + + counter2 = counter4.long().unsqueeze(1) + indcurr = indp_.gather(1, indp_.size(1) - 1 - counter2) + b2 = ( + sb_.gather(1, counter2) - s_.gather(1, counter2) * p_.gather(1, indcurr) + ).squeeze( + 1 + ) # nopep8 + c = b_ - b2 > 0 + + lb = torch.where(c, counter4, lb) + ub = torch.where(c, ub, counter4) + + lb = lb.long() + + if c_l.any(): + lmbd_opt = torch.clamp_min( + (b[c_l] - sb[c_l, -1]) / (-s[c_l, -1]), min=0 + ).unsqueeze(-1) + d[c_l] = (2 * a[c_l] - 1) * lmbd_opt + + lmbd_opt = torch.clamp_min((b[c2] - sb[c2, lb]) / (-s[c2, lb]), min=0).unsqueeze(-1) + d[c2] = torch.min(lmbd_opt, d[c2]) * a[c2] + torch.max(-lmbd_opt, d[c2]) * ( + 1 - a[c2] + ) + + return d * (w != 0).float() + + +def projection_l2(points_to_project, w_hyperplane, b_hyperplane): + device = points_to_project.device + t, w, b = points_to_project, w_hyperplane.clone(), b_hyperplane + + c = (w * t).sum(1) - b + ind2 = 2 * (c >= 0) - 1 + w.mul_(ind2.unsqueeze(1)) + c.mul_(ind2) + + r = torch.max(t / w, (t - 1) / w).clamp(min=-1e12, max=1e12) + r.masked_fill_(w.abs() < 1e-8, 1e12) + r[r == -1e12] *= -1 + rs, indr = torch.sort(r, dim=1) + rs2 = F.pad(rs[:, 1:], (0, 1)) + rs.masked_fill_(rs == 1e12, 0) + rs2.masked_fill_(rs2 == 1e12, 0) + + w3s = (w ** 2).gather(1, indr) + w5 = w3s.sum(dim=1, keepdim=True) + ws = w5 - torch.cumsum(w3s, dim=1) + d = -(r * w) + d.mul_((w.abs() > 1e-8).float()) + s = torch.cat( + (-w5 * rs[:, 0:1], torch.cumsum((-rs2 + rs) * ws, dim=1) - w5 * rs[:, 0:1]), 1 + ) + + c4 = s[:, 0] + c < 0 + c3 = (d * w).sum(dim=1) + c > 0 + c2 = ~(c4 | c3) + + lb = torch.zeros(c2.sum(), device=device) + ub = torch.full_like(lb, w.shape[1] - 1) + nitermax = math.ceil(math.log2(w.shape[1])) + + s_, c_ = s[c2], c[c2] + for counter in range(nitermax): + counter4 = torch.floor((lb + ub) / 2) + counter2 = counter4.long().unsqueeze(1) + c3 = s_.gather(1, counter2).squeeze(1) + c_ > 0 + lb = torch.where(c3, counter4, lb) + ub = torch.where(c3, ub, counter4) + + lb = lb.long() + + if c4.any(): + alpha = c[c4] / w5[c4].squeeze(-1) + d[c4] = -alpha.unsqueeze(-1) * w[c4] + + if c2.any(): + alpha = (s[c2, lb] + c[c2]) / ws[c2, lb] + rs[c2, lb] + alpha[ws[c2, lb] == 0] = 0 + c5 = (alpha.unsqueeze(-1) > r[c2]).float() + d[c2] = d[c2] * c5 - alpha.unsqueeze(-1) * w[c2] * (1 - c5) + + return d * (w.abs() > 1e-8).float() + + +def projection_l1(points_to_project, w_hyperplane, b_hyperplane): + device = points_to_project.device + t, w, b = points_to_project, w_hyperplane.clone(), b_hyperplane + + c = (w * t).sum(1) - b + ind2 = 2 * (c >= 0) - 1 + w.mul_(ind2.unsqueeze(1)) + c.mul_(ind2) + + r = (1 / w).abs().clamp_max(1e12) + indr = torch.argsort(r, dim=1) + indr_rev = torch.argsort(indr) + + c6 = (w < 0).float() + d = (-t + c6) * (w != 0).float() + ds = torch.min(-w * t, w * (1 - t)).gather(1, indr) + ds2 = torch.cat((c.unsqueeze(-1), ds), 1) + s = torch.cumsum(ds2, dim=1) + + c2 = s[:, -1] < 0 + + lb = torch.zeros(c2.sum(), device=device) + ub = torch.full_like(lb, s.shape[1]) + nitermax = math.ceil(math.log2(w.shape[1])) + + s_ = s[c2] + for counter in range(nitermax): + counter4 = torch.floor((lb + ub) / 2) + counter2 = counter4.long().unsqueeze(1) + c3 = s_.gather(1, counter2).squeeze(1) > 0 + lb = torch.where(c3, counter4, lb) + ub = torch.where(c3, ub, counter4) + + lb2 = lb.long() + + if c2.any(): + indr = indr[c2].gather(1, lb2.unsqueeze(1)).squeeze(1) + u = torch.arange(0, w.shape[0], device=device).unsqueeze(1) + u2 = torch.arange(0, w.shape[1], device=device, dtype=torch.float).unsqueeze(0) + alpha = -s[c2, lb2] / w[c2, indr] + c5 = u2 < lb.unsqueeze(-1) + u3 = c5[u[: c5.shape[0]], indr_rev[c2]] + d[c2] = d[c2] * u3.float() + d[c2, indr] = alpha + + return d * (w.abs() > 1e-8).float() + + +def zero_gradients(x): + if isinstance(x, torch.Tensor): + if x.grad is not None: + x.grad.detach_() + x.grad.zero_() + elif isinstance(x, container_abcs.Iterable): + for elem in x: + zero_gradients(elem) diff --git a/torchattacks/attacks/autoattack.py b/torchattacks/attacks/autoattack.py index 2c72cd3c..28f9da2d 100644 --- a/torchattacks/attacks/autoattack.py +++ b/torchattacks/attacks/autoattack.py @@ -4,7 +4,7 @@ from ..wrappers.multiattack import MultiAttack from .apgd import APGD from .apgdt import APGDT -from .fab import FAB +from .afab import AFAB from .square import Square @@ -76,7 +76,7 @@ def __init__( n_classes=n_classes, n_restarts=1, ), - FAB( + AFAB( model, eps=eps, norm=norm, @@ -120,7 +120,7 @@ def __init__( loss="dlr", n_restarts=5, ), - FAB( + AFAB( model, eps=eps, norm=norm, @@ -147,7 +147,7 @@ def __init__( n_classes=n_classes, n_restarts=1, ), - FAB( + AFAB( model, eps=eps, norm=norm, diff --git a/torchattacks/attacks/cw.py b/torchattacks/attacks/cw.py index 8e33f5fc..c2b98da6 100644 --- a/torchattacks/attacks/cw.py +++ b/torchattacks/attacks/cw.py @@ -18,8 +18,9 @@ class CW(Attack): :math:`minimize \Vert\frac{1}{2}(tanh(w)+1)-x\Vert^2_2+c\cdot f(\frac{1}{2}(tanh(w)+1))` kappa (float): kappa (also written as 'confidence') in the paper. (Default: 0) :math:`f(x')=max(max\{Z(x')_i:i\neq t\} -Z(x')_t, - \kappa)` - steps (int): number of steps. (Default: 50) + steps (int): number of steps (also written as 'max_iterations'). (Default: 50) lr (float): learning rate of the Adam optimizer. (Default: 0.01) + abort_early: if true, allows early aborts if gradient descent gets stuck. (Default: True) .. warning:: With default c, you can't easily get adversarial images. Set higher c like 1. @@ -29,19 +30,20 @@ class CW(Attack): - output: :math:`(N, C, H, W)`. Examples:: - >>> attack = torchattacks.CW(model, c=1, kappa=0, steps=50, lr=0.01) + >>> attack = torchattacks.CWL2(model, c=1, kappa=0, steps=50, lr=0.01, abort_early=True) >>> adv_images = attack(images, labels) - .. note:: Binary search for c is NOT IMPLEMENTED methods in the paper due to time consuming. + .. note:: The binary search version of the CW algorithm has been implemented as CWBS. """ - def __init__(self, model, c=1, kappa=0, steps=50, lr=0.01): + def __init__(self, model, c=1, kappa=0, steps=50, lr=0.01, abort_early=True): super().__init__("CW", model) self.c = c self.kappa = kappa self.steps = steps self.lr = lr + self.abort_early = abort_early self.supported_mode = ["default", "targeted"] def forward(self, images, labels): @@ -53,16 +55,16 @@ def forward(self, images, labels): labels = labels.clone().detach().to(self.device) if self.targeted: - target_labels = self.get_target_label(images, labels) + labels = self.get_target_label(images, labels) # w = torch.zeros_like(images).detach() # Requires 2x times w = self.inverse_tanh_space(images).detach() w.requires_grad = True best_adv_images = images.clone().detach() - best_L2 = 1e10 * torch.ones((len(images))).to(self.device) + batch_size = len(images) + best_Lx = torch.full((batch_size, ), 1e10).to(self.device) prev_cost = 1e10 - dim = len(images.shape) MSELoss = nn.MSELoss(reduction="none") Flatten = nn.Flatten() @@ -74,16 +76,18 @@ def forward(self, images, labels): adv_images = self.tanh_space(w) # Calculate loss - current_L2 = MSELoss(Flatten(adv_images), Flatten(images)).sum(dim=1) - L2_loss = current_L2.sum() + current_Lx = MSELoss(Flatten(adv_images), + Flatten(images)).sum(dim=1) + + Lx_loss = current_Lx.sum() outputs = self.get_logits(adv_images) if self.targeted: - f_loss = self.f(outputs, target_labels).sum() + f_loss = self.f(outputs, labels) else: - f_loss = self.f(outputs, labels).sum() + f_loss = self.f(outputs, labels) - cost = L2_loss + self.c * f_loss + cost = Lx_loss + torch.sum(self.c * f_loss) optimizer.zero_grad() cost.backward() @@ -91,30 +95,34 @@ def forward(self, images, labels): # Update adversarial images pre = torch.argmax(outputs.detach(), 1) - if self.targeted: - # We want to let pre == target_labels in a targeted attack - condition = (pre == target_labels).float() - else: - # If the attack is not targeted we simply make these two values unequal - condition = (pre != labels).float() + condition_1 = self.compare(pre, labels) + condition_2 = (current_Lx < best_Lx) # Filter out images that get either correct predictions or non-decreasing loss, # i.e., only images that are both misclassified and loss-decreasing are left - mask = condition * (best_L2 > current_L2.detach()) - best_L2 = mask * current_L2.detach() + (1 - mask) * best_L2 - - mask = mask.view([-1] + [1] * (dim - 1)) - best_adv_images = mask * adv_images.detach() + (1 - mask) * best_adv_images - - # Early stop when loss does not converge. - # max(.,1) To prevent MODULO BY ZERO error in the next step. - if step % max(self.steps // 10, 1) == 0: - if cost.item() > prev_cost: - return best_adv_images - prev_cost = cost.item() - + mask = torch.logical_and(condition_1, condition_2) + best_Lx[mask] = current_Lx[mask] + best_adv_images[mask] = adv_images[mask] + + # Early stop when loss does not converge + if self.abort_early and step % (self.steps // 10) == 0: + if cost > prev_cost * 0.9999: + break + else: + prev_cost = cost + + # print(best_Lx) return best_adv_images + def compare(self, predition, labels): + if self.targeted: + # We want to let pre == target_labels in a targeted attack + ret = (predition == labels) + else: + # If the attack is not targeted we simply make these two values unequal + ret = (predition != labels) + return ret + def tanh_space(self, x): return 1 / 2 * (torch.tanh(x) + 1) @@ -130,10 +138,10 @@ def atanh(self, x): def f(self, outputs, labels): one_hot_labels = torch.eye(outputs.shape[1]).to(self.device)[labels] - # find the max logit other than the target class - other = torch.max((1 - one_hot_labels) * outputs, dim=1)[0] # get the target class's logit - real = torch.max(one_hot_labels * outputs, dim=1)[0] + real = torch.sum(one_hot_labels * outputs, dim=1) + # find the max logit other than the target class + other = torch.max((1 - one_hot_labels) * outputs - one_hot_labels * 1e12, dim=1)[0] # nopep8 if self.targeted: return torch.clamp((other - real), min=-self.kappa) diff --git a/torchattacks/attacks/cwbs.py b/torchattacks/attacks/cwbs.py new file mode 100644 index 00000000..8e9c27d8 --- /dev/null +++ b/torchattacks/attacks/cwbs.py @@ -0,0 +1,188 @@ +import torch +import torch.nn as nn +import torch.optim as optim + +from ..attack import Attack + + +class CWBS(Attack): + r""" + CW (binary search version) in the paper 'Towards Evaluating the Robustness of Neural Networks' + [https://arxiv.org/abs/1608.04644] + + Distance Measure : L2 + + Arguments: + model (nn.Module): model to attack. + init_c (float): init_c (or c) in the paper. parameter for box-constraint. (Default: 1) + :math:`minimize \Vert\frac{1}{2}(tanh(w)+1)-x\Vert^2_2+c\cdot f(\frac{1}{2}(tanh(w)+1))` + kappa (float): kappa (also written as 'confidence') in the paper. (Default: 0) + :math:`f(x')=max(max\{Z(x')_i:i\neq t\} -Z(x')_t, - \kappa)` + steps (int): number of steps (also written as 'max_iterations'). (Default: 50) + lr (float): learning rate of the Adam optimizer. (Default: 0.01) + binary_search_steps (int): The number of times we perform binary search to find the optimal tradeoff-constant between distance and confidence. (Default: 9) + abort_early: if true, allows early aborts if gradient descent gets stuck. (Default: True) + + Shape: + - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. + - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. + - output: :math:`(N, C, H, W)`. + + Examples:: + >>> attack = torchattacks.CWBSL2(model, init_c=1, kappa=0, steps=50, lr=0.01, binary_search_steps=9, abort_early=True) + >>> adv_images = attack(images, labels) + + """ + + def __init__(self, model, init_c=1, kappa=0, steps=50, lr=0.01, binary_search_steps=9, abort_early=True): + super().__init__("CWBS", model) + self.init_c = init_c + self.kappa = kappa + self.steps = steps + self.lr = lr + self.binary_search_steps = binary_search_steps + self.abort_early = abort_early + self.supported_mode = ["default", "targeted"] + + def forward(self, images, labels): + r""" + Overridden. + """ + + images = images.clone().detach().to(self.device) + labels = labels.clone().detach().to(self.device) + + if self.targeted: + labels = self.get_target_label(images, labels) + + # w = torch.zeros_like(images).detach() # Requires 2x times + w = self.inverse_tanh_space(images).detach() + w.requires_grad = True + + o_best_adv_images = images.clone().detach() + + MSELoss = nn.MSELoss(reduction="none") + Flatten = nn.Flatten() + + optimizer = optim.Adam([w], lr=self.lr) + + batch_size = len(images) + lower_bound = torch.zeros((batch_size, )).to(self.device) + const = torch.full((batch_size, ), self.init_c, + dtype=torch.float).to(self.device) + upper_bound = torch.full((batch_size, ), 1e10).to(self.device) + + o_best_score = torch.full( + (batch_size, ), -1, dtype=torch.long).to(self.device) + o_best_Lx = torch.full((batch_size, ), 1e10).to(self.device) + + for _ in range(self.binary_search_steps): + best_score = torch.full( + (batch_size, ), -1, dtype=torch.long).to(self.device) + best_Lx = torch.full((batch_size, ), 1e10).to(self.device) + prev_cost = 1e10 + for step in range(self.steps): + # Get adversarial images + adv_images = self.tanh_space(w) + + # Calculate loss + current_Lx = MSELoss(Flatten(adv_images), + Flatten(images)).sum(dim=1) + + Lx_loss = current_Lx.sum() + + outputs = self.get_logits(adv_images) + if self.targeted: + # f_loss = self.f(outputs, target_labels).sum() + f_loss = self.f(outputs, labels) + else: + # f_loss = self.f(outputs, labels).sum() + f_loss = self.f(outputs, labels) + + cost = Lx_loss + torch.sum(const * f_loss) + + optimizer.zero_grad() + cost.backward() + optimizer.step() + + # Update adversarial images + pre = torch.argmax(outputs.detach(), 1) + condition_1 = self.compare(pre, labels) + condition_2 = (current_Lx < best_Lx) + # Filter out images that get either correct predictions or non-decreasing loss, + # i.e., only images that are both misclassified and loss-decreasing are left + mask_1_2 = torch.logical_and(condition_1, condition_2) + best_Lx[mask_1_2] = current_Lx[mask_1_2] + best_score[mask_1_2] = pre[mask_1_2] + + condition_3 = (current_Lx < o_best_Lx) + o_mask = torch.logical_and(condition_1, condition_3) + o_best_Lx[o_mask] = current_Lx[o_mask] + o_best_score[o_mask] = pre[o_mask] + + o_best_adv_images[o_mask] = adv_images[o_mask] + + # Check if we should abort search if we're getting nowhere. + if self.abort_early and step % (self.steps // 10) == 0: + if cost > prev_cost * 0.9999: + break + else: + prev_cost = cost + + # Adjust the constant as needed + outputs = self.get_logits(adv_images) + pre = torch.argmax(outputs, 1) + + condition_1 = self.compare(pre, labels) + condition_2 = (best_score != -1) + condition_3 = upper_bound < 1e9 + + mask_1_2 = torch.logical_and(condition_1, condition_2) + mask_1_2_3 = torch.logical_and(mask_1_2, condition_3) + const_1 = (lower_bound + upper_bound) / 2.0 + + upper_bound_min = torch.min(upper_bound, const) + upper_bound[mask_1_2] = upper_bound_min[mask_1_2] + const[mask_1_2_3] = const_1[mask_1_2_3] + + mask_n1_n2_3 = torch.logical_and(~mask_1_2, condition_3) + upper_bound_max = torch.max(lower_bound, const) + upper_bound[~mask_1_2] = upper_bound_max[~mask_1_2] + const[mask_n1_n2_3] *= 10 + + # print(o_best_Lx) + return o_best_adv_images + + def compare(self, predition, labels): + if self.targeted: + # We want to let pre == target_labels in a targeted attack + ret = (predition == labels) + else: + # If the attack is not targeted we simply make these two values unequal + ret = (predition != labels) + return ret + + def tanh_space(self, x): + return 1 / 2 * (torch.tanh(x) + 1) + + def inverse_tanh_space(self, x): + # torch.atanh is only for torch >= 1.7.0 + # atanh is defined in the range -1 to 1 + return self.atanh(torch.clamp(x * 2 - 1, min=-1, max=1)) + + def atanh(self, x): + return 0.5 * torch.log((1 + x) / (1 - x)) + + # f-function in the paper + def f(self, outputs, labels): + one_hot_labels = torch.eye(outputs.shape[1]).to(self.device)[labels] + + # get the target class's logit + real = torch.sum(one_hot_labels * outputs, dim=1) + # find the max logit other than the target classs + other = torch.max((1 - one_hot_labels) * outputs - one_hot_labels * 1e12, dim=1)[0] # nopep8 + + if self.targeted: + return torch.clamp((other - real), min=-self.kappa) + else: + return torch.clamp((real - other), min=-self.kappa) diff --git a/torchattacks/attacks/cwbsl0.py b/torchattacks/attacks/cwbsl0.py new file mode 100644 index 00000000..7c1cea31 --- /dev/null +++ b/torchattacks/attacks/cwbsl0.py @@ -0,0 +1,191 @@ +import torch +import torch.nn as nn +import torch.optim as optim + +from ..attack import Attack + + +class CWBSL0(Attack): + r""" + CW (binary search version) in the paper 'Towards Evaluating the Robustness of Neural Networks' + [https://arxiv.org/abs/1608.04644] + + Distance Measure : L0 + + Arguments: + model (nn.Module): model to attack. + init_c (float): init_c (or c) in the paper. parameter for box-constraint. (Default: 1) + :math:`minimize \Vert\frac{1}{2}(tanh(w)+1)-x\Vert^2_2+c\cdot f(\frac{1}{2}(tanh(w)+1))` + kappa (float): kappa (also written as 'confidence') in the paper. (Default: 0) + :math:`f(x')=max(max\{Z(x')_i:i\neq t\} -Z(x')_t, - \kappa)` + steps (int): number of steps (also written as 'max_iterations'). (Default: 50) + lr (float): learning rate of the Adam optimizer. (Default: 0.01) + binary_search_steps (int): The number of times we perform binary search to find the optimal tradeoff-constant between distance and confidence. (Default: 9) + abort_early: if true, allows early aborts if gradient descent gets stuck. (Default: True) + + Shape: + - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. + - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. + - output: :math:`(N, C, H, W)`. + + Examples:: + >>> attack = torchattacks.CWBSL0(model, init_c=1, kappa=0, steps=50, lr=0.01, binary_search_steps=9, abort_early=True) + >>> adv_images = attack(images, labels) + + """ + + def __init__(self, model, init_c=1, kappa=0, steps=50, lr=0.01, binary_search_steps=9, abort_early=True): + super().__init__("CWBSL0", model) + self.init_c = init_c + self.kappa = kappa + self.steps = steps + self.lr = lr + self.binary_search_steps = binary_search_steps + self.abort_early = abort_early + self.supported_mode = ["default", "targeted"] + + def forward(self, images, labels): + r""" + Overridden. + """ + + images = images.clone().detach().to(self.device) + labels = labels.clone().detach().to(self.device) + + if self.targeted: + labels = self.get_target_label(images, labels) + + # w = torch.zeros_like(images).detach() # Requires 2x times + w = self.inverse_tanh_space(images).detach() + w.requires_grad = True + + o_best_adv_images = images.clone().detach() + + optimizer = optim.Adam([w], lr=self.lr) + + batch_size = len(images) + lower_bound = torch.zeros((batch_size, )).to(self.device) + const = torch.full((batch_size, ), self.init_c, + dtype=torch.float).to(self.device) + upper_bound = torch.full((batch_size, ), 1e10).to(self.device) + + o_best_score = torch.full( + (batch_size, ), -1, dtype=torch.long).to(self.device) + o_best_Lx = torch.full((batch_size, ), 1e10).to(self.device) + + threshold = 1e-6 + + for _ in range(self.binary_search_steps): + best_score = torch.full( + (batch_size, ), -1, dtype=torch.long).to(self.device) + best_Lx = torch.full((batch_size, ), 1e10).to(self.device) + prev_cost = 1e10 + for step in range(self.steps): + # Get adversarial images + adv_images = self.tanh_space(w) + + # Calculate loss + l0_abs = torch.abs(adv_images.reshape(-1) - images.reshape(-1)) + l0_norm = (l0_abs > threshold) + # Number of non-zero values + l0_loss = (1.0 / l0_abs.shape[0]) * torch.sum(l0_norm).item() + current_Lx = torch.full( + (batch_size, ), l0_loss).to(self.device) + + Lx_loss = current_Lx.sum() + + outputs = self.get_logits(adv_images) + if self.targeted: + # f_loss = self.f(outputs, target_labels).sum() + f_loss = self.f(outputs, labels) + else: + # f_loss = self.f(outputs, labels).sum() + f_loss = self.f(outputs, labels) + + cost = Lx_loss + torch.sum(const * f_loss) + + optimizer.zero_grad() + cost.backward() + optimizer.step() + + # Update adversarial images + pre = torch.argmax(outputs.detach(), 1) + condition_1 = self.compare(pre, labels) + condition_2 = (current_Lx < best_Lx) + # Filter out images that get either correct predictions or non-decreasing loss, + # i.e., only images that are both misclassified and loss-decreasing are left + mask_1_2 = torch.logical_and(condition_1, condition_2) + best_Lx[mask_1_2] = current_Lx[mask_1_2] + best_score[mask_1_2] = pre[mask_1_2] + + condition_3 = (current_Lx < o_best_Lx) + o_mask = torch.logical_and(condition_1, condition_3) + o_best_Lx[o_mask] = current_Lx[o_mask] + o_best_score[o_mask] = pre[o_mask] + + o_best_adv_images[o_mask] = adv_images[o_mask] + + # Check if we should abort search if we're getting nowhere. + if self.abort_early and step % (self.steps // 10) == 0: + if cost > prev_cost * 0.9999: + break + else: + prev_cost = cost + + # Adjust the constant as needed + outputs = self.get_logits(adv_images) + pre = torch.argmax(outputs, 1) + + condition_1 = self.compare(pre, labels) + condition_2 = (best_score != -1) + condition_3 = upper_bound < 1e9 + + mask_1_2 = torch.logical_and(condition_1, condition_2) + mask_1_2_3 = torch.logical_and(mask_1_2, condition_3) + const_1 = (lower_bound + upper_bound) / 2.0 + + upper_bound_min = torch.min(upper_bound, const) + upper_bound[mask_1_2] = upper_bound_min[mask_1_2] + const[mask_1_2_3] = const_1[mask_1_2_3] + + mask_n1_n2_3 = torch.logical_and(~mask_1_2, condition_3) + upper_bound_max = torch.max(lower_bound, const) + upper_bound[~mask_1_2] = upper_bound_max[~mask_1_2] + const[mask_n1_n2_3] *= 10 + + # print(o_best_Lx) + return o_best_adv_images + + def compare(self, predition, labels): + if self.targeted: + # We want to let pre == target_labels in a targeted attack + ret = (predition == labels) + else: + # If the attack is not targeted we simply make these two values unequal + ret = (predition != labels) + return ret + + def tanh_space(self, x): + return 1 / 2 * (torch.tanh(x) + 1) + + def inverse_tanh_space(self, x): + # torch.atanh is only for torch >= 1.7.0 + # atanh is defined in the range -1 to 1 + return self.atanh(torch.clamp(x * 2 - 1, min=-1, max=1)) + + def atanh(self, x): + return 0.5 * torch.log((1 + x) / (1 - x)) + + # f-function in the paper + def f(self, outputs, labels): + one_hot_labels = torch.eye(outputs.shape[1]).to(self.device)[labels] + + # get the target class's logit + real = torch.sum(one_hot_labels * outputs, dim=1) + # find the max logit other than the target classs + other = torch.max((1 - one_hot_labels) * outputs - one_hot_labels * 1e12, dim=1)[0] # nopep8 + + if self.targeted: + return torch.clamp((other - real), min=-self.kappa) + else: + return torch.clamp((real - other), min=-self.kappa) diff --git a/torchattacks/attacks/cwbslinf.py b/torchattacks/attacks/cwbslinf.py new file mode 100644 index 00000000..a042d535 --- /dev/null +++ b/torchattacks/attacks/cwbslinf.py @@ -0,0 +1,187 @@ +import torch +import torch.nn as nn +import torch.optim as optim + +from ..attack import Attack + + +class CWBSLinf(Attack): + r""" + CW (binary search version) in the paper 'Towards Evaluating the Robustness of Neural Networks' + [https://arxiv.org/abs/1608.04644] + + Distance Measure : Linf + + Arguments: + model (nn.Module): model to attack. + init_c (float): init_c (or c) in the paper. parameter for box-constraint. (Default: 1) + :math:`minimize \Vert\frac{1}{2}(tanh(w)+1)-x\Vert^2_2+c\cdot f(\frac{1}{2}(tanh(w)+1))` + kappa (float): kappa (also written as 'confidence') in the paper. (Default: 0) + :math:`f(x')=max(max\{Z(x')_i:i\neq t\} -Z(x')_t, - \kappa)` + steps (int): number of steps (also written as 'max_iterations'). (Default: 50) + lr (float): learning rate of the Adam optimizer. (Default: 0.01) + binary_search_steps (int): The number of times we perform binary search to find the optimal tradeoff-constant between distance and confidence. (Default: 9) + abort_early: if true, allows early aborts if gradient descent gets stuck. (Default: True) + + Shape: + - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. + - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. + - output: :math:`(N, C, H, W)`. + + Examples:: + >>> attack = torchattacks.CWBSLinf(model, init_c=1, kappa=0, steps=50, lr=0.01, binary_search_steps=9, abort_early=True) + >>> adv_images = attack(images, labels) + + """ + + def __init__(self, model, init_c=1, kappa=0, steps=50, lr=0.01, binary_search_steps=9, abort_early=True): + super().__init__("CWBSLinf", model) + self.init_c = init_c + self.kappa = kappa + self.steps = steps + self.lr = lr + self.binary_search_steps = binary_search_steps + self.abort_early = abort_early + self.supported_mode = ["default", "targeted"] + + def forward(self, images, labels): + r""" + Overridden. + """ + + images = images.clone().detach().to(self.device) + labels = labels.clone().detach().to(self.device) + + if self.targeted: + labels = self.get_target_label(images, labels) + + # w = torch.zeros_like(images).detach() # Requires 2x times + w = self.inverse_tanh_space(images).detach() + w.requires_grad = True + + o_best_adv_images = images.clone().detach() + + optimizer = optim.Adam([w], lr=self.lr) + + batch_size = len(images) + lower_bound = torch.zeros((batch_size, )).to(self.device) + const = torch.full((batch_size, ), self.init_c, + dtype=torch.float).to(self.device) + upper_bound = torch.full((batch_size, ), 1e10).to(self.device) + + o_best_score = torch.full( + (batch_size, ), -1, dtype=torch.long).to(self.device) + o_best_Lx = torch.full((batch_size, ), 1e10).to(self.device) + + for _ in range(self.binary_search_steps): + best_score = torch.full( + (batch_size, ), -1, dtype=torch.long).to(self.device) + best_Lx = torch.full((batch_size, ), 1e10).to(self.device) + prev_cost = 1e10 + for step in range(self.steps): + # Get adversarial images + adv_images = self.tanh_space(w) + + # Calculate loss + linf_norm = torch.max(torch.abs(adv_images - images)) + linf_loss = (1.0 / batch_size) * linf_norm.item() + current_Lx = torch.full( + (batch_size, ), linf_loss).to(self.device) + + Lx_loss = current_Lx.sum() + + outputs = self.get_logits(adv_images) + if self.targeted: + # f_loss = self.f(outputs, target_labels).sum() + f_loss = self.f(outputs, labels) + else: + # f_loss = self.f(outputs, labels).sum() + f_loss = self.f(outputs, labels) + + cost = Lx_loss + torch.sum(const * f_loss) + + optimizer.zero_grad() + cost.backward() + optimizer.step() + + # Update adversarial images + pre = torch.argmax(outputs.detach(), 1) + condition_1 = self.compare(pre, labels) + condition_2 = (current_Lx < best_Lx) + # Filter out images that get either correct predictions or non-decreasing loss, + # i.e., only images that are both misclassified and loss-decreasing are left + mask_1_2 = torch.logical_and(condition_1, condition_2) + best_Lx[mask_1_2] = current_Lx[mask_1_2] + best_score[mask_1_2] = pre[mask_1_2] + + condition_3 = (current_Lx < o_best_Lx) + o_mask = torch.logical_and(condition_1, condition_3) + o_best_Lx[o_mask] = current_Lx[o_mask] + o_best_score[o_mask] = pre[o_mask] + + o_best_adv_images[o_mask] = adv_images[o_mask] + + # Check if we should abort search if we're getting nowhere. + if self.abort_early and step % (self.steps // 10) == 0: + if cost > prev_cost * 0.9999: + break + else: + prev_cost = cost + + # Adjust the constant as needed + outputs = self.get_logits(adv_images) + pre = torch.argmax(outputs, 1) + + condition_1 = self.compare(pre, labels) + condition_2 = (best_score != -1) + condition_3 = upper_bound < 1e9 + + mask_1_2 = torch.logical_and(condition_1, condition_2) + mask_1_2_3 = torch.logical_and(mask_1_2, condition_3) + const_1 = (lower_bound + upper_bound) / 2.0 + + upper_bound_min = torch.min(upper_bound, const) + upper_bound[mask_1_2] = upper_bound_min[mask_1_2] + const[mask_1_2_3] = const_1[mask_1_2_3] + + mask_n1_n2_3 = torch.logical_and(~mask_1_2, condition_3) + upper_bound_max = torch.max(lower_bound, const) + upper_bound[~mask_1_2] = upper_bound_max[~mask_1_2] + const[mask_n1_n2_3] *= 10 + + # print(o_best_Lx) + return o_best_adv_images + + def compare(self, predition, labels): + if self.targeted: + # We want to let pre == target_labels in a targeted attack + ret = (predition == labels) + else: + # If the attack is not targeted we simply make these two values unequal + ret = (predition != labels) + return ret + + def tanh_space(self, x): + return 1 / 2 * (torch.tanh(x) + 1) + + def inverse_tanh_space(self, x): + # torch.atanh is only for torch >= 1.7.0 + # atanh is defined in the range -1 to 1 + return self.atanh(torch.clamp(x * 2 - 1, min=-1, max=1)) + + def atanh(self, x): + return 0.5 * torch.log((1 + x) / (1 - x)) + + # f-function in the paper + def f(self, outputs, labels): + one_hot_labels = torch.eye(outputs.shape[1]).to(self.device)[labels] + + # get the target class's logit + real = torch.sum(one_hot_labels * outputs, dim=1) + # find the max logit other than the target classs + other = torch.max((1 - one_hot_labels) * outputs - one_hot_labels * 1e12, dim=1)[0] # nopep8 + + if self.targeted: + return torch.clamp((other - real), min=-self.kappa) + else: + return torch.clamp((real - other), min=-self.kappa) diff --git a/torchattacks/attacks/cwl0.py b/torchattacks/attacks/cwl0.py new file mode 100644 index 00000000..b031baf4 --- /dev/null +++ b/torchattacks/attacks/cwl0.py @@ -0,0 +1,150 @@ +import torch +import torch.nn as nn +import torch.optim as optim + +from ..attack import Attack + + +class CWL0(Attack): + r""" + CW in the paper 'Towards Evaluating the Robustness of Neural Networks' + [https://arxiv.org/abs/1608.04644] + + Distance Measure : L0 + + Arguments: + model (nn.Module): model to attack. + c (float): c in the paper. parameter for box-constraint. (Default: 1) + :math:`minimize \Vert\frac{1}{2}(tanh(w)+1)-x\Vert^2_2+c\cdot f(\frac{1}{2}(tanh(w)+1))` + kappa (float): kappa (also written as 'confidence') in the paper. (Default: 0) + :math:`f(x')=max(max\{Z(x')_i:i\neq t\} -Z(x')_t, - \kappa)` + steps (int): number of steps (also written as 'max_iterations'). (Default: 50) + lr (float): learning rate of the Adam optimizer. (Default: 0.01) + abort_early: if true, allows early aborts if gradient descent gets stuck. (Default: True) + + .. warning:: With default c, you can't easily get adversarial images. Set higher c like 1. + + Shape: + - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. + - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. + - output: :math:`(N, C, H, W)`. + + Examples:: + >>> attack = torchattacks.CWL0(model, c=1, kappa=0, steps=50, lr=0.01, abort_early=True) + >>> adv_images = attack(images, labels) + + .. note:: The binary search version of the CW algorithm has been implemented as CWBS. + + """ + + def __init__(self, model, c=1, kappa=0, steps=50, lr=0.01, abort_early=True): + super().__init__("CWL0", model) + self.c = c + self.kappa = kappa + self.steps = steps + self.lr = lr + self.abort_early = abort_early + self.supported_mode = ["default", "targeted"] + + def forward(self, images, labels): + r""" + Overridden. + """ + + images = images.clone().detach().to(self.device) + labels = labels.clone().detach().to(self.device) + + if self.targeted: + labels = self.get_target_label(images, labels) + + # w = torch.zeros_like(images).detach() # Requires 2x times + w = self.inverse_tanh_space(images).detach() + w.requires_grad = True + + best_adv_images = images.clone().detach() + batch_size = len(images) + best_Lx = torch.full((batch_size, ), 1e10).to(self.device) + prev_cost = 1e10 + threshold = 1e-6 + + optimizer = optim.Adam([w], lr=self.lr) + + for step in range(self.steps): + # Get adversarial images + adv_images = self.tanh_space(w) + + # Calculate loss + l0_abs = torch.abs(adv_images.reshape(-1) - images.reshape(-1)) + l0_norm = (l0_abs > threshold) + # Number of non-zero values + l0_loss = (1.0 / l0_abs.shape[0]) * torch.sum(l0_norm).item() + current_Lx = torch.full((batch_size, ), l0_loss).to(self.device) + + Lx_loss = current_Lx.sum() + + outputs = self.get_logits(adv_images) + if self.targeted: + f_loss = self.f(outputs, labels) + else: + f_loss = self.f(outputs, labels) + + cost = Lx_loss + torch.sum(self.c * f_loss) + + optimizer.zero_grad() + cost.backward() + optimizer.step() + + # Update adversarial images + pre = torch.argmax(outputs.detach(), 1) + condition_1 = self.compare(pre, labels) + condition_2 = (current_Lx < best_Lx) + + # Filter out images that get either correct predictions or non-decreasing loss, + # i.e., only images that are both misclassified and loss-decreasing are left + mask = torch.logical_and(condition_1, condition_2) + best_Lx[mask] = current_Lx[mask] + best_adv_images[mask] = adv_images[mask] + + # Early stop when loss does not converge + if self.abort_early and step % (self.steps // 10) == 0: + if cost > prev_cost * 0.9999: + break + else: + prev_cost = cost + + # print(best_Lx) + return best_adv_images + + def compare(self, predition, labels): + if self.targeted: + # We want to let pre == target_labels in a targeted attack + ret = (predition == labels) + else: + # If the attack is not targeted we simply make these two values unequal + ret = (predition != labels) + return ret + + def tanh_space(self, x): + return 1 / 2 * (torch.tanh(x) + 1) + + def inverse_tanh_space(self, x): + # torch.atanh is only for torch >= 1.7.0 + # atanh is defined in the range -1 to 1 + return self.atanh(torch.clamp(x * 2 - 1, min=-1, max=1)) + + def atanh(self, x): + return 0.5 * torch.log((1 + x) / (1 - x)) + + # f-function in the paper + def f(self, outputs, labels): + one_hot_labels = torch.eye(outputs.shape[1]).to(self.device)[labels] + + # get the target class's logit + real = torch.sum(one_hot_labels * outputs, dim=1) + # find the max logit other than the target class + other = torch.max((1 - one_hot_labels) * outputs - one_hot_labels * 1e12, dim=1)[0] # nopep8 + + if self.targeted: + return torch.clamp((other - real), min=-self.kappa) + else: + return torch.clamp((real - other), min=-self.kappa) diff --git a/torchattacks/attacks/cwlinf.py b/torchattacks/attacks/cwlinf.py new file mode 100644 index 00000000..51d84177 --- /dev/null +++ b/torchattacks/attacks/cwlinf.py @@ -0,0 +1,147 @@ +import torch +import torch.nn as nn +import torch.optim as optim + +from ..attack import Attack + + +class CWLinf(Attack): + r""" + CW in the paper 'Towards Evaluating the Robustness of Neural Networks' + [https://arxiv.org/abs/1608.04644] + + Distance Measure : Linf + + Arguments: + model (nn.Module): model to attack. + c (float): c in the paper. parameter for box-constraint. (Default: 1) + :math:`minimize \Vert\frac{1}{2}(tanh(w)+1)-x\Vert^2_2+c\cdot f(\frac{1}{2}(tanh(w)+1))` + kappa (float): kappa (also written as 'confidence') in the paper. (Default: 0) + :math:`f(x')=max(max\{Z(x')_i:i\neq t\} -Z(x')_t, - \kappa)` + steps (int): number of steps (also written as 'max_iterations'). (Default: 50) + lr (float): learning rate of the Adam optimizer. (Default: 0.01) + abort_early: if true, allows early aborts if gradient descent gets stuck. (Default: True) + + .. warning:: With default c, you can't easily get adversarial images. Set higher c like 1. + + Shape: + - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. + - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. + - output: :math:`(N, C, H, W)`. + + Examples:: + >>> attack = torchattacks.CWLinf(model, c=1, kappa=0, steps=50, lr=0.01, abort_early=True) + >>> adv_images = attack(images, labels) + + .. note:: The binary search version of the CW algorithm has been implemented as CWBS. + + """ + + def __init__(self, model, c=1, kappa=0, steps=50, lr=0.01, abort_early=True): + super().__init__("CWLinf", model) + self.c = c + self.kappa = kappa + self.steps = steps + self.lr = lr + self.abort_early = abort_early + self.supported_mode = ["default", "targeted"] + + def forward(self, images, labels): + r""" + Overridden. + """ + + images = images.clone().detach().to(self.device) + labels = labels.clone().detach().to(self.device) + + if self.targeted: + labels = self.get_target_label(images, labels) + + # w = torch.zeros_like(images).detach() # Requires 2x times + w = self.inverse_tanh_space(images).detach() + w.requires_grad = True + + best_adv_images = images.clone().detach() + batch_size = len(images) + best_Lx = torch.full((batch_size, ), 1e10).to(self.device) + prev_cost = 1e10 + + optimizer = optim.Adam([w], lr=self.lr) + + for step in range(self.steps): + # Get adversarial images + adv_images = self.tanh_space(w) + + # Calculate loss + linf_norm = torch.max(torch.abs(adv_images - images)) + linf_loss = (1.0 / batch_size) * linf_norm.item() + current_Lx = torch.full((batch_size, ), linf_loss).to(self.device) + + Lx_loss = current_Lx.sum() + + outputs = self.get_logits(adv_images) + if self.targeted: + f_loss = self.f(outputs, labels) + else: + f_loss = self.f(outputs, labels) + + cost = Lx_loss + torch.sum(self.c * f_loss) + + optimizer.zero_grad() + cost.backward() + optimizer.step() + + # Update adversarial images + pre = torch.argmax(outputs.detach(), 1) + condition_1 = self.compare(pre, labels) + condition_2 = (current_Lx < best_Lx) + + # Filter out images that get either correct predictions or non-decreasing loss, + # i.e., only images that are both misclassified and loss-decreasing are left + mask = torch.logical_and(condition_1, condition_2) + best_Lx[mask] = current_Lx[mask] + best_adv_images[mask] = adv_images[mask] + + # Early stop when loss does not converge + if self.abort_early and step % (self.steps // 10) == 0: + if cost > prev_cost * 0.9999: + break + else: + prev_cost = cost + + # print(best_Lx) + return best_adv_images + + def compare(self, predition, labels): + if self.targeted: + # We want to let pre == target_labels in a targeted attack + ret = (predition == labels) + else: + # If the attack is not targeted we simply make these two values unequal + ret = (predition != labels) + return ret + + def tanh_space(self, x): + return 1 / 2 * (torch.tanh(x) + 1) + + def inverse_tanh_space(self, x): + # torch.atanh is only for torch >= 1.7.0 + # atanh is defined in the range -1 to 1 + return self.atanh(torch.clamp(x * 2 - 1, min=-1, max=1)) + + def atanh(self, x): + return 0.5 * torch.log((1 + x) / (1 - x)) + + # f-function in the paper + def f(self, outputs, labels): + one_hot_labels = torch.eye(outputs.shape[1]).to(self.device)[labels] + + # get the target class's logit + real = torch.sum(one_hot_labels * outputs, dim=1) + # find the max logit other than the target class + other = torch.max((1 - one_hot_labels) * outputs - one_hot_labels * 1e12, dim=1)[0] # nopep8 + + if self.targeted: + return torch.clamp((other - real), min=-self.kappa) + else: + return torch.clamp((real - other), min=-self.kappa) diff --git a/torchattacks/attacks/eaden.py b/torchattacks/attacks/eaden.py index 3deb51af..4ef70ae2 100644 --- a/torchattacks/attacks/eaden.py +++ b/torchattacks/attacks/eaden.py @@ -13,13 +13,13 @@ class EADEN(Attack): Arguments: model (nn.Module): model to attack. + init_c(float): the initial constant c to pick as a first guess. (Default: 1) kappa (float): how strong the adversarial example should be (also written as 'confidence'). (Default: 0) + beta (float): hyperparameter trading off L2 minimization for L1 minimization. (Default: 0.001) + steps (int): number of iterations to perform gradient descent. (Default: 10) lr (float): larger values converge faster to less accurate results. (Default: 0.01) binary_search_steps (int): number of times to adjust the constant with binary search. (Default: 9) - max_iterations (int): number of iterations to perform gradient descent. (Default: 100) abort_early (bool): if we stop improving, abort gradient descent early. (Default: True) - initial_const (float): the initial constant c to pick as a first guess. (Default: 0.001) - beta (float): hyperparameter trading off L2 minimization for L1 minimization. (Default: 0.001) Shape: - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. @@ -27,7 +27,7 @@ class EADEN(Attack): - output: :math:`(N, C, H, W)`. Examples:: - >>> attack = torchattacks.EADEN(model, kappa=0, lr=0.01, max_iterations=100) + >>> attack = torchattacks.EADEN(model, init_c=1, kappa=0, steps=10, lr=0.01) >>> adv_images = attack(images, labels) """ @@ -35,22 +35,22 @@ class EADEN(Attack): def __init__( self, model, + init_c=1, kappa=0, + beta=0.001, + steps=10, lr=0.01, binary_search_steps=9, - max_iterations=100, abort_early=True, - initial_const=0.001, - beta=0.001, ): super().__init__("EADEN", model) + self.init_c = init_c self.kappa = kappa + self.beta = beta + self.steps = steps self.lr = lr self.binary_search_steps = binary_search_steps - self.max_iterations = max_iterations self.abort_early = abort_early - self.initial_const = initial_const - self.beta = beta # The last iteration (if we run many steps) repeat the search once. self.repeat = binary_search_steps >= 10 self.supported_mode = ["default", "targeted"] @@ -69,120 +69,137 @@ def forward(self, images, labels): outputs = self.get_logits(images) batch_size = images.shape[0] - lower_bound = torch.zeros(batch_size, device=self.device) - const = torch.ones(batch_size, device=self.device) * self.initial_const - upper_bound = torch.ones(batch_size, device=self.device) * 1e10 + lower_bound = torch.zeros((batch_size, )).to(self.device) + const = torch.ones(batch_size).to(self.device) * self.init_c + upper_bound = torch.ones(batch_size).to(self.device) * 1e10 - final_adv_images = images.clone() - y_one_hot = torch.eye(outputs.shape[1]).to(self.device)[labels] - - o_bestl1 = [1e10] * batch_size - o_bestscore = [-1] * batch_size - o_bestl1 = torch.Tensor(o_bestl1).to(self.device) - o_bestscore = torch.Tensor(o_bestscore).to(self.device) + o_best_adv_images = images.clone() + o_best_score = torch.full( + (batch_size, ), -1, dtype=torch.long).to(self.device) + o_best_EN = torch.full((batch_size, ), 1e10).to(self.device) # Initialization: x^{(0)} = y^{(0)} = x_0 in paper Algorithm 1 part x_k = images.clone().detach() y_k = nn.Parameter(images) # Start binary search for outer_step in range(self.binary_search_steps): - self.global_step = 0 - - bestl1 = [1e10] * batch_size - bestscore = [-1] * batch_size - - bestl1 = torch.Tensor(bestl1).to(self.device) - bestscore = torch.Tensor(bestscore).to(self.device) - prevloss = 1e6 + best_score = torch.full( + (batch_size, ), -1, dtype=torch.long).to(self.device) + best_EN = torch.full((batch_size, ), 1e10).to(self.device) if self.repeat and outer_step == (self.binary_search_steps - 1): const = upper_bound lr = self.lr - for iteration in range(self.max_iterations): + prev_cost = 1e6 + for step in range(self.steps): # reset gradient if y_k.grad is not None: y_k.grad.detach_() y_k.grad.zero_() # Loss over images_parameters with only L2 same as CW - # we don't update L1 loss with SGD because we use ISTA - output = self.get_logits(y_k) + # We don't update L1 loss with SGD because we use ISTA + outputs = self.get_logits(y_k) L2_loss = self.L2_loss(y_k, images) - cost = self.EAD_loss(output, y_one_hot, None, L2_loss, const) - # cost.backward(retain_graph=True) + cost = self.EAD_loss(outputs, labels, None, L2_loss, const) cost.backward() # Gradient step - # y_k.data.add_(-lr, y_k.grad.data) self.global_step += 1 with torch.no_grad(): y_k -= y_k.grad * lr # Ploynomial decay of learning rate - lr = ( - self.lr * (1 - self.global_step / self.max_iterations) ** 0.5 - ) # nopep8 + lr = self.lr * (1 - self.global_step / self.steps) ** 0.5 x_k, y_k = self.FISTA(images, x_k, y_k) # Loss ElasticNet or L1 over x_k with torch.no_grad(): - output = self.get_logits(x_k) + outputs = self.get_logits(x_k) L2_loss = self.L2_loss(x_k, images) L1_loss = self.L1_loss(x_k, images) - loss = self.EAD_loss( - output, y_one_hot, L1_loss, L2_loss, const - ) # nopep8 + cost = self.EAD_loss( + outputs, labels, L1_loss, L2_loss, const) + + # EN attack key step! + current_Lx = L2_loss + (self.beta * L1_loss) + + # Update adversarial images + pre = torch.argmax(outputs.detach(), 1) + condition_1 = self.compare(pre, labels) + condition_2 = (current_Lx < best_EN) + # Filter out images that get either correct predictions or non-decreasing loss, + # i.e., only images that are both misclassified and loss-decreasing are left + mask_1_2 = torch.logical_and(condition_1, condition_2) + best_EN[mask_1_2] = current_Lx[mask_1_2] + best_score[mask_1_2] = pre[mask_1_2] + + condition_3 = (current_Lx < o_best_EN) + o_mask = torch.logical_and(condition_1, condition_3) + o_best_EN[o_mask] = current_Lx[o_mask] + o_best_score[o_mask] = pre[o_mask] + + o_best_adv_images[o_mask] = x_k[o_mask] # print('loss: {}, prevloss: {}'.format(loss, prevloss)) - if ( - self.abort_early - and iteration % (self.max_iterations // 10) == 0 - ): - if loss > prevloss * 0.999999: + if self.abort_early and step % (self.steps // 10) == 0: + if cost > prev_cost * 0.9999: break - prevloss = loss + prev_cost = cost - # EN attack key step! - cost = L2_loss + (L1_loss * self.beta) - self.adjust_best_result( - x_k, - labels, - output, - cost, - bestl1, - bestscore, - o_bestl1, - o_bestscore, - final_adv_images, - ) - - self.adjust_constant(labels, bestscore, const, upper_bound, lower_bound) - - return final_adv_images + # Adjust the constant as needed + outputs = self.get_logits(x_k) + pre = torch.argmax(outputs, 1) + + condition_1 = self.compare(pre, labels) + condition_2 = (best_score != -1) + condition_3 = upper_bound < 1e9 + + mask_1_2 = torch.logical_and(condition_1, condition_2) + mask_1_2_3 = torch.logical_and(mask_1_2, condition_3) + const_1 = (lower_bound + upper_bound) / 2.0 + + upper_bound_min = torch.min(upper_bound, const) + upper_bound[mask_1_2] = upper_bound_min[mask_1_2] + const[mask_1_2_3] = const_1[mask_1_2_3] + + mask_n1_n2_3 = torch.logical_and(~mask_1_2, condition_3) + upper_bound_max = torch.max(lower_bound, const) + upper_bound[~mask_1_2] = upper_bound_max[~mask_1_2] + const[mask_n1_n2_3] *= 10 + + return o_best_adv_images + + def compare(self, predition, labels): + if self.targeted: + # We want to let pre == target_labels in a targeted attack + ret = (predition == labels) + else: + # If the attack is not targeted we simply make these two values unequal + ret = (predition != labels) + return ret def L1_loss(self, x1, x2): Flatten = nn.Flatten() L1_loss = torch.abs(Flatten(x1) - Flatten(x2)).sum(dim=1) - # L1_loss = L1.sum() return L1_loss def L2_loss(self, x1, x2): MSELoss = nn.MSELoss(reduction="none") Flatten = nn.Flatten() L2_loss = MSELoss(Flatten(x1), Flatten(x2)).sum(dim=1) - # L2_loss = L2.sum() return L2_loss - def EAD_loss(self, output, one_hot_labels, L1_loss, L2_loss, const): + def EAD_loss(self, outputs, labels, L1_loss, L2_loss, const): + one_hot_labels = torch.eye(outputs.shape[1]).to(self.device)[labels] - # Not same as CW's f function - other = torch.max( - (1 - one_hot_labels) * output - (one_hot_labels * 1e4), dim=1 - )[0] - real = torch.max(one_hot_labels * output, dim=1)[0] + # get the target class's logit + real = torch.sum(one_hot_labels * outputs, dim=1) + # find the max logit other than the target classs + other = torch.max((1 - one_hot_labels) * outputs - (one_hot_labels * 1e12), dim=1)[0] # nopep8 if self.targeted: F_loss = torch.clamp((other - real), min=-self.kappa) @@ -192,11 +209,7 @@ def EAD_loss(self, output, one_hot_labels, L1_loss, L2_loss, const): if isinstance(L1_loss, type(None)): loss = torch.sum(const * F_loss) + torch.sum(L2_loss) else: - loss = ( - torch.sum(const * F_loss) - + torch.sum(L2_loss) - + torch.sum(self.beta * L1_loss) - ) + loss = torch.sum(const * F_loss) + torch.sum(L2_loss) + torch.sum(self.beta * L1_loss) # nopep8 return loss @@ -208,59 +221,10 @@ def FISTA(self, images, x_k, y_k): lower = torch.clamp(y_k + self.beta, min=0) diff = y_k - images - cond1 = (diff > self.beta).float() - cond2 = (torch.abs(diff) <= self.beta).float() - cond3 = (diff < -self.beta).float() + c1 = diff > self.beta + c2 = torch.abs(diff) <= self.beta + c3 = diff < -self.beta - new_x_k = (cond1 * upper) + (cond2 * images) + (cond3 * lower) + new_x_k = (c1.float() * upper) + (c2.float() * images) + (c3.float() * lower) # nopep8 y_k.data = new_x_k + (zt * (new_x_k - x_k)) return new_x_k, y_k - - def compare(self, output, labels): - if len(output.shape) >= 2: - # output is tensor - output = output.clone().detach() - if self.targeted: - output[:, labels] -= self.kappa - else: - output[:, labels] += self.kappa - output = torch.argmax(output, 1) - else: - # output is int or float - pass - - if self.targeted: - return output == labels - else: - return output != labels - - def adjust_best_result( - self, - adv_img, - labels, - output, - cost, - bestl1, - bestscore, - o_bestl1, - o_bestscore, - final_adv_images, - ): - output_label = torch.argmax(output, 1).float() - mask = (cost < bestl1) & self.compare(output, labels) - bestl1[mask] = cost[mask] - bestscore[mask] = output_label[mask] - - mask = (cost < o_bestl1) & self.compare(output, labels) - o_bestl1[mask] = cost[mask] - o_bestscore[mask] = output_label[mask] - final_adv_images[mask] = adv_img[mask] - - def adjust_constant(self, labels, bestscore, const, upper_bound, lower_bound): - mask = (self.compare(bestscore, labels)) & (bestscore != -1) - upper_bound[mask] = torch.min(upper_bound[mask], const[mask]) - lower_bound[~mask] = torch.max(lower_bound[~mask], const[~mask]) # nopep8 - - mask = upper_bound < 1e9 - const[mask] = (lower_bound[mask] + upper_bound[mask]) / 2 - const[~mask] = const[~mask] * 10 diff --git a/torchattacks/attacks/eadl1.py b/torchattacks/attacks/eadl1.py index 0d6b1520..cdd7bbcf 100644 --- a/torchattacks/attacks/eadl1.py +++ b/torchattacks/attacks/eadl1.py @@ -13,13 +13,13 @@ class EADL1(Attack): Arguments: model (nn.Module): model to attack. + init_c(float): the initial constant c to pick as a first guess. (Default: 1) kappa (float): how strong the adversarial example should be (also written as 'confidence'). (Default: 0) + beta (float): hyperparameter trading off L2 minimization for L1 minimization. (Default: 0.001) + steps (int): number of iterations to perform gradient descent. (Default: 10) lr (float): larger values converge faster to less accurate results. (Default: 0.01) binary_search_steps (int): number of times to adjust the constant with binary search. (Default: 9) - max_iterations (int): number of iterations to perform gradient descent. (Default: 100) abort_early (bool): if we stop improving, abort gradient descent early. (Default: True) - initial_const (float): the initial constant c to pick as a first guess. (Default: 0.001) - beta (float): hyperparameter trading off L2 minimization for L1 minimization. (Default: 0.001) Shape: - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. @@ -27,7 +27,7 @@ class EADL1(Attack): - output: :math:`(N, C, H, W)`. Examples:: - >>> attack = torchattacks.EADL1(model, kappa=0, lr=0.01, max_iterations=100) + >>> attack = torchattacks.EADL1(model, init_c=1, kappa=0, steps=10, lr=0.01) >>> adv_images = attack(images, labels) """ @@ -35,22 +35,22 @@ class EADL1(Attack): def __init__( self, model, + init_c=1, kappa=0, + beta=0.001, + steps=10, lr=0.01, binary_search_steps=9, - max_iterations=100, abort_early=True, - initial_const=0.001, - beta=0.001, ): super().__init__("EADL1", model) + self.init_c = init_c self.kappa = kappa + self.beta = beta + self.steps = steps self.lr = lr self.binary_search_steps = binary_search_steps - self.max_iterations = max_iterations self.abort_early = abort_early - self.initial_const = initial_const - self.beta = beta # The last iteration (if we run many steps) repeat the search once. self.repeat = binary_search_steps >= 10 self.supported_mode = ["default", "targeted"] @@ -69,120 +69,137 @@ def forward(self, images, labels): outputs = self.get_logits(images) batch_size = images.shape[0] - lower_bound = torch.zeros(batch_size, device=self.device) - const = torch.ones(batch_size, device=self.device) * self.initial_const - upper_bound = torch.ones(batch_size, device=self.device) * 1e10 + lower_bound = torch.zeros((batch_size, )).to(self.device) + const = torch.ones(batch_size).to(self.device) * self.init_c + upper_bound = torch.ones(batch_size).to(self.device) * 1e10 - final_adv_images = images.clone() - y_one_hot = torch.eye(outputs.shape[1]).to(self.device)[labels] - - o_bestl1 = [1e10] * batch_size - o_bestscore = [-1] * batch_size - o_bestl1 = torch.Tensor(o_bestl1).to(self.device) - o_bestscore = torch.Tensor(o_bestscore).to(self.device) + o_best_adv_images = images.clone() + o_best_score = torch.full( + (batch_size, ), -1, dtype=torch.long).to(self.device) + o_best_L1 = torch.full((batch_size, ), 1e10).to(self.device) # Initialization: x^{(0)} = y^{(0)} = x_0 in paper Algorithm 1 part x_k = images.clone().detach() y_k = nn.Parameter(images) # Start binary search for outer_step in range(self.binary_search_steps): - self.global_step = 0 - - bestl1 = [1e10] * batch_size - bestscore = [-1] * batch_size - - bestl1 = torch.Tensor(bestl1).to(self.device) - bestscore = torch.Tensor(bestscore).to(self.device) - prevloss = 1e6 + best_score = torch.full( + (batch_size, ), -1, dtype=torch.long).to(self.device) + best_L1 = torch.full((batch_size, ), 1e10).to(self.device) if self.repeat and outer_step == (self.binary_search_steps - 1): const = upper_bound lr = self.lr - for iteration in range(self.max_iterations): + prev_cost = 1e6 + for step in range(self.steps): # reset gradient if y_k.grad is not None: y_k.grad.detach_() y_k.grad.zero_() # Loss over images_parameters with only L2 same as CW - # we don't update L1 loss with SGD because we use ISTA - output = self.get_logits(y_k) + # We don't update L1 loss with SGD because we use ISTA + outputs = self.get_logits(y_k) L2_loss = self.L2_loss(y_k, images) - cost = self.EAD_loss(output, y_one_hot, None, L2_loss, const) - # cost.backward(retain_graph=True) + cost = self.EAD_loss(outputs, labels, None, L2_loss, const) cost.backward() # Gradient step - # y_k.data.add_(-lr, y_k.grad.data) self.global_step += 1 with torch.no_grad(): y_k -= y_k.grad * lr # Ploynomial decay of learning rate - lr = ( - self.lr * (1 - self.global_step / self.max_iterations) ** 0.5 - ) # nopep8 + lr = self.lr * (1 - self.global_step / self.steps) ** 0.5 x_k, y_k = self.FISTA(images, x_k, y_k) # Loss ElasticNet or L1 over x_k with torch.no_grad(): - output = self.get_logits(x_k) + outputs = self.get_logits(x_k) L2_loss = self.L2_loss(x_k, images) L1_loss = self.L1_loss(x_k, images) - loss = self.EAD_loss( - output, y_one_hot, L1_loss, L2_loss, const - ) # nopep8 + cost = self.EAD_loss( + outputs, labels, L1_loss, L2_loss, const) + + # L1 attack key step! + current_Lx = L1_loss + + # Update adversarial images + pre = torch.argmax(outputs.detach(), 1) + condition_1 = self.compare(pre, labels) + condition_2 = (current_Lx < best_L1) + # Filter out images that get either correct predictions or non-decreasing loss, + # i.e., only images that are both misclassified and loss-decreasing are left + mask_1_2 = torch.logical_and(condition_1, condition_2) + best_L1[mask_1_2] = current_Lx[mask_1_2] + best_score[mask_1_2] = pre[mask_1_2] + + condition_3 = (current_Lx < o_best_L1) + o_mask = torch.logical_and(condition_1, condition_3) + o_best_L1[o_mask] = current_Lx[o_mask] + o_best_score[o_mask] = pre[o_mask] + + o_best_adv_images[o_mask] = x_k[o_mask] # print('loss: {}, prevloss: {}'.format(loss, prevloss)) - if ( - self.abort_early - and iteration % (self.max_iterations // 10) == 0 - ): - if loss > prevloss * 0.999999: + if self.abort_early and step % (self.steps // 10) == 0: + if cost > prev_cost * 0.9999: break - prevloss = loss + prev_cost = cost - # L1 attack key step! - cost = L1_loss - self.adjust_best_result( - x_k, - labels, - output, - cost, - bestl1, - bestscore, - o_bestl1, - o_bestscore, - final_adv_images, - ) - - self.adjust_constant(labels, bestscore, const, upper_bound, lower_bound) - - return final_adv_images + # Adjust the constant as needed + outputs = self.get_logits(x_k) + pre = torch.argmax(outputs, 1) + + condition_1 = self.compare(pre, labels) + condition_2 = (best_score != -1) + condition_3 = upper_bound < 1e9 + + mask_1_2 = torch.logical_and(condition_1, condition_2) + mask_1_2_3 = torch.logical_and(mask_1_2, condition_3) + const_1 = (lower_bound + upper_bound) / 2.0 + + upper_bound_min = torch.min(upper_bound, const) + upper_bound[mask_1_2] = upper_bound_min[mask_1_2] + const[mask_1_2_3] = const_1[mask_1_2_3] + + mask_n1_n2_3 = torch.logical_and(~mask_1_2, condition_3) + upper_bound_max = torch.max(lower_bound, const) + upper_bound[~mask_1_2] = upper_bound_max[~mask_1_2] + const[mask_n1_n2_3] *= 10 + + return o_best_adv_images + + def compare(self, predition, labels): + if self.targeted: + # We want to let pre == target_labels in a targeted attack + ret = (predition == labels) + else: + # If the attack is not targeted we simply make these two values unequal + ret = (predition != labels) + return ret def L1_loss(self, x1, x2): Flatten = nn.Flatten() L1_loss = torch.abs(Flatten(x1) - Flatten(x2)).sum(dim=1) - # L1_loss = L1.sum() return L1_loss def L2_loss(self, x1, x2): MSELoss = nn.MSELoss(reduction="none") Flatten = nn.Flatten() L2_loss = MSELoss(Flatten(x1), Flatten(x2)).sum(dim=1) - # L2_loss = L2.sum() return L2_loss - def EAD_loss(self, output, one_hot_labels, L1_loss, L2_loss, const): + def EAD_loss(self, outputs, labels, L1_loss, L2_loss, const): + one_hot_labels = torch.eye(outputs.shape[1]).to(self.device)[labels] - # Not same as CW's f function - other = torch.max( - (1 - one_hot_labels) * output - (one_hot_labels * 1e4), dim=1 - )[0] - real = torch.max(one_hot_labels * output, dim=1)[0] + # get the target class's logit + real = torch.sum(one_hot_labels * outputs, dim=1) + # find the max logit other than the target classs + other = torch.max((1 - one_hot_labels) * outputs - (one_hot_labels * 1e12), dim=1)[0] # nopep8 if self.targeted: F_loss = torch.clamp((other - real), min=-self.kappa) @@ -192,11 +209,7 @@ def EAD_loss(self, output, one_hot_labels, L1_loss, L2_loss, const): if isinstance(L1_loss, type(None)): loss = torch.sum(const * F_loss) + torch.sum(L2_loss) else: - loss = ( - torch.sum(const * F_loss) - + torch.sum(L2_loss) - + torch.sum(self.beta * L1_loss) - ) + loss = torch.sum(const * F_loss) + torch.sum(L2_loss) + torch.sum(self.beta * L1_loss) # nopep8 return loss @@ -208,59 +221,10 @@ def FISTA(self, images, x_k, y_k): lower = torch.clamp(y_k + self.beta, min=0) diff = y_k - images - cond1 = (diff > self.beta).float() - cond2 = (torch.abs(diff) <= self.beta).float() - cond3 = (diff < -self.beta).float() + c1 = diff > self.beta + c2 = torch.abs(diff) <= self.beta + c3 = diff < -self.beta - new_x_k = (cond1 * upper) + (cond2 * images) + (cond3 * lower) + new_x_k = (c1.float() * upper) + (c2.float() * images) + (c3.float() * lower) # nopep8 y_k.data = new_x_k + (zt * (new_x_k - x_k)) return new_x_k, y_k - - def compare(self, output, labels): - if len(output.shape) >= 2: - # output is tensor - output = output.clone().detach() - if self.targeted: - output[:, labels] -= self.kappa - else: - output[:, labels] += self.kappa - output = torch.argmax(output, 1) - else: - # output is int or float - pass - - if self.targeted: - return output == labels - else: - return output != labels - - def adjust_best_result( - self, - adv_img, - labels, - output, - cost, - bestl1, - bestscore, - o_bestl1, - o_bestscore, - final_adv_images, - ): - output_label = torch.argmax(output, 1).float() - mask = (cost < bestl1) & self.compare(output, labels) - bestl1[mask] = cost[mask] - bestscore[mask] = output_label[mask] - - mask = (cost < o_bestl1) & self.compare(output, labels) - o_bestl1[mask] = cost[mask] - o_bestscore[mask] = output_label[mask] - final_adv_images[mask] = adv_img[mask] - - def adjust_constant(self, labels, bestscore, const, upper_bound, lower_bound): - mask = (self.compare(bestscore, labels)) & (bestscore != -1) - upper_bound[mask] = torch.min(upper_bound[mask], const[mask]) - lower_bound[~mask] = torch.max(lower_bound[~mask], const[~mask]) # nopep8 - - mask = upper_bound < 1e9 - const[mask] = (lower_bound[mask] + upper_bound[mask]) / 2 - const[~mask] = const[~mask] * 10 diff --git a/torchattacks/attacks/espgd.py b/torchattacks/attacks/espgd.py new file mode 100644 index 00000000..dfa67d74 --- /dev/null +++ b/torchattacks/attacks/espgd.py @@ -0,0 +1,92 @@ +import torch +import torch.nn as nn + +from ..attack import Attack + + +class ESPGD(Attack): + r""" + Early-stopped PGD in the paper 'Attacks Which Do Not Kill Training Make Adversarial Learning Stronger' + [https://arxiv.org/abs/2002.11242] + + Distance Measure : Linf + + Arguments: + model (nn.Module): model to attack. + eps (float): maximum perturbation. (Default: 8/255) + alpha (float): step size. (Default: 2/255) + steps (int): number of steps. (Default: 10) + tau (int): the step controlling how early we should stop interations when wrong adv data is found. (Default: 3) + random_start (bool): using random initialization of delta. (Default: True) + + Shape: + - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. + - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. + - output: :math:`(N, C, H, W)`. + + Examples:: + >>> attack = torchattacks.ESPGD(model, eps=8/255, alpha=1/255, steps=10, tau=10, random_start=True) + >>> adv_images = attack(images, labels) + + """ + + def __init__(self, model, eps=8 / 255, alpha=2 / 255, steps=10, tau=3, random_start=True): + super().__init__("PGD", model) + self.eps = eps + self.alpha = alpha + self.steps = steps + self.tau = tau + self.random_start = random_start + self.supported_mode = ["default", "targeted"] + + def forward(self, images, labels): + r""" + Overridden. + """ + + images = images.clone().detach().to(self.device) + labels = labels.clone().detach().to(self.device) + + if self.targeted: + target_labels = self.get_target_label(images, labels) + + loss = nn.CrossEntropyLoss() + control = torch.ones(labels.shape[0]).to(self.device) * self.tau + adv_images = images.clone().detach() + output_images = images.clone().detach() + + if self.random_start: + # Starting at a uniformly random point + adv_images = adv_images + torch.empty_like(adv_images).uniform_( + -self.eps, self.eps + ) + adv_images = torch.clamp(adv_images, min=0, max=1).detach() + + for _ in range(self.steps): + adv_images.requires_grad = True + outputs = self.get_logits(adv_images) + + pred = torch.argmax(outputs, dim=1) + ind_1 = control == 0 + ind_2 = pred != labels + ind_all = torch.logical_and(ind_1, ind_2) + output_images[ind_all] = adv_images[ind_all] + control[ind_2] = control[ind_2] - 1 + + # Calculate loss + if self.targeted: + cost = -loss(outputs, target_labels) + else: + cost = loss(outputs, labels) + + # Update adversarial images + grad = torch.autograd.grad( + cost, adv_images, retain_graph=False, create_graph=False + )[0] + + adv_images = adv_images.detach() + self.alpha * grad.sign() + delta = torch.clamp(adv_images - images, + min=-self.eps, max=self.eps) + adv_images = torch.clamp(images + delta, min=0, max=1).detach() + + return output_images diff --git a/torchattacks/attacks/fab.py b/torchattacks/attacks/fab.py index 02b39984..ea17603b 100644 --- a/torchattacks/attacks/fab.py +++ b/torchattacks/attacks/fab.py @@ -1,43 +1,26 @@ -from __future__ import absolute_import -from __future__ import division -from __future__ import print_function -from __future__ import unicode_literals - -import time -import math - import torch -import torch.nn.functional as F - -# zero_gradients deprecated in torch >= 1.9. -# zero_gradients is re-defined in the bottom of the code. -# from torch.autograd.gradcheck import zero_gradients -from collections import abc as container_abcs +import torch.distributions.uniform as uniform from ..attack import Attack class FAB(Attack): r""" - Fast Adaptive Boundary Attack in the paper 'Minimally distorted Adversarial Examples with a Fast Adaptive Boundary Attack' + Fast Adaptive Boundary Attack (FAB) in the paper 'Minimally distorted Adversarial Examples with a Fast Adaptive Boundary Attack' [https://arxiv.org/abs/1907.02044] - [https://github.com/fra31/auto-attack] + [https://github.com/fra31/fab-attack] - Distance Measure : Linf, L2, L1 + Distance Measure : Linf Arguments: model (nn.Module): model to attack. - norm (str) : Lp-norm to minimize. ['Linf', 'L2', 'L1'] (Default: 'Linf') eps (float): maximum perturbation. (Default: 8/255) - steps (int): number of steps. (Default: 10) n_restarts (int): number of random restarts. (Default: 1) + n_iter (int): number of steps. (Default: 10) alpha_max (float): alpha_max. (Default: 0.1) eta (float): overshooting. (Default: 1.05) beta (float): backward step. (Default: 0.9) - verbose (bool): print progress. (Default: False) - seed (int): random seed for the starting point. (Default: 0) - targeted (bool): targeted attack for every wrong classes. (Default: False) - n_classes (int): number of classes. (Default: 10) + las (bool): final search. (Default: False) Shape: - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. @@ -45,40 +28,20 @@ class FAB(Attack): - output: :math:`(N, C, H, W)`. Examples:: - >>> attack = torchattacks.FAB(model, norm='Linf', steps=10, eps=8/255, n_restarts=1, alpha_max=0.1, eta=1.05, beta=0.9, loss_fn=None, verbose=False, seed=0, targeted=False, n_classes=10) + >>> attack = torchattacks.FAB(model, eps=8/255, n_restarts=1, n_iter=10, alpha_max=0.1, eta=1.05, beta=0.9, las=False) >>> adv_images = attack(images, labels) """ - def __init__( - self, - model, - norm="Linf", - eps=8 / 255, - steps=10, - n_restarts=1, - alpha_max=0.1, - eta=1.05, - beta=0.9, - verbose=False, - seed=0, - multi_targeted=False, - n_classes=10, - ): + def __init__(self, model, eps=8/255, n_restarts=1, n_iter=10, alpha_max=0.1, eta=1.05, beta=0.9, las=False): super().__init__("FAB", model) - self.norm = norm + self.eps = eps self.n_restarts = n_restarts - Default_EPS_DICT_BY_NORM = {"Linf": 0.3, "L2": 1.0, "L1": 5.0} - self.eps = eps if eps is not None else Default_EPS_DICT_BY_NORM[norm] + self.n_iter = n_iter self.alpha_max = alpha_max self.eta = eta self.beta = beta - self.steps = steps - self.verbose = verbose - self.seed = seed - self.target_class = None - self.multi_targeted = multi_targeted - self.n_target_classes = n_classes - 1 + self.las = las self.supported_mode = ["default", "targeted"] def forward(self, images, labels): @@ -92,818 +55,204 @@ def forward(self, images, labels): return adv_images - def _get_predicted_label(self, x): - with torch.no_grad(): - outputs = self.get_logits(x) - _, y = torch.max(outputs, dim=1) - return y - - def check_shape(self, x): - return x if len(x.shape) > 0 else x.unsqueeze(0) - - def get_diff_logits_grads_batch(self, imgs, la): - im = imgs.clone().requires_grad_() - with torch.enable_grad(): - y = self.get_logits(im) - - g2 = torch.zeros([y.shape[-1], *imgs.size()]).to(self.device) - grad_mask = torch.zeros_like(y) - for counter in range(y.shape[-1]): - zero_gradients(im) - grad_mask[:, counter] = 1.0 - y.backward(grad_mask, retain_graph=True) - grad_mask[:, counter] = 0.0 - g2[counter] = im.grad.data - - g2 = torch.transpose(g2, 0, 1).detach() - # y2 = self.get_logits(imgs).detach() - y2 = y.detach() - df = y2 - y2[torch.arange(imgs.shape[0]), la].unsqueeze(1) - dg = g2 - g2[torch.arange(imgs.shape[0]), la].unsqueeze(1) - df[torch.arange(imgs.shape[0]), la] = 1e10 - - return df, dg - - def get_diff_logits_grads_batch_targeted(self, imgs, la, la_target): - u = torch.arange(imgs.shape[0]) - im = imgs.clone().requires_grad_() - with torch.enable_grad(): - y = self.get_logits(im) - diffy = -(y[u, la] - y[u, la_target]) - sumdiffy = diffy.sum() - - zero_gradients(im) - sumdiffy.backward() - graddiffy = im.grad.data - df = diffy.detach().unsqueeze(1) - dg = graddiffy.unsqueeze(1) - - return df, dg - - def attack_single_run(self, x, y=None, use_rand_start=False): - """ - :param x: clean images - :param y: clean labels, if None we use the predicted labels - """ - - # self.device = x.device - self.orig_dim = list(x.shape[1:]) - self.ndims = len(self.orig_dim) - - x = x.detach().clone().float().to(self.device) - # assert next(self.model.parameters()).device == x.device - - y_pred = self._get_predicted_label(x) - if y is None: - y = y_pred.detach().clone().long().to(self.device) - else: - y = y.detach().clone().long().to(self.device) - pred = y_pred == y - corr_classified = pred.float().sum() - if self.verbose: - print("Clean accuracy: {:.2%}".format(pred.float().mean())) - if pred.sum() == 0: - return x - pred = self.check_shape(pred.nonzero().squeeze()) - - startt = time.time() - # runs the attack only on correctly classified points - im2 = x[pred].detach().clone() - la2 = y[pred].detach().clone() - if len(im2.shape) == self.ndims: - im2 = im2.unsqueeze(0) - bs = im2.shape[0] + def perturb(self, images, labels): + logits = self.get_logits(images) + pred = torch.argmax(logits, 1) == labels + pred1 = pred.clone() + im2 = images[pred].clone() + la2 = labels[pred].clone() + bs = torch.sum(pred) u1 = torch.arange(bs) - adv = im2.clone() - adv_c = x.clone() - res2 = 1e10 * torch.ones([bs]).to(self.device) - res_c = torch.zeros([x.shape[0]]).to(self.device) - x1 = im2.clone() - x0 = im2.clone().reshape([bs, -1]) - counter_restarts = 0 - - while counter_restarts < 1: - if use_rand_start: - if self.norm == "Linf": - t = 2 * torch.rand(x1.shape).to(self.device) - 1 - x1 = ( - im2 - + ( - torch.min( - res2, self.eps * torch.ones(res2.shape).to(self.device) - ).reshape([-1, *[1] * self.ndims]) - ) - * t - / ( - t.reshape([t.shape[0], -1]) - .abs() - .max(dim=1, keepdim=True)[0] - .reshape([-1, *[1] * self.ndims]) - ) - * 0.5 - ) - elif self.norm == "L2": - t = torch.randn(x1.shape).to(self.device) - x1 = ( - im2 - + ( - torch.min( - res2, self.eps * torch.ones(res2.shape).to(self.device) - ).reshape([-1, *[1] * self.ndims]) - ) - * t - / ( - (t ** 2) - .view(t.shape[0], -1) - .sum(dim=-1) - .sqrt() - .view(t.shape[0], *[1] * self.ndims) - ) - * 0.5 - ) - elif self.norm == "L1": - t = torch.randn(x1.shape).to(self.device) - x1 = ( - im2 - + ( - torch.min( - res2, self.eps * torch.ones(res2.shape).to(self.device) - ).reshape([-1, *[1] * self.ndims]) - ) - * t - / ( - t.abs() - .view(t.shape[0], -1) - .sum(dim=-1) - .view(t.shape[0], *[1] * self.ndims) - ) - / 2 - ) - - x1 = x1.clamp(0.0, 1.0) - - counter_iter = 0 - while counter_iter < self.steps: - with torch.no_grad(): - df, dg = self.get_diff_logits_grads_batch(x1, la2) - if self.norm == "Linf": - dist1 = df.abs() / ( - 1e-12 - + dg.abs().view(dg.shape[0], dg.shape[1], -1).sum(dim=-1) - ) - elif self.norm == "L2": - dist1 = df.abs() / ( - 1e-12 - + (dg ** 2) - .view(dg.shape[0], dg.shape[1], -1) - .sum(dim=-1) - .sqrt() - ) - elif self.norm == "L1": - dist1 = df.abs() / ( - 1e-12 - + dg.abs() - .reshape([df.shape[0], df.shape[1], -1]) - .max(dim=2)[0] - ) - else: - raise ValueError("norm not supported") - ind = dist1.min(dim=1)[1] - dg2 = dg[u1, ind] - b = -df[u1, ind] + (dg2 * x1).view(x1.shape[0], -1).sum(dim=-1) - w = dg2.reshape([bs, -1]) - - if self.norm == "Linf": - d3 = projection_linf( - torch.cat((x1.reshape([bs, -1]), x0), 0), - torch.cat((w, w), 0), - torch.cat((b, b), 0), - ) - elif self.norm == "L2": - d3 = projection_l2( - torch.cat((x1.reshape([bs, -1]), x0), 0), - torch.cat((w, w), 0), - torch.cat((b, b), 0), - ) - elif self.norm == "L1": - d3 = projection_l1( - torch.cat((x1.reshape([bs, -1]), x0), 0), - torch.cat((w, w), 0), - torch.cat((b, b), 0), - ) - d1 = torch.reshape(d3[:bs], x1.shape) - d2 = torch.reshape(d3[-bs:], x1.shape) - if self.norm == "Linf": - a0 = ( - d3.abs() - .max(dim=1, keepdim=True)[0] - .view(-1, *[1] * self.ndims) - ) - elif self.norm == "L2": - a0 = ( - (d3 ** 2) - .sum(dim=1, keepdim=True) - .sqrt() - .view(-1, *[1] * self.ndims) - ) - elif self.norm == "L1": - a0 = ( - d3.abs() - .sum(dim=1, keepdim=True) - .view(-1, *[1] * self.ndims) - ) - a0 = torch.max(a0, 1e-8 * torch.ones(a0.shape).to(self.device)) - a1 = a0[:bs] - a2 = a0[-bs:] - alpha = torch.min( - torch.max( - a1 / (a1 + a2), torch.zeros(a1.shape).to(self.device) - ), - self.alpha_max * torch.ones(a1.shape).to(self.device), - ) - x1 = ( - (x1 + self.eta * d1) * (1 - alpha) - + (im2 + d2 * self.eta) * alpha - ).clamp(0.0, 1.0) - - is_adv = self._get_predicted_label(x1) != la2 - - if is_adv.sum() > 0: - ind_adv = is_adv.nonzero().squeeze() - ind_adv = self.check_shape(ind_adv) - if self.norm == "Linf": - t = ( - (x1[ind_adv] - im2[ind_adv]) - .reshape([ind_adv.shape[0], -1]) - .abs() - .max(dim=1)[0] - ) - elif self.norm == "L2": - t = ( - ((x1[ind_adv] - im2[ind_adv]) ** 2) - .view(ind_adv.shape[0], -1) - .sum(dim=-1) - .sqrt() - ) - elif self.norm == "L1": - t = ( - (x1[ind_adv] - im2[ind_adv]) - .abs() - .view(ind_adv.shape[0], -1) - .sum(dim=-1) - ) - adv[ind_adv] = x1[ind_adv] * ( - t < res2[ind_adv] - ).float().reshape([-1, *[1] * self.ndims]) + adv[ind_adv] * ( - t >= res2[ind_adv] - ).float().reshape( - [-1, *[1] * self.ndims] - ) - res2[ind_adv] = ( - t * (t < res2[ind_adv]).float() - + res2[ind_adv] * (t >= res2[ind_adv]).float() - ) - x1[ind_adv] = ( - im2[ind_adv] + (x1[ind_adv] - im2[ind_adv]) * self.beta - ) - - counter_iter += 1 - - counter_restarts += 1 - - ind_succ = res2 < 1e10 - if self.verbose: - print( - "success rate: {:.0f}/{:.0f}".format( - ind_succ.float().sum(), corr_classified - ) - + " (on correctly classified points) in {:.1f} s".format( - time.time() - startt - ) - ) - - res_c[pred] = res2 * ind_succ.float() + 1e10 * (1 - ind_succ.float()) - ind_succ = self.check_shape(ind_succ.nonzero().squeeze()) - adv_c[pred[ind_succ]] = adv[ind_succ].clone() + adv = im2.clone() + adv_c = images.clone() + res2 = torch.full((bs, ), 1e10, device=self.device) + x1 = torch.clone(im2) + x0 = torch.clone(im2).reshape(bs, -1) + eps = torch.full(res2.shape, self.eps, device=self.device) + + if self.targeted: + # The code provided in the original paper does not implement the target attack code, + # and the code here is implemented based on the relevant code in the original author's subsequent autoattack work. + # https://github.com/fra31/auto-attack + target_labels = self.get_target_label(images, labels) + la_target2 = target_labels[pred].detach().clone() + else: + la_target2 = None + + for counter_restarts in range(self.n_restarts): + if counter_restarts > 0: + t = uniform.Uniform(-1, 1).sample(x1.shape).to(self.device) + a = torch.min(res2, eps).reshape((-1, 1, 1, 1)) * t + b = torch.abs(t.view(t.shape[0], -1)).max(dim=1, keepdim=True)[0].view((-1, 1, 1, 1)) # nopep8 + x1 = im2 + a / b * 0.5 + x1 = torch.clamp(x1, min=0.0, max=1.0) + + for _ in range(self.n_iter): + # print(i) + df, dg = self.get_diff_logits_grads_batch(x1, la2, la_target2) + dist1 = torch.abs(df) / (1e-8 + torch.sum(torch.abs(dg).view(dg.shape[0], dg.shape[1], -1), -1)) # nopep8 + ind = torch.argmin(dist1, 1) + b = - df[u1, ind] + torch.sum(torch.reshape(dg[u1, ind] * x1, (bs, -1)), 1).to(self.device) # nopep8 + w = torch.reshape(dg[u1, ind], [bs, -1]).to(self.device) + x2 = torch.reshape(x1, (bs, -1)) + d3 = self.projection_linf(torch.cat((x2, x0), 0), torch.cat((w, w), 0), torch.cat((b, b), 0)) # nopep8 + d1 = torch.reshape(d3[:bs], x1.shape) + d2 = torch.reshape(d3[-bs:], x1.shape) + a0 = torch.abs(d3).max(dim=1, keepdim=True)[0].view(-1, 1, 1, 1) # nopep8 + a0 = torch.max(a0, 1e-8 * torch.ones(a0.shape, device=self.device)) # nopep8 + a1 = a0[:bs] + a2 = a0[-bs:] + temp_var_1 = torch.max(a1 / (a1 + a2), torch.zeros(a1.shape, device=self.device)) # nopep8 + temp_var_2 = self.alpha_max * torch.ones(a1.shape, device=self.device) # nopep8 + alpha = torch.min(temp_var_1, temp_var_2) + x1 = torch.clamp((x1 + self.eta * d1) * (1 - alpha) + (im2 + d2 * self.eta) * alpha, min=0.0, max=1.0) # nopep8 + is_adv = torch.argmax(self.get_logits(x1), 1) != la2 + if torch.sum(is_adv) > 0: + temp_var = torch.reshape(x1[is_adv] - im2[is_adv], (torch.sum(is_adv), -1)) # nopep8 + t = torch.max(torch.abs(temp_var), 1)[0] + temp_var_3 = x1[is_adv] * (t < res2[is_adv]).float().reshape([-1, 1, 1, 1]) # nopep8 + temp_var_4 = adv[is_adv] * (t >= res2[is_adv]).float().reshape([-1, 1, 1, 1]) # nopep8 + adv[is_adv] = temp_var_3 + temp_var_4 + res2[is_adv] = t * (t < res2[is_adv]).float() + res2[is_adv] * (t >= res2[is_adv]).float() # nopep8 + x1[is_adv] = im2[is_adv] + (x1[is_adv] - im2[is_adv]) * self.beta # nopep8 + + if self.las: + adv = self.linear_approximation_search(im2, la2, adv, 3) + + adv_c[pred1] = adv return adv_c - def attack_single_run_targeted(self, x, y=None, use_rand_start=False): - """ - :param x: clean images - :param y: clean labels, if None we use the predicted labels - """ + def get_diff_logits_grads_batch(self, images, labels, target_labels=None): + images = images.clone().detach().requires_grad_() # make sure its was leaf node + # print(images.is_leaf) + + if not self.targeted: + logits = self.get_logits(images) + g2 = self.compute_jacobian(images, logits) + y2 = logits + df = y2 - torch.unsqueeze(y2[torch.arange(images.shape[0]), labels], 1) # nopep8 + dg = g2 - torch.unsqueeze(g2[torch.arange(images.shape[0]), labels], 1) # nopep8 + df[torch.arange(images.shape[0]), labels] = 1e10 + else: + u = torch.arange(images.shape[0]) + logits = self.get_logits(images) + diff_logits = -(logits[u, labels] - logits[u, target_labels]) + sum_diff = torch.sum(diff_logits) + + # jacobian + self.zero_gradients(images) + sum_diff.backward() + grad_diff = images.grad.data + df = torch.unsqueeze(diff_logits.detach(), 1) + dg = torch.unsqueeze(grad_diff, 1) - if self.device is None: - self.device = x.device - self.orig_dim = list(x.shape[1:]) - self.ndims = len(self.orig_dim) + return df, dg - x = x.detach().clone().float().to(self.device) - # assert next(self.model.parameters()).device == x.device + def compute_jacobian(self, images, logits): + num_classes = logits.shape[1] + jacobian = torch.zeros(num_classes, *images.size()).to(self.device) + grad_output = torch.zeros_like(logits).to(self.device) - y_pred = self._get_predicted_label(x) - if y is None: - y = y_pred.detach().clone().long().to(self.device) - else: - y = y.detach().clone().long().to(self.device) - pred = y_pred == y - corr_classified = pred.float().sum() - if self.verbose: - print("Clean accuracy: {:.2%}".format(pred.float().mean())) - if pred.sum() == 0: - return x - pred = self.check_shape(pred.nonzero().squeeze()) - - output = self.get_logits(x) - if self.multi_targeted: - la_target = output.sort(dim=-1)[1][:, -self.target_class] - else: - la_target = self.target_class - - startt = time.time() - # runs the attack only on correctly classified points - im2 = x[pred].detach().clone() - la2 = y[pred].detach().clone() - la_target2 = la_target[pred].detach().clone() - if len(im2.shape) == self.ndims: - im2 = im2.unsqueeze(0) - bs = im2.shape[0] - u1 = torch.arange(bs) - adv = im2.clone() - adv_c = x.clone() - res2 = 1e10 * torch.ones([bs]).to(self.device) - res_c = torch.zeros([x.shape[0]]).to(self.device) - x1 = im2.clone() - x0 = im2.clone().reshape([bs, -1]) - counter_restarts = 0 - - while counter_restarts < 1: - if use_rand_start: - if self.norm == "Linf": - t = 2 * torch.rand(x1.shape).to(self.device) - 1 - x1 = ( - im2 - + ( - torch.min( - res2, self.eps * torch.ones(res2.shape).to(self.device) - ).reshape([-1, *[1] * self.ndims]) - ) - * t - / ( - t.reshape([t.shape[0], -1]) - .abs() - .max(dim=1, keepdim=True)[0] - .reshape([-1, *[1] * self.ndims]) - ) - * 0.5 - ) - elif self.norm == "L2": - t = torch.randn(x1.shape).to(self.device) - x1 = ( - im2 - + ( - torch.min( - res2, self.eps * torch.ones(res2.shape).to(self.device) - ).reshape([-1, *[1] * self.ndims]) - ) - * t - / ( - (t ** 2) - .view(t.shape[0], -1) - .sum(dim=-1) - .sqrt() - .view(t.shape[0], *[1] * self.ndims) - ) - * 0.5 - ) - elif self.norm == "L1": - t = torch.randn(x1.shape).to(self.device) - x1 = ( - im2 - + ( - torch.min( - res2, self.eps * torch.ones(res2.shape).to(self.device) - ).reshape([-1, *[1] * self.ndims]) - ) - * t - / ( - t.abs() - .view(t.shape[0], -1) - .sum(dim=-1) - .view(t.shape[0], *[1] * self.ndims) - ) - / 2 - ) - - x1 = x1.clamp(0.0, 1.0) - - counter_iter = 0 - while counter_iter < self.steps: - with torch.no_grad(): - df, dg = self.get_diff_logits_grads_batch_targeted( - x1, la2, la_target2 - ) - if self.norm == "Linf": - dist1 = df.abs() / ( - 1e-12 - + dg.abs().view(dg.shape[0], dg.shape[1], -1).sum(dim=-1) - ) - elif self.norm == "L2": - dist1 = df.abs() / ( - 1e-12 - + (dg ** 2) - .view(dg.shape[0], dg.shape[1], -1) - .sum(dim=-1) - .sqrt() - ) - elif self.norm == "L1": - dist1 = df.abs() / ( - 1e-12 - + dg.abs() - .reshape([df.shape[0], df.shape[1], -1]) - .max(dim=2)[0] - ) - else: - raise ValueError("norm not supported") - ind = dist1.min(dim=1)[1] - - dg2 = dg[u1, ind] - b = -df[u1, ind] + (dg2 * x1).view(x1.shape[0], -1).sum(dim=-1) - w = dg2.reshape([bs, -1]) - - if self.norm == "Linf": - d3 = projection_linf( - torch.cat((x1.reshape([bs, -1]), x0), 0), - torch.cat((w, w), 0), - torch.cat((b, b), 0), - ) - elif self.norm == "L2": - d3 = projection_l2( - torch.cat((x1.reshape([bs, -1]), x0), 0), - torch.cat((w, w), 0), - torch.cat((b, b), 0), - ) - elif self.norm == "L1": - d3 = projection_l1( - torch.cat((x1.reshape([bs, -1]), x0), 0), - torch.cat((w, w), 0), - torch.cat((b, b), 0), - ) - d1 = torch.reshape(d3[:bs], x1.shape) - d2 = torch.reshape(d3[-bs:], x1.shape) - if self.norm == "Linf": - a0 = ( - d3.abs() - .max(dim=1, keepdim=True)[0] - .view(-1, *[1] * self.ndims) - ) - elif self.norm == "L2": - a0 = ( - (d3 ** 2) - .sum(dim=1, keepdim=True) - .sqrt() - .view(-1, *[1] * self.ndims) - ) - elif self.norm == "L1": - a0 = ( - d3.abs() - .sum(dim=1, keepdim=True) - .view(-1, *[1] * self.ndims) - ) - a0 = torch.max(a0, 1e-8 * torch.ones(a0.shape).to(self.device)) - a1 = a0[:bs] - a2 = a0[-bs:] - alpha = torch.min( - torch.max( - a1 / (a1 + a2), torch.zeros(a1.shape).to(self.device) - ), - self.alpha_max * torch.ones(a1.shape).to(self.device), - ) - x1 = ( - (x1 + self.eta * d1) * (1 - alpha) - + (im2 + d2 * self.eta) * alpha - ).clamp(0.0, 1.0) - - is_adv = self._get_predicted_label(x1) != la2 - - if is_adv.sum() > 0: - ind_adv = is_adv.nonzero().squeeze() - ind_adv = self.check_shape(ind_adv) - if self.norm == "Linf": - t = ( - (x1[ind_adv] - im2[ind_adv]) - .reshape([ind_adv.shape[0], -1]) - .abs() - .max(dim=1)[0] - ) - elif self.norm == "L2": - t = ( - ((x1[ind_adv] - im2[ind_adv]) ** 2) - .view(ind_adv.shape[0], -1) - .sum(dim=-1) - .sqrt() - ) - elif self.norm == "L1": - t = ( - (x1[ind_adv] - im2[ind_adv]) - .abs() - .view(ind_adv.shape[0], -1) - .sum(dim=-1) - ) - adv[ind_adv] = x1[ind_adv] * ( - t < res2[ind_adv] - ).float().reshape([-1, *[1] * self.ndims]) + adv[ind_adv] * ( - t >= res2[ind_adv] - ).float().reshape( - [-1, *[1] * self.ndims] - ) - res2[ind_adv] = ( - t * (t < res2[ind_adv]).float() - + res2[ind_adv] * (t >= res2[ind_adv]).float() - ) - x1[ind_adv] = ( - im2[ind_adv] + (x1[ind_adv] - im2[ind_adv]) * self.beta - ) - - counter_iter += 1 - - counter_restarts += 1 - - ind_succ = res2 < 1e10 - if self.verbose: - print( - "success rate: {:.0f}/{:.0f}".format( - ind_succ.float().sum(), corr_classified - ) - + " (on correctly classified points) in {:.1f} s".format( - time.time() - startt - ) - ) - - res_c[pred] = res2 * ind_succ.float() + 1e10 * (1 - ind_succ.float()) - ind_succ = self.check_shape(ind_succ.nonzero().squeeze()) - adv_c[pred[ind_succ]] = adv[ind_succ].clone() + for i in range(num_classes): + self.zero_gradients(images) + grad_output.zero_() + grad_output[:, i] = 1 + logits.backward(grad_output, retain_graph=True) + jacobian[i] = images.grad.data - return adv_c + return torch.transpose(jacobian, dim0=0, dim1=1) - def perturb(self, x, y): - adv = x.clone() - with torch.no_grad(): - acc = self.get_logits(x).max(1)[1] == y - - startt = time.time() - - torch.random.manual_seed(self.seed) - torch.cuda.random.manual_seed(self.seed) - - def inner_perturb(targeted): - for counter in range(self.n_restarts): - ind_to_fool = acc.nonzero().squeeze() - if len(ind_to_fool.shape) == 0: - ind_to_fool = ind_to_fool.unsqueeze(0) - if ind_to_fool.numel() != 0: - x_to_fool, y_to_fool = ( - x[ind_to_fool].clone(), - y[ind_to_fool].clone(), - ) # nopep8 - - if targeted: - adv_curr = self.attack_single_run_targeted( - x_to_fool, y_to_fool, use_rand_start=(counter > 0) - ) - else: - adv_curr = self.attack_single_run( - x_to_fool, y_to_fool, use_rand_start=(counter > 0) - ) - - acc_curr = self.get_logits(adv_curr).max(1)[1] == y_to_fool - if self.norm == "Linf": - res = ( - (x_to_fool - adv_curr) - .abs() - .view(x_to_fool.shape[0], -1) - .max(1)[0] - ) # nopep8 - elif self.norm == "L2": - res = ( - ((x_to_fool - adv_curr) ** 2) - .view(x_to_fool.shape[0], -1) - .sum(dim=-1) - .sqrt() - ) # nopep8 - acc_curr = torch.max(acc_curr, res > self.eps) - - ind_curr = (acc_curr == 0).nonzero().squeeze() - acc[ind_to_fool[ind_curr]] = 0 - adv[ind_to_fool[ind_curr]] = adv_curr[ind_curr].clone() - - if self.verbose: - if targeted: - print( - "restart {} - target_class {} - robust accuracy: {:.2%} at eps = {:.5f} - cum. time: {:.1f} s".format( - counter, - self.target_class, - acc.float().mean(), - self.eps, - time.time() - startt, - ) - ) - else: - print( - "restart {} - robust accuracy: {:.2%} at eps = {:.5f} - cum. time: {:.1f} s".format( - counter, - acc.float().mean(), - self.eps, - time.time() - startt, - ) - ) - - if self.multi_targeted: - for target_class in range(2, self.n_target_classes + 2): - self.target_class = target_class - inner_perturb(targeted=True) - elif self.targeted: - self.target_class = self.get_target_label(x, y) - inner_perturb(targeted=True) - else: - inner_perturb(targeted=False) - return adv - - -def projection_linf(points_to_project, w_hyperplane, b_hyperplane): - device = points_to_project.device - t, w, b = points_to_project, w_hyperplane.clone(), b_hyperplane.clone() - - sign = 2 * ((w * t).sum(1) - b >= 0) - 1 - w.mul_(sign.unsqueeze(1)) - b.mul_(sign) - - a = (w < 0).float() - d = (a - t) * (w != 0).float() - - p = a - t * (2 * a - 1) - indp = torch.argsort(p, dim=1) - - b = b - (w * t).sum(1) - b0 = (w * d).sum(1) - - indp2 = indp.flip((1,)) - ws = w.gather(1, indp2) - bs2 = -ws * d.gather(1, indp2) - - s = torch.cumsum(ws.abs(), dim=1) - sb = torch.cumsum(bs2, dim=1) + b0.unsqueeze(1) - - b2 = sb[:, -1] - s[:, -1] * p.gather(1, indp[:, 0:1]).squeeze(1) - c_l = b - b2 > 0 - c2 = (b - b0 > 0) & (~c_l) - lb = torch.zeros(c2.sum(), device=device) - ub = torch.full_like(lb, w.shape[1] - 1) - nitermax = math.ceil(math.log2(w.shape[1])) - - indp_, sb_, s_, p_, b_ = indp[c2], sb[c2], s[c2], p[c2], b[c2] - for counter in range(nitermax): - counter4 = torch.floor((lb + ub) / 2) - - counter2 = counter4.long().unsqueeze(1) - indcurr = indp_.gather(1, indp_.size(1) - 1 - counter2) - b2 = ( - sb_.gather(1, counter2) - s_.gather(1, counter2) * p_.gather(1, indcurr) - ).squeeze( - 1 - ) # nopep8 - c = b_ - b2 > 0 - - lb = torch.where(c, counter4, lb) - ub = torch.where(c, ub, counter4) - - lb = lb.long() - - if c_l.any(): - lmbd_opt = torch.clamp_min( - (b[c_l] - sb[c_l, -1]) / (-s[c_l, -1]), min=0 - ).unsqueeze(-1) - d[c_l] = (2 * a[c_l] - 1) * lmbd_opt - - lmbd_opt = torch.clamp_min((b[c2] - sb[c2, lb]) / (-s[c2, lb]), min=0).unsqueeze(-1) - d[c2] = torch.min(lmbd_opt, d[c2]) * a[c2] + torch.max(-lmbd_opt, d[c2]) * ( - 1 - a[c2] - ) - - return d * (w != 0).float() - - -def projection_l2(points_to_project, w_hyperplane, b_hyperplane): - device = points_to_project.device - t, w, b = points_to_project, w_hyperplane.clone(), b_hyperplane - - c = (w * t).sum(1) - b - ind2 = 2 * (c >= 0) - 1 - w.mul_(ind2.unsqueeze(1)) - c.mul_(ind2) - - r = torch.max(t / w, (t - 1) / w).clamp(min=-1e12, max=1e12) - r.masked_fill_(w.abs() < 1e-8, 1e12) - r[r == -1e12] *= -1 - rs, indr = torch.sort(r, dim=1) - rs2 = F.pad(rs[:, 1:], (0, 1)) - rs.masked_fill_(rs == 1e12, 0) - rs2.masked_fill_(rs2 == 1e12, 0) - - w3s = (w ** 2).gather(1, indr) - w5 = w3s.sum(dim=1, keepdim=True) - ws = w5 - torch.cumsum(w3s, dim=1) - d = -(r * w) - d.mul_((w.abs() > 1e-8).float()) - s = torch.cat( - (-w5 * rs[:, 0:1], torch.cumsum((-rs2 + rs) * ws, dim=1) - w5 * rs[:, 0:1]), 1 - ) - - c4 = s[:, 0] + c < 0 - c3 = (d * w).sum(dim=1) + c > 0 - c2 = ~(c4 | c3) - - lb = torch.zeros(c2.sum(), device=device) - ub = torch.full_like(lb, w.shape[1] - 1) - nitermax = math.ceil(math.log2(w.shape[1])) - - s_, c_ = s[c2], c[c2] - for counter in range(nitermax): - counter4 = torch.floor((lb + ub) / 2) - counter2 = counter4.long().unsqueeze(1) - c3 = s_.gather(1, counter2).squeeze(1) + c_ > 0 - lb = torch.where(c3, counter4, lb) - ub = torch.where(c3, ub, counter4) - - lb = lb.long() - - if c4.any(): - alpha = c[c4] / w5[c4].squeeze(-1) - d[c4] = -alpha.unsqueeze(-1) * w[c4] - - if c2.any(): - alpha = (s[c2, lb] + c[c2]) / ws[c2, lb] + rs[c2, lb] - alpha[ws[c2, lb] == 0] = 0 - c5 = (alpha.unsqueeze(-1) > r[c2]).float() - d[c2] = d[c2] * c5 - alpha.unsqueeze(-1) * w[c2] * (1 - c5) - - return d * (w.abs() > 1e-8).float() - - -def projection_l1(points_to_project, w_hyperplane, b_hyperplane): - device = points_to_project.device - t, w, b = points_to_project, w_hyperplane.clone(), b_hyperplane - - c = (w * t).sum(1) - b - ind2 = 2 * (c >= 0) - 1 - w.mul_(ind2.unsqueeze(1)) - c.mul_(ind2) - - r = (1 / w).abs().clamp_max(1e12) - indr = torch.argsort(r, dim=1) - indr_rev = torch.argsort(indr) - - c6 = (w < 0).float() - d = (-t + c6) * (w != 0).float() - ds = torch.min(-w * t, w * (1 - t)).gather(1, indr) - ds2 = torch.cat((c.unsqueeze(-1), ds), 1) - s = torch.cumsum(ds2, dim=1) - - c2 = s[:, -1] < 0 - - lb = torch.zeros(c2.sum(), device=device) - ub = torch.full_like(lb, s.shape[1]) - nitermax = math.ceil(math.log2(w.shape[1])) - - s_ = s[c2] - for counter in range(nitermax): - counter4 = torch.floor((lb + ub) / 2) - counter2 = counter4.long().unsqueeze(1) - c3 = s_.gather(1, counter2).squeeze(1) > 0 - lb = torch.where(c3, counter4, lb) - ub = torch.where(c3, ub, counter4) - - lb2 = lb.long() - - if c2.any(): - indr = indr[c2].gather(1, lb2.unsqueeze(1)).squeeze(1) - u = torch.arange(0, w.shape[0], device=device).unsqueeze(1) - u2 = torch.arange(0, w.shape[1], device=device, dtype=torch.float).unsqueeze(0) - alpha = -s[c2, lb2] / w[c2, indr] - c5 = u2 < lb.unsqueeze(-1) - u3 = c5[u[: c5.shape[0]], indr_rev[c2]] - d[c2] = d[c2] * u3.float() - d[c2, indr] = alpha - - return d * (w.abs() > 1e-8).float() - - -def zero_gradients(x): - if isinstance(x, torch.Tensor): + def zero_gradients(self, x): if x.grad is not None: x.grad.detach_() x.grad.zero_() - elif isinstance(x, container_abcs.Iterable): - for elem in x: - zero_gradients(elem) + + def projection_linf(self, t2, w2, b2): + t = t2.clone().float() + w = w2.clone().float() + b = b2.clone().float() + + ind2 = torch.nonzero(torch.sum(w * t, 1) - b < 0) + w[ind2] *= -1 + b[ind2] *= -1 + + c5 = (w < 0).type(torch.FloatTensor).to(self.device) + a = torch.ones(t.shape).to(self.device) + d = (a * c5 - t) * (w != 0).type(torch.FloatTensor).to(self.device) + a -= a * (1 - c5) + + p = torch.ones(t.shape, device=self.device) * c5 - t * (2 * c5 - 1) + indp = torch.argsort(p, dim=1) + + b = b - torch.sum(w * t, 1) + b0 = torch.sum(w * d, 1) + b1 = b0.clone() + + indp2 = indp.unsqueeze(-1).flip(dims=(1, 2)).squeeze() + u = torch.arange(0, w.shape[0]) + ws = w[u.unsqueeze(1), indp2] + bs2 = -ws * d[u.unsqueeze(1), indp2] + + s = torch.cumsum(ws.abs(), dim=1) + sb = torch.cumsum(bs2, dim=1) + b0.unsqueeze(1) + + b2 = sb[u, -1] - s[u, -1] * p[u, indp[u, 0]] + c_l = torch.nonzero(b - b2 > 0).squeeze() + c2 = torch.nonzero((b - b1 > 0) * (b - b2 <= 0)).squeeze() + + lb = torch.zeros(c2.shape[0], device=self.device) + ub = torch.ones(c2.shape[0], device=self.device) * (w.shape[1] - 1) + nitermax = torch.ceil(torch.log2(torch.tensor(w.shape[1]).float())) + + for _ in range(int(nitermax.item())): + counter4 = torch.floor((lb + ub) / 2) + counter2 = counter4.type(torch.LongTensor) + indcurr = indp[c2, -counter2 - 1] + b2 = sb[c2, counter2] - s[c2, counter2] * p[c2, indcurr] + ind3 = b[c2] - b2 > 0 + ind32 = ~ind3 + lb[ind3] = counter4[ind3] + ub[ind32] = counter4[ind32] + + lb = lb.cpu().numpy().astype(int) + + if c_l.nelement != 0: + m = torch.max((b[c_l] - sb[c_l, -1]) / (-s[c_l, -1]), torch.zeros(sb[c_l, -1].shape, device=self.device)) # nopep8 + lmbd_opt = torch.unsqueeze(m, -1) + d[c_l] = (2 * a[c_l] - 1) * lmbd_opt + + m = torch.max((b[c2] - sb[c2, lb])/(-s[c2, lb]), torch.zeros(sb[c2, lb].shape, device=self.device)) # nopep8 + lmbd_opt = torch.unsqueeze(m, -1) + d[c2] = torch.min(lmbd_opt, d[c2]) * c5[c2] + torch.max(-lmbd_opt, d[c2]) * (1 - c5[c2]) # nopep8 + + return d * (w != 0).type(torch.FloatTensor).to(self.device) + + def linear_approximation_search(self, clean_images, clean_labels, adv_images, niter): + a1 = clean_images.clone() + a2 = adv_images.clone() + u = torch.arange(clean_images.shape[0]) + y1 = self.get_logits(a1) + y2 = self.get_logits(a2) + la2 = torch.argmax(y2, 1) + + for _ in range(niter): + t1 = (y1[u, clean_labels] - y1[u, la2]).reshape([-1, 1, 1, 1]) + t2 = (-(y2[u, clean_labels] - y2[u, la2])).reshape([-1, 1, 1, 1]) + + t3 = t1 / (t1 + t2 + 1e-10) + c3 = torch.logical_and(0.0 <= t3, t3 <= 1.0) + t3[~c3] = 1.0 + + a3 = a1 * (1.0 - t3) + a2 * t3 + y3 = self.get_logits(a3) + la3 = torch.argmax(y3, 1) + pred = la3 == clean_labels + + y1[pred] = y3[pred] + 0 + a1[pred] = a3[pred] + 0 + y2[~pred] = y3[~pred] + 0 + la2[~pred] = la3[~pred] + 0 + a2[~pred] = a3[~pred] + 0 + + return a2 diff --git a/torchattacks/attacks/fabl1.py b/torchattacks/attacks/fabl1.py new file mode 100644 index 00000000..ce3a3df0 --- /dev/null +++ b/torchattacks/attacks/fabl1.py @@ -0,0 +1,250 @@ +import torch +import torch.distributions.uniform as uniform + +from ..attack import Attack + + +class FABL1(Attack): + r""" + Fast Adaptive Boundary Attack (FAB) in the paper 'Minimally distorted Adversarial Examples with a Fast Adaptive Boundary Attack' + [https://arxiv.org/abs/1907.02044] + [https://github.com/fra31/fab-attack] + + Distance Measure : L1 + + Arguments: + model (nn.Module): model to attack. + eps (float): maximum perturbation. (Default: 8/255) + n_restarts (int): number of random restarts. (Default: 1) + n_iter (int): number of steps. (Default: 10) + alpha_max (float): alpha_max. (Default: 0.1) + eta (float): overshooting. (Default: 1.05) + beta (float): backward step. (Default: 0.9) + las (bool): final search. (Default: False) + + Shape: + - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. + - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. + - output: :math:`(N, C, H, W)`. + + Examples:: + >>> attack = torchattacks.FABL1(model, eps=8/255, n_restarts=1, n_iter=10, alpha_max=0.1, eta=1.05, beta=0.9, las=False) + >>> adv_images = attack(images, labels) + + """ + + def __init__(self, model, eps=8/255, n_restarts=1, n_iter=10, alpha_max=0.1, eta=1.05, beta=0.9, las=False): + super().__init__("FABL1", model) + self.eps = eps + self.n_restarts = n_restarts + self.n_iter = n_iter + self.alpha_max = alpha_max + self.eta = eta + self.beta = beta + self.las = las + self.supported_mode = ["default", "targeted"] + + def forward(self, images, labels): + r""" + Overridden. + """ + + images = images.clone().detach().to(self.device) + labels = labels.clone().detach().to(self.device) + adv_images = self.perturb(images, labels) + + return adv_images + + def perturb(self, images, labels): + logits = self.get_logits(images) + pred = torch.argmax(logits, 1) == labels + pred1 = pred.clone() + im2 = images[pred].clone() + la2 = labels[pred].clone() + bs = torch.sum(pred) + u1 = torch.arange(bs) + + adv = im2.clone() + adv_c = images.clone() + res2 = torch.full((bs, ), 1e10, device=self.device) + x1 = torch.clone(im2) + x0 = torch.clone(im2).reshape(bs, -1) + eps = torch.full(res2.shape, self.eps, device=self.device) + + if self.targeted: + # The code provided in the original paper does not implement the target attack code, + # and the code here is implemented based on the relevant code in the original author's subsequent autoattack work. + # https://github.com/fra31/auto-attack + target_labels = self.get_target_label(images, labels) + la_target2 = target_labels[pred].detach().clone() + else: + la_target2 = None + + for counter_restarts in range(self.n_restarts): + if counter_restarts > 0: + # t = torch.rand(x1.shape[0], x1.shape[1], x1.shape[2], x1.shape[3]) # nopep8 + t = uniform.Uniform(-1, 1).sample(x1.shape).to(self.device) + a = torch.min(res2, eps).reshape((-1, 1, 1, 1)) * t + b = torch.sum(torch.abs(t).view(t.shape[0], -1), -1).view(t.shape[0], 1, 1, 1) # nopep8 + x1 = im2 + a / b * 0.5 + x1 = torch.clamp(x1, min=0.0, max=1.0) + + for _ in range(self.n_iter): + # print(i) + df, dg = self.get_diff_logits_grads_batch(x1, la2, la_target2) + dist1 = torch.abs(df) / torch.max(1e-8 + torch.abs(dg).reshape((df.shape[0], df.shape[1], -1)), 2)[0] # nopep8 + ind = torch.argmin(dist1, 1) + b = - df[u1, ind] + torch.sum(torch.reshape(dg[u1, ind] * x1, (bs, -1)), 1).to(self.device) # nopep8 + w = torch.reshape(dg[u1, ind], [bs, -1]).to(self.device) + x2 = torch.reshape(x1, (bs, -1)) + d3 = self.projection_l1(torch.cat((x2, x0), 0), torch.cat((w, w), 0), torch.cat((b, b), 0)) # nopep8 + d1 = torch.reshape(d3[:bs], x1.shape) + d2 = torch.reshape(d3[-bs:], x1.shape) + a0 = torch.sum(torch.abs(d3), dim=1, keepdim=True).view(-1, 1, 1, 1) # nopep8 + a0 = torch.max(a0, 1e-8 * torch.ones(a0.shape, device=self.device)) # nopep8 + a1 = a0[:bs] + a2 = a0[-bs:] + temp_var_1 = torch.max(a1 / (a1 + a2), torch.zeros(a1.shape, device=self.device)) # nopep8 + temp_var_2 = self.alpha_max * torch.ones(a1.shape, device=self.device) # nopep8 + alpha = torch.min(temp_var_1, temp_var_2) + x1 = torch.clamp((x1 + self.eta * d1) * (1 - alpha) + (im2 + d2 * self.eta) * alpha, min=0.0, max=1.0) # nopep8 + is_adv = torch.argmax(self.get_logits(x1), 1) != la2 + if torch.sum(is_adv) > 0: + temp_var = torch.reshape(x1[is_adv] - im2[is_adv], (torch.sum(is_adv), -1)) # nopep8 + t = torch.sum(torch.abs(temp_var).view(torch.sum(is_adv), -1), -1) # nopep8 + temp_var_3 = x1[is_adv] * (t < res2[is_adv]).float().reshape([-1, 1, 1, 1]) # nopep8 + temp_var_4 = adv[is_adv] * (t >= res2[is_adv]).float().reshape([-1, 1, 1, 1]) # nopep8 + adv[is_adv] = temp_var_3 + temp_var_4 + res2[is_adv] = t * (t < res2[is_adv]).float() + res2[is_adv] * (t >= res2[is_adv]).float() # nopep8 + x1[is_adv] = im2[is_adv] + (x1[is_adv] - im2[is_adv]) * self.beta # nopep8 + + if self.las: + adv = self.linear_approximation_search(im2, la2, adv, 3) + + adv_c[pred1] = adv + return adv_c + + def get_diff_logits_grads_batch(self, images, labels, target_labels=None): + images = images.clone().detach().requires_grad_() # make sure its was leaf node + # print(images.is_leaf) + + if not self.targeted: + logits = self.get_logits(images) + g2 = self.compute_jacobian(images, logits) + y2 = logits + df = y2 - torch.unsqueeze(y2[torch.arange(images.shape[0]), labels], 1) # nopep8 + dg = g2 - torch.unsqueeze(g2[torch.arange(images.shape[0]), labels], 1) # nopep8 + df[torch.arange(images.shape[0]), labels] = 1e10 + else: + u = torch.arange(images.shape[0]) + logits = self.get_logits(images) + diff_logits = -(logits[u, labels] - logits[u, target_labels]) + sum_diff = torch.sum(diff_logits) + + # jacobian + self.zero_gradients(images) + sum_diff.backward() + grad_diff = images.grad.data + df = torch.unsqueeze(diff_logits.detach(), 1) + dg = torch.unsqueeze(grad_diff, 1) + + return df, dg + + def compute_jacobian(self, images, logits): + num_classes = logits.shape[1] + jacobian = torch.zeros(num_classes, *images.size()).to(self.device) + grad_output = torch.zeros_like(logits).to(self.device) + + for i in range(num_classes): + self.zero_gradients(images) + grad_output.zero_() + grad_output[:, i] = 1 + logits.backward(grad_output, retain_graph=True) + jacobian[i] = images.grad.data + + return torch.transpose(jacobian, dim0=0, dim1=1) + + def zero_gradients(self, x): + if x.grad is not None: + x.grad.detach_() + x.grad.zero_() + + def projection_l1(self, t2, w2, b2): + t = t2.clone().float() + w = w2.clone().float() + b = b2.clone().float() + + c = torch.sum(w * t, 1) - b + ind2 = c < 0 + w[ind2] *= -1 + c[ind2] *= -1 + + r = torch.max(1 / w, -1 / w) + r = torch.min(r, 1e20 * torch.ones(r.shape, device=self.device)) # nopep8 + indr = torch.argsort(r, dim=1) + indr_rev = torch.argsort(indr) + + u = torch.arange(0, w.shape[0]).unsqueeze(1).to(self.device) + u2 = torch.arange(0, w.shape[1]).repeat(w.shape[0], 1).to(self.device) # nopep8 + + c6 = (w < 0).float() + d = (-t + c6) * (w != 0).float().to(self.device) + d2 = torch.min(-w * t, w * (1 - t)) + ds = d2[u, indr] + ds2 = torch.cat((c.unsqueeze(-1), ds), 1) + s = torch.cumsum(ds2, dim=1) + + c4 = s[:, -1] < 0 + c2 = c4.nonzero().squeeze(-1) + + lb = torch.zeros(c2.shape[0], device=self.device) + ub = torch.ones(c2.shape[0], device=self.device) * (s.shape[1]) + nitermax = torch.ceil(torch.log2(torch.tensor(s.shape[1]).float())) + counter2 = torch.zeros(lb.shape).type(torch.LongTensor) + + for _ in range(int(nitermax.item())): + counter4 = torch.floor((lb + ub)/2) + counter2 = counter4.type(torch.LongTensor) + ind3 = s[c2, counter2] > 0 + ind32 = ~ind3 + lb[ind3] = counter4[ind3] + ub[ind32] = counter4[ind32] + + lb2 = lb.cpu().numpy().astype(int) + if c2.nelement() != 0: + alpha = -s[c2, lb2] / w[c2, indr[c2, lb2]] + c5 = u2[c2].float() < lb.unsqueeze(-1).float() + u3 = c5[u[:c5.shape[0]], indr_rev[c2]] + d[c2] = d[c2] * u3.float() + d[c2, indr[c2, lb2]] = alpha + + return d + + def linear_approximation_search(self, clean_images, clean_labels, adv_images, niter): + a1 = clean_images.clone() + a2 = adv_images.clone() + u = torch.arange(clean_images.shape[0]) + y1 = self.get_logits(a1) + y2 = self.get_logits(a2) + la2 = torch.argmax(y2, 1) + + for _ in range(niter): + t1 = (y1[u, clean_labels] - y1[u, la2]).reshape([-1, 1, 1, 1]) + t2 = (-(y2[u, clean_labels] - y2[u, la2])).reshape([-1, 1, 1, 1]) + + t3 = t1 / (t1 + t2 + 1e-10) + c3 = torch.logical_and(0.0 <= t3, t3 <= 1.0) + t3[~c3] = 1.0 + + a3 = a1 * (1.0 - t3) + a2 * t3 + y3 = self.get_logits(a3) + la3 = torch.argmax(y3, 1) + pred = la3 == clean_labels + + y1[pred] = y3[pred] + 0 + a1[pred] = a3[pred] + 0 + y2[~pred] = y3[~pred] + 0 + la2[~pred] = la3[~pred] + 0 + a2[~pred] = a3[~pred] + 0 + + return a2 diff --git a/torchattacks/attacks/fabl2.py b/torchattacks/attacks/fabl2.py new file mode 100644 index 00000000..34d54810 --- /dev/null +++ b/torchattacks/attacks/fabl2.py @@ -0,0 +1,257 @@ +import torch +import torch.nn.functional as F +import torch.distributions.uniform as uniform + + +from ..attack import Attack + + +class FABL2(Attack): + r""" + Fast Adaptive Boundary Attack (FAB) in the paper 'Minimally distorted Adversarial Examples with a Fast Adaptive Boundary Attack' + [https://arxiv.org/abs/1907.02044] + [https://github.com/fra31/fab-attack] + + Distance Measure : L2 + + Arguments: + model (nn.Module): model to attack. + eps (float): maximum perturbation. (Default: 8/255) + n_restarts (int): number of random restarts. (Default: 1) + n_iter (int): number of steps. (Default: 10) + alpha_max (float): alpha_max. (Default: 0.1) + eta (float): overshooting. (Default: 1.05) + beta (float): backward step. (Default: 0.9) + las (bool): final search. (Default: False) + + Shape: + - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. + - labels: :math:`(N)` where each value :math:`y_i` is :math:`0 \leq y_i \leq` `number of labels`. + - output: :math:`(N, C, H, W)`. + + Examples:: + >>> attack = torchattacks.FABL2(model, eps=8/255, n_restarts=1, n_iter=10, alpha_max=0.1, eta=1.05, beta=0.9, las=False) + >>> adv_images = attack(images, labels) + + """ + + def __init__(self, model, eps=8/255, n_restarts=1, n_iter=10, alpha_max=0.1, eta=1.05, beta=0.9, las=False): + super().__init__("FABL2", model) + self.eps = eps + self.n_restarts = n_restarts + self.n_iter = n_iter + self.alpha_max = alpha_max + self.eta = eta + self.beta = beta + self.las = las + self.supported_mode = ["default", "targeted"] + + def forward(self, images, labels): + r""" + Overridden. + """ + + images = images.clone().detach().to(self.device) + labels = labels.clone().detach().to(self.device) + adv_images = self.perturb(images, labels) + + return adv_images + + def perturb(self, images, labels): + logits = self.get_logits(images) + pred = torch.argmax(logits, 1) == labels + pred1 = pred.clone() + im2 = images[pred].clone() + la2 = labels[pred].clone() + bs = torch.sum(pred) + u1 = torch.arange(bs) + + adv = im2.clone() + adv_c = images.clone() + res2 = torch.full((bs, ), 1e10, device=self.device) + x1 = torch.clone(im2) + x0 = torch.clone(im2).reshape(bs, -1) + eps = torch.full(res2.shape, self.eps, device=self.device) + + if self.targeted: + # The code provided in the original paper does not implement the target attack code, + # and the code here is implemented based on the relevant code in the original author's subsequent autoattack work. + # https://github.com/fra31/auto-attack + target_labels = self.get_target_label(images, labels) + la_target2 = target_labels[pred].detach().clone() + else: + la_target2 = None + + for counter_restarts in range(self.n_restarts): + if counter_restarts > 0: + # t = torch.rand(x1.shape[0], x1.shape[1], x1.shape[2], x1.shape[3]) # nopep8 + t = uniform.Uniform(-1, 1).sample(x1.shape).to(self.device) + a = torch.min(res2, eps).reshape((-1, 1, 1, 1)) * t + b1 = torch.sum(torch.square(t).reshape(t.shape[0], -1), -1) + b2 = torch.sqrt(b1).reshape((-1, 1, 1, 1)) * 0.5 + x1 = im2 + a / b2 * 0.5 + x1 = torch.clamp(x1, min=0.0, max=1.0) + + for _ in range(self.n_iter): + # print(i) + df, dg = self.get_diff_logits_grads_batch(x1, la2, la_target2) + dist1 = torch.abs(df) / torch.sqrt(torch.sum(1e-12 + torch.square(dg).reshape(dg.shape[0], dg.shape[1], -1), -1)) # nopep8 + ind = torch.argmin(dist1, 1) + b = - df[u1, ind] + torch.sum(torch.reshape(dg[u1, ind] * x1, (bs, -1)), 1).to(self.device) # nopep8 + w = torch.reshape(dg[u1, ind], [bs, -1]).to(self.device) + x2 = torch.reshape(x1, (bs, -1)) + d3 = self.projection_l2(torch.cat((x2, x0), 0), torch.cat((w, w), 0), torch.cat((b, b), 0)) # nopep8 + d1 = torch.reshape(d3[:bs], x1.shape) + d2 = torch.reshape(d3[-bs:], x1.shape) + a0 = torch.sqrt(torch.sum(torch.square(d3), dim=1, keepdim=True)).view(-1, 1, 1, 1) # nopep8 + a0 = torch.max(a0, 1e-8 * torch.ones(a0.shape, device=self.device)) # nopep8 + a1 = a0[:bs] + a2 = a0[-bs:] + temp_var_1 = torch.max(a1 / (a1 + a2), torch.zeros(a1.shape, device=self.device)) # nopep8 + temp_var_2 = self.alpha_max * torch.ones(a1.shape, device=self.device) # nopep8 + alpha = torch.min(temp_var_1, temp_var_2) + x1 = torch.clamp((x1 + self.eta * d1) * (1 - alpha) + (im2 + d2 * self.eta) * alpha, min=0.0, max=1.0) # nopep8 + is_adv = torch.argmax(self.get_logits(x1), 1) != la2 + if torch.sum(is_adv) > 0: + temp_var = torch.reshape(x1[is_adv] - im2[is_adv], (torch.sum(is_adv), -1)) # nopep8 + t = torch.sqrt(torch.sum(torch.square(temp_var).view(torch.sum(is_adv), -1), -1)) # nopep8 + temp_var_3 = x1[is_adv] * (t < res2[is_adv]).float().reshape([-1, 1, 1, 1]) # nopep8 + temp_var_4 = adv[is_adv] * (t >= res2[is_adv]).float().reshape([-1, 1, 1, 1]) # nopep8 + adv[is_adv] = temp_var_3 + temp_var_4 + res2[is_adv] = t * (t < res2[is_adv]).float() + res2[is_adv] * (t >= res2[is_adv]).float() # nopep8 + x1[is_adv] = im2[is_adv] + (x1[is_adv] - im2[is_adv]) * self.beta # nopep8 + + if self.las: + adv = self.linear_approximation_search(im2, la2, adv, 3) + + adv_c[pred1] = adv + return adv_c + + def get_diff_logits_grads_batch(self, images, labels, target_labels=None): + images = images.clone().detach().requires_grad_() # make sure its was leaf node + # print(images.is_leaf) + + if not self.targeted: + logits = self.get_logits(images) + g2 = self.compute_jacobian(images, logits) + y2 = logits + df = y2 - torch.unsqueeze(y2[torch.arange(images.shape[0]), labels], 1) # nopep8 + dg = g2 - torch.unsqueeze(g2[torch.arange(images.shape[0]), labels], 1) # nopep8 + df[torch.arange(images.shape[0]), labels] = 1e10 + else: + u = torch.arange(images.shape[0]) + logits = self.get_logits(images) + diff_logits = -(logits[u, labels] - logits[u, target_labels]) + sum_diff = torch.sum(diff_logits) + + # jacobian + self.zero_gradients(images) + sum_diff.backward() + grad_diff = images.grad.data + df = torch.unsqueeze(diff_logits.detach(), 1) + dg = torch.unsqueeze(grad_diff, 1) + + return df, dg + + def compute_jacobian(self, images, logits): + num_classes = logits.shape[1] + jacobian = torch.zeros(num_classes, *images.size()).to(self.device) + grad_output = torch.zeros_like(logits).to(self.device) + + for i in range(num_classes): + self.zero_gradients(images) + grad_output.zero_() + grad_output[:, i] = 1 + logits.backward(grad_output, retain_graph=True) + jacobian[i] = images.grad.data + + return torch.transpose(jacobian, dim0=0, dim1=1) + + def zero_gradients(self, x): + if x.grad is not None: + x.grad.detach_() + x.grad.zero_() + + def projection_l2(self, t2, w2, b2): + t, w, b = t2, w2.clone(), b2 + + c = torch.sum(w * t, 1) - b + ind2 = 2 * (c >= 0) - 1 + w = torch.mul(w, torch.unsqueeze(ind2, 1)) + c = torch.mul(c, ind2) + + r = torch.clamp(torch.max(t / w, (t - 1) / w), min=-1e12, max=1e12) + r[torch.abs(w) < 1e-8] = 1e12 + r[r == -1e12] *= -1 + rs, indr = torch.sort(r, dim=1) + rs2 = F.pad(rs[:, 1:], (0, 1)) + rs[rs == 1e12] = 0 + rs2[rs2 == 1e12] = 0 + + w3s = (w ** 2).gather(1, indr) + w5 = w3s.sum(dim=1, keepdim=True) + ws = w5 - torch.cumsum(w3s, dim=1) + d = -(r * w) + d = torch.mul(d, (torch.abs(w) > 1e-8).float()) + temp_var_1 = -w5 * rs[:, 0:1] + temp_var_2 = torch.cumsum((-rs2 + rs) * ws, dim=1) - w5 * rs[:, 0:1] + s = torch.cat((temp_var_1, temp_var_2), 1) + + c4 = s[:, 0] + c < 0 + c3 = torch.sum(d * w, 1) + c > 0 + c2 = ~(c4 | c3) + + lb = torch.zeros(c2.sum(), device=self.device) + ub = torch.full_like(lb, w.shape[1] - 1) + nitermax = torch.ceil(torch.log2(torch.tensor(s.shape[1]).float())) + + s_, c_ = s[c2], c[c2] + for _ in range(int(nitermax.item())): + counter4 = torch.floor((lb + ub) / 2) + counter2 = counter4.long().unsqueeze(1) + c3 = s_.gather(1, counter2).squeeze(1) + c_ > 0 + lb = torch.where(c3, counter4, lb) + ub = torch.where(c3, ub, counter4) + + lb = lb.long() + + if c4.any(): + alpha = c[c4] / w5[c4].squeeze(-1) + d[c4] = -alpha.unsqueeze(-1) * w[c4] + + if c2.any(): + alpha = (s[c2, lb] + c[c2]) / ws[c2, lb] + rs[c2, lb] + alpha[ws[c2, lb] == 0] = 0 + c5 = (alpha.unsqueeze(-1) > r[c2]).float() + d[c2] = d[c2] * c5 - alpha.unsqueeze(-1) * w[c2] * (1 - c5) + + return d * (w != 0).type(torch.FloatTensor).to(self.device) + + def linear_approximation_search(self, clean_images, clean_labels, adv_images, niter): + a1 = clean_images.clone() + a2 = adv_images.clone() + u = torch.arange(clean_images.shape[0]) + y1 = self.get_logits(a1) + y2 = self.get_logits(a2) + la2 = torch.argmax(y2, 1) + + for _ in range(niter): + t1 = (y1[u, clean_labels] - y1[u, la2]).reshape([-1, 1, 1, 1]) + t2 = (-(y2[u, clean_labels] - y2[u, la2])).reshape([-1, 1, 1, 1]) + + t3 = t1 / (t1 + t2 + 1e-10) + c3 = torch.logical_and(0.0 <= t3, t3 <= 1.0) + t3[~c3] = 1.0 + + a3 = a1 * (1.0 - t3) + a2 * t3 + y3 = self.get_logits(a3) + la3 = torch.argmax(y3, 1) + pred = la3 == clean_labels + + y1[pred] = y3[pred] + 0 + a1[pred] = a3[pred] + 0 + y2[~pred] = y3[~pred] + 0 + la2[~pred] = la3[~pred] + 0 + a2[~pred] = a3[~pred] + 0 + + return a2 diff --git a/torchattacks/attacks/jsma.py b/torchattacks/attacks/jsma.py index 72f86e9f..62dae7c4 100644 --- a/torchattacks/attacks/jsma.py +++ b/torchattacks/attacks/jsma.py @@ -13,8 +13,9 @@ class JSMA(Attack): Arguments: model (nn.Module): model to attack. - theta (float): perturb length, range is either [theta, 0], [0, theta]. (Default: 1.0) - gamma (float): highest percentage of pixels can be modified. (Default: 0.1) + theta (float): the change made to pixels. (Default: 1.0) + gamma (float): the maximum distortion. (Default: 0.1) + increasing (bool): crafting perturbation by increasing or decreasing pixel intensities. (Default: True) Shape: - images: :math:`(N, C, H, W)` where `N = number of batches`, `C = number of channels`, `H = height` and `W = width`. It must have a range [0, 1]. @@ -22,15 +23,16 @@ class JSMA(Attack): - output: :math:`(N, C, H, W)`. Examples:: - >>> attack = torchattacks.JSMA(model, theta=1.0, gamma=0.1) + >>> attack = torchattacks.JSMA(model, theta=1.0, gamma=0.1, increasing=True) >>> adv_images = attack(images, labels) """ - def __init__(self, model, theta=1.0, gamma=0.1): + def __init__(self, model, theta=1.0, gamma=0.1, increasing=True): super().__init__("JSMA", model) self.theta = theta self.gamma = gamma + self.increasing = increasing self.supported_mode = ["default", "targeted"] def forward(self, images, labels): @@ -40,6 +42,8 @@ def forward(self, images, labels): images = images.clone().detach().to(self.device) labels = labels.clone().detach().to(self.device) + # only predict one image + class_num = self.get_logits(torch.unsqueeze(images[0], 0)).shape[1] if self.targeted: target_labels = self.get_target_label(images, labels) @@ -49,163 +53,115 @@ def forward(self, images, labels): # (we have no control over the convergence of the attack to a data point that is NOT equal to the original class), # so we make the default setting of the target label is right circular shift # to make attack work if user didn't set target label. - target_labels = (labels + 1) % 10 - - adv_images = None - for im, tl in zip(images, target_labels): - # Since the attack uses the Jacobian-matrix, - # if we input a large number of images directly into it, - # the processing will be very complicated, - # here, in order to simplify the processing, - # we only process one image at a time. - # Shape of MNIST is [-1, 1, 28, 28], - # and shape of CIFAR10 is [-1, 3, 32, 32]. - pert_image = self.perturbation_single( - torch.unsqueeze(im, 0), torch.unsqueeze(tl, 0) - ) - try: - adv_images = torch.cat((adv_images, pert_image), 0) - except Exception: - adv_images = pert_image + target_labels = (labels + 1) % class_num + + adv_images = images + dim_x = int(np.prod(images.shape[1:])) + max_iter = int(dim_x * self.gamma / 2) + search_space = torch.ones(images.shape[0], dim_x).to(self.device) + adv_prediction = torch.argmax(self.get_logits(adv_images), 1) + + # Algorithm 2 + i = 0 + while torch.sum(adv_prediction != target_labels) != 0 and i < max_iter and torch.sum(search_space != 0) != 0: + grads_target, grads_other = self.compute_forward_derivative( + adv_images, target_labels, class_num) + p1, p2 = self.saliency_map( + search_space, grads_target, grads_other, target_labels) + + cond = (adv_prediction != target_labels) + self.update_search_space(search_space, p1, p2, cond) + adv_images = self.update_adv_images(adv_images, p1, p2, cond) + adv_prediction = torch.argmax(self.get_logits(adv_images), 1) + i += 1 adv_images = torch.clamp(adv_images, min=0, max=1) return adv_images - def compute_jacobian(self, image): - var_image = image.clone().detach() - var_image.requires_grad = True - output = self.get_logits(var_image) - - num_features = int(np.prod(var_image.shape[1:])) - jacobian = torch.zeros([output.shape[1], num_features]) - for i in range(output.shape[1]): - if var_image.grad is not None: - var_image.grad.zero_() - output[0][i].backward(retain_graph=True) - # Copy the derivative to the target place - jacobian[i] = ( - var_image.grad.squeeze().view(-1, num_features).clone() - ) # nopep8 - - return jacobian.to(self.device) - - @torch.no_grad() - def saliency_map( - self, jacobian, target_label, increasing, search_space, nb_features - ): - # The search domain - domain = torch.eq(search_space, 1).float() - # The sum of all features' derivative with respect to each class - all_sum = torch.sum(jacobian, dim=0, keepdim=True) - # The forward derivative of the target class - target_grad = jacobian[target_label] - # The sum of forward derivative of other classes - others_grad = all_sum - target_grad - - # This list blanks out those that are not in the search domain - if increasing: - increase_coef = 2 * (torch.eq(domain, 0)).float().to(self.device) - else: - increase_coef = -1 * 2 * (torch.eq(domain, 0)).float().to(self.device) - increase_coef = increase_coef.view(-1, nb_features) - - # Calculate sum of target forward derivative of any 2 features. - target_tmp = target_grad.clone() - target_tmp -= increase_coef * torch.max(torch.abs(target_grad)) - # PyTorch will automatically extend the dimensions - alpha = target_tmp.view(-1, 1, nb_features) + target_tmp.view( - -1, nb_features, 1 - ) - # Calculate sum of other forward derivative of any 2 features. - others_tmp = others_grad.clone() - others_tmp += increase_coef * torch.max(torch.abs(others_grad)) - beta = others_tmp.view(-1, 1, nb_features) + others_tmp.view(-1, nb_features, 1) - - # Zero out the situation where a feature sums with itself - tmp = np.ones((nb_features, nb_features), int) - np.fill_diagonal(tmp, 0) - zero_diagonal = torch.from_numpy(tmp).byte().to(self.device) - - # According to the definition of saliency map in the paper (formulas 8 and 9), - # those elements in the saliency map that doesn't satisfy the requirement will be blanked out. - if increasing: - mask1 = torch.gt(alpha, 0.0) - mask2 = torch.lt(beta, 0.0) - else: - mask1 = torch.lt(alpha, 0.0) - mask2 = torch.gt(beta, 0.0) - - # Apply the mask to the saliency map - mask = torch.mul(torch.mul(mask1, mask2), zero_diagonal.view_as(mask1)) - # Do the multiplication according to formula 10 in the paper - saliency_map = torch.mul(torch.mul(alpha, torch.abs(beta)), mask.float()) - # Get the most significant two pixels - max_idx = torch.argmax(saliency_map.view(-1, nb_features * nb_features), dim=1) - # p = max_idx // nb_features - p = torch.div(max_idx, nb_features, rounding_mode="floor") - # q = max_idx % nb_features - q = max_idx - p * nb_features - return p, q - - def perturbation_single(self, image, target_label): - """ - image: only one element - label: only one element - """ - var_image = image - var_label = target_label - var_image = var_image.to(self.device) - var_label = var_label.to(self.device) - - if self.theta > 0: - increasing = True - else: - increasing = False - - num_features = int(np.prod(var_image.shape[1:])) - shape = var_image.shape + def update_adv_images(self, adv_images, p1, p2, cond): + origin_shape = adv_images.shape + adv_images = adv_images.view(adv_images.shape[0], -1) + for idx in range(adv_images.shape[0]): + if cond[idx]: + if self.increasing: + # Section IV, A + adv_images[idx, p1[idx]] += self.theta + adv_images[idx, p2[idx]] += self.theta + else: + # Section IV, B + adv_images[idx, p1[idx]] -= self.theta + adv_images[idx, p2[idx]] -= self.theta - # Perturb two pixels in one iteration, thus max_iters is divided by 2 - max_iters = int(np.ceil(num_features * self.gamma / 2.0)) + adv_images = torch.clamp(adv_images, min=0, max=1) + adv_images = adv_images.view(origin_shape) + return adv_images - # Masked search domain, if the pixel has already reached the top or bottom, we don't bother to modify it - if increasing: - search_domain = torch.lt(var_image, 0.99) + def update_search_space(self, search_space, p1, p2, cond): + # Algorithm 2 line 10 and line 11 + p1_cond = torch.logical_or(p1 == 0, p1 == 1) + p2_cond = torch.logical_or(p2 == 0, p2 == 1) + + # Early stop + p1_cond = torch.logical_or(p1_cond, cond) + p2_cond = torch.logical_or(p2_cond, cond) + + for ind in range(search_space.shape[0]): + if p1_cond[ind]: + search_space[ind, p1[ind]] = False + if p2_cond[ind]: + search_space[ind, p2[ind]] = False + + def jacobian(self, adv_images, class_num): + tmp_images = adv_images.detach().clone() + tmp_images.requires_grad = True + jacobians = [] + output = self.get_logits(tmp_images) + + for n in range(class_num): + if tmp_images.grad is not None: + tmp_images.grad.zero_() + torch.sum(output[:, n]).backward(retain_graph=True) + grad = tmp_images.grad.detach().clone() + jacobians.append(grad) + + jacobians = torch.stack(jacobians, 0) + return jacobians + + def compute_forward_derivative(self, adv_images, target_labels, class_num): + jacobians = self.jacobian(adv_images, class_num) + grads = jacobians.view((jacobians.shape[0], jacobians.shape[1], -1)) + grads_target = grads[target_labels, range(len(target_labels)), :] + grads_other = grads.sum(dim=0) - grads_target + return grads_target, grads_other + + def sum_pair(self, grads, dim): + # Eq 8 and Eq 9 + return grads.view(-1, dim, 1) + grads.view(-1, 1, dim) + + def and_pair(self, cond, dim): + return cond.view(-1, dim, 1) & cond.view(-1, 1, dim) + + def saliency_map(self, search_space, grads_target, grads_other, y): + dim = search_space.shape[1] + # alpha in Algorithm 3 line 2 + gradsum_target = self.sum_pair(grads_target, dim) + # beta in Algorithm 3 line 3 + gradsum_other = self.sum_pair(grads_other, dim) + + # Algorithm 3 line 4 + if self.increasing: + scores_mask = torch.logical_and( + gradsum_target > 0, gradsum_other < 0) else: - search_domain = torch.gt(var_image, 0.01) - - search_domain = search_domain.view(num_features) - output = self.get_logits(var_image) - current_pred = torch.argmax(output.data, 1) - - iter = 0 - while ( - (iter < max_iters) - and (current_pred != target_label) - and (search_domain.sum() != 0) - ): - # Calculate Jacobian matrix of forward derivative - jacobian = self.compute_jacobian(var_image) - # Get the saliency map and calculate the two pixels that have the greatest influence - p1, p2 = self.saliency_map( - jacobian, var_label, increasing, search_domain, num_features - ) - # Apply modifications - # var_sample_flatten = var_image.view(-1, num_features).clone().detach_() - var_sample_flatten = var_image.view(-1, num_features) - var_sample_flatten[0, p1] += self.theta - var_sample_flatten[0, p2] += self.theta - - new_image = torch.clamp(var_sample_flatten, min=0.0, max=1.0) - new_image = new_image.view(shape) - search_domain[p1] = 0 - search_domain[p2] = 0 - # var_image = new_image.clone().detach().to(self.device) - var_image = new_image.to(self.device) - - output = self.get_logits(var_image) - current_pred = torch.argmax(output.data, 1) - iter += 1 - - adv_image = var_image - return adv_image + scores_mask = torch.logical_and( + gradsum_target < 0, gradsum_other > 0) + + search_space_mask = self.and_pair(search_space != 0, dim) + scores_mask = torch.logical_and(scores_mask, search_space_mask) + scores_mask[:, range(dim), range(dim)] = 0 + scores = scores_mask.float() * (-1 * gradsum_target * gradsum_other) + best_indices = torch.argmax(scores.view(-1, dim * dim), 1) + + p1 = torch.remainder(best_indices, dim) + p2 = ((best_indices - p1) / dim).to(torch.long) + return p1, p2 diff --git a/torchattacks/attacks/mifgsm.py b/torchattacks/attacks/mifgsm.py index fc65d191..4d2baa6d 100644 --- a/torchattacks/attacks/mifgsm.py +++ b/torchattacks/attacks/mifgsm.py @@ -48,14 +48,13 @@ def forward(self, images, labels): if self.targeted: target_labels = self.get_target_label(images, labels) - momentum = torch.zeros_like(images).detach().to(self.device) loss = nn.CrossEntropyLoss() - adv_images = images.clone().detach() + adv_images.requires_grad = True + momentum = torch.zeros_like(images).detach().to(self.device) for _ in range(self.steps): - adv_images.requires_grad = True outputs = self.get_logits(adv_images) # Calculate loss @@ -70,11 +69,10 @@ def forward(self, images, labels): )[0] grad = grad / torch.mean(torch.abs(grad), dim=(1, 2, 3), keepdim=True) - grad = grad + momentum * self.decay - momentum = grad + momentum = self.decay * momentum + grad - adv_images = adv_images.detach() + self.alpha * grad.sign() + adv_images = adv_images + self.alpha * torch.sign(momentum) delta = torch.clamp(adv_images - images, min=-self.eps, max=self.eps) - adv_images = torch.clamp(images + delta, min=0, max=1).detach() + adv_images = torch.clamp(images + delta, min=0, max=1) return adv_images