Copy_of_cifar10_mobilenet.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Copy of cifar10_mobilenet.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/amina01/Learning-Neural-Activations/blob/master/Copy_of_cifar10_mobilenet.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nMOLTySJyerG",
        "colab_type": "code",
        "outputId": "2602516f-41f0-4393-ce68-c70a8948550c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "# -*- coding: utf-8 -*-\n",
        "\"\"\"\n",
        "\n",
        "MobileNetv2 code taken from https://github.com/digantamisra98/Mish/blob/master/Notebooks/cifar-10-mobilenetv2-mish.ipynb\n",
        "\n",
        "\"\"\"\n",
        "\n",
        "import numpy as np # linear algebra\n",
        "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
        "\n",
        "import os\n",
        "#print(os.listdir(\"../input\"))\n",
        "\n",
        "import time\n",
        "\n",
        "# import pytorch\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torch.optim import SGD,Adam,lr_scheduler\n",
        "from torch.utils.data import random_split\n",
        "import torchvision\n",
        "from torchvision import transforms, datasets\n",
        "from torch.utils.data import DataLoader\n",
        "\n",
        "\n",
        "\n",
        "train_transform = transforms.Compose([\n",
        "    transforms.RandomHorizontalFlip(p=.40),\n",
        "    transforms.RandomRotation(30),\n",
        "    transforms.ToTensor(),\n",
        "    transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])])\n",
        "\n",
        "# define transformations for test\n",
        "test_transform = transforms.Compose([\n",
        "    transforms.ToTensor(),\n",
        "    transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])])\n",
        "\n",
        "# define training dataloader\n",
        "def get_training_dataloader(train_transform, batch_size=128, num_workers=0, shuffle=True):\n",
        "    \"\"\" return training dataloader\n",
        "    Args:\n",
        "        train_transform: transfroms for train dataset\n",
        "        path: path to cifar100 training python dataset\n",
        "        batch_size: dataloader batchsize\n",
        "        num_workers: dataloader num_works\n",
        "        shuffle: whether to shuffle \n",
        "    Returns: train_data_loader:torch dataloader object\n",
        "    \"\"\"\n",
        "\n",
        "    transform_train = train_transform\n",
        "    cifar10_training = torchvision.datasets.CIFAR10(root='.', train=True, download=True, transform=transform_train)\n",
        "    cifar10_training_loader = DataLoader(\n",
        "        cifar10_training, shuffle=shuffle, num_workers=num_workers, batch_size=batch_size)\n",
        "\n",
        "    return cifar10_training_loader\n",
        "\n",
        "# define test dataloader\n",
        "def get_testing_dataloader(test_transform, batch_size=128, num_workers=0, shuffle=True):\n",
        "    \"\"\" return training dataloader\n",
        "    Args:\n",
        "        test_transform: transforms for test dataset\n",
        "        path: path to cifar100 test python dataset\n",
        "        batch_size: dataloader batchsize\n",
        "        num_workers: dataloader num_works\n",
        "        shuffle: whether to shuffle \n",
        "    Returns: cifar100_test_loader:torch dataloader object\n",
        "    \"\"\"\n",
        "\n",
        "    transform_test = test_transform\n",
        "    cifar10_test = torchvision.datasets.CIFAR10(root='.', train=False, download=True, transform=transform_test)\n",
        "    cifar10_test_loader = DataLoader(\n",
        "        cifar10_test, shuffle=shuffle, num_workers=num_workers, batch_size=batch_size)\n",
        "\n",
        "    return cifar10_test_loader\n",
        "\n",
        "\n",
        "\n",
        "# Define AFU\n",
        "class AF(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(AF, self).__init__()\n",
        "        self.fc1 = nn.Linear(1, 8)\n",
        "        self.fc2 = nn.Linear(8, 1)\n",
        "        \n",
        "    def forward(self, x):\n",
        "        \n",
        "\n",
        "        z = x.shape\n",
        "        x = x.flatten().unsqueeze(1)\n",
        "        \n",
        "        x = self.fc1(x)\n",
        "        x = torch.relu(x)\n",
        "        x = self.fc2(x)\n",
        "        \n",
        "        return x.view(z)\n",
        "\n",
        "\n",
        "\n",
        "def f_mish(input):\n",
        "    '''\n",
        "    Applies the mish function element-wise:\n",
        "    mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + exp(x)))\n",
        "    '''\n",
        "    return input * torch.tanh(F.softplus(input))\n",
        "\n",
        "# implement class wrapper for mish activation function\n",
        "class mish(nn.Module):\n",
        "    '''\n",
        "    Applies the mish function element-wise:\n",
        "    mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + exp(x)))\n",
        "\n",
        "    Shape:\n",
        "        - Input: (N, *) where * means, any number of additional\n",
        "          dimensions\n",
        "        - Output: (N, *), same shape as the input\n",
        "\n",
        "    Examples:\n",
        "        >>> m = mish()\n",
        "        >>> input = torch.randn(2)\n",
        "        >>> output = m(input)\n",
        "\n",
        "    '''\n",
        "    def __init__(self):\n",
        "        '''\n",
        "        Init method.\n",
        "        '''\n",
        "        super().__init__()\n",
        "\n",
        "    def forward(self, input):\n",
        "        '''\n",
        "        Forward pass of the function.\n",
        "        '''\n",
        "        return f_mish(input)\n",
        "    \n",
        "    \n",
        "    \n",
        "    \n",
        "# implement swish activation function\n",
        "def f_swish(input):\n",
        "    '''\n",
        "    Applies the swish function element-wise:\n",
        "    swish(x) = x * sigmoid(x)\n",
        "    '''\n",
        "    return input * torch.sigmoid(input)\n",
        "\n",
        "# implement class wrapper for swish activation function\n",
        "class swish(nn.Module):\n",
        "    '''\n",
        "    Applies the swish function element-wise:\n",
        "    swish(x) = x * sigmoid(x)\n",
        "\n",
        "    Shape:\n",
        "        - Input: (N, *) where * means, any number of additional\n",
        "          dimensions\n",
        "        - Output: (N, *), same shape as the input\n",
        "\n",
        "    Examples:\n",
        "        >>> m = swish()\n",
        "        >>> input = torch.randn(2)\n",
        "        >>> output = m(input)\n",
        "\n",
        "    '''\n",
        "    def __init__(self):\n",
        "        '''\n",
        "        Init method.\n",
        "        '''\n",
        "        super().__init__()\n",
        "\n",
        "    def forward(self, input):\n",
        "        '''\n",
        "        Forward pass of the function.\n",
        "        '''\n",
        "        return f_swish(input)\n",
        "    \n",
        "    \n",
        "    \n",
        "    \n",
        "class DepthSeperabelConv2d(nn.Module):\n",
        "\n",
        "    def __init__(self, input_channels, output_channels, kernel_size, activation = 'relu', **kwargs):\n",
        "        super().__init__()\n",
        "        \n",
        "        \n",
        "        if (activation == 'relu'):\n",
        "            f_activation =  nn.ReLU(inplace=True)\n",
        "            \n",
        "        if (activation == 'swish'):\n",
        "            f_activation =  swish()\n",
        "            \n",
        "        if (activation == 'mish'):\n",
        "            f_activation =  mish()\n",
        "        if (activation == 'AF'):\n",
        "            f_activation =  AF()\n",
        "        \n",
        "        self.depthwise = nn.Sequential(\n",
        "            nn.Conv2d(\n",
        "                input_channels,\n",
        "                input_channels,\n",
        "                kernel_size,\n",
        "                groups=input_channels,\n",
        "                **kwargs),\n",
        "            nn.BatchNorm2d(input_channels),\n",
        "            f_activation\n",
        "        )\n",
        "\n",
        "        self.pointwise = nn.Sequential(\n",
        "            nn.Conv2d(input_channels, output_channels, 1),\n",
        "            nn.BatchNorm2d(output_channels),\n",
        "            f_activation\n",
        "        )\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.depthwise(x)\n",
        "        x = self.pointwise(x)\n",
        "\n",
        "        return x\n",
        "    \n",
        "    \n",
        "class BasicConv2d(nn.Module):\n",
        "\n",
        "    def __init__(self, input_channels, output_channels, kernel_size, activation = 'relu', **kwargs):\n",
        "\n",
        "        super().__init__()\n",
        "        self.conv = nn.Conv2d(\n",
        "            input_channels, output_channels, kernel_size, **kwargs)\n",
        "        self.bn = nn.BatchNorm2d(output_channels)\n",
        "        \n",
        "        if (activation == 'relu'):\n",
        "            self.activation =  nn.ReLU(inplace=True)\n",
        "            \n",
        "        if (activation == 'swish'):\n",
        "            self.activation =  swish()\n",
        "            \n",
        "        if (activation == 'mish'):\n",
        "            self.activation =  mish()\n",
        "        if (activation == 'AF'):\n",
        "            self.activation =  AF()\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.conv(x)\n",
        "        x = self.bn(x)\n",
        "        x = self.activation(x)\n",
        "\n",
        "        return x\n",
        "    \n",
        "    \n",
        "    \n",
        "\n",
        "\n",
        "class MobileNet(nn.Module):\n",
        "\n",
        "    \"\"\"\n",
        "    Args:\n",
        "        width multipler: The role of the width multiplier α is to thin \n",
        "                         a network uniformly at each layer. For a given \n",
        "                         layer and width multiplier α, the number of \n",
        "                         input channels M becomes αM and the number of \n",
        "                         output channels N becomes αN.\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, width_multiplier=1, class_num=100, activation = 'relu'):\n",
        "        super().__init__()\n",
        "        \n",
        "        alpha = width_multiplier\n",
        "        self.stem = nn.Sequential(\n",
        "           BasicConv2d(3, int(32 * alpha), 3, padding=1, bias=False, activation = activation),\n",
        "           DepthSeperabelConv2d(\n",
        "               int(32 * alpha),\n",
        "               int(64 * alpha),\n",
        "               3,\n",
        "               padding=1,\n",
        "               bias=False\n",
        "           )\n",
        "       )\n",
        "        self.conv1 = nn.Sequential(\n",
        "           DepthSeperabelConv2d(\n",
        "               int(64 * alpha),\n",
        "               int(128 * alpha),\n",
        "               3,\n",
        "               stride=2,\n",
        "               padding=1,\n",
        "               bias=False,\n",
        "               activation = activation\n",
        "           ),\n",
        "           DepthSeperabelConv2d(\n",
        "               int(128 * alpha),\n",
        "               int(128 * alpha),\n",
        "               3,\n",
        "               padding=1,\n",
        "               bias=False,\n",
        "               activation = activation\n",
        "           )\n",
        "       )\n",
        "        self.conv2 = nn.Sequential(\n",
        "           DepthSeperabelConv2d(\n",
        "               int(128 * alpha),\n",
        "               int(256 * alpha),\n",
        "               3,\n",
        "               stride=2,\n",
        "               padding=1,\n",
        "               bias=False,\n",
        "               activation = activation\n",
        "           ),\n",
        "           DepthSeperabelConv2d(\n",
        "               int(256 * alpha),\n",
        "               int(256 * alpha),\n",
        "               3,\n",
        "               padding=1,\n",
        "               bias=False,\n",
        "               activation = activation\n",
        "           )\n",
        "       )\n",
        "        self.conv3 = nn.Sequential(\n",
        "           DepthSeperabelConv2d(\n",
        "               int(256 * alpha),\n",
        "               int(512 * alpha),\n",
        "               3,\n",
        "               stride=2,\n",
        "               padding=1,\n",
        "               bias=False,\n",
        "               activation = activation\n",
        "           ),\n",
        "\n",
        "           DepthSeperabelConv2d(\n",
        "               int(512 * alpha),\n",
        "               int(512 * alpha),\n",
        "               3,\n",
        "               padding=1,\n",
        "               bias=False,\n",
        "               activation = activation\n",
        "           ),\n",
        "           DepthSeperabelConv2d(\n",
        "               int(512 * alpha),\n",
        "               int(512 * alpha),\n",
        "               3,\n",
        "               padding=1,\n",
        "               bias=False,\n",
        "               activation = activation\n",
        "           ),\n",
        "           DepthSeperabelConv2d(\n",
        "               int(512 * alpha),\n",
        "               int(512 * alpha),\n",
        "               3,\n",
        "               padding=1,\n",
        "               bias=False,\n",
        "               activation = activation\n",
        "           ),\n",
        "           DepthSeperabelConv2d(\n",
        "               int(512 * alpha),\n",
        "               int(512 * alpha),\n",
        "               3,\n",
        "               padding=1,\n",
        "               bias=False,\n",
        "               activation = activation\n",
        "           ),\n",
        "           DepthSeperabelConv2d(\n",
        "               int(512 * alpha),\n",
        "               int(512 * alpha),\n",
        "               3,\n",
        "               padding=1,\n",
        "               bias=False,\n",
        "               activation = activation\n",
        "           )\n",
        "       )\n",
        "        self.conv4 = nn.Sequential(\n",
        "           DepthSeperabelConv2d(\n",
        "               int(512 * alpha),\n",
        "               int(1024 * alpha),\n",
        "               3,\n",
        "               stride=2,\n",
        "               padding=1,\n",
        "               bias=False,\n",
        "               activation = activation\n",
        "           ),\n",
        "           DepthSeperabelConv2d(\n",
        "               int(1024 * alpha),\n",
        "               int(1024 * alpha),\n",
        "               3,\n",
        "               padding=1,\n",
        "               bias=False,\n",
        "               activation = activation\n",
        "           )\n",
        "       )\n",
        "        self.fc = nn.Linear(int(1024 * alpha), class_num)\n",
        "        self.avg = nn.AdaptiveAvgPool2d(1)\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.stem(x)\n",
        "\n",
        "        x = self.conv1(x)\n",
        "        x = self.conv2(x)\n",
        "        x = self.conv3(x)\n",
        "        x = self.conv4(x)\n",
        "\n",
        "        x = self.avg(x)\n",
        "        x = x.view(x.size(0), -1)\n",
        "        x = self.fc(x)\n",
        "        return x\n",
        "\n",
        "\n",
        "def mobilenet(alpha=1, class_num=10, activation = 'relu'):\n",
        "    return MobileNet(alpha, class_num, activation)\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "trainloader = get_training_dataloader(train_transform)\n",
        "testloader = get_testing_dataloader(test_transform)\n",
        "\n",
        "\n",
        "\n",
        "epochs = 100\n",
        "batch_size = 128\n",
        "learning_rate = 0.001\n",
        "device = torch.device('cuda:0' if torch.cuda.is_available() else \"cpu\")\n",
        "\n",
        "\n",
        "\n",
        "model = mobilenet(activation = 'AF')\n",
        "\n",
        "\n",
        "# set loss function\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "\n",
        "# set optimizer, only train the classifier parameters, feature parameters are frozen\n",
        "optimizer = Adam(model.parameters(), lr=learning_rate)\n",
        "\n",
        "\n",
        "\n",
        "train_stats = pd.DataFrame(columns = ['Epoch', 'Time per epoch', 'Avg time per step', 'Train loss', 'Train accuracy', 'Train top-3 accuracy','Test loss', 'Test accuracy', 'Test top-3 accuracy'])\n",
        "\n",
        "\n",
        "model.to(device)\n",
        "\n",
        "\n",
        "steps = 0\n",
        "running_loss = 0\n",
        "for epoch in range(epochs):\n",
        "    \n",
        "    since = time.time()\n",
        "    \n",
        "    train_accuracy = 0\n",
        "    top3_train_accuracy = 0 \n",
        "    for inputs, labels in trainloader:\n",
        "        steps += 1\n",
        "        # Move input and label tensors to the default device\n",
        "        inputs, labels = inputs.to(device), labels.to(device)\n",
        "        \n",
        "        optimizer.zero_grad()\n",
        "        \n",
        "        logps = model.forward(inputs)\n",
        "        loss = criterion(logps, labels)\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        running_loss += loss.item()\n",
        "        \n",
        "        # calculate train top-1 accuracy\n",
        "        ps = torch.exp(logps)\n",
        "        top_p, top_class = ps.topk(1, dim=1)\n",
        "        equals = top_class == labels.view(*top_class.shape)\n",
        "        train_accuracy += torch.mean(equals.type(torch.FloatTensor)).item()\n",
        "        \n",
        "        # Calculate train top-3 accuracy\n",
        "        np_top3_class = ps.topk(3, dim=1)[1].cpu().numpy()\n",
        "        target_numpy = labels.cpu().numpy()\n",
        "        top3_train_accuracy += np.mean([1 if target_numpy[i] in np_top3_class[i] else 0 for i in range(0, len(target_numpy))])\n",
        "        \n",
        "    time_elapsed = time.time() - since\n",
        "    \n",
        "    test_loss = 0\n",
        "    test_accuracy = 0\n",
        "    top3_test_accuracy = 0\n",
        "    model.eval()\n",
        "    with torch.no_grad():\n",
        "        for inputs, labels in testloader:\n",
        "            inputs, labels = inputs.to(device), labels.to(device)\n",
        "            logps = model.forward(inputs)\n",
        "            batch_loss = criterion(logps, labels)\n",
        "\n",
        "            test_loss += batch_loss.item()\n",
        "\n",
        "            # Calculate test top-1 accuracy\n",
        "            ps = torch.exp(logps)\n",
        "            top_p, top_class = ps.topk(1, dim=1)\n",
        "            equals = top_class == labels.view(*top_class.shape)\n",
        "            test_accuracy += torch.mean(equals.type(torch.FloatTensor)).item()\n",
        "            \n",
        "            # Calculate test top-3 accuracy\n",
        "            np_top3_class = ps.topk(3, dim=1)[1].cpu().numpy()\n",
        "            target_numpy = labels.cpu().numpy()\n",
        "            top3_test_accuracy += np.mean([1 if target_numpy[i] in np_top3_class[i] else 0 for i in range(0, len(target_numpy))])\n",
        "\n",
        "    print(f\"Epoch {epoch+1}/{epochs}.. \"\n",
        "          f\"Time per epoch: {time_elapsed:.4f}.. \"\n",
        "          f\"Average time per step: {time_elapsed/len(trainloader):.4f}.. \"\n",
        "          f\"Train loss: {running_loss/len(trainloader):.4f}.. \"\n",
        "          f\"Train accuracy: {train_accuracy/len(trainloader):.4f}.. \"\n",
        "          f\"Top-3 train accuracy: {top3_train_accuracy/len(trainloader):.4f}.. \"\n",
        "          f\"Test loss: {test_loss/len(testloader):.4f}.. \"\n",
        "          f\"Test accuracy: {test_accuracy/len(testloader):.4f}.. \"\n",
        "          f\"Top-3 test accuracy: {top3_test_accuracy/len(testloader):.4f}\")\n",
        "\n",
        "    train_stats = train_stats.append({'Epoch': epoch, 'Time per epoch':time_elapsed, 'Avg time per step': time_elapsed/len(trainloader), 'Train loss' : running_loss/len(trainloader), 'Train accuracy': train_accuracy/len(trainloader), 'Train top-3 accuracy':top3_train_accuracy/len(trainloader),'Test loss' : test_loss/len(testloader), 'Test accuracy': test_accuracy/len(testloader), 'Test top-3 accuracy':top3_test_accuracy/len(testloader)}, ignore_index=True)\n",
        "\n",
        "    running_loss = 0\n",
        "    model.train()\n",
        "    \n",
        "train_stats.to_csv('train_log_MobileNet_Mish.csv')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Files already downloaded and verified\n",
            "Files already downloaded and verified\n",
            "Epoch 1/100.. Time per epoch: 158.4601.. Average time per step: 0.4053.. Train loss: 1.7890.. Train accuracy: 0.3162.. Top-3 train accuracy: 0.6506.. Test loss: 1.6113.. Test accuracy: 0.4273.. Top-3 test accuracy: 0.7505\n",
            "Epoch 2/100.. Time per epoch: 158.1166.. Average time per step: 0.4044.. Train loss: 1.3409.. Train accuracy: 0.5073.. Top-3 train accuracy: 0.8287.. Test loss: 1.2527.. Test accuracy: 0.5546.. Top-3 test accuracy: 0.8512\n",
            "Epoch 3/100.. Time per epoch: 158.2539.. Average time per step: 0.4047.. Train loss: 1.1485.. Train accuracy: 0.5864.. Top-3 train accuracy: 0.8697.. Test loss: 1.0447.. Test accuracy: 0.6221.. Top-3 test accuracy: 0.8918\n",
            "Epoch 4/100.. Time per epoch: 158.3521.. Average time per step: 0.4050.. Train loss: 1.0086.. Train accuracy: 0.6403.. Top-3 train accuracy: 0.8986.. Test loss: 0.9281.. Test accuracy: 0.6745.. Top-3 test accuracy: 0.9125\n",
            "Epoch 5/100.. Time per epoch: 158.3686.. Average time per step: 0.4050.. Train loss: 0.9013.. Train accuracy: 0.6801.. Top-3 train accuracy: 0.9134.. Test loss: 0.8653.. Test accuracy: 0.6980.. Top-3 test accuracy: 0.9183\n",
            "Epoch 6/100.. Time per epoch: 158.4382.. Average time per step: 0.4052.. Train loss: 0.8243.. Train accuracy: 0.7122.. Top-3 train accuracy: 0.9259.. Test loss: 0.7707.. Test accuracy: 0.7341.. Top-3 test accuracy: 0.9314\n",
            "Epoch 7/100.. Time per epoch: 158.5474.. Average time per step: 0.4055.. Train loss: 0.7650.. Train accuracy: 0.7332.. Top-3 train accuracy: 0.9341.. Test loss: 0.7912.. Test accuracy: 0.7291.. Top-3 test accuracy: 0.9254\n",
            "Epoch 8/100.. Time per epoch: 158.6441.. Average time per step: 0.4057.. Train loss: 0.7062.. Train accuracy: 0.7543.. Top-3 train accuracy: 0.9421.. Test loss: 0.6927.. Test accuracy: 0.7604.. Top-3 test accuracy: 0.9408\n",
            "Epoch 9/100.. Time per epoch: 158.7873.. Average time per step: 0.4061.. Train loss: 0.6652.. Train accuracy: 0.7683.. Top-3 train accuracy: 0.9490.. Test loss: 0.6464.. Test accuracy: 0.7756.. Top-3 test accuracy: 0.9505\n",
            "Epoch 10/100.. Time per epoch: 158.7507.. Average time per step: 0.4060.. Train loss: 0.6245.. Train accuracy: 0.7844.. Top-3 train accuracy: 0.9527.. Test loss: 0.6744.. Test accuracy: 0.7756.. Top-3 test accuracy: 0.9437\n",
            "Epoch 11/100.. Time per epoch: 158.9456.. Average time per step: 0.4065.. Train loss: 0.5900.. Train accuracy: 0.7962.. Top-3 train accuracy: 0.9568.. Test loss: 0.6130.. Test accuracy: 0.7921.. Top-3 test accuracy: 0.9513\n",
            "Epoch 12/100.. Time per epoch: 158.9113.. Average time per step: 0.4064.. Train loss: 0.5634.. Train accuracy: 0.8029.. Top-3 train accuracy: 0.9598.. Test loss: 0.6256.. Test accuracy: 0.7915.. Top-3 test accuracy: 0.9492\n",
            "Epoch 13/100.. Time per epoch: 158.6394.. Average time per step: 0.4057.. Train loss: 0.5398.. Train accuracy: 0.8128.. Top-3 train accuracy: 0.9630.. Test loss: 0.5789.. Test accuracy: 0.8064.. Top-3 test accuracy: 0.9543\n",
            "Epoch 14/100.. Time per epoch: 158.7628.. Average time per step: 0.4060.. Train loss: 0.5118.. Train accuracy: 0.8215.. Top-3 train accuracy: 0.9670.. Test loss: 0.6217.. Test accuracy: 0.7903.. Top-3 test accuracy: 0.9519\n",
            "Epoch 15/100.. Time per epoch: 158.6122.. Average time per step: 0.4057.. Train loss: 0.4928.. Train accuracy: 0.8285.. Top-3 train accuracy: 0.9686.. Test loss: 0.5657.. Test accuracy: 0.8085.. Top-3 test accuracy: 0.9565\n",
            "Epoch 16/100.. Time per epoch: 158.6759.. Average time per step: 0.4058.. Train loss: 0.4698.. Train accuracy: 0.8383.. Top-3 train accuracy: 0.9705.. Test loss: 0.5464.. Test accuracy: 0.8132.. Top-3 test accuracy: 0.9600\n",
            "Epoch 17/100.. Time per epoch: 158.7341.. Average time per step: 0.4060.. Train loss: 0.4522.. Train accuracy: 0.8431.. Top-3 train accuracy: 0.9725.. Test loss: 0.5736.. Test accuracy: 0.8071.. Top-3 test accuracy: 0.9579\n",
            "Epoch 18/100.. Time per epoch: 158.6519.. Average time per step: 0.4058.. Train loss: 0.4354.. Train accuracy: 0.8488.. Top-3 train accuracy: 0.9749.. Test loss: 0.5785.. Test accuracy: 0.8061.. Top-3 test accuracy: 0.9560\n",
            "Epoch 19/100.. Time per epoch: 158.7078.. Average time per step: 0.4059.. Train loss: 0.4142.. Train accuracy: 0.8564.. Top-3 train accuracy: 0.9765.. Test loss: 0.6195.. Test accuracy: 0.7958.. Top-3 test accuracy: 0.9577\n",
            "Epoch 20/100.. Time per epoch: 158.9152.. Average time per step: 0.4064.. Train loss: 0.4028.. Train accuracy: 0.8603.. Top-3 train accuracy: 0.9782.. Test loss: 0.5612.. Test accuracy: 0.8173.. Top-3 test accuracy: 0.9616\n",
            "Epoch 21/100.. Time per epoch: 159.0084.. Average time per step: 0.4067.. Train loss: 0.3835.. Train accuracy: 0.8682.. Top-3 train accuracy: 0.9805.. Test loss: 0.5475.. Test accuracy: 0.8194.. Top-3 test accuracy: 0.9625\n",
            "Epoch 22/100.. Time per epoch: 158.7562.. Average time per step: 0.4060.. Train loss: 0.3712.. Train accuracy: 0.8712.. Top-3 train accuracy: 0.9813.. Test loss: 0.5701.. Test accuracy: 0.8109.. Top-3 test accuracy: 0.9577\n",
            "Epoch 23/100.. Time per epoch: 158.7791.. Average time per step: 0.4061.. Train loss: 0.3599.. Train accuracy: 0.8739.. Top-3 train accuracy: 0.9819.. Test loss: 0.5888.. Test accuracy: 0.8095.. Top-3 test accuracy: 0.9578\n",
            "Epoch 24/100.. Time per epoch: 158.7665.. Average time per step: 0.4061.. Train loss: 0.3447.. Train accuracy: 0.8803.. Top-3 train accuracy: 0.9839.. Test loss: 0.5329.. Test accuracy: 0.8310.. Top-3 test accuracy: 0.9637\n",
            "Epoch 25/100.. Time per epoch: 158.6079.. Average time per step: 0.4056.. Train loss: 0.3338.. Train accuracy: 0.8827.. Top-3 train accuracy: 0.9850.. Test loss: 0.5362.. Test accuracy: 0.8275.. Top-3 test accuracy: 0.9617\n",
            "Epoch 26/100.. Time per epoch: 158.6757.. Average time per step: 0.4058.. Train loss: 0.3193.. Train accuracy: 0.8890.. Top-3 train accuracy: 0.9853.. Test loss: 0.5623.. Test accuracy: 0.8259.. Top-3 test accuracy: 0.9595\n",
            "Epoch 27/100.. Time per epoch: 158.5225.. Average time per step: 0.4054.. Train loss: 0.3041.. Train accuracy: 0.8949.. Top-3 train accuracy: 0.9871.. Test loss: 0.6011.. Test accuracy: 0.8166.. Top-3 test accuracy: 0.9572\n",
            "Epoch 28/100.. Time per epoch: 158.6485.. Average time per step: 0.4058.. Train loss: 0.2996.. Train accuracy: 0.8949.. Top-3 train accuracy: 0.9873.. Test loss: 0.5541.. Test accuracy: 0.8220.. Top-3 test accuracy: 0.9575\n",
            "Epoch 29/100.. Time per epoch: 158.8926.. Average time per step: 0.4064.. Train loss: 0.2889.. Train accuracy: 0.8980.. Top-3 train accuracy: 0.9876.. Test loss: 0.5643.. Test accuracy: 0.8230.. Top-3 test accuracy: 0.9592\n",
            "Epoch 30/100.. Time per epoch: 159.0356.. Average time per step: 0.4067.. Train loss: 0.2795.. Train accuracy: 0.9036.. Top-3 train accuracy: 0.9884.. Test loss: 0.5482.. Test accuracy: 0.8318.. Top-3 test accuracy: 0.9617\n",
            "Epoch 31/100.. Time per epoch: 158.9866.. Average time per step: 0.4066.. Train loss: 0.2695.. Train accuracy: 0.9064.. Top-3 train accuracy: 0.9897.. Test loss: 0.5724.. Test accuracy: 0.8308.. Top-3 test accuracy: 0.9626\n",
            "Epoch 32/100.. Time per epoch: 159.1644.. Average time per step: 0.4071.. Train loss: 0.2594.. Train accuracy: 0.9111.. Top-3 train accuracy: 0.9896.. Test loss: 0.5610.. Test accuracy: 0.8293.. Top-3 test accuracy: 0.9621\n",
            "Epoch 33/100.. Time per epoch: 159.0115.. Average time per step: 0.4067.. Train loss: 0.2521.. Train accuracy: 0.9112.. Top-3 train accuracy: 0.9908.. Test loss: 0.5478.. Test accuracy: 0.8307.. Top-3 test accuracy: 0.9640\n",
            "Epoch 34/100.. Time per epoch: 158.9669.. Average time per step: 0.4066.. Train loss: 0.2425.. Train accuracy: 0.9163.. Top-3 train accuracy: 0.9914.. Test loss: 0.6557.. Test accuracy: 0.8218.. Top-3 test accuracy: 0.9597\n",
            "Epoch 35/100.. Time per epoch: 159.1020.. Average time per step: 0.4069.. Train loss: 0.2379.. Train accuracy: 0.9166.. Top-3 train accuracy: 0.9915.. Test loss: 0.6319.. Test accuracy: 0.8257.. Top-3 test accuracy: 0.9579\n",
            "Epoch 36/100.. Time per epoch: 159.0254.. Average time per step: 0.4067.. Train loss: 0.2324.. Train accuracy: 0.9181.. Top-3 train accuracy: 0.9918.. Test loss: 0.5368.. Test accuracy: 0.8401.. Top-3 test accuracy: 0.9665\n",
            "Epoch 37/100.. Time per epoch: 159.1210.. Average time per step: 0.4070.. Train loss: 0.2240.. Train accuracy: 0.9228.. Top-3 train accuracy: 0.9922.. Test loss: 0.5372.. Test accuracy: 0.8390.. Top-3 test accuracy: 0.9669\n",
            "Epoch 38/100.. Time per epoch: 159.0956.. Average time per step: 0.4069.. Train loss: 0.2200.. Train accuracy: 0.9242.. Top-3 train accuracy: 0.9930.. Test loss: 0.5257.. Test accuracy: 0.8429.. Top-3 test accuracy: 0.9671\n",
            "Epoch 39/100.. Time per epoch: 159.0526.. Average time per step: 0.4068.. Train loss: 0.2112.. Train accuracy: 0.9268.. Top-3 train accuracy: 0.9930.. Test loss: 0.5838.. Test accuracy: 0.8409.. Top-3 test accuracy: 0.9661\n",
            "Epoch 40/100.. Time per epoch: 159.1705.. Average time per step: 0.4071.. Train loss: 0.2105.. Train accuracy: 0.9279.. Top-3 train accuracy: 0.9929.. Test loss: 0.5461.. Test accuracy: 0.8371.. Top-3 test accuracy: 0.9651\n",
            "Epoch 41/100.. Time per epoch: 159.0284.. Average time per step: 0.4067.. Train loss: 0.2043.. Train accuracy: 0.9294.. Top-3 train accuracy: 0.9933.. Test loss: 0.5577.. Test accuracy: 0.8447.. Top-3 test accuracy: 0.9668\n",
            "Epoch 42/100.. Time per epoch: 159.2158.. Average time per step: 0.4072.. Train loss: 0.1968.. Train accuracy: 0.9324.. Top-3 train accuracy: 0.9938.. Test loss: 0.5833.. Test accuracy: 0.8329.. Top-3 test accuracy: 0.9672\n",
            "Epoch 43/100.. Time per epoch: 159.2218.. Average time per step: 0.4072.. Train loss: 0.1906.. Train accuracy: 0.9338.. Top-3 train accuracy: 0.9947.. Test loss: 0.6461.. Test accuracy: 0.8279.. Top-3 test accuracy: 0.9637\n",
            "Epoch 44/100.. Time per epoch: 159.1359.. Average time per step: 0.4070.. Train loss: 0.1868.. Train accuracy: 0.9350.. Top-3 train accuracy: 0.9946.. Test loss: 0.5566.. Test accuracy: 0.8407.. Top-3 test accuracy: 0.9643\n",
            "Epoch 45/100.. Time per epoch: 159.3844.. Average time per step: 0.4076.. Train loss: 0.1880.. Train accuracy: 0.9348.. Top-3 train accuracy: 0.9946.. Test loss: 0.5486.. Test accuracy: 0.8459.. Top-3 test accuracy: 0.9690\n",
            "Epoch 46/100.. Time per epoch: 159.2684.. Average time per step: 0.4073.. Train loss: 0.1799.. Train accuracy: 0.9366.. Top-3 train accuracy: 0.9949.. Test loss: 0.6246.. Test accuracy: 0.8306.. Top-3 test accuracy: 0.9626\n",
            "Epoch 47/100.. Time per epoch: 159.1291.. Average time per step: 0.4070.. Train loss: 0.1718.. Train accuracy: 0.9405.. Top-3 train accuracy: 0.9956.. Test loss: 0.5696.. Test accuracy: 0.8415.. Top-3 test accuracy: 0.9688\n",
            "Epoch 48/100.. Time per epoch: 159.3549.. Average time per step: 0.4076.. Train loss: 0.1732.. Train accuracy: 0.9400.. Top-3 train accuracy: 0.9951.. Test loss: 0.7859.. Test accuracy: 0.7988.. Top-3 test accuracy: 0.9529\n",
            "Epoch 49/100.. Time per epoch: 159.2322.. Average time per step: 0.4072.. Train loss: 0.1681.. Train accuracy: 0.9412.. Top-3 train accuracy: 0.9955.. Test loss: 0.6572.. Test accuracy: 0.8319.. Top-3 test accuracy: 0.9602\n",
            "Epoch 50/100.. Time per epoch: 159.2051.. Average time per step: 0.4072.. Train loss: 0.1624.. Train accuracy: 0.9429.. Top-3 train accuracy: 0.9957.. Test loss: 0.6005.. Test accuracy: 0.8362.. Top-3 test accuracy: 0.9675\n",
            "Epoch 51/100.. Time per epoch: 159.1446.. Average time per step: 0.4070.. Train loss: 0.1602.. Train accuracy: 0.9446.. Top-3 train accuracy: 0.9957.. Test loss: 0.6057.. Test accuracy: 0.8353.. Top-3 test accuracy: 0.9633\n",
            "Epoch 52/100.. Time per epoch: 159.0505.. Average time per step: 0.4068.. Train loss: 0.1547.. Train accuracy: 0.9460.. Top-3 train accuracy: 0.9961.. Test loss: 0.5787.. Test accuracy: 0.8424.. Top-3 test accuracy: 0.9653\n",
            "Epoch 53/100.. Time per epoch: 158.8772.. Average time per step: 0.4063.. Train loss: 0.1496.. Train accuracy: 0.9483.. Top-3 train accuracy: 0.9962.. Test loss: 0.5572.. Test accuracy: 0.8463.. Top-3 test accuracy: 0.9654\n",
            "Epoch 54/100.. Time per epoch: 158.9021.. Average time per step: 0.4064.. Train loss: 0.1546.. Train accuracy: 0.9471.. Top-3 train accuracy: 0.9961.. Test loss: 0.6018.. Test accuracy: 0.8400.. Top-3 test accuracy: 0.9658\n",
            "Epoch 55/100.. Time per epoch: 158.7581.. Average time per step: 0.4060.. Train loss: 0.1472.. Train accuracy: 0.9486.. Top-3 train accuracy: 0.9961.. Test loss: 0.5872.. Test accuracy: 0.8454.. Top-3 test accuracy: 0.9642\n",
            "Epoch 56/100.. Time per epoch: 158.8603.. Average time per step: 0.4063.. Train loss: 0.1434.. Train accuracy: 0.9498.. Top-3 train accuracy: 0.9965.. Test loss: 0.6092.. Test accuracy: 0.8436.. Top-3 test accuracy: 0.9670\n",
            "Epoch 57/100.. Time per epoch: 158.6999.. Average time per step: 0.4059.. Train loss: 0.1447.. Train accuracy: 0.9497.. Top-3 train accuracy: 0.9968.. Test loss: 0.6037.. Test accuracy: 0.8363.. Top-3 test accuracy: 0.9659\n",
            "Epoch 58/100.. Time per epoch: 158.6641.. Average time per step: 0.4058.. Train loss: 0.1408.. Train accuracy: 0.9518.. Top-3 train accuracy: 0.9970.. Test loss: 0.6000.. Test accuracy: 0.8467.. Top-3 test accuracy: 0.9658\n",
            "Epoch 59/100.. Time per epoch: 158.5460.. Average time per step: 0.4055.. Train loss: 0.1361.. Train accuracy: 0.9532.. Top-3 train accuracy: 0.9968.. Test loss: 0.6008.. Test accuracy: 0.8416.. Top-3 test accuracy: 0.9657\n",
            "Epoch 60/100.. Time per epoch: 158.4962.. Average time per step: 0.4054.. Train loss: 0.1320.. Train accuracy: 0.9542.. Top-3 train accuracy: 0.9969.. Test loss: 0.6030.. Test accuracy: 0.8440.. Top-3 test accuracy: 0.9678\n",
            "Epoch 61/100.. Time per epoch: 158.4860.. Average time per step: 0.4053.. Train loss: 0.1341.. Train accuracy: 0.9522.. Top-3 train accuracy: 0.9972.. Test loss: 0.5934.. Test accuracy: 0.8441.. Top-3 test accuracy: 0.9679\n",
            "Epoch 62/100.. Time per epoch: 158.5069.. Average time per step: 0.4054.. Train loss: 0.1269.. Train accuracy: 0.9557.. Top-3 train accuracy: 0.9976.. Test loss: 0.6283.. Test accuracy: 0.8383.. Top-3 test accuracy: 0.9637\n",
            "Epoch 63/100.. Time per epoch: 158.5852.. Average time per step: 0.4056.. Train loss: 0.1310.. Train accuracy: 0.9539.. Top-3 train accuracy: 0.9971.. Test loss: 0.7277.. Test accuracy: 0.8218.. Top-3 test accuracy: 0.9607\n",
            "Epoch 64/100.. Time per epoch: 158.4845.. Average time per step: 0.4053.. Train loss: 0.1227.. Train accuracy: 0.9571.. Top-3 train accuracy: 0.9979.. Test loss: 0.6173.. Test accuracy: 0.8378.. Top-3 test accuracy: 0.9655\n",
            "Epoch 65/100.. Time per epoch: 158.4576.. Average time per step: 0.4053.. Train loss: 0.1211.. Train accuracy: 0.9577.. Top-3 train accuracy: 0.9975.. Test loss: 0.5678.. Test accuracy: 0.8518.. Top-3 test accuracy: 0.9694\n",
            "Epoch 66/100.. Time per epoch: 158.7300.. Average time per step: 0.4060.. Train loss: 0.1190.. Train accuracy: 0.9583.. Top-3 train accuracy: 0.9975.. Test loss: 0.6264.. Test accuracy: 0.8450.. Top-3 test accuracy: 0.9658\n",
            "Epoch 67/100.. Time per epoch: 158.1866.. Average time per step: 0.4046.. Train loss: 0.1211.. Train accuracy: 0.9580.. Top-3 train accuracy: 0.9978.. Test loss: 0.6170.. Test accuracy: 0.8416.. Top-3 test accuracy: 0.9663\n",
            "Epoch 68/100.. Time per epoch: 158.2456.. Average time per step: 0.4047.. Train loss: 0.1144.. Train accuracy: 0.9602.. Top-3 train accuracy: 0.9977.. Test loss: 0.6453.. Test accuracy: 0.8350.. Top-3 test accuracy: 0.9642\n",
            "Epoch 69/100.. Time per epoch: 158.1405.. Average time per step: 0.4045.. Train loss: 0.1108.. Train accuracy: 0.9619.. Top-3 train accuracy: 0.9977.. Test loss: 0.6682.. Test accuracy: 0.8422.. Top-3 test accuracy: 0.9657\n",
            "Epoch 70/100.. Time per epoch: 158.1509.. Average time per step: 0.4045.. Train loss: 0.1116.. Train accuracy: 0.9611.. Top-3 train accuracy: 0.9979.. Test loss: 0.6366.. Test accuracy: 0.8432.. Top-3 test accuracy: 0.9656\n",
            "Epoch 71/100.. Time per epoch: 158.1480.. Average time per step: 0.4045.. Train loss: 0.1080.. Train accuracy: 0.9632.. Top-3 train accuracy: 0.9978.. Test loss: 0.6587.. Test accuracy: 0.8448.. Top-3 test accuracy: 0.9677\n",
            "Epoch 72/100.. Time per epoch: 158.0383.. Average time per step: 0.4042.. Train loss: 0.1066.. Train accuracy: 0.9625.. Top-3 train accuracy: 0.9979.. Test loss: 0.6162.. Test accuracy: 0.8421.. Top-3 test accuracy: 0.9654\n",
            "Epoch 73/100.. Time per epoch: 157.9917.. Average time per step: 0.4041.. Train loss: 0.1084.. Train accuracy: 0.9626.. Top-3 train accuracy: 0.9981.. Test loss: 0.6329.. Test accuracy: 0.8425.. Top-3 test accuracy: 0.9683\n",
            "Epoch 74/100.. Time per epoch: 158.2310.. Average time per step: 0.4047.. Train loss: 0.1075.. Train accuracy: 0.9622.. Top-3 train accuracy: 0.9980.. Test loss: 0.6681.. Test accuracy: 0.8391.. Top-3 test accuracy: 0.9650\n",
            "Epoch 75/100.. Time per epoch: 158.2344.. Average time per step: 0.4047.. Train loss: 0.1033.. Train accuracy: 0.9639.. Top-3 train accuracy: 0.9985.. Test loss: 0.6416.. Test accuracy: 0.8454.. Top-3 test accuracy: 0.9679\n",
            "Epoch 76/100.. Time per epoch: 158.2132.. Average time per step: 0.4046.. Train loss: 0.1032.. Train accuracy: 0.9643.. Top-3 train accuracy: 0.9983.. Test loss: 0.6018.. Test accuracy: 0.8461.. Top-3 test accuracy: 0.9646\n",
            "Epoch 77/100.. Time per epoch: 158.1548.. Average time per step: 0.4045.. Train loss: 0.1037.. Train accuracy: 0.9639.. Top-3 train accuracy: 0.9984.. Test loss: 0.6852.. Test accuracy: 0.8382.. Top-3 test accuracy: 0.9647\n",
            "Epoch 78/100.. Time per epoch: 157.9943.. Average time per step: 0.4041.. Train loss: 0.0965.. Train accuracy: 0.9667.. Top-3 train accuracy: 0.9986.. Test loss: 0.6293.. Test accuracy: 0.8477.. Top-3 test accuracy: 0.9679\n",
            "Epoch 79/100.. Time per epoch: 158.2605.. Average time per step: 0.4048.. Train loss: 0.0984.. Train accuracy: 0.9656.. Top-3 train accuracy: 0.9983.. Test loss: 0.6379.. Test accuracy: 0.8440.. Top-3 test accuracy: 0.9670\n",
            "Epoch 80/100.. Time per epoch: 158.0003.. Average time per step: 0.4041.. Train loss: 0.0962.. Train accuracy: 0.9663.. Top-3 train accuracy: 0.9981.. Test loss: 0.6869.. Test accuracy: 0.8432.. Top-3 test accuracy: 0.9659\n",
            "Epoch 81/100.. Time per epoch: 158.1061.. Average time per step: 0.4044.. Train loss: 0.0956.. Train accuracy: 0.9669.. Top-3 train accuracy: 0.9986.. Test loss: 0.7050.. Test accuracy: 0.8357.. Top-3 test accuracy: 0.9648\n",
            "Epoch 82/100.. Time per epoch: 158.3123.. Average time per step: 0.4049.. Train loss: 0.0962.. Train accuracy: 0.9672.. Top-3 train accuracy: 0.9982.. Test loss: 0.6658.. Test accuracy: 0.8442.. Top-3 test accuracy: 0.9660\n",
            "Epoch 83/100.. Time per epoch: 158.0962.. Average time per step: 0.4043.. Train loss: 0.0945.. Train accuracy: 0.9681.. Top-3 train accuracy: 0.9986.. Test loss: 0.6571.. Test accuracy: 0.8463.. Top-3 test accuracy: 0.9680\n",
            "Epoch 84/100.. Time per epoch: 158.4174.. Average time per step: 0.4052.. Train loss: 0.0906.. Train accuracy: 0.9681.. Top-3 train accuracy: 0.9984.. Test loss: 0.6965.. Test accuracy: 0.8470.. Top-3 test accuracy: 0.9689\n",
            "Epoch 85/100.. Time per epoch: 158.5204.. Average time per step: 0.4054.. Train loss: 0.0922.. Train accuracy: 0.9684.. Top-3 train accuracy: 0.9984.. Test loss: 0.6594.. Test accuracy: 0.8434.. Top-3 test accuracy: 0.9685\n",
            "Epoch 86/100.. Time per epoch: 158.8547.. Average time per step: 0.4063.. Train loss: 0.0934.. Train accuracy: 0.9680.. Top-3 train accuracy: 0.9983.. Test loss: 0.6669.. Test accuracy: 0.8459.. Top-3 test accuracy: 0.9654\n",
            "Epoch 87/100.. Time per epoch: 158.8635.. Average time per step: 0.4063.. Train loss: 0.0863.. Train accuracy: 0.9702.. Top-3 train accuracy: 0.9986.. Test loss: 0.6551.. Test accuracy: 0.8467.. Top-3 test accuracy: 0.9646\n",
            "Epoch 88/100.. Time per epoch: 158.9338.. Average time per step: 0.4065.. Train loss: 0.0854.. Train accuracy: 0.9706.. Top-3 train accuracy: 0.9987.. Test loss: 0.6865.. Test accuracy: 0.8407.. Top-3 test accuracy: 0.9667\n",
            "Epoch 89/100.. Time per epoch: 159.0522.. Average time per step: 0.4068.. Train loss: 0.0883.. Train accuracy: 0.9687.. Top-3 train accuracy: 0.9988.. Test loss: 0.7711.. Test accuracy: 0.8255.. Top-3 test accuracy: 0.9624\n",
            "Epoch 90/100.. Time per epoch: 159.0356.. Average time per step: 0.4067.. Train loss: 0.0880.. Train accuracy: 0.9708.. Top-3 train accuracy: 0.9987.. Test loss: 0.6401.. Test accuracy: 0.8493.. Top-3 test accuracy: 0.9680\n",
            "Epoch 91/100.. Time per epoch: 159.2041.. Average time per step: 0.4072.. Train loss: 0.0842.. Train accuracy: 0.9707.. Top-3 train accuracy: 0.9984.. Test loss: 0.6487.. Test accuracy: 0.8461.. Top-3 test accuracy: 0.9674\n",
            "Epoch 92/100.. Time per epoch: 159.1447.. Average time per step: 0.4070.. Train loss: 0.0867.. Train accuracy: 0.9702.. Top-3 train accuracy: 0.9988.. Test loss: 0.6211.. Test accuracy: 0.8508.. Top-3 test accuracy: 0.9676\n",
            "Epoch 93/100.. Time per epoch: 159.0623.. Average time per step: 0.4068.. Train loss: 0.0806.. Train accuracy: 0.9725.. Top-3 train accuracy: 0.9990.. Test loss: 0.6816.. Test accuracy: 0.8477.. Top-3 test accuracy: 0.9679\n",
            "Epoch 94/100.. Time per epoch: 158.9915.. Average time per step: 0.4066.. Train loss: 0.0807.. Train accuracy: 0.9713.. Top-3 train accuracy: 0.9990.. Test loss: 0.7555.. Test accuracy: 0.8411.. Top-3 test accuracy: 0.9630\n",
            "Epoch 95/100.. Time per epoch: 159.1508.. Average time per step: 0.4070.. Train loss: 0.0818.. Train accuracy: 0.9715.. Top-3 train accuracy: 0.9988.. Test loss: 0.6714.. Test accuracy: 0.8464.. Top-3 test accuracy: 0.9681\n",
            "Epoch 96/100.. Time per epoch: 159.1185.. Average time per step: 0.4070.. Train loss: 0.0813.. Train accuracy: 0.9722.. Top-3 train accuracy: 0.9990.. Test loss: 0.6434.. Test accuracy: 0.8509.. Top-3 test accuracy: 0.9669\n",
            "Epoch 97/100.. Time per epoch: 159.1653.. Average time per step: 0.4071.. Train loss: 0.0764.. Train accuracy: 0.9733.. Top-3 train accuracy: 0.9987.. Test loss: 0.7102.. Test accuracy: 0.8423.. Top-3 test accuracy: 0.9645\n",
            "Epoch 98/100.. Time per epoch: 159.2401.. Average time per step: 0.4073.. Train loss: 0.0804.. Train accuracy: 0.9722.. Top-3 train accuracy: 0.9989.. Test loss: 0.7311.. Test accuracy: 0.8443.. Top-3 test accuracy: 0.9675\n",
            "Epoch 99/100.. Time per epoch: 159.1692.. Average time per step: 0.4071.. Train loss: 0.0796.. Train accuracy: 0.9721.. Top-3 train accuracy: 0.9988.. Test loss: 0.6087.. Test accuracy: 0.8487.. Top-3 test accuracy: 0.9695\n",
            "Epoch 100/100.. Time per epoch: 159.2997.. Average time per step: 0.4074.. Train loss: 0.0727.. Train accuracy: 0.9745.. Top-3 train accuracy: 0.9989.. Test loss: 0.6973.. Test accuracy: 0.8427.. Top-3 test accuracy: 0.9658\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YfJk0014_JTj",
        "colab_type": "code",
        "outputId": "eb2e015f-922e-4b92-d3c0-aacc637cfc21",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 312
        }
      },
      "source": [
        "# Plotting the learnt AFU\n",
        "import matplotlib.pyplot as plt\n",
        "from torch.autograd import Variable\n",
        "plt.figure();z = torch.linspace(-5,5,1000); \n",
        "z=Variable(z).type(torch.cuda.FloatTensor)\n",
        "plt.plot(z.cpu().numpy(),model.stem[0].activation(z.unsqueeze(-1)).detach().cpu().numpy());\n",
        "plt.grid(); plt.title(\"post\"); plt.xlabel('z'); plt.ylabel('g(z)')  \n",
        "\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Text(0, 0.5, 'g(z)')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3dZ3RU5f728e+PkNBCL6F3kI4QqiiC\niqIoKFZAEBSiHrFgL8dy1GP9iw0VQRBRJCpYEDgKKoigIIRepIP0TiC0tPt5kfE8HKQEyM6ecn3W\nymJmz87kuteQubLL3Nucc4iISOTK43cAERHxl4pARCTCqQhERCKcikBEJMKpCEREIpyKQEQkwqkI\nREQinIpAJBeZ2Qgze97vHCJHUxGIiEQ4FYHISZjZOjN7zMyWmtkeM/vQzPIHHutnZqvMbLeZjTOz\n8oHlZmavm9l2M9tnZovMrIGZJQA9gIfNLMXMvvVzbCJ/MU0xIXJiZrYOSAEuBw4A3wJTgJ+Az4FL\ngSXA/wGNnXNtzewy4AXgYiAZqAPsdc5tMbMRwEbn3D9zeSgiJ5TX7wAiIWCQc24DgJn9G3gbKAcM\nd87NDSx/DNhjZlWBNKAwWQXwu3NumR+hRbJLu4ZETm3DUbfXA+UDX+v/WuicSwF2ARWccz8Bg4B3\ngO1mNsTMiuRiXpHToiIQObVKR92uDGwOfFX5a6GZFQJKApsAnHNvOefigXpAbeChwKraFytBR0Ug\ncmp3mVlFMysBPAF8BowG+pjZuWaWj6xjArOcc+vMrLmZtTSzaLKOKxwGMgPPtQ2o7sMYRE5IRSBy\nap8Ck4A1wGrgeefcD8CTwFhgC1ADuCmwfhFgKLCHrN1Hu4BXA48NA+qZ2V4z+zrXRiByEjprSOQk\nAmcN9Q288YuEJW0RiIhEOBWBiEiE064hEZEIpy0CEZEIF3KfLC5VqpSrWrWq3zFO24EDByhUqJDf\nMXKVxhz+Im28ELpjTkpK2umcK328x0KuCKpWrcqcOXP8jnHapk6dSrt27fyOkas05vAXaeOF0B2z\nma0/0WPaNSQiEuFUBCIiEU5FICIS4VQEIiIRTkUgIhLhVAQiIhFORSAiEuFUBCIiQc45x1s/rmTp\n5n2ePH/IfaBMRCSSOOd48T9/MGTaGg6mZlCvfM5f9VRFICISpJxzPD9hGcOmr6VX6yo80vEcT36O\nikBEJAg553h2/FI+nLGO3udV5emr6mFmnvwsFYGISJBxzvHMuCV89Nt6bm1TjSevrOtZCYCKQEQk\nqGRmOp4et4SPZ66n3wXVePwKb0sAVAQiIkEjM9Px5DeLGTXrT26/sDqPdqzjeQmAikBEJChkZjqe\n+HoRo3/fwJ3tavDwZefkSgmAikBExHeZmY7HvlzEZ3M20L99TR64tHaulQCoCEREfJWR6Xhk7ELG\nJG3knotrMeCSWrlaAqAiEBHxTUam46ExC/hy7ibuu6QW911S25ccKgIRER9kZDoe/GIBX83bxP0d\nanPPxbV8y6IiEBHJZekZmTzwxQK+mb+ZBy+tTf+L/CsB8HDSOTOrZGZTzGypmS0xs3uPs047M0s2\ns/mBr6e8yiMiEgzSMzIZ8HlWCTzc8RzfSwC83SJIBx5wzs01s8JAkplNds4tPWa9X5xzV3qYQ0Qk\nKKRlZHJf4nwmLNrCY5fX4fYLa/gdCfBwi8A5t8U5Nzdwez+wDKjg1c8TEQlmaRmZ3DN6HhMWbeGJ\nK+oGTQkAmHPO+x9iVhWYBjRwzu07ank7YCywEdgMPOicW3Kc708AEgDi4uLiExMTPc+c01JSUoiN\njfU7Rq7SmMNfpI0XzmzM6ZmO9xYcIWlbBt3qxHBZ1WiP0p1Y+/btk5xzzY77oHPO0y8gFkgCuh7n\nsSJAbOD2FcDKUz1ffHy8C0VTpkzxO0Ku05jDX6SN17nTH/ORtAzX96PZrsoj493w6Wu8CZUNwBx3\ngvdVT69QZmbRZP3FP8o59+VxSmifcy4lcHsiEG1mpbzMJCKSW46kZ/CPUUlMXrqNZ7vUp0+ban5H\nOi4vzxoyYBiwzDk38ATrlA2sh5m1COTZ5VUmEZHccjgtgzs/mcsPy7bz3NUN6NW6qt+RTsjLs4ba\nAD2BRWY2P7DscaAygHNuMHAdcKeZpQOHgJsCmzAiIiHrcFoGd3ySxNTlO3jhmoZ0b1nZ70gn5VkR\nOOemAyedMMM5NwgY5FUGEZHcdjgtg4SPk5i2YgcvdW3ITS2CuwRAnywWEckxh1IzSPh4DtNX7eSV\naxtxQ/NKfkfKFhWBiEgOOJSaQd+Rs/l19S5eubYR1zcLjRIAFYGIyFk7mJrObSPmMHPtLv7vusZc\nG1/R70inRUUgInIWDhxJ59YRs5m9bjev33AuVzcJvQkUVAQiImfowJF0+nw4mznrd/P6jefS5dzQ\nKwFQEYiInJGUI+n0Hv478zbs5c2bmnBV4/J+RzpjKgIRkdO0/3AavT+czfwNe3nrpiZ0alTO70hn\nRUUgInIaDqY5eg3/nUUbkxnUrQmXNwztEgAVgYhItiUfSuP/5hzmz/2HGNS9KR0blPU7Uo5QEYiI\nZEPywTR6Dp/F+n2ZDO7ZjA714vyOlGNUBCIip7D3YCo9h/3O8q37ubtJvrAqAVARiIic1J4Dqdw8\nbBYrt6Xwfs94bOuxV9sNfZ5ej0BEJJTtPpBK9w9msXJ7CkN6xdO+Thm/I3lCWwQiIsexK+UIPT6Y\nxdqdBxjaqxkX1i7tdyTPqAhERI6xM+UIPYbOYt2uAwy7pTnn1wrvCyeqCEREjrJj/xG6D53Jhj0H\nGd67OW1qhncJgIpAROS/tu8/TPehs9i05xDDezfnvBrhXwKgIhARAWD7vsN0GzqTLcmH+bBPc1pV\nL+l3pFyjIhCRiLdt32G6DZnJ1n2HGdGnBS2qlfA7Uq5SEYhIRNuanLUlsH3fYUbe2oJmVSOrBEBF\nICIRbPPeQ3QbOpNdKamMvK0F8VUirwRARSAiEWrT3kN0GzKTPQeySqBp5eJ+R/KNikBEIs7GPQfp\nNnQmew+m8XHflpxbqZjfkXylIhCRiLJh90FuGjKT/YfTGNW3JY0qRnYJgIdzDZlZJTObYmZLzWyJ\nmd17nHXMzN4ys1VmttDMmnqVR0Tkz11ZJZByJJ1RfVupBAK83CJIBx5wzs01s8JAkplNds4dPXXf\n5UCtwFdL4L3AvyIiOWr9rgN0GzKTg2kZjOrbkgYVivodKWh4tkXgnNvinJsbuL0fWAZUOGa1LsBI\nl2UmUMzMQv+6byISVNbtPMBNQ2ZyKC2DT/u2Ugkcw5xz3v8Qs6rANKCBc27fUcvHAy8556YH7v8I\nPOKcm3PM9ycACQBxcXHxiYmJnmfOaSkpKcTGxvodI1dpzOEvFMa79UAmL/1+mIxMx8MtClCp8Nn9\n/RsKYz6e9u3bJznnmh3vMc8PFptZLDAWuO/oEjgdzrkhwBCAZs2auXbt2uVcwFwydepUQjH32dCY\nw1+wj3f1jhQeHjKTqLzRfNavFeeULXzWzxnsYz4TnhaBmUWTVQKjnHNfHmeVTUClo+5XDCwTETkr\nq7bvp9vQWTjnGJ3QitpxZ18C4crLs4YMGAYsc84NPMFq44BegbOHWgHJzrktXmUSkciwctt+bhoy\nC+dgdD+VwKl4uUXQBugJLDKz+YFljwOVAZxzg4GJwBXAKuAg0MfDPCISAZZv3U/3oTOJymN82q8V\nNcuE3v783OZZEQQOANsp1nHAXV5lEJHI8sfWfXQfOovoKGN0v1ZUL60SyA5dvF5EwsLSzfvoNmQm\nMVF5SExorRI4DSoCEQl5SzYn0/2DmeSPjiIxoRXVShXyO1JIURGISEhbvCmZ7kNnUTBQAlVVAqdN\nk86JSMhatDGZHh/MpHD+aBITWlGpREG/I4UkbRGISEhasGEv3T+YSZECKoGzpS0CEQk58/7cQ69h\nv1OsUDSj+7WiYnGVwNlQEYhISElav4dbhv9OydgYRvdrRfliBfyOFPJUBCISMpLW7+aW4bMpFRvD\n6IRWlCuqEsgJOkYgIiFh9rrd9Br2O2UK5yMxobVKIAdpi0BEgt6sNbvoM2I2ZYvmJ7FfK8oUye93\npLCiLQIRCWq/rd5F7w9nU65ofhITVAJe0BaBiAStX1ft5NaPZlOpeEE+7deK0oXz+R0pLGmLQESC\n0vSVO+kzYjaVSxRkdIJKwEvaIhCRoDNtxQ76jZxDtVKFGNW3JSVjVQJeUhGISFD5OVACNUrHMqpv\nS0oUivE7UtjTriERCRpTlm+n38g51Cwdy6cqgVyjLQIRCQo/LtvGnZ/MpXbZWD65rSXFCqoEcou2\nCETEdz8s3cYdnyRRp1xhRt3WSiWQy1QEIuKrSUu2cueoJOqVK8LHt7WkaMFovyNFHO0aEhHffLd4\nC/0/nUeDCkUZeVsLiuRXCfhBWwQi4ouJi7Zw16fzaFSxKB+rBHylLQIRyXXjF27m3sT5NKlUjBG3\ntiA2n96K/KQtAhHJVeMWZJVA08oqgWChIhCRXPPN/E3clziP+CrFGdFHJRAsPCsCMxtuZtvNbPEJ\nHm9nZslmNj/w9ZRXWUTEf1/N28iAz+bToloJRvRpTiGVQNDw8pUYAQwCRp5knV+cc1d6mEFEgsCY\npI08NGYBrauXZNgtzSkQE+V3JDmKZ1sEzrlpwG6vnl9EQsPnczbw0JgFtKlRSiUQpMw5592Tm1UF\nxjvnGhznsXbAWGAjsBl40Dm35ATPkwAkAMTFxcUnJiZ6lNg7KSkpxMbG+h0jV2nM4e9U4/15Yxoj\nFqdSv2QU9zTNR0yU5WI6b4Tqa9y+ffsk51yz4z7onPPsC6gKLD7BY0WA2MDtK4CV2XnO+Ph4F4qm\nTJnid4RcpzGHv5ONd9TM9a7KI+Ndr2Gz3KHU9NwL5bFQfY2BOe4E76u+nTXknNvnnEsJ3J4IRJtZ\nKb/yiEjO+WTmeh7/ahHtzynN+z3jyR+t3UHBzLciMLOyZmaB2y0CWXb5lUdEcsbI39bxz68Xc3Gd\nMgxWCYQEz84aMrPRQDuglJltBJ4GogGcc4OB64A7zSwdOATcFNh8EZEQNWLGWp75dimX1I3jnR5N\nyJdXJRAKPCsC51y3Uzw+iKzTS0UkDAyfvpZnxy/l0npxDOrelJi8+rxqqNAnOkTkrH3wyxqen7CM\njvXL8nb3JkRHqQRCiYpARM7KkGmreWHiH1zRsCxv3qQSCEUqAhE5YxPWpPLFij/o1Kgcb9x4rkog\nROlVE5Ez8s6UVXyxIo3OjcvzpkogpGmLQERO29s/ruS1yStoXS6KgTc0Jq9KIKSpCETktLzxwwre\n+GElXZtU4Moye1QCYSBbr6CZlTGza8zsLjO71cxamJlefZEI4pxj4OSsErguviKvXt+YPBb6cwfJ\nKbYIzKw98ChQApgHbAfyA1cDNcxsDPCac26f10FFxD9/lcDbP63ihmYVealrI/LkUQmEi1PtGroC\n6Oec+/PYB8wsL3Al0IGsWURFJAw553j1++W8O3U1NzWvxAvXNFQJhJmTFoFz7qGTPJYOfJ3jiUQk\naDjnePm75Qz+eTXdW1bm+S4NVAJhKLvHCDLM7KW/JokLLJvrXSwR8Ztzjhf/8weDf17Nza1UAuEs\nuwd8lwTWnWRmJQLL9D9CJEw553h+wjKGTFtDr9ZVeE4lENayWwTpzrmHgQ+AX8wsHtBMoSJhyDnH\ns+OXMmz6WnqfV5V/da6P6eygsJbdzxEYgHPuMzNbAnwKVPYslYj45q0fV/HhjHXc2qYaT15ZVyUQ\nAbJbBH3/uuGcW2xmFwBdvIkkIn75ecUO3vhxBdc0qaASiCAn3TVkZucDOOeSjl7unEt2zo00syJm\n9rcL04tI6Nm09xD3Jc7jnLjCvHBNQ5VABDnVFsG1ZvYK8B2QBOwg6wNlNYH2QBXgAU8TiojnjqRn\n8I9Rc0nLcLzboykFYnRlsUhyqs8RDAicJXQtcD1QlqzLSi4D3nfOTfc+ooh47d8TlrFgw14G39yU\n6qVj/Y4jueyUxwicc7vNrAiwEFj012LgHDNLcc7N9zKgiHjrm/mbGPnbevpdUI2ODcr5HUd8kN3T\nR+OBO4ByQHngdqAjMNTMHvYom4h4bMW2/Tw6dhHNqxbn4Y51/I4jPsnuWUMVgabOuRQAM3samAC0\nJevYwSvexBMRr6QcSeeOT5IolC8vg7o31YVlIlh2X/kywJGj7qcBcc65Q8csF5EQ4JzjkbELWbfz\nAG93a0Jckfx+RxIfZXeLYBQwy8y+Cdy/CvjUzAoBSz1JJiKe+XDGOiYs3MIjHevQukZJv+OIz7JV\nBM6558zsP0CbwKI7nHNzArd7eJJMRDyRtH43L0xcxiV147jjwup+x5EgkO1LVQbe+OeccsUAMxtO\n1vUKtjvn/vahs8BMpm+Sdc2Dg0Bv55xmNBXx0M6UI9w1ah7lixXgtRsa60NjAmT/GMGZGEHWmUUn\ncjlQK/CVALznYRaRiJeR6bg3cR57Dqby3s1NKVog2u9IEiQ8KwLn3DRg90lW6QKMdFlmAsXMTCcx\ni3jk9ckrmLFqF891aUD98kX9jiNBxM/zxSoAG466vzGwTERy2E9/bGPQlKzrDd/QvJLfcSTIZPsY\ngZ/MLIGs3UfExcUxdepUfwOdgZSUlJDMfTY05uCw42Amz/x2iMqF83BJ8d05mi8Yx+u1cByzn0Ww\nCTj6T5OKgWV/45wbAgwBaNasmWvXrp3n4XLa1KlTCcXcZ0Nj9t/htAyuG/wreaLS+OSOC6hcsmCO\nPn+wjTc3hOOY/dw1NA7oZVlaAcnOuS0+5hEJO//6dimLN+1j4A3n5ngJSPjwbIvAzEYD7YBSZrYR\neBqIBnDODQYmknXq6CqyTh/t41UWkUg0Jmkjo3//kzvb1aBDvTi/40gQ86wInHPdTvG4A+7y6ueL\nRLJlW/bxxFeLaF29JA90qO13HAlymmVKJMzsO5zGnZ8kUbRANG91a0JeTSYnpxASZw2JSPY453jo\niwVs2HOIxIRWlC6cz+9IEgL0p4JIGBn6yxq+X7KNxy6vQ/OqJfyOIyFCRSASJpLW7+bl75ZzeYOy\n3HZ+Nb/jSAhREYiEgcNpGTz0xULKFc3PK9c10mRycloipgicc/y566DfMUQ88daPK1mz8wAvdm1I\n4fyaTE5OT8QUwbgFm7l44FRen7yC1PRMv+OI5Jglm5N5f9oarouvyAW1SvsdR0JQxBTBBbVK06lh\nOd78cSVXvT2dBRv2+h1J5KylZ2TyyNiFFC8Ywz871fU7joSoiCmCEoVieOOmJgy7pRnJh9K45t0Z\nvDBxGYdSM/yOJnLGPpi+lsWb9vFsl/oUKxjjdxwJURFTBH+5uG4ck+5vy43NKzFk2houf3Mas9bs\n8juWyGlbu/MAr09ewWX147i8QVm/40gIi7giACiSP5oXuzbi074tyXCOG4fM5J9fL2L/4TS/o4lk\nS2am49GxC4nJm4dnuzTQWUJyViKyCP5yXs1SfH9fW25tU41Rs/7kstenMXX5dr9jiZzS6Nl/Mmvt\nbv7ZqS5xRfL7HUdCXEQXAUDBmLw8dVU9xtxxHgXz5aX3h7O5//P57D2Y6nc0kePaknyIlyb+wXk1\nSnJDM11tTM5exBfBX+KrFGfCPefTv31Nvpm/mUsGTuM/i3R5BAkuzjme/HoxaZmZvNi1oXYJSY5Q\nERwlX94oHrzsHMb1b0NckXzcOWoud36SxPb9h/2OJgLA+IVb+GHZdh7ocA5VShbyO46ECRXBcdQv\nX5Sv72rDQ5edw49/bKfDwGmMTdpI1iUURPyx50Aqz4xbQuOKRenTpqrfcSSMqAhOIDoqD3e1r8nE\ney6gZplYHvhiAX1GzGbz3kN+R5MI9dz4pSQfSuOlaxvpGgOSo/S/6RRqlonl89tb8/RV9Zi1ZjeX\nvj6NT2auJzNTWweSe6Yu386X8zbxj3Y1qFuuiN9xJMyoCLIhKo/Rp001Jg1oS+NKRfnn14vpNnQm\n63Ye8DuaRICUI+k88dViapaJ5a6LavodR8KQiuA0VCpRkE9ua8nL1zZk6ZZ9dHxzGkOnrSFDWwfi\noVe/+4PNyYd4+dqG5Msb5XccCUMqgtNkZtzYvDKTB1zI+TVL8e+Jy+j63q8s37rf72gShuas283I\nmeu5pXVV4qvoimPiDRXBGSpbND9DezXjrW5N2LD7IFe+/Qtv/rBSU1xLjjmclsEjYxdSvmgBHrrs\nHL/jSBhTEZwFM6Nz4/JMHtCWyxuU4/UfVtB50HQWbtQU13L2Bv20itU7DvBC14YUypfX7zgSxlQE\nOaBkbD7e6taEob2asftAKle/M4MX/7OMw2ma4lrOzNLN+xj882q6Nq3AhbV1sRnxloogB3WoF8fk\n+y/k+vhKvP/zGi5/8xd+X7vb71gSYv662EzRAtE82ame33EkAnhaBGbW0cyWm9kqM3v0OI/3NrMd\nZjY/8NXXyzy5oWiBaF6+rhGf3NaStIxMbnj/N576ZjGH0nVmkWTP8BlrWbQpmX91qU/xQrrYjHjP\nsx2PZhYFvAN0ADYCs81snHNu6TGrfuac6+9VDr+cXytriutXv1/OR7+tY0I+o1DlHbTVZr6cxLqd\nB3ht0gouqRtHp4bl/I4jEcLLLYIWwCrn3BrnXCqQCHTx8OcFnUL58vJM5/p8cXtroqOg1/DfefCL\nBSQf1AVw5O+cczz65UJiovLw/NW62IzkHvNqIjUzuw7o6JzrG7jfE2h59F//ZtYbeBHYAawABjjn\nNhznuRKABIC4uLj4xMRETzJ7ac++FH7cGsPEtWkUjjF61YshPi68zwRJSUkhNjbW7xi56mzGPHVD\nGiOWpNK7fgztKkXncDJv6DUOHe3bt09yzjU73mN+vxN9C4x2zh0xs9uBj4CLjl3JOTcEGALQrFkz\n165du1wNmROmTp3KoM7tWLwpmYfGLOTtefvo1LAEz3SuT+nC+fyO54mpU6cSiq/V2TjTMW9NPszd\nU36mVfUSPNWjFXnyhMbWgF7j8ODlrqFNwNGXT6oYWPZfzrldzrkjgbsfAPEe5gkKDSoUZVz/Njx4\naW0mL91Gh9d/5qt5muI6kjnnePKbxaRmZPJS10YhUwISPrwsgtlALTOrZmYxwE3AuKNXMLOjj4Z1\nBpZ5mCdoREflof9FtZhwz/lUK1WIAZ8t4LaP5rAlWVNcR6KJi7Yyeek27u9Qm6qldLEZyX2eFYFz\nLh3oD3xP1hv85865JWb2rJl1Dqx2j5ktMbMFwD1Ab6/yBKNacYUZc8d5PHllPX5dvZNLB07j01l/\naorrCLLnQCpPj1tMwwpFue38an7HkQjl6TEC59xEYOIxy5466vZjwGNeZgh2UXmM286vxiV1y/Do\n2EU8/tUixi3YxMvXNtKlCCPA8xOWsfdgGiNvbamLzYhv9D8vSFQpWYhP+7Xkxa4NWbJpH5e9MY0P\nftEU1+Hs5xU7GDt3I7dfWJ165XWxGfGPiiCImBndWlRm0v1tOa9GKZ6fsIzrBv/Kym2a4jrcHDiS\nzuNfLqJ66ULcfVEtv+NIhFMRBKFyRQsw7JZmvHHjuazbeYBOb03n7R9XkpahKa7DxavfL2fT3kO8\nfG0j8kfrYjPiLxVBkDIzrm5Sgcn3X0iH+nG8NnkFnQfNYPGmZL+jyVlKWr+Hj35bR6/WVWheVReb\nEf+pCIJcqdh8vNO9Ke/3jGdnyhG6vDODl7/7Q1Nch6gj6VkXmylXJD8Pd6zjdxwRQEUQMi6rX5Yf\nBlxI1yYVeG/qaq546xfmrNMU16HmnSmrWbU9hX9f05BYXWxGgoSKIIQULRjNq9c3ZuStLTiSlsn1\n7//GM+OWcOBIut/RJBv+2LqPd6es4upzy9O+Thm/44j8l4ogBLWtXZpJA9rSq1UVRvy6jsvemMYv\nK3f4HUtOIiPT8ciYhRQpEM1TV9X3O47I/1ARhKhC+fLyry4N+OKO1sRE5aHnsN95eMwCkg9piutg\n9OGMtSzYmMzTV9WjhC42I0FGRRDimlctwcR7L+COC2swdu4mOgz8mUlLtvodS46yZHMyr36/nIvq\nlKFz4/J+xxH5GxVBGMgfHcWjl9fh63+0oUShGBI+TqL/p3PZlXLk1N8sntp9IJWEkUkULxjDy9c2\n0sVmJCipCMJIw4pFGdf/fO7vUJvvl2zlkoE/8838TZri2ifpGZncPXouO/YfYXDP+LC97oSEPhVB\nmInJm4d7Lq7FhHsuoHLJQtybOJ++H81ha/Jhv6NFnFe+X86MVbt4/uoGnFupmN9xRE5IRRCmascV\n5ss7z+OfneoyY/VOOgz8mdG//6mtg1wybsFmhkxbQ89WVbiheaVTf4OIj1QEYSwqj9H3gup8d29b\n6pUvwmNfLqLHB7P4c9dBv6OFtaWb9/HwmAU0r1qcJ6+s53cckVNSEUSAqqUKMbpfK/59TQMWbkzm\nsjemMXz6Wk1x7YGUVMftn8yhaIFo3unRlJi8+hWT4Kf/pREiTx6jR8sqTBrQllbVS/Ds+KVcP/hX\nVm3XFNc5JT0jk/cWHGZb8hEG3xxPmcL5/Y4kki0qgghTvlgBhvduzsAbGrNm5wGueHM670xZpSmu\nc8Crk5azZFcmz11dnyaVi/sdRyTbVAQRyMzo2rQikwdcyCX1yvDq98vpoimuz8r4hZt5/+c1tK+U\nlxubV/Y7jshpURFEsNKF8/Fuj3gG39yU7fuzprh+9XtNcX26lm3Zx0NfLCS+SnF61NX0ERJ6VARC\nxwbl+OH+tlx9bgXembKaTm/9QtL6PX7HCgl7D6Zy+8dJFM6fl/d6NCVvHn1yWEKPikAAKFYwhtdu\naMyIPs05nJbJdYN/5V/fLuFgqqa4PpGMTMfdo+exJfkQ790cT5kiOjgsoUlFIP+j3Tll+H5AW25u\nWYUPZ2RNcT1j1U6/YwWl/5u0nF9W7uTZLg2Ir6KDwxK6VATyN7H58vLc1Q34LKEVUWb0+GAWj45d\nyL7DmuL6LxMWbuG9qavp1qIy3Vro4LCENk+LwMw6mtlyM1tlZo8e5/F8ZvZZ4PFZZlbVyzxyelpW\nL8l397Xl9rbV+XzOBjoM/Jkflm7zO5bvlm/dz0NjFtC0cjGe6axPDkvo86wIzCwKeAe4HKgHdDOz\nY39rbgP2OOdqAq8DL3uVR0a1FGEAAAjwSURBVM5M/ugoHruiLl/9ow3FCsTQd+Qc7hk9L2KnuE4+\nmEbCx3MolC8v790cT768UX5HEjlrXm4RtABWOefWOOdSgUSgyzHrdAE+CtweA1xsmrA9KDWuVIxv\n7z6f+y6pxX8Wb6HD69MYt2BzRE1il5HpuCdxHpv3HmLwzU2J08FhCRPm1S+ymV0HdHTO9Q3c7wm0\ndM71P2qdxYF1Ngburw6ss/OY50oAEgDi4uLiExMTPcnspZSUFGJjY/2OkSM27M9k+OIjrE3OpEmZ\nKHrVi6F4/r//TRFOYwYYsyKV8WvSuKVeDO0rRx93nXAb86lE2nghdMfcvn37JOdcs+M9lje3w5wJ\n59wQYAhAs2bNXLt27fwNdAamTp1KKOY+kW5XZDJ8xlpem7SCp2am8WSnelzfrOL/XIErnMb89bxN\njF8zn24tKvGvro1OuF44jTk7Im28EJ5j9nLX0Cbg6InYKwaWHXcdM8sLFAV2eZhJckjeqDwktK3B\nd/e1pW7ZIjw8diG9hv/Oht3hN8X1xzPXM+Dz+bSsVoJnOtf3O45IjvOyCGYDtcysmpnFADcB445Z\nZxxwS+D2dcBPLpJ2OoeBaqUKkZjQiueubsDc9Xu47I1pjJixlswwmOLaOcdbP67kya8Xc3GdMnx0\nawsdHJaw5NmuIedcupn1B74HooDhzrklZvYsMMc5Nw4YBnxsZquA3WSVhYSYPHmMnq2qcFGdMjz+\n5SKe+XYp4xdu4dpKoTujaWam49nxSxnx6zq6Nq3Ay9c2IjpKH7uR8OTpMQLn3ERg4jHLnjrq9mHg\nei8zSO6pUKwAI/o0Z+zcTTw3filPbkhjT+wqEi6oTt4QehNNy8jkoS8W8PX8zdzaphr/7FSXPJpD\nSMJY6Px2SkgwM66Lr8jk+9vSuHQUr3y3nKvfncHSzfv8jpYth1IzSBg5h6/nb+ahy87hyStVAhL+\nVATiiTKF83N3k/y826MpW5MP03nQdF6btJwj6cE7xXXyoTR6DpvF1BU7+Pc1DbirfU30sRaJBCoC\n8dQVDcsxecCFdG5cnrd/WsWVb01n7p/BN8X19n2HufH931iwcS+DujWlR8sqfkcSyTUqAvFc8UIx\nDLzxXD7s3ZyUI+lc+96vPDd+KYdSg2PrYP2uA1w3+Df+3H2Q4b2b06lROb8jieQqFYHkmvZ1yjBp\nQFu6t6jMsOlrueyNafy62t8prpdu3se17/3GvsNpfNqvFRfUKu1rHhE/qAgkVxXOH82/r2nI6H6t\nMIPuQ2fx2JeLfJnieva63dw45Deio4wxd7Tm3ErFcj2DSDBQEYgvWtcoyXf3tqXfBdX4bPafXDpw\nGj/9kXtTXP/0xzZu/mAWpQvnY8yd51GzTOFc+9kiwUZFIL4pEBPFE53qMfbO8yhSIC+3jpjDfYnz\n2HMg1bOfmZnpGJu0kX4jk6gdV5gvbm9NhWIFPPt5IqEgJCadk/DWpHJxvr37fN6Zspp3p6zil5U7\n+VeX+nRqWO60T990zrEzJZWNew6ycc8hNgT+3bjnEBt3H2Tj3kOkpmdyXo2SDOnVjNh8+hUQ0W+B\nBIV8eaO4v0NtOtYvyyNjF9L/03mMq7eZZ7s0oGzR/z/vv3OO3QdS//+b+56D//tmv+cgh9P+d2qL\nEoViqFi8AHXKFaZDvTiqlSrE1U0qkD9a8waJgIpAgky98kX46h/n8cH0tQycvIK2r06hXNH8pKZn\nkpqeyYHU9L+90RcrGE3F4gWoWTqWdrVLU6lEQSoWL0DF4ln/FtJf/SInpd8QCTp5o/Jwx4U16NSw\nHMOmr2XPwVRiovKQLzoPBaKjKFe0wH/f7CsUL0CR/Me/SIyIZI+KQIJWpRIFNf+/SC7QWUMiIhFO\nRSAiEuFUBCIiEU5FICIS4VQEIiIRTkUgIhLhVAQiIhFORSAiEuHMOed3htNiZjuA9X7nOAOlAH+v\nwpL7NObwF2njhdAdcxXn3HGvvBRyRRCqzGyOc66Z3zlyk8Yc/iJtvBCeY9auIRGRCKciEBGJcCqC\n3DPE7wA+0JjDX6SNF8JwzDpGICIS4bRFICIS4VQEIiIRTkXgAzN7wMycmZXyO4uXzOxVM/vDzBaa\n2VdmVszvTF4xs45mttzMVpnZo37n8ZqZVTKzKWa21MyWmNm9fmfKLWYWZWbzzGy831lyioogl5lZ\nJeBS4E+/s+SCyUAD51wjYAXwmM95PGFmUcA7wOVAPaCbmdXzN5Xn0oEHnHP1gFbAXREw5r/cCyzz\nO0ROUhHkvteBh4GwP0rvnJvknEsP3J0JVPQzj4daAKucc2ucc6lAItDF50yecs5tcc7NDdzeT9Yb\nYwV/U3nPzCoCnYAP/M6Sk1QEucjMugCbnHML/M7ig1uB//gdwiMVgA1H3d9IBLwp/sXMqgJNgFn+\nJskVb5D1h1ym30Fyki5en8PM7Aeg7HEeegJ4nKzdQmHjZON1zn0TWOcJsnYljMrNbOI9M4sFxgL3\nOef2+Z3HS2Z2JbDdOZdkZu38zpOTVAQ5zDl3yfGWm1lDoBqwwMwgazfJXDNr4ZzbmosRc9SJxvsX\nM+sNXAlc7ML3QyubgEpH3a8YWBbWzCyarBIY5Zz70u88uaAN0NnMrgDyA0XM7BPn3M0+5zpr+kCZ\nT8xsHdDMOReKsxhmi5l1BAYCFzrndvidxytmlpesg+EXk1UAs4HuzrklvgbzkGX9NfMRsNs5d5/f\neXJbYIvgQefclX5nyQk6RiBeGgQUBiab2XwzG+x3IC8EDoj3B74n66Dp5+FcAgFtgJ7ARYHXdn7g\nL2UJQdoiEBGJcNoiEBGJcCoCEZEIpyIQEYlwKgIRkQinIhARiXAqAhGRCKciEBGJcCoCkbNkZncc\n9aGqtWY2xe9MIqdDHygTySGBuXd+Al5xzn3rdx6R7NIWgUjOeRP4SSUgoUazj4rkgMAsq1XImnNI\nJKRo15DIWTKzeLJm4rzAObfH7zwip0u7hkTOXn+gBDAlcMA4rC5jKOFPWwQiIhFOWwQiIhFORSAi\nEuFUBCIiEU5FICIS4VQEIiIRTkUgIhLhVAQiIhHu/wGSqOQ1UTUO/AAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JwuEJg8E-kiC",
        "colab_type": "code",
        "outputId": "93df54b1-d886-4669-8da3-78e8ba92b381",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "print (model.stem[0].activation)"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "AF(\n",
            "  (fc1): Linear(in_features=1, out_features=8, bias=True)\n",
            "  (fc2): Linear(in_features=8, out_features=1, bias=True)\n",
            ")\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}