Faster_RCNN from scratch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T02:59:53.861942Z",
     "start_time": "2019-08-26T02:59:46.436125Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torchvision\n",
    "\n",
    "from utils import *\n",
    "\n",
    "%load_ext autoreload"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Extraction"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Set the input image (H, W) = (800, 800)\n",
    "Use the VGG16 to extrcat features -> (50, 50)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**bbox and anchor -> y1, x1, y2, x2**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T02:59:53.900978Z",
     "start_time": "2019-08-26T02:59:53.863898Z"
    }
   },
   "outputs": [],
   "source": [
    "image = torch.zeros((1, 3, 800, 800)).float()\n",
    "image_size = (800, 800)\n",
    "\n",
    "# bbox -> y1, x1, y2, x2\n",
    "bbox = torch.FloatTensor([[20, 30, 400, 500], [300, 400, 500, 600]])\n",
    "labels = torch.LongTensor([6, 8])\n",
    "\n",
    "sub_sample = 16"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use the first 30 layers of VGG16 to exact features. \n",
    "\n",
    "Because there are 4 maxpooling layers, (H, W) -> (H//16, W//16)\n",
    "\n",
    "13 conv layers, 13 ReLU layers, 4 Maxpooling layers -> 30 layers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:14.620894Z",
     "start_time": "2019-08-26T02:59:53.907961Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sequential(\n",
      "  (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "  (1): ReLU(inplace)\n",
      "  (2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "  (3): ReLU(inplace)\n",
      "  (4): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "  (5): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "  (6): ReLU(inplace)\n",
      "  (7): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "  (8): ReLU(inplace)\n",
      "  (9): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "  (10): Conv2d(128, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "  (11): ReLU(inplace)\n",
      "  (12): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "  (13): ReLU(inplace)\n",
      "  (14): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "  (15): ReLU(inplace)\n",
      "  (16): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "  (17): Conv2d(256, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "  (18): ReLU(inplace)\n",
      "  (19): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "  (20): ReLU(inplace)\n",
      "  (21): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "  (22): ReLU(inplace)\n",
      "  (23): MaxPool2d(kernel_size=2, stride=2, padding=0, dilation=1, ceil_mode=False)\n",
      "  (24): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "  (25): ReLU(inplace)\n",
      "  (26): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "  (27): ReLU(inplace)\n",
      "  (28): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
      "  (29): ReLU(inplace)\n",
      ")\n",
      "torch.Size([1, 512, 50, 50])\n"
     ]
    }
   ],
   "source": [
    "vgg16 = torchvision.models.vgg16(pretrained=True)\n",
    "req_features = vgg16.features[:30]\n",
    "print(req_features)\n",
    "output_map = req_features(image)\n",
    "print(output_map.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Target Anchor Box"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Assign template anchor boxes with origin (0, 0)\n",
    "\n",
    "We will use `anchor_scales(feature map)` of **8, 16, 32**, `ratio` of **0.5, 1, 2** and `sub sampling` of **16** (Since we have pooled our image from 800 px to 50px). Now every pixel in the output feature map maps to corresponding 16 * 16 pixels in the image.\n",
    "\n",
    "anchor_template = (9, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:14.857767Z",
     "start_time": "2019-08-26T03:00:14.694669Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ -45.254834    -90.50966799   45.254834     90.50966799]\n",
      " [ -64.          -64.           64.           64.        ]\n",
      " [ -90.50966799  -45.254834     90.50966799   45.254834  ]\n",
      " [ -90.50966799 -181.01933598   90.50966799  181.01933598]\n",
      " [-128.         -128.          128.          128.        ]\n",
      " [-181.01933598  -90.50966799  181.01933598   90.50966799]\n",
      " [-181.01933598 -362.03867197  181.01933598  362.03867197]\n",
      " [-256.         -256.          256.          256.        ]\n",
      " [-362.03867197 -181.01933598  362.03867197  181.01933598]]\n"
     ]
    }
   ],
   "source": [
    "anchor_scale = [8, 16, 32]\n",
    "ratio = [0.5, 1, 2] # H/W\n",
    "\n",
    "len_anchor_scale = len(anchor_scale)\n",
    "len_ratio = len(ratio)\n",
    "len_anchor_template = len_anchor_scale * len_ratio\n",
    "anchor_template = np.zeros((9, 4))\n",
    "\n",
    "for idx, scale in enumerate(anchor_scale):\n",
    "    h = scale * np.sqrt(ratio) * sub_sample\n",
    "    w = scale / np.sqrt(ratio) * sub_sample\n",
    "    y1 = -h/2\n",
    "    x1 = -w/2\n",
    "    y2 = h/2\n",
    "    x2 = w/2\n",
    "    anchor_template[idx*len_ratio:(idx+1)*len_ratio, 0] = y1\n",
    "    anchor_template[idx*len_ratio:(idx+1)*len_ratio, 1] = x1\n",
    "    anchor_template[idx*len_ratio:(idx+1)*len_ratio, 2] = y2\n",
    "    anchor_template[idx*len_ratio:(idx+1)*len_ratio, 3] = x2\n",
    "\n",
    "print(anchor_template)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:14.873755Z",
     "start_time": "2019-08-26T03:00:14.859796Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.70710678, 1.        , 1.41421356])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.sqrt(ratio)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate anchor at all the feature map location"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate center coors for all the feature map pixels\n",
    "ctr -> (50, 50, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:14.879709Z",
     "start_time": "2019-08-26T03:00:14.875752Z"
    }
   },
   "outputs": [],
   "source": [
    "feature_map_size = (50, 50)\n",
    "# The first center coors is (8, 8)\n",
    "ctr_y = np.arange(8, 800, 16)\n",
    "ctr_x = np.arange(8, 800, 16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:14.891685Z",
     "start_time": "2019-08-26T03:00:14.881704Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50, 50, 2)\n"
     ]
    }
   ],
   "source": [
    "ctr = np.zeros((*feature_map_size, 2))\n",
    "for idx, y in enumerate(ctr_y):\n",
    "    ctr[idx, :, 0] = y\n",
    "    ctr[idx, :, 1] = ctr_x\n",
    "print(ctr.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add center coors and previous anchor boxes coors\n",
    "\n",
    "anchors -> (50, 50, 9, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:14.922592Z",
     "start_time": "2019-08-26T03:00:14.895665Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50, 50, 9, 4)\n"
     ]
    }
   ],
   "source": [
    "anchors = np.zeros((*feature_map_size, 9, 4))\n",
    "\n",
    "for idx_y in range(feature_map_size[0]):\n",
    "    for idx_x in range(feature_map_size[1]):\n",
    "        anchors[idx_y, idx_x] = (ctr[idx_y, idx_x] + anchor_template.reshape(-1, 2, 2)).reshape(-1, 4)\n",
    "print(anchors.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:14.931570Z",
     "start_time": "2019-08-26T03:00:14.925584Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(22500, 4)\n"
     ]
    }
   ],
   "source": [
    "anchors = anchors.reshape(-1, 4)\n",
    "print(anchors.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Assign labels and location of objects to each and every anchor"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Find the index of all valid anchor boxes\n",
    "\n",
    "valid anchor boxes -> inside the image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:14.945542Z",
     "start_time": "2019-08-26T03:00:14.933570Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8940,)\n"
     ]
    }
   ],
   "source": [
    "valid_index = np.where((anchors[:, 0] >= 0)\n",
    "                      &(anchors[:, 1] >= 0)\n",
    "                      &(anchors[:, 2] <= 800)\n",
    "                      &(anchors[:, 3] <= 800))[0]\n",
    "print(valid_index.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We assign a **positive label to two kind of anchors** a) The anchor/anchors with the highest Intersection-over-Union(IoU) overlap with a ground-truth-box or b) An anchor that has an IoU overlap higher than 0.7 with ground-truth box.\n",
    "\n",
    "\n",
    "**Note that single ground-truth object may assign positive labels to multiple anchors.**\n",
    "\n",
    "\n",
    "c) We assign a **negative label** to a non-positive anchor if its IoU ratio is lower than 0.3 for all ground-truth boxes. \n",
    "\n",
    "d) Anchors that are **neither positive nor negitive** do not contribute to the training objective."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### create an empty label array with inside_index shape and fill with -1. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:14.952514Z",
     "start_time": "2019-08-26T03:00:14.947536Z"
    }
   },
   "outputs": [],
   "source": [
    "valid_labels = np.empty((valid_index.shape[0],), dtype=np.int32)\n",
    "valid_labels.fill(-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create valid anchors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:14.958498Z",
     "start_time": "2019-08-26T03:00:14.954510Z"
    }
   },
   "outputs": [],
   "source": [
    "valid_anchors = anchors[valid_index]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:14.969473Z",
     "start_time": "2019-08-26T03:00:14.962512Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8940, 4)\n",
      "torch.Size([2, 4])\n"
     ]
    }
   ],
   "source": [
    "print(valid_anchors.shape)\n",
    "print(bbox.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate ious matrix between valid_anchors and bbox"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:15.293603Z",
     "start_time": "2019-08-26T03:00:14.972458Z"
    }
   },
   "outputs": [],
   "source": [
    "ious = bbox_iou(valid_anchors, bbox.numpy())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Label pos and neg anchors\n",
    "\n",
    "Considering the scenarios of a and b, we need to find two things here\n",
    "- the highest iou for each gt_box and its corresponding anchor box\n",
    "- the highest iou for each anchor box and its corresponding ground truth box"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Set `pos_iou_thres` and `neg_iou_thres` to set anchors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:15.304572Z",
     "start_time": "2019-08-26T03:00:15.296592Z"
    }
   },
   "outputs": [],
   "source": [
    "pos_iou_thres = 0.7\n",
    "neg_iou_thred = 0.3\n",
    "\n",
    "anchor_max_iou = np.amax(ious, axis=1)\n",
    "pos_iou_anchor_label = np.where(anchor_max_iou >= pos_iou_thres)[0]\n",
    "neg_iou_anchor_label = np.where(anchor_max_iou < neg_iou_thred)[0]\n",
    "valid_labels[pos_iou_anchor_label] = 1\n",
    "valid_labels[neg_iou_anchor_label] = 0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Assign labels to the anchors with highest iou with box"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:15.314545Z",
     "start_time": "2019-08-26T03:00:15.306566Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2264 2271 4097 4105 4113 4121 4371 4379 4387 4395 4645 4653 4661 4669\n",
      " 4919 4927 4935 4943]\n"
     ]
    }
   ],
   "source": [
    "gt_max_iou = np.amax(ious, axis=0)\n",
    "gt_max_iou_anchor_label = np.where(ious == gt_max_iou)[0]\n",
    "print(gt_max_iou_anchor_label)\n",
    "valid_labels[gt_max_iou_anchor_label] = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sample positive and negtive anchors\n",
    "\n",
    "> we randomly sample 256 anchors in an image to compute the loss function of a mini-batch, where the sampled positive and negative anchors have a ratio of up to 1:1. If there are fewer than 128 positive samples in an image, we pad the mini-batch with negitive ones."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:15.329506Z",
     "start_time": "2019-08-26T03:00:15.321528Z"
    }
   },
   "outputs": [],
   "source": [
    "n_sample_anchors = 256\n",
    "pos_ratio = 0.5\n",
    "\n",
    "total_n_pos = len(np.where(valid_labels == 1)[0])\n",
    "n_pos_sample = n_sample_anchors*pos_ratio if total_n_pos > n_sample_anchors*pos_ratio else total_n_pos\n",
    "n_neg_sample = n_sample_anchors - n_pos_sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:15.402310Z",
     "start_time": "2019-08-26T03:00:15.335492Z"
    }
   },
   "outputs": [],
   "source": [
    "pos_index = np.where(valid_labels == 1)[0]\n",
    "if len(pos_index) > n_sample_anchors*pos_ratio:\n",
    "    disable_index = np.random.choice(pos_index, size=len(pos_index)-n_pos_sample, replace=False)\n",
    "    valid_labels[disable_index] = -1\n",
    "\n",
    "neg_index = np.where(valid_labels == 0)[0]\n",
    "disable_index = np.random.choice(neg_index, size=len(neg_index) - n_neg_sample, replace=False)\n",
    "valid_labels[disable_index] = -1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Assigning locations to anchor boxes\n",
    "\n",
    "```\n",
    "t_{x} = (x - x_{a})/w_{a}\n",
    "t_{y} = (y - y_{a})/h_{a}\n",
    "t_{w} = log(w/ w_a)\n",
    "t_{h} = log(h/ h_a)\n",
    "```\n",
    "**x, y , w, h** are the groud truth box center co-ordinates which has maxmimum iou with corresponding anchor, width and height. **x_a, y_a, h_a and w_a** and anchor boxes center cooridinates, width and height."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:15.663612Z",
     "start_time": "2019-08-26T03:00:15.404305Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8940, 4)\n",
      "(8940, 4)\n"
     ]
    }
   ],
   "source": [
    "# Each anchor corresponds to a box\n",
    "\n",
    "argmax_iou = np.argmax(ious, axis=1)\n",
    "max_iou_box = bbox[argmax_iou].numpy()\n",
    "print(max_iou_box.shape)\n",
    "print(valid_anchors.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:15.673587Z",
     "start_time": "2019-08-26T03:00:15.665606Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8940, 4)\n"
     ]
    }
   ],
   "source": [
    "anchor_loc_format_target = format_loc(valid_anchors, max_iou_box)\n",
    "print(anchor_loc_format_target.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Final labels and locations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:15.683558Z",
     "start_time": "2019-08-26T03:00:15.675580Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(22500,)\n",
      "(22500, 4)\n"
     ]
    }
   ],
   "source": [
    "anchor_target_labels = np.empty((len(anchors),), dtype=np.int32)\n",
    "anchor_target_format_locations = np.zeros((len(anchors), 4), dtype=np.float32)\n",
    "\n",
    "anchor_target_labels.fill(-1)\n",
    "anchor_target_labels[valid_index] = valid_labels\n",
    "\n",
    "anchor_target_format_locations[valid_index] = anchor_loc_format_target\n",
    "\n",
    "print(anchor_target_labels.shape)\n",
    "print(anchor_target_format_locations.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Region Proposal Network\n",
    "\n",
    "To generate region proposals, we **slide a small network over the convolutional feature map output** that we obtained in the feature extraction module. This small network takes as input an n x n spatial window of the input convolutional feature map. Each sliding window is mapped to a lower-dimensional feature [512 features]. **This feature is fed into two sibling fully connected layers**\n",
    "- A box regrression layer\n",
    "- A box classification layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:15.929932Z",
     "start_time": "2019-08-26T03:00:15.685555Z"
    }
   },
   "outputs": [],
   "source": [
    "mid_channel = 512\n",
    "in_channel = 512\n",
    "n_anchor = 9\n",
    "\n",
    "conv1 = nn.Conv2d(in_channel, mid_channel, 3, 1, 1)\n",
    "reg_layer = nn.Conv2d(mid_channel, n_anchor*4, 1, 1, 0)\n",
    "cls_layer = nn.Conv2d(mid_channel, n_anchor*2, 1, 1, 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialize weights and bias\n",
    "The paper tells that they initialized these layers with zero mean and 0.01 standard deviation for weights and zeros for base."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:16.494946Z",
     "start_time": "2019-08-26T03:00:15.931920Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "conv1.weight.data.normal_(0, 0.01)\n",
    "conv1.bias.data.zero_()\n",
    "\n",
    "reg_layer.weight.data.normal_(0, 0.01)\n",
    "reg_layer.bias.data.zero_()\n",
    "\n",
    "cls_layer.weight.data.normal_(0, 0.01)\n",
    "cls_layer.bias.data.zero_()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:16.740261Z",
     "start_time": "2019-08-26T03:00:16.496910Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 36, 50, 50])\n",
      "torch.Size([1, 18, 50, 50])\n"
     ]
    }
   ],
   "source": [
    "x = conv1(output_map)\n",
    "anchor_pred_format_locations = reg_layer(x)\n",
    "anchor_pred_scores = cls_layer(x)\n",
    "\n",
    "print(anchor_pred_format_locations.shape)\n",
    "print(anchor_pred_scores.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reformat the output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lets **reformat these a bit and make it align with our anchor targets we designed previously**. We will also find the objectness scores for each anchor box, as this is used to for proposal layer which we will discuss in the next section"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:16.916788Z",
     "start_time": "2019-08-26T03:00:16.742255Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 22500, 4])\n",
      "torch.Size([1, 22500, 2])\n",
      "torch.Size([1, 22500])\n"
     ]
    }
   ],
   "source": [
    "anchor_pred_format_locations = anchor_pred_format_locations.permute(0, 2, 3, 1).contiguous().view(1, -1, 4)\n",
    "print(anchor_pred_format_locations.shape)\n",
    "\n",
    "anchor_pred_scores = anchor_pred_scores.permute(0, 2, 3, 1).contiguous().view(1, -1, 2)\n",
    "print(anchor_pred_scores.shape)\n",
    "\n",
    "objectness_pred_scores = anchor_pred_scores[:, :, 1]\n",
    "print(objectness_pred_scores.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RPN Loss\n",
    "\n",
    "![](./RPN-loss.png)\n",
    "\n",
    "where `p_{i}` is the predicted class label and `p_{i}^*` is the actual class score. `t_{i}` and `t_{i}^*` are the predicted co-oridinates and actual co-ordinates. The ground-truth label `p_{i}^*` is 1 if the the anchor is positive and 0 if the anchor is negative. We will see how this is done in Pytorch."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:16.927771Z",
     "start_time": "2019-08-26T03:00:16.918783Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(22500,)\n",
      "(22500, 4)\n",
      "torch.Size([1, 22500, 2])\n",
      "torch.Size([1, 22500, 4])\n"
     ]
    }
   ],
   "source": [
    "print(anchor_target_labels.shape)\n",
    "print(anchor_target_format_locations.shape)\n",
    "print(anchor_pred_scores.shape)\n",
    "print(anchor_pred_format_locations.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:16.948713Z",
     "start_time": "2019-08-26T03:00:16.931755Z"
    }
   },
   "outputs": [],
   "source": [
    "gt_rpn_format_locs = torch.from_numpy(anchor_target_format_locations)\n",
    "gt_rpn_scores = torch.from_numpy(anchor_target_labels)\n",
    "\n",
    "rpn_format_locs = anchor_pred_format_locations[0]\n",
    "rpn_scores = anchor_pred_scores[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### cls loss\n",
    "pred_cls_scores and anchor_labels are the predited objectness score and actual objectness score of the RPN network. We will use the following loss functions for Regression and classification respectively.\n",
    "For classification we use cross-entropy loss\n",
    "![](./Cross_entropy-loss.png)\n",
    "Cross Entropy Loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:17.501103Z",
     "start_time": "2019-08-26T03:00:16.950698Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.6935, grad_fn=<NllLossBackward>)\n"
     ]
    }
   ],
   "source": [
    "rpn_cls_loss = F.cross_entropy(rpn_scores, gt_rpn_scores.long(), ignore_index=-1)\n",
    "print(rpn_cls_loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### reg loss for positive target anchors\n",
    "For Regression we use smooth L1 loss as defined in the Fast RCNN paper,\n",
    "![](Smooth-L1-loss.png)\n",
    "Smooth L1 Loss\n",
    "They used L1 loss instead of L2 loss because the values of predicted regression head of RPN are not bounded. **Regression loss is also applied to the bounding boxes which have positive label**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:17.551099Z",
     "start_time": "2019-08-26T03:00:17.502069Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([18, 4])\n",
      "torch.Size([18, 4])\n"
     ]
    }
   ],
   "source": [
    "mask = gt_rpn_scores > 0\n",
    "mask_target_format_locs = gt_rpn_format_locs[mask]\n",
    "mask_pred_format_locs = rpn_format_locs[mask]\n",
    "\n",
    "print(mask_target_format_locs.shape)\n",
    "print(mask_pred_format_locs.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:17.731524Z",
     "start_time": "2019-08-26T03:00:17.553094Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(1.1740, grad_fn=<SumBackward0>)\n"
     ]
    }
   ],
   "source": [
    "x = torch.abs(mask_target_format_locs - mask_pred_format_locs)\n",
    "rpn_loc_loss = ((x<0.5).float()*(x**2)*0.5 + (x>0.5).float()*(x-0.5)).sum()\n",
    "print(rpn_loc_loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RPN total loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:17.765438Z",
     "start_time": "2019-08-26T03:00:17.733517Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(1.3457, grad_fn=<AddBackward0>)\n"
     ]
    }
   ],
   "source": [
    "rpn_lambda = 10\n",
    "N_reg = mask.float().sum()\n",
    "\n",
    "rpn_loss = rpn_cls_loss + rpn_lambda / N_reg * rpn_loc_loss\n",
    "print(rpn_loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generating proposals to feed Fast R-CNN network\n",
    "\n",
    "The proposal function will take the following parameters\n",
    "- Weather training_mode or testing mode\n",
    "- nms_thresh\n",
    "- n_train_pre_nms — number of bboxes before nms during training\n",
    "- n_train_post_nms — number of bboxes after nms during training\n",
    "- n_test_pre_nms — number of bboxes before nms during testing\n",
    "- n_test_post_nms — number of bboxes after nms during testing\n",
    "- min_size — minimum height of the object required to create a proposal."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Faster R_CNN says, RPN proposals highly overlap with each other. To reduced redundancy, we adopt non-maximum supression (NMS) on the proposal regions based on their cls scores. We fix the IoU threshold for NMS at 0.7, which leaves us about 2000 proposal regions per image. After an ablation study, the authors show that NMS does not harm the ultimate detection accuracy, but substantially reduces the number of proposals. After NMS, we use the top-N ranked proposal regions for detection. In the following we training Fast R-CNN using 2000 RPN proposals. During testing they evaluate only 300 proposals, they have tested this with various numbers and obtained this."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:17.772414Z",
     "start_time": "2019-08-26T03:00:17.768426Z"
    }
   },
   "outputs": [],
   "source": [
    "nms_thresh = 0.7\n",
    "n_train_pre_nms = 12000\n",
    "n_train_post_nms = 2000\n",
    "n_test_pre_nms = 6000\n",
    "n_test_post_nms = 300\n",
    "min_size = 16"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T19:35:00.042227Z",
     "start_time": "2019-08-25T19:35:00.034248Z"
    }
   },
   "source": [
    "- convert the loc predictions from the rpn network to bbox [y1, x1, y2, x2] format.\n",
    "- clip the predicted boxes to the image\n",
    "- Remove predicted boxes with either height or width < threshold (min_size).\n",
    "- Sort all (proposal, score) pairs by score from highest to lowest.\n",
    "- Take top pre_nms_topN (e.g. 12000 while training and 300 while testing).\n",
    "- Apply nms threshold > 0.7\n",
    "- Take top pos_nms_topN (e.g. 2000 while training and 300 while testing)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## convert the loc predictions from the rpn net to [y1, x1, y2, x2] format\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:17.786376Z",
     "start_time": "2019-08-26T03:00:17.776418Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(22500, 4)\n",
      "torch.Size([1, 22500, 4])\n"
     ]
    }
   ],
   "source": [
    "print(anchors.shape)\n",
    "print(anchor_pred_format_locations.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:17.802334Z",
     "start_time": "2019-08-26T03:00:17.788372Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(22500, 4)\n",
      "[[ -37.56205856  -83.65124834   55.51502551   96.9647187 ]\n",
      " [ -59.50866938  -56.68875009   64.91222143   72.23375052]\n",
      " [ -81.40298363  -41.99777969   96.39533509   49.35743635]\n",
      " ...\n",
      " [ 610.35422226  414.3952291   979.0893042  1163.98340092]\n",
      " [ 538.20066833  564.81064224 1041.29725647 1063.15491104]\n",
      " [ 432.48094419  606.7697889  1166.24708388  973.39356325]]\n"
     ]
    }
   ],
   "source": [
    "rois = deformat_loc(anchors=anchors, formatted_base_anchor=anchor_pred_format_locations[0].data.numpy())\n",
    "print(rois.shape)\n",
    "print(rois)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## clip the predicted boxes to the image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:17.833253Z",
     "start_time": "2019-08-26T03:00:17.804329Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[  0.           0.          55.51502551  96.9647187 ]\n",
      " [  0.           0.          64.91222143  72.23375052]\n",
      " [  0.           0.          96.39533509  49.35743635]\n",
      " ...\n",
      " [610.35422226 414.3952291  800.         800.        ]\n",
      " [538.20066833 564.81064224 800.         800.        ]\n",
      " [432.48094419 606.7697889  800.         800.        ]]\n"
     ]
    }
   ],
   "source": [
    "rois[:, 0:4:2] = np.clip(rois[:, 0:4:2], a_min=0, a_max=image_size[0])\n",
    "rois[:, 1:4:2] = np.clip(rois[:, 1:4:2], a_min=0, a_max=image_size[1])\n",
    "print(rois)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Remove predicted boxes with either height or width < threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:17.846217Z",
     "start_time": "2019-08-26T03:00:17.835246Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(22500, 4)\n",
      "(22500,)\n"
     ]
    }
   ],
   "source": [
    "h = rois[:, 2] - rois[:, 0]\n",
    "w = rois[:, 3] - rois[:, 1]\n",
    "\n",
    "valid_index = np.where((h>min_size)&(w>min_size))[0]\n",
    "valid_rois = rois[valid_index]\n",
    "valid_scores = objectness_pred_scores[0][valid_index].data.numpy()\n",
    "\n",
    "print(valid_rois.shape)\n",
    "print(valid_scores.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## sort all pairs by score from highest to lowest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:17.883118Z",
     "start_time": "2019-08-26T03:00:17.849210Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[433  16   1 ... 913 432   9]\n"
     ]
    }
   ],
   "source": [
    "valid_score_order = valid_scores.ravel().argsort()[::-1]\n",
    "print(valid_score_order)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Take top pre_nms_topN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:17.893098Z",
     "start_time": "2019-08-26T03:00:17.885129Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(12000, 4)\n",
      "(12000,)\n",
      "(12000,)\n"
     ]
    }
   ],
   "source": [
    "pre_train_valid_score_order = valid_score_order[:n_train_pre_nms]\n",
    "pre_train_valid_rois = valid_rois[pre_train_valid_score_order]\n",
    "pre_train_valid_scores = valid_scores[pre_train_valid_score_order]\n",
    "\n",
    "print(pre_train_valid_rois.shape)\n",
    "print(pre_train_valid_scores.shape)\n",
    "print(pre_train_valid_score_order.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Apply NMS threshold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Take all the roi boxes [roi_array]\n",
    "- Find the areas of all the boxes [roi_area]\n",
    "- Take the indexes of order the probability score in descending order [order_array]\n",
    "keep = []\n",
    "while order_array.size > 0:\n",
    "  - take the first element in order_array and append that to keep  \n",
    "  - Find the area with all other boxes\n",
    "  - Find the index of all the boxes which have high overlap with this box\n",
    "  - Remove them from order array\n",
    "  - Iterate this till we get the order_size to zero (while loop)\n",
    "- Ouput the keep variable which tells what indexes to consider."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:18.948273Z",
     "start_time": "2019-08-26T03:00:17.896084Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2000, 4)\n",
      "(2000,)\n"
     ]
    }
   ],
   "source": [
    "keep_index = nms(rois=pre_train_valid_rois, scores=pre_train_valid_scores, nms_thresh=nms_thresh)\n",
    "post_train_valid_rois = pre_train_valid_rois[keep_index][:n_train_post_nms]\n",
    "post_train_valid_scores = pre_train_valid_scores[keep_index][:n_train_post_nms]\n",
    "print(post_train_valid_rois.shape)\n",
    "print(post_train_valid_scores.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Proposal targets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Fast R-CNN network takes the region proposals (obtained from proposal layer in previous section), ground truth boxes and their respective labels as inputs. It will take the following parameters\n",
    "- n_sample: Number of samples to sample from roi, The default value is 128.\n",
    "- pos_ratio: the number of positive examples out of the n_samples. The default values is 0.25.\n",
    "- pos_iou_thesh: The minimum overlap of region proposal with any groundtruth object to consider it as positive label.\n",
    "- [neg_iou_threshold_lo, neg_iou_threshold_hi] : [0.0, 0.5], The overlap value bounding required to consider a region proposal as negitive [background object]."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**This Step I think is used in the training process. In test step, we just use the proposal after NMS as the input of Fast RCNN network.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:18.956250Z",
     "start_time": "2019-08-26T03:00:18.950266Z"
    }
   },
   "outputs": [],
   "source": [
    "n_sample = 128\n",
    "pos_ratio = 0.25\n",
    "pos_iou_thresh = 0.5\n",
    "neg_iou_thresh_hi = 0.5\n",
    "neg_iou_thresh_lo = 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:18.965227Z",
     "start_time": "2019-08-26T03:00:18.958245Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2000, 4)\n",
      "(2000,)\n"
     ]
    }
   ],
   "source": [
    "print(post_train_valid_rois.shape)\n",
    "print(post_train_valid_scores.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Find the iou of each ground truth object with the region proposals\n",
    "\n",
    "same as 2.3.4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:19.617492Z",
     "start_time": "2019-08-26T03:00:18.966239Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2000, 2)\n"
     ]
    }
   ],
   "source": [
    "ious = bbox_iou(post_train_valid_rois, bbox)\n",
    "print(ious.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T00:49:07.040885Z",
     "start_time": "2019-08-26T00:49:07.036892Z"
    }
   },
   "source": [
    "## Assign labels for each proposal\n",
    "\n",
    "same as 2.3.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:19.625489Z",
     "start_time": "2019-08-26T03:00:19.619477Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([2000])\n"
     ]
    }
   ],
   "source": [
    "bbox_assignments = ious.argmax(axis=1)\n",
    "roi_max_ious = ious.max(axis=1)\n",
    "roi_target_labels = labels[bbox_assignments]\n",
    "print(roi_target_labels.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sample pos and neg samples\n",
    "\n",
    "same as 2.3.6\n",
    "\n",
    "- Select the foreground rois as per the pos_iou_thesh. We also want only n_sample x pos_ratio (128 x 0.25 = 32) foreground samples. So incase if we get less than 32 positive samples we will leave it as it is, Incase if we get more than 32 foreground samples, we will sample 32 samples from the positive samples. This is done using the following code."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:19.635435Z",
     "start_time": "2019-08-26T03:00:19.627483Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\n",
      "118\n"
     ]
    }
   ],
   "source": [
    "total_n_pos = len(np.where(roi_max_ious >= pos_iou_thresh)[0])\n",
    "n_pos_sample = n_sample*pos_ratio if total_n_pos > n_sample*pos_ratio else total_n_pos\n",
    "n_neg_sample = n_sample - n_pos_sample\n",
    "\n",
    "print(n_pos_sample)\n",
    "print(n_neg_sample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:19.650399Z",
     "start_time": "2019-08-26T03:00:19.637430Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10,)\n",
      "(118,)\n"
     ]
    }
   ],
   "source": [
    "pos_index = np.where(roi_max_ious >= pos_iou_thresh)[0]\n",
    "pos_index = np.random.choice(pos_index, size=n_pos_sample, replace=False)\n",
    "\n",
    "neg_index = np.where((roi_max_ious < neg_iou_thresh_hi) & (roi_max_ious > neg_iou_thresh_lo))[0]\n",
    "neg_index = np.random.choice(neg_index, size=n_neg_sample, replace=False)\n",
    "\n",
    "print(pos_index.shape)\n",
    "print(neg_index.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gather positve samples index and negitive samples index, their respective labels and region proposals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:19.658400Z",
     "start_time": "2019-08-26T03:00:19.653388Z"
    }
   },
   "outputs": [],
   "source": [
    "keep_index = np.append(pos_index, neg_index)\n",
    "post_sample_target_labels = roi_target_labels[keep_index].data.numpy()\n",
    "post_sample_target_labels[len(pos_index):] = 0\n",
    "post_sample_rois = post_train_valid_rois[keep_index]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pick the ground truth objects for these sample_roi and later parameterize\n",
    "\n",
    "same as 2.3.7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:19.669357Z",
     "start_time": "2019-08-26T03:00:19.663362Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(128, 4)\n"
     ]
    }
   ],
   "source": [
    "post_sample_bbox = bbox[bbox_assignments[keep_index]]\n",
    "post_sample_format_rois = format_loc(anchors=post_sample_rois, base_anchors=post_sample_bbox.data.numpy())\n",
    "print(post_sample_format_rois.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fast R-CNN\n",
    "\n",
    "Fast R-CNN used ROI pooling to extract features for each and every proposal suggested by selective search (Fast RCNN) or Region Proposal network (RPN in Faster R- CNN). "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Region of interest pooling (also known as RoI pooling) purpose is to perform max pooling on inputs of non-uniform sizes to obtain fixed-size feature maps (e.g. 7×7). This layer takes two inputs\n",
    "\n",
    "- A fixed-size feature map obtained from a deep convolutional network with several convolutions and max-pooling layers\n",
    "- An Nx5 matrix of representing a list of regions of interest, where N is the number of RoIs. The first column represents the image index and the remaining four are the co-ordinates of the top left and bottom right corners of the region."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "What does the RoI pooling actually do? For every region of interest from the input list, it takes a section of the input feature map that corresponds to it and scales it to some pre-defined size (e.g., 7×7). The scaling is done by:\n",
    "- Dividing the region proposal into equal-sized sections (the number of which is the same as the dimension of the output)\n",
    "- Finding the largest value in each section\n",
    "- Copying these max values to the output buffer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:19.679319Z",
     "start_time": "2019-08-26T03:00:19.671339Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([128, 4])\n"
     ]
    }
   ],
   "source": [
    "rois = torch.from_numpy(post_sample_rois).float()\n",
    "print(rois.shape)\n",
    "# roi_indices = torch.zeros((len(rois),1), dtype=torch.float32)\n",
    "# print(rois.shape, roi_indices.shape)\n",
    "\n",
    "# indices_and_rois = torch.cat([roi_indices, rois], dim=1)\n",
    "# print(indices_and_rois.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we need to pass this array to the roi_pooling layer. We will briefly discuss the workings of it here. The sudo code is as follows\n",
    "\n",
    "- Multiply the dimensions of rois with the sub_sampling ratio (16 in this case)\n",
    "- Empty output Tensor\n",
    "- Take each roi\n",
    "    - subset the **feature map** based on the roi dimension\n",
    "    - Apply AdaptiveMaxPool2d to this subset Tensor.\n",
    "    - Add the outputs to the output Tensor\n",
    "- Empty output Tensor goes to the network\n",
    "\n",
    "We will define the size to be 7 x 7 and define adaptive_max_pool"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ROI Pooling layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:19.802257Z",
     "start_time": "2019-08-26T03:00:19.682315Z"
    }
   },
   "outputs": [],
   "source": [
    "size = (7, 7)\n",
    "adaptive_max_pool = nn.AdaptiveMaxPool2d(size)\n",
    "\n",
    "# correspond to feature map\n",
    "rois.mul_(1/16.0)\n",
    "rois = rois.long()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:20.121572Z",
     "start_time": "2019-08-26T03:00:19.804253Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([128, 512, 7, 7])\n"
     ]
    }
   ],
   "source": [
    "output = []\n",
    "num_rois = len(rois)\n",
    "for roi in rois:\n",
    "    roi_feature = output_map[..., roi[0]:roi[2]+1, roi[1]:roi[3]+1]\n",
    "    output.append(adaptive_max_pool(roi_feature))\n",
    "output = torch.cat(output, 0)\n",
    "print(output.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:20.129549Z",
     "start_time": "2019-08-26T03:00:20.123566Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([128, 25088])\n"
     ]
    }
   ],
   "source": [
    "output_ROI_pooling = output.view(output.size(0), -1)\n",
    "print(output_ROI_pooling.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loc and Cls layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:21.172784Z",
     "start_time": "2019-08-26T03:00:20.131544Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.])"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roi_head = nn.Sequential(nn.Linear(25088, 4096),\n",
    "                        nn.Linear(4096, 4096))\n",
    "\n",
    "cls_loc = nn.Linear(4096, 21*4)\n",
    "cls_loc.weight.data.normal_(0, 0.01)\n",
    "cls_loc.bias.data.zero_()\n",
    "\n",
    "cls_score = nn.Linear(4096, 21)\n",
    "cls_score.weight.data.normal_(0, 0.01)\n",
    "cls_score.bias.data.zero_()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:21.597940Z",
     "start_time": "2019-08-26T03:00:21.176757Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([128, 84]) torch.Size([128, 21])\n"
     ]
    }
   ],
   "source": [
    "x = roi_head(output_ROI_pooling)\n",
    "roi_cls_loc = cls_loc(x)\n",
    "roi_cls_score = cls_score(x)\n",
    "\n",
    "print(roi_cls_loc.shape, roi_cls_score.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fast R-CNN loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- predicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:21.605933Z",
     "start_time": "2019-08-26T03:00:21.599935Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([128, 84])\n",
      "torch.Size([128, 21])\n"
     ]
    }
   ],
   "source": [
    "print(roi_cls_loc.shape)\n",
    "print(roi_cls_score.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:00:21.615898Z",
     "start_time": "2019-08-26T03:00:21.608910Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(128, 4)\n",
      "(128,)\n"
     ]
    }
   ],
   "source": [
    "print(post_sample_format_rois.shape)\n",
    "print(post_sample_target_labels.shape)\n",
    "\n",
    "gt_roi_cls_loc = torch.from_numpy(post_sample_format_rois).float()\n",
    "gt_roi_cls_label = torch.from_numpy(post_sample_target_labels).long()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### cls loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:01:07.294447Z",
     "start_time": "2019-08-26T03:01:07.286492Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(3.0664, grad_fn=<NllLossBackward>)\n"
     ]
    }
   ],
   "source": [
    "roi_cls_loss = F.cross_entropy(roi_cls_score, gt_roi_cls_label)\n",
    "print(roi_cls_loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### reg loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:01:08.211650Z",
     "start_time": "2019-08-26T03:01:08.058534Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([128, 4])\n"
     ]
    }
   ],
   "source": [
    "num_roi = roi_cls_loc.size(0)\n",
    "roi_cls_loc = roi_cls_loc.view(-1, 21, 4)\n",
    "roi_cls_loc = roi_cls_loc[torch.arange(num_roi), gt_roi_cls_label]\n",
    "print(roi_cls_loc.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:01:08.325907Z",
     "start_time": "2019-08-26T03:01:08.315926Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([10, 4])\n",
      "torch.Size([10, 4])\n"
     ]
    }
   ],
   "source": [
    "mask = gt_roi_cls_label>0\n",
    "mask_loc_pred = roi_cls_loc[mask]\n",
    "mask_loc_target = gt_roi_cls_loc[mask]\n",
    "\n",
    "print(mask_loc_pred.shape)\n",
    "print(mask_loc_target.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:01:08.519546Z",
     "start_time": "2019-08-26T03:01:08.511580Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.6413, grad_fn=<SumBackward0>)\n"
     ]
    }
   ],
   "source": [
    "x = torch.abs(mask_loc_pred-mask_loc_target)\n",
    "roi_loc_loss = ((x<0.5).float()*x**2*0.5 + (x>0.5).float()*(x-0.5)).sum()\n",
    "print(roi_loc_loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### total loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:01:08.880641Z",
     "start_time": "2019-08-26T03:01:08.875660Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(3.7076, grad_fn=<AddBackward0>)\n"
     ]
    }
   ],
   "source": [
    "roi_lambda = 10\n",
    "N_reg = (gt_roi_cls_label>0).float().sum()\n",
    "roi_loss = roi_cls_loss + roi_lambda / N_reg * roi_loc_loss\n",
    "print(roi_loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Total loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-26T03:01:10.115228Z",
     "start_time": "2019-08-26T03:01:10.110244Z"
    }
   },
   "outputs": [],
   "source": [
    "total_loss = rpn_loss + roi_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "279.273px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}