Faster_RCNN from scratch_v1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T16:33:37.690149Z",
     "start_time": "2019-08-25T16:33:24.745127Z"
    }
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import numpy as np\n",
    "%load_ext autoreload"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T16:33:50.267307Z",
     "start_time": "2019-08-25T16:33:50.068536Z"
    }
   },
   "outputs": [],
   "source": [
    "image = torch.zeros((1, 3, 800, 800)).float()\n",
    "\n",
    "# [y1, x1, y2, x2] format\n",
    "bbox = torch.FloatTensor([[20, 30, 400, 500], [300, 400, 500, 600]])\n",
    "labels = torch.LongTensor([6, 8])\n",
    "\n",
    "# 1x1 in feature map -> 16x16 in image\n",
    "sub_sample = 16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T16:33:51.478998Z",
     "start_time": "2019-08-25T16:33:51.357325Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 3, 800, 800])\n"
     ]
    }
   ],
   "source": [
    "dummy_img = torch.zeros((1, 3, 800, 800)).float()\n",
    "print(dummy_img.size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Use the first 30 layers of VGG16 to exact features. \n",
    "\n",
    "Because there are 4 maxpooling layers, (H, W) -> (H//16, W//16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T00:11:46.231800Z",
     "start_time": "2019-08-24T00:11:43.733814Z"
    }
   },
   "outputs": [],
   "source": [
    "model = torchvision.models.vgg16(pretrained=True)\n",
    "fe = list(model.features)\n",
    "req_features = fe[:30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T00:11:46.240776Z",
     "start_time": "2019-08-24T00:11:46.235798Z"
    }
   },
   "outputs": [],
   "source": [
    "faster_rcnn_feature = nn.Sequential(*req_features)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sample output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T00:11:59.081607Z",
     "start_time": "2019-08-24T00:11:46.242771Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 512, 50, 50])\n"
     ]
    }
   ],
   "source": [
    "sample_output = faster_rcnn_feature(dummy_img)\n",
    "print(sample_output.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Anchor boxes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Assign template anchor boxes with origin (0, 0)\n",
    "At each point in the feature map, we assign 9 anchor boxes, each has 4 values(y1, x1, y2, x2)\n",
    "The following step will add the template anchor boxes coors and center coors to get anchor boxes.\n",
    "\n",
    "`anchor_sizes = [8, 16, 32]` is the size in feature map, but we need to get the anchors for image, so that we need multiply 16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T16:33:55.767115Z",
     "start_time": "2019-08-25T16:33:55.755146Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total anchor #: 9\n"
     ]
    }
   ],
   "source": [
    "# ration = h/w\n",
    "ratios = [0.5, 1, 2]\n",
    "# anchor_size 是 feature map上的size\n",
    "anchor_sizes = [8, 16, 32]\n",
    "anchor_number = len(ratios) * len(anchor_sizes)\n",
    "print(f'Total anchor #: {anchor_number}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T16:33:55.967585Z",
     "start_time": "2019-08-25T16:33:55.959624Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ -45.254834    -90.50966799   45.254834     90.50966799]\n",
      " [ -90.50966799 -181.01933598   90.50966799  181.01933598]\n",
      " [-181.01933598 -362.03867197  181.01933598  362.03867197]\n",
      " [ -64.          -64.           64.           64.        ]\n",
      " [-128.         -128.          128.          128.        ]\n",
      " [-256.         -256.          256.          256.        ]\n",
      " [ -90.50966799  -45.254834     90.50966799   45.254834  ]\n",
      " [-181.01933598  -90.50966799  181.01933598   90.50966799]\n",
      " [-362.03867197 -181.01933598  362.03867197  181.01933598]]\n"
     ]
    }
   ],
   "source": [
    "# assign value to anchor_shape \n",
    "\n",
    "# center coors as for each 16x16 patch in original image\n",
    "ctr_y = sub_sample / 2\n",
    "ctr_x = sub_sample / 2\n",
    "\n",
    "# all 9 anchors in original image\n",
    "anchors_template = np.zeros((9, 4))\n",
    "\n",
    "\n",
    "for i, ratio in enumerate(ratios):\n",
    "    for j, size in enumerate(anchor_sizes):\n",
    "#         get the h and w in original image\n",
    "        h = size * np.sqrt(ratio) * sub_sample\n",
    "        w = size / np.sqrt(ratio) * sub_sample\n",
    "#         anchor coors in original image\n",
    "        y1 = -h/2\n",
    "        x1 = -w/2\n",
    "        y2 =  h/2\n",
    "        x2 =  w/2\n",
    "        \n",
    "        anchor = [y1, x1, y2, x2]\n",
    "        anchors_template[i*len(ratios) + j] = anchor\n",
    "\n",
    "print(anchors_template)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "These are the **anchor locations at the first feature map pixel**, we have to now generate these anchors at all the locations of feature map. Also note that **negitive values** mean that the anchor boxes are outside image dimension. In the later section we will **label them with -1 and remove them when calculating the loss the functions and generating proposals for anchor boxes**. Also Since we got 9 anchors at each location and there **50 * 50** such locations inside an image, We will get **17500 (50 * 50 * 9)** anchors in total. Lets generate other anchors now,"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generate anchor at all the feature map location"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generate center coors for all the feature map pixels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T16:33:58.848495Z",
     "start_time": "2019-08-25T16:33:58.842541Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50,)\n"
     ]
    }
   ],
   "source": [
    "feature_map_size = 800 // 16\n",
    "\n",
    "ctr_x_all = np.arange(8, (feature_map_size + 1) * 16 - 8, 16)\n",
    "ctr_y_all = np.arange(8, (feature_map_size + 1) * 16 - 8, 16)\n",
    "\n",
    "print(ctr_x_all.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T16:33:59.364162Z",
     "start_time": "2019-08-25T16:33:59.354157Z"
    }
   },
   "outputs": [],
   "source": [
    "ctr = np.zeros((feature_map_size, feature_map_size, 2), dtype=np.float32)\n",
    "\n",
    "for y in range(feature_map_size):\n",
    "    for x in range(feature_map_size):\n",
    "        ctr[y, x] = np.array([ctr_y_all[y], ctr_x_all[x]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Add center coors and previous anchor boxes coors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T16:34:00.221133Z",
     "start_time": "2019-08-25T16:34:00.217118Z"
    }
   },
   "outputs": [],
   "source": [
    "# anchors -> (H//16, W//16, 9, 4)\n",
    "anchors = np.zeros((feature_map_size, feature_map_size, 9, 4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T16:34:00.437987Z",
     "start_time": "2019-08-25T16:34:00.425993Z"
    }
   },
   "outputs": [],
   "source": [
    "for y in range(feature_map_size):\n",
    "    for x in range(feature_map_size):\n",
    "#         use broadcast to get anchor boxes corresponding to feature(y, x)\n",
    "#         two reshape operations\n",
    "#         (y1, x1, y2, x2) -> ((y1, x1), (y2, x2))\n",
    "#         ((y1, x1), (y2, x2)) -> (y1, x1, y2, x2)\n",
    "        anchors[y, x] = (ctr[y, x] + anchors_template.reshape(-1, 2, 2)).reshape(-1, 4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T16:34:00.626649Z",
     "start_time": "2019-08-25T16:34:00.621677Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(50, 50, 9, 4)\n"
     ]
    }
   ],
   "source": [
    "print(anchors.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Assign labels and location of objects to each and every anchor"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We assign a **positive label to two kind of anchors** a) The anchor/anchors with the highest Intersection-over-Union(IoU) overlap with a ground-truth-box or b) An anchor that has an IoU overlap higher than 0.7 with ground-truth box.\n",
    "\n",
    "\n",
    "**Note that single ground-truth object may assign positive labels to multiple anchors.**\n",
    "\n",
    "\n",
    "c) We assign a **negative label** to a non-positive anchor if its IoU ratio is lower than 0.3 for all ground-truth boxes. \n",
    "\n",
    "d) Anchors that are **neither positive nor negitive** do not contribute to the training objective."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T16:34:01.995738Z",
     "start_time": "2019-08-25T16:34:01.991751Z"
    }
   },
   "outputs": [],
   "source": [
    "# generate two ground_truth_bbox and labels\n",
    "bbox = np.asarray([[20, 30, 400, 500], [300, 400, 500, 600]], dtype=np.float32) # [y1, x1, y2, x2] format\n",
    "labels = np.asarray([6, 8], dtype=np.int8) # 0 represents background"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Find the index of all valid anchor boxes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T16:34:03.471066Z",
     "start_time": "2019-08-25T16:34:03.466115Z"
    }
   },
   "outputs": [],
   "source": [
    "# convert anchors from (H//16, W//16, 4) -> (-1, 4) in order to get the anchors inside the image\n",
    "# because some anchors have negative coors\n",
    "anchors = anchors.reshape(-1, 4)\n",
    "\n",
    "index_inside = np.where((anchors[:, 0] >= 0) &\n",
    "                       (anchors[:, 1] >= 0) &\n",
    "                       (anchors[:, 2] <= 800) &\n",
    "                       (anchors[:, 3] <= 800))[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T17:01:59.227672Z",
     "start_time": "2019-08-25T17:01:59.214681Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8940,)\n"
     ]
    }
   ],
   "source": [
    "print(index_inside.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### create an empty label array with inside_index shape and fill with -1. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T17:01:59.776675Z",
     "start_time": "2019-08-25T17:01:59.769697Z"
    }
   },
   "outputs": [],
   "source": [
    "label = np.empty((len(index_inside),), dtype=np.int32)\n",
    "label.fill(-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### create an array with valid anchor boxes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T17:02:00.189913Z",
     "start_time": "2019-08-25T17:02:00.177915Z"
    }
   },
   "outputs": [],
   "source": [
    "valid_anchors = anchors[index_inside]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### For each anchor box calculate IOU with each ground truth object.\n",
    "\n",
    "ince we have 8940 anchor boxes and 2 ground truth objects, we should get an array with (8490, 2) as the output.\n",
    "\n",
    "```\n",
    "- Find the max of x1 and y1 in both the boxes (xn1, yn1)\n",
    "- Find the min of x2 and y2 in both the boxes (xn2, yn2)\n",
    "- Now both the boxes are intersecting only\n",
    " if (xn1 < xn2) and (yn2 < yn1)\n",
    "      - iou_area will be (xn2 - xn1) * (yn2 - yn1)\n",
    " else\n",
    "      - iuo_area will be 0\n",
    "- similarly calculate area for anchor box and ground truth object\n",
    "- iou = iou_area/(anchor_box_area + ground_truth_area - iou_area)\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T17:02:01.011196Z",
     "start_time": "2019-08-25T17:02:00.612892Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8940, 2)\n"
     ]
    }
   ],
   "source": [
    "ious = np.zeros((len(valid_anchors), len(bbox)), dtype=np.float32)\n",
    "\n",
    "for index_v, valid_anchor in enumerate(valid_anchors):\n",
    "    y1_v, x1_v, y2_v, x2_v = valid_anchor\n",
    "    anchor_area = (y2_v - y1_v) * (x2_v - x1_v)\n",
    "    \n",
    "    for index_t, bbox_t in enumerate(bbox):\n",
    "        y1_t, x1_t, y2_t, x2_t = bbox_t\n",
    "        bbox_area = (y2_t - y1_t) * (x2_t - x1_t)\n",
    "        \n",
    "        max_y1 = max([y1_v, y1_t])\n",
    "        max_x1 = max([x1_v, x1_t])\n",
    "        min_y2 = min([y2_v, y2_t])\n",
    "        min_x2 = min([x2_v, x2_t])\n",
    "        \n",
    "        if max_x1 < min_x2 and max_y1 < min_y2:\n",
    "            inter_area = (min_y2 - max_y1) * (min_x2 - max_x1)\n",
    "            iou = inter_area / (anchor_area + bbox_area - inter_area)\n",
    "        else:\n",
    "            iou = 0\n",
    "        \n",
    "        ious[index_v, index_t] = iou\n",
    "print(ious.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T17:15:09.695880Z",
     "start_time": "2019-08-25T17:15:09.690883Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.61035156\n"
     ]
    }
   ],
   "source": [
    "print(max(ious[:, 1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Assign labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Considering the scenarios of a and b, we need to find two things here\n",
    "- the highest iou for each gt_box and its corresponding anchor box\n",
    "- the highest iou for each anchor box and its corresponding ground truth box"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T17:02:01.440204Z",
     "start_time": "2019-08-25T17:02:01.427229Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2262 4096]\n",
      "[0.68130493 0.61035156]\n"
     ]
    }
   ],
   "source": [
    "# case 1\n",
    "gt_argmax_ious = ious.argmax(axis=0)\n",
    "print(gt_argmax_ious)\n",
    "\n",
    "gt_max_ious = ious[gt_argmax_ious, np.arange(len(bbox))]\n",
    "print(gt_max_ious)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T17:02:01.622843Z",
     "start_time": "2019-08-25T17:02:01.613878Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 0 0 ... 0 0 0]\n",
      "[0.07360116 0.08134178 0.08159019 ... 0.         0.         0.        ]\n"
     ]
    }
   ],
   "source": [
    "# case 2\n",
    "argmax_ious = ious.argmax(axis=1)\n",
    "print(argmax_ious)\n",
    "\n",
    "max_ious = ious[np.arange(ious.shape[0]), argmax_ious]\n",
    "print(max_ious)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Find the valid anchors which have the gt_max_ious**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T17:02:02.177932Z",
     "start_time": "2019-08-25T17:02:02.168922Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2262 2269 4096 4104 4112 4120 4370 4378 4386 4394 4644 4652 4660 4668\n",
      " 4918 4926 4934 4942]\n"
     ]
    }
   ],
   "source": [
    "# ious == gt_max_ious -> 并不是要求gt_max_ious一行的所有value都等于 gt_max_ious的value\n",
    "# 而是会生成一个 0/1 matrix 作为condition\n",
    "gt_argmax_ious = np.where(ious == gt_max_ious)[0]\n",
    "print(gt_argmax_ious)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we have three arrays\n",
    "- argmax_ious — Tells which ground truth object has max iou with each anchor.\n",
    "- max_ious — Tells the max_iou with ground truth object with each anchor.\n",
    "- gt_argmax_ious — Tells the anchors with the highest Intersection-over-Union (IoU) overlap with a ground-truth box."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T17:02:02.845864Z",
     "start_time": "2019-08-25T17:02:02.841903Z"
    }
   },
   "outputs": [],
   "source": [
    "pos_iou_threshold = 0.7\n",
    "neg_iou_threshold = 0.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T17:02:03.064664Z",
     "start_time": "2019-08-25T17:02:03.061637Z"
    }
   },
   "outputs": [],
   "source": [
    "label[max_ious < neg_iou_threshold] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T17:02:03.261452Z",
     "start_time": "2019-08-25T17:02:03.257464Z"
    }
   },
   "outputs": [],
   "source": [
    "label[gt_argmax_ious] = 1\n",
    "label[max_ious >= pos_iou_threshold] = 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sample positive and negtive anchors\n",
    "\n",
    "> we randomly sample 256 anchors in an image to compute the loss function of a mini-batch, where the sampled positive and negative anchors have a ratio of up to 1:1. If there are fewer than 128 positive samples in an image, we pad the mini-batch with negitive ones."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T17:02:05.023930Z",
     "start_time": "2019-08-25T17:02:05.020904Z"
    }
   },
   "outputs": [],
   "source": [
    "pos_ratio = 0.5\n",
    "n_sample = 256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T17:02:05.192761Z",
     "start_time": "2019-08-25T17:02:05.187748Z"
    }
   },
   "outputs": [],
   "source": [
    "n_pos = pos_ratio * n_sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T17:02:05.361567Z",
     "start_time": "2019-08-25T17:02:05.355561Z"
    }
   },
   "outputs": [],
   "source": [
    "# positive samples\n",
    "\n",
    "pos_index = np.where(label == 1)[0]\n",
    "\n",
    "if len(pos_index) > n_pos:\n",
    "    disable_index = np.random.choice(pos_index, size=len(pos_index) - n_pos, replace=False)\n",
    "    label[disable_index] = -1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T17:02:06.179959Z",
     "start_time": "2019-08-25T17:02:06.044609Z"
    }
   },
   "outputs": [],
   "source": [
    "# negative samples\n",
    "\n",
    "n_neg = n_sample - len(np.where(label==1)[0])\n",
    "\n",
    "neg_index = np.where(label == 0)[0]\n",
    "\n",
    "if len(neg_index) > n_neg:\n",
    "    disable_index = np.random.choice(neg_index, size=len(neg_index) - n_neg, replace=False)\n",
    "    label[disable_index] = -1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Assigning locations to anchor boxes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```\n",
    "t_{x} = (x - x_{a})/w_{a}\n",
    "t_{y} = (y - y_{a})/h_{a}\n",
    "t_{w} = log(w/ w_a)\n",
    "t_{h} = log(h/ h_a)\n",
    "```\n",
    "**x, y , w, h** are the groud truth box center co-ordinates which has maxmimum iou with corresponding anchor, width and height. **x_a, y_a, h_a and w_a** and anchor boxes center cooridinates, width and height."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T00:12:19.798896Z",
     "start_time": "2019-08-24T00:12:19.794879Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8940, 4)\n"
     ]
    }
   ],
   "source": [
    "# For each anchor box, find the groundtruth object which has max_iou as base\n",
    "max_iou_bbox = bbox[argmax_ious]\n",
    "print(max_iou_bbox.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T00:12:19.976199Z",
     "start_time": "2019-08-24T00:12:19.969189Z"
    }
   },
   "outputs": [],
   "source": [
    "height = valid_anchors[:, 2] - valid_anchors[:, 0]\n",
    "width = valid_anchors[:, 3] - valid_anchors[:, 1]\n",
    "ctr_y = valid_anchors[:, 0] + height*0.5\n",
    "ctr_x = valid_anchors[:, 1] + width*0.5\n",
    "\n",
    "base_height = max_iou_bbox[:, 2] - max_iou_bbox[:, 0]\n",
    "base_width = max_iou_bbox[:, 3] - max_iou_bbox[:, 1]\n",
    "base_ctr_y = max_iou_bbox[:, 0] + base_height*0.5\n",
    "base_ctr_x = max_iou_bbox[:, 1] + base_width*0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T00:12:20.188355Z",
     "start_time": "2019-08-24T00:12:20.179371Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(8940, 4)\n"
     ]
    }
   ],
   "source": [
    "eps = np.finfo(height.dtype).eps\n",
    "height = np.maximum(height, eps)\n",
    "width = np.maximum(width, eps)\n",
    "\n",
    "dy = (base_ctr_y - ctr_y) / height\n",
    "dx = (base_ctr_x - ctr_x) / width\n",
    "dh = np.log(base_height / height)\n",
    "dw = np.log(base_width / width)\n",
    "\n",
    "anchor_locs = np.vstack((dy, dx, dh, dw)).transpose()\n",
    "print(anchor_locs.shape)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Final labels and locations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T00:12:20.573039Z",
     "start_time": "2019-08-24T00:12:20.568051Z"
    }
   },
   "outputs": [],
   "source": [
    "# Final labels for all the anchors\n",
    "anchor_labels = np.empty((len(anchors),), dtype=np.int32)\n",
    "anchor_labels.fill(-1)\n",
    "anchor_labels[index_inside] = label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T00:12:20.752400Z",
     "start_time": "2019-08-24T00:12:20.748411Z"
    }
   },
   "outputs": [],
   "source": [
    "#] Final loc for all the anchors\n",
    "anchor_locations = np.empty((len(anchors), 4), dtype=np.float32)\n",
    "anchor_locations.fill(0)\n",
    "anchor_locations[index_inside] = anchor_locs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T00:12:20.955003Z",
     "start_time": "2019-08-24T00:12:20.949019Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(22500,)\n",
      "(22500, 4)\n"
     ]
    }
   ],
   "source": [
    "print(anchor_labels.shape)\n",
    "print(anchor_locations.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Region Proposal Network"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To generate region proposals, we **slide a small network over the convolutional feature map output** that we obtained in the feature extraction module. This small network takes as input an n x n spatial window of the input convolutional feature map. Each sliding window is mapped to a lower-dimensional feature [512 features]. **This feature is fed into two sibling fully connected layers**\n",
    "- A box regrression layer\n",
    "- A box classification layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T00:12:21.880797Z",
     "start_time": "2019-08-24T00:12:21.511053Z"
    }
   },
   "outputs": [],
   "source": [
    "mid_channels = 512\n",
    "in_channels = 512\n",
    "n_anchor = 9 # Number of anchors at each location in the feature map\n",
    "\n",
    "conv1 = nn.Conv2d(in_channels, mid_channels, 3, 1, 1)\n",
    "reg_layer = nn.Conv2d(mid_channels, n_anchor*4, 1, 1, 0)\n",
    "cls_layer = nn.Conv2d(mid_channels, n_anchor*2, 1, 1, 0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialize weights and bias\n",
    "The paper tells that they initialized these layers with zero mean and 0.01 standard deviation for weights and zeros for base."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T00:12:22.571485Z",
     "start_time": "2019-08-24T00:12:21.884743Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# conv1 sliding layer\n",
    "conv1.weight.data.normal_(0, 0.01)\n",
    "conv1.bias.data.zero_()\n",
    "\n",
    "# reg_layer\n",
    "reg_layer.weight.data.normal_(0, 0.01)\n",
    "reg_layer.bias.data.zero_()\n",
    "\n",
    "# cls_layer\n",
    "cls_layer.weight.data.normal_(0, 0.01)\n",
    "cls_layer.bias.data.zero_()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T00:12:22.827802Z",
     "start_time": "2019-08-24T00:12:22.574478Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 36, 50, 50]) torch.Size([1, 18, 50, 50])\n"
     ]
    }
   ],
   "source": [
    "x = conv1(sample_output) # sample_output is obtained in Section 1.2\n",
    "pred_anchor_locs = reg_layer(x)\n",
    "pred_cls_scores = cls_layer(x)\n",
    "\n",
    "print(pred_anchor_locs.shape, pred_cls_scores.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reformat the output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Lets **reformat these a bit and make it align with our anchor targets we designed previously**. We will also find the objectness scores for each anchor box, as this is used to for proposal layer which we will discuss in the next section"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T00:12:23.008323Z",
     "start_time": "2019-08-24T00:12:22.832787Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 22500, 4])\n",
      "torch.Size([1, 50, 50, 18])\n",
      "torch.Size([1, 22500])\n",
      "torch.Size([1, 22500, 2])\n"
     ]
    }
   ],
   "source": [
    "pred_anchor_locs = pred_anchor_locs.permute(0, 2, 3, 1).contiguous().view(1, -1, 4)\n",
    "print(pred_anchor_locs.shape)\n",
    "\n",
    "pred_cls_scores = pred_cls_scores.permute(0, 2, 3, 1).contiguous()\n",
    "print(pred_cls_scores.shape)\n",
    "\n",
    "objectness_scores = pred_cls_scores.view(1, 50, 50, 9, 2)[:, :, :, :, 1].contiguous().view(1, -1)\n",
    "print(objectness_scores.shape)\n",
    "\n",
    "pred_cls_scores  = pred_cls_scores.view(1, -1, 2)\n",
    "print(pred_cls_scores.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generating proposals to feed Fast R-CNN network\n",
    "\n",
    "The proposal function will take the following parameters\n",
    "- Weather training_mode or testing mode\n",
    "- nms_thresh\n",
    "- n_train_pre_nms — number of bboxes before nms during training\n",
    "- n_train_post_nms — number of bboxes after nms during training\n",
    "- n_test_pre_nms — number of bboxes before nms during testing\n",
    "- n_test_post_nms — number of bboxes after nms during testing\n",
    "- min_size — minimum height of the object required to create a proposal."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Faster R_CNN says, RPN proposals highly overlap with each other. To reduced redundancy, we adopt non-maximum supression (NMS) on the proposal regions based on their cls scores. We fix the IoU threshold for NMS at 0.7, which leaves us about 2000 proposal regions per image. After an ablation study, the authors show that NMS does not harm the ultimate detection accuracy, but substantially reduces the number of proposals. After NMS, we use the top-N ranked proposal regions for detection. In the following we training Fast R-CNN using 2000 RPN proposals. During testing they evaluate only 300 proposals, they have tested this with various numbers and obtained this."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T00:36:12.592414Z",
     "start_time": "2019-08-24T00:36:12.588428Z"
    }
   },
   "outputs": [],
   "source": [
    "nms_thresh = 0.7\n",
    "n_train_pre_nms = 12000\n",
    "n_train_post_nms = 2000\n",
    "n_test_pre_nms = 6000\n",
    "n_test_post_nms = 300\n",
    "min_size = 16"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- convert the loc predictions from the rpn network to bbox [y1, x1, y2, x2] format.\n",
    "- clip the predicted boxes to the image\n",
    "- Remove predicted boxes with either height or width < threshold (min_size).\n",
    "- Sort all (proposal, score) pairs by score from highest to lowest.\n",
    "- Take top pre_nms_topN (e.g. 12000 while training and 300 while testing).\n",
    "- Apply nms threshold > 0.7\n",
    "- Take top pos_nms_topN (e.g. 2000 while training and 300 while testing)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## convert the loc predictions from the rpn net to [y1, x1, y2, x2] format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T00:36:13.284161Z",
     "start_time": "2019-08-24T00:36:13.279144Z"
    }
   },
   "outputs": [],
   "source": [
    "anc_height = anchors[:, 2] - anchors[:, 0]\n",
    "anc_width = anchors[:, 3] - anchors[:, 1]\n",
    "anc_ctr_y = anchors[:, 0] + anc_height*0.5\n",
    "anc_ctr_x = anchors[:, 1] + anc_width*0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T00:36:13.646708Z",
     "start_time": "2019-08-24T00:36:13.637732Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 22500, 4])\n",
      "torch.Size([1, 22500, 2])\n",
      "torch.Size([1, 22500])\n"
     ]
    }
   ],
   "source": [
    "print(pred_anchor_locs.shape)\n",
    "print(pred_cls_scores.shape)\n",
    "print(objectness_scores.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T00:36:13.945621Z",
     "start_time": "2019-08-24T00:36:13.938611Z"
    }
   },
   "outputs": [],
   "source": [
    "pred_anchor_locs_np = pred_anchor_locs[0].data.numpy()\n",
    "objectness_score_np = objectness_scores[0].data.numpy()\n",
    "\n",
    "dy = pred_anchor_locs_np[:, 0]\n",
    "dx = pred_anchor_locs_np[:, 1]\n",
    "dh = pred_anchor_locs_np[:, 2]\n",
    "dw = pred_anchor_locs_np[:, 3]\n",
    "\n",
    "ctr_y = dy * anc_height + anc_ctr_y\n",
    "ctr_x = dx * anc_width + anc_ctr_x\n",
    "h = np.exp(dh) * anc_height\n",
    "w = np.exp(dw) * anc_width"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T00:36:14.402757Z",
     "start_time": "2019-08-24T00:36:14.394779Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ -36.8519559 ,  -79.62706699,   51.95098727,  102.68216486],\n",
       "       [ -88.07382143, -169.48897501,   96.38739479,  199.29125059],\n",
       "       [-171.5806405 , -356.84217547,  191.98967897,  378.18893614],\n",
       "       ...,\n",
       "       [ 704.94185882,  745.34040758,  885.73096646,  837.74671343],\n",
       "       [ 614.3390355 ,  695.5290976 ,  977.66343855,  881.61952082],\n",
       "       [ 419.37618682,  610.85832571, 1145.80790631,  975.66940523]])"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# generate roi(region of interest) matrix\n",
    "roi = np.zeros(pred_anchor_locs_np.shape, dtype=anchors.dtype)\n",
    "\n",
    "roi[:, 0] = ctr_y - 0.5*h\n",
    "roi[:, 1] = ctr_x - 0.5*w\n",
    "roi[:, 2] = ctr_y + 0.5*h\n",
    "roi[:, 3] = ctr_x + 0.5*w\n",
    "\n",
    "roi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## clip the predicted boxes to the image\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T00:36:15.075682Z",
     "start_time": "2019-08-24T00:36:15.068696Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  0.        ,   0.        ,  51.95098727, 102.68216486],\n",
       "       [  0.        ,   0.        ,  96.38739479, 199.29125059],\n",
       "       [  0.        ,   0.        , 191.98967897, 378.18893614],\n",
       "       ...,\n",
       "       [704.94185882, 745.34040758, 800.        , 800.        ],\n",
       "       [614.3390355 , 695.5290976 , 800.        , 800.        ],\n",
       "       [419.37618682, 610.85832571, 800.        , 800.        ]])"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "img_size = (800, 800)\n",
    "\n",
    "roi[:, 0:4:2] = np.clip(roi[:, 0:4:2],a_min=0, a_max=img_size[0])\n",
    "roi[:, 1:4:2] = np.clip(roi[:, 1:4:2],a_min=0, a_max=img_size[1])\n",
    "\n",
    "roi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Remove predicted boxes with either height or width < threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T00:36:15.916625Z",
     "start_time": "2019-08-24T00:36:15.909642Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(22500,)\n"
     ]
    }
   ],
   "source": [
    "keep = np.where((h >= min_size) & (w >= min_size))[0]\n",
    "roi = roi[keep]\n",
    "score = objectness_score_np[keep]\n",
    "\n",
    "print(score.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## sort all pairs by score from highest to lowest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T00:36:16.648962Z",
     "start_time": "2019-08-24T00:36:16.642968Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[  451   461   875 ... 21609 22049 22064]\n"
     ]
    }
   ],
   "source": [
    "order = score.ravel().argsort()[::-1]\n",
    "print(order)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Take top pre_nms_topN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T00:36:17.153356Z",
     "start_time": "2019-08-24T00:36:17.147389Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(12000, 4)\n",
      "(12000,)\n"
     ]
    }
   ],
   "source": [
    "order = order[:n_train_pre_nms]\n",
    "roi = roi[order]\n",
    "score = score[order]\n",
    "\n",
    "print(roi.shape)\n",
    "print(score.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Apply NMS threshold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Take all the roi boxes [roi_array]\n",
    "- Find the areas of all the boxes [roi_area]\n",
    "- Take the indexes of order the probability score in descending order [order_array]\n",
    "keep = []\n",
    "while order_array.size > 0:\n",
    "  - take the first element in order_array and append that to keep  \n",
    "  - Find the area with all other boxes\n",
    "  - Find the index of all the boxes which have high overlap with this box\n",
    "  - Remove them from order array\n",
    "  - Iterate this till we get the order_size to zero (while loop)\n",
    "- Ouput the keep variable which tells what indexes to consider."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T00:36:18.558374Z",
     "start_time": "2019-08-24T00:36:18.173396Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2000, 4)\n"
     ]
    }
   ],
   "source": [
    "y1, x1, y2, x2 = roi.T\n",
    "\n",
    "areas = (y2-y1+1) * (x2-x1+1)\n",
    "\n",
    "# 根据score降序排列\n",
    "order = score.argsort()[::-1]\n",
    "\n",
    "keep = []\n",
    "\n",
    "while order.size > 0:\n",
    "    i = order[0]\n",
    "    keep.append(i)\n",
    "    \n",
    "    # 计算当前score最大anchor与其他anchor的IOU\n",
    "    yy1 = np.maximum(y1[i], y1[order[1:]])\n",
    "    xx1 = np.maximum(x1[i], x1[order[1:]])\n",
    "    yy2 = np.maximum(y2[i], y2[order[1:]])\n",
    "    xx2 = np.maximum(y2[i], y2[order[1:]])\n",
    "    \n",
    "    h = np.maximum(0.0, yy2-yy1)\n",
    "    w = np.maximum(0.0, xx2-xx1)\n",
    "    \n",
    "    inter = h*w\n",
    "    iou = inter / (areas[i] + areas[order[1:]] - inter)\n",
    "    \n",
    "    # IOU threshold\n",
    "    inds = np.where(iou <= nms_thresh)[0]\n",
    "    # 要注意这里inds+1, 是因为加上order[0]\n",
    "    order = order[inds+1]\n",
    "\n",
    "keep = keep[:n_train_post_nms]\n",
    "roi = roi[keep]\n",
    "print(roi.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Proposal targets"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Fast R-CNN network takes the region proposals (obtained from proposal layer in previous section), ground truth boxes and their respective labels as inputs. It will take the following parameters\n",
    "- n_sample: Number of samples to sample from roi, The default value is 128.\n",
    "- pos_ratio: the number of positive examples out of the n_samples. The default values is 0.25.\n",
    "- pos_iou_thesh: The minimum overlap of region proposal with any groundtruth object to consider it as positive label.\n",
    "- [neg_iou_threshold_lo, neg_iou_threshold_hi] : [0.0, 0.5], The overlap value bounding required to consider a region proposal as negitive [background object]."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**This Step I think is used in the training process. In test step, we just use the proposal after NMS as the input of Fast RCNN network.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T00:40:54.680541Z",
     "start_time": "2019-08-24T00:40:54.677551Z"
    }
   },
   "outputs": [],
   "source": [
    "n_sample = 128\n",
    "pos_ratio = 0.25\n",
    "pos_iou_thresh = 0.5\n",
    "neg_iou_thresh_hi = 0.5\n",
    "neg_iou_thresh_lo = 0.0"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Find the iou of each ground truth object with the region proposals\n",
    "\n",
    "same as 2.3.4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T16:09:32.639516Z",
     "start_time": "2019-08-24T16:09:32.630540Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(2000, 2)\n"
     ]
    }
   ],
   "source": [
    "ious = np.empty((len(roi), 2), dtype = np.float32)\n",
    "ious.fill(0)\n",
    "\n",
    "for index, box in enumerate(bbox):\n",
    "    y1, x1, y2, x2 = box\n",
    "    \n",
    "    inter_y1 = np.maximum(y1, roi[:, 0])\n",
    "    inter_x1 = np.maximum(x1, roi[:, 1])\n",
    "    inter_y2 = np.minimum(y2, roi[:, 2])\n",
    "    inter_x2 = np.minimum(x2, roi[:, 3])\n",
    "    \n",
    "    h = np.maximum(0.0, inter_y2-inter_y1)\n",
    "    w = np.maximum(0.0, inter_x2-inter_x1)\n",
    "    \n",
    "    eps = np.finfo(np.float32).eps\n",
    "    iou = h*w / ((y2-y1)*(x2-x1)+(roi[:, 2]-roi[:, 0])*(roi[:, 3]-roi[:, 1]) - h*w + eps)\n",
    "        \n",
    "    ious[:, index] = iou\n",
    "print(ious.shape)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Assign labels for each proposal\n",
    "\n",
    "same as 2.3.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T16:13:47.837732Z",
     "start_time": "2019-08-24T16:13:47.829757Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 0 0 ... 0 0 0]\n",
      "[0.08455282 0.11251243 0.37005857 ... 0.         0.         0.        ]\n"
     ]
    }
   ],
   "source": [
    "gt_assignment = ious.argmax(axis=1)\n",
    "max_iou = ious.max(axis=1)\n",
    "print(gt_assignment)\n",
    "print(max_iou)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T16:13:52.724097Z",
     "start_time": "2019-08-24T16:13:52.720106Z"
    }
   },
   "outputs": [],
   "source": [
    "gt_roi_label = labels[gt_assignment]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sample pos and neg samples\n",
    "\n",
    "same as 2.3.6\n",
    "\n",
    "- Select the foreground rois as per the pos_iou_thesh. We also want only n_sample x pos_ratio (128 x 0.25 = 32) foreground samples. So incase if we get less than 32 positive samples we will leave it as it is, Incase if we get more than 32 foreground samples, we will sample 32 samples from the positive samples. This is done using the following code."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T16:30:07.447534Z",
     "start_time": "2019-08-24T16:30:07.439557Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "32\n",
      "(32,)\n"
     ]
    }
   ],
   "source": [
    "pos_index = np.where(max_iou>=pos_iou_thresh)[0]\n",
    "pos_roi_per_this_image = n_sample * pos_ratio\n",
    "pos_roi_per_this_image = int(min(pos_roi_per_this_image, pos_index.size))\n",
    "if pos_index.size > 0:\n",
    "    pos_index = np.random.choice(pos_index, size=pos_roi_per_this_image, replace=False)\n",
    "print(pos_roi_per_this_image)\n",
    "print(pos_index.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T16:30:19.995675Z",
     "start_time": "2019-08-24T16:30:19.987696Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "96\n",
      "(96,)\n"
     ]
    }
   ],
   "source": [
    "neg_index = np.where((max_iou < neg_iou_thresh_hi) & (max_iou > neg_iou_thresh_lo))[0]\n",
    "neg_roi_per_this_image = n_sample - pos_roi_per_this_image\n",
    "\n",
    "if neg_index.size > 0:\n",
    "    neg_index = np.random.choice(neg_index, size=neg_roi_per_this_image, replace=False)\n",
    "print(neg_roi_per_this_image)\n",
    "print(neg_index.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gather positve samples index and negitive samples index, their respective labels and region proposals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T16:30:21.038598Z",
     "start_time": "2019-08-24T16:30:21.032618Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "128\n",
      "(128, 4)\n"
     ]
    }
   ],
   "source": [
    "keep_index = np.append(pos_index, neg_index)\n",
    "print(len(keep_index))\n",
    "gt_roi_labels = gt_roi_label[keep_index]\n",
    "gt_roi_labels[pos_roi_per_this_image:] = 0\n",
    "sample_roi = roi[keep_index]\n",
    "print(sample_roi.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pick the ground truth objects for these sample_roi and later parameterize\n",
    "\n",
    "same as 2.3.7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T16:47:40.266878Z",
     "start_time": "2019-08-24T16:47:40.260895Z"
    }
   },
   "outputs": [],
   "source": [
    "bbox_for_sampled_roi = bbox[gt_assignment[keep_index]]\n",
    "\n",
    "height = sample_roi[:, 2] - sample_roi[:, 0]\n",
    "width = sample_roi[:, 3] - sample_roi[:, 2]\n",
    "ctr_y = sample_roi[:, 0] + height*0.5\n",
    "ctr_x = sample_roi[:, 1] + width*0.5\n",
    "\n",
    "base_height = bbox_for_sampled_roi[:, 2] - bbox_for_sampled_roi[:, 0]\n",
    "base_width = bbox_for_sampled_roi[:, 3] - bbox_for_sampled_roi[:, 1]\n",
    "base_ctr_y = bbox_for_sampled_roi[:, 0] + base_height*0.5\n",
    "base_ctr_x = bbox_for_sampled_roi[:, 1] + base_width*0.5\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- We will use the following formulation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```\n",
    "t_{x} = (x - x_{a})/w_{a}\n",
    "t_{y} = (y - y_{a})/h_{a}\n",
    "t_{w} = log(w/ w_a)\n",
    "t_{h} = log(h/ h_a)\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T16:47:41.248955Z",
     "start_time": "2019-08-24T16:47:41.237988Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.31523577  0.37119662  0.3887858   0.43511528]\n",
      " [ 0.26899167  0.28475489  0.33038858  0.33061902]\n",
      " [ 0.31523577  0.32766118  0.3887858   0.38385146]\n",
      " [ 0.18874494  0.13455423  0.22017946  0.11817034]\n",
      " [ 0.18874494  0.18721215  0.22017946  0.19789063]\n",
      " [ 0.09506534  0.0431238   0.07397966  0.11817034]\n",
      " [-0.04187217  0.03015921 -0.23882236  0.32849712]\n",
      " [ 0.08413659  0.077143   -0.23882236  0.2231373 ]\n",
      " [ 0.15440454  0.05998208  0.16903416  0.08057334]\n",
      " [ 0.02584132 -0.03067134  0.0297298   0.12421907]\n",
      " [ 0.18874494  0.15983427  0.22017946  0.15723628]\n",
      " [ 0.15440454  0.00446868  0.16903416  0.04433881]\n",
      " [ 0.31523577  0.41956631  0.3887858   0.48914975]\n",
      " [-0.01753452 -0.09203664  0.0297298   0.16352675]\n",
      " [ 0.26899167  0.36690531  0.33038858  0.43017735]\n",
      " [ 0.02113221  0.34642664 -0.23882236  0.44627929]\n",
      " [ 0.08413659  0.25237989 -0.23882236  0.32849712]\n",
      " [ 0.22688444  0.18723246  0.27407595  0.19792019]\n",
      " [ 0.18874494  0.2169603   0.22017946  0.24026806]\n",
      " [ 0.22688444  0.21698241  0.27407595  0.24029889]\n",
      " [ 0.09506534 -0.01381594  0.07397966  0.08057334]\n",
      " [ 0.22688444  0.15985299  0.27407595  0.15726466]\n",
      " [ 0.31523577  0.28826977  0.3887858   0.33508797]\n",
      " [ 0.14714097  0.17714237 -0.23882236  0.2231373 ]\n",
      " [ 0.3767586   0.40381774  0.46154015  0.47187532]\n",
      " [ 0.06921715 -0.04809027  0.0297298   0.08639823]\n",
      " [ 0.02113221 -0.02285638 -0.23882236  0.2231373 ]\n",
      " [ 0.20623911  0.20455486  0.24526231  0.55250338]\n",
      " [ 0.3767586   0.5807473   0.46154015  0.65065561]\n",
      " [ 0.14714097  0.02467575 -0.23882236  0.12782769]\n",
      " [ 0.20623911  0.02423141  0.24526231  0.44068968]\n",
      " [ 0.15440454  0.11974922  0.16903416  0.11817034]\n",
      " [ 0.3767586   0.08472989  0.46154015  0.03639761]\n",
      " [ 0.2180209  -0.66200132  0.40303152  0.48976874]\n",
      " [ 0.18874494 -0.39735924  0.22017946 -0.05202218]\n",
      " [ 1.35656879 -0.8408912   1.21179377 -0.37943872]\n",
      " [ 0.46216288 -0.29047483 -0.23882236 -0.36464659]\n",
      " [ 0.06914156 -0.67932499  0.02979719  0.08643806]\n",
      " [ 0.59219779  1.10662902  0.68125571  1.55837387]\n",
      " [ 0.26446028 -0.16500691 -0.6149233  -0.44748051]\n",
      " [ 0.62772792 -0.54581062  0.71326864  0.28699844]\n",
      " [-0.06091036 -0.49264629  0.0297298   0.20444296]\n",
      " [ 0.33615412  0.0208795  -0.23882236 -0.03922552]\n",
      " [ 1.89095821 -0.92859633  1.46475793 -0.11763872]\n",
      " [ 0.45220443 -0.37903563  0.54408819 -0.20936243]\n",
      " [ 0.46216288  0.47921862 -0.23882236  0.12782769]\n",
      " [ 0.09201214 -0.5654307   0.40303152  0.74108218]\n",
      " [ 0.31523577  0.04049291  0.3887858  -0.02142694]\n",
      " [ 0.59772835  0.9090571   0.68630663  0.91592362]\n",
      " [ 0.06921715 -0.19654854  0.0297298   0.08639823]\n",
      " [ 0.90460369 -0.38836176  0.93281892  0.32461311]\n",
      " [ 0.45220443 -0.29619976  0.54408819 -0.20936243]\n",
      " [ 0.25148229 -0.74970663  0.30735609 -0.10215766]\n",
      " [ 0.77718479 -0.54387611 -0.23882236 -0.8064781 ]\n",
      " [ 0.39422405 -0.20956459 -0.6149233  -0.59020918]\n",
      " [ 0.84018917 -0.54236315 -0.23882236 -0.84156934]\n",
      " [ 0.31523577 -0.13950695  0.3887858  -0.14328175]\n",
      " [ 0.45220443  0.16621602  0.54408819  0.16686157]\n",
      " [ 1.35656879 -0.57671841  1.21179377 -0.15621305]\n",
      " [ 0.20623911 -0.29457173  0.24526231  0.2060343 ]\n",
      " [-0.1042862  -0.27440006  0.0297298   0.247105  ]\n",
      " [-0.19103787 -0.53934021  0.0297298   0.33831092]\n",
      " [ 0.90460369 -0.63328384  0.93281892  0.07573455]\n",
      " [ 0.22120569 -0.14695244 -0.6149233  -0.39498762]\n",
      " [-0.23441371 -0.34074718  0.0297298   0.38723575]\n",
      " [ 0.28067436  0.09485625 -0.61212409 -0.46717958]\n",
      " [ 1.10748245 -0.13841287  1.06773298  0.25105925]\n",
      " [ 0.62772792 -0.82242007  0.71326864 -0.0553362 ]\n",
      " [ 0.3767586  -0.55531626  0.46154015 -0.17606335]\n",
      " [-0.01753452 -0.33257932  0.0297298   0.16352675]\n",
      " [ 0.22688444 -0.36925566  0.27407595 -0.08382427]\n",
      " [ 0.3767586   0.01276118  0.46154015 -0.06427394]\n",
      " [ 0.26899167  0.08930132  0.33038858  0.04418523]\n",
      " [ 0.26899167 -0.25164976  0.33038858 -0.11430677]\n",
      " [ 1.10748245 -0.75664193  1.06773298 -0.35359074]\n",
      " [ 0.21014535 -0.61142635 -0.23882236 -0.47000676]\n",
      " [ 1.89095821 -0.91076343  1.46475793 -0.08690688]\n",
      " [-0.16000539 -0.59160282  0.40303152  1.07755263]\n",
      " [ 0.45220443  0.42732281  0.54408819  0.49754933]\n",
      " [ 0.59752199  0.15387154  0.68611862  0.1481585 ]\n",
      " [ 0.02900776 -0.89580046  0.40303152  0.38440857]\n",
      " [ 0.31523577  0.25245757  0.3887858   0.2885922 ]\n",
      " [ 0.45257238 -0.68345374  0.54447454 -0.20950943]\n",
      " [ 0.22688444 -0.27533979  0.27407595 -0.08382427]\n",
      " [-0.35689408 -0.62285262 -0.23882236  0.2231373 ]\n",
      " [ 1.10748245  0.59495131  1.06773298  0.68193854]\n",
      " [ 0.22688444 -0.55708739  0.27407595 -0.08382427]\n",
      " [ 0.20623911 -0.34297176  0.24526231  0.16505424]\n",
      " [ 0.20623911 -0.78931858  0.24526231 -0.06716197]\n",
      " [-0.03399662 -0.97219069  0.40303152  0.33561856]\n",
      " [ 0.15440454 -0.29374308  0.16903416 -0.0191716 ]\n",
      " [-0.20575014 -0.81196866  0.08616871 -0.3019673 ]\n",
      " [ 0.31523577 -0.09950435  0.3887858  -0.11670317]\n",
      " [ 0.21970124 -0.83249382  0.26414462 -0.07816649]\n",
      " [ 0.2180209  -0.78171506  0.40303152  0.33561856]\n",
      " [ 0.45220443 -0.4618715   0.54408819 -0.20936243]\n",
      " [-0.35688737 -0.87095752 -0.23882843  0.09216121]\n",
      " [ 0.97082301 -0.8063083   0.97888583 -0.17631886]\n",
      " [ 0.62772792  0.59393771  0.71326864  1.07293792]\n",
      " [ 0.1938555   0.0493113  -0.61205669 -0.36144497]\n",
      " [ 0.32411216 -0.07638894 -0.61120614 -0.51635022]\n",
      " [ 0.09506534 -0.24319434  0.07397966  0.04995582]\n",
      " [ 1.07375118 -0.95059881  1.08966976  0.01321229]\n",
      " [ 0.58817164 -0.50952222 -0.23882236 -0.65232779]\n",
      " [ 0.71418041 -0.54550115 -0.23882236 -0.77011054]\n",
      " [-0.29294935 -0.83158136  0.08616871 -0.24099682]\n",
      " [ 0.1939198   0.10541161 -0.61212795 -0.36149956]\n",
      " [ 1.51597808 -0.83965742  1.2941682   0.08162398]\n",
      " [-0.03399662 -0.39160407  0.40303152  1.07755263]\n",
      " [ 0.46526709 -0.92173353  0.55771328 -0.11643669]\n",
      " [-0.03399662 -0.91663199  0.40303152  0.4357017 ]\n",
      " [ 0.65117603 -0.61783206 -0.23882236 -0.78479439]\n",
      " [ 0.33615412 -0.60129671 -0.23882236 -0.56531665]\n",
      " [-0.1042862  -0.36157035  0.0297298   0.247105  ]\n",
      " [ 1.35656879 -0.24881729  1.21179377  0.14769724]\n",
      " [ 0.27314974  0.84642274 -0.23882236  0.44627929]\n",
      " [-0.23088532 -0.3031718  -0.23882236  0.32849712]\n",
      " [ 0.31523577 -0.16900516  0.3887858  -0.14328175]\n",
      " [-0.10487655 -0.64856831 -0.23882236 -0.18232573]\n",
      " [ 0.06904508 -0.71571981  0.03064775  0.08620714]\n",
      " [ 0.14718736 -0.76952825 -0.23886118 -0.43954295]\n",
      " [ 0.71416147 -0.27019041  0.78711741  0.50973062]\n",
      " [-0.02951535 -0.17666995 -0.61451135 -0.02083828]\n",
      " [ 0.58817164 -0.58571264 -0.23882236 -0.7323703 ]\n",
      " [ 0.97082301 -0.56654425  0.97888583  0.12054706]\n",
      " [-0.554547   -0.90865469  0.08616871 -0.03199951]\n",
      " [ 0.94683043 -0.96739462  1.08846165  0.04892084]\n",
      " [ 0.90460369 -0.74428584  0.93281892 -0.06131377]]\n"
     ]
    }
   ],
   "source": [
    "dy = (base_ctr_y - ctr_y) / height\n",
    "dx = (base_ctr_x - ctr_x) / width\n",
    "dh = np.log(base_height/height)\n",
    "dw = np.log(base_width/width)\n",
    "\n",
    "gt_roi_locs = np.stack((dy, dx, dh, dw), axis=1)\n",
    "print(gt_roi_locs)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fast R-CNN\n",
    "\n",
    "Fast R-CNN used ROI pooling to extract features for each and every proposal suggested by selective search (Fast RCNN) or Region Proposal network (RPN in Faster R- CNN). "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Region of interest pooling (also known as RoI pooling) purpose is to perform max pooling on inputs of non-uniform sizes to obtain fixed-size feature maps (e.g. 7×7). This layer takes two inputs\n",
    "\n",
    "- A fixed-size feature map obtained from a deep convolutional network with several convolutions and max-pooling layers\n",
    "- An Nx5 matrix of representing a list of regions of interest, where N is the number of RoIs. The first column represents the image index and the remaining four are the co-ordinates of the top left and bottom right corners of the region."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "What does the RoI pooling actually do? For every region of interest from the input list, it takes a section of the input feature map that corresponds to it and scales it to some pre-defined size (e.g., 7×7). The scaling is done by:\n",
    "- Dividing the region proposal into equal-sized sections (the number of which is the same as the dimension of the output)\n",
    "- Finding the largest value in each section\n",
    "- Copying these max values to the output buffer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T22:14:40.719585Z",
     "start_time": "2019-08-24T22:14:40.679690Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([128, 4]) torch.Size([128, 1])\n"
     ]
    }
   ],
   "source": [
    "rois = torch.from_numpy(sample_roi).float()\n",
    "# roi_indices is 0 because there is only one image\n",
    "roi_indices = 0 * np.ones((len(rois), 1), dtype=np.int32)\n",
    "roi_indices = torch.from_numpy(roi_indices).float()\n",
    "print(rois.shape, roi_indices.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "concat rois and roi_indices, so that we get the tensor with shape [N, 5] (index, y1, x1, y2, x2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T22:15:56.428189Z",
     "start_time": "2019-08-24T22:15:56.422206Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([128, 5])\n"
     ]
    }
   ],
   "source": [
    "indices_and_rois = torch.cat([roi_indices, rois], dim=1)\n",
    "print(indices_and_rois.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we need to pass this array to the roi_pooling layer. We will briefly discuss the workings of it here. The sudo code is as follows\n",
    "\n",
    "- Multiply the dimensions of rois with the sub_sampling ratio (16 in this case)\n",
    "- Empty output Tensor\n",
    "- Take each roi\n",
    "    - subset the **feature map** based on the roi dimension\n",
    "    - Apply AdaptiveMaxPool2d to this subset Tensor.\n",
    "    - Add the outputs to the output Tensor\n",
    "- Empty output Tensor goes to the network\n",
    "\n",
    "We will define the size to be 7 x 7 and define adaptive_max_pool"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ROI layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T22:32:42.582427Z",
     "start_time": "2019-08-24T22:32:42.181899Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "128\n"
     ]
    }
   ],
   "source": [
    "size = (7, 7)\n",
    "adaptive_max_pool = nn.AdaptiveAvgPool2d(size)\n",
    "\n",
    "output = []\n",
    "rois = indices_and_rois.data.float()\n",
    "rois[:, 1:].mul_(1/16.0)\n",
    "rois = rois.long()\n",
    "num_rois = rois.size(0)\n",
    "print(num_rois)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T22:32:42.910833Z",
     "start_time": "2019-08-24T22:32:42.814117Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([128, 512, 7, 7])\n"
     ]
    }
   ],
   "source": [
    "for i in range(num_rois):\n",
    "    roi = rois[i]\n",
    "    im_idx = roi[0]\n",
    "    im = sample_output[..., roi[1]:roi[3]+1, roi[2]:roi[4]+1]\n",
    "    output.append(adaptive_max_pool(im))\n",
    "\n",
    "output = torch.cat(output, 0)\n",
    "print(output.size())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T22:33:52.884494Z",
     "start_time": "2019-08-24T22:33:52.759823Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([128, 25088])\n"
     ]
    }
   ],
   "source": [
    "# Reshape the tensor so that we can pass it through the feed forward layer.\n",
    "k = output.view(output.size(0), -1)\n",
    "print(k.size())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loc and Cls layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T22:41:11.458472Z",
     "start_time": "2019-08-24T22:41:09.847210Z"
    }
   },
   "outputs": [],
   "source": [
    "roi_head_classifier = nn.Sequential(nn.Linear(25088, 4096),\n",
    "                                    nn.Linear(4096, 4096))\n",
    "\n",
    "# loc网络\n",
    "cls_loc = nn.Linear(4096, 21*4) # (VOC 20 classes + 1 background. Each will have 4 co-ordinates)\n",
    "\n",
    "cls_loc.weight.data.normal_(0, 0.01)\n",
    "cls_loc.bias.data.zero_()\n",
    "\n",
    "# cls网络\n",
    "score = nn.Linear(4096, 21)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T22:43:14.278861Z",
     "start_time": "2019-08-24T22:43:11.718934Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([128, 84]) torch.Size([128, 21])\n"
     ]
    }
   ],
   "source": [
    "k = roi_head_classifier(k)\n",
    "roi_cls_loc = cls_loc(k)\n",
    "roi_cls_score = score(k)\n",
    "print(roi_cls_loc.shape, roi_cls_score.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**roi_cls_loc** and **roi_cls_score** are two ouput tensors from which we can get actual bounding boxes, We will see this section 8. In section 7, We will compute the losses for both the RPN and Fast RCNN networks. This will complete the Faster R-CNN implementation."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loss functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We have two networks, RPN and Fast-RCNN, which further have two outputs each (Regression head, and classification head). The Loss function for both the network is defined as\n",
    "\n",
    "![](./Faster-RCNN-loss.png)\n",
    "\n",
    "Faster RCNN loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RPN Loss\n",
    "![](./RPN-loss.png)\n",
    "\n",
    "where `p_{i}` is the predicted class label and `p_{i}^*` is the actual class score. `t_{i}` and `t_{i}^*` are the predicted co-oridinates and actual co-ordinates. The ground-truth label `p_{i}^*` is 1 if the the anchor is positive and 0 if the anchor is negative. We will see how this is done in Pytorch."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T22:50:47.735598Z",
     "start_time": "2019-08-24T22:50:47.726599Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 22500, 4])\n",
      "torch.Size([1, 22500, 2])\n",
      "(22500, 4)\n",
      "(22500,)\n"
     ]
    }
   ],
   "source": [
    "print(pred_anchor_locs.shape)\n",
    "print(pred_cls_scores.shape)\n",
    "print(anchor_locations.shape)\n",
    "print(anchor_labels.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T22:52:46.029787Z",
     "start_time": "2019-08-24T22:52:46.024831Z"
    }
   },
   "outputs": [],
   "source": [
    "rpn_loc = pred_anchor_locs[0]\n",
    "rpn_score = pred_cls_scores[0]\n",
    "\n",
    "gt_rpn_loc = torch.from_numpy(anchor_locations)\n",
    "gt_rpn_score = torch.from_numpy(anchor_labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "pred_cls_scores and anchor_labels are the predited objectness score and actual objectness score of the RPN network. We will use the following loss functions for Regression and classification respectively.\n",
    "For classification we use cross-entropy loss\n",
    "![](./Cross_entropy-loss.png)\n",
    "Cross Entropy Loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T23:44:21.651295Z",
     "start_time": "2019-08-24T23:44:21.643318Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(0.6951, grad_fn=<NllLossBackward>)\n"
     ]
    }
   ],
   "source": [
    "# F.cross_entropy already average the loss for each batch\n",
    "rpn_cls_loss = F.cross_entropy(rpn_score, gt_rpn_score.long(), ignore_index=-1)\n",
    "print(rpn_cls_loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For Regression we use smooth L1 loss as defined in the Fast RCNN paper,\n",
    "![](Smooth-L1-loss.png)\n",
    "Smooth L1 Loss\n",
    "They used L1 loss instead of L2 loss because the values of predicted regression head of RPN are not bounded. **Regression loss is also applied to the bounding boxes which have positive label**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T23:44:22.951069Z",
     "start_time": "2019-08-24T23:44:22.945083Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([22500])\n"
     ]
    }
   ],
   "source": [
    "mask = gt_rpn_score > 0\n",
    "print(mask.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now take those bounding boxes which have positve labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T23:44:23.549519Z",
     "start_time": "2019-08-24T23:44:23.540533Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([18, 4]) torch.Size([18, 4])\n"
     ]
    }
   ],
   "source": [
    "mask_loc_preds = rpn_loc[mask]\n",
    "mask_loc_targets = gt_rpn_loc[mask]\n",
    "print(mask_loc_preds.shape, mask_loc_preds.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T23:44:23.741066Z",
     "start_time": "2019-08-24T23:44:23.732089Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(1.1647, grad_fn=<SumBackward0>)\n"
     ]
    }
   ],
   "source": [
    "x = torch.abs(mask_loc_targets-mask_loc_preds)\n",
    "rpn_loc_loss = (x<1).float() * 0.5 * x**2 + (x>1).float()*(x-0.5)\n",
    "print(rpn_loc_loss.sum())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Combining both the rpn_cls_loss and rpn_reg_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T23:44:26.186635Z",
     "start_time": "2019-08-24T23:44:24.094023Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(1.3421, grad_fn=<AddBackward0>)\n"
     ]
    }
   ],
   "source": [
    "rpn_lambda = 10\n",
    "N_reg = (gt_rpn_score>0).float().sum()\n",
    "rpn_loc_loss = rpn_loc_loss.sum()/N_reg\n",
    "rpn_loss = rpn_cls_loss + rpn_lambda*rpn_loc_loss\n",
    "\n",
    "print(rpn_loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Fast R-CNN loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- predicted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-24T23:47:12.475196Z",
     "start_time": "2019-08-24T23:47:12.469206Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([128, 84])\n",
      "torch.Size([128, 21])\n"
     ]
    }
   ],
   "source": [
    "print(roi_cls_loc.shape)\n",
    "print(roi_cls_score.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- actual"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T00:02:22.077088Z",
     "start_time": "2019-08-25T00:02:22.069081Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([128, 4]) torch.Size([128])\n"
     ]
    }
   ],
   "source": [
    "gt_roi_loc = torch.from_numpy(gt_roi_locs).float()\n",
    "gt_roi_label = torch.from_numpy(gt_roi_labels).long()\n",
    "print(gt_roi_loc.shape, gt_roi_label.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Classification loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T00:02:22.635050Z",
     "start_time": "2019-08-25T00:02:22.629065Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(3.0134, grad_fn=<NllLossBackward>)\n"
     ]
    }
   ],
   "source": [
    "roi_cls_loss = F.cross_entropy(roi_cls_score, gt_roi_label, ignore_index=-1)\n",
    "print(roi_cls_loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- Regression loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T00:02:23.255891Z",
     "start_time": "2019-08-25T00:02:23.248895Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([128, 21, 4])\n",
      "torch.Size([128, 4])\n"
     ]
    }
   ],
   "source": [
    "n_sample = roi_cls_loc.shape[0]\n",
    "roi_loc = roi_cls_loc.view(n_sample, -1, 4)\n",
    "print(roi_loc.shape)\n",
    "\n",
    "roi_loc = roi_loc[torch.arange(0, n_sample).long(), gt_roi_label]\n",
    "print(roi_loc.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T00:20:01.777122Z",
     "start_time": "2019-08-25T00:20:01.649482Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([32, 4]) torch.Size([32, 4])\n"
     ]
    }
   ],
   "source": [
    "mask = gt_roi_label>0\n",
    "mask_loc_preds = roi_loc[mask].view(-1, 4)\n",
    "mask_loc_targets = gt_roi_loc[mask].view(-1, 4)\n",
    "\n",
    "print(mask_loc_preds.shape, mask_loc_targets.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T00:20:54.518881Z",
     "start_time": "2019-08-25T00:20:54.506922Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(3.9434, grad_fn=<SumBackward0>)\n"
     ]
    }
   ],
   "source": [
    "x = torch.abs(mask_loc_preds - mask_loc_targets)\n",
    "roi_loc_loss = (x<1).float()*0.5*x**2 + (x>=1).float()*(x-0.5)\n",
    "roi_loc_loss = roi_loc_loss\n",
    "print(roi_loc_loss.sum())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- total roi loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T00:22:10.111048Z",
     "start_time": "2019-08-25T00:22:10.103069Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(4.2457, grad_fn=<AddBackward0>)\n"
     ]
    }
   ],
   "source": [
    "roi_lambda = 10.\n",
    "N_reg = (gt_roi_label>0).float().sum()\n",
    "roi_loc_loss = roi_loc_loss.sum() / N_reg\n",
    "roi_loss = roi_cls_loss + roi_lambda * roi_loc_loss\n",
    "print(roi_loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Total loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T00:22:44.289043Z",
     "start_time": "2019-08-25T00:22:44.284056Z"
    }
   },
   "outputs": [],
   "source": [
    "total_loss = rpn_loss + roi_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-08-25T00:22:49.584745Z",
     "start_time": "2019-08-25T00:22:49.573782Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(5.5878, grad_fn=<AddBackward0>)\n"
     ]
    }
   ],
   "source": [
    "print(total_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "403.807px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}