submissionv3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ensemble of 5 models\n"
     ]
    }
   ],
   "source": [
    "from utils import pickle_load\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import cv2\n",
    "import pandas as pd\n",
    "import os\n",
    "from glob import glob\n",
    "from evaluate import compute_sim, evaluate, l2norm_numpy\n",
    "from evaluate import dict2list\n",
    "\n",
    "def l2norm(embs):\n",
    "    return {k: v/np.linalg.norm(v) for k, v in embs.items()}\n",
    "\n",
    "infer_dirs = glob('infer/*')[:]\n",
    "# infer_dirs = infer_dirs[1:]\n",
    "infer_dirs = [x for x in infer_dirs if 'bk' not in x and 'eval' not in x]\n",
    "weights = [1.0] * len(infer_dirs)\n",
    "# weights = [1.0, 0.5, 0.5]\n",
    "\n",
    "print(f\"Ensemble of {len(infer_dirs)} models\")\n",
    "\n",
    "submit_file = f'submission.csv'\n",
    "if os.path.exists('D:/whale_data/train_images-384-384/train_images-384-384'):\n",
    "    train_img_dir = 'D:/whale_data/train_images-384-384/train_images-384-384'\n",
    "else:\n",
    "    train_img_dir = '/Users/macbook/works/train_images-384-384'\n",
    "\n",
    "norm=True\n",
    "method = 'cat'\n",
    "\n",
    "def get_emb(infer_dirs, subset):\n",
    "    li = []\n",
    "    for infer_dir in infer_dirs:\n",
    "        if isinstance(subset, str):\n",
    "            embs = pickle_load(f\"{infer_dir}/{subset}_emb.pkl\")\n",
    "        else:\n",
    "            embs = {}\n",
    "            for s in subset:\n",
    "                p = f\"{infer_dir}/{s}_emb.pkl\"\n",
    "                if os.path.exists(p):\n",
    "                    embs = {**embs, **pickle_load(p)}\n",
    "                elif s != \"val\":\n",
    "                    raise FileNotFoundError(p)\n",
    "\n",
    "        li.append(embs)\n",
    "\n",
    "    if len(li) == 1:\n",
    "        return li[0]\n",
    "\n",
    "    print(f'Ensemble from {len(infer_dirs)} models {infer_dirs}')\n",
    "    li0 = li[0]\n",
    "    di = {}\n",
    "    for k in li0.keys():\n",
    "        di[k] = []\n",
    "        for i in range(len(li)):\n",
    "            e = li[i][k]\n",
    "            di[k].append(e)\n",
    "\n",
    "    return di\n",
    "\n",
    "train_df = pd.read_csv('data/train_kfold.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import normalize, QuantileTransformer\n",
    "\n",
    "def add_border(img, color):\n",
    "    bordersize = 14\n",
    "    return cv2.copyMakeBorder(\n",
    "        img,\n",
    "        top=bordersize,\n",
    "        bottom=bordersize,\n",
    "        left=bordersize,\n",
    "        right=bordersize,\n",
    "        borderType=cv2.BORDER_CONSTANT,\n",
    "        value=color\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "nens = len(infer_dirs)\n",
    "def load(infer_dirs, subset):\n",
    "    embs = get_emb(infer_dirs, subset)\n",
    "    k, v = dict2list(embs)\n",
    "\n",
    "    # for i in range(nens):\n",
    "    #     v[:, i, :] = normalize(v[:, i, :])\n",
    "    # v = v.reshape(-1, 512 * nens)\n",
    "    return k, v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ensemble from 5 models ['infer\\\\b5_cutout_800_lb0.832', 'infer\\\\b5_pseudo_cutout_lb0.836', 'infer\\\\b5_pseudo_lb0.832', 'infer\\\\b5_pseudo_lb0.833', 'infer\\\\b6_pseudo_lb0.83x']\n",
      "Ensemble from 5 models ['infer\\\\b5_cutout_800_lb0.832', 'infer\\\\b5_pseudo_cutout_lb0.836', 'infer\\\\b5_pseudo_lb0.832', 'infer\\\\b5_pseudo_lb0.833', 'infer\\\\b6_pseudo_lb0.83x']\n"
     ]
    }
   ],
   "source": [
    "# assert False\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "n_neighbors = 100\n",
    "knn = NearestNeighbors(n_neighbors=n_neighbors,metric='cosine')\n",
    "\n",
    "train_k, train_v = load(infer_dirs, 'train')\n",
    "test_k, test_v = load(infer_dirs, 'test')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 5/5 [05:50<00:00, 70.00s/it]\n"
     ]
    }
   ],
   "source": [
    "from tqdm.auto import tqdm\n",
    "\n",
    "distances_list = []\n",
    "idxs_list = []\n",
    "for i in tqdm(range(nens)):\n",
    "    traini, testi = train_v[:, i, :], test_v[:, i, :]\n",
    "    f = QuantileTransformer(output_distribution=\"normal\")\n",
    "    f.fit(testi)\n",
    "    traini = f.transform(traini)\n",
    "    testi = f.transform(testi)\n",
    "    knn.fit(traini)\n",
    "    d, idx = knn.kneighbors(testi, n_neighbors, return_distance=True)\n",
    "    distances_list.append(d)\n",
    "    idxs_list.append(idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "distances = np.stack(distances_list)\n",
    "idxs = np.stack(idxs_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 27956/27956 [00:03<00:00, 7072.05it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image</th>\n",
       "      <th>target</th>\n",
       "      <th>confidence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>dd5f7eb1cbe207.jpg</td>\n",
       "      <td>84a261c0e5cf</td>\n",
       "      <td>0.999468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>7a785b700b0339.jpg</td>\n",
       "      <td>c93996835aa8</td>\n",
       "      <td>0.999455</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>fe41a5bf4593bc.jpg</td>\n",
       "      <td>84a261c0e5cf</td>\n",
       "      <td>0.999355</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                image        target  confidence\n",
       "0  dd5f7eb1cbe207.jpg  84a261c0e5cf    0.999468\n",
       "1  7a785b700b0339.jpg  c93996835aa8    0.999455\n",
       "2  fe41a5bf4593bc.jpg  84a261c0e5cf    0.999355"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "test_df = []\n",
    "train_k = np.asarray(train_k)\n",
    "img2id = dict(zip(train_df.image, train_df.individual_id))\n",
    "\n",
    "for i in tqdm(range(len(test_k))):\n",
    "    dist, idx = distances[i], idxs[i]\n",
    "    for d, id in zip(dist, idx):\n",
    "        img_id = train_k[id]\n",
    "        tar = img2id[img_id]\n",
    "        test_df.append([test_k[i], tar, d])\n",
    "\n",
    "    \n",
    "cols = ['image', 'target', 'distances']\n",
    "test_df = pd.DataFrame(test_df, columns=cols)\n",
    "raw_test_df = test_df.copy()\n",
    "test_df['confidence'] = 1-test_df['distances']\n",
    "test_df = test_df.groupby(cols[:2]).confidence.max().reset_index()\n",
    "test_df = test_df.sort_values('confidence',ascending=False).reset_index(drop=True)\n",
    "test_df.to_csv('test_neighbors.csv')\n",
    "test_df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image</th>\n",
       "      <th>target</th>\n",
       "      <th>confidence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>fb4f83e1a357ed.jpg</td>\n",
       "      <td>6a3af6e0c55c</td>\n",
       "      <td>24.200880</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5acba135e9222c.jpg</td>\n",
       "      <td>6a3af6e0c55c</td>\n",
       "      <td>23.923040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bcd9dfc85d2d7a.jpg</td>\n",
       "      <td>6a3af6e0c55c</td>\n",
       "      <td>23.779226</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                image        target  confidence\n",
       "0  fb4f83e1a357ed.jpg  6a3af6e0c55c   24.200880\n",
       "1  5acba135e9222c.jpg  6a3af6e0c55c   23.923040\n",
       "2  bcd9dfc85d2d7a.jpg  6a3af6e0c55c   23.779226"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def accum(x):\n",
    "    # if len(x) > 3 and np.max(x) < 0.51:\n",
    "    #     return np.max(x) - 0.002 * len(x)\n",
    "    r = np.sum([v ** (8 * (i + 1)) for i, v in enumerate(x)])\n",
    "    return r\n",
    "\n",
    "raw_test_df['confidence'] = 1 - raw_test_df['distances']\n",
    "test_df = raw_test_df.groupby(['image','target']).confidence.agg(accum).reset_index()\n",
    "test_df = test_df.sort_values(['confidence'], ascending=False).reset_index(drop=True)\n",
    "test_df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "THR: 0.0039000000000000003: 0.21548147088281586\n"
     ]
    }
   ],
   "source": [
    "xt = test_df.groupby('image').confidence.max().reset_index()\n",
    "for thr in np.arange(0.0, 0.55, 0.0001):\n",
    "    r = len(xt[xt['confidence'] <= thr]) / len(xt)\n",
    "    if abs(r - 0.22) < 0.005:\n",
    "        print(f\"THR: {thr}: {r}\")\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1207833/1207833 [01:00<00:00, 19947.78it/s]\n",
      "100%|██████████| 27956/27956 [00:00<00:00, 1330745.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1104\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>image</th>\n",
       "      <th>predictions</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>fb4f83e1a357ed.jpg</td>\n",
       "      <td>6a3af6e0c55c new_individual 938b7e931166 5bf17...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5acba135e9222c.jpg</td>\n",
       "      <td>6a3af6e0c55c new_individual 938b7e931166 5bf17...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bcd9dfc85d2d7a.jpg</td>\n",
       "      <td>6a3af6e0c55c new_individual 938b7e931166 5bf17...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>78773f7edcf992.jpg</td>\n",
       "      <td>6a3af6e0c55c new_individual 938b7e931166 5bf17...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0454ac14150afc.jpg</td>\n",
       "      <td>6a3af6e0c55c new_individual 938b7e931166 5bf17...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                image                                        predictions\n",
       "0  fb4f83e1a357ed.jpg  6a3af6e0c55c new_individual 938b7e931166 5bf17...\n",
       "1  5acba135e9222c.jpg  6a3af6e0c55c new_individual 938b7e931166 5bf17...\n",
       "2  bcd9dfc85d2d7a.jpg  6a3af6e0c55c new_individual 938b7e931166 5bf17...\n",
       "3  78773f7edcf992.jpg  6a3af6e0c55c new_individual 938b7e931166 5bf17...\n",
       "4  0454ac14150afc.jpg  6a3af6e0c55c new_individual 938b7e931166 5bf17..."
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions = {}\n",
    "sample_list = ['938b7e931166', '5bf17305f073', '7593d2aee842', '7362d7a01d00','956562ff2888']\n",
    "\n",
    "for i,row in tqdm(test_df.iterrows(), total=len(test_df)):\n",
    "    if row.image in predictions:\n",
    "        if len(predictions[row.image])==5:\n",
    "            continue\n",
    "        predictions[row.image].append(row.target)\n",
    "    elif row.confidence>0.0:\n",
    "        predictions[row.image] = [row.target,'new_individual']\n",
    "    else:\n",
    "        predictions[row.image] = ['new_individual',row.target]\n",
    "\n",
    "c = 0\n",
    "for x in tqdm(predictions):\n",
    "    if len(predictions[x])<5:\n",
    "        c+=1\n",
    "        remaining = [y for y in sample_list if y not in predictions]\n",
    "        predictions[x] = predictions[x]+remaining\n",
    "        predictions[x] = predictions[x][:5]\n",
    "    predictions[x] = ' '.join(predictions[x])\n",
    "\n",
    "print(c)\n",
    "predictions = pd.Series(predictions).reset_index()\n",
    "predictions.columns = ['image','predictions']\n",
    "predictions.to_csv('submissionv2.csv',index=False)\n",
    "predictions.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6054871941622549"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v1 = pd.read_csv('submission.csv')\n",
    "v2 = predictions\n",
    "\n",
    "m = v1.merge(v2, on='image', how='left')\n",
    "(m.predictions_x == m.predictions_y).mean()"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "0d4bc70c65d06e1543861fe65a0e2a7420176491a9e60a2c8babf4d7456b2d28"
  },
  "kernelspec": {
   "display_name": "Python 3.7.9 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}