13.baseline_v3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] ='0'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import gc\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "out_dir = './out/'\n",
    "if not os.path.exists(out_dir):\n",
    "    os.mkdir(out_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import LabelEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from catboost import CatBoostClassifier, Pool"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = pd.read_pickle('./pkl/train_data.pkl')\n",
    "test_data = pd.read_pickle('./pkl/valid_data.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.concat([train_data, test_data]).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>author_id</th>\n",
       "      <th>author_name</th>\n",
       "      <th>author_org</th>\n",
       "      <th>label</th>\n",
       "      <th>paper_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8GjtUmBs</td>\n",
       "      <td>li_guo</td>\n",
       "      <td>Institute of Pharmacology and Toxicology</td>\n",
       "      <td>0.0</td>\n",
       "      <td>P9a1gcvg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>EShnTfSe</td>\n",
       "      <td>li_guo</td>\n",
       "      <td>Institute of Pharmacology and Toxicology</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P9a1gcvg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>t1ruuB9N</td>\n",
       "      <td>li_guo</td>\n",
       "      <td>Institute of Pharmacology and Toxicology</td>\n",
       "      <td>0.0</td>\n",
       "      <td>P9a1gcvg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>xLLXKy6I</td>\n",
       "      <td>li_guo</td>\n",
       "      <td>Institute of Pharmacology and Toxicology</td>\n",
       "      <td>0.0</td>\n",
       "      <td>P9a1gcvg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>jTu2AZES</td>\n",
       "      <td>li_guo</td>\n",
       "      <td>Institute of Pharmacology and Toxicology</td>\n",
       "      <td>0.0</td>\n",
       "      <td>P9a1gcvg</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  author_id author_name                                author_org  label  \\\n",
       "0  8GjtUmBs      li_guo  Institute of Pharmacology and Toxicology    0.0   \n",
       "1  EShnTfSe      li_guo  Institute of Pharmacology and Toxicology    1.0   \n",
       "2  t1ruuB9N      li_guo  Institute of Pharmacology and Toxicology    0.0   \n",
       "3  xLLXKy6I      li_guo  Institute of Pharmacology and Toxicology    0.0   \n",
       "4  jTu2AZES      li_guo  Institute of Pharmacology and Toxicology    0.0   \n",
       "\n",
       "   paper_id  \n",
       "0  P9a1gcvg  \n",
       "1  P9a1gcvg  \n",
       "2  P9a1gcvg  \n",
       "3  P9a1gcvg  \n",
       "4  P9a1gcvg  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1194996, 5)\n",
      "(452786, 4)\n",
      "(1647782, 5)\n"
     ]
    }
   ],
   "source": [
    "print(train_data.shape)\n",
    "print(test_data.shape)\n",
    "print(data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "##\n",
    "time_feat_a = pd.read_pickle('./feat/time_feat_a.pkl')\n",
    "rank_feat = pd.read_pickle('./feat/rank_feat3.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "feats = [time_feat_a, rank_feat]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.concat([data] + feats, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>author_id</th>\n",
       "      <th>author_name</th>\n",
       "      <th>author_org</th>\n",
       "      <th>label</th>\n",
       "      <th>paper_id</th>\n",
       "      <th>year_a</th>\n",
       "      <th>year_b_min</th>\n",
       "      <th>year_b_max</th>\n",
       "      <th>year_b_mean</th>\n",
       "      <th>year_b_std</th>\n",
       "      <th>...</th>\n",
       "      <th>authors_sims_mm2_bert_rank_b</th>\n",
       "      <th>orgs_sims_min_bert_rank_b</th>\n",
       "      <th>orgs_sims_max_bert_rank_b</th>\n",
       "      <th>orgs_sims_mean_bert_rank_b</th>\n",
       "      <th>orgs_sims_std_bert_rank_b</th>\n",
       "      <th>orgs_sims_mm2_bert_rank_b</th>\n",
       "      <th>orgs_sims_max2_bert_rank_b</th>\n",
       "      <th>orgs_sims_min2_bert_rank_b</th>\n",
       "      <th>orgs_sims_mean2_bert_rank_b</th>\n",
       "      <th>orgs_sims_std2_bert_rank_b</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8GjtUmBs</td>\n",
       "      <td>li_guo</td>\n",
       "      <td>Institute of Pharmacology and Toxicology</td>\n",
       "      <td>0.0</td>\n",
       "      <td>P9a1gcvg</td>\n",
       "      <td>1996</td>\n",
       "      <td>2013</td>\n",
       "      <td>2013</td>\n",
       "      <td>2013.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>EShnTfSe</td>\n",
       "      <td>li_guo</td>\n",
       "      <td>Institute of Pharmacology and Toxicology</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P9a1gcvg</td>\n",
       "      <td>1996</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>3.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>t1ruuB9N</td>\n",
       "      <td>li_guo</td>\n",
       "      <td>Institute of Pharmacology and Toxicology</td>\n",
       "      <td>0.0</td>\n",
       "      <td>P9a1gcvg</td>\n",
       "      <td>1996</td>\n",
       "      <td>2011</td>\n",
       "      <td>2011</td>\n",
       "      <td>2011.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>xLLXKy6I</td>\n",
       "      <td>li_guo</td>\n",
       "      <td>Institute of Pharmacology and Toxicology</td>\n",
       "      <td>0.0</td>\n",
       "      <td>P9a1gcvg</td>\n",
       "      <td>1996</td>\n",
       "      <td>2008</td>\n",
       "      <td>2008</td>\n",
       "      <td>2008.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>jTu2AZES</td>\n",
       "      <td>li_guo</td>\n",
       "      <td>Institute of Pharmacology and Toxicology</td>\n",
       "      <td>0.0</td>\n",
       "      <td>P9a1gcvg</td>\n",
       "      <td>1996</td>\n",
       "      <td>2011</td>\n",
       "      <td>2011</td>\n",
       "      <td>2011.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 168 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  author_id author_name                                author_org  label  \\\n",
       "0  8GjtUmBs      li_guo  Institute of Pharmacology and Toxicology    0.0   \n",
       "1  EShnTfSe      li_guo  Institute of Pharmacology and Toxicology    1.0   \n",
       "2  t1ruuB9N      li_guo  Institute of Pharmacology and Toxicology    0.0   \n",
       "3  xLLXKy6I      li_guo  Institute of Pharmacology and Toxicology    0.0   \n",
       "4  jTu2AZES      li_guo  Institute of Pharmacology and Toxicology    0.0   \n",
       "\n",
       "   paper_id  year_a  year_b_min  year_b_max  year_b_mean  year_b_std  \\\n",
       "0  P9a1gcvg    1996        2013        2013       2013.0         0.0   \n",
       "1  P9a1gcvg    1996           0           0          0.0         0.0   \n",
       "2  P9a1gcvg    1996        2011        2011       2011.0         0.0   \n",
       "3  P9a1gcvg    1996        2008        2008       2008.0         0.0   \n",
       "4  P9a1gcvg    1996        2011        2011       2011.0         0.0   \n",
       "\n",
       "              ...              authors_sims_mm2_bert_rank_b  \\\n",
       "0             ...                                       5.0   \n",
       "1             ...                                       6.0   \n",
       "2             ...                                       4.0   \n",
       "3             ...                                       1.0   \n",
       "4             ...                                       3.0   \n",
       "\n",
       "   orgs_sims_min_bert_rank_b  orgs_sims_max_bert_rank_b  \\\n",
       "0                        4.0                        4.0   \n",
       "1                        6.0                        6.0   \n",
       "2                        5.0                        5.0   \n",
       "3                        3.0                        3.0   \n",
       "4                        1.0                        1.0   \n",
       "\n",
       "   orgs_sims_mean_bert_rank_b  orgs_sims_std_bert_rank_b  \\\n",
       "0                         4.0                        4.0   \n",
       "1                         6.0                        4.0   \n",
       "2                         5.0                        4.0   \n",
       "3                         3.0                        4.0   \n",
       "4                         1.0                        4.0   \n",
       "\n",
       "   orgs_sims_mm2_bert_rank_b  orgs_sims_max2_bert_rank_b  \\\n",
       "0                        4.0                         4.0   \n",
       "1                        6.0                         6.0   \n",
       "2                        5.0                         5.0   \n",
       "3                        3.0                         3.0   \n",
       "4                        1.0                         1.0   \n",
       "\n",
       "   orgs_sims_min2_bert_rank_b  orgs_sims_mean2_bert_rank_b  \\\n",
       "0                         4.0                          4.0   \n",
       "1                         6.0                          6.0   \n",
       "2                         5.0                          5.0   \n",
       "3                         3.0                          3.0   \n",
       "4                         1.0                          1.0   \n",
       "\n",
       "   orgs_sims_std2_bert_rank_b  \n",
       "0                         3.5  \n",
       "1                         3.5  \n",
       "2                         3.5  \n",
       "3                         3.5  \n",
       "4                         3.5  \n",
       "\n",
       "[5 rows x 168 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "95\n",
      "['year_a', 'year_b_min', 'year_b_max', 'year_b_mean', 'year_b_std', 'year_b_mm2', 'year_b_min-year_a', 'year_b_max-year_a', 'year_b_mean-year_a', 'year_b_mm2-year_a', 'year_inside_range', 'paper_num_rank_a', 'author_org_in_orgs_b_times_rank_a', 'author_interset_num_rank_a', 'author_interset_num/paper_ids_len_rank_a', 'venue_a_in_venue_b_num_rank_a', 'keywords_interset_num_rank_a', 'venue_a_in_venue_b_num/paper_ids_len_rank_a', 'keywords_interset_num/paper_ids_len_rank_a', 'abstract_sims_min_bert_rank_a', 'abstract_sims_max_bert_rank_a', 'abstract_sims_mean_bert_rank_a', 'abstract_sims_std_bert_rank_a', 'abstract_sims_mm2_bert_rank_a', 'keywords_sims_min_bert_rank_a', 'keywords_sims_max_bert_rank_a', 'keywords_sims_mean_bert_rank_a', 'keywords_sims_std_bert_rank_a', 'keywords_sims_mm2_bert_rank_a', 'title_sims_min_bert_rank_a', 'title_sims_max_bert_rank_a', 'title_sims_mean_bert_rank_a', 'title_sims_std_bert_rank_a', 'title_sims_mm2_bert_rank_a', 'venue_sims_min_bert_rank_a', 'venue_sims_max_bert_rank_a', 'venue_sims_mean_bert_rank_a', 'venue_sims_std_bert_rank_a', 'venue_sims_mm2_bert_rank_a', 'authors_sims_min_bert_rank_a', 'authors_sims_max_bert_rank_a', 'authors_sims_mean_bert_rank_a', 'authors_sims_std_bert_rank_a', 'authors_sims_mm2_bert_rank_a', 'orgs_sims_min_bert_rank_a', 'orgs_sims_max_bert_rank_a', 'orgs_sims_mean_bert_rank_a', 'orgs_sims_std_bert_rank_a', 'orgs_sims_mm2_bert_rank_a', 'orgs_sims_max2_bert_rank_a', 'orgs_sims_min2_bert_rank_a', 'orgs_sims_mean2_bert_rank_a', 'orgs_sims_std2_bert_rank_a', 'paper_num_rank_b', 'author_org_in_orgs_b_times_rank_b', 'author_interset_num_rank_b', 'author_interset_num/paper_ids_len_rank_b', 'venue_a_in_venue_b_num_rank_b', 'keywords_interset_num_rank_b', 'venue_a_in_venue_b_num/paper_ids_len_rank_b', 'keywords_interset_num/paper_ids_len_rank_b', 'abstract_sims_min_bert_rank_b', 'abstract_sims_max_bert_rank_b', 'abstract_sims_mean_bert_rank_b', 'abstract_sims_std_bert_rank_b', 'abstract_sims_mm2_bert_rank_b', 'keywords_sims_min_bert_rank_b', 'keywords_sims_max_bert_rank_b', 'keywords_sims_mean_bert_rank_b', 'keywords_sims_std_bert_rank_b', 'keywords_sims_mm2_bert_rank_b', 'title_sims_min_bert_rank_b', 'title_sims_max_bert_rank_b', 'title_sims_mean_bert_rank_b', 'title_sims_std_bert_rank_b', 'title_sims_mm2_bert_rank_b', 'venue_sims_min_bert_rank_b', 'venue_sims_max_bert_rank_b', 'venue_sims_mean_bert_rank_b', 'venue_sims_std_bert_rank_b', 'venue_sims_mm2_bert_rank_b', 'authors_sims_min_bert_rank_b', 'authors_sims_max_bert_rank_b', 'authors_sims_mean_bert_rank_b', 'authors_sims_std_bert_rank_b', 'authors_sims_mm2_bert_rank_b', 'orgs_sims_min_bert_rank_b', 'orgs_sims_max_bert_rank_b', 'orgs_sims_mean_bert_rank_b', 'orgs_sims_std_bert_rank_b', 'orgs_sims_mm2_bert_rank_b', 'orgs_sims_max2_bert_rank_b', 'orgs_sims_min2_bert_rank_b', 'orgs_sims_mean2_bert_rank_b', 'orgs_sims_std2_bert_rank_b']\n"
     ]
    }
   ],
   "source": [
    "drop_feat = ['author_id', 'author_name', 'author_org', 'paper_id', 'label']\n",
    "used_feat = [c for c in data.columns if c not in drop_feat]\n",
    "print(len(used_feat))\n",
    "print(used_feat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = data[:len(train_data)]\n",
    "test = data[len(train_data):]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_x = test[used_feat]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "204\n"
     ]
    }
   ],
   "source": [
    "# cv split according to author names\n",
    "train_author_name = train['author_name'].unique()\n",
    "print(len(train_author_name))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def gen_dict(df, label):\n",
    "    df = df[['paper_id', 'author_name', 'author_id', label]]\n",
    "    res = df.groupby(['paper_id', 'author_name'])[label].apply(np.argmax).reset_index()\n",
    "    res.columns = ['paper_id', 'author_name', 'index']\n",
    "    idx_name = df[['author_id']].reset_index()\n",
    "    res = res.merge(idx_name, 'left', 'index')\n",
    "    from collections import defaultdict\n",
    "    res_dict = defaultdict(list)\n",
    "    for pid, aid in res[['paper_id', 'author_id']].values:\n",
    "        res_dict[aid].append(pid)\n",
    "    return res_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def f1_score(pred_dict, true_dict):\n",
    "    total_unassigned_paper = np.sum([len(l) for l in true_dict.values()])\n",
    "    print('total_unassigned_paper: ', total_unassigned_paper)\n",
    "    print('true author num: ', len(true_dict))\n",
    "    author_weight = dict((k, len(v) / total_unassigned_paper) for k, v in true_dict.items())\n",
    "    author_precision = {}\n",
    "    author_recall = {}\n",
    "    for author in author_weight.keys():\n",
    "        # total pred, total belong, correct pred\n",
    "        total_belong = len(true_dict[author])\n",
    "        total_pred = (len(pred_dict[author]) if author in pred_dict else 0)\n",
    "        correct_pred = len(set(true_dict[author]) & (set(pred_dict[author]) if author in pred_dict else set()))\n",
    "        author_precision[author] = (correct_pred/total_pred) if total_pred > 0 else 0\n",
    "        author_recall[author] = correct_pred / total_belong\n",
    "        \n",
    "    weighted_precision = 0\n",
    "    weighted_recall = 0\n",
    "    for author, weight in author_weight.items():\n",
    "        weighted_precision += weight * author_precision[author]\n",
    "        weighted_recall += weight * author_recall[author]\n",
    "    weighted_f1 = 2 * weighted_precision * weighted_recall / (weighted_precision + weighted_recall)\n",
    "    print('weighted_precision: %f, weighted_recall: %f, weighted_f1: %f' %(weighted_precision, weighted_recall, weighted_f1))\n",
    "    return weighted_precision, weighted_recall, weighted_f1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "******************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: less than 75% gpu memory available for training. Free: 4388.5625 Total: 4388.5625\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 0.9975497\ttest: 0.9971379\tbest: 0.9971379 (0)\ttotal: 15.8ms\tremaining: 2m 38s\n",
      "100:\tlearn: 0.9997494\ttest: 0.9996755\tbest: 0.9996755 (100)\ttotal: 1.31s\tremaining: 2m 8s\n",
      "200:\tlearn: 0.9998242\ttest: 0.9997479\tbest: 0.9997479 (200)\ttotal: 2.56s\tremaining: 2m 5s\n",
      "300:\tlearn: 0.9998572\ttest: 0.9997733\tbest: 0.9997734 (299)\ttotal: 3.83s\tremaining: 2m 3s\n",
      "400:\tlearn: 0.9998775\ttest: 0.9997894\tbest: 0.9997895 (399)\ttotal: 5.11s\tremaining: 2m 2s\n",
      "500:\tlearn: 0.9998966\ttest: 0.9997950\tbest: 0.9997954 (489)\ttotal: 6.39s\tremaining: 2m 1s\n",
      "600:\tlearn: 0.9999099\ttest: 0.9998027\tbest: 0.9998028 (595)\ttotal: 7.71s\tremaining: 2m\n",
      "700:\tlearn: 0.9999231\ttest: 0.9998082\tbest: 0.9998084 (695)\ttotal: 8.99s\tremaining: 1m 59s\n",
      "800:\tlearn: 0.9999328\ttest: 0.9998132\tbest: 0.9998132 (800)\ttotal: 10.3s\tremaining: 1m 58s\n",
      "900:\tlearn: 0.9999403\ttest: 0.9998142\tbest: 0.9998147 (874)\ttotal: 11.6s\tremaining: 1m 56s\n",
      "1000:\tlearn: 0.9999473\ttest: 0.9998164\tbest: 0.9998177 (989)\ttotal: 12.8s\tremaining: 1m 55s\n",
      "1100:\tlearn: 0.9999529\ttest: 0.9998193\tbest: 0.9998197 (1098)\ttotal: 14.1s\tremaining: 1m 54s\n",
      "1200:\tlearn: 0.9999587\ttest: 0.9998221\tbest: 0.9998222 (1198)\ttotal: 15.4s\tremaining: 1m 52s\n",
      "1300:\tlearn: 0.9999638\ttest: 0.9998223\tbest: 0.9998235 (1254)\ttotal: 16.7s\tremaining: 1m 51s\n",
      "1400:\tlearn: 0.9999681\ttest: 0.9998237\tbest: 0.9998238 (1389)\ttotal: 18s\tremaining: 1m 50s\n",
      "1500:\tlearn: 0.9999716\ttest: 0.9998228\tbest: 0.9998238 (1389)\ttotal: 19.2s\tremaining: 1m 48s\n",
      "1600:\tlearn: 0.9999748\ttest: 0.9998226\tbest: 0.9998238 (1389)\ttotal: 20.5s\tremaining: 1m 47s\n",
      "1700:\tlearn: 0.9999783\ttest: 0.9998230\tbest: 0.9998247 (1649)\ttotal: 21.8s\tremaining: 1m 46s\n",
      "1800:\tlearn: 0.9999810\ttest: 0.9998214\tbest: 0.9998247 (1649)\ttotal: 23.2s\tremaining: 1m 45s\n",
      "1900:\tlearn: 0.9999835\ttest: 0.9998207\tbest: 0.9998247 (1649)\ttotal: 24.5s\tremaining: 1m 44s\n",
      "bestTest = 0.9998247027\n",
      "bestIteration = 1649\n",
      "Shrink model to first 1650 iterations.\n",
      "total_unassigned_paper:  42757\n",
      "true author num:  4867\n",
      "weighted_precision: 0.994035, weighted_recall: 0.993779, weighted_f1: 0.993907\n",
      "fold 1 round 1649 : auc: 0.999825 | mean auc 0.999825 | F1: 0.993907 | mean F1: 0.993907\n",
      "******************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: less than 75% gpu memory available for training. Free: 4388.5625 Total: 4388.5625\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 0.9972583\ttest: 0.9969720\tbest: 0.9969720 (0)\ttotal: 46.3ms\tremaining: 7m 42s\n",
      "100:\tlearn: 0.9997396\ttest: 0.9996647\tbest: 0.9996647 (100)\ttotal: 2.63s\tremaining: 4m 18s\n",
      "200:\tlearn: 0.9998118\ttest: 0.9997188\tbest: 0.9997193 (197)\ttotal: 5.29s\tremaining: 4m 18s\n",
      "300:\tlearn: 0.9998518\ttest: 0.9997449\tbest: 0.9997452 (298)\ttotal: 7.98s\tremaining: 4m 17s\n",
      "400:\tlearn: 0.9998756\ttest: 0.9997635\tbest: 0.9997640 (399)\ttotal: 10.7s\tremaining: 4m 17s\n",
      "500:\tlearn: 0.9998925\ttest: 0.9997688\tbest: 0.9997688 (500)\ttotal: 13.5s\tremaining: 4m 16s\n",
      "600:\tlearn: 0.9999071\ttest: 0.9997710\tbest: 0.9997720 (557)\ttotal: 16.1s\tremaining: 4m 12s\n",
      "700:\tlearn: 0.9999195\ttest: 0.9997789\tbest: 0.9997789 (700)\ttotal: 18.8s\tremaining: 4m 9s\n",
      "800:\tlearn: 0.9999287\ttest: 0.9997822\tbest: 0.9997822 (796)\ttotal: 21.4s\tremaining: 4m 5s\n",
      "900:\tlearn: 0.9999379\ttest: 0.9997869\tbest: 0.9997870 (895)\ttotal: 24s\tremaining: 4m 2s\n",
      "1000:\tlearn: 0.9999461\ttest: 0.9997880\tbest: 0.9997884 (988)\ttotal: 26.8s\tremaining: 4m 1s\n",
      "1100:\tlearn: 0.9999518\ttest: 0.9997918\tbest: 0.9997920 (1096)\ttotal: 29.4s\tremaining: 3m 57s\n",
      "1200:\tlearn: 0.9999573\ttest: 0.9997933\tbest: 0.9997943 (1153)\ttotal: 32s\tremaining: 3m 54s\n",
      "1300:\tlearn: 0.9999630\ttest: 0.9997929\tbest: 0.9997943 (1153)\ttotal: 34.8s\tremaining: 3m 52s\n",
      "1400:\tlearn: 0.9999673\ttest: 0.9997923\tbest: 0.9997943 (1153)\ttotal: 37.3s\tremaining: 3m 48s\n",
      "bestTest = 0.9997943044\n",
      "bestIteration = 1153\n",
      "Shrink model to first 1154 iterations.\n",
      "total_unassigned_paper:  36561\n",
      "true author num:  4496\n",
      "weighted_precision: 0.993350, weighted_recall: 0.993107, weighted_f1: 0.993229\n",
      "fold 2 round 1153 : auc: 0.999794 | mean auc 0.999810 | F1: 0.993229 | mean F1: 0.993568\n",
      "******************************\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: less than 75% gpu memory available for training. Free: 4388.5625 Total: 4388.5625\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 0.9974340\ttest: 0.9970190\tbest: 0.9970190 (0)\ttotal: 17ms\tremaining: 2m 49s\n",
      "100:\tlearn: 0.9997501\ttest: 0.9996758\tbest: 0.9996758 (100)\ttotal: 2.5s\tremaining: 4m 4s\n",
      "200:\tlearn: 0.9998254\ttest: 0.9997292\tbest: 0.9997292 (200)\ttotal: 5.11s\tremaining: 4m 9s\n",
      "300:\tlearn: 0.9998624\ttest: 0.9997545\tbest: 0.9997545 (300)\ttotal: 7.62s\tremaining: 4m 5s\n",
      "400:\tlearn: 0.9998850\ttest: 0.9997683\tbest: 0.9997689 (396)\ttotal: 10.1s\tremaining: 4m 1s\n",
      "500:\tlearn: 0.9999032\ttest: 0.9997736\tbest: 0.9997737 (497)\ttotal: 12.7s\tremaining: 4m 1s\n",
      "600:\tlearn: 0.9999161\ttest: 0.9997758\tbest: 0.9997759 (597)\ttotal: 15.3s\tremaining: 3m 59s\n",
      "700:\tlearn: 0.9999282\ttest: 0.9997799\tbest: 0.9997799 (683)\ttotal: 17.9s\tremaining: 3m 58s\n",
      "800:\tlearn: 0.9999378\ttest: 0.9997867\tbest: 0.9997867 (800)\ttotal: 20.6s\tremaining: 3m 56s\n",
      "900:\tlearn: 0.9999455\ttest: 0.9997877\tbest: 0.9997891 (884)\ttotal: 23.1s\tremaining: 3m 53s\n",
      "1000:\tlearn: 0.9999517\ttest: 0.9997847\tbest: 0.9997891 (884)\ttotal: 25.7s\tremaining: 3m 51s\n",
      "1100:\tlearn: 0.9999573\ttest: 0.9997888\tbest: 0.9997891 (884)\ttotal: 28.4s\tremaining: 3m 49s\n",
      "1200:\tlearn: 0.9999627\ttest: 0.9997897\tbest: 0.9997905 (1195)\ttotal: 31s\tremaining: 3m 46s\n",
      "1300:\tlearn: 0.9999671\ttest: 0.9997894\tbest: 0.9997905 (1195)\ttotal: 33.6s\tremaining: 3m 44s\n",
      "1400:\tlearn: 0.9999710\ttest: 0.9997894\tbest: 0.9997905 (1195)\ttotal: 36.1s\tremaining: 3m 41s\n",
      "1500:\tlearn: 0.9999746\ttest: 0.9997917\tbest: 0.9997923 (1453)\ttotal: 38.5s\tremaining: 3m 38s\n",
      "1600:\tlearn: 0.9999779\ttest: 0.9997886\tbest: 0.9997923 (1453)\ttotal: 41.2s\tremaining: 3m 36s\n",
      "1700:\tlearn: 0.9999806\ttest: 0.9997883\tbest: 0.9997923 (1453)\ttotal: 43.9s\tremaining: 3m 34s\n",
      "bestTest = 0.9997922778\n",
      "bestIteration = 1453\n",
      "Shrink model to first 1454 iterations.\n",
      "total_unassigned_paper:  42971\n",
      "true author num:  4468\n",
      "weighted_precision: 0.994087, weighted_recall: 0.993647, weighted_f1: 0.993867\n",
      "fold 3 round 1453 : auc: 0.999792 | mean auc 0.999804 | F1: 0.993867 | mean F1: 0.993668\n",
      "******************************\n",
      "0:\tlearn: 0.9974858\ttest: 0.9985495\tbest: 0.9985495 (0)\ttotal: 16.7ms\tremaining: 2m 46s\n",
      "100:\tlearn: 0.9997184\ttest: 0.9998342\tbest: 0.9998342 (100)\ttotal: 1.32s\tremaining: 2m 9s\n",
      "200:\tlearn: 0.9997941\ttest: 0.9998685\tbest: 0.9998685 (199)\ttotal: 2.62s\tremaining: 2m 7s\n",
      "300:\tlearn: 0.9998313\ttest: 0.9998823\tbest: 0.9998823 (299)\ttotal: 3.88s\tremaining: 2m 5s\n",
      "400:\tlearn: 0.9998623\ttest: 0.9998891\tbest: 0.9998891 (392)\ttotal: 5.14s\tremaining: 2m 3s\n",
      "500:\tlearn: 0.9998828\ttest: 0.9998931\tbest: 0.9998939 (482)\ttotal: 6.42s\tremaining: 2m 1s\n",
      "600:\tlearn: 0.9998984\ttest: 0.9998953\tbest: 0.9998955 (581)\ttotal: 7.71s\tremaining: 2m\n",
      "700:\tlearn: 0.9999109\ttest: 0.9998994\tbest: 0.9998997 (695)\ttotal: 9.02s\tremaining: 1m 59s\n",
      "800:\tlearn: 0.9999220\ttest: 0.9999006\tbest: 0.9999010 (779)\ttotal: 10.3s\tremaining: 1m 58s\n",
      "900:\tlearn: 0.9999312\ttest: 0.9999031\tbest: 0.9999034 (897)\ttotal: 11.6s\tremaining: 1m 57s\n",
      "1000:\tlearn: 0.9999390\ttest: 0.9999030\tbest: 0.9999034 (897)\ttotal: 12.9s\tremaining: 1m 55s\n",
      "1100:\tlearn: 0.9999466\ttest: 0.9999055\tbest: 0.9999055 (1099)\ttotal: 14.1s\tremaining: 1m 54s\n",
      "1200:\tlearn: 0.9999529\ttest: 0.9999067\tbest: 0.9999068 (1195)\ttotal: 15.4s\tremaining: 1m 52s\n",
      "1300:\tlearn: 0.9999585\ttest: 0.9999071\tbest: 0.9999074 (1257)\ttotal: 16.7s\tremaining: 1m 51s\n",
      "1400:\tlearn: 0.9999635\ttest: 0.9999079\tbest: 0.9999081 (1361)\ttotal: 18s\tremaining: 1m 50s\n",
      "1500:\tlearn: 0.9999673\ttest: 0.9999080\tbest: 0.9999081 (1361)\ttotal: 19.4s\tremaining: 1m 49s\n",
      "1600:\tlearn: 0.9999709\ttest: 0.9999073\tbest: 0.9999081 (1361)\ttotal: 20.7s\tremaining: 1m 48s\n",
      "bestTest = 0.9999081492\n",
      "bestIteration = 1361\n",
      "Shrink model to first 1362 iterations.\n",
      "total_unassigned_paper:  44391\n",
      "true author num:  4719\n",
      "weighted_precision: 0.995167, weighted_recall: 0.995044, weighted_f1: 0.995106\n",
      "fold 4 round 1361 : auc: 0.999908 | mean auc 0.999830 | F1: 0.995106 | mean F1: 0.994027\n",
      "******************************\n",
      "0:\tlearn: 0.9977620\ttest: 0.9972097\tbest: 0.9972097 (0)\ttotal: 17ms\tremaining: 2m 49s\n",
      "100:\tlearn: 0.9997423\ttest: 0.9996644\tbest: 0.9996644 (100)\ttotal: 1.42s\tremaining: 2m 19s\n",
      "200:\tlearn: 0.9998112\ttest: 0.9997213\tbest: 0.9997213 (200)\ttotal: 2.82s\tremaining: 2m 17s\n",
      "300:\tlearn: 0.9998519\ttest: 0.9997436\tbest: 0.9997436 (300)\ttotal: 4.24s\tremaining: 2m 16s\n",
      "400:\tlearn: 0.9998763\ttest: 0.9997526\tbest: 0.9997527 (399)\ttotal: 5.71s\tremaining: 2m 16s\n",
      "500:\tlearn: 0.9998931\ttest: 0.9997582\tbest: 0.9997586 (491)\ttotal: 7.11s\tremaining: 2m 14s\n",
      "600:\tlearn: 0.9999086\ttest: 0.9997628\tbest: 0.9997630 (595)\ttotal: 8.52s\tremaining: 2m 13s\n",
      "700:\tlearn: 0.9999208\ttest: 0.9997649\tbest: 0.9997666 (630)\ttotal: 9.92s\tremaining: 2m 11s\n",
      "800:\tlearn: 0.9999307\ttest: 0.9997659\tbest: 0.9997666 (630)\ttotal: 11.3s\tremaining: 2m 9s\n",
      "900:\tlearn: 0.9999394\ttest: 0.9997656\tbest: 0.9997668 (882)\ttotal: 12.8s\tremaining: 2m 8s\n",
      "1000:\tlearn: 0.9999467\ttest: 0.9997666\tbest: 0.9997668 (882)\ttotal: 14.2s\tremaining: 2m 7s\n",
      "1100:\tlearn: 0.9999532\ttest: 0.9997687\tbest: 0.9997687 (1099)\ttotal: 15.6s\tremaining: 2m 5s\n",
      "1200:\tlearn: 0.9999597\ttest: 0.9997683\tbest: 0.9997691 (1147)\ttotal: 17s\tremaining: 2m 4s\n",
      "1300:\tlearn: 0.9999647\ttest: 0.9997692\tbest: 0.9997693 (1298)\ttotal: 18.4s\tremaining: 2m 3s\n",
      "1400:\tlearn: 0.9999691\ttest: 0.9997697\tbest: 0.9997705 (1389)\ttotal: 19.9s\tremaining: 2m 1s\n",
      "1500:\tlearn: 0.9999728\ttest: 0.9997672\tbest: 0.9997705 (1389)\ttotal: 21.3s\tremaining: 2m\n",
      "1600:\tlearn: 0.9999760\ttest: 0.9997669\tbest: 0.9997705 (1389)\ttotal: 22.7s\tremaining: 1m 59s\n",
      "bestTest = 0.9997704625\n",
      "bestIteration = 1389\n",
      "Shrink model to first 1390 iterations.\n",
      "total_unassigned_paper:  35840\n",
      "true author num:  4214\n",
      "weighted_precision: 0.991886, weighted_recall: 0.991406, weighted_f1: 0.991646\n",
      "fold 5 round 1389 : auc: 0.999770 | mean auc 0.999818 | F1: 0.991646 | mean F1: 0.993551\n"
     ]
    }
   ],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "preds = np.zeros((test.shape[0], 2))\n",
    "scores = []\n",
    "f1_scores = []\n",
    "has_saved = False\n",
    "imp = pd.DataFrame()\n",
    "imp['feat'] = used_feat\n",
    "\n",
    "kfold = KFold(n_splits=5, shuffle=True, random_state=42)\n",
    "for index, (tr_idx, va_idx) in enumerate(kfold.split(train_author_name)):\n",
    "    print('*' * 30)\n",
    "    trn_aname, val_aname = train_author_name[tr_idx], train_author_name[va_idx]\n",
    "    trn_dat = train[train['author_name'].isin(trn_aname)]\n",
    "    val_dat = train[train['author_name'].isin(val_aname)]\n",
    "    X_train, y_train, X_valid, y_valid = trn_dat[used_feat], trn_dat['label'], val_dat[used_feat], val_dat['label']\n",
    "    cate_features = []\n",
    "    train_pool = Pool(X_train, y_train, cat_features=cate_features)\n",
    "    eval_pool = Pool(X_valid, y_valid,cat_features=cate_features)\n",
    "    if not has_saved: \n",
    "        cbt_model = CatBoostClassifier(iterations=10000,\n",
    "                           learning_rate=0.1,\n",
    "                           eval_metric='AUC',\n",
    "                           use_best_model=True,\n",
    "                           random_seed=42,\n",
    "                           logging_level='Verbose',\n",
    "                           task_type='GPU',\n",
    "                           devices='0',\n",
    "#                            gpu_ram_part=0.5,\n",
    "                           early_stopping_rounds=300,\n",
    "                           loss_function='Logloss',\n",
    "#                            depth=12,\n",
    "                           )\n",
    "        cbt_model.fit(train_pool, eval_set=eval_pool, verbose=100)\n",
    "#         with open('./models/fold%d_cbt_v1.mdl' % index, 'wb') as file:\n",
    "#             pickle.dump(cbt_model, file)\n",
    "#     else:\n",
    "#         with open('./models/fold%d_cbt_v1.mdl' % index, 'rb') as file:\n",
    "#             cbt_model = pickle.load(file)\n",
    "    \n",
    "    imp['score%d' % (index+1)] = cbt_model.feature_importances_\n",
    "    \n",
    "    val_dat['pred'] = cbt_model.predict_proba(X_valid)[:, 1]\n",
    "    val_pred_dict = gen_dict(val_dat, 'pred')\n",
    "    val_true_dict = gen_dict(val_dat, 'label')\n",
    "    precision, recall, f1 = f1_score(val_pred_dict, val_true_dict)\n",
    "    f1_scores.append(f1)\n",
    "    \n",
    "    score = cbt_model.best_score_['validation']['AUC']\n",
    "    scores.append(score)\n",
    "    print('fold %d round %d : auc: %.6f | mean auc %.6f | F1: %.6f | mean F1: %.6f' % (index+1, cbt_model.best_iteration_, score,np.mean(scores), f1, np.mean(f1_scores))) \n",
    "    preds += cbt_model.predict_proba(test_x)  \n",
    "#     break\n",
    "    del cbt_model, train_pool, eval_pool\n",
    "    del X_train, y_train, X_valid, y_valid\n",
    "    import gc\n",
    "    gc.collect()\n",
    "    \n",
    "#     mdls.append(cbt_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feat</th>\n",
       "      <th>score1</th>\n",
       "      <th>score2</th>\n",
       "      <th>score3</th>\n",
       "      <th>score4</th>\n",
       "      <th>score5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>author_interset_num_rank_b</td>\n",
       "      <td>8.298452</td>\n",
       "      <td>9.578436</td>\n",
       "      <td>7.598800</td>\n",
       "      <td>7.475774</td>\n",
       "      <td>8.116196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>year_b_max</td>\n",
       "      <td>5.735328</td>\n",
       "      <td>6.535335</td>\n",
       "      <td>3.740404</td>\n",
       "      <td>5.142827</td>\n",
       "      <td>8.352572</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>orgs_sims_max2_bert_rank_b</td>\n",
       "      <td>5.313472</td>\n",
       "      <td>5.694128</td>\n",
       "      <td>7.393179</td>\n",
       "      <td>7.184583</td>\n",
       "      <td>7.615661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>author_interset_num_rank_a</td>\n",
       "      <td>4.948066</td>\n",
       "      <td>4.680774</td>\n",
       "      <td>5.389117</td>\n",
       "      <td>5.553717</td>\n",
       "      <td>5.218932</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>year_b_std</td>\n",
       "      <td>3.532221</td>\n",
       "      <td>3.057120</td>\n",
       "      <td>3.247804</td>\n",
       "      <td>3.318082</td>\n",
       "      <td>3.121897</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>orgs_sims_max2_bert_rank_a</td>\n",
       "      <td>3.236305</td>\n",
       "      <td>4.152073</td>\n",
       "      <td>5.122202</td>\n",
       "      <td>3.326928</td>\n",
       "      <td>3.131560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>paper_num_rank_a</td>\n",
       "      <td>2.297650</td>\n",
       "      <td>2.131351</td>\n",
       "      <td>2.037330</td>\n",
       "      <td>2.180784</td>\n",
       "      <td>2.117218</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>year_b_mean</td>\n",
       "      <td>2.252438</td>\n",
       "      <td>1.168923</td>\n",
       "      <td>2.083037</td>\n",
       "      <td>4.591285</td>\n",
       "      <td>1.958823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>orgs_sims_std2_bert_rank_a</td>\n",
       "      <td>2.173362</td>\n",
       "      <td>2.504588</td>\n",
       "      <td>1.786622</td>\n",
       "      <td>2.619718</td>\n",
       "      <td>2.884649</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>venue_a_in_venue_b_num_rank_b</td>\n",
       "      <td>2.056701</td>\n",
       "      <td>3.294738</td>\n",
       "      <td>1.887856</td>\n",
       "      <td>3.237639</td>\n",
       "      <td>1.778620</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>year_b_mm2-year_a</td>\n",
       "      <td>2.008491</td>\n",
       "      <td>2.512041</td>\n",
       "      <td>2.005812</td>\n",
       "      <td>0.340251</td>\n",
       "      <td>1.566271</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>author_interset_num/paper_ids_len_rank_b</td>\n",
       "      <td>2.004028</td>\n",
       "      <td>1.760037</td>\n",
       "      <td>2.080413</td>\n",
       "      <td>1.738537</td>\n",
       "      <td>1.345648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>paper_num_rank_b</td>\n",
       "      <td>2.000704</td>\n",
       "      <td>1.781295</td>\n",
       "      <td>2.089642</td>\n",
       "      <td>2.097227</td>\n",
       "      <td>2.281512</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>title_sims_max_bert_rank_b</td>\n",
       "      <td>1.969042</td>\n",
       "      <td>1.576145</td>\n",
       "      <td>1.828183</td>\n",
       "      <td>1.726377</td>\n",
       "      <td>1.657107</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>year_b_mm2</td>\n",
       "      <td>1.968963</td>\n",
       "      <td>1.787924</td>\n",
       "      <td>2.457097</td>\n",
       "      <td>1.011037</td>\n",
       "      <td>0.838681</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>venue_a_in_venue_b_num/paper_ids_len_rank_b</td>\n",
       "      <td>1.839003</td>\n",
       "      <td>1.257734</td>\n",
       "      <td>0.877650</td>\n",
       "      <td>1.444203</td>\n",
       "      <td>1.700464</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>author_org_in_orgs_b_times_rank_a</td>\n",
       "      <td>1.735343</td>\n",
       "      <td>1.402116</td>\n",
       "      <td>1.317598</td>\n",
       "      <td>1.164456</td>\n",
       "      <td>1.336619</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>author_interset_num/paper_ids_len_rank_a</td>\n",
       "      <td>1.691743</td>\n",
       "      <td>1.515626</td>\n",
       "      <td>1.528875</td>\n",
       "      <td>1.187903</td>\n",
       "      <td>1.584299</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>year_inside_range</td>\n",
       "      <td>1.624007</td>\n",
       "      <td>0.786667</td>\n",
       "      <td>1.251221</td>\n",
       "      <td>0.779662</td>\n",
       "      <td>0.813357</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>year_b_mean-year_a</td>\n",
       "      <td>1.588200</td>\n",
       "      <td>0.692718</td>\n",
       "      <td>1.507300</td>\n",
       "      <td>1.081650</td>\n",
       "      <td>0.781860</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>year_a</td>\n",
       "      <td>1.541591</td>\n",
       "      <td>2.442837</td>\n",
       "      <td>1.538997</td>\n",
       "      <td>1.454740</td>\n",
       "      <td>1.818557</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>year_b_max-year_a</td>\n",
       "      <td>1.519968</td>\n",
       "      <td>2.755024</td>\n",
       "      <td>2.632201</td>\n",
       "      <td>3.376249</td>\n",
       "      <td>1.921294</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>orgs_sims_mean2_bert_rank_a</td>\n",
       "      <td>1.450139</td>\n",
       "      <td>1.120694</td>\n",
       "      <td>0.931383</td>\n",
       "      <td>0.924929</td>\n",
       "      <td>0.579085</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>authors_sims_max_bert_rank_b</td>\n",
       "      <td>1.322112</td>\n",
       "      <td>1.558610</td>\n",
       "      <td>1.226965</td>\n",
       "      <td>1.473427</td>\n",
       "      <td>1.604767</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>author_org_in_orgs_b_times_rank_b</td>\n",
       "      <td>1.175386</td>\n",
       "      <td>1.311069</td>\n",
       "      <td>2.164446</td>\n",
       "      <td>1.160218</td>\n",
       "      <td>1.275777</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>year_b_min-year_a</td>\n",
       "      <td>1.151827</td>\n",
       "      <td>1.009140</td>\n",
       "      <td>0.958247</td>\n",
       "      <td>1.096359</td>\n",
       "      <td>1.348554</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>orgs_sims_mean2_bert_rank_b</td>\n",
       "      <td>1.107941</td>\n",
       "      <td>0.489296</td>\n",
       "      <td>1.025109</td>\n",
       "      <td>0.628833</td>\n",
       "      <td>0.429666</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>orgs_sims_min_bert_rank_a</td>\n",
       "      <td>1.081552</td>\n",
       "      <td>0.751255</td>\n",
       "      <td>0.696623</td>\n",
       "      <td>0.914205</td>\n",
       "      <td>0.529106</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>venue_sims_max_bert_rank_b</td>\n",
       "      <td>0.889462</td>\n",
       "      <td>1.069687</td>\n",
       "      <td>0.991642</td>\n",
       "      <td>0.916391</td>\n",
       "      <td>1.006488</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>orgs_sims_std2_bert_rank_b</td>\n",
       "      <td>0.866779</td>\n",
       "      <td>0.980031</td>\n",
       "      <td>0.952244</td>\n",
       "      <td>1.515145</td>\n",
       "      <td>1.104695</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>authors_sims_min_bert_rank_a</td>\n",
       "      <td>0.410576</td>\n",
       "      <td>0.251978</td>\n",
       "      <td>0.466578</td>\n",
       "      <td>0.328818</td>\n",
       "      <td>0.430818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>orgs_sims_std_bert_rank_b</td>\n",
       "      <td>0.405391</td>\n",
       "      <td>0.327544</td>\n",
       "      <td>0.219746</td>\n",
       "      <td>0.381034</td>\n",
       "      <td>0.363560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>venue_a_in_venue_b_num/paper_ids_len_rank_a</td>\n",
       "      <td>0.393324</td>\n",
       "      <td>0.632492</td>\n",
       "      <td>0.800952</td>\n",
       "      <td>0.348476</td>\n",
       "      <td>0.440102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>orgs_sims_std_bert_rank_a</td>\n",
       "      <td>0.392175</td>\n",
       "      <td>0.344238</td>\n",
       "      <td>0.508873</td>\n",
       "      <td>0.466276</td>\n",
       "      <td>0.330494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>abstract_sims_mm2_bert_rank_a</td>\n",
       "      <td>0.379302</td>\n",
       "      <td>0.169454</td>\n",
       "      <td>0.346927</td>\n",
       "      <td>0.363078</td>\n",
       "      <td>0.377655</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>venue_sims_min_bert_rank_b</td>\n",
       "      <td>0.379167</td>\n",
       "      <td>0.191031</td>\n",
       "      <td>0.245128</td>\n",
       "      <td>0.205836</td>\n",
       "      <td>0.281396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>venue_sims_mm2_bert_rank_b</td>\n",
       "      <td>0.376931</td>\n",
       "      <td>0.201727</td>\n",
       "      <td>0.384620</td>\n",
       "      <td>0.357203</td>\n",
       "      <td>0.518069</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>orgs_sims_mean_bert_rank_a</td>\n",
       "      <td>0.369830</td>\n",
       "      <td>0.114240</td>\n",
       "      <td>0.230646</td>\n",
       "      <td>0.412384</td>\n",
       "      <td>0.277407</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>keywords_sims_mean_bert_rank_b</td>\n",
       "      <td>0.361449</td>\n",
       "      <td>0.357742</td>\n",
       "      <td>0.192362</td>\n",
       "      <td>0.281239</td>\n",
       "      <td>0.261446</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>abstract_sims_min_bert_rank_b</td>\n",
       "      <td>0.349830</td>\n",
       "      <td>0.327280</td>\n",
       "      <td>0.239938</td>\n",
       "      <td>0.237655</td>\n",
       "      <td>0.206550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>authors_sims_std_bert_rank_a</td>\n",
       "      <td>0.345005</td>\n",
       "      <td>0.213104</td>\n",
       "      <td>0.484045</td>\n",
       "      <td>0.291294</td>\n",
       "      <td>0.559067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>venue_sims_std_bert_rank_a</td>\n",
       "      <td>0.335925</td>\n",
       "      <td>0.280561</td>\n",
       "      <td>0.219818</td>\n",
       "      <td>0.193559</td>\n",
       "      <td>0.355494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>abstract_sims_mean_bert_rank_b</td>\n",
       "      <td>0.333190</td>\n",
       "      <td>0.221689</td>\n",
       "      <td>0.307498</td>\n",
       "      <td>0.184629</td>\n",
       "      <td>0.217488</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>keywords_sims_std_bert_rank_b</td>\n",
       "      <td>0.332738</td>\n",
       "      <td>0.326019</td>\n",
       "      <td>0.476710</td>\n",
       "      <td>0.398757</td>\n",
       "      <td>0.332403</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>abstract_sims_min_bert_rank_a</td>\n",
       "      <td>0.318525</td>\n",
       "      <td>0.407954</td>\n",
       "      <td>0.421188</td>\n",
       "      <td>0.154015</td>\n",
       "      <td>0.323406</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>venue_sims_mm2_bert_rank_a</td>\n",
       "      <td>0.316765</td>\n",
       "      <td>0.146759</td>\n",
       "      <td>0.191536</td>\n",
       "      <td>0.320332</td>\n",
       "      <td>0.433849</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>authors_sims_min_bert_rank_b</td>\n",
       "      <td>0.306232</td>\n",
       "      <td>0.400674</td>\n",
       "      <td>0.360392</td>\n",
       "      <td>0.519989</td>\n",
       "      <td>0.255747</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>authors_sims_std_bert_rank_b</td>\n",
       "      <td>0.303092</td>\n",
       "      <td>0.210011</td>\n",
       "      <td>0.217949</td>\n",
       "      <td>0.438402</td>\n",
       "      <td>0.369037</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>venue_sims_std_bert_rank_b</td>\n",
       "      <td>0.302069</td>\n",
       "      <td>0.303074</td>\n",
       "      <td>0.416861</td>\n",
       "      <td>0.334152</td>\n",
       "      <td>0.410621</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>authors_sims_mean_bert_rank_a</td>\n",
       "      <td>0.300849</td>\n",
       "      <td>0.327235</td>\n",
       "      <td>0.211685</td>\n",
       "      <td>0.332910</td>\n",
       "      <td>0.179289</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>title_sims_min_bert_rank_b</td>\n",
       "      <td>0.300268</td>\n",
       "      <td>0.244979</td>\n",
       "      <td>0.382181</td>\n",
       "      <td>0.382663</td>\n",
       "      <td>0.348616</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>keywords_sims_mean_bert_rank_a</td>\n",
       "      <td>0.279161</td>\n",
       "      <td>0.189983</td>\n",
       "      <td>0.100759</td>\n",
       "      <td>0.218952</td>\n",
       "      <td>0.205560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>authors_sims_mm2_bert_rank_a</td>\n",
       "      <td>0.270357</td>\n",
       "      <td>0.253769</td>\n",
       "      <td>0.341063</td>\n",
       "      <td>0.309190</td>\n",
       "      <td>0.204094</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>authors_sims_mm2_bert_rank_b</td>\n",
       "      <td>0.247467</td>\n",
       "      <td>0.155342</td>\n",
       "      <td>0.281182</td>\n",
       "      <td>0.194120</td>\n",
       "      <td>0.193588</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>keywords_sims_mm2_bert_rank_a</td>\n",
       "      <td>0.233275</td>\n",
       "      <td>0.211766</td>\n",
       "      <td>0.154922</td>\n",
       "      <td>0.227651</td>\n",
       "      <td>0.134563</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>keywords_sims_mm2_bert_rank_b</td>\n",
       "      <td>0.224910</td>\n",
       "      <td>0.228418</td>\n",
       "      <td>0.262136</td>\n",
       "      <td>0.230497</td>\n",
       "      <td>0.149155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>orgs_sims_mm2_bert_rank_b</td>\n",
       "      <td>0.224068</td>\n",
       "      <td>0.138086</td>\n",
       "      <td>0.219107</td>\n",
       "      <td>0.294393</td>\n",
       "      <td>0.212926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>orgs_sims_mm2_bert_rank_a</td>\n",
       "      <td>0.211761</td>\n",
       "      <td>0.291360</td>\n",
       "      <td>0.194420</td>\n",
       "      <td>0.113114</td>\n",
       "      <td>0.170742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>abstract_sims_mm2_bert_rank_b</td>\n",
       "      <td>0.201753</td>\n",
       "      <td>0.201010</td>\n",
       "      <td>0.290671</td>\n",
       "      <td>0.235395</td>\n",
       "      <td>0.279009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>venue_sims_mean_bert_rank_a</td>\n",
       "      <td>0.164030</td>\n",
       "      <td>0.207942</td>\n",
       "      <td>0.266810</td>\n",
       "      <td>0.165573</td>\n",
       "      <td>0.198246</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>95 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           feat    score1    score2    score3  \\\n",
       "55                   author_interset_num_rank_b  8.298452  9.578436  7.598800   \n",
       "2                                    year_b_max  5.735328  6.535335  3.740404   \n",
       "91                   orgs_sims_max2_bert_rank_b  5.313472  5.694128  7.393179   \n",
       "13                   author_interset_num_rank_a  4.948066  4.680774  5.389117   \n",
       "4                                    year_b_std  3.532221  3.057120  3.247804   \n",
       "49                   orgs_sims_max2_bert_rank_a  3.236305  4.152073  5.122202   \n",
       "11                             paper_num_rank_a  2.297650  2.131351  2.037330   \n",
       "3                                   year_b_mean  2.252438  1.168923  2.083037   \n",
       "52                   orgs_sims_std2_bert_rank_a  2.173362  2.504588  1.786622   \n",
       "57                venue_a_in_venue_b_num_rank_b  2.056701  3.294738  1.887856   \n",
       "9                             year_b_mm2-year_a  2.008491  2.512041  2.005812   \n",
       "56     author_interset_num/paper_ids_len_rank_b  2.004028  1.760037  2.080413   \n",
       "53                             paper_num_rank_b  2.000704  1.781295  2.089642   \n",
       "72                   title_sims_max_bert_rank_b  1.969042  1.576145  1.828183   \n",
       "5                                    year_b_mm2  1.968963  1.787924  2.457097   \n",
       "59  venue_a_in_venue_b_num/paper_ids_len_rank_b  1.839003  1.257734  0.877650   \n",
       "12            author_org_in_orgs_b_times_rank_a  1.735343  1.402116  1.317598   \n",
       "14     author_interset_num/paper_ids_len_rank_a  1.691743  1.515626  1.528875   \n",
       "10                            year_inside_range  1.624007  0.786667  1.251221   \n",
       "8                            year_b_mean-year_a  1.588200  0.692718  1.507300   \n",
       "0                                        year_a  1.541591  2.442837  1.538997   \n",
       "7                             year_b_max-year_a  1.519968  2.755024  2.632201   \n",
       "51                  orgs_sims_mean2_bert_rank_a  1.450139  1.120694  0.931383   \n",
       "82                 authors_sims_max_bert_rank_b  1.322112  1.558610  1.226965   \n",
       "54            author_org_in_orgs_b_times_rank_b  1.175386  1.311069  2.164446   \n",
       "6                             year_b_min-year_a  1.151827  1.009140  0.958247   \n",
       "93                  orgs_sims_mean2_bert_rank_b  1.107941  0.489296  1.025109   \n",
       "44                    orgs_sims_min_bert_rank_a  1.081552  0.751255  0.696623   \n",
       "77                   venue_sims_max_bert_rank_b  0.889462  1.069687  0.991642   \n",
       "94                   orgs_sims_std2_bert_rank_b  0.866779  0.980031  0.952244   \n",
       "..                                          ...       ...       ...       ...   \n",
       "39                 authors_sims_min_bert_rank_a  0.410576  0.251978  0.466578   \n",
       "89                    orgs_sims_std_bert_rank_b  0.405391  0.327544  0.219746   \n",
       "17  venue_a_in_venue_b_num/paper_ids_len_rank_a  0.393324  0.632492  0.800952   \n",
       "47                    orgs_sims_std_bert_rank_a  0.392175  0.344238  0.508873   \n",
       "23                abstract_sims_mm2_bert_rank_a  0.379302  0.169454  0.346927   \n",
       "76                   venue_sims_min_bert_rank_b  0.379167  0.191031  0.245128   \n",
       "80                   venue_sims_mm2_bert_rank_b  0.376931  0.201727  0.384620   \n",
       "46                   orgs_sims_mean_bert_rank_a  0.369830  0.114240  0.230646   \n",
       "68               keywords_sims_mean_bert_rank_b  0.361449  0.357742  0.192362   \n",
       "61                abstract_sims_min_bert_rank_b  0.349830  0.327280  0.239938   \n",
       "42                 authors_sims_std_bert_rank_a  0.345005  0.213104  0.484045   \n",
       "37                   venue_sims_std_bert_rank_a  0.335925  0.280561  0.219818   \n",
       "63               abstract_sims_mean_bert_rank_b  0.333190  0.221689  0.307498   \n",
       "69                keywords_sims_std_bert_rank_b  0.332738  0.326019  0.476710   \n",
       "19                abstract_sims_min_bert_rank_a  0.318525  0.407954  0.421188   \n",
       "38                   venue_sims_mm2_bert_rank_a  0.316765  0.146759  0.191536   \n",
       "81                 authors_sims_min_bert_rank_b  0.306232  0.400674  0.360392   \n",
       "84                 authors_sims_std_bert_rank_b  0.303092  0.210011  0.217949   \n",
       "79                   venue_sims_std_bert_rank_b  0.302069  0.303074  0.416861   \n",
       "41                authors_sims_mean_bert_rank_a  0.300849  0.327235  0.211685   \n",
       "71                   title_sims_min_bert_rank_b  0.300268  0.244979  0.382181   \n",
       "26               keywords_sims_mean_bert_rank_a  0.279161  0.189983  0.100759   \n",
       "43                 authors_sims_mm2_bert_rank_a  0.270357  0.253769  0.341063   \n",
       "85                 authors_sims_mm2_bert_rank_b  0.247467  0.155342  0.281182   \n",
       "28                keywords_sims_mm2_bert_rank_a  0.233275  0.211766  0.154922   \n",
       "70                keywords_sims_mm2_bert_rank_b  0.224910  0.228418  0.262136   \n",
       "90                    orgs_sims_mm2_bert_rank_b  0.224068  0.138086  0.219107   \n",
       "48                    orgs_sims_mm2_bert_rank_a  0.211761  0.291360  0.194420   \n",
       "65                abstract_sims_mm2_bert_rank_b  0.201753  0.201010  0.290671   \n",
       "36                  venue_sims_mean_bert_rank_a  0.164030  0.207942  0.266810   \n",
       "\n",
       "      score4    score5  \n",
       "55  7.475774  8.116196  \n",
       "2   5.142827  8.352572  \n",
       "91  7.184583  7.615661  \n",
       "13  5.553717  5.218932  \n",
       "4   3.318082  3.121897  \n",
       "49  3.326928  3.131560  \n",
       "11  2.180784  2.117218  \n",
       "3   4.591285  1.958823  \n",
       "52  2.619718  2.884649  \n",
       "57  3.237639  1.778620  \n",
       "9   0.340251  1.566271  \n",
       "56  1.738537  1.345648  \n",
       "53  2.097227  2.281512  \n",
       "72  1.726377  1.657107  \n",
       "5   1.011037  0.838681  \n",
       "59  1.444203  1.700464  \n",
       "12  1.164456  1.336619  \n",
       "14  1.187903  1.584299  \n",
       "10  0.779662  0.813357  \n",
       "8   1.081650  0.781860  \n",
       "0   1.454740  1.818557  \n",
       "7   3.376249  1.921294  \n",
       "51  0.924929  0.579085  \n",
       "82  1.473427  1.604767  \n",
       "54  1.160218  1.275777  \n",
       "6   1.096359  1.348554  \n",
       "93  0.628833  0.429666  \n",
       "44  0.914205  0.529106  \n",
       "77  0.916391  1.006488  \n",
       "94  1.515145  1.104695  \n",
       "..       ...       ...  \n",
       "39  0.328818  0.430818  \n",
       "89  0.381034  0.363560  \n",
       "17  0.348476  0.440102  \n",
       "47  0.466276  0.330494  \n",
       "23  0.363078  0.377655  \n",
       "76  0.205836  0.281396  \n",
       "80  0.357203  0.518069  \n",
       "46  0.412384  0.277407  \n",
       "68  0.281239  0.261446  \n",
       "61  0.237655  0.206550  \n",
       "42  0.291294  0.559067  \n",
       "37  0.193559  0.355494  \n",
       "63  0.184629  0.217488  \n",
       "69  0.398757  0.332403  \n",
       "19  0.154015  0.323406  \n",
       "38  0.320332  0.433849  \n",
       "81  0.519989  0.255747  \n",
       "84  0.438402  0.369037  \n",
       "79  0.334152  0.410621  \n",
       "41  0.332910  0.179289  \n",
       "71  0.382663  0.348616  \n",
       "26  0.218952  0.205560  \n",
       "43  0.309190  0.204094  \n",
       "85  0.194120  0.193588  \n",
       "28  0.227651  0.134563  \n",
       "70  0.230497  0.149155  \n",
       "90  0.294393  0.212926  \n",
       "48  0.113114  0.170742  \n",
       "65  0.235395  0.279009  \n",
       "36  0.165573  0.198246  \n",
       "\n",
       "[95 rows x 6 columns]"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imp.sort_values(by='score1', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "test_data['pred'] = preds[:, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>paper_id</th>\n",
       "      <th>author_name</th>\n",
       "      <th>author_org</th>\n",
       "      <th>author_id</th>\n",
       "      <th>pred</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>F3Mha4HG</td>\n",
       "      <td>lei_shi</td>\n",
       "      <td>State Key Laboratory of Catalysis</td>\n",
       "      <td>0gL4hj4n</td>\n",
       "      <td>0.041708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>F3Mha4HG</td>\n",
       "      <td>lei_shi</td>\n",
       "      <td>State Key Laboratory of Catalysis</td>\n",
       "      <td>0jknXeWe</td>\n",
       "      <td>0.058157</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>F3Mha4HG</td>\n",
       "      <td>lei_shi</td>\n",
       "      <td>State Key Laboratory of Catalysis</td>\n",
       "      <td>1vE3Drg0</td>\n",
       "      <td>0.457224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>F3Mha4HG</td>\n",
       "      <td>lei_shi</td>\n",
       "      <td>State Key Laboratory of Catalysis</td>\n",
       "      <td>2IVNIGb6</td>\n",
       "      <td>0.456184</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>F3Mha4HG</td>\n",
       "      <td>lei_shi</td>\n",
       "      <td>State Key Laboratory of Catalysis</td>\n",
       "      <td>5QgbLXhl</td>\n",
       "      <td>0.434320</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   paper_id author_name                         author_org author_id      pred\n",
       "0  F3Mha4HG     lei_shi  State Key Laboratory of Catalysis  0gL4hj4n  0.041708\n",
       "1  F3Mha4HG     lei_shi  State Key Laboratory of Catalysis  0jknXeWe  0.058157\n",
       "2  F3Mha4HG     lei_shi  State Key Laboratory of Catalysis  1vE3Drg0  0.457224\n",
       "3  F3Mha4HG     lei_shi  State Key Laboratory of Catalysis  2IVNIGb6  0.456184\n",
       "4  F3Mha4HG     lei_shi  State Key Laboratory of Catalysis  5QgbLXhl  0.434320"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_dict = gen_dict(test_data, 'pred')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1291"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(result_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import time\n",
    "localtime = time.localtime(time.time())\n",
    "save_path = './out/result_%02d%02d%02d%02d.json' % (localtime[1], localtime[2], localtime[3], localtime[4])\n",
    "with open(save_path, 'w') as file:\n",
    "    file.write(json.dumps(result_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}