6.gen_feat_v1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import gc\n",
    "import time\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "feat_dir = './feat/'\n",
    "if not os.path.exists(feat_dir):\n",
    "    os.mkdir(feat_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = pd.read_pickle('./pkl/train_data.pkl')\n",
    "test_data = pd.read_pickle('./pkl/valid_data.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "pub_info = pd.read_pickle('./pkl/pub_info.pkl')\n",
    "author_pub_detail = pd.read_pickle('./pkl/author_pub_detail.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>paper_id</th>\n",
       "      <th>author_name</th>\n",
       "      <th>author_org</th>\n",
       "      <th>author_id</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>P9a1gcvg</td>\n",
       "      <td>li_guo</td>\n",
       "      <td>Institute of Pharmacology and Toxicology</td>\n",
       "      <td>8GjtUmBs</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>P9a1gcvg</td>\n",
       "      <td>li_guo</td>\n",
       "      <td>Institute of Pharmacology and Toxicology</td>\n",
       "      <td>EShnTfSe</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>P9a1gcvg</td>\n",
       "      <td>li_guo</td>\n",
       "      <td>Institute of Pharmacology and Toxicology</td>\n",
       "      <td>t1ruuB9N</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   paper_id author_name                                author_org author_id  \\\n",
       "0  P9a1gcvg      li_guo  Institute of Pharmacology and Toxicology  8GjtUmBs   \n",
       "1  P9a1gcvg      li_guo  Institute of Pharmacology and Toxicology  EShnTfSe   \n",
       "2  P9a1gcvg      li_guo  Institute of Pharmacology and Toxicology  t1ruuB9N   \n",
       "\n",
       "   label  \n",
       "0      0  \n",
       "1      1  \n",
       "2      0  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>paper_id</th>\n",
       "      <th>author_name</th>\n",
       "      <th>author_org</th>\n",
       "      <th>author_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>F3Mha4HG</td>\n",
       "      <td>lei_shi</td>\n",
       "      <td>State Key Laboratory of Catalysis</td>\n",
       "      <td>0gL4hj4n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>F3Mha4HG</td>\n",
       "      <td>lei_shi</td>\n",
       "      <td>State Key Laboratory of Catalysis</td>\n",
       "      <td>0jknXeWe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>F3Mha4HG</td>\n",
       "      <td>lei_shi</td>\n",
       "      <td>State Key Laboratory of Catalysis</td>\n",
       "      <td>1vE3Drg0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   paper_id author_name                         author_org author_id\n",
       "0  F3Mha4HG     lei_shi  State Key Laboratory of Catalysis  0gL4hj4n\n",
       "1  F3Mha4HG     lei_shi  State Key Laboratory of Catalysis  0jknXeWe\n",
       "2  F3Mha4HG     lei_shi  State Key Laboratory of Catalysis  1vE3Drg0"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1194996, 5)\n",
      "(452786, 4)\n"
     ]
    }
   ],
   "source": [
    "print(train_data.shape)\n",
    "print(test_data.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.concat([train_data, test_data]).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>author_id</th>\n",
       "      <th>author_name</th>\n",
       "      <th>author_org</th>\n",
       "      <th>label</th>\n",
       "      <th>paper_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8GjtUmBs</td>\n",
       "      <td>li_guo</td>\n",
       "      <td>Institute of Pharmacology and Toxicology</td>\n",
       "      <td>0.0</td>\n",
       "      <td>P9a1gcvg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>EShnTfSe</td>\n",
       "      <td>li_guo</td>\n",
       "      <td>Institute of Pharmacology and Toxicology</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P9a1gcvg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>t1ruuB9N</td>\n",
       "      <td>li_guo</td>\n",
       "      <td>Institute of Pharmacology and Toxicology</td>\n",
       "      <td>0.0</td>\n",
       "      <td>P9a1gcvg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>xLLXKy6I</td>\n",
       "      <td>li_guo</td>\n",
       "      <td>Institute of Pharmacology and Toxicology</td>\n",
       "      <td>0.0</td>\n",
       "      <td>P9a1gcvg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>jTu2AZES</td>\n",
       "      <td>li_guo</td>\n",
       "      <td>Institute of Pharmacology and Toxicology</td>\n",
       "      <td>0.0</td>\n",
       "      <td>P9a1gcvg</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  author_id author_name                                author_org  label  \\\n",
       "0  8GjtUmBs      li_guo  Institute of Pharmacology and Toxicology    0.0   \n",
       "1  EShnTfSe      li_guo  Institute of Pharmacology and Toxicology    1.0   \n",
       "2  t1ruuB9N      li_guo  Institute of Pharmacology and Toxicology    0.0   \n",
       "3  xLLXKy6I      li_guo  Institute of Pharmacology and Toxicology    0.0   \n",
       "4  jTu2AZES      li_guo  Institute of Pharmacology and Toxicology    0.0   \n",
       "\n",
       "   paper_id  \n",
       "0  P9a1gcvg  \n",
       "1  P9a1gcvg  \n",
       "2  P9a1gcvg  \n",
       "3  P9a1gcvg  \n",
       "4  P9a1gcvg  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>abstract_a</th>\n",
       "      <th>authors_a</th>\n",
       "      <th>keywords_a</th>\n",
       "      <th>paper_id</th>\n",
       "      <th>title_a</th>\n",
       "      <th>venue_a</th>\n",
       "      <th>year_a</th>\n",
       "      <th>orgs_a</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Extremely long states of bound solitons consis...</td>\n",
       "      <td>[Ya Liu, Xin Zhao, Jiansheng Liu, Guoqing Hu]</td>\n",
       "      <td>[carbon nanotubes, fibre lasers, laser cavity ...</td>\n",
       "      <td>0009qJgC</td>\n",
       "      <td>Generating ultra-long bound soliton sequences ...</td>\n",
       "      <td>Lasers and Electro-Optics</td>\n",
       "      <td>2014</td>\n",
       "      <td>[Sch. of Electron. &amp; Inf. Eng., Beihang Univ.,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td></td>\n",
       "      <td>[Hao He, Jun Yang]</td>\n",
       "      <td>[different part, data graph, variety ofstructu...</td>\n",
       "      <td>000BMUBY</td>\n",
       "      <td>Multiresolution Indexing of XML for Frequent Q...</td>\n",
       "      <td>icde</td>\n",
       "      <td>2004</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The Yellow River (YR) supplies a large amount ...</td>\n",
       "      <td>[Weifeng Yang, Min Chen, Guangxue Li, Jianping...</td>\n",
       "      <td>[13C, 15N, East China Sea, Trace element, Yell...</td>\n",
       "      <td>000Fmw78</td>\n",
       "      <td>Relocation of the Yellow River as revealed by ...</td>\n",
       "      <td>Marine Pollution Bulletin</td>\n",
       "      <td>2009</td>\n",
       "      <td>[State Key Laboratory of Marine Environmental ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Our findings suggest that Th17 cells may be in...</td>\n",
       "      <td>[Xiaowei Wang, Xiaojun Chen, Heng Tang, Jifeng...</td>\n",
       "      <td>[Mycoplasma Pneumoniae, Th17 cells, children, ...</td>\n",
       "      <td>000HsdYy</td>\n",
       "      <td>Increased Frequency of Th17 Cells in Children ...</td>\n",
       "      <td>Journal of clinical laboratory analysis</td>\n",
       "      <td>2016</td>\n",
       "      <td>[Department of Blood Transfusion, Nanjing Chil...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td></td>\n",
       "      <td>[jun chen, xianran xing, andrew d watson, wei ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>000NKgQT</td>\n",
       "      <td>Rapid Synthesis of Multiferroic BiFeO 3 Single...</td>\n",
       "      <td>Chemistry of Materials</td>\n",
       "      <td>2007</td>\n",
       "      <td>[, , , , , , , , ]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          abstract_a  \\\n",
       "0  Extremely long states of bound solitons consis...   \n",
       "1                                                      \n",
       "2  The Yellow River (YR) supplies a large amount ...   \n",
       "3  Our findings suggest that Th17 cells may be in...   \n",
       "4                                                      \n",
       "\n",
       "                                           authors_a  \\\n",
       "0      [Ya Liu, Xin Zhao, Jiansheng Liu, Guoqing Hu]   \n",
       "1                                 [Hao He, Jun Yang]   \n",
       "2  [Weifeng Yang, Min Chen, Guangxue Li, Jianping...   \n",
       "3  [Xiaowei Wang, Xiaojun Chen, Heng Tang, Jifeng...   \n",
       "4  [jun chen, xianran xing, andrew d watson, wei ...   \n",
       "\n",
       "                                          keywords_a  paper_id  \\\n",
       "0  [carbon nanotubes, fibre lasers, laser cavity ...  0009qJgC   \n",
       "1  [different part, data graph, variety ofstructu...  000BMUBY   \n",
       "2  [13C, 15N, East China Sea, Trace element, Yell...  000Fmw78   \n",
       "3  [Mycoplasma Pneumoniae, Th17 cells, children, ...  000HsdYy   \n",
       "4                                                 []  000NKgQT   \n",
       "\n",
       "                                             title_a  \\\n",
       "0  Generating ultra-long bound soliton sequences ...   \n",
       "1  Multiresolution Indexing of XML for Frequent Q...   \n",
       "2  Relocation of the Yellow River as revealed by ...   \n",
       "3  Increased Frequency of Th17 Cells in Children ...   \n",
       "4  Rapid Synthesis of Multiferroic BiFeO 3 Single...   \n",
       "\n",
       "                                   venue_a  year_a  \\\n",
       "0                Lasers and Electro-Optics    2014   \n",
       "1                                     icde    2004   \n",
       "2                Marine Pollution Bulletin    2009   \n",
       "3  Journal of clinical laboratory analysis    2016   \n",
       "4                   Chemistry of Materials    2007   \n",
       "\n",
       "                                              orgs_a  \n",
       "0  [Sch. of Electron. & Inf. Eng., Beihang Univ.,...  \n",
       "1                                                 []  \n",
       "2  [State Key Laboratory of Marine Environmental ...  \n",
       "3  [Department of Blood Transfusion, Nanjing Chil...  \n",
       "4                                 [, , , , , , , , ]  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pub_info.columns = ['abstract_a', 'authors_a', 'keywords_a', 'paper_id', 'title_a', 'venue_a', 'year_a', 'orgs_a']\n",
    "pub_info.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>author_id</th>\n",
       "      <th>paper_ids</th>\n",
       "      <th>paper_ids_len</th>\n",
       "      <th>abstract_b</th>\n",
       "      <th>keywords_b</th>\n",
       "      <th>title_b</th>\n",
       "      <th>venue_b</th>\n",
       "      <th>year_b</th>\n",
       "      <th>authors_b</th>\n",
       "      <th>orgs_b</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>004mBKh6</td>\n",
       "      <td>[TY0MYj83, lRl7m2tU, S6rklGvj, dVAKsHla, 2aTAK...</td>\n",
       "      <td>9</td>\n",
       "      <td>[Robot-assisted single port access surgery (SP...</td>\n",
       "      <td>[[Manipulators, Surgery, Grippers, Kinematics,...</td>\n",
       "      <td>[A Robotic System With Multichannel Flexible P...</td>\n",
       "      <td>[IEEE Transactions on Industrial Informatics, ...</td>\n",
       "      <td>[2019, 2018, 2018, 2019, 2019, 2019, 2018, 201...</td>\n",
       "      <td>[[Changsheng Li, Xiaoyi Gu, Xiao Xiao, Chwee M...</td>\n",
       "      <td>[[Department of Biomedical Engineering, Nation...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>008b5hIc</td>\n",
       "      <td>[nPuf0kYs]</td>\n",
       "      <td>1</td>\n",
       "      <td>[With the development of nuclear power industr...</td>\n",
       "      <td>[[Ecological security, Groundwater, Radioactiv...</td>\n",
       "      <td>[A review on the study of groundwater and its ...</td>\n",
       "      <td>[Earth Science Frontiers]</td>\n",
       "      <td>[2014]</td>\n",
       "      <td>[[Zhanxue Sun, Yuanyuan Liu, Wenjie Ma, Bai Gao]]</td>\n",
       "      <td>[[State Key Laboratory Breeding Base of Nuclea...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>00BAgHFr</td>\n",
       "      <td>[TuEkjf5i, kn6wo3ot, 0f2awX7U]</td>\n",
       "      <td>3</td>\n",
       "      <td>[An elevations selection approach of area targ...</td>\n",
       "      <td>[[EOS application, GA, Optimization], [Feature...</td>\n",
       "      <td>[Elevations selection approach of area target ...</td>\n",
       "      <td>[Geomatics and Information Science of Wuhan Un...</td>\n",
       "      <td>[2006, 2006, 2007]</td>\n",
       "      <td>[[Jianghan Zhu, Xi Li, Chilong Mao, Lining Zha...</td>\n",
       "      <td>[[, , , ], [, , ], [School of Information Syst...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00M93JDI</td>\n",
       "      <td>[7IiMiMhZ, rbdaKJow, nbE0DmMh, PJeWna4W, hBHTP...</td>\n",
       "      <td>309</td>\n",
       "      <td>[ ,  ,  ,  ,  ,  ,  ,  ,  ,  ,  ,  ,  ,  ,  , ...</td>\n",
       "      <td>[[nanoparticles, luminescence], [macropore arr...</td>\n",
       "      <td>[Blue Luminescence of ZnO Nanoparticles Based ...</td>\n",
       "      <td>[advanced functional materials, journal of col...</td>\n",
       "      <td>[2010, 2007, 2007, 1999, 2009, 2003, 2017, 200...</td>\n",
       "      <td>[[Haibo Zeng, Guotao Duan, Yue Li, Shikuan Yan...</td>\n",
       "      <td>[[], [], [], [chinese academy of sciences, chi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>00ShLPmg</td>\n",
       "      <td>[inFGp3mc, LV5duZL5, jGsCF9Uy, Nii27VXk, uyJus...</td>\n",
       "      <td>18</td>\n",
       "      <td>[ ,  ,  ,  ,  ,  ,  ,  ,  ,  ,  ,  ,  ,  ,  , ...</td>\n",
       "      <td>[[Protein kinase C, Idiopathic thrombocytopeni...</td>\n",
       "      <td>[Determination of protein kinase C activity of...</td>\n",
       "      <td>[laboratory medicine, journal of clinical hema...</td>\n",
       "      <td>[2006, 2007, 2012, 2011, 2006, 2006, 2009, 200...</td>\n",
       "      <td>[[LIU Fang, WU Changlin, XIAO Hong, CHEN Qun],...</td>\n",
       "      <td>[[.Department of Microbiology and Immunology,G...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  author_id                                          paper_ids  paper_ids_len  \\\n",
       "0  004mBKh6  [TY0MYj83, lRl7m2tU, S6rklGvj, dVAKsHla, 2aTAK...              9   \n",
       "1  008b5hIc                                         [nPuf0kYs]              1   \n",
       "2  00BAgHFr                     [TuEkjf5i, kn6wo3ot, 0f2awX7U]              3   \n",
       "3  00M93JDI  [7IiMiMhZ, rbdaKJow, nbE0DmMh, PJeWna4W, hBHTP...            309   \n",
       "4  00ShLPmg  [inFGp3mc, LV5duZL5, jGsCF9Uy, Nii27VXk, uyJus...             18   \n",
       "\n",
       "                                          abstract_b  \\\n",
       "0  [Robot-assisted single port access surgery (SP...   \n",
       "1  [With the development of nuclear power industr...   \n",
       "2  [An elevations selection approach of area targ...   \n",
       "3  [ ,  ,  ,  ,  ,  ,  ,  ,  ,  ,  ,  ,  ,  ,  , ...   \n",
       "4  [ ,  ,  ,  ,  ,  ,  ,  ,  ,  ,  ,  ,  ,  ,  , ...   \n",
       "\n",
       "                                          keywords_b  \\\n",
       "0  [[Manipulators, Surgery, Grippers, Kinematics,...   \n",
       "1  [[Ecological security, Groundwater, Radioactiv...   \n",
       "2  [[EOS application, GA, Optimization], [Feature...   \n",
       "3  [[nanoparticles, luminescence], [macropore arr...   \n",
       "4  [[Protein kinase C, Idiopathic thrombocytopeni...   \n",
       "\n",
       "                                             title_b  \\\n",
       "0  [A Robotic System With Multichannel Flexible P...   \n",
       "1  [A review on the study of groundwater and its ...   \n",
       "2  [Elevations selection approach of area target ...   \n",
       "3  [Blue Luminescence of ZnO Nanoparticles Based ...   \n",
       "4  [Determination of protein kinase C activity of...   \n",
       "\n",
       "                                             venue_b  \\\n",
       "0  [IEEE Transactions on Industrial Informatics, ...   \n",
       "1                          [Earth Science Frontiers]   \n",
       "2  [Geomatics and Information Science of Wuhan Un...   \n",
       "3  [advanced functional materials, journal of col...   \n",
       "4  [laboratory medicine, journal of clinical hema...   \n",
       "\n",
       "                                              year_b  \\\n",
       "0  [2019, 2018, 2018, 2019, 2019, 2019, 2018, 201...   \n",
       "1                                             [2014]   \n",
       "2                                 [2006, 2006, 2007]   \n",
       "3  [2010, 2007, 2007, 1999, 2009, 2003, 2017, 200...   \n",
       "4  [2006, 2007, 2012, 2011, 2006, 2006, 2009, 200...   \n",
       "\n",
       "                                           authors_b  \\\n",
       "0  [[Changsheng Li, Xiaoyi Gu, Xiao Xiao, Chwee M...   \n",
       "1  [[Zhanxue Sun, Yuanyuan Liu, Wenjie Ma, Bai Gao]]   \n",
       "2  [[Jianghan Zhu, Xi Li, Chilong Mao, Lining Zha...   \n",
       "3  [[Haibo Zeng, Guotao Duan, Yue Li, Shikuan Yan...   \n",
       "4  [[LIU Fang, WU Changlin, XIAO Hong, CHEN Qun],...   \n",
       "\n",
       "                                              orgs_b  \n",
       "0  [[Department of Biomedical Engineering, Nation...  \n",
       "1  [[State Key Laboratory Breeding Base of Nuclea...  \n",
       "2  [[, , , ], [, , ], [School of Information Syst...  \n",
       "3  [[], [], [], [chinese academy of sciences, chi...  \n",
       "4  [[.Department of Microbiology and Immunology,G...  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "author_pub_detail.columns = ['author_id', 'paper_ids', 'paper_ids_len', 'abstract_b', 'keywords_b', 'title_b', 'venue_b', 'year_b', 'authors_b', 'orgs_b']\n",
    "author_pub_detail.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = data.merge(pub_info, 'left', 'paper_id').merge(author_pub_detail, 'left', 'author_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1647782, 21)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>author_id</th>\n",
       "      <th>author_name</th>\n",
       "      <th>author_org</th>\n",
       "      <th>label</th>\n",
       "      <th>paper_id</th>\n",
       "      <th>abstract_a</th>\n",
       "      <th>authors_a</th>\n",
       "      <th>keywords_a</th>\n",
       "      <th>title_a</th>\n",
       "      <th>venue_a</th>\n",
       "      <th>...</th>\n",
       "      <th>orgs_a</th>\n",
       "      <th>paper_ids</th>\n",
       "      <th>paper_ids_len</th>\n",
       "      <th>abstract_b</th>\n",
       "      <th>keywords_b</th>\n",
       "      <th>title_b</th>\n",
       "      <th>venue_b</th>\n",
       "      <th>year_b</th>\n",
       "      <th>authors_b</th>\n",
       "      <th>orgs_b</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8GjtUmBs</td>\n",
       "      <td>li_guo</td>\n",
       "      <td>Institute of Pharmacology and Toxicology</td>\n",
       "      <td>0.0</td>\n",
       "      <td>P9a1gcvg</td>\n",
       "      <td>Objective: To establish a simultaneous determi...</td>\n",
       "      <td>[Fenghe Qiu, Li Liu, Li Guo]</td>\n",
       "      <td>[central nervous drugs, GC-FID, GC-MS, solid-p...</td>\n",
       "      <td>Rapid determination of central nervous drugs i...</td>\n",
       "      <td>Chinese Pharmaceutical Journal</td>\n",
       "      <td>...</td>\n",
       "      <td>[Institute of Pharmacology and Toxicology, Ins...</td>\n",
       "      <td>[J5KPOi3Y]</td>\n",
       "      <td>1</td>\n",
       "      <td>[ ]</td>\n",
       "      <td>[[Beijing, China, Drop trend, Soil Hg]]</td>\n",
       "      <td>[Mercury drop trend in urban soils in Beijing,...</td>\n",
       "      <td>[Journal of Geochemical Exploration]</td>\n",
       "      <td>[2013]</td>\n",
       "      <td>[[Hangxin Cheng, Chuandong Zhao, Fei Liu, Ke Y...</td>\n",
       "      <td>[[Key Laboratory of Geochemical Cycling of Car...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>EShnTfSe</td>\n",
       "      <td>li_guo</td>\n",
       "      <td>Institute of Pharmacology and Toxicology</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P9a1gcvg</td>\n",
       "      <td>Objective: To establish a simultaneous determi...</td>\n",
       "      <td>[Fenghe Qiu, Li Liu, Li Guo]</td>\n",
       "      <td>[central nervous drugs, GC-FID, GC-MS, solid-p...</td>\n",
       "      <td>Rapid determination of central nervous drugs i...</td>\n",
       "      <td>Chinese Pharmaceutical Journal</td>\n",
       "      <td>...</td>\n",
       "      <td>[Institute of Pharmacology and Toxicology, Ins...</td>\n",
       "      <td>[P9a1gcvg]</td>\n",
       "      <td>1</td>\n",
       "      <td>[Objective: To establish a simultaneous determ...</td>\n",
       "      <td>[[central nervous drugs, GC-FID, GC-MS, solid-...</td>\n",
       "      <td>[Rapid determination of central nervous drugs ...</td>\n",
       "      <td>[Chinese Pharmaceutical Journal]</td>\n",
       "      <td>[1996]</td>\n",
       "      <td>[[Fenghe Qiu, Li Liu, Li Guo]]</td>\n",
       "      <td>[[Institute of Pharmacology and Toxicology, In...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>t1ruuB9N</td>\n",
       "      <td>li_guo</td>\n",
       "      <td>Institute of Pharmacology and Toxicology</td>\n",
       "      <td>0.0</td>\n",
       "      <td>P9a1gcvg</td>\n",
       "      <td>Objective: To establish a simultaneous determi...</td>\n",
       "      <td>[Fenghe Qiu, Li Liu, Li Guo]</td>\n",
       "      <td>[central nervous drugs, GC-FID, GC-MS, solid-p...</td>\n",
       "      <td>Rapid determination of central nervous drugs i...</td>\n",
       "      <td>Chinese Pharmaceutical Journal</td>\n",
       "      <td>...</td>\n",
       "      <td>[Institute of Pharmacology and Toxicology, Ins...</td>\n",
       "      <td>[T35c9ZAd]</td>\n",
       "      <td>1</td>\n",
       "      <td>[Beijing Plain is composed of a suite of consi...</td>\n",
       "      <td>[[Beijing, Cenozoic, Stratigraphie classificat...</td>\n",
       "      <td>[Core characteristics and preliminary stratigr...</td>\n",
       "      <td>[Geology in China]</td>\n",
       "      <td>[2011]</td>\n",
       "      <td>[[Ying-Bo Luan, Xiang-Min Cai, Gao-Xuan Guo, Y...</td>\n",
       "      <td>[[Bureau of Geological Mineral Resources Explo...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  author_id author_name                                author_org  label  \\\n",
       "0  8GjtUmBs      li_guo  Institute of Pharmacology and Toxicology    0.0   \n",
       "1  EShnTfSe      li_guo  Institute of Pharmacology and Toxicology    1.0   \n",
       "2  t1ruuB9N      li_guo  Institute of Pharmacology and Toxicology    0.0   \n",
       "\n",
       "   paper_id                                         abstract_a  \\\n",
       "0  P9a1gcvg  Objective: To establish a simultaneous determi...   \n",
       "1  P9a1gcvg  Objective: To establish a simultaneous determi...   \n",
       "2  P9a1gcvg  Objective: To establish a simultaneous determi...   \n",
       "\n",
       "                      authors_a  \\\n",
       "0  [Fenghe Qiu, Li Liu, Li Guo]   \n",
       "1  [Fenghe Qiu, Li Liu, Li Guo]   \n",
       "2  [Fenghe Qiu, Li Liu, Li Guo]   \n",
       "\n",
       "                                          keywords_a  \\\n",
       "0  [central nervous drugs, GC-FID, GC-MS, solid-p...   \n",
       "1  [central nervous drugs, GC-FID, GC-MS, solid-p...   \n",
       "2  [central nervous drugs, GC-FID, GC-MS, solid-p...   \n",
       "\n",
       "                                             title_a  \\\n",
       "0  Rapid determination of central nervous drugs i...   \n",
       "1  Rapid determination of central nervous drugs i...   \n",
       "2  Rapid determination of central nervous drugs i...   \n",
       "\n",
       "                          venue_a  \\\n",
       "0  Chinese Pharmaceutical Journal   \n",
       "1  Chinese Pharmaceutical Journal   \n",
       "2  Chinese Pharmaceutical Journal   \n",
       "\n",
       "                         ...                          \\\n",
       "0                        ...                           \n",
       "1                        ...                           \n",
       "2                        ...                           \n",
       "\n",
       "                                              orgs_a   paper_ids  \\\n",
       "0  [Institute of Pharmacology and Toxicology, Ins...  [J5KPOi3Y]   \n",
       "1  [Institute of Pharmacology and Toxicology, Ins...  [P9a1gcvg]   \n",
       "2  [Institute of Pharmacology and Toxicology, Ins...  [T35c9ZAd]   \n",
       "\n",
       "  paper_ids_len                                         abstract_b  \\\n",
       "0             1                                                [ ]   \n",
       "1             1  [Objective: To establish a simultaneous determ...   \n",
       "2             1  [Beijing Plain is composed of a suite of consi...   \n",
       "\n",
       "                                          keywords_b  \\\n",
       "0            [[Beijing, China, Drop trend, Soil Hg]]   \n",
       "1  [[central nervous drugs, GC-FID, GC-MS, solid-...   \n",
       "2  [[Beijing, Cenozoic, Stratigraphie classificat...   \n",
       "\n",
       "                                             title_b  \\\n",
       "0  [Mercury drop trend in urban soils in Beijing,...   \n",
       "1  [Rapid determination of central nervous drugs ...   \n",
       "2  [Core characteristics and preliminary stratigr...   \n",
       "\n",
       "                                venue_b  year_b  \\\n",
       "0  [Journal of Geochemical Exploration]  [2013]   \n",
       "1      [Chinese Pharmaceutical Journal]  [1996]   \n",
       "2                    [Geology in China]  [2011]   \n",
       "\n",
       "                                           authors_b  \\\n",
       "0  [[Hangxin Cheng, Chuandong Zhao, Fei Liu, Ke Y...   \n",
       "1                     [[Fenghe Qiu, Li Liu, Li Guo]]   \n",
       "2  [[Ying-Bo Luan, Xiang-Min Cai, Gao-Xuan Guo, Y...   \n",
       "\n",
       "                                              orgs_b  \n",
       "0  [[Key Laboratory of Geochemical Cycling of Car...  \n",
       "1  [[Institute of Pharmacology and Toxicology, In...  \n",
       "2  [[Bureau of Geological Mineral Resources Explo...  \n",
       "\n",
       "[3 rows x 21 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### delete paper_id in pos sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pidx(p, ps):\n",
    "    ans = np.nan\n",
    "    for i, p2 in enumerate(ps):\n",
    "        if p == p2:\n",
    "            ans = i\n",
    "            break\n",
    "    return ans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 16s, sys: 1.61 s, total: 1min 18s\n",
      "Wall time: 1min 18s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "data['idx'] = data.apply(lambda row: pidx(row['paper_id'], row['paper_ids']) if row['label'] == 1 else np.nan, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "23d42b15ceb246d0b3b43639316632f6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1647782), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm_notebook\n",
    "\n",
    "cols = ['abstract_b', 'keywords_b', 'title_b', 'venue_b', 'year_b', 'authors_b', 'orgs_b']\n",
    "for i in tqdm_notebook(range(len(data))):\n",
    "    if pd.isna(data.loc[i, 'idx']):\n",
    "        continue\n",
    "    for c in cols:\n",
    "        v = list(data.loc[i, c])\n",
    "        del v[data.loc[i, 'idx'].astype(int)]\n",
    "        data.set_value(i, c, v)\n",
    "#         data.loc[i, c] = frozenset(v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_pickle('./pkl/data.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['author_id', 'author_name', 'author_org', 'label', 'paper_id',\n",
      "       'abstract_a', 'authors_a', 'keywords_a', 'title_a', 'venue_a', 'year_a',\n",
      "       'orgs_a', 'paper_ids', 'paper_ids_len', 'abstract_b', 'keywords_b',\n",
      "       'title_b', 'venue_b', 'year_b', 'authors_b', 'orgs_b', 'idx'],\n",
      "      dtype='object')\n"
     ]
    }
   ],
   "source": [
    "print(data.columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### year"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_pickle('./pkl/data.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['year_b'] = data['year_b'].apply(lambda x: [0] if len(x) == 0 else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['year_b_min'] = data['year_b'].apply(np.min)\n",
    "data['year_b_max'] = data['year_b'].apply(np.max)\n",
    "data['year_b_mean'] = data['year_b'].apply(np.mean)\n",
    "data['year_b_std'] = data['year_b'].apply(np.std)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['year_b_mm2'] = (data['year_b_min'] + data['year_b_max']) / 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "for c in ['year_b_min', 'year_b_max', 'year_b_mean', 'year_b_mm2']:\n",
    "    data[c + '-year_a'] = data[c] - data['year_a']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['year_inside_range'] = ((data['year_b_min-year_a'] <= 0) & (data['year_b_max-year_a'] >= 0)).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "cols = ['year_a', 'year_b_min', 'year_b_max', 'year_b_mean', 'year_b_std', 'year_b_mm2',\n",
    "       'year_b_min-year_a', 'year_b_max-year_a', 'year_b_mean-year_a',\n",
    "       'year_b_mm2-year_a', 'year_inside_range']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "data[cols].to_pickle('./feat/time_feat_a.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 'authors', 'orgs'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_pickle('./pkl/data.pkl')\n",
    "tmp = data[['author_org', 'authors_a', 'orgs_a', 'authors_b', 'orgs_b']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import multiprocessing as mp\n",
    "\n",
    "# def split_df(df, n):\n",
    "#     chunk_size = int(np.ceil(len(df) / n))\n",
    "#     return [df[i*chunk_size:(i+1)*chunk_size] for i in range(n)]\n",
    "\n",
    "# chunk_list = split_df(tmp, 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def func(a, b):\n",
    "    cnt = 0\n",
    "    for x in b:\n",
    "        for y in x:\n",
    "            if a == y:\n",
    "                cnt += 1\n",
    "    return cnt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# def process(df):\n",
    "#     return df.apply(lambda row: func(row['author_org'], row['orgs_b']), axis=1)\n",
    " \n",
    "# with mp.Pool(8) as pool:\n",
    "#     ret = pool.map(process, chunk_list)\n",
    "#     print(len(ret))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4min 11s, sys: 2.64 s, total: 4min 14s\n",
      "Wall time: 4min 14s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "tmp['author_org_in_orgs_b_times'] = tmp.apply(lambda row: func(row['author_org'], row['orgs_b']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def func(a, b):\n",
    "    b = set([y for x in b for y in x])\n",
    "#     b = set([x for x in b])\n",
    "    a = set(a)\n",
    "    return len(a & b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 7min 2s, sys: 3.65 s, total: 7min 5s\n",
      "Wall time: 7min 6s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "tmp['author_interset_num'] = tmp.apply(lambda row: func(row['authors_a'], row['authors_b']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp['author_interset_num/paper_ids_len'] = (tmp['author_interset_num'] / (data['paper_ids_len'] - (data['label'] == 1).astype(int))).fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1       1049629\n",
       "0        397246\n",
       "2         44245\n",
       "3         40378\n",
       "4         35455\n",
       "5         26070\n",
       "6         17612\n",
       "7         11221\n",
       "8          7733\n",
       "9          4788\n",
       "10         2903\n",
       "11         1800\n",
       "12         1191\n",
       "13          680\n",
       "14          513\n",
       "15          384\n",
       "16          279\n",
       "17          238\n",
       "20          141\n",
       "18          138\n",
       "19          137\n",
       "21           79\n",
       "24           63\n",
       "25           62\n",
       "22           56\n",
       "26           48\n",
       "2409         46\n",
       "99           46\n",
       "23           42\n",
       "52           37\n",
       "         ...   \n",
       "1451          1\n",
       "424           1\n",
       "973           1\n",
       "1413          1\n",
       "439           1\n",
       "2436          1\n",
       "2045          1\n",
       "977           1\n",
       "3026          1\n",
       "979           1\n",
       "981           1\n",
       "432           1\n",
       "446           1\n",
       "1513          1\n",
       "1482          1\n",
       "484           1\n",
       "482           1\n",
       "480           1\n",
       "1491          1\n",
       "1490          1\n",
       "1487          1\n",
       "1480          1\n",
       "447           1\n",
       "1478          1\n",
       "1476          1\n",
       "1475          1\n",
       "451           1\n",
       "449           1\n",
       "448           1\n",
       "1022          1\n",
       "Name: author_interset_num, Length: 1098, dtype: int64"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tmp['author_interset_num'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp[['author_org_in_orgs_b_times', 'author_interset_num', 'author_interset_num/paper_ids_len']].to_pickle('./feat/author_org_cross.pkl')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### venue, keywords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>venue_a</th>\n",
       "      <th>venue_b</th>\n",
       "      <th>keywords_a</th>\n",
       "      <th>keywords_b</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Chinese Pharmaceutical Journal</td>\n",
       "      <td>[Journal of Geochemical Exploration]</td>\n",
       "      <td>[central nervous drugs, GC-FID, GC-MS, solid-p...</td>\n",
       "      <td>[[Beijing, China, Drop trend, Soil Hg]]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Chinese Pharmaceutical Journal</td>\n",
       "      <td>[]</td>\n",
       "      <td>[central nervous drugs, GC-FID, GC-MS, solid-p...</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Chinese Pharmaceutical Journal</td>\n",
       "      <td>[Geology in China]</td>\n",
       "      <td>[central nervous drugs, GC-FID, GC-MS, solid-p...</td>\n",
       "      <td>[[Beijing, Cenozoic, Stratigraphie classificat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Chinese Pharmaceutical Journal</td>\n",
       "      <td>[Jianzhu Cailiao Xuebao/Journal of Building Ma...</td>\n",
       "      <td>[central nervous drugs, GC-FID, GC-MS, solid-p...</td>\n",
       "      <td>[[Carbonated concrete, Experimental study, Rea...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Chinese Pharmaceutical Journal</td>\n",
       "      <td>[Chinese Journal of New Drugs]</td>\n",
       "      <td>[central nervous drugs, GC-FID, GC-MS, solid-p...</td>\n",
       "      <td>[[Intramuscular in children, Recombinant human...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          venue_a  \\\n",
       "0  Chinese Pharmaceutical Journal   \n",
       "1  Chinese Pharmaceutical Journal   \n",
       "2  Chinese Pharmaceutical Journal   \n",
       "3  Chinese Pharmaceutical Journal   \n",
       "4  Chinese Pharmaceutical Journal   \n",
       "\n",
       "                                             venue_b  \\\n",
       "0               [Journal of Geochemical Exploration]   \n",
       "1                                                 []   \n",
       "2                                 [Geology in China]   \n",
       "3  [Jianzhu Cailiao Xuebao/Journal of Building Ma...   \n",
       "4                     [Chinese Journal of New Drugs]   \n",
       "\n",
       "                                          keywords_a  \\\n",
       "0  [central nervous drugs, GC-FID, GC-MS, solid-p...   \n",
       "1  [central nervous drugs, GC-FID, GC-MS, solid-p...   \n",
       "2  [central nervous drugs, GC-FID, GC-MS, solid-p...   \n",
       "3  [central nervous drugs, GC-FID, GC-MS, solid-p...   \n",
       "4  [central nervous drugs, GC-FID, GC-MS, solid-p...   \n",
       "\n",
       "                                          keywords_b  \n",
       "0            [[Beijing, China, Drop trend, Soil Hg]]  \n",
       "1                                                 []  \n",
       "2  [[Beijing, Cenozoic, Stratigraphie classificat...  \n",
       "3  [[Carbonated concrete, Experimental study, Rea...  \n",
       "4  [[Intramuscular in children, Recombinant human...  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# data = pd.read_pickle('./pkl/data.pkl')\n",
    "tmp = data[['venue_a', 'venue_b', 'keywords_a', 'keywords_b']]\n",
    "tmp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "def func(a, b):\n",
    "    cnt = 0\n",
    "    for x in b:\n",
    "        if a.lower() == x.lower():\n",
    "            cnt += 1\n",
    "    return cnt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp['venue_a_in_venue_b_num'] = tmp.apply(lambda row: func(row['venue_a'], row['venue_b']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "def func(a, b):\n",
    "    bl = []\n",
    "    for x in b:\n",
    "        if type(x) == list:\n",
    "            for y in x:\n",
    "                if not pd.isna(y):\n",
    "                    bl.append(y.lower())\n",
    "    b = set(bl)\n",
    "    al = []\n",
    "    if type(a) != float:\n",
    "        for x in a:\n",
    "            al.append(x.lower())\n",
    "    a = set(al)\n",
    "    return len(a & b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 9min 43s, sys: 5.54 s, total: 9min 49s\n",
      "Wall time: 9min 50s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "tmp['keywords_interset_num'] = tmp.apply(lambda row: func(row['keywords_a'], row['keywords_b']), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp['venue_a_in_venue_b_num/paper_ids_len'] = (tmp['venue_a_in_venue_b_num'] / (data['paper_ids_len'] - (data['label'] == 1).astype(int))).fillna(0)\n",
    "tmp['keywords_interset_num/paper_ids_len'] = (tmp['keywords_interset_num'] / (data['paper_ids_len'] - (data['label'] == 1).astype(int))).fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>venue_a</th>\n",
       "      <th>venue_b</th>\n",
       "      <th>keywords_a</th>\n",
       "      <th>keywords_b</th>\n",
       "      <th>venue_a_in_venue_b_num</th>\n",
       "      <th>keywords_interset_num</th>\n",
       "      <th>venue_a_in_venue_b_num/paper_ids_len</th>\n",
       "      <th>keywords_interset_num/paper_ids_len</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Chinese Pharmaceutical Journal</td>\n",
       "      <td>[Journal of Geochemical Exploration]</td>\n",
       "      <td>[central nervous drugs, GC-FID, GC-MS, solid-p...</td>\n",
       "      <td>[[Beijing, China, Drop trend, Soil Hg]]</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Chinese Pharmaceutical Journal</td>\n",
       "      <td>[]</td>\n",
       "      <td>[central nervous drugs, GC-FID, GC-MS, solid-p...</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Chinese Pharmaceutical Journal</td>\n",
       "      <td>[Geology in China]</td>\n",
       "      <td>[central nervous drugs, GC-FID, GC-MS, solid-p...</td>\n",
       "      <td>[[Beijing, Cenozoic, Stratigraphie classificat...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          venue_a                               venue_b  \\\n",
       "0  Chinese Pharmaceutical Journal  [Journal of Geochemical Exploration]   \n",
       "1  Chinese Pharmaceutical Journal                                    []   \n",
       "2  Chinese Pharmaceutical Journal                    [Geology in China]   \n",
       "\n",
       "                                          keywords_a  \\\n",
       "0  [central nervous drugs, GC-FID, GC-MS, solid-p...   \n",
       "1  [central nervous drugs, GC-FID, GC-MS, solid-p...   \n",
       "2  [central nervous drugs, GC-FID, GC-MS, solid-p...   \n",
       "\n",
       "                                          keywords_b  venue_a_in_venue_b_num  \\\n",
       "0            [[Beijing, China, Drop trend, Soil Hg]]                       0   \n",
       "1                                                 []                       0   \n",
       "2  [[Beijing, Cenozoic, Stratigraphie classificat...                       0   \n",
       "\n",
       "   keywords_interset_num  venue_a_in_venue_b_num/paper_ids_len  \\\n",
       "0                      0                                   0.0   \n",
       "1                      0                                   0.0   \n",
       "2                      0                                   0.0   \n",
       "\n",
       "   keywords_interset_num/paper_ids_len  \n",
       "0                                  0.0  \n",
       "1                                  0.0  \n",
       "2                                  0.0  "
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tmp.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp[['venue_a_in_venue_b_num', 'keywords_interset_num', 'venue_a_in_venue_b_num/paper_ids_len', 'keywords_interset_num/paper_ids_len']].to_pickle('./feat/keyword_venue_cross.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>author_id</th>\n",
       "      <th>author_name</th>\n",
       "      <th>author_org</th>\n",
       "      <th>label</th>\n",
       "      <th>paper_id</th>\n",
       "      <th>abstract_a</th>\n",
       "      <th>authors_a</th>\n",
       "      <th>keywords_a</th>\n",
       "      <th>title_a</th>\n",
       "      <th>venue_a</th>\n",
       "      <th>...</th>\n",
       "      <th>paper_ids</th>\n",
       "      <th>paper_ids_len</th>\n",
       "      <th>abstract_b</th>\n",
       "      <th>keywords_b</th>\n",
       "      <th>title_b</th>\n",
       "      <th>venue_b</th>\n",
       "      <th>year_b</th>\n",
       "      <th>authors_b</th>\n",
       "      <th>orgs_b</th>\n",
       "      <th>idx</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8GjtUmBs</td>\n",
       "      <td>li_guo</td>\n",
       "      <td>Institute of Pharmacology and Toxicology</td>\n",
       "      <td>0.0</td>\n",
       "      <td>P9a1gcvg</td>\n",
       "      <td>Objective: To establish a simultaneous determi...</td>\n",
       "      <td>[Fenghe Qiu, Li Liu, Li Guo]</td>\n",
       "      <td>[central nervous drugs, GC-FID, GC-MS, solid-p...</td>\n",
       "      <td>Rapid determination of central nervous drugs i...</td>\n",
       "      <td>Chinese Pharmaceutical Journal</td>\n",
       "      <td>...</td>\n",
       "      <td>[J5KPOi3Y]</td>\n",
       "      <td>1</td>\n",
       "      <td>[ ]</td>\n",
       "      <td>[[Beijing, China, Drop trend, Soil Hg]]</td>\n",
       "      <td>[Mercury drop trend in urban soils in Beijing,...</td>\n",
       "      <td>[Journal of Geochemical Exploration]</td>\n",
       "      <td>[2013]</td>\n",
       "      <td>[[Hangxin Cheng, Chuandong Zhao, Fei Liu, Ke Y...</td>\n",
       "      <td>[[Key Laboratory of Geochemical Cycling of Car...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>EShnTfSe</td>\n",
       "      <td>li_guo</td>\n",
       "      <td>Institute of Pharmacology and Toxicology</td>\n",
       "      <td>1.0</td>\n",
       "      <td>P9a1gcvg</td>\n",
       "      <td>Objective: To establish a simultaneous determi...</td>\n",
       "      <td>[Fenghe Qiu, Li Liu, Li Guo]</td>\n",
       "      <td>[central nervous drugs, GC-FID, GC-MS, solid-p...</td>\n",
       "      <td>Rapid determination of central nervous drugs i...</td>\n",
       "      <td>Chinese Pharmaceutical Journal</td>\n",
       "      <td>...</td>\n",
       "      <td>[P9a1gcvg]</td>\n",
       "      <td>1</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>t1ruuB9N</td>\n",
       "      <td>li_guo</td>\n",
       "      <td>Institute of Pharmacology and Toxicology</td>\n",
       "      <td>0.0</td>\n",
       "      <td>P9a1gcvg</td>\n",
       "      <td>Objective: To establish a simultaneous determi...</td>\n",
       "      <td>[Fenghe Qiu, Li Liu, Li Guo]</td>\n",
       "      <td>[central nervous drugs, GC-FID, GC-MS, solid-p...</td>\n",
       "      <td>Rapid determination of central nervous drugs i...</td>\n",
       "      <td>Chinese Pharmaceutical Journal</td>\n",
       "      <td>...</td>\n",
       "      <td>[T35c9ZAd]</td>\n",
       "      <td>1</td>\n",
       "      <td>[Beijing Plain is composed of a suite of consi...</td>\n",
       "      <td>[[Beijing, Cenozoic, Stratigraphie classificat...</td>\n",
       "      <td>[Core characteristics and preliminary stratigr...</td>\n",
       "      <td>[Geology in China]</td>\n",
       "      <td>[2011]</td>\n",
       "      <td>[[Ying-Bo Luan, Xiang-Min Cai, Gao-Xuan Guo, Y...</td>\n",
       "      <td>[[Bureau of Geological Mineral Resources Explo...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>xLLXKy6I</td>\n",
       "      <td>li_guo</td>\n",
       "      <td>Institute of Pharmacology and Toxicology</td>\n",
       "      <td>0.0</td>\n",
       "      <td>P9a1gcvg</td>\n",
       "      <td>Objective: To establish a simultaneous determi...</td>\n",
       "      <td>[Fenghe Qiu, Li Liu, Li Guo]</td>\n",
       "      <td>[central nervous drugs, GC-FID, GC-MS, solid-p...</td>\n",
       "      <td>Rapid determination of central nervous drugs i...</td>\n",
       "      <td>Chinese Pharmaceutical Journal</td>\n",
       "      <td>...</td>\n",
       "      <td>[OQ7F7UAh]</td>\n",
       "      <td>1</td>\n",
       "      <td>[According to the experiment of realkalization...</td>\n",
       "      <td>[[Carbonated concrete, Experimental study, Rea...</td>\n",
       "      <td>[Influencing factor of realkalization techniqu...</td>\n",
       "      <td>[Jianzhu Cailiao Xuebao/Journal of Building Ma...</td>\n",
       "      <td>[2008]</td>\n",
       "      <td>[[Wen-Jun Qu, Yan Xiong, Li Guo]]</td>\n",
       "      <td>[[School of Civil Engineering, School of Civil...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>jTu2AZES</td>\n",
       "      <td>li_guo</td>\n",
       "      <td>Institute of Pharmacology and Toxicology</td>\n",
       "      <td>0.0</td>\n",
       "      <td>P9a1gcvg</td>\n",
       "      <td>Objective: To establish a simultaneous determi...</td>\n",
       "      <td>[Fenghe Qiu, Li Liu, Li Guo]</td>\n",
       "      <td>[central nervous drugs, GC-FID, GC-MS, solid-p...</td>\n",
       "      <td>Rapid determination of central nervous drugs i...</td>\n",
       "      <td>Chinese Pharmaceutical Journal</td>\n",
       "      <td>...</td>\n",
       "      <td>[NRR6kM3z]</td>\n",
       "      <td>1</td>\n",
       "      <td>[Objective: To evaluate the tolerance and safe...</td>\n",
       "      <td>[[Intramuscular in children, Recombinant human...</td>\n",
       "      <td>[Tolerance and safety of recombinant human int...</td>\n",
       "      <td>[Chinese Journal of New Drugs]</td>\n",
       "      <td>[2011]</td>\n",
       "      <td>[[Wei Sun, Yi Qiao, Li Guo, Yu-Hong Cao, Lin Y...</td>\n",
       "      <td>[[Department of Pharmacy, Department of Pharma...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 22 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  author_id author_name                                author_org  label  \\\n",
       "0  8GjtUmBs      li_guo  Institute of Pharmacology and Toxicology    0.0   \n",
       "1  EShnTfSe      li_guo  Institute of Pharmacology and Toxicology    1.0   \n",
       "2  t1ruuB9N      li_guo  Institute of Pharmacology and Toxicology    0.0   \n",
       "3  xLLXKy6I      li_guo  Institute of Pharmacology and Toxicology    0.0   \n",
       "4  jTu2AZES      li_guo  Institute of Pharmacology and Toxicology    0.0   \n",
       "\n",
       "   paper_id                                         abstract_a  \\\n",
       "0  P9a1gcvg  Objective: To establish a simultaneous determi...   \n",
       "1  P9a1gcvg  Objective: To establish a simultaneous determi...   \n",
       "2  P9a1gcvg  Objective: To establish a simultaneous determi...   \n",
       "3  P9a1gcvg  Objective: To establish a simultaneous determi...   \n",
       "4  P9a1gcvg  Objective: To establish a simultaneous determi...   \n",
       "\n",
       "                      authors_a  \\\n",
       "0  [Fenghe Qiu, Li Liu, Li Guo]   \n",
       "1  [Fenghe Qiu, Li Liu, Li Guo]   \n",
       "2  [Fenghe Qiu, Li Liu, Li Guo]   \n",
       "3  [Fenghe Qiu, Li Liu, Li Guo]   \n",
       "4  [Fenghe Qiu, Li Liu, Li Guo]   \n",
       "\n",
       "                                          keywords_a  \\\n",
       "0  [central nervous drugs, GC-FID, GC-MS, solid-p...   \n",
       "1  [central nervous drugs, GC-FID, GC-MS, solid-p...   \n",
       "2  [central nervous drugs, GC-FID, GC-MS, solid-p...   \n",
       "3  [central nervous drugs, GC-FID, GC-MS, solid-p...   \n",
       "4  [central nervous drugs, GC-FID, GC-MS, solid-p...   \n",
       "\n",
       "                                             title_a  \\\n",
       "0  Rapid determination of central nervous drugs i...   \n",
       "1  Rapid determination of central nervous drugs i...   \n",
       "2  Rapid determination of central nervous drugs i...   \n",
       "3  Rapid determination of central nervous drugs i...   \n",
       "4  Rapid determination of central nervous drugs i...   \n",
       "\n",
       "                          venue_a ...    paper_ids paper_ids_len  \\\n",
       "0  Chinese Pharmaceutical Journal ...   [J5KPOi3Y]             1   \n",
       "1  Chinese Pharmaceutical Journal ...   [P9a1gcvg]             1   \n",
       "2  Chinese Pharmaceutical Journal ...   [T35c9ZAd]             1   \n",
       "3  Chinese Pharmaceutical Journal ...   [OQ7F7UAh]             1   \n",
       "4  Chinese Pharmaceutical Journal ...   [NRR6kM3z]             1   \n",
       "\n",
       "                                          abstract_b  \\\n",
       "0                                                [ ]   \n",
       "1                                                 []   \n",
       "2  [Beijing Plain is composed of a suite of consi...   \n",
       "3  [According to the experiment of realkalization...   \n",
       "4  [Objective: To evaluate the tolerance and safe...   \n",
       "\n",
       "                                          keywords_b  \\\n",
       "0            [[Beijing, China, Drop trend, Soil Hg]]   \n",
       "1                                                 []   \n",
       "2  [[Beijing, Cenozoic, Stratigraphie classificat...   \n",
       "3  [[Carbonated concrete, Experimental study, Rea...   \n",
       "4  [[Intramuscular in children, Recombinant human...   \n",
       "\n",
       "                                             title_b  \\\n",
       "0  [Mercury drop trend in urban soils in Beijing,...   \n",
       "1                                                 []   \n",
       "2  [Core characteristics and preliminary stratigr...   \n",
       "3  [Influencing factor of realkalization techniqu...   \n",
       "4  [Tolerance and safety of recombinant human int...   \n",
       "\n",
       "                                             venue_b  year_b  \\\n",
       "0               [Journal of Geochemical Exploration]  [2013]   \n",
       "1                                                 []      []   \n",
       "2                                 [Geology in China]  [2011]   \n",
       "3  [Jianzhu Cailiao Xuebao/Journal of Building Ma...  [2008]   \n",
       "4                     [Chinese Journal of New Drugs]  [2011]   \n",
       "\n",
       "                                           authors_b  \\\n",
       "0  [[Hangxin Cheng, Chuandong Zhao, Fei Liu, Ke Y...   \n",
       "1                                                 []   \n",
       "2  [[Ying-Bo Luan, Xiang-Min Cai, Gao-Xuan Guo, Y...   \n",
       "3                  [[Wen-Jun Qu, Yan Xiong, Li Guo]]   \n",
       "4  [[Wei Sun, Yi Qiao, Li Guo, Yu-Hong Cao, Lin Y...   \n",
       "\n",
       "                                              orgs_b  idx  \n",
       "0  [[Key Laboratory of Geochemical Cycling of Car...  NaN  \n",
       "1                                                 []  0.0  \n",
       "2  [[Bureau of Geological Mineral Resources Explo...  NaN  \n",
       "3  [[School of Civil Engineering, School of Civil...  NaN  \n",
       "4  [[Department of Pharmacy, Department of Pharma...  NaN  \n",
       "\n",
       "[5 rows x 22 columns]"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data = pd.read_pickle('./pkl/data.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data['paper_num'] = data['paper_ids_len'] - data['label']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data[['paper_num']].to_pickle('./feat/paper_num.pkl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}