2_classical_ml_approach.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "-CHBpwUVJ5uR"
   },
   "source": [
    "# Classical Machine Learning Approach\n",
    "\n",
    "In this notebook we will be learning to\n",
    "  1. Create a Naive TF - IDF based Bag of Words representation of text.\n",
    "  2. Use classical ML models to solve text classification.\n",
    "  3. Use a One Vs Rest strategy to solve multi-label text classification.\n",
    "\n",
    "\n",
    "  **HOT TIP** : *Save them as pickle for easy rendering for experiments*\n",
    "\n",
    "  This Notebook uses code from https://github.com/susanli2016/Machine-Learning-with-Python/blob/master/Multi%20label%20text%20classification.ipynb\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 496
    },
    "colab_type": "code",
    "id": "R5d2a5ctKzaT",
    "outputId": "516ed33f-f70d-4ce0-b1ee-adfc82f816e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting contractions\n",
      "  Downloading https://files.pythonhosted.org/packages/85/41/c3dfd5feb91a8d587ed1a59f553f07c05f95ad4e5d00ab78702fbf8fe48a/contractions-0.0.24-py2.py3-none-any.whl\n",
      "Collecting textsearch\n",
      "  Downloading https://files.pythonhosted.org/packages/42/a8/03407021f9555043de5492a2bd7a35c56cc03c2510092b5ec018cae1bbf1/textsearch-0.0.17-py2.py3-none-any.whl\n",
      "Collecting Unidecode\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d0/42/d9edfed04228bacea2d824904cae367ee9efd05e6cce7ceaaedd0b0ad964/Unidecode-1.1.1-py2.py3-none-any.whl (238kB)\n",
      "\u001b[K     |████████████████████████████████| 245kB 4.0MB/s \n",
      "\u001b[?25hCollecting pyahocorasick\n",
      "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f4/9f/f0d8e8850e12829eea2e778f1c90e3c53a9a799b7f412082a5d21cd19ae1/pyahocorasick-1.4.0.tar.gz (312kB)\n",
      "\u001b[K     |████████████████████████████████| 317kB 58.8MB/s \n",
      "\u001b[?25hBuilding wheels for collected packages: pyahocorasick\n",
      "  Building wheel for pyahocorasick (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
      "  Created wheel for pyahocorasick: filename=pyahocorasick-1.4.0-cp36-cp36m-linux_x86_64.whl size=81698 sha256=f163f074e78b5cd47a2b71398341f597c7b77d8aa2b7781783d0cc381dc4a4a6\n",
      "  Stored in directory: /root/.cache/pip/wheels/0a/90/61/87a55f5b459792fbb2b7ba6b31721b06ff5cf6bde541b40994\n",
      "Successfully built pyahocorasick\n",
      "Installing collected packages: Unidecode, pyahocorasick, textsearch, contractions\n",
      "Successfully installed Unidecode-1.1.1 contractions-0.0.24 pyahocorasick-1.4.0 textsearch-0.0.17\n",
      "Requirement already satisfied: textsearch in /usr/local/lib/python3.6/dist-packages (0.0.17)\n",
      "Requirement already satisfied: Unidecode in /usr/local/lib/python3.6/dist-packages (from textsearch) (1.1.1)\n",
      "Requirement already satisfied: pyahocorasick in /usr/local/lib/python3.6/dist-packages (from textsearch) (1.4.0)\n",
      "Requirement already satisfied: tqdm in /usr/local/lib/python3.6/dist-packages (4.28.1)\n",
      "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
      "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
      "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
      "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/externals/joblib/__init__.py:15: FutureWarning: sklearn.externals.joblib is deprecated in 0.21 and will be removed in 0.23. Please import this functionality directly from joblib, which can be installed with: pip install joblib. If this warning is raised when loading pickled models, you may need to re-serialize those models with scikit-learn 0.21+.\n",
      "  warnings.warn(msg, category=FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "# Installing packages.\n",
    "!pip install contractions\n",
    "!pip install textsearch\n",
    "!pip install tqdm\n",
    "\n",
    "# Importing packages.\n",
    "import nltk\n",
    "nltk.download('punkt')\n",
    "nltk.download('stopwords')\n",
    "%matplotlib inline\n",
    "import re\n",
    "import matplotlib\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.multiclass import OneVsRestClassifier\n",
    "from nltk.corpus import stopwords\n",
    "stop_words = set(stopwords.words('english'))\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.pipeline import Pipeline\n",
    "import seaborn as sns\n",
    "from sklearn.metrics import confusion_matrix, classification_report\n",
    "import pickle\n",
    "import ast\n",
    "from sklearn.externals import joblib\n",
    "from datetime import datetime\n",
    "from sklearn.preprocessing import MultiLabelBinarizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "2d7M7yfALpCN",
    "outputId": "a72e1301-8efa-4c37-d7b1-420d8a782837"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mounted at /content/drive\n"
     ]
    }
   ],
   "source": [
    "# Let's mount our G-Drive.\n",
    "\n",
    "from google.colab import drive\n",
    "drive.mount('/content/drive', force_remount=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Z75PYl6NkYsn"
   },
   "outputs": [],
   "source": [
    "# Data read and preparation.\n",
    "# Mentioning where is our data located on G-Drive. Make sure to rectify your path\n",
    "path = '/content/drive/My Drive/ICDMAI_Tutorial/notebook/'\n",
    "data ='filtered_data/question_tag_text_mapping.pkl'\n",
    "ml_model = path + 'ml_model/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 142
    },
    "colab_type": "code",
    "id": "Z_JUXpN2kce_",
    "outputId": "87558729-9257-42d1-dfa4-8a1136b7626a"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>OwnerUserId</th>\n",
       "      <th>CreationDate</th>\n",
       "      <th>ClosedDate</th>\n",
       "      <th>Score</th>\n",
       "      <th>Title</th>\n",
       "      <th>Body</th>\n",
       "      <th>Tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>120</td>\n",
       "      <td>83.0</td>\n",
       "      <td>2008-08-01T15:50:08Z</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21</td>\n",
       "      <td>ASP.NET Site Maps</td>\n",
       "      <td>&lt;p&gt;Has anyone got experience creating &lt;strong&gt;...</td>\n",
       "      <td>[sql, asp.net]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>260</td>\n",
       "      <td>91.0</td>\n",
       "      <td>2008-08-01T23:22:08Z</td>\n",
       "      <td>NaN</td>\n",
       "      <td>49</td>\n",
       "      <td>Adding scripting functionality to .NET applica...</td>\n",
       "      <td>&lt;p&gt;I have a little game written in C#. It uses...</td>\n",
       "      <td>[c#, .net]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>330</td>\n",
       "      <td>63.0</td>\n",
       "      <td>2008-08-02T02:51:36Z</td>\n",
       "      <td>NaN</td>\n",
       "      <td>29</td>\n",
       "      <td>Should I use nested classes in this case?</td>\n",
       "      <td>&lt;p&gt;I am working on a collection of classes use...</td>\n",
       "      <td>[c++]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    Id  ...             Tag\n",
       "0  120  ...  [sql, asp.net]\n",
       "1  260  ...      [c#, .net]\n",
       "2  330  ...           [c++]\n",
       "\n",
       "[3 rows x 8 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let us quickly load our question tag data\n",
    "question_tag = pd.read_pickle(path+data)\n",
    "question_tag.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "YklJh3CKtR3p"
   },
   "source": [
    "### Creating one hot encoding from multilabelled tagged data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 480
    },
    "colab_type": "code",
    "id": "AJckJ6dNlF2P",
    "outputId": "6c1e3d94-bf03-4638-d1cb-248c2150f12f"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>OwnerUserId</th>\n",
       "      <th>CreationDate</th>\n",
       "      <th>ClosedDate</th>\n",
       "      <th>Score</th>\n",
       "      <th>Title</th>\n",
       "      <th>Body</th>\n",
       "      <th>Tag</th>\n",
       "      <th>.net</th>\n",
       "      <th>agile</th>\n",
       "      <th>ajax</th>\n",
       "      <th>amazon-web-services</th>\n",
       "      <th>android</th>\n",
       "      <th>android-studio</th>\n",
       "      <th>angular2</th>\n",
       "      <th>angularjs</th>\n",
       "      <th>apache</th>\n",
       "      <th>apache-spark</th>\n",
       "      <th>api</th>\n",
       "      <th>asp.net</th>\n",
       "      <th>asp.net-web-api</th>\n",
       "      <th>azure</th>\n",
       "      <th>bash</th>\n",
       "      <th>c</th>\n",
       "      <th>c#</th>\n",
       "      <th>c++</th>\n",
       "      <th>cloud</th>\n",
       "      <th>codeigniter</th>\n",
       "      <th>css</th>\n",
       "      <th>devops</th>\n",
       "      <th>django</th>\n",
       "      <th>docker</th>\n",
       "      <th>drupal</th>\n",
       "      <th>eclipse</th>\n",
       "      <th>elasticsearch</th>\n",
       "      <th>embedded</th>\n",
       "      <th>entity-framework</th>\n",
       "      <th>excel</th>\n",
       "      <th>excel-vba</th>\n",
       "      <th>express</th>\n",
       "      <th>...</th>\n",
       "      <th>qt</th>\n",
       "      <th>r</th>\n",
       "      <th>react-native</th>\n",
       "      <th>reactjs</th>\n",
       "      <th>redis</th>\n",
       "      <th>redux</th>\n",
       "      <th>regex</th>\n",
       "      <th>rest</th>\n",
       "      <th>ruby</th>\n",
       "      <th>ruby-on-rails</th>\n",
       "      <th>sass</th>\n",
       "      <th>scala</th>\n",
       "      <th>selenium</th>\n",
       "      <th>shell</th>\n",
       "      <th>spring</th>\n",
       "      <th>spring-boot</th>\n",
       "      <th>spring-mvc</th>\n",
       "      <th>sql</th>\n",
       "      <th>sql-server</th>\n",
       "      <th>swift</th>\n",
       "      <th>tdd</th>\n",
       "      <th>testing</th>\n",
       "      <th>twitter-bootstrap</th>\n",
       "      <th>twitter-bootstrap-3</th>\n",
       "      <th>typescript</th>\n",
       "      <th>ubuntu</th>\n",
       "      <th>unity3d</th>\n",
       "      <th>unix</th>\n",
       "      <th>vb.net</th>\n",
       "      <th>vba</th>\n",
       "      <th>visual-studio</th>\n",
       "      <th>vue.js</th>\n",
       "      <th>wcf</th>\n",
       "      <th>web-services</th>\n",
       "      <th>windows</th>\n",
       "      <th>wordpress</th>\n",
       "      <th>wpf</th>\n",
       "      <th>xamarin</th>\n",
       "      <th>xcode</th>\n",
       "      <th>xml</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>120</td>\n",
       "      <td>83.0</td>\n",
       "      <td>2008-08-01T15:50:08Z</td>\n",
       "      <td>NaN</td>\n",
       "      <td>21</td>\n",
       "      <td>ASP.NET Site Maps</td>\n",
       "      <td>&lt;p&gt;Has anyone got experience creating &lt;strong&gt;...</td>\n",
       "      <td>[sql, asp.net]</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>260</td>\n",
       "      <td>91.0</td>\n",
       "      <td>2008-08-01T23:22:08Z</td>\n",
       "      <td>NaN</td>\n",
       "      <td>49</td>\n",
       "      <td>Adding scripting functionality to .NET applica...</td>\n",
       "      <td>&lt;p&gt;I have a little game written in C#. It uses...</td>\n",
       "      <td>[c#, .net]</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>330</td>\n",
       "      <td>63.0</td>\n",
       "      <td>2008-08-02T02:51:36Z</td>\n",
       "      <td>NaN</td>\n",
       "      <td>29</td>\n",
       "      <td>Should I use nested classes in this case?</td>\n",
       "      <td>&lt;p&gt;I am working on a collection of classes use...</td>\n",
       "      <td>[c++]</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 120 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    Id  OwnerUserId          CreationDate ClosedDate  ...  wpf xamarin xcode xml\n",
       "0  120         83.0  2008-08-01T15:50:08Z        NaN  ...    0       0     0   0\n",
       "1  260         91.0  2008-08-01T23:22:08Z        NaN  ...    0       0     0   0\n",
       "2  330         63.0  2008-08-02T02:51:36Z        NaN  ...    0       0     0   0\n",
       "\n",
       "[3 rows x 120 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# In order to use one vs rest strategy we will need to one hot encoding each tag across all documents.\n",
    "mlb = MultiLabelBinarizer()\n",
    "question_tag['Tag_pop'] = question_tag['Tag']\n",
    "question_tag = question_tag.join(pd.DataFrame(mlb.fit_transform(question_tag.pop('Tag_pop')),\n",
    "                          columns=mlb.classes_,\n",
    "                          index=question_tag.index))\n",
    "question_tag.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "3fZrAeH8lIJX"
   },
   "outputs": [],
   "source": [
    "# Creating a list of all existing 'Tags'\n",
    "dummy = question_tag.drop(['Id', 'OwnerUserId', 'CreationDate', 'ClosedDate', 'Score', 'Title','Body','Tag'], axis=1)\n",
    "categories = list(dummy.columns.values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "cVJoetuCsR6m"
   },
   "source": [
    "### Text preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "fBDeeV8FmGna"
   },
   "outputs": [],
   "source": [
    "# Let us createa a very basic text preprocessor which we will use for cleaning text.\n",
    "def clean_text(text):\n",
    "    text = text.lower()\n",
    "    text = re.sub(r\"what's\", \"what is \", text)\n",
    "    text = re.sub(r\"\\'s\", \" \", text)\n",
    "    text = re.sub(r\"\\'ve\", \" have \", text)\n",
    "    text = re.sub(r\"can't\", \"can not \", text)\n",
    "    text = re.sub(r\"n't\", \" not \", text)\n",
    "    text = re.sub(r\"i'm\", \"i am \", text)\n",
    "    text = re.sub(r\"\\'re\", \" are \", text)\n",
    "    text = re.sub(r\"\\'d\", \" would \", text)\n",
    "    text = re.sub(r\"\\'ll\", \" will \", text)\n",
    "    text = re.sub(r\"\\'scuse\", \" excuse \", text)\n",
    "    text = re.sub('\\W', ' ', text)\n",
    "    text = re.sub('\\s+', ' ', text)\n",
    "    text = text.strip(' ')\n",
    "    return text\n",
    "\n",
    "question_tag['Body'] = question_tag['Body'].map(lambda com : clean_text(com))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "oAPgabDns4mf"
   },
   "source": [
    "### Creating a 70/30 Train-Test Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "-FSyIUnElXrf",
    "outputId": "615764aa-ef9c-48f1-8b3f-426548b9aca7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train data shape : (736394,)\n",
      "Test data shape : (315598,)\n"
     ]
    }
   ],
   "source": [
    "train, test = train_test_split(question_tag, random_state=42, test_size=0.30, shuffle=True)\n",
    "\n",
    "X_train = train.Body\n",
    "X_test = test.Body\n",
    "\n",
    "print(\"Train data shape : {}\".format(X_train.shape))\n",
    "print(\"Test data shape : {}\".format(X_test.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "02ojVok0nROs"
   },
   "source": [
    "# Creating Bag of Words representation using TF - IDF\n",
    "  1. Initializing the Vectorizer object\n",
    "  2. Create a corpus from training data.\n",
    "  3. Create a document term matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "uZrbIaqIlY0E"
   },
   "outputs": [],
   "source": [
    "#Initializing the Vectorizer object\n",
    "tfidf = TfidfVectorizer(stop_words=stop_words)\n",
    "\n",
    "#Create a corpus from training data\n",
    "#Create a document term matrix of training data based on the corpus.\n",
    "X_train_dtm = tfidf.fit_transform(X_train)\n",
    "\n",
    "#Create a document term matrix of test data based on the corpus.\n",
    "#Note that the dimensions/columns of DTM of the test data will be based on the training data corpus only.\n",
    "X_test_dtm = tfidf.transform(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "LHy98X0LrQPV"
   },
   "source": [
    "## Pipeline\n",
    "scikit-learn provides a Pipeline utility to help automate machine learning workflows. Pipelines are very common in Machine Learning systems, since there is a lot of data to manipulate and many data transformations to apply. So we will utilize pipeline to train every classifier.\n",
    "\n",
    "## One Vs Rest Multilabel strategy\n",
    "The Multi-label algorithm accepts a binary mask over multiple labels. The result for each prediction will be an array of 0s and 1s marking which class labels apply to each row input sample.\n",
    "\n",
    "OneVsRest strategy can be used for multilabel learning, where a classifier is used to predict multiple labels for instance. **Naive Bayes**, **SVM**, **Logistic Regression** supports multi-class, but we are in a multi-label scenario, therefore, we wrap them in the OneVsRestClassifier.\n",
    "\n",
    "### We create a Training Pipeline and a Scoring Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Wl9Afjj3Q1LA"
   },
   "outputs": [],
   "source": [
    "def tag_level_training_pipeline(X_train, train, X_test, test, classifier_pipeline, output_directory):\n",
    "  \n",
    "  #1. Create a classifier for each Tag\n",
    "  for category in categories:\n",
    "    print('... Processing {}'.format(category))\n",
    "    \n",
    "    # 1. train the model using X_dtm & y\n",
    "    classifier_pipeline.fit(X_train, train[category])\n",
    "    \n",
    "    # 2. save the model to disk\n",
    "    filename = ml_model + output_directory +str(category)+ '_model.pkl'\n",
    "    joblib.dump(classifier_pipeline, filename, compress = 1)\n",
    "    \n",
    "    # 3. compute the testing accuracy\n",
    "    prediction = classifier_pipeline.predict(X_test)\n",
    "    print('Test accuracy is {}'.format(accuracy_score(test[category], prediction)))\n",
    "    print(classification_report(test[category], prediction))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "IHJwCF94SEEg"
   },
   "outputs": [],
   "source": [
    "def tag_level_predict(X_train, train, X_test, test, model_directory):\n",
    "  prediction_df = pd.DataFrame(columns=['dummy1'])\n",
    "  \n",
    "  #Score the document across classifier for each Tag\n",
    "  for category in categories:\n",
    "    \n",
    "    # 1. load the model\n",
    "    filename = ml_model + model_directory +str(category)+ '_model.pkl'\n",
    "    classifier_pipeline = joblib.load(filename)\n",
    "    \n",
    "    # 2. predict on the test data.\n",
    "    prediction = classifier_pipeline.predict(X_test)\n",
    "    prediction_df[str(category)] = prediction\n",
    "\n",
    "  # Remember We had encoded the labels. It time to bring them back to their original form.\n",
    "  for category in categories:\n",
    "    prediction_df.loc[prediction_df[str(category)] == 1, str(category)] = category\n",
    "  prediction_df['predicted_labels'] = prediction_df[[str(i) for i in categories]].values.tolist()\n",
    "  prediction_df['predicted_labels'] =  prediction_df['predicted_labels'].apply(lambda x : list(set(x)))\n",
    "  # prediction_df['predicted_labels'] = prediction_df['predicted_labels'].apply(lambda x: x.remove(0) if (0 in x) else x )\n",
    "  \n",
    "  # We create result having orignal labels and predicted labels for metrics Evaluation\n",
    "  final_pred_df = pd.concat([test[['Id','Tag']].reset_index(), prediction_df[['predicted_labels']].reset_index()], axis=1)\n",
    "  final_pred_df['original_labels'] = final_pred_df['Tag']\n",
    "  # prediction_df[['Id']] = test[['Id']]\n",
    "  final_pred_df_result = final_pred_df[['Id','original_labels','predicted_labels']]\n",
    "  return final_pred_df_result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8Jl8Vg24W2zy"
   },
   "outputs": [],
   "source": [
    "# importing os module \n",
    "import os\n",
    "try:\n",
    "  os.rename('/content/drive/My Drive/ICDMAI_Tutorial/notebook/ml_model/SVM/_net_model.pkl', '/content/drive/My Drive/ICDMAI_Tutorial/notebook/ml_model/SVM/.net_model.pkl')\n",
    "except :\n",
    "  print(\"Already in proper filename!\")  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 309
    },
    "colab_type": "code",
    "id": "AR8oVNlMkQkq",
    "outputId": "a961a467-5f73-4608-ceaf-e77943a275ab"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/utils/deprecation.py:144: FutureWarning: The sklearn.svm.classes module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.svm. Anything that cannot be imported from sklearn.svm is now part of the private API.\n",
      "  warnings.warn(message, FutureWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/base.py:318: UserWarning: Trying to unpickle estimator LinearSVC from version 0.21.3 when using version 0.22.1. This might lead to breaking code or invalid results. Use at your own risk.\n",
      "  UserWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/utils/deprecation.py:144: FutureWarning: The sklearn.preprocessing.label module is  deprecated in version 0.22 and will be removed in version 0.24. The corresponding classes / functions should instead be imported from sklearn.preprocessing. Anything that cannot be imported from sklearn.preprocessing is now part of the private API.\n",
      "  warnings.warn(message, FutureWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/base.py:318: UserWarning: Trying to unpickle estimator LabelBinarizer from version 0.21.3 when using version 0.22.1. This might lead to breaking code or invalid results. Use at your own risk.\n",
      "  UserWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/base.py:318: UserWarning: Trying to unpickle estimator OneVsRestClassifier from version 0.21.3 when using version 0.22.1. This might lead to breaking code or invalid results. Use at your own risk.\n",
      "  UserWarning)\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/base.py:318: UserWarning: Trying to unpickle estimator Pipeline from version 0.21.3 when using version 0.22.1. This might lead to breaking code or invalid results. Use at your own risk.\n",
      "  UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input [ How to handle memory locking ? ] || Predicted classes:  [0]\n",
      "Input [ How to handle memory locking in java ? ] || Predicted classes:  [0, 'java']\n",
      "Input [ How to handle memory locking in java python ? ] || Predicted classes:  [0, 'python', 'java']\n",
      "Input [ This post is not about java ] || Predicted classes:  [0, 'java']\n"
     ]
    }
   ],
   "source": [
    "## A Dummy example.\n",
    "X_test = [\"How to handle memory locking ?\", \"How to handle memory locking in java ?\", \"How to handle memory locking in java python ?\",\"This post is not about java\"]\n",
    "X_test_dtm = tfidf.transform(X_test)\n",
    "result = tag_level_predict(X_train_dtm, train, X_test_dtm, test.head(1), 'SVM/')\n",
    "\n",
    "for i in range(result.shape[0]):\n",
    "  print(\"Input [\",X_test[i],\"] || Predicted classes: \",result.predicted_labels[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "99lEvcIcVtg-"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "xF5rVRHS0JWV"
   },
   "source": [
    "# Evaluating our results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "w5mMGJ9prC0i"
   },
   "outputs": [],
   "source": [
    "# Here we define precision, recall, f1 measure at a single document level.\n",
    "def document_evaluation_metrics(prd_grp,grp,metric=\"precision\"):\n",
    "    pred_group = prd_grp\n",
    "    if 0 in pred_group: pred_group.remove(0)\n",
    "    group = grp\n",
    "\n",
    "    set_pred_group = set(pred_group)\n",
    "    set_group = set(group)\n",
    "    intrsct = set_group.intersection(set_pred_group)\n",
    "    accuracy = len(intrsct) / float(len(set_pred_group) if len(set_pred_group)>1 else 1)\n",
    "    recall = len(intrsct) / float(len(set_group) if len(set_group)>1 else 1)\n",
    "    if metric == \"precision\":\n",
    "      return accuracy\n",
    "    elif metric == \"recall\":\n",
    "      return recall\n",
    "    elif metric == \"f1_measure\":\n",
    "      if accuracy == 0 or recall == 0:\n",
    "        return 0\n",
    "      elif accuracy > 0 and recall >0 :\n",
    "        f1_measure = 2*accuracy*recall/(float(accuracy + recall))\n",
    "        return f1_measure\n",
    "    \n",
    "    return -1\n",
    "\n",
    "# Provide overall average stats and populate document level metrics.\n",
    "def model_evaluation_stats(final_pred_df, model_name=\"default\"):\n",
    "  final_pred_df['doc_precision'] = final_pred_df.apply(lambda x: document_evaluation_metrics(x.predicted_labels, x.original_labels, \"precision\"), axis=1)\n",
    "  final_pred_df['doc_recall'] = final_pred_df.apply(lambda x: document_evaluation_metrics(x.predicted_labels, x.original_labels, \"recall\"), axis=1)\n",
    "  final_pred_df['doc_f1_measure'] = final_pred_df.apply(lambda x: document_evaluation_metrics(x.predicted_labels, x.original_labels, \"f1_measure\"), axis=1)\n",
    "  \n",
    "  print('Avearge precision across documents is {}'.format(final_pred_df['doc_precision'].mean()))\n",
    "  print('Avearge recall across documents is {}'.format(final_pred_df['doc_recall'].mean()))\n",
    "  print('Avearge f1 measure across documents is {}'.format(final_pred_df['doc_f1_measure'].mean()))\n",
    "  pickle.dump(final_pred_df, open(ml_model + model_name + \".pkl\", 'wb'))\n",
    "  # final_pred_df.to_csv(ml_model + 'SVM_Tag_predictions.txt',sep='\\t',index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "yR9IC53B0TNL"
   },
   "source": [
    "# Let us train, score and evaluate Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZmnLqd0b67nI"
   },
   "outputs": [],
   "source": [
    "#Naive Bayes Classifier\n",
    "NB_pipeline = Pipeline([\n",
    "                ('clf', OneVsRestClassifier(MultinomialNB(\n",
    "                    fit_prior=True, class_prior=None))),\n",
    "            ])\n",
    "\n",
    "tag_level_training_pipeline(X_train_dtm, train, X_test_dtm, test, NB_pipeline, 'NaiveBayes/')\n",
    "result = tag_level_predict(X_train_dtm, train, X_test_dtm, test, 'NaiveBayes/')\n",
    "model_evaluation_stats(result, \"NaiveBayes\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ScCKvvqd0i9K"
   },
   "source": [
    "# Let us train, score and evaluate Support Vector Machines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "VrZ8XXiORO6z",
    "outputId": "c49f0be6-67bd-42ac-88f0-31b26df818f6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "... Processing .net\n",
      "Test accuracy is 0.9771893358006071\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      1.00      0.99    308362\n",
      "           1       0.51      0.09      0.15      7236\n",
      "\n",
      "    accuracy                           0.98    315598\n",
      "   macro avg       0.75      0.54      0.57    315598\n",
      "weighted avg       0.97      0.98      0.97    315598\n",
      "\n",
      "... Processing agile\n",
      "Test accuracy is 0.9999429654180318\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    315573\n",
      "           1       0.89      0.32      0.47        25\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.94      0.66      0.74    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing ajax\n",
      "Test accuracy is 0.9887356700612805\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99    310952\n",
      "           1       0.70      0.41      0.52      4646\n",
      "\n",
      "    accuracy                           0.99    315598\n",
      "   macro avg       0.84      0.71      0.76    315598\n",
      "weighted avg       0.99      0.99      0.99    315598\n",
      "\n",
      "... Processing amazon-web-services\n",
      "Test accuracy is 0.9981780619649048\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    314643\n",
      "           1       0.79      0.55      0.65       955\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.89      0.77      0.82    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing android\n",
      "Test accuracy is 0.9829815144582665\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99    288322\n",
      "           1       0.96      0.84      0.90     27276\n",
      "\n",
      "    accuracy                           0.98    315598\n",
      "   macro avg       0.97      0.92      0.94    315598\n",
      "weighted avg       0.98      0.98      0.98    315598\n",
      "\n",
      "... Processing android-studio\n",
      "Test accuracy is 0.9972401599503166\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    314589\n",
      "           1       0.67      0.27      0.38      1009\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.84      0.63      0.69    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing angular2\n",
      "Test accuracy is 0.9991064582158315\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    314898\n",
      "           1       0.94      0.64      0.76       700\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.97      0.82      0.88    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing angularjs\n",
      "Test accuracy is 0.9949904625504598\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    309420\n",
      "           1       0.93      0.80      0.86      6178\n",
      "\n",
      "    accuracy                           0.99    315598\n",
      "   macro avg       0.96      0.90      0.93    315598\n",
      "weighted avg       0.99      0.99      0.99    315598\n",
      "\n",
      "... Processing apache\n",
      "Test accuracy is 0.9952788040481879\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    313590\n",
      "           1       0.71      0.43      0.54      2008\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.85      0.72      0.77    315598\n",
      "weighted avg       0.99      1.00      0.99    315598\n",
      "\n",
      "... Processing apache-spark\n",
      "Test accuracy is 0.999404305477221\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    314985\n",
      "           1       0.93      0.75      0.83       613\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.97      0.87      0.91    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing api\n",
      "Test accuracy is 0.9951900835873485\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    314094\n",
      "           1       0.47      0.07      0.12      1504\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.73      0.53      0.56    315598\n",
      "weighted avg       0.99      1.00      0.99    315598\n",
      "\n",
      "... Processing asp.net\n",
      "Test accuracy is 0.9815905043758199\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99    306664\n",
      "           1       0.79      0.48      0.60      8934\n",
      "\n",
      "    accuracy                           0.98    315598\n",
      "   macro avg       0.89      0.74      0.79    315598\n",
      "weighted avg       0.98      0.98      0.98    315598\n",
      "\n",
      "... Processing asp.net-web-api\n",
      "Test accuracy is 0.9985804726265692\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    314999\n",
      "           1       0.72      0.41      0.52       599\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.86      0.71      0.76    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing azure\n",
      "Test accuracy is 0.9987769250755708\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    314517\n",
      "           1       0.91      0.71      0.80      1081\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.95      0.86      0.90    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing bash\n",
      "Test accuracy is 0.995047497132428\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    313311\n",
      "           1       0.76      0.46      0.58      2287\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.88      0.73      0.79    315598\n",
      "weighted avg       0.99      1.00      0.99    315598\n",
      "\n",
      "... Processing c\n",
      "Test accuracy is 0.9872432651664459\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99    308691\n",
      "           1       0.81      0.55      0.65      6907\n",
      "\n",
      "    accuracy                           0.99    315598\n",
      "   macro avg       0.90      0.77      0.82    315598\n",
      "weighted avg       0.99      0.99      0.99    315598\n",
      "\n",
      "... Processing c#\n",
      "Test accuracy is 0.9414159785549971\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.98      0.97    285147\n",
      "           1       0.77      0.56      0.65     30451\n",
      "\n",
      "    accuracy                           0.94    315598\n",
      "   macro avg       0.86      0.77      0.81    315598\n",
      "weighted avg       0.94      0.94      0.94    315598\n",
      "\n",
      "... Processing c++\n",
      "Test accuracy is 0.9785581657678439\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.99      0.99    301367\n",
      "           1       0.85      0.63      0.73     14231\n",
      "\n",
      "    accuracy                           0.98    315598\n",
      "   macro avg       0.92      0.81      0.86    315598\n",
      "weighted avg       0.98      0.98      0.98    315598\n",
      "\n",
      "... Processing cloud\n",
      "Test accuracy is 0.9995722406352385\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    315459\n",
      "           1       0.75      0.04      0.08       139\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.87      0.52      0.54    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing codeigniter\n",
      "Test accuracy is 0.9979055634066122\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    314150\n",
      "           1       0.90      0.61      0.73      1448\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.95      0.80      0.86    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing css\n",
      "Test accuracy is 0.9785454914162954\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.99      0.99    302936\n",
      "           1       0.78      0.64      0.71     12662\n",
      "\n",
      "    accuracy                           0.98    315598\n",
      "   macro avg       0.88      0.82      0.85    315598\n",
      "weighted avg       0.98      0.98      0.98    315598\n",
      "\n",
      "... Processing devops\n",
      "Test accuracy is 0.9999524711816932\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/metrics/classification.py:1437: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n",
      "  'precision', 'predicted', average, warn_for)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    315583\n",
      "           1       0.00      0.00      0.00        15\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.50      0.50      0.50    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing django\n",
      "Test accuracy is 0.9972718458291878\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    311726\n",
      "           1       0.96      0.81      0.88      3872\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.98      0.90      0.94    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing docker\n",
      "Test accuracy is 0.9996546239203037\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    315127\n",
      "           1       0.93      0.83      0.88       471\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.97      0.92      0.94    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing drupal\n",
      "Test accuracy is 0.9992078530282195\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    315063\n",
      "           1       0.91      0.59      0.72       535\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.95      0.80      0.86    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing eclipse\n",
      "Test accuracy is 0.9935899467043517\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00    312691\n",
      "           1       0.76      0.45      0.56      2907\n",
      "\n",
      "    accuracy                           0.99    315598\n",
      "   macro avg       0.88      0.72      0.78    315598\n",
      "weighted avg       0.99      0.99      0.99    315598\n",
      "\n",
      "... Processing elasticsearch\n",
      "Test accuracy is 0.9994328227682051\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    314963\n",
      "           1       0.95      0.76      0.84       635\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.97      0.88      0.92    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing embedded\n",
      "Test accuracy is 0.9994106426529953\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    315413\n",
      "           1       0.46      0.03      0.06       185\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.73      0.52      0.53    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing entity-framework\n",
      "Test accuracy is 0.9963022579357284\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    313799\n",
      "           1       0.75      0.52      0.62      1799\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.87      0.76      0.81    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing excel\n",
      "Test accuracy is 0.9937895677412404\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    312452\n",
      "           1       0.75      0.57      0.65      3146\n",
      "\n",
      "    accuracy                           0.99    315598\n",
      "   macro avg       0.87      0.79      0.82    315598\n",
      "weighted avg       0.99      0.99      0.99    315598\n",
      "\n",
      "... Processing excel-vba\n",
      "Test accuracy is 0.9964955417968429\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    314013\n",
      "           1       0.71      0.52      0.60      1585\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.85      0.76      0.80    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing express\n",
      "Test accuracy is 0.9980956786798395\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    314782\n",
      "           1       0.74      0.41      0.53       816\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.87      0.71      0.76    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing flask\n",
      "Test accuracy is 0.9993599452468013\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    315154\n",
      "           1       0.87      0.64      0.74       444\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.93      0.82      0.87    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing git\n",
      "Test accuracy is 0.9980354755099842\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    313325\n",
      "           1       0.90      0.81      0.86      2273\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.95      0.91      0.93    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing github\n",
      "Test accuracy is 0.9986438443843116\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    314995\n",
      "           1       0.71      0.49      0.58       603\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.86      0.74      0.79    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing go\n",
      "Test accuracy is 0.9992712247859619\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    315037\n",
      "           1       0.96      0.61      0.75       561\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.98      0.81      0.87    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing hadoop\n",
      "Test accuracy is 0.9988814884758459\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    314687\n",
      "           1       0.88      0.71      0.79       911\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.94      0.85      0.89    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing haskell\n",
      "Test accuracy is 0.9991920100887838\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    314664\n",
      "           1       0.98      0.74      0.84       934\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.99      0.87      0.92    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing hibernate\n",
      "Test accuracy is 0.9974999841570605\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    313870\n",
      "           1       0.85      0.66      0.74      1728\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.92      0.83      0.87    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing html\n",
      "Test accuracy is 0.95084886469496\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.99      0.97    297994\n",
      "           1       0.60      0.36      0.45     17604\n",
      "\n",
      "    accuracy                           0.95    315598\n",
      "   macro avg       0.78      0.67      0.71    315598\n",
      "weighted avg       0.94      0.95      0.95    315598\n",
      "\n",
      "... Processing html5\n",
      "Test accuracy is 0.9918884150089671\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00    312749\n",
      "           1       0.68      0.19      0.30      2849\n",
      "\n",
      "    accuracy                           0.99    315598\n",
      "   macro avg       0.84      0.60      0.65    315598\n",
      "weighted avg       0.99      0.99      0.99    315598\n",
      "\n",
      "... Processing ionic-framework\n",
      "Test accuracy is 0.9990304121065406\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    315153\n",
      "           1       0.73      0.49      0.59       445\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.87      0.74      0.79    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing ios\n",
      "Test accuracy is 0.9735422911425294\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.99      0.99    301642\n",
      "           1       0.74      0.62      0.68     13956\n",
      "\n",
      "    accuracy                           0.97    315598\n",
      "   macro avg       0.86      0.81      0.83    315598\n",
      "weighted avg       0.97      0.97      0.97    315598\n",
      "\n",
      "... Processing iphone\n",
      "Test accuracy is 0.9816063473152555\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99    309185\n",
      "           1       0.59      0.31      0.41      6413\n",
      "\n",
      "    accuracy                           0.98    315598\n",
      "   macro avg       0.79      0.65      0.70    315598\n",
      "weighted avg       0.98      0.98      0.98    315598\n",
      "\n",
      "... Processing java\n",
      "Test accuracy is 0.9439033200463881\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.98      0.97    280946\n",
      "           1       0.81      0.63      0.71     34652\n",
      "\n",
      "    accuracy                           0.94    315598\n",
      "   macro avg       0.89      0.81      0.84    315598\n",
      "weighted avg       0.94      0.94      0.94    315598\n",
      "\n",
      "... Processing java-ee\n",
      "Test accuracy is 0.9974936469812863\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    314792\n",
      "           1       0.55      0.10      0.16       806\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.78      0.55      0.58    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing javascript\n",
      "Test accuracy is 0.925493824422208\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.94      0.97      0.96    278136\n",
      "           1       0.74      0.58      0.65     37462\n",
      "\n",
      "    accuracy                           0.93    315598\n",
      "   macro avg       0.84      0.77      0.80    315598\n",
      "weighted avg       0.92      0.93      0.92    315598\n",
      "\n",
      "... Processing jenkins\n",
      "Test accuracy is 0.9994771829986249\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    315021\n",
      "           1       0.94      0.77      0.84       577\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.97      0.88      0.92    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing jquery\n",
      "Test accuracy is 0.9630922882908003\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.99      0.98    291958\n",
      "           1       0.82      0.65      0.72     23640\n",
      "\n",
      "    accuracy                           0.96    315598\n",
      "   macro avg       0.90      0.82      0.85    315598\n",
      "weighted avg       0.96      0.96      0.96    315598\n",
      "\n",
      "... Processing json\n",
      "Test accuracy is 0.9870816671842026\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99    310273\n",
      "           1       0.70      0.41      0.51      5325\n",
      "\n",
      "    accuracy                           0.99    315598\n",
      "   macro avg       0.85      0.70      0.75    315598\n",
      "weighted avg       0.99      0.99      0.99    315598\n",
      "\n",
      "... Processing jsp\n",
      "Test accuracy is 0.9971926311320097\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    314281\n",
      "           1       0.77      0.47      0.58      1317\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.88      0.73      0.79    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing laravel\n",
      "Test accuracy is 0.9970848991438476\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    314097\n",
      "           1       0.81      0.50      0.62      1501\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.90      0.75      0.81    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing less\n",
      "Test accuracy is 0.9995944207504484\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    315408\n",
      "           1       0.87      0.38      0.53       190\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.93      0.69      0.77    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing linq\n",
      "Test accuracy is 0.9957826095222403\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    313774\n",
      "           1       0.73      0.42      0.54      1824\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.87      0.71      0.77    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing linux\n",
      "Test accuracy is 0.9885582291396017\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99    311589\n",
      "           1       0.61      0.27      0.37      4009\n",
      "\n",
      "    accuracy                           0.99    315598\n",
      "   macro avg       0.80      0.63      0.68    315598\n",
      "weighted avg       0.99      0.99      0.99    315598\n",
      "\n",
      "... Processing machine-learning\n",
      "Test accuracy is 0.9988751513000717\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    315209\n",
      "           1       0.60      0.25      0.36       389\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.80      0.63      0.68    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing matlab\n",
      "Test accuracy is 0.9979372492854834\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    313640\n",
      "           1       0.94      0.71      0.81      1958\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.97      0.86      0.90    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing maven\n",
      "Test accuracy is 0.9979087319944994\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    314193\n",
      "           1       0.84      0.66      0.74      1405\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.92      0.83      0.87    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing mongodb\n",
      "Test accuracy is 0.9975063213328348\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    313440\n",
      "           1       0.90      0.72      0.80      2158\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.95      0.86      0.90    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing mysql\n",
      "Test accuracy is 0.9750790562677837\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.99      0.99    302773\n",
      "           1       0.77      0.55      0.64     12825\n",
      "\n",
      "    accuracy                           0.98    315598\n",
      "   macro avg       0.88      0.77      0.81    315598\n",
      "weighted avg       0.97      0.98      0.97    315598\n",
      "\n",
      "... Processing nginx\n",
      "Test accuracy is 0.999220527379768\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    314927\n",
      "           1       0.90      0.71      0.79       671\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.95      0.86      0.90    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing node.js\n",
      "Test accuracy is 0.9932382334488812\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00    311324\n",
      "           1       0.84      0.62      0.71      4274\n",
      "\n",
      "    accuracy                           0.99    315598\n",
      "   macro avg       0.92      0.81      0.85    315598\n",
      "weighted avg       0.99      0.99      0.99    315598\n",
      "\n",
      "... Processing objective-c\n",
      "Test accuracy is 0.9780353487664687\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.99      0.99    307593\n",
      "           1       0.60      0.40      0.48      8005\n",
      "\n",
      "    accuracy                           0.98    315598\n",
      "   macro avg       0.79      0.70      0.73    315598\n",
      "weighted avg       0.97      0.98      0.98    315598\n",
      "\n",
      "... Processing oracle\n",
      "Test accuracy is 0.9954720879093023\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    313356\n",
      "           1       0.80      0.48      0.60      2242\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.90      0.74      0.80    315598\n",
      "weighted avg       0.99      1.00      0.99    315598\n",
      "\n",
      "... Processing osx\n",
      "Test accuracy is 0.9942236642817762\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    313483\n",
      "           1       0.64      0.32      0.43      2115\n",
      "\n",
      "    accuracy                           0.99    315598\n",
      "   macro avg       0.82      0.66      0.71    315598\n",
      "weighted avg       0.99      0.99      0.99    315598\n",
      "\n",
      "... Processing perl\n",
      "Test accuracy is 0.9979721037522418\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    314042\n",
      "           1       0.93      0.64      0.76      1556\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.96      0.82      0.88    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing photoshop\n",
      "Test accuracy is 0.9998479077814182\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    315539\n",
      "           1       0.72      0.31      0.43        59\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.86      0.65      0.71    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing php\n",
      "Test accuracy is 0.9627152263322328\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.97      0.99      0.98    285890\n",
      "           1       0.88      0.70      0.78     29708\n",
      "\n",
      "    accuracy                           0.96    315598\n",
      "   macro avg       0.92      0.85      0.88    315598\n",
      "weighted avg       0.96      0.96      0.96    315598\n",
      "\n",
      "... Processing plsql\n",
      "Test accuracy is 0.9989195115304913\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    315188\n",
      "           1       0.69      0.31      0.42       410\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.84      0.65      0.71    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing postgresql\n",
      "Test accuracy is 0.9965937680213436\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    313790\n",
      "           1       0.87      0.48      0.62      1808\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.93      0.74      0.81    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing powershell\n",
      "Test accuracy is 0.9987642507240223\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    314463\n",
      "           1       0.94      0.70      0.80      1135\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.97      0.85      0.90    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing python\n",
      "Test accuracy is 0.9801963257054861\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.99      0.99    296188\n",
      "           1       0.91      0.75      0.82     19410\n",
      "\n",
      "    accuracy                           0.98    315598\n",
      "   macro avg       0.95      0.87      0.91    315598\n",
      "weighted avg       0.98      0.98      0.98    315598\n",
      "\n",
      "... Processing qt\n",
      "Test accuracy is 0.9980703299767426\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    314015\n",
      "           1       0.89      0.70      0.79      1583\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.94      0.85      0.89    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing r\n",
      "Test accuracy is 0.9951995893510098\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    310983\n",
      "           1       0.94      0.72      0.81      4615\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.97      0.86      0.91    315598\n",
      "weighted avg       1.00      1.00      0.99    315598\n",
      "\n",
      "... Processing react-native\n",
      "Test accuracy is 0.9996292752172067\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    315371\n",
      "           1       0.94      0.52      0.67       227\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.97      0.76      0.83    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing reactjs\n",
      "Test accuracy is 0.9991888415008967\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    314823\n",
      "           1       0.91      0.74      0.82       775\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.96      0.87      0.91    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing redis\n",
      "Test accuracy is 0.9995183746411574\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    315290\n",
      "           1       0.82      0.64      0.72       308\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.91      0.82      0.86    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing redux\n",
      "Test accuracy is 0.9998003789631112\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    315479\n",
      "           1       0.82      0.61      0.70       119\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.91      0.80      0.85    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing regex\n",
      "Test accuracy is 0.9923858833072453\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00    310998\n",
      "           1       0.85      0.58      0.69      4600\n",
      "\n",
      "    accuracy                           0.99    315598\n",
      "   macro avg       0.92      0.79      0.84    315598\n",
      "weighted avg       0.99      0.99      0.99    315598\n",
      "\n",
      "... Processing rest\n",
      "Test accuracy is 0.9958554870436441\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    314095\n",
      "           1       0.66      0.26      0.38      1503\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.83      0.63      0.69    315598\n",
      "weighted avg       0.99      1.00      0.99    315598\n",
      "\n",
      "... Processing ruby\n",
      "Test accuracy is 0.9877850936951438\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99    310503\n",
      "           1       0.70      0.43      0.53      5095\n",
      "\n",
      "    accuracy                           0.99    315598\n",
      "   macro avg       0.84      0.71      0.76    315598\n",
      "weighted avg       0.99      0.99      0.99    315598\n",
      "\n",
      "... Processing ruby-on-rails\n",
      "Test accuracy is 0.9911976628495744\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00    307768\n",
      "           1       0.90      0.73      0.80      7830\n",
      "\n",
      "    accuracy                           0.99    315598\n",
      "   macro avg       0.94      0.86      0.90    315598\n",
      "weighted avg       0.99      0.99      0.99    315598\n",
      "\n",
      "... Processing sass\n",
      "Test accuracy is 0.9994359913560923\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    315302\n",
      "           1       0.80      0.53      0.64       296\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.90      0.76      0.82    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing scala\n",
      "Test accuracy is 0.9978580345883054\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    313889\n",
      "           1       0.91      0.67      0.77      1709\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.96      0.83      0.89    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing selenium\n",
      "Test accuracy is 0.9981717247891305\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    314532\n",
      "           1       0.83      0.58      0.68      1066\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.91      0.79      0.84    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing shell\n",
      "Test accuracy is 0.995487930848738\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    314094\n",
      "           1       0.57      0.20      0.30      1504\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.79      0.60      0.65    315598\n",
      "weighted avg       0.99      1.00      0.99    315598\n",
      "\n",
      "... Processing spring\n",
      "Test accuracy is 0.9944708141369717\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    312645\n",
      "           1       0.77      0.58      0.66      2953\n",
      "\n",
      "    accuracy                           0.99    315598\n",
      "   macro avg       0.88      0.79      0.83    315598\n",
      "weighted avg       0.99      0.99      0.99    315598\n",
      "\n",
      "... Processing spring-boot\n",
      "Test accuracy is 0.9990525922217505\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    315188\n",
      "           1       0.76      0.40      0.52       410\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.88      0.70      0.76    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing spring-mvc\n",
      "Test accuracy is 0.9970405389134278\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    314490\n",
      "           1       0.64      0.36      0.46      1108\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.82      0.68      0.73    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing sql\n",
      "Test accuracy is 0.9694896672349\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.99      0.98    304845\n",
      "           1       0.59      0.36      0.44     10753\n",
      "\n",
      "    accuracy                           0.97    315598\n",
      "   macro avg       0.78      0.67      0.71    315598\n",
      "weighted avg       0.96      0.97      0.97    315598\n",
      "\n",
      "... Processing sql-server\n",
      "Test accuracy is 0.9862990259760835\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      0.99    310135\n",
      "           1       0.68      0.39      0.49      5463\n",
      "\n",
      "    accuracy                           0.99    315598\n",
      "   macro avg       0.84      0.69      0.74    315598\n",
      "weighted avg       0.98      0.99      0.98    315598\n",
      "\n",
      "... Processing swift\n",
      "Test accuracy is 0.9952376124056553\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    312095\n",
      "           1       0.89      0.65      0.75      3503\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.94      0.83      0.88    315598\n",
      "weighted avg       0.99      1.00      0.99    315598\n",
      "\n",
      "... Processing tdd\n",
      "Test accuracy is 0.9995944207504484\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    315458\n",
      "           1       0.71      0.14      0.24       140\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.86      0.57      0.62    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing testing\n",
      "Test accuracy is 0.997693268018175\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    314875\n",
      "           1       0.45      0.03      0.06       723\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.73      0.52      0.53    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing twitter-bootstrap\n",
      "Test accuracy is 0.9953770302726886\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    313447\n",
      "           1       0.73      0.50      0.60      2151\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.87      0.75      0.80    315598\n",
      "weighted avg       0.99      1.00      0.99    315598\n",
      "\n",
      "... Processing twitter-bootstrap-3\n",
      "Test accuracy is 0.998393525941229\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    315100\n",
      "           1       0.38      0.03      0.05       498\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.69      0.51      0.53    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing typescript\n",
      "Test accuracy is 0.9990209063428792\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    315077\n",
      "           1       0.86      0.49      0.62       521\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.93      0.74      0.81    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing ubuntu\n",
      "Test accuracy is 0.9969549870404756\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    314643\n",
      "           1       0.49      0.13      0.21       955\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.74      0.57      0.60    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing unity3d\n",
      "Test accuracy is 0.9991064582158315\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    314988\n",
      "           1       0.86      0.64      0.73       610\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.93      0.82      0.87    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing unix\n",
      "Test accuracy is 0.9970468760892021\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    314644\n",
      "           1       0.58      0.09      0.15       954\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.79      0.54      0.58    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing vb.net\n",
      "Test accuracy is 0.994011368893339\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    312484\n",
      "           1       0.81      0.51      0.63      3114\n",
      "\n",
      "    accuracy                           0.99    315598\n",
      "   macro avg       0.90      0.75      0.81    315598\n",
      "weighted avg       0.99      0.99      0.99    315598\n",
      "\n",
      "... Processing vba\n",
      "Test accuracy is 0.996086793959404\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    313501\n",
      "           1       0.77      0.59      0.67      2097\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.88      0.79      0.83    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing visual-studio\n",
      "Test accuracy is 0.9943250590941641\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    313762\n",
      "           1       0.54      0.17      0.26      1836\n",
      "\n",
      "    accuracy                           0.99    315598\n",
      "   macro avg       0.77      0.59      0.63    315598\n",
      "weighted avg       0.99      0.99      0.99    315598\n",
      "\n",
      "... Processing vue.js\n",
      "Test accuracy is 0.9999144481270477\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    315549\n",
      "           1       0.81      0.59      0.68        49\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.90      0.80      0.84    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing wcf\n",
      "Test accuracy is 0.99846957205052\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    314208\n",
      "           1       0.91      0.72      0.81      1390\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.95      0.86      0.90    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing web-services\n",
      "Test accuracy is 0.9954530763819797\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    313994\n",
      "           1       0.64      0.25      0.36      1604\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.82      0.62      0.68    315598\n",
      "weighted avg       0.99      1.00      0.99    315598\n",
      "\n",
      "... Processing windows\n",
      "Test accuracy is 0.9906368227935538\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00    312574\n",
      "           1       0.54      0.15      0.23      3024\n",
      "\n",
      "    accuracy                           0.99    315598\n",
      "   macro avg       0.77      0.57      0.61    315598\n",
      "weighted avg       0.99      0.99      0.99    315598\n",
      "\n",
      "... Processing wordpress\n",
      "Test accuracy is 0.9965018789726171\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    312646\n",
      "           1       0.89      0.71      0.79      2952\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.95      0.86      0.90    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing wpf\n",
      "Test accuracy is 0.9957192377644979\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    311919\n",
      "           1       0.88      0.73      0.80      3679\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.94      0.86      0.90    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing xamarin\n",
      "Test accuracy is 0.9989226801183785\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00    315088\n",
      "           1       0.79      0.46      0.58       510\n",
      "\n",
      "    accuracy                           1.00    315598\n",
      "   macro avg       0.89      0.73      0.79    315598\n",
      "weighted avg       1.00      1.00      1.00    315598\n",
      "\n",
      "... Processing xcode\n",
      "Test accuracy is 0.9912261801405585\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00    312421\n",
      "           1       0.67      0.25      0.36      3177\n",
      "\n",
      "    accuracy                           0.99    315598\n",
      "   macro avg       0.83      0.62      0.68    315598\n",
      "weighted avg       0.99      0.99      0.99    315598\n",
      "\n",
      "... Processing xml\n",
      "Test accuracy is 0.9903738299989227\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      1.00      1.00    311288\n",
      "           1       0.72      0.49      0.58      4310\n",
      "\n",
      "    accuracy                           0.99    315598\n",
      "   macro avg       0.85      0.74      0.79    315598\n",
      "weighted avg       0.99      0.99      0.99    315598\n",
      "\n",
      "... Processing .net\n",
      "... Processing agile\n",
      "... Processing ajax\n",
      "... Processing amazon-web-services\n",
      "... Processing android\n",
      "... Processing android-studio\n",
      "... Processing angular2\n",
      "... Processing angularjs\n",
      "... Processing apache\n",
      "... Processing apache-spark\n",
      "... Processing api\n",
      "... Processing asp.net\n",
      "... Processing asp.net-web-api\n",
      "... Processing azure\n",
      "... Processing bash\n",
      "... Processing c\n",
      "... Processing c#\n",
      "... Processing c++\n",
      "... Processing cloud\n",
      "... Processing codeigniter\n",
      "... Processing css\n",
      "... Processing devops\n",
      "... Processing django\n",
      "... Processing docker\n",
      "... Processing drupal\n",
      "... Processing eclipse\n",
      "... Processing elasticsearch\n",
      "... Processing embedded\n",
      "... Processing entity-framework\n",
      "... Processing excel\n",
      "... Processing excel-vba\n",
      "... Processing express\n",
      "... Processing flask\n",
      "... Processing git\n",
      "... Processing github\n",
      "... Processing go\n",
      "... Processing hadoop\n",
      "... Processing haskell\n",
      "... Processing hibernate\n",
      "... Processing html\n",
      "... Processing html5\n",
      "... Processing ionic-framework\n",
      "... Processing ios\n",
      "... Processing iphone\n",
      "... Processing java\n",
      "... Processing java-ee\n",
      "... Processing javascript\n",
      "... Processing jenkins\n",
      "... Processing jquery\n",
      "... Processing json\n",
      "... Processing jsp\n",
      "... Processing laravel\n",
      "... Processing less\n",
      "... Processing linq\n",
      "... Processing linux\n",
      "... Processing machine-learning\n",
      "... Processing matlab\n",
      "... Processing maven\n",
      "... Processing mongodb\n",
      "... Processing mysql\n",
      "... Processing nginx\n",
      "... Processing node.js\n",
      "... Processing objective-c\n",
      "... Processing oracle\n",
      "... Processing osx\n",
      "... Processing perl\n",
      "... Processing photoshop\n",
      "... Processing php\n",
      "... Processing plsql\n",
      "... Processing postgresql\n",
      "... Processing powershell\n",
      "... Processing python\n",
      "... Processing qt\n",
      "... Processing r\n",
      "... Processing react-native\n",
      "... Processing reactjs\n",
      "... Processing redis\n",
      "... Processing redux\n",
      "... Processing regex\n",
      "... Processing rest\n",
      "... Processing ruby\n",
      "... Processing ruby-on-rails\n",
      "... Processing sass\n",
      "... Processing scala\n",
      "... Processing selenium\n",
      "... Processing shell\n",
      "... Processing spring\n",
      "... Processing spring-boot\n",
      "... Processing spring-mvc\n",
      "... Processing sql\n",
      "... Processing sql-server\n",
      "... Processing swift\n",
      "... Processing tdd\n",
      "... Processing testing\n",
      "... Processing twitter-bootstrap\n",
      "... Processing twitter-bootstrap-3\n",
      "... Processing typescript\n",
      "... Processing ubuntu\n",
      "... Processing unity3d\n",
      "... Processing unix\n",
      "... Processing vb.net\n",
      "... Processing vba\n",
      "... Processing visual-studio\n",
      "... Processing vue.js\n",
      "... Processing wcf\n",
      "... Processing web-services\n",
      "... Processing windows\n",
      "... Processing wordpress\n",
      "... Processing wpf\n",
      "... Processing xamarin\n",
      "... Processing xcode\n",
      "... Processing xml\n",
      "Avearge precision across documents is 0.6659770763228201\n",
      "Avearge recall across documents is 0.6147943480841764\n",
      "Avearge f1 measure across documents is 0.6143742008740732\n"
     ]
    }
   ],
   "source": [
    "#SVM Classifier\n",
    "SVC_pipeline = Pipeline([\n",
    "                ('clf', OneVsRestClassifier(LinearSVC(), n_jobs=1)),\n",
    "            ])\n",
    "\n",
    "tag_level_training_pipeline(X_train_dtm, train, X_test_dtm, test, SVC_pipeline, 'SVM/')\n",
    "result = tag_level_predict(X_train_dtm, train, X_test_dtm, test, 'SVM/')\n",
    "model_evaluation_stats(result, \"SVM\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "hNOMgocp0oHj"
   },
   "source": [
    "# Let us train, score and evaluate Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PHb8X6iRRZ3T"
   },
   "outputs": [],
   "source": [
    "#Logistic Regression Classifier\n",
    "LogReg_pipeline = Pipeline([\n",
    "                ('clf', OneVsRestClassifier(LogisticRegression(solver='sag'), n_jobs=1)),\n",
    "            ])\n",
    "\n",
    "tag_level_training_pipeline(X_train_dtm, train, X_test_dtm, test, LogReg_pipeline, 'LogisticRegression/')\n",
    "result = tag_level_predict(X_train_dtm, train, X_test_dtm, test, 'LogisticRegression/')\n",
    "model_evaluation_stats(result, \"LogisticRegression\")"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "machine_shape": "hm",
   "name": "2_classical_ml_approach.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}