Decision Tree.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\anaconda3\\lib\\site-packages\\sklearn\\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.\n",
      "  \"This module will be removed in 0.20.\", DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sklearn.tree import DecisionTreeClassifier as DTC\n",
    "from sklearn.cross_validation import StratifiedKFold"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The header may contain special chars, e.g. \"(kg/m²)\", which can cause some errors, so we need to remove them"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Peptide1',\n",
       " 'Peptide2',\n",
       " 'Peptide3',\n",
       " 'Peptide4',\n",
       " 'Peptide5',\n",
       " 'Peptide6',\n",
       " 'Peptide7',\n",
       " 'Peptide8',\n",
       " 'Peptide9',\n",
       " 'Peptide10',\n",
       " 'Peptide11',\n",
       " 'Peptide12',\n",
       " 'Peptide13',\n",
       " 'Peptide14',\n",
       " 'Peptide15',\n",
       " 'Peptide16',\n",
       " 'Peptide17',\n",
       " 'Peptide18',\n",
       " 'Peptide21',\n",
       " 'Peptide22',\n",
       " 'Peptide23',\n",
       " 'Peptide24',\n",
       " 'Peptide25',\n",
       " 'Peptide26',\n",
       " 'Peptide27',\n",
       " 'Peptide29',\n",
       " 'Peptide30',\n",
       " 'Age',\n",
       " 'Diagnosis',\n",
       " 'BMI',\n",
       " 'HbA1c',\n",
       " 'Gender',\n",
       " 'Height',\n",
       " 'Bodyweight',\n",
       " 'BMI',\n",
       " 'Bodyfat',\n",
       " 'Fatfreemass',\n",
       " 'Waist',\n",
       " 'Hip',\n",
       " 'WHR',\n",
       " 'Hemoglobin',\n",
       " 'Erythrozyten',\n",
       " 'Thrombozyten',\n",
       " 'Leukocytes',\n",
       " 'ALAT',\n",
       " 'ASAT',\n",
       " 'gGT',\n",
       " 'Fastingplasmaglucose',\n",
       " 'Fastingplasmainsulin',\n",
       " 'C-Peptide',\n",
       " 'Proinsulin',\n",
       " 'Creatinin',\n",
       " 'Triglycerides',\n",
       " 'Cholesteroltotal',\n",
       " 'HDL-Cholesterol',\n",
       " 'LDL-Cholesterol',\n",
       " 'Proteinstotal',\n",
       " 'Albumin',\n",
       " 'TSH',\n",
       " 'fT3',\n",
       " 'fT4',\n",
       " 'Cortisol',\n",
       " 'Testosteron',\n",
       " 'HOMA-IR',\n",
       " 'Freefattyacids',\n",
       " 'RRsys',\n",
       " 'RRdia',\n",
       " 'ssCRP']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "header = open(\"newly_diagnosed.csv\").readlines()[0]\n",
    "header = header.strip().split(\",\")[1:]\n",
    "for i in range(len(header)):\n",
    "    header[i] = header[i].split(\"(\")[0].replace(' ','')\n",
    "header"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "read data, skip the first row, and give the col names manually"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "E:\\anaconda3\\lib\\site-packages\\pandas\\io\\parsers.py:709: UserWarning: Duplicate names specified. This will raise an error in the future.\n",
      "  return _read(filepath_or_buffer, kwds)\n"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv(\"newly_diagnosed.csv\",index_col=0,skiprows=[0],header=None,names=header)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(96, 68)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Peptide1</th>\n",
       "      <th>Peptide2</th>\n",
       "      <th>Peptide3</th>\n",
       "      <th>Peptide4</th>\n",
       "      <th>Peptide5</th>\n",
       "      <th>Peptide6</th>\n",
       "      <th>Peptide7</th>\n",
       "      <th>Peptide8</th>\n",
       "      <th>Peptide9</th>\n",
       "      <th>Peptide10</th>\n",
       "      <th>...</th>\n",
       "      <th>TSH</th>\n",
       "      <th>fT3</th>\n",
       "      <th>fT4</th>\n",
       "      <th>Cortisol</th>\n",
       "      <th>Testosteron</th>\n",
       "      <th>HOMA-IR</th>\n",
       "      <th>Freefattyacids</th>\n",
       "      <th>RRsys</th>\n",
       "      <th>RRdia</th>\n",
       "      <th>ssCRP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>sample 2</th>\n",
       "      <td>33.58</td>\n",
       "      <td>7.18</td>\n",
       "      <td>9.35</td>\n",
       "      <td>3.57</td>\n",
       "      <td>94.44</td>\n",
       "      <td>14.91</td>\n",
       "      <td>153.05</td>\n",
       "      <td>35.52</td>\n",
       "      <td>9.76</td>\n",
       "      <td>71.20</td>\n",
       "      <td>...</td>\n",
       "      <td>0.72</td>\n",
       "      <td>5.23</td>\n",
       "      <td>21.5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.2</td>\n",
       "      <td>0.30</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sample 3</th>\n",
       "      <td>37.57</td>\n",
       "      <td>8.70</td>\n",
       "      <td>10.79</td>\n",
       "      <td>3.36</td>\n",
       "      <td>94.11</td>\n",
       "      <td>15.99</td>\n",
       "      <td>198.88</td>\n",
       "      <td>39.65</td>\n",
       "      <td>8.62</td>\n",
       "      <td>75.48</td>\n",
       "      <td>...</td>\n",
       "      <td>0.97</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.3</td>\n",
       "      <td>1.04</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7.15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sample 4</th>\n",
       "      <td>27.31</td>\n",
       "      <td>5.42</td>\n",
       "      <td>5.64</td>\n",
       "      <td>2.75</td>\n",
       "      <td>67.01</td>\n",
       "      <td>11.91</td>\n",
       "      <td>148.32</td>\n",
       "      <td>28.90</td>\n",
       "      <td>5.64</td>\n",
       "      <td>59.26</td>\n",
       "      <td>...</td>\n",
       "      <td>1.02</td>\n",
       "      <td>5.01</td>\n",
       "      <td>17.6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.1</td>\n",
       "      <td>0.37</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sample 5</th>\n",
       "      <td>29.09</td>\n",
       "      <td>5.81</td>\n",
       "      <td>4.69</td>\n",
       "      <td>3.61</td>\n",
       "      <td>65.99</td>\n",
       "      <td>12.42</td>\n",
       "      <td>154.55</td>\n",
       "      <td>26.48</td>\n",
       "      <td>6.38</td>\n",
       "      <td>53.50</td>\n",
       "      <td>...</td>\n",
       "      <td>1.00</td>\n",
       "      <td>4.61</td>\n",
       "      <td>15.7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>24.5</td>\n",
       "      <td>0.99</td>\n",
       "      <td>168.0</td>\n",
       "      <td>95.0</td>\n",
       "      <td>0.20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sample 6</th>\n",
       "      <td>41.13</td>\n",
       "      <td>8.40</td>\n",
       "      <td>8.85</td>\n",
       "      <td>3.56</td>\n",
       "      <td>109.13</td>\n",
       "      <td>17.60</td>\n",
       "      <td>209.62</td>\n",
       "      <td>44.35</td>\n",
       "      <td>10.43</td>\n",
       "      <td>84.78</td>\n",
       "      <td>...</td>\n",
       "      <td>1.30</td>\n",
       "      <td>5.43</td>\n",
       "      <td>19.2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11.1</td>\n",
       "      <td>0.54</td>\n",
       "      <td>165.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>0.30</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 68 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          Peptide1  Peptide2  Peptide3  Peptide4  Peptide5  Peptide6  \\\n",
       "sample 2     33.58      7.18      9.35      3.57     94.44     14.91   \n",
       "sample 3     37.57      8.70     10.79      3.36     94.11     15.99   \n",
       "sample 4     27.31      5.42      5.64      2.75     67.01     11.91   \n",
       "sample 5     29.09      5.81      4.69      3.61     65.99     12.42   \n",
       "sample 6     41.13      8.40      8.85      3.56    109.13     17.60   \n",
       "\n",
       "          Peptide7  Peptide8  Peptide9  Peptide10  ...     TSH   fT3   fT4  \\\n",
       "sample 2    153.05     35.52      9.76      71.20  ...    0.72  5.23  21.5   \n",
       "sample 3    198.88     39.65      8.62      75.48  ...    0.97   NaN   NaN   \n",
       "sample 4    148.32     28.90      5.64      59.26  ...    1.02  5.01  17.6   \n",
       "sample 5    154.55     26.48      6.38      53.50  ...    1.00  4.61  15.7   \n",
       "sample 6    209.62     44.35     10.43      84.78  ...    1.30  5.43  19.2   \n",
       "\n",
       "          Cortisol  Testosteron  HOMA-IR  Freefattyacids  RRsys  RRdia  ssCRP  \n",
       "sample 2       NaN          NaN      7.2            0.30    NaN    NaN   0.48  \n",
       "sample 3       NaN          NaN      9.3            1.04    NaN    NaN   7.15  \n",
       "sample 4       NaN          NaN      9.1            0.37    NaN    NaN    NaN  \n",
       "sample 5       NaN          NaN     24.5            0.99  168.0   95.0   0.20  \n",
       "sample 6       NaN          NaN     11.1            0.54  165.0   84.0   0.30  \n",
       "\n",
       "[5 rows x 68 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The WHR column has some spaces, which causes the read_csv function not able to read it as float\n",
    "\n",
    "This situation is very common in data science where users didn't provide clean dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Peptide1</th>\n",
       "      <th>Peptide2</th>\n",
       "      <th>Peptide3</th>\n",
       "      <th>Peptide4</th>\n",
       "      <th>Peptide5</th>\n",
       "      <th>Peptide6</th>\n",
       "      <th>Peptide7</th>\n",
       "      <th>Peptide8</th>\n",
       "      <th>Peptide9</th>\n",
       "      <th>Peptide10</th>\n",
       "      <th>...</th>\n",
       "      <th>TSH</th>\n",
       "      <th>fT3</th>\n",
       "      <th>fT4</th>\n",
       "      <th>Cortisol</th>\n",
       "      <th>Testosteron</th>\n",
       "      <th>HOMA-IR</th>\n",
       "      <th>Freefattyacids</th>\n",
       "      <th>RRsys</th>\n",
       "      <th>RRdia</th>\n",
       "      <th>ssCRP</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>sample 2</th>\n",
       "      <td>33.58</td>\n",
       "      <td>7.18</td>\n",
       "      <td>9.35</td>\n",
       "      <td>3.57</td>\n",
       "      <td>94.44</td>\n",
       "      <td>14.91</td>\n",
       "      <td>153.05</td>\n",
       "      <td>35.52</td>\n",
       "      <td>9.76</td>\n",
       "      <td>71.20</td>\n",
       "      <td>...</td>\n",
       "      <td>0.72</td>\n",
       "      <td>5.230000</td>\n",
       "      <td>21.500000</td>\n",
       "      <td>256.9552</td>\n",
       "      <td>38.047609</td>\n",
       "      <td>7.2</td>\n",
       "      <td>0.30</td>\n",
       "      <td>133.980392</td>\n",
       "      <td>84.862745</td>\n",
       "      <td>0.480000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sample 3</th>\n",
       "      <td>37.57</td>\n",
       "      <td>8.70</td>\n",
       "      <td>10.79</td>\n",
       "      <td>3.36</td>\n",
       "      <td>94.11</td>\n",
       "      <td>15.99</td>\n",
       "      <td>198.88</td>\n",
       "      <td>39.65</td>\n",
       "      <td>8.62</td>\n",
       "      <td>75.48</td>\n",
       "      <td>...</td>\n",
       "      <td>0.97</td>\n",
       "      <td>5.151061</td>\n",
       "      <td>16.621212</td>\n",
       "      <td>256.9552</td>\n",
       "      <td>38.047609</td>\n",
       "      <td>9.3</td>\n",
       "      <td>1.04</td>\n",
       "      <td>133.980392</td>\n",
       "      <td>84.862745</td>\n",
       "      <td>7.150000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sample 4</th>\n",
       "      <td>27.31</td>\n",
       "      <td>5.42</td>\n",
       "      <td>5.64</td>\n",
       "      <td>2.75</td>\n",
       "      <td>67.01</td>\n",
       "      <td>11.91</td>\n",
       "      <td>148.32</td>\n",
       "      <td>28.90</td>\n",
       "      <td>5.64</td>\n",
       "      <td>59.26</td>\n",
       "      <td>...</td>\n",
       "      <td>1.02</td>\n",
       "      <td>5.010000</td>\n",
       "      <td>17.600000</td>\n",
       "      <td>256.9552</td>\n",
       "      <td>38.047609</td>\n",
       "      <td>9.1</td>\n",
       "      <td>0.37</td>\n",
       "      <td>133.980392</td>\n",
       "      <td>84.862745</td>\n",
       "      <td>2.520263</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sample 5</th>\n",
       "      <td>29.09</td>\n",
       "      <td>5.81</td>\n",
       "      <td>4.69</td>\n",
       "      <td>3.61</td>\n",
       "      <td>65.99</td>\n",
       "      <td>12.42</td>\n",
       "      <td>154.55</td>\n",
       "      <td>26.48</td>\n",
       "      <td>6.38</td>\n",
       "      <td>53.50</td>\n",
       "      <td>...</td>\n",
       "      <td>1.00</td>\n",
       "      <td>4.610000</td>\n",
       "      <td>15.700000</td>\n",
       "      <td>256.9552</td>\n",
       "      <td>38.047609</td>\n",
       "      <td>24.5</td>\n",
       "      <td>0.99</td>\n",
       "      <td>168.000000</td>\n",
       "      <td>95.000000</td>\n",
       "      <td>0.200000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>sample 6</th>\n",
       "      <td>41.13</td>\n",
       "      <td>8.40</td>\n",
       "      <td>8.85</td>\n",
       "      <td>3.56</td>\n",
       "      <td>109.13</td>\n",
       "      <td>17.60</td>\n",
       "      <td>209.62</td>\n",
       "      <td>44.35</td>\n",
       "      <td>10.43</td>\n",
       "      <td>84.78</td>\n",
       "      <td>...</td>\n",
       "      <td>1.30</td>\n",
       "      <td>5.430000</td>\n",
       "      <td>19.200000</td>\n",
       "      <td>256.9552</td>\n",
       "      <td>38.047609</td>\n",
       "      <td>11.1</td>\n",
       "      <td>0.54</td>\n",
       "      <td>165.000000</td>\n",
       "      <td>84.000000</td>\n",
       "      <td>0.300000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 68 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          Peptide1  Peptide2  Peptide3  Peptide4  Peptide5  Peptide6  \\\n",
       "sample 2     33.58      7.18      9.35      3.57     94.44     14.91   \n",
       "sample 3     37.57      8.70     10.79      3.36     94.11     15.99   \n",
       "sample 4     27.31      5.42      5.64      2.75     67.01     11.91   \n",
       "sample 5     29.09      5.81      4.69      3.61     65.99     12.42   \n",
       "sample 6     41.13      8.40      8.85      3.56    109.13     17.60   \n",
       "\n",
       "          Peptide7  Peptide8  Peptide9  Peptide10    ...      TSH       fT3  \\\n",
       "sample 2    153.05     35.52      9.76      71.20    ...     0.72  5.230000   \n",
       "sample 3    198.88     39.65      8.62      75.48    ...     0.97  5.151061   \n",
       "sample 4    148.32     28.90      5.64      59.26    ...     1.02  5.010000   \n",
       "sample 5    154.55     26.48      6.38      53.50    ...     1.00  4.610000   \n",
       "sample 6    209.62     44.35     10.43      84.78    ...     1.30  5.430000   \n",
       "\n",
       "                fT4  Cortisol  Testosteron  HOMA-IR  Freefattyacids  \\\n",
       "sample 2  21.500000  256.9552    38.047609      7.2            0.30   \n",
       "sample 3  16.621212  256.9552    38.047609      9.3            1.04   \n",
       "sample 4  17.600000  256.9552    38.047609      9.1            0.37   \n",
       "sample 5  15.700000  256.9552    38.047609     24.5            0.99   \n",
       "sample 6  19.200000  256.9552    38.047609     11.1            0.54   \n",
       "\n",
       "               RRsys      RRdia     ssCRP  \n",
       "sample 2  133.980392  84.862745  0.480000  \n",
       "sample 3  133.980392  84.862745  7.150000  \n",
       "sample 4  133.980392  84.862745  2.520263  \n",
       "sample 5  168.000000  95.000000  0.200000  \n",
       "sample 6  165.000000  84.000000  0.300000  \n",
       "\n",
       "[5 rows x 68 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "df['WHR'] = pd.to_numeric(df['WHR'].str.replace(' ',''))\n",
    "df = df.fillna(df.mean())\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "All the gender column is male, not informative, therefore we removed it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df.drop('Gender',axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(96, 67)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classification\n",
    "\n",
    "## Concepts\n",
    "\n",
    "### Target variable\n",
    "\n",
    "Target variable is the label, basically, you want to build a model so that it can predict the target value.\n",
    "\n",
    "This variable is often times refered as 'Y'.\n",
    "\n",
    "### Feature variables\n",
    "\n",
    "Feature variables are the measurements. Basically, you want to use these features to predict the target value.\n",
    "\n",
    "The features together, is refered as 'X'."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sample 2      1\n",
       "sample 3      1\n",
       "sample 4      1\n",
       "sample 5      1\n",
       "sample 6      1\n",
       "sample 7      1\n",
       "sample 8      1\n",
       "sample 9      0\n",
       "sample 10     1\n",
       "sample 11     0\n",
       "Sample 12     1\n",
       "Sample 13     1\n",
       "Sample 14     1\n",
       "Sample 15     0\n",
       "Sample 16     1\n",
       "Sample 17     1\n",
       "Sample 18     1\n",
       "Sample 19     1\n",
       "Sample 20     1\n",
       "Sample 21     1\n",
       "Sample 22     1\n",
       "Sample 23     0\n",
       "Sample 24     1\n",
       "Sample 25     1\n",
       "Sample 26     0\n",
       "Sample 27     0\n",
       "Sample 28     1\n",
       "Sample 29     1\n",
       "Sample 30     1\n",
       "Sample 31     0\n",
       "             ..\n",
       "Sample 71     1\n",
       "Sample 72     1\n",
       "Sample 73     1\n",
       "Sample 74     1\n",
       "Sample 75     1\n",
       "Sample 76     1\n",
       "Sample 77     1\n",
       "Sample 78     1\n",
       "Sample 79     0\n",
       "Sample 80     0\n",
       "Sample 81     1\n",
       "Sample 82     1\n",
       "Sample 83     1\n",
       "Sample 84     0\n",
       "Sample 85     0\n",
       "Sample 86     0\n",
       "Sample 87     0\n",
       "Sample 88     0\n",
       "Sample 89     0\n",
       "Sample 90     0\n",
       "Sample 91     1\n",
       "Sample 92     1\n",
       "Sample 93     0\n",
       "Sample 94     0\n",
       "Sample 95     0\n",
       "Sample 96     1\n",
       "Sample 97     0\n",
       "Sample 98     0\n",
       "Sample 99     1\n",
       "Sample 100    0\n",
       "Name: Diagnosis, Length: 96, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mapping = {} # the key value will be replace to 1 or 0\n",
    "mapping['T2D'] = 1\n",
    "mapping['NGT'] = 0\n",
    "target = df.Diagnosis.replace(mapping) # here I'm using target, instead of Y, just for ease of understanding. I don't do this usually.\n",
    "target"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "X should not contain the target values, so let's remove that column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df.drop('Diagnosis',axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## K-fold cross validation\n",
    "\n",
    "### K-fold\n",
    "\n",
    "Divide your data into k pieces\n",
    "\n",
    "### cross validation\n",
    "\n",
    "Use (k-1) folds to train and use the remaining 1 fold to test\n",
    "\n",
    "Therefore you will have a list of accuracies. In the end, you tell us the average.\n",
    "\n",
    "This procedure is called cross validation, or CV.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.845"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "import numpy as np\n",
    "acc_list = []\n",
    "se_list = []\n",
    "sp_list = []\n",
    "for train_index, test_index in StratifiedKFold(target, n_folds=10,shuffle=True):\n",
    "    train_x = X.iloc[train_index,:]\n",
    "    train_y = target[train_index]\n",
    "    test_x = X.iloc[test_index,:]\n",
    "    test_y = target[test_index]\n",
    "    dt_clf=DTC()\n",
    "    dt_clf.fit(train_x, train_y)\n",
    "    y_pred_dt = dt_clf.predict(test_x)\n",
    "    tn, fp, fn, tp = confusion_matrix(test_y, y_pred_dt).ravel()\n",
    "    accuracy = (tp+tn)/(tn+tp+fn+fp)\n",
    "    sensitivity = tp/(fn+tp)\n",
    "    specificity = tn/(tn+fp)\n",
    "    acc_list.append(accuracy)\n",
    "    se_list.append(sensitivity)\n",
    "    sp_list.append(specificity) \n",
    "    \n",
    "np.mean(acc_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This above accuracy is not good, this is because the parameters are not good.\n",
    "\n",
    "Parameter tuning is often the most difficult part in machine learning. There are more than 1 answers, or there's no answer at all."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Right now, you can just use the best parameter I found"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.96"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "acc_list = []\n",
    "se_list = []\n",
    "sp_list = []\n",
    "subX = X[['Peptide27','HbA1c']]\n",
    "for train_index, test_index in StratifiedKFold(target, n_folds=10,shuffle=True):\n",
    "    train_x = subX.iloc[train_index,:]\n",
    "    train_y = target[train_index]\n",
    "    test_x = subX.iloc[test_index,:]\n",
    "    test_y = target[test_index]\n",
    "    dt_clf=DTC(max_depth = 2)\n",
    "    dt_clf.fit(train_x, train_y)\n",
    "    y_pred_dt = dt_clf.predict(test_x)\n",
    "    tn, fp, fn, tp = confusion_matrix(test_y, y_pred_dt).ravel()\n",
    "    accuracy = (tp+tn)/(tn+tp+fn+fp)\n",
    "    sensitivity = tp/(fn+tp)\n",
    "    specificity = tn/(tn+fp)\n",
    "    acc_list.append(accuracy)\n",
    "    se_list.append(sensitivity)\n",
    "    sp_list.append(specificity)  \n",
    "np.mean(acc_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### install graphviz to visulize your tree\n",
    "\n",
    "conda install graphviz\n",
    "\n",
    "conda install python-graphviz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<svg height=\"269pt\" viewBox=\"0.00 0.00 296.00 269.00\" width=\"296pt\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g class=\"graph\" id=\"graph0\" transform=\"scale(1 1) rotate(0) translate(4 265)\">\n",
       "<title>Tree</title>\n",
       "<polygon fill=\"white\" points=\"-4,4 -4,-265 292,-265 292,4 -4,4\" stroke=\"none\"/>\n",
       "<!-- 0 -->\n",
       "<g class=\"node\" id=\"node1\"><title>0</title>\n",
       "<polygon fill=\"none\" points=\"224.5,-261 119.5,-261 119.5,-193 224.5,-193 224.5,-261\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"172\" y=\"-245.8\">HbA1c &lt;= 6.05</text>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"172\" y=\"-230.8\">gini = 0.5</text>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"172\" y=\"-215.8\">samples = 88</text>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"172\" y=\"-200.8\">value = [44, 44]</text>\n",
       "</g>\n",
       "<!-- 1 -->\n",
       "<g class=\"node\" id=\"node2\"><title>1</title>\n",
       "<polygon fill=\"none\" points=\"172,-157 38,-157 38,-89 172,-89 172,-157\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"105\" y=\"-141.8\">Peptide27 &lt;= 30.025</text>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"105\" y=\"-126.8\">gini = 0.302</text>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"105\" y=\"-111.8\">samples = 54</text>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"105\" y=\"-96.8\">value = [44, 10]</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;1 -->\n",
       "<g class=\"edge\" id=\"edge1\"><title>0-&gt;1</title>\n",
       "<path d=\"M150.247,-192.884C144.572,-184.243 138.381,-174.819 132.452,-165.793\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"135.289,-163.736 126.873,-157.299 129.438,-167.579 135.289,-163.736\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"121.803\" y=\"-178.08\">True</text>\n",
       "</g>\n",
       "<!-- 4 -->\n",
       "<g class=\"node\" id=\"node5\"><title>4</title>\n",
       "<polygon fill=\"none\" points=\"288,-149.5 190,-149.5 190,-96.5 288,-96.5 288,-149.5\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"239\" y=\"-134.3\">gini = 0.0</text>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"239\" y=\"-119.3\">samples = 34</text>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"239\" y=\"-104.3\">value = [0, 34]</text>\n",
       "</g>\n",
       "<!-- 0&#45;&gt;4 -->\n",
       "<g class=\"edge\" id=\"edge4\"><title>0-&gt;4</title>\n",
       "<path d=\"M193.753,-192.884C201.049,-181.776 209.197,-169.372 216.546,-158.184\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"219.672,-159.8 222.237,-149.52 213.821,-155.957 219.672,-159.8\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"227.307\" y=\"-170.3\">False</text>\n",
       "</g>\n",
       "<!-- 2 -->\n",
       "<g class=\"node\" id=\"node3\"><title>2</title>\n",
       "<polygon fill=\"none\" points=\"98,-53 0,-53 0,-0 98,-0 98,-53\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"49\" y=\"-37.8\">gini = 0.122</text>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"49\" y=\"-22.8\">samples = 46</text>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"49\" y=\"-7.8\">value = [43, 3]</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;2 -->\n",
       "<g class=\"edge\" id=\"edge2\"><title>1-&gt;2</title>\n",
       "<path d=\"M85.4139,-88.9485C80.2639,-80.2579 74.6953,-70.8608 69.4988,-62.0917\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"72.3605,-60.0553 64.2514,-53.2367 66.3384,-63.624 72.3605,-60.0553\" stroke=\"black\"/>\n",
       "</g>\n",
       "<!-- 3 -->\n",
       "<g class=\"node\" id=\"node4\"><title>3</title>\n",
       "<polygon fill=\"none\" points=\"207.5,-53 116.5,-53 116.5,-0 207.5,-0 207.5,-53\" stroke=\"black\"/>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"162\" y=\"-37.8\">gini = 0.219</text>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"162\" y=\"-22.8\">samples = 8</text>\n",
       "<text font-family=\"Times New Roman,serif\" font-size=\"14.00\" text-anchor=\"middle\" x=\"162\" y=\"-7.8\">value = [1, 7]</text>\n",
       "</g>\n",
       "<!-- 1&#45;&gt;3 -->\n",
       "<g class=\"edge\" id=\"edge3\"><title>1-&gt;3</title>\n",
       "<path d=\"M124.936,-88.9485C130.233,-80.1664 135.965,-70.6629 141.302,-61.815\" fill=\"none\" stroke=\"black\"/>\n",
       "<polygon fill=\"black\" points=\"144.308,-63.6074 146.476,-53.2367 138.314,-59.9919 144.308,-63.6074\" stroke=\"black\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>"
      ],
      "text/plain": [
       "<IPython.core.display.SVG object>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.tree import export_graphviz\n",
    "from IPython.display import SVG\n",
    "from graphviz import Source\n",
    "graph = Source(export_graphviz(dt_clf, out_file=None, feature_names=['Peptide27','HbA1c']))\n",
    "SVG(graph.pipe(format='svg'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}