Base_model_Sanjeev.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import scipy as sp\n",
    "from scipy.stats import mode\n",
    "from sklearn import linear_model\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn import preprocessing\n",
    "import sklearn as sk\n",
    "import sklearn.discriminant_analysis as da\n",
    "import sklearn.neighbors as knn\n",
    "from IPython.display import Markdown, display\n",
    "from sklearn.cross_validation import train_test_split\n",
    "from sklearn.preprocessing import Imputer\n",
    "from sklearn.linear_model import LinearRegression\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv('listings.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>scrape_id</th>\n",
       "      <th>last_scraped</th>\n",
       "      <th>name</th>\n",
       "      <th>picture_url</th>\n",
       "      <th>host_id</th>\n",
       "      <th>host_name</th>\n",
       "      <th>host_since</th>\n",
       "      <th>host_picture_url</th>\n",
       "      <th>street</th>\n",
       "      <th>...</th>\n",
       "      <th>last_review</th>\n",
       "      <th>review_scores_rating</th>\n",
       "      <th>review_scores_accuracy</th>\n",
       "      <th>review_scores_cleanliness</th>\n",
       "      <th>review_scores_checkin</th>\n",
       "      <th>review_scores_communication</th>\n",
       "      <th>review_scores_location</th>\n",
       "      <th>review_scores_value</th>\n",
       "      <th>host_listing_count</th>\n",
       "      <th>log_price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1069266</td>\n",
       "      <td>20150101184336</td>\n",
       "      <td>2015-01-02</td>\n",
       "      <td>Stay like a real New Yorker!</td>\n",
       "      <td>https://a0.muscache.com/pictures/50276484/larg...</td>\n",
       "      <td>5867023</td>\n",
       "      <td>Michael</td>\n",
       "      <td>2013-04-10</td>\n",
       "      <td>https://a2.muscache.com/ic/users/5867023/profi...</td>\n",
       "      <td>East 53rd Street, New York, NY 10022, United S...</td>\n",
       "      <td>...</td>\n",
       "      <td>2014-12-17</td>\n",
       "      <td>86.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1</td>\n",
       "      <td>5.075174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1846722</td>\n",
       "      <td>20150101184336</td>\n",
       "      <td>2015-01-02</td>\n",
       "      <td>Apartment 20 Minutes Times Square</td>\n",
       "      <td>https://a1.muscache.com/pictures/35865039/larg...</td>\n",
       "      <td>2631556</td>\n",
       "      <td>Denise</td>\n",
       "      <td>2012-06-13</td>\n",
       "      <td>https://a2.muscache.com/ic/users/2631556/profi...</td>\n",
       "      <td>West 155th Street, New York, NY, United States</td>\n",
       "      <td>...</td>\n",
       "      <td>2014-12-29</td>\n",
       "      <td>85.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>2</td>\n",
       "      <td>4.653960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2061725</td>\n",
       "      <td>20150101184336</td>\n",
       "      <td>2015-01-02</td>\n",
       "      <td>Option of 2 Beds w Private Bathroom</td>\n",
       "      <td>https://a2.muscache.com/pictures/50650147/larg...</td>\n",
       "      <td>4601412</td>\n",
       "      <td>Miao</td>\n",
       "      <td>2013-01-05</td>\n",
       "      <td>https://a0.muscache.com/ic/users/4601412/profi...</td>\n",
       "      <td>Van Buren Street, Brooklyn, NY 11221, United S...</td>\n",
       "      <td>...</td>\n",
       "      <td>2014-12-29</td>\n",
       "      <td>98.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>4</td>\n",
       "      <td>4.060443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>44974</td>\n",
       "      <td>20150101184336</td>\n",
       "      <td>2015-01-03</td>\n",
       "      <td>Charming Bright West Village Studio</td>\n",
       "      <td>https://a1.muscache.com/pictures/20489905/larg...</td>\n",
       "      <td>198425</td>\n",
       "      <td>Sara</td>\n",
       "      <td>2010-08-11</td>\n",
       "      <td>https://a0.muscache.com/ic/users/198425/profil...</td>\n",
       "      <td>Greenwich Ave, New York, NY 10011, United States</td>\n",
       "      <td>...</td>\n",
       "      <td>2014-10-30</td>\n",
       "      <td>96.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1</td>\n",
       "      <td>5.220356</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4701675</td>\n",
       "      <td>20150101184336</td>\n",
       "      <td>2015-01-02</td>\n",
       "      <td>Charming Apartment in Chelsea</td>\n",
       "      <td>https://a2.muscache.com/pictures/60588955/larg...</td>\n",
       "      <td>22590025</td>\n",
       "      <td>Charles</td>\n",
       "      <td>2014-10-15</td>\n",
       "      <td>https://a2.muscache.com/ic/users/22590025/prof...</td>\n",
       "      <td>West 22nd Street, New York, NY 10011, United S...</td>\n",
       "      <td>...</td>\n",
       "      <td>2014-12-08</td>\n",
       "      <td>100.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1</td>\n",
       "      <td>5.273000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 53 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        id       scrape_id last_scraped                                 name  \\\n",
       "0  1069266  20150101184336   2015-01-02        Stay like a real New Yorker!    \n",
       "1  1846722  20150101184336   2015-01-02    Apartment 20 Minutes Times Square   \n",
       "2  2061725  20150101184336   2015-01-02  Option of 2 Beds w Private Bathroom   \n",
       "3    44974  20150101184336   2015-01-03  Charming Bright West Village Studio   \n",
       "4  4701675  20150101184336   2015-01-02        Charming Apartment in Chelsea   \n",
       "\n",
       "                                         picture_url   host_id host_name  \\\n",
       "0  https://a0.muscache.com/pictures/50276484/larg...   5867023   Michael   \n",
       "1  https://a1.muscache.com/pictures/35865039/larg...   2631556    Denise   \n",
       "2  https://a2.muscache.com/pictures/50650147/larg...   4601412      Miao   \n",
       "3  https://a1.muscache.com/pictures/20489905/larg...    198425      Sara   \n",
       "4  https://a2.muscache.com/pictures/60588955/larg...  22590025   Charles   \n",
       "\n",
       "   host_since                                   host_picture_url  \\\n",
       "0  2013-04-10  https://a2.muscache.com/ic/users/5867023/profi...   \n",
       "1  2012-06-13  https://a2.muscache.com/ic/users/2631556/profi...   \n",
       "2  2013-01-05  https://a0.muscache.com/ic/users/4601412/profi...   \n",
       "3  2010-08-11  https://a0.muscache.com/ic/users/198425/profil...   \n",
       "4  2014-10-15  https://a2.muscache.com/ic/users/22590025/prof...   \n",
       "\n",
       "                                              street    ...    last_review  \\\n",
       "0  East 53rd Street, New York, NY 10022, United S...    ...     2014-12-17   \n",
       "1     West 155th Street, New York, NY, United States    ...     2014-12-29   \n",
       "2  Van Buren Street, Brooklyn, NY 11221, United S...    ...     2014-12-29   \n",
       "3   Greenwich Ave, New York, NY 10011, United States    ...     2014-10-30   \n",
       "4  West 22nd Street, New York, NY 10011, United S...    ...     2014-12-08   \n",
       "\n",
       "  review_scores_rating review_scores_accuracy review_scores_cleanliness  \\\n",
       "0                 86.0                    9.0                       7.0   \n",
       "1                 85.0                    8.0                       8.0   \n",
       "2                 98.0                   10.0                      10.0   \n",
       "3                 96.0                   10.0                       9.0   \n",
       "4                100.0                   10.0                      10.0   \n",
       "\n",
       "  review_scores_checkin review_scores_communication review_scores_location  \\\n",
       "0                   9.0                         9.0                   10.0   \n",
       "1                   9.0                         8.0                    7.0   \n",
       "2                  10.0                        10.0                    9.0   \n",
       "3                  10.0                        10.0                   10.0   \n",
       "4                  10.0                        10.0                   10.0   \n",
       "\n",
       "   review_scores_value  host_listing_count log_price  \n",
       "0                  9.0                   1  5.075174  \n",
       "1                  8.0                   2  4.653960  \n",
       "2                 10.0                   4  4.060443  \n",
       "3                  9.0                   1  5.220356  \n",
       "4                 10.0                   1  5.273000  \n",
       "\n",
       "[5 rows x 53 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "Index([u'id', u'scrape_id', u'last_scraped', u'name', u'picture_url',\n",
       "       u'host_id', u'host_name', u'host_since', u'host_picture_url', u'street',\n",
       "       u'neighbourhood', u'neighbourhood_cleansed', u'city', u'state',\n",
       "       u'zipcode', u'market', u'country', u'latitude', u'longitude',\n",
       "       u'is_location_exact', u'property_type', u'room_type', u'accommodates',\n",
       "       u'bathrooms', u'bedrooms', u'beds', u'bed_type', u'square_feet',\n",
       "       u'price', u'weekly_price', u'monthly_price', u'guests_included',\n",
       "       u'extra_people', u'minimum_nights', u'maximum_nights',\n",
       "       u'calendar_updated', u'availability_30', u'availability_60',\n",
       "       u'availability_90', u'availability_365', u'calendar_last_scraped',\n",
       "       u'number_of_reviews', u'first_review', u'last_review',\n",
       "       u'review_scores_rating', u'review_scores_accuracy',\n",
       "       u'review_scores_cleanliness', u'review_scores_checkin',\n",
       "       u'review_scores_communication', u'review_scores_location',\n",
       "       u'review_scores_value', u'host_listing_count', u'log_price'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['price'] = df['price'].replace('[\\$,)]','', regex=True).replace('[(]','-', regex=True).astype(float)\n",
    "df['log_price'] = np.log(df['price'].values)\n",
    "display(df.head())\n",
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/markdown": [
       "Looks like a normal distribution"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEACAYAAACznAEdAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGNNJREFUeJzt3H+wXGV9x/H3B9KAKHCjLbk1AS5WIonV3kYNtuhwW5Qf\n2klop41Bq97CtFPBgnbGJqF/RP8Rwow1WoUZK80NDsiPqAMdYwiZZHXsiEQlgiZCrJOQBHOtImkd\nZyjBb//Y55Il5MfunnvPs0/285rZyZ4n59znk5uz+93zfHdXEYGZmdkJuQOYmVlvcEEwMzPABcHM\nzBIXBDMzA1wQzMwscUEwMzOgjYIg6VZJ45IeaRm7SdJ2SVslfUnSaS1/t1zSjvT3F7eMz5f0iKTH\nJa1qGZ8u6c50zLcknTWZ/0AzM2tPO1cIq4FLDhnbALw2IoaBHcByAEnzgMXAXOAy4GZJSsfcAlwV\nEXOAOZImfuZVwFMRcS6wCripwr/HzMy6dMyCEBHfBH55yNjGiPhN2nwQmJ3uLwTujIgDEbGTZrFY\nIGkQODUitqT9bgMuT/cXAWvS/bXARV3+W8zMrILJ6CFcCaxL92cBu1v+bm8amwXsaRnfk8ZecExE\nPAc8Lenlk5DLzMw6UKkgSPpn4NmI+OIk5QHQsXcxM7PJNq3bAyWNAu8A/rRleC9wZsv27DR2pPHW\nY56UdCJwWkQ8dYQ5/cVLZmZdiIhjvthu9wpBtLxyl3Qp8BFgYUQ807LffcCS9M6hc4BXAw9FxD5g\nv6QFqcn8PuDelmPen+7/FbDpaEEiotjbihUrsmdw/vw5+i278+e/teuYVwiS7gBGgFdIegJYAVwP\nTAceSG8iejAiro6IbZLuBrYBzwJXx8E01wBjwMnAuohYn8ZvBb4gaQfwC2BJ2+kLs3PnztwRKnH+\nfErODs5fimMWhIh492GGVx9l/xuAGw4z/l3gdYcZf4bmW1XNzCwjf1K5RqOjo7kjVOL8+ZScHZy/\nFOpkfSk3SVFSXjOzXiCJmMSmsk2CRqORO0Ilzp9PydnB+UvhgmBmZoCXjMzMjnteMjIzs464INSo\n9HVI58+n5Ozg/KVwQTAzM8A9BDOz4557CGZm1hEXhBqVvg7p/PmUnB2cvxQuCGZmBriHYGZ23HMP\nwczMOuKCUKPS1yGdP5+Ss4Pzl8IFwczMAPcQzMyOe+4hmJlZR1wQalT6OqTz51NydnD+UrggmJkZ\n4B6Cmdlxzz0EMzPriAtCjUpfh3T+fErODs5fChcEMzMD3EMwMzvuuYdgPWNwcAhJ2W6Dg0O5fwVm\nRXBBqFHp65Dd5h8f3wVEtltz/rJ//yVnB+cvhQuCmZkBbfQQJN0K/BkwHhGvT2MzgLuAs4GdwOKI\n2J/+bjlwJXAAuC4iNqTx+cAYcDKwLiI+lManA7cBbwB+DrwrIp44Qhb3EAokiear9WwJ8Hlj/Wwy\newirgUsOGVsGbIyI1wCbgOVp0nnAYmAucBlws5rPBgC3AFdFxBxgjqSJn3kV8FREnAusAm5qI5OZ\nmU2yYxaEiPgm8MtDhhcBa9L9NcDl6f5C4M6IOBARO4EdwAJJg8CpEbEl7XdbyzGtP2stcFEX/44i\nlL4O6fz5lJwdnL8U3fYQzoiIcYCI2AeckcZnAbtb9tubxmYBe1rG96SxFxwTEc8BT0t6eZe5zMys\nS5PVVJ7MBdpjrnOVamRkJHeESpw/n5Kzg/OXYlqXx41LmhkR42k56GdpfC9wZst+s9PYkcZbj3lS\n0onAaRHx1JEmHh0dZWhoCICBgQGGh4ef/8+auKzzdm9tHzSxPVLzNh3l9ba3S99uNBqMjY0BPP98\n2ZaIOOYNGAIebdleCSxN95cCN6b784CHgenAOcCPOfhOpgeBBTSvANYBl6bxq4Gb0/0lNHsQR8oR\nJdu8eXPuCJV0mx8IiIw3KuXvBSVnj3D+3NJj4JjP9ce8QpB0B82XXK+Q9ASwArgRuEfSlcAumu8s\nIiK2Sbob2AY8C1ydwgBcwwvfdro+jd8KfEHSDuAXqSiYmVnN/F1GNuX8OQSzvPxdRmZm1hEXhBq9\nuMlalnLzn1T8l+uV+7tvcv4yuCBYH3iG5pLVZnJ+uZ5Zr3MPwaZcL/QQcs/v89Zycg/BzMw64oJQ\no9LXIUvPf+gH1UpS+u/e+cvggmBmZoB7CFYD9xDcQ7C83EMwM7OOuCDUqPR1yNLzu4eQj/OXwQXB\nzMwA9xCsBu4huIdgebmHYGZmHXFBqFHp65Cl53cPIR/nL4MLgpmZAe4hWA3cQ3APwfJyD8HMzDri\nglCj0tchS8/vHkI+zl8GFwQzMwPcQ7AauIfgHoLl5R6CmZl1xAWhRqWvQ5ae3z2EfJy/DC4IZmYG\nuIdgNXAPwT0Ey8s9BDMz64gLQo1KX4csPb97CPk4fxlcEMzMDHAPwWrgHoJ7CJaXewhmZtaRSgVB\n0ocl/UDSI5JulzRd0gxJGyQ9Jul+Sae37L9c0g5J2yVd3DI+P/2MxyWtqpKpl5W+Dll6fvcQ8nH+\nMnRdECS9EvgHYH5EvB6YBlwBLAM2RsRrgE3A8rT/PGAxMBe4DLhZzbUEgFuAqyJiDjBH0iXd5jIz\ns+503UNIBeFbwDDwv8CXgU8DnwEujIhxSYNAIyLOk7QMiIhYmY7/GvBRYBewKSLmpfEl6fgPHGZO\n9xAK5B6CewiW15T3ECLiSeATwBPAXmB/RGwEZkbEeNpnH3BGOmQWsLvlR+xNY7OAPS3je9KYmZnV\naFq3B0oaABYBZwP7gXskvYcXvxSb1JdGo6OjDA0NATAwMMDw8DAjIyPAwXW+Xt1etWpVUXknK/9B\nE9sjNW9PWEXzgjbP/FV+/62/y145H5y/d/IdLu/Y2BjA88+XbYmIrm7AXwL/1rL9XuCzwHaaVwkA\ng8D2dH8ZsLRl//XA+a37pPElwC1HmDNKtnnz5twRKuk2PxAQGW8T82/ONn+u332vcP680jl4zOf1\nKj2EBcCtwJuAZ4DVwBbgLOCpiFgpaSkwIyKWpaby7akIzAIeAM6NiJD0IHBtOv6rwKcjYv1h5oxu\n81o+7iG4h2B5tdtD6HrJKCIekrQWeBh4Nv35OeBU4G5JV9JsGC9O+2+TdDewLe1/dcuz+zXAGHAy\nsO5wxcDMzKaWP6lco0aj8fx6X4m6zd87VwgNDq7v1zt/1fO2X8+dXlF6fn9S2czMOuIrBJtyvXOF\nkG9+n7eWk68QzMysIy4INXrx+/LLUnp+f5dRPs5fBhcEMzMD3EOwGriH4B6C5eUegpmZdcQFoUal\nr0OWnt89hHycvwwuCGZmBriHYDVwD8E9BMvLPQQzM+uIC0KNSl+HLD2/ewj5OH8ZXBDMzAxwD8Fq\n4B6CewiWl3sIZmbWEReEGpW+Dll6fvcQ8nH+MrggmJkZ4B6C1cA9BPcQLC/3EMzMrCMuCDUqfR2y\n9PzuIeTj/GVwQTAzM8A9BKuBewjuIVhe7iGYmVlHXBBqVPo6ZOn53UPIx/nL4IJgZmaAewhWA/cQ\n3EOwvNxDMDOzjrgg1Kj0dcjS87uHkI/zl6FSQZB0uqR7JG2X9ENJ50uaIWmDpMck3S/p9Jb9l0va\nkfa/uGV8vqRHJD0uaVWVTGZm1p1KPQRJY8DXI2K1pGnAS4HrgV9ExE2SlgIzImKZpHnA7cCbgNnA\nRuDciAhJ3wY+GBFbJK0DPhUR9x9mPvcQCuQegnsIlteU9xAknQa8NSJWA0TEgYjYDywC1qTd1gCX\np/sLgTvTfjuBHcACSYPAqRGxJe13W8sxZmZWkypLRucAP5e0WtL3JH1O0inAzIgYB4iIfcAZaf9Z\nwO6W4/emsVnAnpbxPWnsuFP6OmTp+d1DyMf5yzCt4rHzgWsi4juSPgks48XX5pN6rTw6OsrQ0BAA\nAwMDDA8PMzIyAhz8T+vV7a1bt/ZUnrryHzSxPVLz9oStWefP/f/n7f7ZbjQajI2NATz/fNmOrnsI\nkmYC34qIV6Xtt9AsCL8HjETEeFoO2hwRcyUtAyIiVqb91wMrgF0T+6TxJcCFEfGBw8zpHkKB3ENw\nD8HymvIeQloW2i1pThq6CPghcB8wmsbeD9yb7t8HLJE0XdI5wKuBh9Ky0n5JC9R85nhfyzFmZlaT\nqp9DuBa4XdJW4A+AjwMrgbdLeoxmkbgRICK2AXcD24B1wNUtL/evAW4FHgd2RMT6irl60ouXUMpS\nen73EPJx/jJU6SEQEd+n+TbSQ73tCPvfANxwmPHvAq+rksXMzKrxdxnZlHMPwT0Ey8vfZWRmZh1x\nQahR6euQped3DyEf5y+DC4KZmQHuIVgN3ENwD8Hycg/BzMw64oJQo9LXIUvP7x5CPs5fBhcEMzMD\n3EOwGriH4B6C5eUegpmZdcQFoUalr0OWnt89hHycvwwuCGZmBriHYDVwD8E9BMvLPQQzM+uIC0KN\nSl+HLD2/ewj5OH8ZXBDMzAxwD8Fq4B6CewiWl3sIZmbWEReEGpW+Dll6fvcQ8nH+MrggmJkZ4B6C\n1cA9BPcQLC/3EMzMrCMuCDUqfR2y9PzuIeTj/GVwQTAzM8A9BKuBewjuIVhe7iGYmVlHXBBqVPo6\nZOn53UPIx/nL4IJgZmbAJPQQJJ0AfAfYExELJc0A7gLOBnYCiyNif9p3OXAlcAC4LiI2pPH5wBhw\nMrAuIj50hLncQyiQewjuIVhedfYQrgO2tWwvAzZGxGuATcDyFGgesBiYC1wG3KzmMwXALcBVETEH\nmCPpkknIZcng4BCSst3MrAyVCoKk2cA7gM+3DC8C1qT7a4DL0/2FwJ0RcSAidgI7gAWSBoFTI2JL\n2u+2lmOOK7nWIcfHd9F8hVz1trnL43pFI3eArpW+hu38Zah6hfBJ4CO88FE/MyLGASJiH3BGGp8F\n7G7Zb28amwXsaRnfk8bMzKxG07o9UNI7gfGI2Cpp5Ci7TupLxNHRUYaGhgAYGBhgeHiYkZHm9BNV\nvFe3J8bqnv+gie2RLrcnxjo9frLm73b70Ax55q/y/zkyMpL9/HX+3slzrO1Go8HY2BjA88+X7ei6\nqSzp48Bf02wQvwQ4FfgK8EZgJCLG03LQ5oiYK2kZEBGxMh2/HlgB7JrYJ40vAS6MiA8cZk43lbvg\npm7++X3eWk5T3lSOiOsj4qyIeBWwBNgUEe8F/gMYTbu9H7g33b8PWCJpuqRzgFcDD6Vlpf2SFqQm\n8/tajjmulL8O2cgdoKJG7gBdK/3ccf4ydL1kdBQ3AndLupLmq//FABGxTdLdNN+R9CxwdcvL/Wt4\n4dtO109BLjMzOwp/l1Ef8JJR/vl93lpO/i4jMzPriAtCjcpfh2zkDlBRI3eArpV+7jh/GVwQzMwM\ncA+hL7iHkH9+n7eWk3sIZmbWEReEGpW/DtnIHaCiRu4AXSv93HH+MrggmJkZ4B5CX3APIf/8Pm8t\nJ/cQzMysIy4INSp/HbKRO0BFjdwBulb6ueP8ZXBBMDMzwD2EvuAeQv75fd5aTu4hmJlZR1wQalT+\nOmQjd4CKGrkDdK30c8f5y+CCYGZmgHsIfcE9hPzz+7y1nNxDMDOzjrgg1Kj8dchG7gAVNXIH6Frp\n547zl8EFwczMAPcQ+oJ7CPnn93lrObmHYGZmHXFBqFH565CN3AEqauQO0LXSzx3nL4MLgpmZAe4h\n9AX3EPLP7/PWcnIPwczMOuKCUKPy1yEbuQNU1MgdoGulnzvOXwYXBDMzA9xD6AvuIeSf3+et5TTl\nPQRJsyVtkvRDSY9KujaNz5C0QdJjku6XdHrLMcsl7ZC0XdLFLePzJT0i6XFJq7rNZGZm3auyZHQA\n+MeIeC3wR8A1ks4DlgEbI+I1wCZgOYCkecBiYC5wGXCzmi9dAW4BroqIOcAcSZdUyNWzyl+HbOQO\nUFEjd4CulX7uOH8ZpnV7YETsA/al+7+StB2YDSwCLky7raH5KFwGLATujIgDwE5JO4AFknYBp0bE\nlnTMbcDlwP3dZjPrLSdx8LVP/WbOPJt9+3Zmm9/KMSk9BElDNJ/4fx/YHREzWv7uqYh4uaR/Bb4V\nEXek8c8D64BdwA0RcXEafwvwTxGx8DDzuIfQBfcQPL8fN/2tts8hSHoZsBa4LiJ+xYvPfJ+JZmYF\n6HrJCEDSNJrF4AsRcW8aHpc0MyLGJQ0CP0vje4EzWw6fncaONH5Yo6OjDA0NATAwMMDw8DAjIyPA\nwXW+Xt1etWpVlrwHTWyPdLm9Chju4vjJmr/b7Qnd5p+s+av8vNaf1d38Oc//1nOxVx6Px3P+RqPB\n2NgYwPPPl22JiK5vNNf7/+WQsZXA0nR/KXBjuj8PeBiYDpwD/JiDS1YPAgtoXluvAy49wnxRss2b\nN2eZFwiISbht7vK4yZq/29vE/N3mn6z5c/zum/Pnluvcnyyl50/nwIueUw+9dd1DkHQB8A3g0eYJ\nTwDXAw8Bd9N81b8LWBwRT6djlgNXAc/SXGLakMbfAIwBJwPrIuK6I8wZ3ebtZ+4heH4/bvpbuz0E\nfzCtD7ggeH4/bvqbv9yuB5X/XuZG7gAVNXIHqKCRO0AlpZ/7pedvlwuCmZkBXjLqC14y8vx+3PQ3\nLxmZmVlHXBBqVP46ZCN3gIoauQNU0MgdoJLSz/3S87fLBcHMzAD3EPqCewie34+b/uYegpmZdcQF\noUblr0M2cgeoqJE7QAWN3AEqKf3cLz1/u1wQzMwMcA+hL7iH4Pn9uOlv7iGYmVlHXBBqVP46ZCN3\ngIoauQNU0MgdoJLSz/3S87fLBcHMzAD3EPqCewie34+b/uYegpmZdcQFoUblr0M2cgeoqJE7QAWN\n3AEqKf3cLz1/u1wQzMwMcA+hL7iH4Pn9uOlv7iGYmVlHXBBqVP46ZCN3gIoauQNU0MgdoJLSz/3S\n87fLBcHMzAD3EPqCewie34+b/uYegpmZdcQFoUblr0M2cgeoqJE7QAWN3AEqKf3cLz1/u1wQzI57\nJyEp221wcCj3L8Da5B5CTQYHhxgf35UxQX+voXt+9zD6Wbs9BBeEmuRt7OZ/QvD8/T1/qY/b40Vx\nTWVJl0r6kaTHJS3NnWdqNHIHqKiRO0BFjdwBKmjkDlBJ6WvwpedvV08UBEknAJ8BLgFeC1wh6by8\nqabC1twBKnL+fErODlu3On8JeqIgAAuAHRGxKyKeBe4EFmXONAWezh2gIufPp+Ts8PTTzl+CXikI\ns4DdLdt70piZFe8kPvaxj/ldTgXolYIwpd7+9kuyvu3uda97fUqyM+evYRLszB2gop25A1SwM3eA\nCp4B3k+zsV3/bXx8X+XHcJWCVlJB6ol3GUl6M/DRiLg0bS8DIiJWHrJf/rBmZgUq5m2nkk4EHgMu\nAn4KPARcERHbswYzM+sj03IHAIiI5yR9ENhAcxnrVhcDM7N69cQVgpmZ5VdEU1nSbEmbJP1Q0qOS\nrs2dqV2STpL0bUkPp+wrcmfqhqQTJH1P0n25s3RK0k5J30//Bw/lztMpSadLukfS9vQYOD93pnZJ\nmpN+799Lf+4v7PH7YUk/kPSIpNslTc+dqROSrkvPO209bxZxhSBpEBiMiK2SXgZ8F1gUET/KHK0t\nkk6JiF+nXsl/AtdGRFFPTJI+DLwBOC0iFubO0wlJPwHeEBG/zJ2lG5LGgK9HxGpJ04BTIuJ/Msfq\nmJofQN0DnB8Ru4+1f26SXgl8EzgvIv5P0l3AVyPitszR2iLptcAXgTcBB4CvAX8fET850jFFXCFE\nxL6I2Jru/wrYTkGfU4iIX6e7J9Hs2/R+FW4haTbwDuDzubN0SRRyrh9K0mnAWyNiNUBEHCixGCRv\nA/6rhGLQ4kTgpROFGHgyc55OzAW+HRHPRMRzwDeAvzjaAcU9SCQNAcPAt/MmaV9abnkY2Ac8EBFb\ncmfq0CeBj1BYIWsRwAOStkj629xhOnQO8HNJq9Oyy+ckvSR3qC69i+Yr1iJExJPAJ4AngL3A0xGx\nMW+qjvwAeKukGZJOofmi7syjHVBUQUjLRWuB69KVQhEi4jcR8YfAbOB8SfNyZ2qXpHcC4+kKTelW\nmgsiYj7NB8Q1kt6SO1AHpgHzgc+mf8OvgWV5I3VO0m8BC4F7cmdpl6QBml+hczbwSuBlkt6dN1X7\n0pL6SuABYB3wMPDc0Y4ppiCkS7a1wBci4t7cebqRLvU3A5fmztKBC4CFaR3+i8CfSCpiDXVCRPw0\n/fnfwFdofndWKfYAuyPiO2l7Lc0CUZrLgO+m/4NSvA34SUQ8lZZcvgz8ceZMHYmI1RHxxogYofmF\nWI8fbf9iCgLw78C2iPhU7iCdkPTbkk5P918CvB0oohkOEBHXR8RZEfEqYAmwKSLelztXuySdkq4s\nkfRS4GKal9JFiIhxYLekOWnoImBbxkjduoKClouSJ4A3SzpZkmj+7ov6fJSk30l/ngX8OXDH0fbv\niQ+mHYukC4D3AI+mtfgAro+I9XmTteV3gTXpHRYnAHdFxLrMmfrJTOAr6WtPpgG3R8SGzJk6dS1w\ne1p2+QnwN5nzdCStX78N+LvcWToREQ9JWktzqeXZ9Ofn8qbq2JckvZxm/quP9YaEIt52amZmU6+k\nJSMzM5tCLghmZga4IJiZWeKCYGZmgAuCmZklLghmZga4IJiZWeKCYGZmAPw/W7Mtr6rzJH8AAAAA\nSUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xb627048>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Lets plot a histogram of the log(price) \n",
    "df.log_price.hist()\n",
    "Markdown('Looks like a normal distribution')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "not_interesting_cols = ['id','last_scraped','state','scrape_id','name','picture_url', 'host_id', 'host_name', 'host_since', 'host_picture_url', 'street',\n",
    "       'neighbourhood', 'country', 'price', 'weekly_price', 'monthly_price',\n",
    "        'calendar_updated', 'availability_30', 'availability_60',\n",
    "       'availability_90', 'availability_365', 'calendar_last_scraped', 'last_review', 'first_review']\n",
    "\n",
    "data = df.drop(not_interesting_cols, axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Missing Values "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "List of Predictors and their respective percentages of missing values\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>DataType</th>\n",
       "      <th>NumberOfNulls</th>\n",
       "      <th>PercentageNulls</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>review_scores_value</td>\n",
       "      <td>float64</td>\n",
       "      <td>8734</td>\n",
       "      <td>31.885222</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>review_scores_location</td>\n",
       "      <td>float64</td>\n",
       "      <td>8732</td>\n",
       "      <td>31.877921</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>review_scores_cleanliness</td>\n",
       "      <td>float64</td>\n",
       "      <td>8731</td>\n",
       "      <td>31.874270</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>review_scores_communication</td>\n",
       "      <td>float64</td>\n",
       "      <td>8731</td>\n",
       "      <td>31.874270</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>review_scores_checkin</td>\n",
       "      <td>float64</td>\n",
       "      <td>8729</td>\n",
       "      <td>31.866968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>review_scores_accuracy</td>\n",
       "      <td>float64</td>\n",
       "      <td>8727</td>\n",
       "      <td>31.859667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>review_scores_rating</td>\n",
       "      <td>float64</td>\n",
       "      <td>8657</td>\n",
       "      <td>31.604118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>bathrooms</td>\n",
       "      <td>float64</td>\n",
       "      <td>463</td>\n",
       "      <td>1.690275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>zipcode</td>\n",
       "      <td>object</td>\n",
       "      <td>162</td>\n",
       "      <td>0.591414</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>bedrooms</td>\n",
       "      <td>float64</td>\n",
       "      <td>140</td>\n",
       "      <td>0.511098</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           Name DataType  NumberOfNulls  PercentageNulls\n",
       "26          review_scores_value  float64           8734        31.885222\n",
       "25       review_scores_location  float64           8732        31.877921\n",
       "22    review_scores_cleanliness  float64           8731        31.874270\n",
       "24  review_scores_communication  float64           8731        31.874270\n",
       "23        review_scores_checkin  float64           8729        31.866968\n",
       "21       review_scores_accuracy  float64           8727        31.859667\n",
       "20         review_scores_rating  float64           8657        31.604118\n",
       "10                    bathrooms  float64            463         1.690275\n",
       "2                       zipcode   object            162         0.591414\n",
       "11                     bedrooms  float64            140         0.511098"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>neighbourhood_cleansed</th>\n",
       "      <th>city</th>\n",
       "      <th>zipcode</th>\n",
       "      <th>market</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>is_location_exact</th>\n",
       "      <th>property_type</th>\n",
       "      <th>room_type</th>\n",
       "      <th>accommodates</th>\n",
       "      <th>...</th>\n",
       "      <th>number_of_reviews</th>\n",
       "      <th>review_scores_rating</th>\n",
       "      <th>review_scores_accuracy</th>\n",
       "      <th>review_scores_cleanliness</th>\n",
       "      <th>review_scores_checkin</th>\n",
       "      <th>review_scores_communication</th>\n",
       "      <th>review_scores_location</th>\n",
       "      <th>review_scores_value</th>\n",
       "      <th>host_listing_count</th>\n",
       "      <th>log_price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Midtown East</td>\n",
       "      <td>New York</td>\n",
       "      <td>10022</td>\n",
       "      <td>New York</td>\n",
       "      <td>40.756852</td>\n",
       "      <td>-73.964754</td>\n",
       "      <td>t</td>\n",
       "      <td>Apartment</td>\n",
       "      <td>Entire home/apt</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>62</td>\n",
       "      <td>86.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1</td>\n",
       "      <td>5.075174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Hamilton Heights</td>\n",
       "      <td>New York</td>\n",
       "      <td>NaN</td>\n",
       "      <td>New York</td>\n",
       "      <td>40.830599</td>\n",
       "      <td>-73.941014</td>\n",
       "      <td>f</td>\n",
       "      <td>Apartment</td>\n",
       "      <td>Entire home/apt</td>\n",
       "      <td>10</td>\n",
       "      <td>...</td>\n",
       "      <td>22</td>\n",
       "      <td>85.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>2</td>\n",
       "      <td>4.653960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Bushwick</td>\n",
       "      <td>Brooklyn</td>\n",
       "      <td>11221</td>\n",
       "      <td>New York</td>\n",
       "      <td>40.692189</td>\n",
       "      <td>-73.924120</td>\n",
       "      <td>t</td>\n",
       "      <td>Apartment</td>\n",
       "      <td>Private room</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>35</td>\n",
       "      <td>98.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>4</td>\n",
       "      <td>4.060443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>West Village</td>\n",
       "      <td>New York</td>\n",
       "      <td>10011</td>\n",
       "      <td>New York</td>\n",
       "      <td>40.734751</td>\n",
       "      <td>-74.002592</td>\n",
       "      <td>t</td>\n",
       "      <td>Apartment</td>\n",
       "      <td>Entire home/apt</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>26</td>\n",
       "      <td>96.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1</td>\n",
       "      <td>5.220356</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Chelsea</td>\n",
       "      <td>New York</td>\n",
       "      <td>10011</td>\n",
       "      <td>New York</td>\n",
       "      <td>40.745282</td>\n",
       "      <td>-73.997836</td>\n",
       "      <td>t</td>\n",
       "      <td>Apartment</td>\n",
       "      <td>Entire home/apt</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>100.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1</td>\n",
       "      <td>5.273000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  neighbourhood_cleansed      city zipcode    market   latitude  longitude  \\\n",
       "0           Midtown East  New York   10022  New York  40.756852 -73.964754   \n",
       "1       Hamilton Heights  New York     NaN  New York  40.830599 -73.941014   \n",
       "2               Bushwick  Brooklyn   11221  New York  40.692189 -73.924120   \n",
       "3           West Village  New York   10011  New York  40.734751 -74.002592   \n",
       "4                Chelsea  New York   10011  New York  40.745282 -73.997836   \n",
       "\n",
       "  is_location_exact property_type        room_type  accommodates    ...      \\\n",
       "0                 t     Apartment  Entire home/apt             2    ...       \n",
       "1                 f     Apartment  Entire home/apt            10    ...       \n",
       "2                 t     Apartment     Private room             2    ...       \n",
       "3                 t     Apartment  Entire home/apt             2    ...       \n",
       "4                 t     Apartment  Entire home/apt             2    ...       \n",
       "\n",
       "   number_of_reviews  review_scores_rating  review_scores_accuracy  \\\n",
       "0                 62                  86.0                     9.0   \n",
       "1                 22                  85.0                     8.0   \n",
       "2                 35                  98.0                    10.0   \n",
       "3                 26                  96.0                    10.0   \n",
       "4                  1                 100.0                    10.0   \n",
       "\n",
       "  review_scores_cleanliness  review_scores_checkin  \\\n",
       "0                       7.0                    9.0   \n",
       "1                       8.0                    9.0   \n",
       "2                      10.0                   10.0   \n",
       "3                       9.0                   10.0   \n",
       "4                      10.0                   10.0   \n",
       "\n",
       "  review_scores_communication  review_scores_location  review_scores_value  \\\n",
       "0                         9.0                    10.0                  9.0   \n",
       "1                         8.0                     7.0                  8.0   \n",
       "2                        10.0                     9.0                 10.0   \n",
       "3                        10.0                    10.0                  9.0   \n",
       "4                        10.0                    10.0                 10.0   \n",
       "\n",
       "   host_listing_count  log_price  \n",
       "0                   1   5.075174  \n",
       "1                   2   4.653960  \n",
       "2                   4   4.060443  \n",
       "3                   1   5.220356  \n",
       "4                   1   5.273000  \n",
       "\n",
       "[5 rows x 28 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Replace blanks with NaNs\n",
    "data = data.replace('_', np.nan)\n",
    "\n",
    "col_analysis = []\n",
    "for column in data.columns:\n",
    "    numNulls = len(data[column][data[column].isnull()])\n",
    "    totalLength = len(data[column])\n",
    "    dict1 = {'Name':column,'DataType':data[column].dtype, 'NumberOfNulls':numNulls, 'PercentageNulls':numNulls*100.0/totalLength}\n",
    "    col_analysis.append(dict1)\n",
    "    \n",
    "col_anal_df = pd.DataFrame(col_analysis)[['Name', 'DataType','NumberOfNulls','PercentageNulls']].sort_values(by='PercentageNulls', ascending=False)\n",
    "\n",
    "useful_cols = col_anal_df[col_anal_df.PercentageNulls < 50.0]\n",
    "\n",
    "print 'List of Predictors and their respective percentages of missing values'\n",
    "display(useful_cols.head(10))\n",
    "\n",
    "for cols in data.columns.values:\n",
    "    if (np.any(useful_cols.Name.values == cols) == False):\n",
    "        data.drop(cols, axis=1, inplace=True)\n",
    "        \n",
    "data.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Impute Missing Values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>neighbourhood_cleansed</th>\n",
       "      <th>city</th>\n",
       "      <th>zipcode</th>\n",
       "      <th>market</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>is_location_exact</th>\n",
       "      <th>property_type</th>\n",
       "      <th>room_type</th>\n",
       "      <th>accommodates</th>\n",
       "      <th>...</th>\n",
       "      <th>number_of_reviews</th>\n",
       "      <th>review_scores_rating</th>\n",
       "      <th>review_scores_accuracy</th>\n",
       "      <th>review_scores_cleanliness</th>\n",
       "      <th>review_scores_checkin</th>\n",
       "      <th>review_scores_communication</th>\n",
       "      <th>review_scores_location</th>\n",
       "      <th>review_scores_value</th>\n",
       "      <th>host_listing_count</th>\n",
       "      <th>log_price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Midtown East</td>\n",
       "      <td>New York</td>\n",
       "      <td>10022</td>\n",
       "      <td>New York</td>\n",
       "      <td>40.756852</td>\n",
       "      <td>-73.964754</td>\n",
       "      <td>t</td>\n",
       "      <td>Apartment</td>\n",
       "      <td>Entire home/apt</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>62</td>\n",
       "      <td>86.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1</td>\n",
       "      <td>5.075174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Hamilton Heights</td>\n",
       "      <td>New York</td>\n",
       "      <td>11211</td>\n",
       "      <td>New York</td>\n",
       "      <td>40.830599</td>\n",
       "      <td>-73.941014</td>\n",
       "      <td>f</td>\n",
       "      <td>Apartment</td>\n",
       "      <td>Entire home/apt</td>\n",
       "      <td>10</td>\n",
       "      <td>...</td>\n",
       "      <td>22</td>\n",
       "      <td>85.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>2</td>\n",
       "      <td>4.653960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Bushwick</td>\n",
       "      <td>Brooklyn</td>\n",
       "      <td>11221</td>\n",
       "      <td>New York</td>\n",
       "      <td>40.692189</td>\n",
       "      <td>-73.924120</td>\n",
       "      <td>t</td>\n",
       "      <td>Apartment</td>\n",
       "      <td>Private room</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>35</td>\n",
       "      <td>98.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>4</td>\n",
       "      <td>4.060443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>West Village</td>\n",
       "      <td>New York</td>\n",
       "      <td>10011</td>\n",
       "      <td>New York</td>\n",
       "      <td>40.734751</td>\n",
       "      <td>-74.002592</td>\n",
       "      <td>t</td>\n",
       "      <td>Apartment</td>\n",
       "      <td>Entire home/apt</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>26</td>\n",
       "      <td>96.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1</td>\n",
       "      <td>5.220356</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Chelsea</td>\n",
       "      <td>New York</td>\n",
       "      <td>10011</td>\n",
       "      <td>New York</td>\n",
       "      <td>40.745282</td>\n",
       "      <td>-73.997836</td>\n",
       "      <td>t</td>\n",
       "      <td>Apartment</td>\n",
       "      <td>Entire home/apt</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>100.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1</td>\n",
       "      <td>5.273000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  neighbourhood_cleansed      city zipcode    market   latitude  longitude  \\\n",
       "0           Midtown East  New York   10022  New York  40.756852 -73.964754   \n",
       "1       Hamilton Heights  New York   11211  New York  40.830599 -73.941014   \n",
       "2               Bushwick  Brooklyn   11221  New York  40.692189 -73.924120   \n",
       "3           West Village  New York   10011  New York  40.734751 -74.002592   \n",
       "4                Chelsea  New York   10011  New York  40.745282 -73.997836   \n",
       "\n",
       "  is_location_exact property_type        room_type  accommodates    ...      \\\n",
       "0                 t     Apartment  Entire home/apt             2    ...       \n",
       "1                 f     Apartment  Entire home/apt            10    ...       \n",
       "2                 t     Apartment     Private room             2    ...       \n",
       "3                 t     Apartment  Entire home/apt             2    ...       \n",
       "4                 t     Apartment  Entire home/apt             2    ...       \n",
       "\n",
       "   number_of_reviews  review_scores_rating  review_scores_accuracy  \\\n",
       "0                 62                  86.0                     9.0   \n",
       "1                 22                  85.0                     8.0   \n",
       "2                 35                  98.0                    10.0   \n",
       "3                 26                  96.0                    10.0   \n",
       "4                  1                 100.0                    10.0   \n",
       "\n",
       "  review_scores_cleanliness  review_scores_checkin  \\\n",
       "0                       7.0                    9.0   \n",
       "1                       8.0                    9.0   \n",
       "2                      10.0                   10.0   \n",
       "3                       9.0                   10.0   \n",
       "4                      10.0                   10.0   \n",
       "\n",
       "  review_scores_communication  review_scores_location  review_scores_value  \\\n",
       "0                         9.0                    10.0                  9.0   \n",
       "1                         8.0                     7.0                  8.0   \n",
       "2                        10.0                     9.0                 10.0   \n",
       "3                        10.0                    10.0                  9.0   \n",
       "4                        10.0                    10.0                 10.0   \n",
       "\n",
       "   host_listing_count  log_price  \n",
       "0                   1   5.075174  \n",
       "1                   2   4.653960  \n",
       "2                   4   4.060443  \n",
       "3                   1   5.220356  \n",
       "4                   1   5.273000  \n",
       "\n",
       "[5 rows x 28 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Use Mean for Real values Columns\n",
    "real_value_cols = useful_cols[useful_cols.DataType == 'float64']\n",
    "imp = Imputer(missing_values='NaN', strategy='mean', axis=0)\n",
    "data[real_value_cols.Name.values] = imp.fit_transform(data[real_value_cols.Name.values])\n",
    "\n",
    "\n",
    "#Use Highest frequency for categorical columns\n",
    "categorical_value_cols = useful_cols[useful_cols.DataType == 'object'].Name.values\n",
    "data[categorical_value_cols] = data[categorical_value_cols].apply(lambda x:x.fillna(x.value_counts().index[0]))\n",
    "\n",
    "data.head()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Convert Categorical Variables to dummy integer values here\n",
    "- We convert the categorical variables to numeric here such that we can run models that work only with numbers\n",
    "\n",
    "## One-Hot Encoding\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final Dataset ready for modelling after filling in missing values, and encoding categorical variables\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>neighbourhood_cleansed</th>\n",
       "      <th>city</th>\n",
       "      <th>zipcode</th>\n",
       "      <th>market</th>\n",
       "      <th>latitude</th>\n",
       "      <th>longitude</th>\n",
       "      <th>is_location_exact</th>\n",
       "      <th>property_type</th>\n",
       "      <th>room_type</th>\n",
       "      <th>accommodates</th>\n",
       "      <th>...</th>\n",
       "      <th>number_of_reviews</th>\n",
       "      <th>review_scores_rating</th>\n",
       "      <th>review_scores_accuracy</th>\n",
       "      <th>review_scores_cleanliness</th>\n",
       "      <th>review_scores_checkin</th>\n",
       "      <th>review_scores_communication</th>\n",
       "      <th>review_scores_location</th>\n",
       "      <th>review_scores_value</th>\n",
       "      <th>host_listing_count</th>\n",
       "      <th>log_price</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>112</td>\n",
       "      <td>104</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "      <td>40.756852</td>\n",
       "      <td>-73.964754</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>62</td>\n",
       "      <td>86.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1</td>\n",
       "      <td>5.075174</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>80</td>\n",
       "      <td>104</td>\n",
       "      <td>106</td>\n",
       "      <td>0</td>\n",
       "      <td>40.830599</td>\n",
       "      <td>-73.941014</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>...</td>\n",
       "      <td>22</td>\n",
       "      <td>85.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>2</td>\n",
       "      <td>4.653960</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>20</td>\n",
       "      <td>32</td>\n",
       "      <td>116</td>\n",
       "      <td>0</td>\n",
       "      <td>40.692189</td>\n",
       "      <td>-73.924120</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>35</td>\n",
       "      <td>98.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>4</td>\n",
       "      <td>4.060443</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>176</td>\n",
       "      <td>104</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>40.734751</td>\n",
       "      <td>-74.002592</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>26</td>\n",
       "      <td>96.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1</td>\n",
       "      <td>5.220356</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>25</td>\n",
       "      <td>104</td>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>40.745282</td>\n",
       "      <td>-73.997836</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>100.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>1</td>\n",
       "      <td>5.273000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   neighbourhood_cleansed  city  zipcode  market   latitude  longitude  \\\n",
       "0                     112   104       21       0  40.756852 -73.964754   \n",
       "1                      80   104      106       0  40.830599 -73.941014   \n",
       "2                      20    32      116       0  40.692189 -73.924120   \n",
       "3                     176   104       12       0  40.734751 -74.002592   \n",
       "4                      25   104       12       0  40.745282 -73.997836   \n",
       "\n",
       "   is_location_exact  property_type  room_type  accommodates    ...      \\\n",
       "0                  1              0          0             2    ...       \n",
       "1                  0              0          0            10    ...       \n",
       "2                  1              0          1             2    ...       \n",
       "3                  1              0          0             2    ...       \n",
       "4                  1              0          0             2    ...       \n",
       "\n",
       "   number_of_reviews  review_scores_rating  review_scores_accuracy  \\\n",
       "0                 62                  86.0                     9.0   \n",
       "1                 22                  85.0                     8.0   \n",
       "2                 35                  98.0                    10.0   \n",
       "3                 26                  96.0                    10.0   \n",
       "4                  1                 100.0                    10.0   \n",
       "\n",
       "   review_scores_cleanliness  review_scores_checkin  \\\n",
       "0                        7.0                    9.0   \n",
       "1                        8.0                    9.0   \n",
       "2                       10.0                   10.0   \n",
       "3                        9.0                   10.0   \n",
       "4                       10.0                   10.0   \n",
       "\n",
       "   review_scores_communication  review_scores_location  review_scores_value  \\\n",
       "0                          9.0                    10.0                  9.0   \n",
       "1                          8.0                     7.0                  8.0   \n",
       "2                         10.0                     9.0                 10.0   \n",
       "3                         10.0                    10.0                  9.0   \n",
       "4                         10.0                    10.0                 10.0   \n",
       "\n",
       "   host_listing_count  log_price  \n",
       "0                   1   5.075174  \n",
       "1                   2   4.653960  \n",
       "2                   4   4.060443  \n",
       "3                   1   5.220356  \n",
       "4                   1   5.273000  \n",
       "\n",
       "[5 rows x 28 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_ohe = data.copy(deep= True)\n",
    "\n",
    "#Encode categorical variables\n",
    "def encode_categorical(array):\n",
    "    return preprocessing.LabelEncoder().fit_transform(array) \n",
    "\n",
    "\n",
    "categorical_value_cols = useful_cols[useful_cols.DataType == 'object'].Name.values\n",
    "\n",
    "\n",
    "#Convert Categories to numbers here\n",
    "data_ohe[categorical_value_cols] = data_ohe[categorical_value_cols].apply(encode_categorical)\n",
    "\n",
    "# Apply one hot endcoing\n",
    "# Leads to inferior performance and hence we disable for now\n",
    "#data_ohe = pd.get_dummies(data_ohe.ix[:,:-1], columns=categorical_value_cols)\n",
    "\n",
    "print 'Final Dataset ready for modelling after filling in missing values, and encoding categorical variables'\n",
    "data_ohe.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Separate response from predictors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "x = data_ohe.values[:, :-1]\n",
    "y = data_ohe.values[:, -1]\n",
    "\n",
    "#response = df_filtered[['log_price']]\n",
    "#predictors =  df_filtered.drop(['log_price'], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Split into train/test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n",
    "x_train, x_test, y_train, y_test = train_test_split(x, y, test_size=0.4, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Simple Regression Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sklearn: R2 score for Linear Regression is: 0.614574354441\n"
     ]
    }
   ],
   "source": [
    "#OLS regression\n",
    "clf = LinearRegression(copy_X=True, fit_intercept=True, n_jobs=1, normalize=False)\n",
    "clf.fit(x_train, y_train)\n",
    "predicted = clf.predict(x_test)\n",
    "\n",
    "score = sk.metrics.r2_score(y_test, predicted)\n",
    "print('sklearn: R2 score for Linear Regression is: {}'.format(score))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}