SVD++.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import torch\n",
    "from sklearn.utils import shuffle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userId</th>\n",
       "      <th>movieId</th>\n",
       "      <th>rating</th>\n",
       "      <th>TimeStamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1193</td>\n",
       "      <td>5</td>\n",
       "      <td>978300760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>661</td>\n",
       "      <td>3</td>\n",
       "      <td>978302109</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>914</td>\n",
       "      <td>3</td>\n",
       "      <td>978301968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>3408</td>\n",
       "      <td>4</td>\n",
       "      <td>978300275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>2355</td>\n",
       "      <td>5</td>\n",
       "      <td>978824291</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000204</th>\n",
       "      <td>6040</td>\n",
       "      <td>1091</td>\n",
       "      <td>1</td>\n",
       "      <td>956716541</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000205</th>\n",
       "      <td>6040</td>\n",
       "      <td>1094</td>\n",
       "      <td>5</td>\n",
       "      <td>956704887</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000206</th>\n",
       "      <td>6040</td>\n",
       "      <td>562</td>\n",
       "      <td>5</td>\n",
       "      <td>956704746</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000207</th>\n",
       "      <td>6040</td>\n",
       "      <td>1096</td>\n",
       "      <td>4</td>\n",
       "      <td>956715648</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1000208</th>\n",
       "      <td>6040</td>\n",
       "      <td>1097</td>\n",
       "      <td>4</td>\n",
       "      <td>956715569</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1000209 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         userId  movieId  rating  TimeStamp\n",
       "0             1     1193       5  978300760\n",
       "1             1      661       3  978302109\n",
       "2             1      914       3  978301968\n",
       "3             1     3408       4  978300275\n",
       "4             1     2355       5  978824291\n",
       "...         ...      ...     ...        ...\n",
       "1000204    6040     1091       1  956716541\n",
       "1000205    6040     1094       5  956704887\n",
       "1000206    6040      562       5  956704746\n",
       "1000207    6040     1096       4  956715648\n",
       "1000208    6040     1097       4  956715569\n",
       "\n",
       "[1000209 rows x 4 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "rnames = ['userId', 'movieId', 'rating', \"TimeStamp\"]\n",
    "ratings = pd.read_table(\"ml-1m.inter\", header=0, names=rnames, engine='python')\n",
    "ratings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings = shuffle(ratings)\n",
    "ratio = 0.8\n",
    "# ratings\n",
    "train = ratings.copy()\n",
    "test = ratings.copy()\n",
    "train.iloc[int(ratio*len(ratings)):,2] = None\n",
    "# train.iloc[int(ratio*len(ratings)):,'rating'] = 0\n",
    "test.iloc[:int(ratio*len(ratings)),2] = None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>userId</th>\n",
       "      <th>movieId</th>\n",
       "      <th>rating</th>\n",
       "      <th>TimeStamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1.000209e+06</td>\n",
       "      <td>1.000209e+06</td>\n",
       "      <td>1.000209e+06</td>\n",
       "      <td>1.000209e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>3.024512e+03</td>\n",
       "      <td>1.865540e+03</td>\n",
       "      <td>3.581564e+00</td>\n",
       "      <td>9.722437e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>1.728413e+03</td>\n",
       "      <td>1.096041e+03</td>\n",
       "      <td>1.117102e+00</td>\n",
       "      <td>1.215256e+07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>1.000000e+00</td>\n",
       "      <td>9.567039e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.506000e+03</td>\n",
       "      <td>1.030000e+03</td>\n",
       "      <td>3.000000e+00</td>\n",
       "      <td>9.653026e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>3.070000e+03</td>\n",
       "      <td>1.835000e+03</td>\n",
       "      <td>4.000000e+00</td>\n",
       "      <td>9.730180e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>4.476000e+03</td>\n",
       "      <td>2.770000e+03</td>\n",
       "      <td>4.000000e+00</td>\n",
       "      <td>9.752209e+08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>6.040000e+03</td>\n",
       "      <td>3.952000e+03</td>\n",
       "      <td>5.000000e+00</td>\n",
       "      <td>1.046455e+09</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             userId       movieId        rating     TimeStamp\n",
       "count  1.000209e+06  1.000209e+06  1.000209e+06  1.000209e+06\n",
       "mean   3.024512e+03  1.865540e+03  3.581564e+00  9.722437e+08\n",
       "std    1.728413e+03  1.096041e+03  1.117102e+00  1.215256e+07\n",
       "min    1.000000e+00  1.000000e+00  1.000000e+00  9.567039e+08\n",
       "25%    1.506000e+03  1.030000e+03  3.000000e+00  9.653026e+08\n",
       "50%    3.070000e+03  1.835000e+03  4.000000e+00  9.730180e+08\n",
       "75%    4.476000e+03  2.770000e+03  4.000000e+00  9.752209e+08\n",
       "max    6.040000e+03  3.952000e+03  5.000000e+00  1.046455e+09"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "movieId  1     2     3     4     5     6     7     8     9     10    ...   \n",
      "userId                                                               ...   \n",
      "1         NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  ...  \\\n",
      "2         NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
      "3         NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
      "4         NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
      "5         NaN   NaN   NaN   NaN   NaN  0.25   NaN   NaN   NaN   NaN  ...   \n",
      "...       ...   ...   ...   ...   ...   ...   ...   ...   ...   ...  ...   \n",
      "6036      NaN   NaN   NaN  0.25   NaN  0.50   NaN   NaN   NaN   NaN  ...   \n",
      "6037      NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
      "6038      NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
      "6039      NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
      "6040      0.5   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
      "\n",
      "movieId  3943  3944  3945  3946  3947  3948  3949  3950  3951  3952  \n",
      "userId                                                               \n",
      "1         NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
      "2         NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
      "3         NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
      "4         NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
      "5         NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
      "...       ...   ...   ...   ...   ...   ...   ...   ...   ...   ...  \n",
      "6036      NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
      "6037      NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
      "6038      NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
      "6039      NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
      "6040      NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
      "\n",
      "[6040 rows x 3706 columns]\n",
      "800167\n"
     ]
    }
   ],
   "source": [
    "rating_matrix = train.pivot(index='userId', columns='movieId', values='rating')\n",
    "n_users, n_movies = rating_matrix.shape\n",
    "# Scaling ratings to between 0 and 1, this helps our model by constraining predictions\n",
    "min_rating, max_rating = train['rating'].min(), train['rating'].max()\n",
    "rating_matrix = (rating_matrix - min_rating) / (max_rating - min_rating)\n",
    "print(rating_matrix)\n",
    "print(n_users*n_movies-rating_matrix.isnull().values.sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-1.0000, -1.0000, -1.0000,  ..., -1.0000, -1.0000, -1.0000],\n",
      "        [-1.0000, -1.0000, -1.0000,  ..., -1.0000, -1.0000, -1.0000],\n",
      "        [-1.0000, -1.0000, -1.0000,  ..., -1.0000, -1.0000, -1.0000],\n",
      "        ...,\n",
      "        [-1.0000, -1.0000, -1.0000,  ..., -1.0000, -1.0000, -1.0000],\n",
      "        [-1.0000, -1.0000, -1.0000,  ..., -1.0000, -1.0000, -1.0000],\n",
      "        [ 0.5000, -1.0000, -1.0000,  ..., -1.0000, -1.0000, -1.0000]])\n"
     ]
    }
   ],
   "source": [
    "# Replacing missing ratings with -1 so we can filter them out later\n",
    "\n",
    "rating_matrix[rating_matrix.isnull()] = -1\n",
    "rating_matrix = torch.FloatTensor(rating_matrix.values)\n",
    "print(rating_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[1., 1., 1.],\n",
      "        [1., 1., 1.],\n",
      "        [1., 1., 1.],\n",
      "        [1., 1., 1.],\n",
      "        [1., 1., 1.]])\n"
     ]
    }
   ],
   "source": [
    "a = torch.ones(3,4)\n",
    "b = torch.ones(3)\n",
    "print(b.expand(5, 3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SVD(torch.nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        \n",
    "    def forward(self, u_features, v_features, u_bias, v_bias):\n",
    "        predicted_rating = torch.sigmoid(torch.mm(u_features, v_features.t())+u_bias.expand(n_movies, n_users).T+(v_bias.expand(n_users, n_movies)))\n",
    "        \n",
    "        return predicted_rating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SVDLoss(torch.nn.Module):\n",
    "    def __init__(self, lam_u=0.3, lam_v=0.3, lam_1 = 0.1, lam_2 = 0.1):\n",
    "        super().__init__()\n",
    "        self.lam_u = lam_u\n",
    "        self.lam_v = lam_v\n",
    "        self.lam_1 = lam_1\n",
    "        self.lam_2 = lam_2\n",
    "\n",
    "    def forward(self, matrix, predicted, u_features, v_features, u_bias, v_bias):\n",
    "        non_zero_mask = (matrix != -1).type(torch.FloatTensor)\n",
    "        diff = (matrix - predicted)**2\n",
    "        prediction_error = torch.sum(diff*non_zero_mask)\n",
    "\n",
    "        u_regularization = self.lam_u * torch.sum(u_features.norm(dim=1))\n",
    "        v_regularization = self.lam_v * torch.sum(v_features.norm(dim=1))\n",
    "\n",
    "        u_bias_regularization = self.lam_1 * (u_bias.norm(dim=0))    \n",
    "        v_bias_regularization = self.lam_2 * (v_bias.norm(dim=0))    \n",
    "        return prediction_error + u_regularization + v_regularization + u_bias_regularization + v_bias_regularization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 0, 79428.867\n",
      "Step 10, 63252.473\n",
      "Step 20, 46335.078\n",
      "Step 30, 40934.484\n",
      "Step 40, 38457.824\n",
      "Step 50, 36578.383\n",
      "Step 60, 34731.324\n",
      "Step 70, 32945.469\n",
      "Step 80, 31295.244\n",
      "Step 90, 29854.148\n"
     ]
    }
   ],
   "source": [
    "SVD_plus = SVD()\n",
    "svdloss = SVDLoss(lam_u=0.1, lam_v=0.1, lam_1 = 0.1, lam_2 = 0.1)\n",
    "num_epoch = 100\n",
    "latent_vectors = 20\n",
    "\n",
    "user_features = torch.randn(n_users, latent_vectors, requires_grad=True)\n",
    "movie_features = torch.randn(n_movies, latent_vectors, requires_grad=True)\n",
    "user_bias = torch.randn(n_users, requires_grad=True)\n",
    "movie_bias = torch.randn(n_movies, requires_grad=True)\n",
    "user_features.data.mul_(0.01)\n",
    "movie_features.data.mul_(0.01)\n",
    "user_bias.data.mul_(0.01)\n",
    "movie_bias.data.mul_(0.01)\n",
    "\n",
    "optimizer = torch.optim.Adam([user_features, movie_features,user_bias,movie_bias], lr=0.01)\n",
    "for step, epoch in enumerate(range(num_epoch)):\n",
    "    optimizer.zero_grad()\n",
    "    pred = SVD_plus(user_features, movie_features, user_bias, movie_bias)\n",
    "    loss = svdloss(rating_matrix, pred, user_features, movie_features, user_bias, movie_bias)\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "    if step % 10 == 0:\n",
    "        print(f\"Step {step}, {loss:.3f}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 5., -1., -1.,  ..., -1., -1., -1.],\n",
       "        [-1., -1., -1.,  ..., -1., -1., -1.],\n",
       "        [-1., -1., -1.,  ..., -1., -1., -1.],\n",
       "        ...,\n",
       "        [-1., -1., -1.,  ..., -1., -1., -1.],\n",
       "        [-1., -1., -1.,  ..., -1., -1., -1.],\n",
       "        [-1., -1., -1.,  ..., -1., -1., -1.]])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_rating_matrix = test.pivot(index='userId', columns='movieId', values='rating')\n",
    "\n",
    "test_rating_matrix[test_rating_matrix.isnull()] = -1\n",
    "test_rating_matrix = torch.FloatTensor(test_rating_matrix.values)\n",
    "test_rating_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_MAE = 0.69334364\n",
      "test_RMSE = 0.88743484\n"
     ]
    }
   ],
   "source": [
    "## test SVD\n",
    "non_zero_mask = (test_rating_matrix != -1).type(torch.FloatTensor)\n",
    "num = torch.sum(non_zero_mask)\n",
    "predicted_ratings = torch.sigmoid(torch.mm(user_features, movie_features.t())+user_bias.expand(n_movies, n_users).T+(movie_bias.expand(n_users, n_movies)))\n",
    "predicted_ratings = (predicted_ratings*(max_rating - min_rating) + min_rating)*non_zero_mask\n",
    "actual_ratings = test_rating_matrix*non_zero_mask\n",
    "\n",
    "AE_diff = torch.abs(predicted_ratings - actual_ratings)\n",
    "SE_diff = (predicted_ratings - actual_ratings)**2\n",
    "\n",
    "test_MAE = torch.sum(AE_diff)/num\n",
    "test_RMSE = torch.sqrt(torch.sum(SE_diff)/num)\n",
    "\n",
    "print('test_MAE =', test_MAE.data.numpy())\n",
    "print('test_RMSE =', test_RMSE.data.numpy())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}