Movielens Recommender Metrics.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from surprise import SVD\n",
    "from surprise import KNNBaseline\n",
    "from surprise.model_selection import train_test_split\n",
    "from surprise.model_selection import LeaveOneOut\n",
    "from surprise import Reader\n",
    "from surprise import Dataset\n",
    "from surprise import accuracy\n",
    "from surprise.model_selection import train_test_split\n",
    "from collections import defaultdict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ratings prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE:  0.8997274890435606\n",
      "MAE:  0.6939386166639191\n"
     ]
    }
   ],
   "source": [
    "movies = pd.read_csv('movielens_data/movies.csv')\n",
    "ratings = pd.read_csv('movielens_data/ratings.csv')\n",
    "df = pd.merge(movies, ratings, on='movieId', how='inner')\n",
    "reader = Reader(rating_scale=(0.5, 5))\n",
    "data = Dataset.load_from_df(df[['userId', 'movieId', 'rating']], reader)\n",
    "trainSet, testSet = train_test_split(data, test_size=.25, random_state=0)\n",
    "algo = SVD(random_state=0)\n",
    "algo.fit(trainSet)\n",
    "predictions = algo.test(testSet)\n",
    "\n",
    "def MAE(predictions):\n",
    "        return accuracy.mae(predictions, verbose=False)\n",
    "def RMSE(predictions):\n",
    "        return accuracy.rmse(predictions, verbose=False)\n",
    "    \n",
    "print(\"RMSE: \", RMSE(predictions))\n",
    "print(\"MAE: \", MAE(predictions))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Top-N"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def GetTopN(predictions, n=10, minimumRating=4.0):\n",
    "    topN = defaultdict(list)\n",
    "    for userID, movieID, actualRating, estimatedRating, _ in predictions:\n",
    "        if (estimatedRating >= minimumRating):\n",
    "            topN[int(userID)].append((int(movieID), estimatedRating))\n",
    "\n",
    "    for userID, ratings in topN.items():\n",
    "        ratings.sort(key=lambda x: x[1], reverse=True)\n",
    "        topN[int(userID)] = ratings[:n]\n",
    "\n",
    "    return topN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "LOOCV = LeaveOneOut(n_splits=1, random_state=1)\n",
    "\n",
    "for trainSet, testSet in LOOCV.split(data):\n",
    "    # Train model without left-out ratings\n",
    "    algo.fit(trainSet)\n",
    "    # Predicts ratings for left-out ratings only\n",
    "    leftOutPredictions = algo.test(testSet)\n",
    "    # Build predictions for all ratings not in the training set\n",
    "    bigTestSet = trainSet.build_anti_testset()\n",
    "    allPredictions = algo.test(bigTestSet)\n",
    "    # Compute top 10 recs for each user\n",
    "    topNPredicted = GetTopN(allPredictions, n=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hit Rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Hit Rate:  0.010432190760059613\n"
     ]
    }
   ],
   "source": [
    "# See how often we recommended a movie the user actually rated\n",
    "def HitRate(topNPredicted, leftOutPredictions):\n",
    "    hits = 0\n",
    "    total = 0\n",
    "\n",
    " # For each left-out rating\n",
    "    for leftOut in leftOutPredictions:\n",
    "        userID = leftOut[0]\n",
    "        leftOutMovieID = leftOut[1]\n",
    "        # Is it in the predicted top 10 for this user?\n",
    "        hit = False\n",
    "        for movieID, predictedRating in topNPredicted[int(userID)]:\n",
    "            if (int(leftOutMovieID) == int(movieID)):\n",
    "                hit = True\n",
    "                break\n",
    "        if (hit) :\n",
    "            hits += 1\n",
    "\n",
    "        total += 1\n",
    "\n",
    "    # Compute overall precision\n",
    "    return hits/total\n",
    "print(\"\\nHit Rate: \", HitRate(topNPredicted, leftOutPredictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Hit Rate by Rating value: \n",
      "3.0 0.015267175572519083\n",
      "4.0 0.0049261083743842365\n",
      "5.0 0.030303030303030304\n"
     ]
    }
   ],
   "source": [
    "def RatingHitRate(topNPredicted, leftOutPredictions):\n",
    "    hits = defaultdict(float)\n",
    "    total = defaultdict(float)\n",
    "    # For each left-out rating\n",
    "    for userID, leftOutMovieID, actualRating, estimatedRating, _ in leftOutPredictions:\n",
    "        # Is it in the predicted top N for this user?\n",
    "        hit = False\n",
    "        for movieID, predictedRating in topNPredicted[int(userID)]:\n",
    "            if (int(leftOutMovieID) == movieID):\n",
    "                hit = True\n",
    "                break\n",
    "        if (hit) :\n",
    "            hits[actualRating] += 1\n",
    "        total[actualRating] += 1\n",
    "\n",
    "    # Compute overall precision\n",
    "    for rating in sorted(hits.keys()):\n",
    "        print(rating, hits[rating] / total[rating])\n",
    "print(\"Hit Rate by Rating value: \")\n",
    "RatingHitRate(topNPredicted, leftOutPredictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cumulative Hit Rate (rating >= 4):  0.013157894736842105\n"
     ]
    }
   ],
   "source": [
    "def CumulativeHitRate(topNPredicted, leftOutPredictions, ratingCutoff=0):\n",
    "    hits = 0\n",
    "    total = 0\n",
    "    # For each left-out rating\n",
    "    for userID, leftOutMovieID, actualRating, estimatedRating, _ in leftOutPredictions:\n",
    "        # Only look at ability to recommend things the users actually liked...\n",
    "        if (actualRating >= ratingCutoff):\n",
    "            # Is it in the predicted top 10 for this user?\n",
    "            hit = False\n",
    "            for movieID, predictedRating in topNPredicted[int(userID)]:\n",
    "                if (int(leftOutMovieID) == movieID):\n",
    "                    hit = True\n",
    "                    break\n",
    "            if (hit) :\n",
    "                hits += 1\n",
    "            total += 1\n",
    "\n",
    "        # Compute overall precision\n",
    "    return hits/total\n",
    "print(\"Cumulative Hit Rate (rating >= 4): \", CumulativeHitRate(topNPredicted, leftOutPredictions, 4.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average Reciprocal Hit Rank:  0.0031710548103990725\n"
     ]
    }
   ],
   "source": [
    "# Compute ARHR\n",
    "def AverageReciprocalHitRank(topNPredicted, leftOutPredictions):\n",
    "    summation = 0\n",
    "    total = 0\n",
    "        # For each left-out rating\n",
    "    for userID, leftOutMovieID, actualRating, estimatedRating, _ in leftOutPredictions:\n",
    "        # Is it in the predicted top N for this user?\n",
    "        hitRank = 0\n",
    "        rank = 0\n",
    "        for movieID, predictedRating in topNPredicted[int(userID)]:\n",
    "            rank = rank + 1\n",
    "            if (int(leftOutMovieID) == movieID):\n",
    "                hitRank = rank\n",
    "                break\n",
    "        if (hitRank > 0) :\n",
    "                summation += 1.0 / hitRank\n",
    "\n",
    "        total += 1\n",
    "\n",
    "    return summation / total\n",
    "\n",
    "print(\"Average Reciprocal Hit Rank: \", AverageReciprocalHitRank(topNPredicted, leftOutPredictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<surprise.prediction_algorithms.knns.KNNBaseline at 0x29dc37567f0>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fullTrainSet = data.build_full_trainset()\n",
    "sim_options = {'name': 'pearson_baseline', 'user_based': False}\n",
    "simsAlgo = KNNBaseline(sim_options=sim_options)\n",
    "simsAlgo.fit(fullTrainSet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Computing complete recommendations, no hold outs...\n"
     ]
    }
   ],
   "source": [
    "algo.fit(fullTrainSet)\n",
    "bigTestSet = fullTrainSet.build_anti_testset()\n",
    "allPredictions = algo.test(bigTestSet)\n",
    "topNPredicted = GetTopN(allPredictions, n=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "User coverage:  0.9538002980625931\n"
     ]
    }
   ],
   "source": [
    "# Print user coverage with a minimum predicted rating of 4.0:\n",
    "def UserCoverage(topNPredicted, numUsers, ratingThreshold=0):\n",
    "    hits = 0\n",
    "    for userID in topNPredicted.keys():\n",
    "        hit = False\n",
    "        for movieID, predictedRating in topNPredicted[userID]:\n",
    "            if (predictedRating >= ratingThreshold):\n",
    "                hit = True\n",
    "                break\n",
    "        if (hit):\n",
    "            hits += 1\n",
    "\n",
    "    return hits / numUsers\n",
    "print(\"\\nUser coverage: \", UserCoverage(topNPredicted, fullTrainSet.n_users, ratingThreshold=4.0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Novelty (average popularity rank):  525.1527572016461\n"
     ]
    }
   ],
   "source": [
    "import csv\n",
    "\n",
    "def getPopularityRanks():\n",
    "        ratings = defaultdict(int)\n",
    "        rankings = defaultdict(int)\n",
    "        with open('movielens_data/ratings.csv', newline='') as csvfile:\n",
    "            ratingReader = csv.reader(csvfile)\n",
    "            next(ratingReader)\n",
    "            for row in ratingReader:\n",
    "                movieID = int(row[1])\n",
    "                ratings[movieID] += 1\n",
    "        rank = 1\n",
    "        for movieID, ratingCount in sorted(ratings.items(), key=lambda x: x[1], reverse=True):\n",
    "            rankings[movieID] = rank\n",
    "            rank += 1\n",
    "        return rankings\n",
    "rankings = getPopularityRanks()\n",
    "# Measure novelty (average popularity rank of recommendations):\n",
    "def Novelty(topNPredicted, rankings):\n",
    "    n = 0\n",
    "    total = 0\n",
    "    for userID in topNPredicted.keys():\n",
    "        for rating in topNPredicted[userID]:\n",
    "            movieID = rating[0]\n",
    "            rank = rankings[movieID]\n",
    "            total += rank\n",
    "            n += 1\n",
    "    return total / n\n",
    "print(\"\\nNovelty (average popularity rank): \", Novelty(topNPredicted, rankings))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}