diff --git "a/Week16_\353\263\265\354\212\265\352\263\274\354\240\234_\354\236\245\354\204\234\354\227\260.ipynb" "b/Week16_\353\263\265\354\212\265\352\263\274\354\240\234_\354\236\245\354\204\234\354\227\260.ipynb" new file mode 100644 index 0000000..fbce32f --- /dev/null +++ "b/Week16_\353\263\265\354\212\265\352\263\274\354\240\234_\354\236\245\354\204\234\354\227\260.ipynb" @@ -0,0 +1,6424 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "_wmZjF2tKmfv" + }, + "source": [ + "# 9.5장 콘텐츠 기반 필터링 실습 - TMDB 5000 영화 데이터 세트" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "XYUi_i-vLJQh" + }, + "source": [ + "## 데이터 로딩 및 가공" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "hHpP117WdZAW", + "outputId": "6d582492-0394-482f-929c-ebe8048ec787" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Requirement already satisfied: kagglehub[pandas-datasets] in /usr/local/lib/python3.11/dist-packages (0.3.12)\n", + "Requirement already satisfied: packaging in /usr/local/lib/python3.11/dist-packages (from kagglehub[pandas-datasets]) (24.2)\n", + "Requirement already satisfied: pyyaml in /usr/local/lib/python3.11/dist-packages (from kagglehub[pandas-datasets]) (6.0.2)\n", + "Requirement already satisfied: requests in /usr/local/lib/python3.11/dist-packages (from kagglehub[pandas-datasets]) (2.32.3)\n", + "Requirement already satisfied: tqdm in /usr/local/lib/python3.11/dist-packages (from kagglehub[pandas-datasets]) (4.67.1)\n", + "Requirement already satisfied: pandas in /usr/local/lib/python3.11/dist-packages (from kagglehub[pandas-datasets]) (2.2.2)\n", + "Requirement already satisfied: numpy>=1.23.2 in /usr/local/lib/python3.11/dist-packages (from pandas->kagglehub[pandas-datasets]) (2.0.2)\n", + "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.11/dist-packages (from pandas->kagglehub[pandas-datasets]) (2.9.0.post0)\n", + "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.11/dist-packages (from pandas->kagglehub[pandas-datasets]) (2025.2)\n", + "Requirement already satisfied: tzdata>=2022.7 in /usr/local/lib/python3.11/dist-packages (from pandas->kagglehub[pandas-datasets]) (2025.2)\n", + "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.11/dist-packages (from requests->kagglehub[pandas-datasets]) (3.4.2)\n", + "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.11/dist-packages (from requests->kagglehub[pandas-datasets]) (3.10)\n", + "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.11/dist-packages (from requests->kagglehub[pandas-datasets]) (2.4.0)\n", + "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.11/dist-packages (from requests->kagglehub[pandas-datasets]) (2025.6.15)\n", + "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.11/dist-packages (from python-dateutil>=2.8.2->pandas->kagglehub[pandas-datasets]) (1.17.0)\n" + ] + } + ], + "source": [ + "!pip install --upgrade kagglehub[pandas-datasets]" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 206 + }, + "id": "6SKQZsIdKeYY", + "outputId": "ee715718-d5c9-4750-d05a-f3e413661438" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "(4803, 20)\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + " budget genres \\\n", + "0 237000000 [{\"id\": 28, \"name\": \"Action\"}, {\"id\": 12, \"nam... \n", + "\n", + " homepage id \\\n", + "0 http://www.avatarmovie.com/ 19995 \n", + "\n", + " keywords original_language \\\n", + "0 [{\"id\": 1463, \"name\": \"culture clash\"}, {\"id\":... en \n", + "\n", + " original_title overview \\\n", + "0 Avatar In the 22nd century, a paraplegic Marine is di... \n", + "\n", + " popularity production_companies \\\n", + "0 150.437577 [{\"name\": \"Ingenious Film Partners\", \"id\": 289... \n", + "\n", + " production_countries release_date revenue \\\n", + "0 [{\"iso_3166_1\": \"US\", \"name\": \"United States o... 2009-12-10 2787965087 \n", + "\n", + " runtime spoken_languages status \\\n", + "0 162.0 [{\"iso_639_1\": \"en\", \"name\": \"English\"}, {\"iso... Released \n", + "\n", + " tagline title vote_average vote_count \n", + "0 Enter the World of Pandora. Avatar 7.2 11800 " + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
budgetgenreshomepageidkeywordsoriginal_languageoriginal_titleoverviewpopularityproduction_companiesproduction_countriesrelease_daterevenueruntimespoken_languagesstatustaglinetitlevote_averagevote_count
0237000000[{\"id\": 28, \"name\": \"Action\"}, {\"id\": 12, \"nam...http://www.avatarmovie.com/19995[{\"id\": 1463, \"name\": \"culture clash\"}, {\"id\":...enAvatarIn the 22nd century, a paraplegic Marine is di...150.437577[{\"name\": \"Ingenious Film Partners\", \"id\": 289...[{\"iso_3166_1\": \"US\", \"name\": \"United States o...2009-12-102787965087162.0[{\"iso_639_1\": \"en\", \"name\": \"English\"}, {\"iso...ReleasedEnter the World of Pandora.Avatar7.211800
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + "
\n" + ], + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "dataframe", + "variable_name": "movies", + "summary": "{\n \"name\": \"movies\",\n \"rows\": 4803,\n \"fields\": [\n {\n \"column\": \"budget\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 40722391,\n \"min\": 0,\n \"max\": 380000000,\n \"num_unique_values\": 436,\n \"samples\": [\n 439000,\n 68000000,\n 700000\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"genres\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 1175,\n \"samples\": [\n \"[{\\\"id\\\": 14, \\\"name\\\": \\\"Fantasy\\\"}, {\\\"id\\\": 12, \\\"name\\\": \\\"Adventure\\\"}, {\\\"id\\\": 16, \\\"name\\\": \\\"Animation\\\"}]\",\n \"[{\\\"id\\\": 28, \\\"name\\\": \\\"Action\\\"}, {\\\"id\\\": 35, \\\"name\\\": \\\"Comedy\\\"}, {\\\"id\\\": 80, \\\"name\\\": \\\"Crime\\\"}, {\\\"id\\\": 18, \\\"name\\\": \\\"Drama\\\"}]\",\n \"[{\\\"id\\\": 12, \\\"name\\\": \\\"Adventure\\\"}, {\\\"id\\\": 16, \\\"name\\\": \\\"Animation\\\"}, {\\\"id\\\": 10751, \\\"name\\\": \\\"Family\\\"}, {\\\"id\\\": 14, \\\"name\\\": \\\"Fantasy\\\"}, {\\\"id\\\": 878, \\\"name\\\": \\\"Science Fiction\\\"}]\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"homepage\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 1691,\n \"samples\": [\n \"https://www.warnerbros.com/running-scared\",\n \"http://www.51birchstreet.com/index.php\",\n \"http://movies2.foxjapan.com/glee/\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"id\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 88694,\n \"min\": 5,\n \"max\": 459488,\n \"num_unique_values\": 4803,\n \"samples\": [\n 8427,\n 13006,\n 18041\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"keywords\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 4222,\n \"samples\": [\n \"[{\\\"id\\\": 782, \\\"name\\\": \\\"assassin\\\"}, {\\\"id\\\": 1872, \\\"name\\\": \\\"loss of father\\\"}, {\\\"id\\\": 2908, \\\"name\\\": \\\"secret society\\\"}, {\\\"id\\\": 3045, \\\"name\\\": \\\"mission of murder\\\"}, {\\\"id\\\": 9748, \\\"name\\\": \\\"revenge\\\"}]\",\n \"[{\\\"id\\\": 2987, \\\"name\\\": \\\"gang war\\\"}, {\\\"id\\\": 4942, \\\"name\\\": \\\"victim of murder\\\"}, {\\\"id\\\": 5332, \\\"name\\\": \\\"greed\\\"}, {\\\"id\\\": 6062, \\\"name\\\": \\\"hostility\\\"}, {\\\"id\\\": 156212, \\\"name\\\": \\\"spaghetti western\\\"}]\",\n \"[{\\\"id\\\": 703, \\\"name\\\": \\\"detective\\\"}, {\\\"id\\\": 1299, \\\"name\\\": \\\"monster\\\"}, {\\\"id\\\": 6101, \\\"name\\\": \\\"engine\\\"}, {\\\"id\\\": 10988, \\\"name\\\": \\\"based on tv series\\\"}, {\\\"id\\\": 15162, \\\"name\\\": \\\"dog\\\"}]\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"original_language\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 37,\n \"samples\": [\n \"xx\",\n \"ta\",\n \"es\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"original_title\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 4801,\n \"samples\": [\n \"I Spy\",\n \"Love Letters\",\n \"Sleepover\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"overview\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 4800,\n \"samples\": [\n \"When the Switchblade, the most sophisticated prototype stealth fighter created yet, is stolen from the U.S. government, one of the United States' top spies, Alex Scott, is called to action. What he doesn't expect is to get teamed up with a cocky civilian, World Class Boxing Champion Kelly Robinson, on a dangerous top secret espionage mission. Their assignment: using equal parts skill and humor, catch Arnold Gundars, one of the world's most successful arms dealers.\",\n \"When \\\"street smart\\\" rapper Christopher \\\"C-Note\\\" Hawkins (Big Boi) applies for a membership to all-white Carolina Pines Country Club, the establishment's proprietors are hardly ready to oblige him.\",\n \"As their first year of high school looms ahead, best friends Julie, Hannah, Yancy and Farrah have one last summer sleepover. Little do they know they're about to embark on the adventure of a lifetime. Desperate to shed their nerdy status, they take part in a night-long scavenger hunt that pits them against their popular archrivals. Everything under the sun goes on -- from taking Yancy's father's car to sneaking into nightclubs!\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"popularity\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 31.816649749537806,\n \"min\": 0.0,\n \"max\": 875.581305,\n \"num_unique_values\": 4802,\n \"samples\": [\n 13.267631,\n 0.010909,\n 5.842299\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"production_companies\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 3697,\n \"samples\": [\n \"[{\\\"name\\\": \\\"Paramount Pictures\\\", \\\"id\\\": 4}, {\\\"name\\\": \\\"Cherry Alley Productions\\\", \\\"id\\\": 2232}]\",\n \"[{\\\"name\\\": \\\"Twentieth Century Fox Film Corporation\\\", \\\"id\\\": 306}, {\\\"name\\\": \\\"Dune Entertainment\\\", \\\"id\\\": 444}, {\\\"name\\\": \\\"Regency Enterprises\\\", \\\"id\\\": 508}, {\\\"name\\\": \\\"Guy Walks into a Bar Productions\\\", \\\"id\\\": 2645}, {\\\"name\\\": \\\"Deep River Productions\\\", \\\"id\\\": 2646}, {\\\"name\\\": \\\"Friendly Films (II)\\\", \\\"id\\\": 81136}]\",\n \"[{\\\"name\\\": \\\"Twentieth Century Fox Film Corporation\\\", \\\"id\\\": 306}]\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"production_countries\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 469,\n \"samples\": [\n \"[{\\\"iso_3166_1\\\": \\\"ES\\\", \\\"name\\\": \\\"Spain\\\"}, {\\\"iso_3166_1\\\": \\\"GB\\\", \\\"name\\\": \\\"United Kingdom\\\"}, {\\\"iso_3166_1\\\": \\\"US\\\", \\\"name\\\": \\\"United States of America\\\"}, {\\\"iso_3166_1\\\": \\\"FR\\\", \\\"name\\\": \\\"France\\\"}]\",\n \"[{\\\"iso_3166_1\\\": \\\"US\\\", \\\"name\\\": \\\"United States of America\\\"}, {\\\"iso_3166_1\\\": \\\"CA\\\", \\\"name\\\": \\\"Canada\\\"}, {\\\"iso_3166_1\\\": \\\"DE\\\", \\\"name\\\": \\\"Germany\\\"}]\",\n \"[{\\\"iso_3166_1\\\": \\\"DE\\\", \\\"name\\\": \\\"Germany\\\"}, {\\\"iso_3166_1\\\": \\\"ES\\\", \\\"name\\\": \\\"Spain\\\"}, {\\\"iso_3166_1\\\": \\\"GB\\\", \\\"name\\\": \\\"United Kingdom\\\"}, {\\\"iso_3166_1\\\": \\\"US\\\", \\\"name\\\": \\\"United States of America\\\"}]\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"release_date\",\n \"properties\": {\n \"dtype\": \"object\",\n \"num_unique_values\": 3280,\n \"samples\": [\n \"1966-10-16\",\n \"1987-07-31\",\n \"1993-09-23\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"revenue\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 162857100,\n \"min\": 0,\n \"max\": 2787965087,\n \"num_unique_values\": 3297,\n \"samples\": [\n 11833696,\n 10462500,\n 17807569\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"runtime\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 22.611934588844207,\n \"min\": 0.0,\n \"max\": 338.0,\n \"num_unique_values\": 156,\n \"samples\": [\n 74.0,\n 85.0,\n 170.0\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"spoken_languages\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 544,\n \"samples\": [\n \"[{\\\"iso_639_1\\\": \\\"es\\\", \\\"name\\\": \\\"Espa\\\\u00f1ol\\\"}, {\\\"iso_639_1\\\": \\\"en\\\", \\\"name\\\": \\\"English\\\"}, {\\\"iso_639_1\\\": \\\"fr\\\", \\\"name\\\": \\\"Fran\\\\u00e7ais\\\"}, {\\\"iso_639_1\\\": \\\"hu\\\", \\\"name\\\": \\\"Magyar\\\"}]\",\n \"[{\\\"iso_639_1\\\": \\\"en\\\", \\\"name\\\": \\\"English\\\"}, {\\\"iso_639_1\\\": \\\"it\\\", \\\"name\\\": \\\"Italiano\\\"}, {\\\"iso_639_1\\\": \\\"pt\\\", \\\"name\\\": \\\"Portugu\\\\u00eas\\\"}]\",\n \"[{\\\"iso_639_1\\\": \\\"de\\\", \\\"name\\\": \\\"Deutsch\\\"}, {\\\"iso_639_1\\\": \\\"it\\\", \\\"name\\\": \\\"Italiano\\\"}, {\\\"iso_639_1\\\": \\\"la\\\", \\\"name\\\": \\\"Latin\\\"}, {\\\"iso_639_1\\\": \\\"pl\\\", \\\"name\\\": \\\"Polski\\\"}]\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"status\",\n \"properties\": {\n \"dtype\": \"category\",\n \"num_unique_values\": 3,\n \"samples\": [\n \"Released\",\n \"Post Production\",\n \"Rumored\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"tagline\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 3944,\n \"samples\": [\n \"When you're 17, every day is war.\",\n \"An Unspeakable Horror. A Creative Genius. Captured For Eternity.\",\n \"May the schwartz be with you\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"title\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 4800,\n \"samples\": [\n \"I Spy\",\n \"Who's Your Caddy?\",\n \"Sleepover\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"vote_average\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 1.1946121628478925,\n \"min\": 0.0,\n \"max\": 10.0,\n \"num_unique_values\": 71,\n \"samples\": [\n 5.1,\n 7.2,\n 4.0\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"vote_count\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 1234,\n \"min\": 0,\n \"max\": 13752,\n \"num_unique_values\": 1609,\n \"samples\": [\n 7604,\n 3428,\n 225\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}" + } + }, + "metadata": {}, + "execution_count": 2 + } + ], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "import warnings; warnings.filterwarnings('ignore')\n", + "\n", + "movies = pd.read_csv('/content/drive/MyDrive/Sample_data/tmdb_movies/tmdb_5000_movies.csv')\n", + "print(movies.shape)\n", + "movies.head(1)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "id": "i3iXAT-VecUN" + }, + "outputs": [], + "source": [ + "movies_df = movies[['id','title','genres','vote_average','vote_count','popularity','keywords','overview']]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "AmBErfwLe8Vh" + }, + "source": [ + "- genres, keywords 과 같은 칼럼들은 파이썬 리스트 내부에 여러개의 딕셔너리가 있는 형태의 문자열임 ; 한번에 여러개의 값 표기\n", + "- genres\n", + ": 여러개의 개별 장르 데이터 가짐, 개별장르의 명칭은 'name'이라는 키로 추출 가능\n", + "- literal_eval() 함수 : list[dict1,dict2] 객체로 만들 수 있음." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 98 + }, + "id": "vQg2S6_ken7p", + "outputId": "2cc4fbda-1618-4cbe-8a64-71de1ff70567" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + " genres \\\n", + "0 [{\"id\": 28, \"name\": \"Action\"}, {\"id\": 12, \"name\": \"Adventure\"}, {\"id\": 14, \"name\": \"Fantasy\"}, {... \n", + "\n", + " keywords \n", + "0 [{\"id\": 1463, \"name\": \"culture clash\"}, {\"id\": 2964, \"name\": \"future\"}, {\"id\": 3386, \"name\": \"sp... " + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
genreskeywords
0[{\"id\": 28, \"name\": \"Action\"}, {\"id\": 12, \"name\": \"Adventure\"}, {\"id\": 14, \"name\": \"Fantasy\"}, {...[{\"id\": 1463, \"name\": \"culture clash\"}, {\"id\": 2964, \"name\": \"future\"}, {\"id\": 3386, \"name\": \"sp...
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + "
\n" + ], + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "dataframe", + "summary": "{\n \"name\": \"movies_df[['genres','keywords']][:1]\",\n \"rows\": 1,\n \"fields\": [\n {\n \"column\": \"genres\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 1,\n \"samples\": [\n \"[{\\\"id\\\": 28, \\\"name\\\": \\\"Action\\\"}, {\\\"id\\\": 12, \\\"name\\\": \\\"Adventure\\\"}, {\\\"id\\\": 14, \\\"name\\\": \\\"Fantasy\\\"}, {\\\"id\\\": 878, \\\"name\\\": \\\"Science Fiction\\\"}]\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"keywords\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 1,\n \"samples\": [\n \"[{\\\"id\\\": 1463, \\\"name\\\": \\\"culture clash\\\"}, {\\\"id\\\": 2964, \\\"name\\\": \\\"future\\\"}, {\\\"id\\\": 3386, \\\"name\\\": \\\"space war\\\"}, {\\\"id\\\": 3388, \\\"name\\\": \\\"space colony\\\"}, {\\\"id\\\": 3679, \\\"name\\\": \\\"society\\\"}, {\\\"id\\\": 3801, \\\"name\\\": \\\"space travel\\\"}, {\\\"id\\\": 9685, \\\"name\\\": \\\"futuristic\\\"}, {\\\"id\\\": 9840, \\\"name\\\": \\\"romance\\\"}, {\\\"id\\\": 9882, \\\"name\\\": \\\"space\\\"}, {\\\"id\\\": 9951, \\\"name\\\": \\\"alien\\\"}, {\\\"id\\\": 10148, \\\"name\\\": \\\"tribe\\\"}, {\\\"id\\\": 10158, \\\"name\\\": \\\"alien planet\\\"}, {\\\"id\\\": 10987, \\\"name\\\": \\\"cgi\\\"}, {\\\"id\\\": 11399, \\\"name\\\": \\\"marine\\\"}, {\\\"id\\\": 13065, \\\"name\\\": \\\"soldier\\\"}, {\\\"id\\\": 14643, \\\"name\\\": \\\"battle\\\"}, {\\\"id\\\": 14720, \\\"name\\\": \\\"love affair\\\"}, {\\\"id\\\": 165431, \\\"name\\\": \\\"anti war\\\"}, {\\\"id\\\": 193554, \\\"name\\\": \\\"power relations\\\"}, {\\\"id\\\": 206690, \\\"name\\\": \\\"mind and soul\\\"}, {\\\"id\\\": 209714, \\\"name\\\": \\\"3d\\\"}]\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}" + } + }, + "metadata": {}, + "execution_count": 4 + } + ], + "source": [ + "pd.set_option('max_colwidth',100)\n", + "movies_df[['genres','keywords']][:1]" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "id": "nSpf0ohSfXOr" + }, + "outputs": [], + "source": [ + "from ast import literal_eval\n", + "# genres : 문자열 X, 실제 리스트 내부에 여러 장르 딕셔너리로 구성된 객체\n", + "movies_df['genres']=movies_df['genres'].apply(literal_eval)\n", + "movies_df['keywords']=movies_df['keywords'].apply(literal_eval)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 98 + }, + "id": "tdrsB-bqfrXc", + "outputId": "1b82c855-b5f3-4a07-b4f7-843e8fd5fa6a" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + " genres \\\n", + "0 [Action, Adventure, Fantasy, Science Fiction] \n", + "\n", + " keywords \n", + "0 [culture clash, future, space war, space colony, society, space travel, futuristic, romance, spa... " + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
genreskeywords
0[Action, Adventure, Fantasy, Science Fiction][culture clash, future, space war, space colony, society, space travel, futuristic, romance, spa...
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + "
\n" + ], + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "dataframe", + "summary": "{\n \"name\": \"movies_df[['genres','keywords']][:1]\",\n \"rows\": 1,\n \"fields\": [\n {\n \"column\": \"genres\",\n \"properties\": {\n \"dtype\": \"object\",\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"keywords\",\n \"properties\": {\n \"dtype\": \"object\",\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}" + } + }, + "metadata": {}, + "execution_count": 6 + } + ], + "source": [ + "movies_df['genres']=movies_df['genres'].apply(lambda x :[y['name'] for y in x])\n", + "movies_df['keywords']=movies_df['keywords'].apply(lambda x :[y['name'] for y in x])\n", + "movies_df[['genres','keywords']][:1]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZalegVRKf8qP" + }, + "source": [ + "## 장르 콘텐츠 유사도 측정\n", + "\n", + "1. genres 문자열 변경 > CountVectorizer로 피처 벡터화\n", + "2. genres 문자열을 피처벡터화 행렬로 변환한 데이터 세트를 코사인 유사도를 통해 비교 / 데이터 세트의 레코드별로 타 레코드와 장르에서 코사인 유사도값을 가지는 객체 생성\n", + "3. 장르 유사도가 높은 영화 중 평점이 높은 순으로 영화 추천" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "KbRrKu8PgSar", + "outputId": "908fab5b-5c56-4187-d454-997eaee2a415" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "(4803, 276)\n" + ] + } + ], + "source": [ + "from sklearn.feature_extraction.text import CountVectorizer\n", + "\n", + "# CountVectorizer 적용 위해 공백문자로 word 단위가 구분되는 문자열로 변환\n", + "movies_df['genres_literal'] = movies_df['genres'].apply(lambda x : (' ').join(x))\n", + "count_vect = CountVectorizer(min_df=0.00000000001, ngram_range=(1,2))\n", + "genre_mat = count_vect.fit_transform(movies_df['genres_literal'])\n", + "print(genre_mat.shape)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "XNGJNgThhiC4" + }, + "source": [ + "- movies_df의 행별 장르 우사도 값 지니는 행렬 생성" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "UphQoEaVhTeq", + "outputId": "6b370064-08f8-477b-dd86-5570b0f28dff" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "(4803, 4803)\n", + "[[1. 0.59628479 0.4472136 ... 0. 0. 0. ]\n", + " [0.59628479 1. 0.4 ... 0. 0. 0. ]]\n" + ] + } + ], + "source": [ + "from sklearn.metrics.pairwise import cosine_similarity\n", + "\n", + "genre_sim = cosine_similarity(genre_mat, genre_mat)\n", + "print(genre_sim.shape)\n", + "print(genre_sim[:2])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "yceUt8xVhn1_" + }, + "source": [ + "- genre_sim 객체의 기준 행별로 비교 대상이 되는 행의 유사도 값이 높은 순으로 정렬된 행렬의 위치 인덱스 값 추출\n", + "- ⬇0번 레코드의 경우 자신인 0번 레코드를 제외하면 46,3494 순으로 유사도가 높고 가장 유사도가 낮은 인덱스는 2031" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "aLBs7sOLhmS9", + "outputId": "ac9fbd55-f3b4-44bd-d42a-d99e845543a2" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "[[ 0 46 3494 ... 3331 3333 2031]]\n" + ] + } + ], + "source": [ + "genre_sim_sorted_ind = genre_sim.argsort()[:, ::-1]\n", + "print(genre_sim_sorted_ind[:1])" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "qtPnwzhiiG9L" + }, + "source": [ + "## 장르 콘텐츠 필터링을 이용한 영화 추천\n", + "\n", + "### find_sim_movie()\n", + "- input : movies_df,genre_sorted_ind, 고객의 추천 기준이 되는 영화제목, 추천할 영화 건수\n", + "- return : 추천영화정보를 가지는 df\n" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "id": "iC8PP5U7iXcg" + }, + "outputs": [], + "source": [ + "def find_sim_movie(df,sorted_ind,title_name,top_n=10):\n", + " # movies_df에서 'title'칼럼이 입력된 title_name값인 df 추출\n", + " title_movie = df[df['title']==title_name]\n", + "\n", + " # title_name가진 df의 인덱스객체 > ndarray\n", + " # genre_sim_sorted_ind 객체에서 유사도순으로 top_n개의 index 추출\n", + " title_index = title_movie.index.values\n", + " similar_indexes = sorted_ind[title_index,:(top_n)]\n", + "\n", + " # top_n 인텍스 출력(2차원데이터) > 1차원 ndarray로 변환\n", + " print(similar_indexes)\n", + " similar_indexes = similar_indexes.reshape(-1)\n", + "\n", + " return df.iloc[similar_indexes]" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 381 + }, + "id": "10Z1nsTljEYu", + "outputId": "be3b9fa4-ab94-4650-c37f-5a931efc55e0" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "[[1881 3378 3866 1370 1464 588 3887 3594 2839 892]]\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + " title vote_average\n", + "1881 The Shawshank Redemption 8.5\n", + "3378 Auto Focus 6.1\n", + "3866 City of God 8.1\n", + "1370 21 6.5\n", + "1464 Black Water Transit 0.0\n", + "588 Wall Street: Money Never Sleeps 5.8\n", + "3887 Trainspotting 7.8\n", + "3594 Spring Breakers 5.0\n", + "2839 Rounders 6.9\n", + "892 Casino 7.8" + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
titlevote_average
1881The Shawshank Redemption8.5
3378Auto Focus6.1
3866City of God8.1
1370216.5
1464Black Water Transit0.0
588Wall Street: Money Never Sleeps5.8
3887Trainspotting7.8
3594Spring Breakers5.0
2839Rounders6.9
892Casino7.8
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "\n", + "
\n", + "
\n" + ], + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "dataframe", + "summary": "{\n \"name\": \"similar_movies[['title', 'vote_average']]\",\n \"rows\": 10,\n \"fields\": [\n {\n \"column\": \"title\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 10,\n \"samples\": [\n \"Rounders\",\n \"Auto Focus\",\n \"Wall Street: Money Never Sleeps\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"vote_average\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 2.4636242498490803,\n \"min\": 0.0,\n \"max\": 8.5,\n \"num_unique_values\": 9,\n \"samples\": [\n 5.0,\n 6.1,\n 5.8\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}" + } + }, + "metadata": {}, + "execution_count": 11 + } + ], + "source": [ + "similar_movies = find_sim_movie(movies_df, genre_sim_sorted_ind, 'The Godfather',10)\n", + "similar_movies[['title', 'vote_average']]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "TAjY0-F_kiu4" + }, + "source": [ + "- 고객에게 왜추천하는지 이해하기 어려운 영화 有 > 평점 0.0\n", + "- 많은 후보군 선정 > 평점에 따라 필터링 후 최종 추천" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ODjR-hkiky_4" + }, + "source": [ + "- vote_average : 0~10, 특정고객이 만점을 부여해 왜곡된 데이터를 가지고 있을 수 있음. > vote_count 고려한 가중 평점 생성\n", + "- 가중평점 = (vote_count/(vote_count+최소투표횟수)) * 개별영화평균평점 + (vote_count/(vote_count+최소투표횟수)) * 전체 영화 평균 평점\n", + "- 가중치 : (vote_count/(vote_count+최소투표횟수))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 362 + }, + "id": "kWEBtDD6kMOo", + "outputId": "e294820d-a6c1-432c-d783-0af26790fa64" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + " title vote_average vote_count\n", + "4662 Little Big Top 10.0 1\n", + "3519 Stiff Upper Lips 10.0 1\n", + "4045 Dancer, Texas Pop. 81 10.0 1\n", + "4247 Me You and Five Bucks 10.0 2\n", + "3992 Sardaarji 9.5 2\n", + "2386 One Man's Hero 9.3 2\n", + "1881 The Shawshank Redemption 8.5 8205\n", + "2970 There Goes My Baby 8.5 2\n", + "3337 The Godfather 8.4 5893\n", + "2796 The Prisoner of Zenda 8.4 11" + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
titlevote_averagevote_count
4662Little Big Top10.01
3519Stiff Upper Lips10.01
4045Dancer, Texas Pop. 8110.01
4247Me You and Five Bucks10.02
3992Sardaarji9.52
2386One Man's Hero9.32
1881The Shawshank Redemption8.58205
2970There Goes My Baby8.52
3337The Godfather8.45893
2796The Prisoner of Zenda8.411
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "\n", + "
\n", + "
\n" + ], + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "dataframe", + "summary": "{\n \"name\": \"movies_df[['title','vote_average','vote_count']]\",\n \"rows\": 10,\n \"fields\": [\n {\n \"column\": \"title\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 10,\n \"samples\": [\n \"The Godfather\",\n \"Stiff Upper Lips\",\n \"One Man's Hero\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"vote_average\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 0.7366591251499343,\n \"min\": 8.4,\n \"max\": 10.0,\n \"num_unique_values\": 5,\n \"samples\": [\n 9.5,\n 8.4,\n 9.3\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"vote_count\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 3020,\n \"min\": 1,\n \"max\": 8205,\n \"num_unique_values\": 5,\n \"samples\": [\n 2,\n 11,\n 8205\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}" + } + }, + "metadata": {}, + "execution_count": 12 + } + ], + "source": [ + "movies_df[['title','vote_average','vote_count']].sort_values('vote_average', ascending=False)[:10]" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ZI51cqn-kN7h", + "outputId": "81bd7a81-535d-4a48-b102-45a0e886e6df" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "C: 6.092 m: 370.2\n" + ] + } + ], + "source": [ + "C = movies_df['vote_average'].mean()\n", + "m = movies_df['vote_count'].quantile(0.6) #상위60퍼\n", + "print('C:',round(C,3), 'm:',round(m,3))" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "id": "1nJIXL0LkREh" + }, + "outputs": [], + "source": [ + "percentile = 0.6\n", + "m = movies_df['vote_count'].quantile(percentile)\n", + "C = movies_df['vote_average'].mean()\n", + "\n", + "def weighted_vote_average(record):\n", + " v = record['vote_count']\n", + " R = record['vote_average']\n", + "\n", + " return ( (v/(v+m)) * R ) + ( (m/(m+v)) * C )\n", + "\n", + "movies_df['weighted_vote'] = movies_df.apply(weighted_vote_average, axis=1)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 362 + }, + "id": "hZcEj9HGkRBd", + "outputId": "a66532b5-9e46-448a-a6b4-ddf880a6cb0b" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + " title vote_average weighted_vote vote_count\n", + "1881 The Shawshank Redemption 8.5 8.396052 8205\n", + "3337 The Godfather 8.4 8.263591 5893\n", + "662 Fight Club 8.3 8.216455 9413\n", + "3232 Pulp Fiction 8.3 8.207102 8428\n", + "65 The Dark Knight 8.2 8.136930 12002\n", + "1818 Schindler's List 8.3 8.126069 4329\n", + "3865 Whiplash 8.3 8.123248 4254\n", + "809 Forrest Gump 8.2 8.105954 7927\n", + "2294 Spirited Away 8.3 8.105867 3840\n", + "2731 The Godfather: Part II 8.3 8.079586 3338" + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
titlevote_averageweighted_votevote_count
1881The Shawshank Redemption8.58.3960528205
3337The Godfather8.48.2635915893
662Fight Club8.38.2164559413
3232Pulp Fiction8.38.2071028428
65The Dark Knight8.28.13693012002
1818Schindler's List8.38.1260694329
3865Whiplash8.38.1232484254
809Forrest Gump8.28.1059547927
2294Spirited Away8.38.1058673840
2731The Godfather: Part II8.38.0795863338
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "\n", + "
\n", + "
\n" + ], + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "dataframe", + "summary": "{\n \"name\": \" ascending=False)[:10]\",\n \"rows\": 10,\n \"fields\": [\n {\n \"column\": \"title\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 10,\n \"samples\": [\n \"Spirited Away\",\n \"The Godfather\",\n \"Schindler's List\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"vote_average\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 0.08755950357709151,\n \"min\": 8.2,\n \"max\": 8.5,\n \"num_unique_values\": 4,\n \"samples\": [\n 8.4,\n 8.2,\n 8.5\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"weighted_vote\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 0.09696608479450805,\n \"min\": 8.07958629828635,\n \"max\": 8.39605162693645,\n \"num_unique_values\": 10,\n \"samples\": [\n 8.105867158639835,\n 8.263590802034972,\n 8.126068673669016\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"vote_count\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 2866,\n \"min\": 3338,\n \"max\": 12002,\n \"num_unique_values\": 10,\n \"samples\": [\n 3840,\n 5893,\n 4329\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}" + } + }, + "metadata": {}, + "execution_count": 15 + } + ], + "source": [ + "movies_df[['title','vote_average','weighted_vote','vote_count']].sort_values('weighted_vote',\n", + " ascending=False)[:10]" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 362 + }, + "id": "_OBhgQ4wkQ9N", + "outputId": "25c2e7d3-4512-4278-bdf9-11585e6e745a" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + " title vote_average weighted_vote\n", + "1881 The Shawshank Redemption 8.5 8.396052\n", + "2731 The Godfather: Part II 8.3 8.079586\n", + "1847 GoodFellas 8.2 7.976937\n", + "3866 City of God 8.1 7.759693\n", + "1663 Once Upon a Time in America 8.2 7.657811\n", + "3887 Trainspotting 7.8 7.591009\n", + "883 Catch Me If You Can 7.7 7.557097\n", + "892 Casino 7.8 7.423040\n", + "4041 This Is England 7.4 6.739664\n", + "1149 American Hustle 6.8 6.717525" + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
titlevote_averageweighted_vote
1881The Shawshank Redemption8.58.396052
2731The Godfather: Part II8.38.079586
1847GoodFellas8.27.976937
3866City of God8.17.759693
1663Once Upon a Time in America8.27.657811
3887Trainspotting7.87.591009
883Catch Me If You Can7.77.557097
892Casino7.87.423040
4041This Is England7.46.739664
1149American Hustle6.86.717525
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "\n", + "
\n", + "
\n" + ], + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "dataframe", + "summary": "{\n \"name\": \"similar_movies[['title', 'vote_average', 'weighted_vote']]\",\n \"rows\": 10,\n \"fields\": [\n {\n \"column\": \"title\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 10,\n \"samples\": [\n \"This Is England\",\n \"The Godfather: Part II\",\n \"Trainspotting\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"vote_average\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 0.5006662228138289,\n \"min\": 6.8,\n \"max\": 8.5,\n \"num_unique_values\": 8,\n \"samples\": [\n 8.3,\n 7.7,\n 8.5\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"weighted_vote\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 0.5360110086094774,\n \"min\": 6.717525466229835,\n \"max\": 8.39605162693645,\n \"num_unique_values\": 10,\n \"samples\": [\n 6.739664363482589,\n 8.07958629828635,\n 7.591009490713154\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}" + } + }, + "metadata": {}, + "execution_count": 16 + } + ], + "source": [ + "# 콘텐츠 기반 필터링\n", + "def find_sim_movie2(df, sorted_ind, title_name, top_n=10):\n", + " title_movie = df[df['title'] == title_name]\n", + " title_index = title_movie.index.values\n", + "\n", + " # top_n의 2배 정도 유사성이 높은 index 추출\n", + " similar_indexes = sorted_ind[title_index, :(top_n*2)]\n", + " similar_indexes = similar_indexes.reshape(-1)\n", + " # 기준 영화 index 제외\n", + " similar_indexes = similar_indexes[similar_indexes != title_index]\n", + "\n", + " # top_n의 2배에 해당하는 후보군에서 weighted_vote 높은 순으로 top_n 만큼 추출\n", + " return df.iloc[similar_indexes].sort_values('weighted_vote', ascending=False)[:top_n]\n", + "\n", + "similar_movies = find_sim_movie2(movies_df, genre_sim_sorted_ind, 'The Godfather',10)\n", + "similar_movies[['title', 'vote_average', 'weighted_vote']]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "d1AynlbkKuSn" + }, + "source": [ + "# 9.6장 아이템 기반 최근접 이웃 협업 필터링 실습" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "z25F-m90lvp1" + }, + "source": [ + "##데이터 가공 및 변환\n", + "\n", + "- 협업 필터링 : 사용자와 아이템 간의 평점에 기반해 추천하는 시스템\n", + "- 행:사용자, 칼럼:영화, 값: 평점인 데이터 세트로 변경" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "RWOcqobhkQ61", + "outputId": "89f1249d-af35-4290-ca6f-9989fd4a002b" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "(9742, 3)\n", + "(100835, 4)\n" + ] + } + ], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "\n", + "movies = pd.read_csv('/content/drive/MyDrive/Sample_data/ml-latest-small/ml_latest_small_movies.csv')\n", + "ratings = pd.read_csv('/content/drive/MyDrive/Sample_data/ml-latest-small/ml_latest_small_ratings.csv')\n", + "print(movies.shape)\n", + "print(ratings.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": { + "id": "egNuTkNNmi-M" + }, + "outputs": [], + "source": [ + "ratings.columns = ['userId','movieId','rating','timestamp']" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "f4812y4Smx8p" + }, + "source": [ + "- Nan값 : 사용자가 평점을 매기지 않은 영화가 컬럼으로 변환되며 값 할당\n", + "- Nan > 0" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 224 + }, + "id": "jLtHulkqmVRR", + "outputId": "efd7fcad-796c-4011-cb5b-d71e21b8494a" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "movieId 1 2 3 4 5 6 7 8 \\\n", + "userId \n", + "1 NaN NaN 4.0 NaN NaN 4.0 NaN NaN \n", + "2 NaN NaN NaN NaN NaN NaN NaN NaN \n", + "3 NaN NaN NaN NaN NaN NaN NaN NaN \n", + "\n", + "movieId 9 10 ... 193565 193567 193571 193573 193579 193581 \\\n", + "userId ... \n", + "1 NaN NaN ... NaN NaN NaN NaN NaN NaN \n", + "2 NaN NaN ... NaN NaN NaN NaN NaN NaN \n", + "3 NaN NaN ... NaN NaN NaN NaN NaN NaN \n", + "\n", + "movieId 193583 193585 193587 193609 \n", + "userId \n", + "1 NaN NaN NaN NaN \n", + "2 NaN NaN NaN NaN \n", + "3 NaN NaN NaN NaN \n", + "\n", + "[3 rows x 9724 columns]" + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
movieId12345678910...193565193567193571193573193579193581193583193585193587193609
userId
1NaNNaN4.0NaNNaN4.0NaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
2NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
3NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN...NaNNaNNaNNaNNaNNaNNaNNaNNaNNaN
\n", + "

3 rows × 9724 columns

\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "\n", + "
\n", + "
\n" + ], + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "dataframe", + "variable_name": "ratings_matrix" + } + }, + "metadata": {}, + "execution_count": 19 + } + ], + "source": [ + "ratings=ratings[['userId','movieId','rating']]\n", + "ratings_matrix = ratings.pivot_table('rating',index='userId',columns='movieId')\n", + "ratings_matrix.head(3)" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": { + "id": "LXadwGtgm40C" + }, + "outputs": [], + "source": [ + "# title > movies와 조인\n", + "rating_movies = pd.merge(ratings,movies,on='movieId')\n", + "# columns='title'칼럼으로 피벗 수행\n", + "ratings_matrix = rating_movies.pivot_table('rating',index='userId',columns='title')" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 408 + }, + "id": "tZ48yBx3nIXH", + "outputId": "3e99c8d2-58b6-4dfc-cd24-f9b3c397cc3a" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "title '71 (2014) 'Hellboy': The Seeds of Creation (2004) \\\n", + "userId \n", + "1 0.0 0.0 \n", + "2 0.0 0.0 \n", + "3 0.0 0.0 \n", + "4 0.0 0.0 \n", + "5 0.0 0.0 \n", + "\n", + "title 'Round Midnight (1986) 'Salem's Lot (2004) \\\n", + "userId \n", + "1 0.0 0.0 \n", + "2 0.0 0.0 \n", + "3 0.0 0.0 \n", + "4 0.0 0.0 \n", + "5 0.0 0.0 \n", + "\n", + "title 'Til There Was You (1997) 'Tis the Season for Love (2015) \\\n", + "userId \n", + "1 0.0 0.0 \n", + "2 0.0 0.0 \n", + "3 0.0 0.0 \n", + "4 0.0 0.0 \n", + "5 0.0 0.0 \n", + "\n", + "title 'burbs, The (1989) 'night Mother (1986) (500) Days of Summer (2009) \\\n", + "userId \n", + "1 0.0 0.0 0.0 \n", + "2 0.0 0.0 0.0 \n", + "3 0.0 0.0 0.0 \n", + "4 0.0 0.0 0.0 \n", + "5 0.0 0.0 0.0 \n", + "\n", + "title *batteries not included (1987) ... Zulu (2013) [REC] (2007) \\\n", + "userId ... \n", + "1 0.0 ... 0.0 0.0 \n", + "2 0.0 ... 0.0 0.0 \n", + "3 0.0 ... 0.0 0.0 \n", + "4 0.0 ... 0.0 0.0 \n", + "5 0.0 ... 0.0 0.0 \n", + "\n", + "title [REC]² (2009) [REC]³ 3 Génesis (2012) \\\n", + "userId \n", + "1 0.0 0.0 \n", + "2 0.0 0.0 \n", + "3 0.0 0.0 \n", + "4 0.0 0.0 \n", + "5 0.0 0.0 \n", + "\n", + "title anohana: The Flower We Saw That Day - The Movie (2013) \\\n", + "userId \n", + "1 0.0 \n", + "2 0.0 \n", + "3 0.0 \n", + "4 0.0 \n", + "5 0.0 \n", + "\n", + "title eXistenZ (1999) xXx (2002) xXx: State of the Union (2005) \\\n", + "userId \n", + "1 0.0 0.0 0.0 \n", + "2 0.0 0.0 0.0 \n", + "3 0.0 0.0 0.0 \n", + "4 0.0 0.0 0.0 \n", + "5 0.0 0.0 0.0 \n", + "\n", + "title ¡Three Amigos! (1986) À nous la liberté (Freedom for Us) (1931) \n", + "userId \n", + "1 4.0 0.0 \n", + "2 0.0 0.0 \n", + "3 0.0 0.0 \n", + "4 0.0 0.0 \n", + "5 0.0 0.0 \n", + "\n", + "[5 rows x 9719 columns]" + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
title'71 (2014)'Hellboy': The Seeds of Creation (2004)'Round Midnight (1986)'Salem's Lot (2004)'Til There Was You (1997)'Tis the Season for Love (2015)'burbs, The (1989)'night Mother (1986)(500) Days of Summer (2009)*batteries not included (1987)...Zulu (2013)[REC] (2007)[REC]² (2009)[REC]³ 3 Génesis (2012)anohana: The Flower We Saw That Day - The Movie (2013)eXistenZ (1999)xXx (2002)xXx: State of the Union (2005)¡Three Amigos! (1986)À nous la liberté (Freedom for Us) (1931)
userId
10.00.00.00.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.04.00.0
20.00.00.00.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.00.0
30.00.00.00.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.00.0
40.00.00.00.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.00.0
50.00.00.00.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.00.0
\n", + "

5 rows × 9719 columns

\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "\n", + "
\n", + "
\n" + ], + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "dataframe", + "variable_name": "ratings_matrix" + } + }, + "metadata": {}, + "execution_count": 21 + } + ], + "source": [ + "# Nan > 0\n", + "ratings_matrix = ratings_matrix.fillna(0)\n", + "ratings_matrix.head()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "rotEn6yznOdK" + }, + "source": [ + "## 영화간 유사도 산출\n", + "\n", + "- 영화간의 유사도 : 코사인유사도(cosine_similarity())\n", + "- ratings_matrix에 적용 시 영화간 ❌, 사용자간 유사도 생성 > transpose로 데이터 행과 열 위치 변경" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 345 + }, + "id": "jZiRR4DUngHx", + "outputId": "2f0c8738-6a49-477e-f254-a3b24293d542" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "userId 1 2 3 4 5 6 7 \\\n", + "title \n", + "'71 (2014) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", + "'Hellboy': The Seeds of Creation (2004) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", + "'Round Midnight (1986) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", + "\n", + "userId 8 9 10 ... 601 602 603 \\\n", + "title ... \n", + "'71 (2014) 0.0 0.0 0.0 ... 0.0 0.0 0.0 \n", + "'Hellboy': The Seeds of Creation (2004) 0.0 0.0 0.0 ... 0.0 0.0 0.0 \n", + "'Round Midnight (1986) 0.0 0.0 0.0 ... 0.0 0.0 0.0 \n", + "\n", + "userId 604 605 606 607 608 609 610 \n", + "title \n", + "'71 (2014) 0.0 0.0 0.0 0.0 0.0 0.0 4.0 \n", + "'Hellboy': The Seeds of Creation (2004) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", + "'Round Midnight (1986) 0.0 0.0 0.0 0.0 0.0 0.0 0.0 \n", + "\n", + "[3 rows x 610 columns]" + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
userId12345678910...601602603604605606607608609610
title
'71 (2014)0.00.00.00.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.04.0
'Hellboy': The Seeds of Creation (2004)0.00.00.00.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.00.0
'Round Midnight (1986)0.00.00.00.00.00.00.00.00.00.0...0.00.00.00.00.00.00.00.00.00.0
\n", + "

3 rows × 610 columns

\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "\n", + "
\n", + "
\n" + ], + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "dataframe", + "variable_name": "ratings_matrix_T" + } + }, + "metadata": {}, + "execution_count": 22 + } + ], + "source": [ + "ratings_matrix_T = ratings_matrix.transpose()\n", + "ratings_matrix_T.head(3)" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 486 + }, + "id": "cc4aXEContKt", + "outputId": "b5c00a58-0bd4-4439-9650-00c8298a61ac" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "(9719, 9719)\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "title '71 (2014) \\\n", + "title \n", + "'71 (2014) 1.0 \n", + "'Hellboy': The Seeds of Creation (2004) 0.0 \n", + "'Round Midnight (1986) 0.0 \n", + "\n", + "title 'Hellboy': The Seeds of Creation (2004) \\\n", + "title \n", + "'71 (2014) 0.000000 \n", + "'Hellboy': The Seeds of Creation (2004) 1.000000 \n", + "'Round Midnight (1986) 0.707107 \n", + "\n", + "title 'Round Midnight (1986) \\\n", + "title \n", + "'71 (2014) 0.000000 \n", + "'Hellboy': The Seeds of Creation (2004) 0.707107 \n", + "'Round Midnight (1986) 1.000000 \n", + "\n", + "title 'Salem's Lot (2004) \\\n", + "title \n", + "'71 (2014) 0.0 \n", + "'Hellboy': The Seeds of Creation (2004) 0.0 \n", + "'Round Midnight (1986) 0.0 \n", + "\n", + "title 'Til There Was You (1997) \\\n", + "title \n", + "'71 (2014) 0.0 \n", + "'Hellboy': The Seeds of Creation (2004) 0.0 \n", + "'Round Midnight (1986) 0.0 \n", + "\n", + "title 'Tis the Season for Love (2015) \\\n", + "title \n", + "'71 (2014) 0.0 \n", + "'Hellboy': The Seeds of Creation (2004) 0.0 \n", + "'Round Midnight (1986) 0.0 \n", + "\n", + "title 'burbs, The (1989) \\\n", + "title \n", + "'71 (2014) 0.000000 \n", + "'Hellboy': The Seeds of Creation (2004) 0.000000 \n", + "'Round Midnight (1986) 0.176777 \n", + "\n", + "title 'night Mother (1986) \\\n", + "title \n", + "'71 (2014) 0.0 \n", + "'Hellboy': The Seeds of Creation (2004) 0.0 \n", + "'Round Midnight (1986) 0.0 \n", + "\n", + "title (500) Days of Summer (2009) \\\n", + "title \n", + "'71 (2014) 0.141653 \n", + "'Hellboy': The Seeds of Creation (2004) 0.000000 \n", + "'Round Midnight (1986) 0.000000 \n", + "\n", + "title *batteries not included (1987) ... \\\n", + "title ... \n", + "'71 (2014) 0.0 ... \n", + "'Hellboy': The Seeds of Creation (2004) 0.0 ... \n", + "'Round Midnight (1986) 0.0 ... \n", + "\n", + "title Zulu (2013) [REC] (2007) \\\n", + "title \n", + "'71 (2014) 0.0 0.342055 \n", + "'Hellboy': The Seeds of Creation (2004) 0.0 0.000000 \n", + "'Round Midnight (1986) 0.0 0.000000 \n", + "\n", + "title [REC]² (2009) \\\n", + "title \n", + "'71 (2014) 0.543305 \n", + "'Hellboy': The Seeds of Creation (2004) 0.000000 \n", + "'Round Midnight (1986) 0.000000 \n", + "\n", + "title [REC]³ 3 Génesis (2012) \\\n", + "title \n", + "'71 (2014) 0.707107 \n", + "'Hellboy': The Seeds of Creation (2004) 0.000000 \n", + "'Round Midnight (1986) 0.000000 \n", + "\n", + "title anohana: The Flower We Saw That Day - The Movie (2013) \\\n", + "title \n", + "'71 (2014) 0.0 \n", + "'Hellboy': The Seeds of Creation (2004) 0.0 \n", + "'Round Midnight (1986) 0.0 \n", + "\n", + "title eXistenZ (1999) xXx (2002) \\\n", + "title \n", + "'71 (2014) 0.0 0.139431 \n", + "'Hellboy': The Seeds of Creation (2004) 0.0 0.000000 \n", + "'Round Midnight (1986) 0.0 0.000000 \n", + "\n", + "title xXx: State of the Union (2005) \\\n", + "title \n", + "'71 (2014) 0.327327 \n", + "'Hellboy': The Seeds of Creation (2004) 0.000000 \n", + "'Round Midnight (1986) 0.000000 \n", + "\n", + "title ¡Three Amigos! (1986) \\\n", + "title \n", + "'71 (2014) 0.0 \n", + "'Hellboy': The Seeds of Creation (2004) 0.0 \n", + "'Round Midnight (1986) 0.0 \n", + "\n", + "title À nous la liberté (Freedom for Us) (1931) \n", + "title \n", + "'71 (2014) 0.0 \n", + "'Hellboy': The Seeds of Creation (2004) 0.0 \n", + "'Round Midnight (1986) 0.0 \n", + "\n", + "[3 rows x 9719 columns]" + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
title'71 (2014)'Hellboy': The Seeds of Creation (2004)'Round Midnight (1986)'Salem's Lot (2004)'Til There Was You (1997)'Tis the Season for Love (2015)'burbs, The (1989)'night Mother (1986)(500) Days of Summer (2009)*batteries not included (1987)...Zulu (2013)[REC] (2007)[REC]² (2009)[REC]³ 3 Génesis (2012)anohana: The Flower We Saw That Day - The Movie (2013)eXistenZ (1999)xXx (2002)xXx: State of the Union (2005)¡Three Amigos! (1986)À nous la liberté (Freedom for Us) (1931)
title
'71 (2014)1.00.0000000.0000000.00.00.00.0000000.00.1416530.0...0.00.3420550.5433050.7071070.00.00.1394310.3273270.00.0
'Hellboy': The Seeds of Creation (2004)0.01.0000000.7071070.00.00.00.0000000.00.0000000.0...0.00.0000000.0000000.0000000.00.00.0000000.0000000.00.0
'Round Midnight (1986)0.00.7071071.0000000.00.00.00.1767770.00.0000000.0...0.00.0000000.0000000.0000000.00.00.0000000.0000000.00.0
\n", + "

3 rows × 9719 columns

\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "\n", + "
\n", + "
\n" + ], + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "dataframe", + "variable_name": "item_sim_df" + } + }, + "metadata": {}, + "execution_count": 23 + } + ], + "source": [ + "from sklearn.metrics.pairwise import cosine_similarity\n", + "\n", + "item_sim = cosine_similarity(ratings_matrix_T, ratings_matrix_T)\n", + "\n", + "# cosine_similarity() 로 반환된 넘파이 행렬을 영화명을 매핑하여 DataFrame으로 변환\n", + "item_sim_df = pd.DataFrame(data=item_sim, index=ratings_matrix.columns,\n", + " columns=ratings_matrix.columns)\n", + "print(item_sim_df.shape)\n", + "item_sim_df.head(3)" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 303 + }, + "id": "TuRXxYl-nwCW", + "outputId": "2058f933-66ce-4f4e-b7b4-6fa2b7f144ee" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "title\n", + "Godfather, The (1972) 1.000000\n", + "Godfather: Part II, The (1974) 0.821773\n", + "Goodfellas (1990) 0.664841\n", + "One Flew Over the Cuckoo's Nest (1975) 0.620536\n", + "Star Wars: Episode IV - A New Hope (1977) 0.595317\n", + "Fargo (1996) 0.588614\n", + "Name: Godfather, The (1972), dtype: float64" + ], + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Godfather, The (1972)
title
Godfather, The (1972)1.000000
Godfather: Part II, The (1974)0.821773
Goodfellas (1990)0.664841
One Flew Over the Cuckoo's Nest (1975)0.620536
Star Wars: Episode IV - A New Hope (1977)0.595317
Fargo (1996)0.588614
\n", + "

" + ] + }, + "metadata": {}, + "execution_count": 24 + } + ], + "source": [ + "item_sim_df[\"Godfather, The (1972)\"].sort_values(ascending=False)[:6]" + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 272 + }, + "id": "P8jXHkOgny9d", + "outputId": "cf84625c-adbe-49b3-d6f9-2222c95cd3db" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "title\n", + "Dark Knight, The (2008) 0.727263\n", + "Inglourious Basterds (2009) 0.646103\n", + "Shutter Island (2010) 0.617736\n", + "Dark Knight Rises, The (2012) 0.617504\n", + "Fight Club (1999) 0.615417\n", + "Name: Inception (2010), dtype: float64" + ], + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Inception (2010)
title
Dark Knight, The (2008)0.727263
Inglourious Basterds (2009)0.646103
Shutter Island (2010)0.617736
Dark Knight Rises, The (2012)0.617504
Fight Club (1999)0.615417
\n", + "

" + ] + }, + "metadata": {}, + "execution_count": 25 + } + ], + "source": [ + "item_sim_df[\"Inception (2010)\"].sort_values(ascending=False)[1:6]" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "4SH9TXJano0H" + }, + "source": [ + "## 아이템 기반 인접 이웃 협업 필터링으로 **개인화**된 영화 추천\n", + "\n", + "$$\\hat{R}_{u,i} = \\sum(S_{(i,N)}*R_{(u,N)}) / \\sum(|S_{(i,N)}|) $$\n", + "\n", + "- $\\hat{R}$ : 개인화된 예측 평점값\n", + "- S : 아이템 i와 가장 유사도가 높은 top-n개 아이템의 유사도 벡터\n", + "- R : 사용자 u 아이템 i와 가장 유사도가 높은 Top-N개 아이템에 대한 실제 평점 벡터\n", + "\n", + "- 예측 평점이 실제 평점과 영화의 코사인 유사도를 내적(+백터합으로 나눔)한 것이기 때매 실제 평점보다 작을 수 있음\n", + "- 예측 평가지표 MSE : get_mse() 사용자 정의 함수" + ] + }, + { + "cell_type": "code", + "execution_count": 26, + "metadata": { + "id": "aYLc3SeMoDbX" + }, + "outputs": [], + "source": [ + "def predict_rating(ratings_arr, item_sim_arr ):\n", + " ratings_pred = ratings_arr.dot(item_sim_arr)/ np.array([np.abs(item_sim_arr).sum(axis=1)])\n", + " return ratings_pred" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 345 + }, + "id": "wtDO7EBxoEmI", + "outputId": "884be5c3-03a6-4fb8-abd3-512a3a47a5a3" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "title '71 (2014) 'Hellboy': The Seeds of Creation (2004) \\\n", + "userId \n", + "1 0.069675 0.571866 \n", + "2 0.018260 0.042744 \n", + "3 0.011884 0.030279 \n", + "\n", + "title 'Round Midnight (1986) 'Salem's Lot (2004) \\\n", + "userId \n", + "1 0.319053 0.227055 \n", + "2 0.018861 0.000000 \n", + "3 0.064437 0.003762 \n", + "\n", + "title 'Til There Was You (1997) 'Tis the Season for Love (2015) \\\n", + "userId \n", + "1 0.206958 0.192404 \n", + "2 0.000000 0.035995 \n", + "3 0.003749 0.002721 \n", + "\n", + "title 'burbs, The (1989) 'night Mother (1986) (500) Days of Summer (2009) \\\n", + "userId \n", + "1 0.248972 0.102214 0.156064 \n", + "2 0.013413 0.002314 0.032213 \n", + "3 0.014625 0.002085 0.005666 \n", + "\n", + "title *batteries not included (1987) ... Zulu (2013) [REC] (2007) \\\n", + "userId ... \n", + "1 0.177548 ... 0.113120 0.180250 \n", + "2 0.014863 ... 0.015640 0.020855 \n", + "3 0.006272 ... 0.006923 0.011665 \n", + "\n", + "title [REC]² (2009) [REC]³ 3 Génesis (2012) \\\n", + "userId \n", + "1 0.133235 0.127794 \n", + "2 0.020119 0.015745 \n", + "3 0.011800 0.012225 \n", + "\n", + "title anohana: The Flower We Saw That Day - The Movie (2013) \\\n", + "userId \n", + "1 0.006179 \n", + "2 0.049983 \n", + "3 0.000000 \n", + "\n", + "title eXistenZ (1999) xXx (2002) xXx: State of the Union (2005) \\\n", + "userId \n", + "1 0.211527 0.191956 0.134909 \n", + "2 0.014876 0.021616 0.024528 \n", + "3 0.008194 0.007017 0.009229 \n", + "\n", + "title ¡Three Amigos! (1986) À nous la liberté (Freedom for Us) (1931) \n", + "userId \n", + "1 0.291970 0.720347 \n", + "2 0.017563 0.000000 \n", + "3 0.010420 0.084501 \n", + "\n", + "[3 rows x 9719 columns]" + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
title'71 (2014)'Hellboy': The Seeds of Creation (2004)'Round Midnight (1986)'Salem's Lot (2004)'Til There Was You (1997)'Tis the Season for Love (2015)'burbs, The (1989)'night Mother (1986)(500) Days of Summer (2009)*batteries not included (1987)...Zulu (2013)[REC] (2007)[REC]² (2009)[REC]³ 3 Génesis (2012)anohana: The Flower We Saw That Day - The Movie (2013)eXistenZ (1999)xXx (2002)xXx: State of the Union (2005)¡Three Amigos! (1986)À nous la liberté (Freedom for Us) (1931)
userId
10.0696750.5718660.3190530.2270550.2069580.1924040.2489720.1022140.1560640.177548...0.1131200.1802500.1332350.1277940.0061790.2115270.1919560.1349090.2919700.720347
20.0182600.0427440.0188610.0000000.0000000.0359950.0134130.0023140.0322130.014863...0.0156400.0208550.0201190.0157450.0499830.0148760.0216160.0245280.0175630.000000
30.0118840.0302790.0644370.0037620.0037490.0027210.0146250.0020850.0056660.006272...0.0069230.0116650.0118000.0122250.0000000.0081940.0070170.0092290.0104200.084501
\n", + "

3 rows × 9719 columns

\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "\n", + "
\n", + "
\n" + ], + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "dataframe", + "variable_name": "ratings_pred_matrix" + } + }, + "metadata": {}, + "execution_count": 27 + } + ], + "source": [ + "ratings_pred = predict_rating(ratings_matrix.values , item_sim_df.values)\n", + "ratings_pred_matrix = pd.DataFrame(data=ratings_pred, index= ratings_matrix.index,\n", + " columns = ratings_matrix.columns)\n", + "ratings_pred_matrix.head(3)" + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "i46Vc6qmoHnQ", + "outputId": "f3b438f9-d386-4f91-8cd5-41c333ad7668" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "아이템 기반 모든 인접 이웃 MSE: 9.895359667092327\n" + ] + } + ], + "source": [ + "from sklearn.metrics import mean_squared_error\n", + "\n", + "# 사용자가 평점을 부여한 영화에 대해서만 예측 성능 평가 MSE 를 구함.\n", + "def get_mse(pred, actual):\n", + " # Ignore nonzero terms.\n", + " pred = pred[actual.nonzero()].flatten()\n", + " actual = actual[actual.nonzero()].flatten()\n", + " return mean_squared_error(pred, actual)\n", + "\n", + "print('아이템 기반 모든 인접 이웃 MSE: ', get_mse(ratings_pred, ratings_matrix.values ))" + ] + }, + { + "cell_type": "code", + "execution_count": 29, + "metadata": { + "id": "ExWZelwLoHk5" + }, + "outputs": [], + "source": [ + "def predict_rating_topsim(ratings_arr, item_sim_arr, n=20):\n", + " # 사용자-아이템 평점 행렬 크기와 같은 예측 행렬 0으로 초기화\n", + " pred = np.zeros(ratings_arr.shape)\n", + "\n", + " # 사용자-아이템 평점 행렬의 열 크기만큼 반복 : 데이터크면 매우 오래걸림\n", + " for col in range(ratings_arr.shape[1]):\n", + " # 유사도 행렬 > 유사도가 큰 순 n개 데이터 행렬의 index 반환\n", + " top_n_items = [np.argsort(item_sim_arr[:, col])[:-n-1:-1]]\n", + " # 예측 평점 계산\n", + " for row in range(ratings_arr.shape[0]):\n", + " pred[row, col] = item_sim_arr[col, :][top_n_items].dot(ratings_arr[row, :][top_n_items].T)\n", + " pred[row, col] /= np.sum(np.abs(item_sim_arr[col, :][top_n_items]))\n", + " return pred" + ] + }, + { + "cell_type": "code", + "execution_count": 30, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Tk0xad46oHfo", + "outputId": "8988da2d-2853-4895-d70c-10d07afb0873" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "아이템 기반 인접 TOP-20 이웃 MSE: 3.6945760072593754\n" + ] + } + ], + "source": [ + "ratings_pred = predict_rating_topsim(ratings_matrix.values , item_sim_df.values, n=20)\n", + "print('아이템 기반 인접 TOP-20 이웃 MSE: ', get_mse(ratings_pred, ratings_matrix.values ))\n", + "\n", + "\n", + "# 행렬 > DataFrame\n", + "ratings_pred_matrix = pd.DataFrame(data=ratings_pred, index= ratings_matrix.index,\n", + " columns = ratings_matrix.columns);" + ] + }, + { + "cell_type": "code", + "execution_count": 31, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 429 + }, + "id": "2DT9N0fHoHdE", + "outputId": "23169d48-cb57-4658-daa5-5e821d3ea349" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "title\n", + "Adaptation (2002) 5.0\n", + "Austin Powers in Goldmember (2002) 5.0\n", + "Back to the Future (1985) 5.0\n", + "Citizen Kane (1941) 5.0\n", + "Lord of the Rings: The Fellowship of the Ring, The (2001) 5.0\n", + "Lord of the Rings: The Two Towers, The (2002) 5.0\n", + "Producers, The (1968) 5.0\n", + "Raiders of the Lost Ark (Indiana Jones and the Raiders of the Lost Ark) (1981) 5.0\n", + "Elling (2001) 4.0\n", + "King of Comedy, The (1983) 4.0\n", + "Name: 9, dtype: float64" + ], + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
9
title
Adaptation (2002)5.0
Austin Powers in Goldmember (2002)5.0
Back to the Future (1985)5.0
Citizen Kane (1941)5.0
Lord of the Rings: The Fellowship of the Ring, The (2001)5.0
Lord of the Rings: The Two Towers, The (2002)5.0
Producers, The (1968)5.0
Raiders of the Lost Ark (Indiana Jones and the Raiders of the Lost Ark) (1981)5.0
Elling (2001)4.0
King of Comedy, The (1983)4.0
\n", + "

" + ] + }, + "metadata": {}, + "execution_count": 31 + } + ], + "source": [ + "user_rating_id = ratings_matrix.loc[9, :]\n", + "user_rating_id[ user_rating_id > 0].sort_values(ascending=False)[:10]" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": { + "id": "dp7u0PnCoOdb" + }, + "outputs": [], + "source": [ + "def get_unseen_movies(ratings_matrix, userId):\n", + " # userId > Series\n", + " # 반환된 user_rating 은 영화명(title)을 index로 가지는 Series 객체임.\n", + " user_rating = ratings_matrix.loc[userId,:]\n", + "\n", + " # user_rating > 0 : 기존에 관람한 영화. 대상 index를 추출 > list 객체\n", + " already_seen = user_rating[ user_rating > 0].index.tolist()\n", + "\n", + " # 모든 영화명을 list 객체로 만듬.\n", + " movies_list = ratings_matrix.columns.tolist()\n", + "\n", + " # already_seen에 해당하는 movie는 movies_list에서 제거\n", + " unseen_list = [ movie for movie in movies_list if movie not in already_seen]\n", + "\n", + " return unseen_list" + ] + }, + { + "cell_type": "code", + "execution_count": 42, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 394 + }, + "id": "Qz0eZsFJoObE", + "outputId": "525cddd9-1e16-4369-cbe2-6873f4ca06e9" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + " pred_score\n", + "title \n", + "Rear Window (1954) 5.704191\n", + "South Park: Bigger, Longer and Uncut (1999) 5.454730\n", + "Rounders (1998) 5.296919\n", + "Blade Runner (1982) 5.244503\n", + "Roger & Me (1989) 5.190057\n", + "Gattaca (1997) 5.184963\n", + "Ben-Hur (1959) 5.130873\n", + "Rosencrantz and Guildenstern Are Dead (1990) 5.088743\n", + "Big Lebowski, The (1998) 5.039282\n", + "Star Wars: Episode V - The Empire Strikes Back (1980) 4.988345" + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
pred_score
title
Rear Window (1954)5.704191
South Park: Bigger, Longer and Uncut (1999)5.454730
Rounders (1998)5.296919
Blade Runner (1982)5.244503
Roger & Me (1989)5.190057
Gattaca (1997)5.184963
Ben-Hur (1959)5.130873
Rosencrantz and Guildenstern Are Dead (1990)5.088743
Big Lebowski, The (1998)5.039282
Star Wars: Episode V - The Empire Strikes Back (1980)4.988345
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "\n", + "
\n", + " \n", + " \n", + " \n", + "
\n", + "\n", + "
\n", + "
\n" + ], + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "dataframe", + "variable_name": "recomm_movies", + "summary": "{\n \"name\": \"recomm_movies\",\n \"rows\": 10,\n \"fields\": [\n {\n \"column\": \"title\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 10,\n \"samples\": [\n \"Big Lebowski, The (1998)\",\n \"South Park: Bigger, Longer and Uncut (1999)\",\n \"Gattaca (1997)\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"pred_score\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 0.21296119475115463,\n \"min\": 4.988345165213479,\n \"max\": 5.7041913343685975,\n \"num_unique_values\": 10,\n \"samples\": [\n 5.039282124749987,\n 5.454730457590612,\n 5.18496338285511\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}" + } + }, + "metadata": {}, + "execution_count": 42 + } + ], + "source": [ + "def recomm_movie_by_userid(pred_df, userId, unseen_list, top_n=10):\n", + " # 예측 평점 DF에서 사용자id index와 unseen_list로 들어온 영화명 컬럼추출\n", + " # > 높은 순 정렬.\n", + " recomm_movies = pred_df.loc[userId, unseen_list].sort_values(ascending=False)[:top_n]\n", + " return recomm_movies\n", + "\n", + "# 사용자 관람X 영화명 추출\n", + "unseen_list = get_unseen_movies(ratings_matrix, 9)\n", + "\n", + "# 아이템 기반 인접 이웃 협업 필터링 : 영화 추천\n", + "recomm_movies = recomm_movie_by_userid(ratings_pred_matrix, 9, unseen_list, top_n=10)\n", + "\n", + "# 행렬 > DF\n", + "recomm_movies = pd.DataFrame(data=recomm_movies.values,index=recomm_movies.index,columns=['pred_score'])\n", + "recomm_movies" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "_JpYjLzfK0DV" + }, + "source": [ + "# 9.7장 행렬 분해를 이용한 잠재 요인 협업 필터링 실습" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "CWv4UowwoZnh" + }, + "source": [ + "Null data 多> SGD/ALS (SVD,NMF ❌)\n", + "\n", + "### matrix_factorization 함수 ( 행렬 분해 함수 )\n", + "- input : R,K,steps,learning_rate,r_lambda\n", + "- R: 실제 행렬\n", + "- K: 잠재요인수\n", + "- learning_rate: 학습률\n", + "- r_lambda : L2 규제 계수" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": { + "id": "moQFNiKHoOYr" + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "from sklearn.metrics import mean_squared_error\n", + "\n", + "def get_rmse(R, P, Q, non_zeros):\n", + " error = 0\n", + " # P * Q.T > 예측 R\n", + " full_pred_matrix = np.dot(P, Q.T)\n", + "\n", + " # 실제 R 행렬에서 널이 아닌 값의 위치 인덱스 추출\n", + " # > 실제 R - 예측 RMSE 추출\n", + " x_non_zero_ind = [non_zero[0] for non_zero in non_zeros]\n", + " y_non_zero_ind = [non_zero[1] for non_zero in non_zeros]\n", + " R_non_zeros = R[x_non_zero_ind, y_non_zero_ind]\n", + "\n", + " full_pred_matrix_non_zeros = full_pred_matrix[x_non_zero_ind, y_non_zero_ind]\n", + "\n", + " mse = mean_squared_error(R_non_zeros, full_pred_matrix_non_zeros)\n", + " rmse = np.sqrt(mse)\n", + "\n", + " return rmse" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": { + "id": "c9Nd2EsvofEw" + }, + "outputs": [], + "source": [ + "def matrix_factorization(R, K, steps=200, learning_rate=0.01, r_lambda = 0.01):\n", + " num_users, num_items = R.shape\n", + " # P Q 크기 지정/ 정규분포 랜덤값입력\n", + " np.random.seed(1)\n", + " P = np.random.normal(scale=1./K, size=(num_users, K))\n", + " Q = np.random.normal(scale=1./K, size=(num_items, K))\n", + "\n", + " # R > 0 인 행 위치, 열 위치, 값을 non_zeros 리스트 객체에 저장.\n", + " non_zeros = [ (i, j, R[i,j]) for i in range(num_users) for j in range(num_items) if R[i,j] > 0 ]\n", + "\n", + " # SGD기법으로 P, Q 계속 업데이트.\n", + " for step in range(steps):\n", + " for i, j, r in non_zeros:\n", + " # 오류 : 실제 값 - 예측 값\n", + " eij = r - np.dot(P[i, :], Q[j, :].T)\n", + " # Regularization을 반영한 SGD 업데이트 공식 적용\n", + " P[i,:] = P[i,:] + learning_rate*(eij * Q[j, :] - r_lambda*P[i,:])\n", + " Q[j,:] = Q[j,:] + learning_rate*(eij * P[i, :] - r_lambda*Q[j,:])\n", + "\n", + " rmse = get_rmse(R, P, Q, non_zeros)\n", + " if (step % 10) == 0 :\n", + " print(\"### iteration step : \", step,\" rmse : \", rmse)\n", + "\n", + " return P, Q" + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": { + "id": "EubEIBP4ogoV" + }, + "outputs": [], + "source": [ + "import pandas as pd\n", + "import numpy as np\n", + "movies = pd.read_csv('/content/drive/MyDrive/Sample_data/ml-latest-small/ml_latest_small_movies.csv')\n", + "ratings = pd.read_csv('/content/drive/MyDrive/Sample_data/ml-latest-small/ml_latest_small_ratings.csv')\n", + "ratings.columns = ['userId', 'movieId', 'rating','timestamp']\n", + "ratings = ratings[['userId', 'movieId', 'rating']]\n", + "ratings_matrix = ratings.pivot_table('rating', index='userId', columns='movieId')\n", + "\n", + "# title 컬럼을 얻기 이해 movies 와 조인 수행\n", + "rating_movies = pd.merge(ratings, movies, on='movieId')\n", + "\n", + "# columns='title' 로 title 컬럼으로 pivot 수행.\n", + "ratings_matrix = rating_movies.pivot_table('rating', index='userId', columns='title')" + ] + }, + { + "cell_type": "code", + "execution_count": 37, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "2CIdf8ETo3NG", + "outputId": "222f52e9-3134-45ab-98ae-d2b06d376e49" + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "### iteration step : 0 rmse : 2.902361483117536\n", + "### iteration step : 10 rmse : 0.7335761698317158\n", + "### iteration step : 20 rmse : 0.5115603383967597\n", + "### iteration step : 30 rmse : 0.3726213866510964\n", + "### iteration step : 40 rmse : 0.2960862274283514\n", + "### iteration step : 50 rmse : 0.2520402550097047\n", + "### iteration step : 60 rmse : 0.22488030600834652\n", + "### iteration step : 70 rmse : 0.20685967417903406\n", + "### iteration step : 80 rmse : 0.19413888540756688\n", + "### iteration step : 90 rmse : 0.18470501296551115\n", + "### iteration step : 100 rmse : 0.17743297964022212\n", + "### iteration step : 110 rmse : 0.17165553771539724\n", + "### iteration step : 120 rmse : 0.16695470884206962\n", + "### iteration step : 130 rmse : 0.16305548283230267\n", + "### iteration step : 140 rmse : 0.1597691912320195\n", + "### iteration step : 150 rmse : 0.15696188229537253\n", + "### iteration step : 160 rmse : 0.1545357555855267\n", + "### iteration step : 170 rmse : 0.1524177353314282\n", + "### iteration step : 180 rmse : 0.15055214325698385\n", + "### iteration step : 190 rmse : 0.14889583749227855\n" + ] + } + ], + "source": [ + "P, Q = matrix_factorization(ratings_matrix.values, K=50, steps=200, learning_rate=0.01, r_lambda = 0.01)\n", + "pred_matrix = np.dot(P, Q.T)" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 345 + }, + "id": "XMfuemLXo7d2", + "outputId": "476adf79-8a9d-4a74-a2d2-03f2ae110214" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "title '71 (2014) 'Hellboy': The Seeds of Creation (2004) \\\n", + "userId \n", + "1 3.053710 4.093457 \n", + "2 3.170378 3.657983 \n", + "3 2.306473 1.658783 \n", + "\n", + "title 'Round Midnight (1986) 'Salem's Lot (2004) \\\n", + "userId \n", + "1 3.565621 4.501866 \n", + "2 3.308636 4.166149 \n", + "3 1.443389 2.208475 \n", + "\n", + "title 'Til There Was You (1997) 'Tis the Season for Love (2015) \\\n", + "userId \n", + "1 3.979732 1.270481 \n", + "2 4.311504 1.275675 \n", + "3 2.228921 0.780457 \n", + "\n", + "title 'burbs, The (1989) 'night Mother (1986) (500) Days of Summer (2009) \\\n", + "userId \n", + "1 3.610400 2.332452 5.079653 \n", + "2 4.238628 1.900196 3.393151 \n", + "3 1.995356 0.924486 2.975363 \n", + "\n", + "title *batteries not included (1987) ... Zulu (2013) [REC] (2007) \\\n", + "userId ... \n", + "1 3.972328 ... 1.404236 4.216634 \n", + "2 3.647428 ... 0.973841 3.528000 \n", + "3 2.551171 ... 0.520532 1.709802 \n", + "\n", + "title [REC]² (2009) [REC]³ 3 Génesis (2012) \\\n", + "userId \n", + "1 3.707188 2.721879 \n", + "2 3.361705 2.672623 \n", + "3 2.281183 1.782552 \n", + "\n", + "title anohana: The Flower We Saw That Day - The Movie (2013) \\\n", + "userId \n", + "1 2.787248 \n", + "2 2.404345 \n", + "3 1.635064 \n", + "\n", + "title eXistenZ (1999) xXx (2002) xXx: State of the Union (2005) \\\n", + "userId \n", + "1 3.472398 3.245085 2.159882 \n", + "2 4.231644 2.911773 1.634416 \n", + "3 1.318541 2.887611 1.042975 \n", + "\n", + "title ¡Three Amigos! (1986) À nous la liberté (Freedom for Us) (1931) \n", + "userId \n", + "1 4.009836 0.859327 \n", + "2 4.134683 0.725547 \n", + "3 2.294125 0.396788 \n", + "\n", + "[3 rows x 9719 columns]" + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
title'71 (2014)'Hellboy': The Seeds of Creation (2004)'Round Midnight (1986)'Salem's Lot (2004)'Til There Was You (1997)'Tis the Season for Love (2015)'burbs, The (1989)'night Mother (1986)(500) Days of Summer (2009)*batteries not included (1987)...Zulu (2013)[REC] (2007)[REC]² (2009)[REC]³ 3 Génesis (2012)anohana: The Flower We Saw That Day - The Movie (2013)eXistenZ (1999)xXx (2002)xXx: State of the Union (2005)¡Three Amigos! (1986)À nous la liberté (Freedom for Us) (1931)
userId
13.0537104.0934573.5656214.5018663.9797321.2704813.6104002.3324525.0796533.972328...1.4042364.2166343.7071882.7218792.7872483.4723983.2450852.1598824.0098360.859327
23.1703783.6579833.3086364.1661494.3115041.2756754.2386281.9001963.3931513.647428...0.9738413.5280003.3617052.6726232.4043454.2316442.9117731.6344164.1346830.725547
32.3064731.6587831.4433892.2084752.2289210.7804571.9953560.9244862.9753632.551171...0.5205321.7098022.2811831.7825521.6350641.3185412.8876111.0429752.2941250.396788
\n", + "

3 rows × 9719 columns

\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "\n", + "
\n", + "
\n" + ], + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "dataframe", + "variable_name": "ratings_pred_matrix" + } + }, + "metadata": {}, + "execution_count": 38 + } + ], + "source": [ + "# 행렬 > DF\n", + "ratings_pred_matrix = pd.DataFrame(data=pred_matrix, index= ratings_matrix.index,\n", + " columns = ratings_matrix.columns)\n", + "\n", + "ratings_pred_matrix.head(3)" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 394 + }, + "id": "-oCkMZXQo7Sv", + "outputId": "f6cbfae7-3660-47ea-b7a6-5e6cf636936b" + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + " pred_score\n", + "title \n", + "Rear Window (1954) 5.704191\n", + "South Park: Bigger, Longer and Uncut (1999) 5.454730\n", + "Rounders (1998) 5.296919\n", + "Blade Runner (1982) 5.244503\n", + "Roger & Me (1989) 5.190057\n", + "Gattaca (1997) 5.184963\n", + "Ben-Hur (1959) 5.130873\n", + "Rosencrantz and Guildenstern Are Dead (1990) 5.088743\n", + "Big Lebowski, The (1998) 5.039282\n", + "Star Wars: Episode V - The Empire Strikes Back (1980) 4.988345" + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
pred_score
title
Rear Window (1954)5.704191
South Park: Bigger, Longer and Uncut (1999)5.454730
Rounders (1998)5.296919
Blade Runner (1982)5.244503
Roger & Me (1989)5.190057
Gattaca (1997)5.184963
Ben-Hur (1959)5.130873
Rosencrantz and Guildenstern Are Dead (1990)5.088743
Big Lebowski, The (1998)5.039282
Star Wars: Episode V - The Empire Strikes Back (1980)4.988345
\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "\n", + "
\n", + " \n", + " \n", + " \n", + "
\n", + "\n", + "
\n", + "
\n" + ], + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "dataframe", + "variable_name": "recomm_movies", + "summary": "{\n \"name\": \"recomm_movies\",\n \"rows\": 10,\n \"fields\": [\n {\n \"column\": \"title\",\n \"properties\": {\n \"dtype\": \"string\",\n \"num_unique_values\": 10,\n \"samples\": [\n \"Big Lebowski, The (1998)\",\n \"South Park: Bigger, Longer and Uncut (1999)\",\n \"Gattaca (1997)\"\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n },\n {\n \"column\": \"pred_score\",\n \"properties\": {\n \"dtype\": \"number\",\n \"std\": 0.21296119475115463,\n \"min\": 4.988345165213479,\n \"max\": 5.7041913343685975,\n \"num_unique_values\": 10,\n \"samples\": [\n 5.039282124749987,\n 5.454730457590612,\n 5.18496338285511\n ],\n \"semantic_type\": \"\",\n \"description\": \"\"\n }\n }\n ]\n}" + } + }, + "metadata": {}, + "execution_count": 41 + } + ], + "source": [ + "unseen_list = get_unseen_movies(ratings_matrix, 9)\n", + "\n", + "# 아이템 기반의 인접 이웃 협업 필터링으로 영화 추천\n", + "recomm_movies = recomm_movie_by_userid(ratings_pred_matrix, 9, unseen_list, top_n=10)\n", + "\n", + "# 평점 데이타를 DataFrame으로 생성.\n", + "recomm_movies = pd.DataFrame(data=recomm_movies.values,index=recomm_movies.index,columns=['pred_score'])\n", + "recomm_movies" + ] + }, + { + "cell_type": "code", + "execution_count": 41, + "metadata": { + "id": "9dQbMwuTpFS_" + }, + "outputs": [], + "source": [] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} \ No newline at end of file diff --git "a/Week16_\354\230\210\354\212\265\352\263\274\354\240\234_\354\236\245\354\204\234\354\227\260.ipynb" "b/Week16_\354\230\210\354\212\265\352\263\274\354\240\234_\354\236\245\354\204\234\354\227\260.ipynb" new file mode 100644 index 0000000..2eb29c5 --- /dev/null +++ "b/Week16_\354\230\210\354\212\265\352\263\274\354\240\234_\354\236\245\354\204\234\354\227\260.ipynb" @@ -0,0 +1,1468 @@ +{ + "nbformat": 4, + "nbformat_minor": 0, + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "name": "python3", + "display_name": "Python 3" + }, + "language_info": { + "name": "python" + } + }, + "cells": [ + { + "cell_type": "markdown", + "source": [ + "# 9.4 잠재요인 협업 필터링\n", + "## 확률적 경사하강법을 이용한 행렬 분해" + ], + "metadata": { + "id": "ZhhbpXTwnaT0" + } + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "id": "Z5ZBTVwHZhez" + }, + "outputs": [], + "source": [ + "import numpy as np\n", + "\n", + "# 원본 행렬 R 생성, 분해 행렬 P와 Q 초기화, 잠재요인 차원 K는 3 설정.\n", + "R = np.array([[4, np.nan, np.nan, 2, np.nan ],\n", + " [np.nan, 5, np.nan, 3, 1 ],\n", + " [np.nan, np.nan, 3, 4, 4 ],\n", + " [5, 2, 1, 2, np.nan ]])\n", + "num_users, num_items = R.shape\n", + "K=3\n", + "\n", + "# P와 Q 매트릭스의 크기를 지정 / 정규분포를 가진 random값 입력\n", + "\n", + "np.random.seed(1)\n", + "P = np.random.normal(scale=1./K, size=(num_users, K))\n", + "Q = np.random.normal(scale=1./K, size=(num_items, K))" + ] + }, + { + "cell_type": "code", + "source": [ + "from sklearn.metrics import mean_squared_error\n", + "\n", + "def get_rmse(R, P, Q, non_zeros):\n", + " error = 0\n", + " # P * Q.T > 예측 R 행렬 생성\n", + " full_pred_matrix = np.dot(P, Q.T)\n", + "\n", + " # 실제 R 행렬에서 널이 아닌 값의 위치 인덱스 추출 > 실제 R 행렬과 예측 행렬의 RMSE 추출\n", + " x_non_zero_ind = [non_zero[0] for non_zero in non_zeros]\n", + " y_non_zero_ind = [non_zero[1] for non_zero in non_zeros]\n", + " R_non_zeros = R[x_non_zero_ind, y_non_zero_ind]\n", + " full_pred_matrix_non_zeros = full_pred_matrix[x_non_zero_ind, y_non_zero_ind]\n", + "\n", + " mse = mean_squared_error(R_non_zeros, full_pred_matrix_non_zeros)\n", + " rmse = np.sqrt(mse)\n", + "\n", + " return rmse" + ], + "metadata": { + "id": "6f7UKSAsnmET" + }, + "execution_count": 7, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "# R > 0 인 행 위치, 열 위치, 값을 non_zeros 리스트에 저장.\n", + "non_zeros = [ (i, j, R[i,j]) for i in range(num_users) for j in range(num_items) if R[i,j] > 0 ]\n", + "\n", + "steps=1000\n", + "learning_rate=0.01\n", + "r_lambda=0.01\n", + "\n", + "# SGD : P와 Q 계속 업데이트\n", + "for step in range(steps):\n", + " for i, j, r in non_zeros:\n", + " # 실제 값 - 예측 값\n", + " eij = r - np.dot(P[i, :], Q[j, :].T)\n", + " # Regularization을 반영한 SGD 업데이트 공식 적용\n", + " P[i,:] = P[i,:] + learning_rate*(eij * Q[j, :] - r_lambda*P[i,:])\n", + " Q[j,:] = Q[j,:] + learning_rate*(eij * P[i, :] - r_lambda*Q[j,:])\n", + "\n", + " rmse = get_rmse(R, P, Q, non_zeros)\n", + " if (step % 50) == 0 :\n", + " print(\"### iteration step : \", step,\" rmse : \", rmse)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "mlJ1o6uAnn9S", + "outputId": "efdced4a-2ebf-4317-c429-54ac4838f8e1" + }, + "execution_count": 8, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "### iteration step : 0 rmse : 3.2388050277987723\n", + "### iteration step : 50 rmse : 0.4876723101369648\n", + "### iteration step : 100 rmse : 0.1564340384819247\n", + "### iteration step : 150 rmse : 0.07455141311978046\n", + "### iteration step : 200 rmse : 0.04325226798579314\n", + "### iteration step : 250 rmse : 0.029248328780878973\n", + "### iteration step : 300 rmse : 0.022621116143829466\n", + "### iteration step : 350 rmse : 0.019493636196525135\n", + "### iteration step : 400 rmse : 0.018022719092132704\n", + "### iteration step : 450 rmse : 0.01731968595344266\n", + "### iteration step : 500 rmse : 0.016973657887570753\n", + "### iteration step : 550 rmse : 0.016796804595895633\n", + "### iteration step : 600 rmse : 0.01670132290188466\n", + "### iteration step : 650 rmse : 0.01664473691247669\n", + "### iteration step : 700 rmse : 0.016605910068210026\n", + "### iteration step : 750 rmse : 0.016574200475705\n", + "### iteration step : 800 rmse : 0.01654431582921597\n", + "### iteration step : 850 rmse : 0.01651375177473524\n", + "### iteration step : 900 rmse : 0.01648146573819501\n", + "### iteration step : 950 rmse : 0.016447171683479155\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "pred_matrix = np.dot(P, Q.T)\n", + "print('예측 행렬:\\n', np.round(pred_matrix, 3))" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "aWQROvlsoYBp", + "outputId": "475bdba8-61b9-42f6-fc99-5746b7aed92c" + }, + "execution_count": 9, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "예측 행렬:\n", + " [[3.991 0.897 1.306 2.002 1.663]\n", + " [6.696 4.978 0.979 2.981 1.003]\n", + " [6.677 0.391 2.987 3.977 3.986]\n", + " [4.968 2.005 1.006 2.017 1.14 ]]\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "# 9.8 파이썬 추천 시스템 패키지 - Surprise\n" + ], + "metadata": { + "id": "En1IsNHlogcq" + } + }, + { + "cell_type": "code", + "source": [ + "pip install surprise" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "B0W95u65oa7n", + "outputId": "ceb57084-f677-4d93-e92d-6a2b83b408f5" + }, + "execution_count": 10, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Requirement already satisfied: surprise in /usr/local/lib/python3.11/dist-packages (0.1)\n", + "Requirement already satisfied: scikit-surprise in /usr/local/lib/python3.11/dist-packages (from surprise) (1.1.4)\n", + "Requirement already satisfied: joblib>=1.2.0 in /usr/local/lib/python3.11/dist-packages (from scikit-surprise->surprise) (1.5.1)\n", + "Requirement already satisfied: numpy>=1.19.5 in /usr/local/lib/python3.11/dist-packages (from scikit-surprise->surprise) (1.26.4)\n", + "Requirement already satisfied: scipy>=1.6.0 in /usr/local/lib/python3.11/dist-packages (from scikit-surprise->surprise) (1.15.3)\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "!pip install numpy==1.26.4 --upgrade --force-reinstall" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 342 + }, + "id": "ImxcJ99JpwIH", + "outputId": "8dc8a939-7eff-440e-bb63-19cd9ca96d6b" + }, + "execution_count": 11, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Collecting numpy==1.26.4\n", + " Using cached numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (61 kB)\n", + "Using cached numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (18.3 MB)\n", + "Installing collected packages: numpy\n", + " Attempting uninstall: numpy\n", + " Found existing installation: numpy 1.26.4\n", + " Uninstalling numpy-1.26.4:\n", + " Successfully uninstalled numpy-1.26.4\n", + "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n", + "thinc 8.3.6 requires numpy<3.0.0,>=2.0.0, but you have numpy 1.26.4 which is incompatible.\u001b[0m\u001b[31m\n", + "\u001b[0mSuccessfully installed numpy-1.26.4\n" + ] + }, + { + "output_type": "display_data", + "data": { + "application/vnd.colab-display-data+json": { + "pip_warning": { + "packages": [ + "numpy" + ] + }, + "id": "25adbf824dea4049a03a20118a1b87ca" + } + }, + "metadata": {} + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "## Surprise를 이용한 추천 시스템 구축" + ], + "metadata": { + "id": "d1s-eh0Zot1G" + } + }, + { + "cell_type": "code", + "source": [ + "# 관련 모듈 임포트\n", + "from surprise import SVD\n", + "from surprise import Dataset\n", + "from surprise import accuracy\n", + "from surprise.model_selection import train_test_split\n" + ], + "metadata": { + "id": "irY3RVE1oqXs" + }, + "execution_count": 1, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "- 사용 데이터셋 - movieLens사이트 제공 과거 버전의 데이터셋\n", + "- 로우 레벨의 칼럼 데이터를 칼럼 레벨의 데이터로 자체 변경하므로 원본인 로우 레벨의 사용자-아이템 평점 데이터를 데이터셋으로 적용해야함." + ], + "metadata": { + "id": "G38RWz5OpCmA" + } + }, + { + "cell_type": "code", + "source": [ + "data = Dataset.load_builtin('ml-100k')\n", + "# 수행 시마다 동일하게 데이터를 분할하기 위해 random_state 값 부여\n", + "trainset, testset = train_test_split(data, test_size=.25, random_state=0)" + ], + "metadata": { + "id": "r3gP_xc2pH8N" + }, + "execution_count": 2, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "algo = SVD(random_state=0)\n", + "algo.fit(trainset)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "wnUjTZVepQSA", + "outputId": "5f87f644-84f3-4e3c-b22e-9bbbffb13510" + }, + "execution_count": 3, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": {}, + "execution_count": 3 + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "- SVD객체.test() 반환 결과 : 입력인자 데이터셋 크기와 같은 파이썬 리스트\n", + "- Prediction 객체 > surprise 패키지 제공 데이터 타입\n", + " * uid(user id), iid(movie/item id),r_ui(실제평점) 기반해 예측한 예측평점(est)를 튜플형태로 가짐\n", + " * .details - 추천예측 안되는 경우 로그용 데이터 남김\n", + " * was_impossible = True : 예측값 생성할 수 없는 데이터" + ], + "metadata": { + "id": "c-diTwLlrCgm" + } + }, + { + "cell_type": "code", + "source": [ + "predictions = algo.test( testset )\n", + "print('prediction type :',type(predictions), ' size:',len(predictions))\n", + "print('prediction 결과의 최초 5개 추출')\n", + "predictions[:5]" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "k0u5jOScp4U7", + "outputId": "1fe9193b-46d6-414a-d6df-c67b2855eb8c" + }, + "execution_count": 4, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "prediction type : size: 25000\n", + "prediction 결과의 최초 5개 추출\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "[Prediction(uid='120', iid='282', r_ui=4.0, est=3.5114147666251547, details={'was_impossible': False}),\n", + " Prediction(uid='882', iid='291', r_ui=4.0, est=3.573872419581491, details={'was_impossible': False}),\n", + " Prediction(uid='535', iid='507', r_ui=5.0, est=4.033583485472447, details={'was_impossible': False}),\n", + " Prediction(uid='697', iid='244', r_ui=5.0, est=3.8463639495936905, details={'was_impossible': False}),\n", + " Prediction(uid='751', iid='385', r_ui=4.0, est=3.1807542478219157, details={'was_impossible': False})]" + ] + }, + "metadata": {}, + "execution_count": 4 + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "**속성 추출**\n", + "- predict객체.uid\n", + "- predict객체.iid\n", + "- predict객체.est" + ], + "metadata": { + "id": "CkKoZG97rsRM" + } + }, + { + "cell_type": "code", + "source": [ + "[ (pred.uid, pred.iid, pred.est) for pred in predictions[:3] ]" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "PZhTyldwp4TJ", + "outputId": "9e96ce59-1a21-4e70-a194-e2817abb08e9" + }, + "execution_count": 5, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "[('120', '282', 3.5114147666251547),\n", + " ('882', '291', 3.573872419581491),\n", + " ('535', '507', 4.033583485472447)]" + ] + }, + "metadata": {}, + "execution_count": 5 + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "- test가 아닌 predict()로 추천 예측 가능\n", + "- predict() : 개별 사용자의 아이템에 대한 추천 평점 예측\n", + "- input(문자열) : uid, iid (r_ui는 선택)" + ], + "metadata": { + "id": "2Paot01Ur-E0" + } + }, + { + "cell_type": "code", + "source": [ + "# 사용자 아이디, 아이템 아이디는 문자열로 입력해야 함.\n", + "uid = str(196)\n", + "iid = str(302)\n", + "pred = algo.predict(uid, iid)\n", + "print(pred)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "c1yUx49-p4Q3", + "outputId": "60cc292a-7dd3-4038-fcc2-05b209436fb8" + }, + "execution_count": 6, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "user: 196 item: 302 r_ui = None est = 4.49 {'was_impossible': False}\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "- accuracy 모듈은 RMSE, MSE 등으로 추천 시스템의 성능 평가 정보 제공" + ], + "metadata": { + "id": "bMJf4ynDsNz0" + } + }, + { + "cell_type": "code", + "source": [ + "accuracy.rmse(predictions)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "N_KAKnyFqBtO", + "outputId": "e4cce74c-ee66-4d45-a72b-a3971b168fd0" + }, + "execution_count": 7, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "RMSE: 0.9467\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "0.9466860806937948" + ] + }, + "metadata": {}, + "execution_count": 7 + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "## Surprise 주요 모듈 소개\n", + "### Dataset\n", + "1. Dataset.load_builtin\n", + " * 무비렌즈 아카이브 FTP 서버에서 무비렌즈 데이터 내려받음\n", + " * name = ml-100k(default)/ml-1M 내려받을 수 있음\n", + "2. Dataset.load_from_file\n", + " * OS 파일에서 데이터를 로딩할 때 사용\n", + " * 콤마/탭으로 구분된 포맷의 OS파일에서 데이터 로딩\n", + " * parameter - OS 파일명 , Reader로 파일의 포맷 지정\n", + "3. Dataset.load_from_df\n", + " * 판다스의 DataFrame에서 데이터 로딩\n", + " * DataFrame은 반드시 3개의 칼럼인 사용자 아이디, 아이템 아이디, 평점 순으로 칼럼 순서가 정해져있어야 함.\n", + " * parameter - DataFrame 객체, Rader로 파일의 포맷 지정\n", + "⚠️Surprise > OS파일 로딩 시 주의 점 : 칼럼명 헤더 문자열 있으면 안됨" + ], + "metadata": { + "id": "MswknCDzqErK" + } + }, + { + "cell_type": "code", + "source": [ + "import pandas as pd\n", + "\n", + "ratings = pd.read_csv('/content/drive/MyDrive/Sample_data/ml-latest-small/ml_latest_small_ratings.csv')\n", + "# ratings_noh.csv 파일로 unload 시 index 와 header를 모두 제거한 새로운 파일 생성.\n", + "ratings.to_csv('/content/drive/MyDrive/Sample_data/ml-latest-small/ml_latest_small_ratings_noh.csv', index=False, header=False)" + ], + "metadata": { + "id": "hSkGT9aguF1A" + }, + "execution_count": 8, + "outputs": [] + }, + { + "cell_type": "markdown", + "source": [ + "- Reader 클래스 : ratings_noh.csv 파일 파싱 포맷 정의/파싱 정보 알려줌\n", + " * 생성자에 각 필드의 칼럼명,구분자, 최소~최대 평점 입력 ➡ 객체 생성\n", + " * line_format : 칼럼 user item rating timestamp 명시, 문자열을 공백으로 구분\n", + " * sep : 구분자 명시\n", + " * rating_scale : 평점 단위 = 0.5, 최대 평점 = 5\n", + "- Dataset.load_from_file() : Reader 객체 참조 데이터 파일 파싱하며 로딩" + ], + "metadata": { + "id": "dOZs7eISunzI" + } + }, + { + "cell_type": "code", + "source": [ + "from surprise import Reader\n", + "\n", + "reader = Reader(line_format='user item rating timestamp', sep=',', rating_scale=(0.5, 5))\n", + "data=Dataset.load_from_file('/content/drive/MyDrive/Sample_data/ml-latest-small/ml_latest_small_ratings_noh.csv',reader=reader)" + ], + "metadata": { + "id": "AX-S37kGqEdt" + }, + "execution_count": 14, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "trainset, testset = train_test_split(data, test_size=.25, random_state=0)\n", + "\n", + "# 수행시마다 동일한 결과 도출을 위해 random_state 설정\n", + "algo = SVD(n_factors=50, random_state=0)\n", + "\n", + "# 학습/예측/평가\n", + "algo.fit(trainset)\n", + "predictions = algo.test( testset )\n", + "accuracy.rmse(predictions)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "fg4XHsZEqEb1", + "outputId": "91db56b5-acdf-470d-8463-c0be930dbb2b" + }, + "execution_count": 10, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "RMSE: 0.8708\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "0.8708344753692029" + ] + }, + "metadata": {}, + "execution_count": 10 + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "### 판다스 DataFrame에서 Surprise 데이터 세트로 로딩\n", + "- Dataset.load_from_df()" + ], + "metadata": { + "id": "a5WWH7FjvtSe" + } + }, + { + "cell_type": "code", + "source": [ + "import pandas as pd\n", + "from surprise import Reader, Dataset\n", + "\n", + "ratings = pd.read_csv('/content/drive/MyDrive/Sample_data/ml-latest-small/ml_latest_small_ratings.csv')\n", + "reader = Reader(rating_scale=(0.5, 5.0))\n", + "\n" + ], + "metadata": { + "id": "NsEiNUiLqEZ9" + }, + "execution_count": 11, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "ratings" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 424 + }, + "id": "G5ZzvLSqxNL3", + "outputId": "ac46a58a-2895-4909-e1e3-1710f0be4c79" + }, + "execution_count": 17, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + " 1 1.1 4.0 964982703\n", + "0 1 3 4.0 964981247\n", + "1 1 6 4.0 964982224\n", + "2 1 47 5.0 964983815\n", + "3 1 50 5.0 964982931\n", + "4 1 70 3.0 964982400\n", + "... ... ... ... ...\n", + "100830 610 166534 4.0 1493848402\n", + "100831 610 168248 5.0 1493850091\n", + "100832 610 168250 5.0 1494273047\n", + "100833 610 168252 5.0 1493846352\n", + "100834 610 170875 3.0 1493846415\n", + "\n", + "[100835 rows x 4 columns]" + ], + "text/html": [ + "\n", + "
\n", + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
11.14.0964982703
0134.0964981247
1164.0964982224
21475.0964983815
31505.0964982931
41703.0964982400
...............
1008306101665344.01493848402
1008316101682485.01493850091
1008326101682505.01494273047
1008336101682525.01493846352
1008346101708753.01493846415
\n", + "

100835 rows × 4 columns

\n", + "
\n", + "
\n", + "\n", + "
\n", + " \n", + "\n", + " \n", + "\n", + " \n", + "
\n", + "\n", + "\n", + "
\n", + " \n", + "\n", + "\n", + "\n", + " \n", + "
\n", + "\n", + "
\n", + " \n", + " \n", + " \n", + "
\n", + "\n", + "
\n", + "
\n" + ], + "application/vnd.google.colaboratory.intrinsic+json": { + "type": "dataframe", + "variable_name": "ratings" + } + }, + "metadata": {}, + "execution_count": 17 + } + ] + }, + { + "cell_type": "code", + "source": [ + "# ratings DataFrame 에서 컬럼은 사용자 아이디, 아이템 아이디, 평점 순서를 지켜야 합니다.\n", + "ratings.columns = ['userId', 'movieId', 'rating','timestamp']\n", + "\n", + "data = Dataset.load_from_df(ratings[['userId', 'movieId', 'rating']], reader)\n", + "trainset, testset = train_test_split(data, test_size=.25, random_state=0)\n", + "\n", + "algo = SVD(n_factors=50, random_state=0)\n", + "algo.fit(trainset)\n", + "predictions = algo.test( testset )\n", + "accuracy.rmse(predictions)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "OGaKD9WTwKRk", + "outputId": "f4003784-c049-44d8-f4e9-c76f32e79de1" + }, + "execution_count": 19, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "RMSE: 0.8708\n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "0.8708344753692029" + ] + }, + "metadata": {}, + "execution_count": 19 + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "## Surprise 추천 알고리즘 클래스\n", + "\n", + "1. SVD - 행렬 분해, 잠재 요인 협업 필터링\n", + " * 사용자 baseline 편향성 감안한 쳥점 예측에 Regularization 적용\n", + " * 사용자 예측 rating $\\hat{r}_{ui}=\\mu+bu+bi+aTipu$\n", + " * Regularization 적용 비용함수 : $\\sum{({r}_{ui}-\\hat{r}_{ui})2}+\\lambda(b2i+b2u+||qi||2+||pu||2)$\n", + " * **parameters**\n", + " 1. n_factors : 잠재요인 K 수 (default-100), 클수록 과적합 위험 ⬆, 정확도 ⬆\n", + " 2. n_epochs : SGD 수행시 반복 횟수(default - 20)\n", + " 3. biaseed - 베이스라인 사용자 편향 적용 여부 (default-True)\n", + "2. KNNBasic - 최근접 이웃 협업 필터링 KNN알고리즘\n", + "3. BaselineOnly - 사용자/아이템 Bias를 감안한 SGD베이스라인 알고리즘\n", + "4. 이외 알고리즘 - SVD++, NMF, Slpe One, Co-Clustering/ Baseline - 각 개인이 평점을 부여하는 성향 반영해 평점 계산\n", + "\n", + "## 베이스라인 평점\n", + "- 개인의 성향을 반영해 ㅐ아이템 평가에 편향성 요소를 반영해 평점 부과\n", + "- 전체 평균 평점 + 사용자 편향 점수 + 아이템 편향 점수\n", + "- 전체 평균 평점 = 모든 사용자 아이템 평점 평균값\n", + "- 사용자 편향 점수 = 사용자별 아이템 평균값 - 전체 평균 평점\n", + "- 아이템 편향 점수 = 아이템별 평점 평균 값 - 전체 평균 평점" + ], + "metadata": { + "id": "_ccuitb8yL9z" + } + }, + { + "cell_type": "markdown", + "source": [ + "## 교차검증과 하이퍼 파라미터 튜닝\n", + "- suprise.modelsection - cross_validate()와 GridSearchCV클래스 제공" + ], + "metadata": { + "id": "FEJlqTz5qN2p" + } + }, + { + "cell_type": "code", + "source": [ + "from surprise.model_selection import cross_validate\n", + "\n", + "ratings = pd.read_csv('/content/drive/MyDrive/Sample_data/ml-latest-small/ml_latest_small_ratings.csv') # reading data in pandas df\n", + "ratings.columns = ['userId', 'movieId', 'rating','timestamp']\n", + "reader = Reader(rating_scale=(0.5, 5.0))\n", + "data = Dataset.load_from_df(ratings[['userId', 'movieId', 'rating']], reader)\n", + "\n", + "algo = SVD(random_state=0)\n", + "cross_validate(algo, data, measures=['RMSE', 'MAE'], cv=5, verbose=True)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "nUFf4qNEqEXR", + "outputId": "49a01f51-ff8a-4665-e736-8bac019ff0de" + }, + "execution_count": 22, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Evaluating RMSE, MAE of algorithm SVD on 5 split(s).\n", + "\n", + " Fold 1 Fold 2 Fold 3 Fold 4 Fold 5 Mean Std \n", + "RMSE (testset) 0.8717 0.8712 0.8723 0.8724 0.8815 0.8738 0.0039 \n", + "MAE (testset) 0.6696 0.6694 0.6696 0.6680 0.6798 0.6713 0.0043 \n", + "Fit time 2.28 1.39 1.45 1.39 1.51 1.61 0.34 \n", + "Test time 0.18 0.25 0.11 0.27 0.10 0.18 0.07 \n" + ] + }, + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "{'test_rmse': array([0.87171692, 0.87117491, 0.87225411, 0.87238817, 0.88150963]),\n", + " 'test_mae': array([0.66958695, 0.66941221, 0.66962236, 0.66796536, 0.67984648]),\n", + " 'fit_time': (2.2812085151672363,\n", + " 1.3924455642700195,\n", + " 1.449477195739746,\n", + " 1.390857458114624,\n", + " 1.5147252082824707),\n", + " 'test_time': (0.1770644187927246,\n", + " 0.2455298900604248,\n", + " 0.1055753231048584,\n", + " 0.2739133834838867,\n", + " 0.10271763801574707)}" + ] + }, + "metadata": {}, + "execution_count": 22 + } + ] + }, + { + "cell_type": "code", + "source": [ + "from surprise.model_selection import GridSearchCV\n", + "\n", + "param_grid = {'n_epochs': [20, 40, 60], 'n_factors': [50, 100, 200] }\n", + "\n", + "# CV = 3, 성능 평가 = rmse, mse\n", + "gs = GridSearchCV(SVD, param_grid, measures=['rmse', 'mae'], cv=3)\n", + "gs.fit(data)\n", + "\n", + "# 최고 RMSE Evaluation 점수와 그때의 하이퍼 파라미터\n", + "print(gs.best_score['rmse'])\n", + "print(gs.best_params['rmse'])" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "XwFrrgsBqESe", + "outputId": "431db2c5-c05f-4431-ea7c-e015446dd54a" + }, + "execution_count": 23, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "0.8766244571341707\n", + "{'n_epochs': 20, 'n_factors': 50}\n" + ] + } + ] + }, + { + "cell_type": "markdown", + "source": [ + "## Surprise를 이용한 개인화 영화 추천 시스템 구축" + ], + "metadata": { + "id": "1H7FnNLwqVmF" + } + }, + { + "cell_type": "code", + "source": [ + "# 다음 코드는 train_test_split( )으로 분리되지 않는 데이터 세트에 fit( )을 호출해 오류가 발생합니다.\n", + "data = Dataset.load_from_df(ratings[['userId', 'movieId', 'rating']], reader)\n", + "algo = SVD(n_factors=50, random_state=0)\n", + "algo.fit(data)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 245 + }, + "id": "VAlPqA6qqEQI", + "outputId": "7b66f3d7-6a5b-46f0-8916-bd3efd4885f2" + }, + "execution_count": 24, + "outputs": [ + { + "output_type": "error", + "ename": "AttributeError", + "evalue": "'DatasetAutoFolds' object has no attribute 'n_users'", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m/tmp/ipython-input-24-4120753849.py\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 2\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mDataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload_from_df\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mratings\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'userId'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'movieId'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'rating'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0malgo\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSVD\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn_factors\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m50\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrandom_state\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0malgo\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", + "\u001b[0;32m/usr/local/lib/python3.11/dist-packages/surprise/prediction_algorithms/matrix_factorization.pyx\u001b[0m in \u001b[0;36msurprise.prediction_algorithms.matrix_factorization.SVD.fit\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32m/usr/local/lib/python3.11/dist-packages/surprise/prediction_algorithms/matrix_factorization.pyx\u001b[0m in \u001b[0;36msurprise.prediction_algorithms.matrix_factorization.SVD.sgd\u001b[0;34m()\u001b[0m\n", + "\u001b[0;31mAttributeError\u001b[0m: 'DatasetAutoFolds' object has no attribute 'n_users'" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "from surprise.dataset import DatasetAutoFolds\n", + "\n", + "reader = Reader(line_format='user item rating timestamp', sep=',', rating_scale=(0.5, 5))\n", + "# DatasetAutoFolds 클래스를 ratings_noh.csv 파일 기반으로 생성.\n", + "data_folds = DatasetAutoFolds(ratings_file='/content/drive/MyDrive/Sample_data/ml-latest-small/ml_latest_small_ratings_noh.csv', reader=reader)\n", + "\n", + "#전체 데이터를 학습데이터로 생성함.\n", + "trainset = data_folds.build_full_trainset()" + ], + "metadata": { + "id": "Ny-sn_ezqa98" + }, + "execution_count": 26, + "outputs": [] + }, + { + "cell_type": "code", + "source": [ + "algo = SVD(n_epochs=20, n_factors=50, random_state=0)\n", + "algo.fit(trainset)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "0LT3VBa1qdCm", + "outputId": "21c62d22-c964-437d-8178-f9299756c2fd" + }, + "execution_count": 27, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "" + ] + }, + "metadata": {}, + "execution_count": 27 + } + ] + }, + { + "cell_type": "code", + "source": [ + "# 영화에 대한 상세 속성 정보 DataFrame로딩\n", + "movies = pd.read_csv('/content/drive/MyDrive/Sample_data/ml-latest-small/ml_latest_small_movies.csv')\n", + "\n", + "# userId=9 의 movieId 데이터 추출하여 movieId=42 데이터가 있는지 확인.\n", + "movieIds = ratings[ratings['userId']==9]['movieId']\n", + "if movieIds[movieIds==42].count() == 0:\n", + " print('사용자 아이디 9는 영화 아이디 42의 평점 없음')\n", + "\n", + "print(movies[movies['movieId']==42])" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "f2PTEinQqc9t", + "outputId": "af4b8cab-df50-49a2-bc52-628931c23687" + }, + "execution_count": 30, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "사용자 아이디 9는 영화 아이디 42의 평점 없음\n", + " movieId title genres\n", + "38 42 Dead Presidents (1995) Action|Crime|Drama\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "uid = str(9)\n", + "iid = str(42)\n", + "\n", + "pred = algo.predict(uid, iid, verbose=True)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "PV4leey-qc7-", + "outputId": "f11300c1-b108-4f3c-f860-205b66c42991" + }, + "execution_count": 31, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "user: 9 item: 42 r_ui = None est = 3.10 {'was_impossible': False}\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "def get_unseen_surprise(ratings, movies, userId):\n", + " #입력값으로 들어온 userId에 해당하는 사용자가 평점을 매긴 모든 영화를 리스트로 생성\n", + " seen_movies = ratings[ratings['userId']== userId]['movieId'].tolist()\n", + "\n", + " # 모든 영화들의 movieId를 리스트로 생성.\n", + " total_movies = movies['movieId'].tolist()\n", + "\n", + " # 모든 영화들의 movieId중 이미 평점을 매긴 영화의 movieId를 제외하여 리스트로 생성\n", + " unseen_movies= [movie for movie in total_movies if movie not in seen_movies]\n", + " print('평점 매긴 영화수:',len(seen_movies), '추천대상 영화수:',len(unseen_movies), \\\n", + " '전체 영화수:',len(total_movies))\n", + "\n", + " return unseen_movies\n", + "\n", + "unseen_movies = get_unseen_surprise(ratings, movies, 9)" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "ajOImP58qc5a", + "outputId": "1c803b00-1e19-4463-9da8-cda693a58c7d" + }, + "execution_count": 32, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "평점 매긴 영화수: 46 추천대상 영화수: 9696 전체 영화수: 9742\n" + ] + } + ] + }, + { + "cell_type": "code", + "source": [ + "def recomm_movie_by_surprise(algo, userId, unseen_movies, top_n=10):\n", + " # 알고리즘 객체의 predict() 메서드를 평점이 없는 영화에 반복 수행한 후 결과를 list 객체로 저장\n", + " predictions = [algo.predict(str(userId), str(movieId)) for movieId in unseen_movies]\n", + "\n", + " # predictions list 객체는 surprise의 Predictions 객체를 원소로 가지고 있음.\n", + " # [Prediction(uid='9', iid='1', est=3.69), Prediction(uid='9', iid='2', est=2.98),,,,]\n", + " # 이를 est 값으로 정렬하기 위해서 아래의 sortkey_est 함수를 정의함.\n", + "\n", + " # sortkey_est 함수 : list 객체의 sort() 함수의 키 값으로 사용되됨. 정렬 수행.\n", + " def sortkey_est(pred):\n", + " return pred.est\n", + "\n", + " # sortkey_est( ) 반환값의 내림 차순으로 정렬 수행 > top_n 최상위 값 추출.\n", + " predictions.sort(key=sortkey_est, reverse=True)\n", + " top_predictions= predictions[:top_n]\n", + "\n", + " # top_n으로 추출된 영화의 정보 추 >. 영화 아이디, 추천 예상 평점, 제목\n", + " top_movie_ids = [ int(pred.iid) for pred in top_predictions]\n", + " top_movie_rating = [ pred.est for pred in top_predictions]\n", + " top_movie_titles = movies[movies.movieId.isin(top_movie_ids)]['title']\n", + " top_movie_preds = [ (id, title, rating) for id, title, rating in zip(top_movie_ids, top_movie_titles, top_movie_rating)]\n", + "\n", + " return top_movie_preds\n", + "\n", + "unseen_movies = get_unseen_surprise(ratings, movies, 9)\n", + "top_movie_preds = recomm_movie_by_surprise(algo, 9, unseen_movies, top_n=10)\n", + "print('##### Top-10 추천 영화 리스트 #####')\n", + "\n", + "for top_movie in top_movie_preds:\n", + " print(top_movie[1], \":\", top_movie[2])" + ], + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "Oa3aNMF1qc1k", + "outputId": "8120d627-f95c-40e3-8501-01baf03cd7da" + }, + "execution_count": 33, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "평점 매긴 영화수: 46 추천대상 영화수: 9696 전체 영화수: 9742\n", + "##### Top-10 추천 영화 리스트 #####\n", + "Usual Suspects, The (1995) : 4.2267942523743605\n", + "Star Wars: Episode IV - A New Hope (1977) : 4.211016328181448\n", + "Shawshank Redemption, The (1994) : 4.197037380337041\n", + "Dr. Strangelove or: How I Learned to Stop Worrying and Love the Bomb (1964) : 4.142771982890766\n", + "Godfather, The (1972) : 4.136604760890598\n", + "Reservoir Dogs (1992) : 4.124673084995863\n", + "Streetcar Named Desire, A (1951) : 4.120523998974084\n", + "Goodfellas (1990) : 4.069435046627849\n", + "Glory (1989) : 4.067497992056492\n", + "All the President's Men (1976) : 4.062778702133245\n" + ] + } + ] + } + ] +} \ No newline at end of file diff --git "a/Week16_\354\230\210\354\212\265\352\263\274\354\240\234_\354\236\245\354\204\234\354\227\260.pdf" "b/Week16_\354\230\210\354\212\265\352\263\274\354\240\234_\354\236\245\354\204\234\354\227\260.pdf" new file mode 100644 index 0000000..39745dc Binary files /dev/null and "b/Week16_\354\230\210\354\212\265\352\263\274\354\240\234_\354\236\245\354\204\234\354\227\260.pdf" differ