sentiment analysis (1).ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Sentiment Analysis of a book on sale at Amazon.com"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## About the Dataset\n",
    "\n",
    "This dataset contains a set of 10,000 reviews on a particular book which is scrapped from the comment section of Amazon's online website.\n",
    "We aim to find out the type of sentiment a particular reader has towards this book by making use of his/her comments."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Importing the Json file and building the framework of our class which predicts the sentiment."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'POSITIVE'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import json\n",
    "class Sentiment:\n",
    "    NEGATIVE=\"NEGATIVE\"\n",
    "    NEUTRAL=\"NEUTRAL\"\n",
    "    POSITIVE=\"POSITIVE\"\n",
    "    \n",
    "class Review:\n",
    "    def __init__(self,text,score):  \n",
    "        self.text =text\n",
    "        self.score=score\n",
    "        self.sentiment=self.get_sentiment()\n",
    "        \n",
    "    def get_sentiment(self):\n",
    "        if self.score<=2:\n",
    "            return Sentiment.NEGATIVE\n",
    "        elif self.score==3:\n",
    "            return Sentiment.NEUTRAL\n",
    "        else:\n",
    "            return Sentiment.POSITIVE\n",
    "        \n",
    "file_name='C://Users//Nishil07//Documents//Books_small_10000.json'\n",
    "reviews=[]\n",
    "with open(file_name) as f:\n",
    "    for line in f:\n",
    "        review=json.loads(line)\n",
    "        reviews.append(Review(review['reviewText'],review['overall']))\n",
    "reviews[5].sentiment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Splitting the train and test data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "training,test=train_test_split(reviews,test_size=0.33,random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "POSITIVE\n"
     ]
    }
   ],
   "source": [
    "print(training[0].sentiment)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'POSITIVE'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_x=[x.text for x in training]\n",
    "train_y=[x.sentiment for x in training]\n",
    "test_x=[x.text for x in test]\n",
    "test_y=[x.sentiment for x in test]\n",
    "test_y[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Converting the words to vectors and making use of the 'bag of words' concept."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  (0, 932)\t1\n",
      "  (0, 11573)\t1\n",
      "  (0, 16680)\t2\n",
      "  (0, 2155)\t2\n",
      "  (0, 6681)\t4\n",
      "  (0, 951)\t2\n",
      "  (0, 8908)\t5\n",
      "  (0, 8874)\t6\n",
      "  (0, 16593)\t8\n",
      "  (0, 16586)\t4\n",
      "  (0, 18578)\t2\n",
      "  (0, 16824)\t9\n",
      "  (0, 8374)\t1\n",
      "  (0, 7710)\t1\n",
      "  (0, 11267)\t1\n",
      "  (0, 6505)\t1\n",
      "  (0, 7844)\t1\n",
      "  (0, 11409)\t2\n",
      "  (0, 15845)\t2\n",
      "  (0, 7681)\t1\n",
      "  (0, 3393)\t1\n",
      "  (0, 9991)\t3\n",
      "  (0, 15306)\t3\n",
      "  (0, 2541)\t2\n",
      "  (0, 6868)\t4\n",
      "  :\t:\n",
      "  (0, 5312)\t1\n",
      "  (0, 6715)\t1\n",
      "  (0, 12077)\t1\n",
      "  (0, 7629)\t1\n",
      "  (0, 666)\t1\n",
      "  (0, 7768)\t2\n",
      "  (0, 1785)\t1\n",
      "  (0, 372)\t1\n",
      "  (0, 18468)\t1\n",
      "  (0, 5577)\t1\n",
      "  (0, 13335)\t1\n",
      "  (0, 16555)\t1\n",
      "  (0, 877)\t1\n",
      "  (0, 11306)\t1\n",
      "  (0, 8985)\t1\n",
      "  (0, 1443)\t1\n",
      "  (0, 9696)\t1\n",
      "  (0, 9628)\t1\n",
      "  (0, 10389)\t1\n",
      "  (0, 9754)\t1\n",
      "  (0, 16737)\t1\n",
      "  (0, 11834)\t1\n",
      "  (0, 18401)\t1\n",
      "  (0, 10269)\t1\n",
      "  (0, 15765)\t1\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "vectorizer=CountVectorizer()\n",
    "train_x_vectors=vectorizer.fit_transform(train_x)\n",
    "test_x_vectors=vectorizer.fit_transform(test_x)\n",
    "#test_x_vectors=test_x_vectors.toarray()\n",
    "#test_x_vectors=np.reshape(test_x_vectors,(-1,1))\n",
    "print(test_x_vectors[10])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Building the prediction models and fitting the train data into them"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import svm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['POSITIVE'], dtype='<U8')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf_svm=svm.SVC(kernel='linear')\n",
    "#clf_svm.fit(train_x_vectors,train_y)\n",
    "#print(test_x[0])\n",
    "clf_svm.fit(test_x_vectors,test_y)\n",
    "clf_svm.predict(test_x_vectors[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['POSITIVE'], dtype='<U8')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "clf_dec=DecisionTreeClassifier()\n",
    "clf_dec.fit(test_x_vectors,test_y)\n",
    "clf_dec.predict(test_x_vectors[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Nishil07\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array(['POSITIVE'], dtype='<U8')"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "clf_lr=LogisticRegression()\n",
    "clf_lr.fit(test_x_vectors,test_y)\n",
    "clf_lr.predict(test_x_vectors[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Applying various testing metrics to check how accurate our model is......"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0\n",
      "1.0\n",
      "0.9972727272727273\n"
     ]
    }
   ],
   "source": [
    "#mean accuracy\n",
    "print(clf_svm.score(test_x_vectors,test_y))\n",
    "print(clf_dec.score(test_x_vectors,test_y))\n",
    "print(clf_lr.score(test_x_vectors,test_y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.99837633, 0.98911353, 0.99516908])"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#F1-scores\n",
    "from sklearn.metrics import f1_score\n",
    "f1_score(test_y,clf_lr.predict(test_x_vectors),average=None,labels=[Sentiment.POSITIVE,Sentiment.NEUTRAL,Sentiment.NEGATIVE])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}