ML_Classifier_Malayalam_code_mixed.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"ML_Classifier_Malayalam_code_mixed.ipynb","provenance":[],"collapsed_sections":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"accelerator":"GPU"},"cells":[{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"O1S8SYrnMTNI","executionInfo":{"status":"ok","timestamp":1624682778862,"user_tz":-330,"elapsed":21007,"user":{"displayName":"Abhinav Kumar","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhN6WFfi4jSsnU9CzZnfDwqVV_KtgtD8yEnF1qt=s64","userId":"09645300777368779770"}},"outputId":"0d117409-9793-484e-8b65-1c837581ebce"},"source":["from google.colab import drive\n","drive.mount('/content/drive')"],"execution_count":1,"outputs":[{"output_type":"stream","text":["Mounted at /content/drive\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"57R5J3_1OfiA","executionInfo":{"status":"ok","timestamp":1624682781513,"user_tz":-330,"elapsed":1259,"user":{"displayName":"Abhinav Kumar","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhN6WFfi4jSsnU9CzZnfDwqVV_KtgtD8yEnF1qt=s64","userId":"09645300777368779770"}}},"source":["import pandas as pd \n","data_train = pd.read_csv('/content/drive/MyDrive/HASOC_2021/Sentiment_Dravidian/Malyalam/Mal_sentiment_full_train.tsv', sep = '\\t')\n","data_val = pd.read_csv('/content/drive/MyDrive/HASOC_2021/Sentiment_Dravidian/Malyalam/Mal_sentiment_full_dev.tsv', sep = '\\t')\n","data_test = pd.read_csv('/content/drive/MyDrive/HASOC_2021/Sentiment_Dravidian/Malyalam/Mal_sentiment_full_test_withoutlabels.tsv', sep = '\\t')"],"execution_count":2,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"id":"aIWUjapcPoVJ","executionInfo":{"status":"ok","timestamp":1624682781514,"user_tz":-330,"elapsed":21,"user":{"displayName":"Abhinav Kumar","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhN6WFfi4jSsnU9CzZnfDwqVV_KtgtD8yEnF1qt=s64","userId":"09645300777368779770"}},"outputId":"e8baeab0-bdab-4b39-aaff-ad3df348872a"},"source":["data_train.head()"],"execution_count":3,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>text</th>\n","      <th>category</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Sunny chechi fans evide like adichu power kani...</td>\n","      <td>unknown_state</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Angane july month ile ende aadyathe leave njan...</td>\n","      <td>Positive</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>ഏട്ടന്റെ പുതിയ പടത്തിനു വേണ്ടി കാത്തിരിക്കുന്ന...</td>\n","      <td>unknown_state</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>ഇനി ലാലേട്ടന്റെ വേട്ട തുടങ്ങാൻ പോകുന്നു..........</td>\n","      <td>Positive</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Trailer powli oru raksha illa . Pakshea padam ...</td>\n","      <td>Positive</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["                                                text       category\n","0  Sunny chechi fans evide like adichu power kani...  unknown_state\n","1  Angane july month ile ende aadyathe leave njan...       Positive\n","2  ഏട്ടന്റെ പുതിയ പടത്തിനു വേണ്ടി കാത്തിരിക്കുന്ന...  unknown_state\n","3  ഇനി ലാലേട്ടന്റെ വേട്ട തുടങ്ങാൻ പോകുന്നു..........       Positive\n","4  Trailer powli oru raksha illa . Pakshea padam ...       Positive"]},"metadata":{"tags":[]},"execution_count":3}]},{"cell_type":"code","metadata":{"id":"0vk8YlBibEYt","colab":{"base_uri":"https://localhost:8080/","height":206},"executionInfo":{"status":"ok","timestamp":1624682781515,"user_tz":-330,"elapsed":20,"user":{"displayName":"Abhinav Kumar","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhN6WFfi4jSsnU9CzZnfDwqVV_KtgtD8yEnF1qt=s64","userId":"09645300777368779770"}},"outputId":"8c773c32-f24b-4d65-a201-9c9423c3c290"},"source":["data_val.head()"],"execution_count":4,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>text</th>\n","      <th>category</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Mammooka ninghal mass aa pwoli item</td>\n","      <td>Positive</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Waiting for Malayalam movie  For Tamil paiyan</td>\n","      <td>not-malayalam</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>ദളപതി ഫാൻസിന്റെ വക ഒരു ഒന്നൊന്നര വിജയാശംസകൾ...</td>\n","      <td>Positive</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>#Trailer pwolichuuuu ഓണത്തിന് വന്നങ്ങു തകർത്തേ...</td>\n","      <td>Positive</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Mammoookkaaaa polichadukkiii katta waiting nv 21</td>\n","      <td>Positive</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["                                                text       category\n","0                Mammooka ninghal mass aa pwoli item       Positive\n","1      Waiting for Malayalam movie  For Tamil paiyan  not-malayalam\n","2     ദളപതി ഫാൻസിന്റെ വക ഒരു ഒന്നൊന്നര വിജയാശംസകൾ...       Positive\n","3  #Trailer pwolichuuuu ഓണത്തിന് വന്നങ്ങു തകർത്തേ...       Positive\n","4   Mammoookkaaaa polichadukkiii katta waiting nv 21       Positive"]},"metadata":{"tags":[]},"execution_count":4}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"id":"7jeRfw0XPqe5","executionInfo":{"status":"ok","timestamp":1624682781515,"user_tz":-330,"elapsed":19,"user":{"displayName":"Abhinav Kumar","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhN6WFfi4jSsnU9CzZnfDwqVV_KtgtD8yEnF1qt=s64","userId":"09645300777368779770"}},"outputId":"dc3bf79d-effc-4438-e672-c15972f9707e"},"source":["data_test.head()"],"execution_count":5,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>id</th>\n","      <th>text</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>Mal_1</td>\n","      <td>Teaserinu kurach samayamkoodi mathram. Cant wa...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>Mal_2</td>\n","      <td>അപ്പോൾ കഥയുടെ  റൂട്ട് മാറിയല്ലോ...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>Mal_3</td>\n","      <td>മൂത്തോൻ ട്രൈലെർ Trending List വരാത്തത് എന്താ ?</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>Mal_4</td>\n","      <td>Nowadays 944k views is considered as 1M views ...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>Mal_5</td>\n","      <td>Maass.trailer ennu paranja ithaanU makkale.......</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["      id                                               text\n","0  Mal_1  Teaserinu kurach samayamkoodi mathram. Cant wa...\n","1  Mal_2                 അപ്പോൾ കഥയുടെ  റൂട്ട് മാറിയല്ലോ...\n","2  Mal_3     മൂത്തോൻ ട്രൈലെർ Trending List വരാത്തത് എന്താ ?\n","3  Mal_4  Nowadays 944k views is considered as 1M views ...\n","4  Mal_5  Maass.trailer ennu paranja ithaanU makkale......."]},"metadata":{"tags":[]},"execution_count":5}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"tMXviBypwXb0","executionInfo":{"status":"ok","timestamp":1624682781516,"user_tz":-330,"elapsed":19,"user":{"displayName":"Abhinav Kumar","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhN6WFfi4jSsnU9CzZnfDwqVV_KtgtD8yEnF1qt=s64","userId":"09645300777368779770"}},"outputId":"dd53e503-f71f-42b2-b88d-c0ef1c27cd21"},"source":["data_train.shape, data_val.shape, data_test.shape"],"execution_count":6,"outputs":[{"output_type":"execute_result","data":{"text/plain":["((15888, 2), (1766, 2), (1962, 2))"]},"metadata":{"tags":[]},"execution_count":6}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Cr674jk8wyck","executionInfo":{"status":"ok","timestamp":1624682781517,"user_tz":-330,"elapsed":19,"user":{"displayName":"Abhinav Kumar","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhN6WFfi4jSsnU9CzZnfDwqVV_KtgtD8yEnF1qt=s64","userId":"09645300777368779770"}},"outputId":"3c47d663-c6e2-4fb3-fa1c-76ecd3778927"},"source":["data_train['category'].value_counts()"],"execution_count":7,"outputs":[{"output_type":"execute_result","data":{"text/plain":["Positive          6421\n","unknown_state     5279\n","Negative          2105\n","not-malayalam     1157\n","Mixed_feelings     926\n","Name: category, dtype: int64"]},"metadata":{"tags":[]},"execution_count":7}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"tREo3vWbNLQA","executionInfo":{"status":"ok","timestamp":1624682781518,"user_tz":-330,"elapsed":17,"user":{"displayName":"Abhinav Kumar","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhN6WFfi4jSsnU9CzZnfDwqVV_KtgtD8yEnF1qt=s64","userId":"09645300777368779770"}},"outputId":"18e38bd8-6315-43e8-d10d-89ef0bc33926"},"source":["data_val['category'].value_counts()"],"execution_count":8,"outputs":[{"output_type":"execute_result","data":{"text/plain":["Positive          706\n","unknown_state     580\n","Negative          237\n","not-malayalam     141\n","Mixed_feelings    102\n","Name: category, dtype: int64"]},"metadata":{"tags":[]},"execution_count":8}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"zX-6Saf7QPh7","executionInfo":{"status":"ok","timestamp":1624682782802,"user_tz":-330,"elapsed":12,"user":{"displayName":"Abhinav Kumar","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhN6WFfi4jSsnU9CzZnfDwqVV_KtgtD8yEnF1qt=s64","userId":"09645300777368779770"}},"outputId":"dff6a3bc-45be-4822-fded-9267ffc6a2ca"},"source":["from sklearn import preprocessing\n","import numpy as np\n","\n","le = preprocessing.LabelEncoder()\n","\n","le.fit(data_train['category'])\n","print(le.classes_)\n","\n","y_train_index = le.transform(data_train['category'])\n","y_val_index = le.transform(data_val['category'])\n"],"execution_count":9,"outputs":[{"output_type":"stream","text":["['Mixed_feelings' 'Negative' 'Positive' 'not-malayalam' 'unknown_state']\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"mY8Wqps9WAMq"},"source":["For word = ngram_range (1,3)\n","For char = ngram_range (1,6)"]},{"cell_type":"code","metadata":{"id":"jqicHV-qRH4V","executionInfo":{"status":"ok","timestamp":1624682789930,"user_tz":-330,"elapsed":5758,"user":{"displayName":"Abhinav Kumar","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhN6WFfi4jSsnU9CzZnfDwqVV_KtgtD8yEnF1qt=s64","userId":"09645300777368779770"}}},"source":["from sklearn.feature_extraction.text import TfidfVectorizer\n","tfidfvec = TfidfVectorizer(ngram_range=(1,6), analyzer = 'char_wb', max_features = 30000)\n","#tfidfvec = TfidfVectorizer(ngram_range=(1,3), max_features = 30000)\n","tfidf_train = tfidfvec.fit_transform(data_train['text'])\n","tfidf_val = tfidfvec.transform(data_val['text'])"],"execution_count":10,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"iV6FzGg_0leL","executionInfo":{"status":"ok","timestamp":1624682789931,"user_tz":-330,"elapsed":24,"user":{"displayName":"Abhinav Kumar","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhN6WFfi4jSsnU9CzZnfDwqVV_KtgtD8yEnF1qt=s64","userId":"09645300777368779770"}},"outputId":"9afbb63c-228c-4706-d245-41e5a1d2c76d"},"source":["tfidfvec.get_feature_names()"],"execution_count":11,"outputs":[{"output_type":"execute_result","data":{"text/plain":["[' ',\n"," ' !',\n"," ' ! ',\n"," ' !!',\n"," ' !! ',\n"," ' !!!',\n"," ' !!! ',\n"," ' !!!!',\n"," ' #',\n"," ' # ',\n"," ' #f',\n"," ' #i',\n"," ' #l',\n"," ' #la',\n"," ' #lal',\n"," ' #lale',\n"," ' #m',\n"," ' #ma',\n"," ' #mam',\n"," ' #mama',\n"," ' #mamm',\n"," ' #r',\n"," ' #s',\n"," ' #sh',\n"," ' #t',\n"," ' #മ',\n"," ' &',\n"," ' & ',\n"," ' (',\n"," ' ( ',\n"," ' )',\n"," ' ) ',\n"," ' *',\n"," ' +',\n"," ' + ',\n"," ' ,',\n"," ' , ',\n"," ' ,,',\n"," ' ,,,',\n"," ' -',\n"," ' - ',\n"," ' .',\n"," ' . ',\n"," ' ..',\n"," ' .. ',\n"," ' ...',\n"," ' ... ',\n"," ' ....',\n"," ' .... ',\n"," ' .....',\n"," ' .അ',\n"," ' 0',\n"," ' 0.',\n"," ' 0:',\n"," ' 0:4',\n"," ' 0:5',\n"," ' 1',\n"," ' 1 ',\n"," ' 1.',\n"," ' 10',\n"," ' 10 ',\n"," ' 100',\n"," ' 100 ',\n"," ' 100%',\n"," ' 100c',\n"," ' 100cr',\n"," ' 11',\n"," ' 12',\n"," ' 12 ',\n"," ' 14',\n"," ' 15',\n"," ' 16',\n"," ' 17',\n"," ' 17 ',\n"," ' 18',\n"," ' 19',\n"," ' 1:',\n"," ' 1:0',\n"," ' 1:1',\n"," ' 1:2',\n"," ' 1:3',\n"," ' 1:5',\n"," ' 1m',\n"," ' 1m ',\n"," ' 1s',\n"," ' 1st',\n"," ' 1st ',\n"," ' 2',\n"," ' 2 ',\n"," ' 2.',\n"," ' 20',\n"," ' 20 ',\n"," ' 200',\n"," ' 200 ',\n"," ' 201',\n"," ' 2019',\n"," ' 2019 ',\n"," ' 202',\n"," ' 2020',\n"," ' 2020 ',\n"," ' 21',\n"," ' 21 ',\n"," ' 25',\n"," ' 28',\n"," ' 2:',\n"," ' 3',\n"," ' 3 ',\n"," ' 3.',\n"," ' 30',\n"," ' 30 ',\n"," ' 36',\n"," ' 369',\n"," ' 369 ',\n"," ' 4',\n"," ' 4 ',\n"," ' 5',\n"," ' 5 ',\n"," ' 50',\n"," ' 50 ',\n"," ' 500',\n"," ' 500 ',\n"," ' 6',\n"," ' 6 ',\n"," ' 68',\n"," ' 68 ',\n"," ' 7',\n"," ' 7 ',\n"," ' 8',\n"," ' 8 ',\n"," ' 9',\n"," ' :',\n"," ' : ',\n"," ' :)',\n"," ' :) ',\n"," ' :-',\n"," ' :- ',\n"," ' <',\n"," ' <3',\n"," ' <3 ',\n"," ' =',\n"," ' = ',\n"," ' ?',\n"," ' ? ',\n"," ' ??',\n"," ' ?? ',\n"," ' ???',\n"," ' ??? ',\n"," ' @',\n"," ' \\\\',\n"," ' \\\\r',\n"," ' \\\\r ',\n"," ' _',\n"," ' a',\n"," ' a ',\n"," ' a1',\n"," ' a10',\n"," ' a10 ',\n"," ' aa',\n"," ' aa ',\n"," ' aaa',\n"," ' aaa ',\n"," ' aaaa',\n"," ' aaan',\n"," ' aaay',\n"," ' aad',\n"," ' aada',\n"," ' aady',\n"," ' aadya',\n"," ' aag',\n"," ' aah',\n"," ' aak',\n"," ' aaka',\n"," ' aakan',\n"," ' aakat',\n"," ' aaki',\n"," ' aakk',\n"," ' aaku',\n"," ' aakum',\n"," ' aal',\n"," ' aan',\n"," ' aan ',\n"," ' aana',\n"," ' aanal',\n"," ' aane',\n"," ' aane ',\n"," ' aanen',\n"," ' aano',\n"," ' aano ',\n"," ' aanu',\n"," ' aanu ',\n"," ' aanu.',\n"," ' aar',\n"," ' aara',\n"," ' aare',\n"," ' aaren',\n"," ' aark',\n"," ' aaro',\n"," ' aarok',\n"," ' aaru',\n"," ' aarum',\n"," ' aas',\n"," ' aash',\n"," ' aasha',\n"," ' aav',\n"," ' aava',\n"," ' aavat',\n"," ' aavu',\n"," ' aavum',\n"," ' aay',\n"," ' aaya',\n"," ' aaya ',\n"," ' aayal',\n"," ' aayi',\n"," ' aayi ',\n"," ' aayir',\n"," ' aayit',\n"," ' ab',\n"," ' abh',\n"," ' abhi',\n"," ' abhin',\n"," ' abi',\n"," ' abin',\n"," ' abina',\n"," ' abr',\n"," ' abra',\n"," ' abrah',\n"," ' abri',\n"," ' abrid',\n"," ' ac',\n"," ' ach',\n"," ' acha',\n"," ' achan',\n"," ' achay',\n"," ' act',\n"," ' acti',\n"," ' actin',\n"," ' actio',\n"," ' acto',\n"," ' actor',\n"," ' ad',\n"," ' ada',\n"," ' adaa',\n"," ' adaar',\n"," ' adar',\n"," ' add',\n"," ' addi',\n"," ' addic',\n"," ' adh',\n"," ' adhi',\n"," ' adi',\n"," ' adi ',\n"," ' adi.',\n"," ' adi..',\n"," ' adic',\n"," ' adich',\n"," ' adii',\n"," ' adik',\n"," ' adik ',\n"," ' adika',\n"," ' adikk',\n"," ' adiku',\n"," ' adip',\n"," ' adipo',\n"," ' adiy',\n"," ' adu',\n"," ' adut',\n"," ' aduth',\n"," ' ady',\n"," ' af',\n"," ' aft',\n"," ' afte',\n"," ' after',\n"," ' ag',\n"," ' aga',\n"," ' agai',\n"," ' again',\n"," ' age',\n"," ' age ',\n"," ' agr',\n"," ' agra',\n"," ' agrah',\n"," ' ah',\n"," ' aha',\n"," ' ai',\n"," ' aj',\n"," ' aja',\n"," ' ak',\n"," ' aka',\n"," ' akk',\n"," ' akka',\n"," ' aku',\n"," ' akum',\n"," ' akum ',\n"," ' al',\n"," ' ala',\n"," ' alay',\n"," ' alayu',\n"," ' ali',\n"," ' all',\n"," ' all ',\n"," ' alla',\n"," ' alla ',\n"," ' allat',\n"," ' alle',\n"," ' alle ',\n"," ' alw',\n"," ' alwa',\n"," ' alway',\n"," ' am',\n"," ' am ',\n"," ' ama',\n"," ' amb',\n"," ' amm',\n"," ' amma',\n"," ' an',\n"," ' an ',\n"," ' ana',\n"," ' and',\n"," ' and ',\n"," ' ane',\n"," ' anen',\n"," ' anenn',\n"," ' ang',\n"," ' anga',\n"," ' angan',\n"," ' angu',\n"," ' angu ',\n"," ' ani',\n"," ' aniy',\n"," ' aniya',\n"," ' ann',\n"," ' ann ',\n"," ' anna',\n"," ' anna ',\n"," ' annan',\n"," ' anne',\n"," ' annu',\n"," ' annu ',\n"," ' annum',\n"," ' ano',\n"," ' ano ',\n"," ' ant',\n"," ' anth',\n"," ' anu',\n"," ' anu ',\n"," ' anu.',\n"," ' any',\n"," ' ap',\n"," ' apo',\n"," ' apo ',\n"," ' app',\n"," ' appa',\n"," ' appo',\n"," ' appo ',\n"," ' ar',\n"," ' ara',\n"," ' are',\n"," ' are ',\n"," ' ari',\n"," ' arik',\n"," ' arike',\n"," ' ariy',\n"," ' ariya',\n"," ' ariyi',\n"," ' arj',\n"," ' arju',\n"," ' arjun',\n"," ' ark',\n"," ' aro',\n"," ' arok',\n"," ' arokk',\n"," ' aru',\n"," ' arum',\n"," ' arum ',\n"," ' as',\n"," ' as ',\n"," ' ash',\n"," ' asha',\n"," ' asham',\n"," ' ass',\n"," ' at',\n"," ' at ',\n"," ' ath',\n"," ' ath ',\n"," ' atha',\n"," ' athe',\n"," ' athe ',\n"," ' athi',\n"," ' athin',\n"," ' atho',\n"," ' athr',\n"," ' athra',\n"," ' athu',\n"," ' athu ',\n"," ' athuk',\n"," ' au',\n"," ' av',\n"," ' ava',\n"," ' aval',\n"," ' avan',\n"," ' avan ',\n"," ' avar',\n"," ' avas',\n"," ' avasa',\n"," ' avast',\n"," ' avat',\n"," ' avath',\n"," ' ave',\n"," ' avi',\n"," ' avid',\n"," ' avu',\n"," ' avum',\n"," ' aw',\n"," ' awa',\n"," ' awe',\n"," ' awes',\n"," ' aweso',\n"," ' ay',\n"," ' aya',\n"," ' aya ',\n"," ' ayal',\n"," ' ayi',\n"," ' ayi ',\n"," ' ayir',\n"," ' ayiri',\n"," ' ayiru',\n"," ' ayit',\n"," ' ayo',\n"," ' ayy',\n"," ' b',\n"," ' b ',\n"," ' ba',\n"," ' baa',\n"," ' baap',\n"," ' baap ',\n"," ' bab',\n"," ' baby',\n"," ' bac',\n"," ' back',\n"," ' back ',\n"," ' bah',\n"," ' bahu',\n"," ' bahub',\n"," ' bak',\n"," ' ban',\n"," ' bas',\n"," ' be',\n"," ' be ',\n"," ' ben',\n"," ' bes',\n"," ' best',\n"," ' best ',\n"," ' bg',\n"," ' bgm',\n"," ' bgm ',\n"," ' bh',\n"," ' bha',\n"," ' bhas',\n"," ' bhasi',\n"," ' bi',\n"," ' big',\n"," ' big ',\n"," ' bir',\n"," ' birt',\n"," ' birth',\n"," ' bl',\n"," ' blo',\n"," ' bloc',\n"," ' block',\n"," ' bo',\n"," ' bol',\n"," ' bom',\n"," ' bon',\n"," ' bor',\n"," ' bore',\n"," ' bore ',\n"," ' bos',\n"," ' boss',\n"," ' boss ',\n"," ' box',\n"," ' box ',\n"," ' br',\n"," ' bri',\n"," ' bro',\n"," ' bro ',\n"," ' brot',\n"," ' broth',\n"," ' bu',\n"," ' bud',\n"," ' bus',\n"," ' but',\n"," ' but ',\n"," ' butt',\n"," ' butto',\n"," ' by',\n"," ' by ',\n"," ' c',\n"," ' ca',\n"," ' cam',\n"," ' came',\n"," ' camer',\n"," ' can',\n"," ' car',\n"," ' cari',\n"," ' carib',\n"," ' cas',\n"," ' ce',\n"," ' ch',\n"," ' cha',\n"," ' chaa',\n"," ' chak',\n"," ' chal',\n"," ' chali',\n"," ' chan',\n"," ' chang',\n"," ' chann',\n"," ' char',\n"," ' chara',\n"," ' chari',\n"," ' chat',\n"," ' chath',\n"," ' chav',\n"," ' chay',\n"," ' che',\n"," ' chec',\n"," ' chech',\n"," ' chei',\n"," ' chek',\n"," ' chem',\n"," ' chen',\n"," ' chenn',\n"," ' cher',\n"," ' cheri',\n"," ' chet',\n"," ' chett',\n"," ' chey',\n"," ' cheya',\n"," ' cheyt',\n"," ' cheyu',\n"," ' cheyy',\n"," ' chi',\n"," ' chil',\n"," ' chila',\n"," ' chin',\n"," ' chir',\n"," ' chiri',\n"," ' cho',\n"," ' chod',\n"," ' chodi',\n"," ' chor',\n"," ' chr',\n"," ' chri',\n"," ' chris',\n"," ' chu',\n"," ' chud',\n"," ' chudu',\n"," ' chun',\n"," ' chunk',\n"," ' chy',\n"," ' ci',\n"," ' cin',\n"," ' cine',\n"," ' cinem',\n"," ' cini',\n"," ' cinim',\n"," ' cl',\n"," ' cla',\n"," ' clas',\n"," ' class',\n"," ' cli',\n"," ' clim',\n"," ' clima',\n"," ' clu',\n"," ' club',\n"," ' cm',\n"," ' cmn',\n"," ' co',\n"," ' col',\n"," ' coll',\n"," ' colle',\n"," ' com',\n"," ' comb',\n"," ' combo',\n"," ' come',\n"," ' come ',\n"," ' comed',\n"," ' comm',\n"," ' comme',\n"," ' comp',\n"," ' compa',\n"," ' compl',\n"," ' con',\n"," ' cop',\n"," ' copy',\n"," ' copy ',\n"," ' cr',\n"," ' cr ',\n"," ' cro',\n"," ' cror',\n"," ' crore',\n"," ' cu',\n"," ' cut',\n"," ' cut ',\n"," ' d',\n"," ' d ',\n"," ' da',\n"," ' da ',\n"," ' daa',\n"," ' dai',\n"," ' dail',\n"," ' daiv',\n"," ' daiva',\n"," ' dan',\n"," ' danc',\n"," ' dance',\n"," ' dar',\n"," ' dat',\n"," ' date',\n"," ' date ',\n"," ' day',\n"," ' day ',\n"," ' days',\n"," ' days ',\n"," ' de',\n"," ' de ',\n"," ' dea',\n"," ' dec',\n"," ' deg',\n"," ' degr',\n"," ' degra',\n"," ' del',\n"," ' deli',\n"," ' deliv',\n"," ' dev',\n"," ' dh',\n"," ' dha',\n"," ' di',\n"," ' dia',\n"," ' dial',\n"," ' dialo',\n"," ' die',\n"," ' die ',\n"," ' dil',\n"," ' dile',\n"," ' dilee',\n"," ' dilo',\n"," ' dir',\n"," ' dire',\n"," ' direc',\n"," ' dis',\n"," ' disl',\n"," ' disli',\n"," ' div',\n"," ' diva',\n"," ' divas',\n"," ' do',\n"," ' don',\n"," ' dou',\n"," ' doub',\n"," ' doubl',\n"," ' doubt',\n"," ' dq',\n"," ' dq ',\n"," ' dr',\n"," ' dri',\n"," ' du',\n"," ' dub',\n"," ' dub ',\n"," ' dubb',\n"," ' e',\n"," ' e ',\n"," ' ea',\n"," ' eat',\n"," ' eath',\n"," ' eatt',\n"," ' eatta',\n"," ' ed',\n"," ' eda',\n"," ' edh',\n"," ' edi',\n"," ' edit',\n"," ' edit ',\n"," ' editi',\n"," ' edu',\n"," ' eduk',\n"," ' edukk',\n"," ' edut',\n"," ' eduth',\n"," ' ee',\n"," ' ee ',\n"," ' eee',\n"," ' eee ',\n"," ' eet',\n"," ' ef',\n"," ' eff',\n"," ' effe',\n"," ' effec',\n"," ' eg',\n"," ' ej',\n"," ' ejj',\n"," ' ejja',\n"," ' ejjat',\n"," ' ek',\n"," ' ekk',\n"," ' ekka',\n"," ' ekka ',\n"," ' el',\n"," ' ela',\n"," ' ell',\n"," ' ella',\n"," ' ella ',\n"," ' ellaa',\n"," ' ellam',\n"," ' ellar',\n"," ' ellav',\n"," ' en',\n"," ' ena',\n"," ' end',\n"," ' enda',\n"," ' endh',\n"," ' ene',\n"," ' eng',\n"," ' enga',\n"," ' engan',\n"," ' engi',\n"," ' engl',\n"," ' engli',\n"," ' eni',\n"," ' eni ',\n"," ' enik',\n"," ' enik ',\n"," ' enikk',\n"," ' eniku',\n"," ' enk',\n"," ' enki',\n"," ' enkil',\n"," ' enn',\n"," ' enn ',\n"," ' enna',\n"," ' enna ',\n"," ' ennaa',\n"," ' ennal',\n"," ' enne',\n"," ' enne ',\n"," ' enni',\n"," ' enno',\n"," ' ennor',\n"," ' ennu',\n"," ' ennu ',\n"," ' ennul',\n"," ' ennum',\n"," ' ent',\n"," ' enta',\n"," ' enta ',\n"," ' entam',\n"," ' ente',\n"," ' ente ',\n"," ' enter',\n"," ' enth',\n"," ' enth ',\n"," ' entha',\n"," ' enthe',\n"," ' enthi',\n"," ' entho',\n"," ' enthu',\n"," ' entr',\n"," ' enu',\n"," ' enu ',\n"," ' ep',\n"," ' epi',\n"," ' epp',\n"," ' eppo',\n"," ' er',\n"," ' era',\n"," ' erak',\n"," ' eran',\n"," ' erang',\n"," ' es',\n"," ' esh',\n"," ' esht',\n"," ' et',\n"," ' eth',\n"," ' eth ',\n"," ' etha',\n"," ' ethan',\n"," ' ethi',\n"," ' ethil',\n"," ' etho',\n"," ' ethr',\n"," ' ethra',\n"," ' ethu',\n"," ' ethu ',\n"," ' etr',\n"," ' etra',\n"," ' etra ',\n"," ' ett',\n"," ' etta',\n"," ' etta ',\n"," ' ettan',\n"," ' ettav',\n"," ' ev',\n"," ' eva',\n"," ' evd',\n"," ' evde',\n"," ' eve',\n"," ' ever',\n"," ' evi',\n"," ' evid',\n"," ' evide',\n"," ' ex',\n"," ' exp',\n"," ' expe',\n"," ' ez',\n"," ' ezh',\n"," ' ezhu',\n"," ' ezhut',\n"," ' f',\n"," ' fa',\n"," ' fac',\n"," ' face',\n"," ' face ',\n"," ' fah',\n"," ' faha',\n"," ' fahad',\n"," ' fam',\n"," ' fami',\n"," ' famil',\n"," ' fan',\n"," ' fan ',\n"," ' fan.',\n"," ' fan..',\n"," ' fana',\n"," ' fans',\n"," ' fans ',\n"," ' fansi',\n"," ' fas',\n"," ' fd',\n"," ' fdf',\n"," ' fdfs',\n"," ' fdfs ',\n"," ' fe',\n"," ' fee',\n"," ' feel',\n"," ' feel ',\n"," ' feel.',\n"," ' feeli',\n"," ' fi',\n"," ' fig',\n"," ' figh',\n"," ' fight',\n"," ' fil',\n"," ' fili',\n"," ' filim',\n"," ' film',\n"," ' film ',\n"," ' film.',\n"," ' films',\n"," ' fir',\n"," ' firs',\n"," ' first',\n"," ' fl',\n"," ' flo',\n"," ' flop',\n"," ' flop ',\n"," ' fo',\n"," ' for',\n"," ' for ',\n"," ' fr',\n"," ' fra',\n"," ' fre',\n"," ' free',\n"," ' fri',\n"," ' frie',\n"," ' frien',\n"," ' fro',\n"," ' from',\n"," ' from ',\n"," ' fu',\n"," ' ful',\n"," ' full',\n"," ' full ',\n"," ' g',\n"," ' ga',\n"," ' gam',\n"," ' gan',\n"," ' ge',\n"," ' gee',\n"," ' geet',\n"," ' gen',\n"," ' get',\n"," ' get ',\n"," ' gh',\n"," ' gho',\n"," ' ghos',\n"," ' gi',\n"," ' gir',\n"," ' giri',\n"," ' gl',\n"," ' go',\n"," ' god',\n"," ' goo',\n"," ' good',\n"," ' good ',\n"," ' goos',\n"," ' goose',\n"," ' gop',\n"," ' gopi',\n"," ' gopi ',\n"," ' gr',\n"," ' gra',\n"," ' gre',\n"," ' grea',\n"," ' great',\n"," ' gu',\n"," ' guy',\n"," ' guys',\n"," ' h',\n"," ' ha',\n"," ' ha ',\n"," ' hai',\n"," ' han',\n"," ' hang',\n"," ' hang ',\n"," ' hap',\n"," ' happ',\n"," ' happy',\n"," ' har',\n"," ' hard',\n"," ' hard ',\n"," ' hat',\n"," ' hate',\n"," ' hater',\n"," ' he',\n"," ' hea',\n"," ' heav',\n"," ' heavy',\n"," ' her',\n"," ' here',\n"," ' here ',\n"," ' here.',\n"," ' hero',\n"," ' hero ',\n"," ' hi',\n"," ' hig',\n"," ' high',\n"," ' hin',\n"," ' hind',\n"," ' hindi',\n"," ' his',\n"," ' hist',\n"," ' histo',\n"," ' hit',\n"," ' hit ',\n"," ' ho',\n"," ' hol',\n"," ' holl',\n"," ' holly',\n"," ' hop',\n"," ' hou',\n"," ' hour',\n"," ' hr',\n"," ' hri',\n"," ' hrid',\n"," ' hrida',\n"," ' hu',\n"," ' hy',\n"," ' i',\n"," ' i ',\n"," \" i'\",\n"," \" i'm\",\n"," \" i'm \",\n"," ' ia',\n"," ' iam',\n"," ' iam ',\n"," ' ic',\n"," ' ich',\n"," ' icha',\n"," ' ichay',\n"," ' id',\n"," ' ida',\n"," ' idak',\n"," ' idakk',\n"," ' idh',\n"," ' idh ',\n"," ' idi',\n"," ' idi ',\n"," ' idu',\n"," ' ij',\n"," ' ijj',\n"," ' ijja',\n"," ' ijjat',\n"," ' ik',\n"," ' ikk',\n"," ' ikka',\n"," ' ikka ',\n"," ' ikka.',\n"," ' ikkaa',\n"," ' ikkad',\n"," ' ikkan',\n"," ' ikkay',\n"," ...]"]},"metadata":{"tags":[]},"execution_count":11}]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"EDDXkAUI1DF1","executionInfo":{"status":"ok","timestamp":1624682789931,"user_tz":-330,"elapsed":17,"user":{"displayName":"Abhinav Kumar","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhN6WFfi4jSsnU9CzZnfDwqVV_KtgtD8yEnF1qt=s64","userId":"09645300777368779770"}},"outputId":"9999a616-7036-4ebf-e658-3faa21fef2d9"},"source":["tfidf_train.shape"],"execution_count":12,"outputs":[{"output_type":"execute_result","data":{"text/plain":["(15888, 30000)"]},"metadata":{"tags":[]},"execution_count":12}]},{"cell_type":"code","metadata":{"id":"szYsgWh4UFOy","executionInfo":{"status":"ok","timestamp":1624682789932,"user_tz":-330,"elapsed":8,"user":{"displayName":"Abhinav Kumar","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhN6WFfi4jSsnU9CzZnfDwqVV_KtgtD8yEnF1qt=s64","userId":"09645300777368779770"}}},"source":["x_train = tfidf_train\n","x_test = tfidf_val\n","y_train = y_train_index\n","y_test = y_val_index"],"execution_count":13,"outputs":[]},{"cell_type":"code","metadata":{"id":"QbypHNhYSMDy","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1624684748205,"user_tz":-330,"elapsed":1958280,"user":{"displayName":"Abhinav Kumar","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhN6WFfi4jSsnU9CzZnfDwqVV_KtgtD8yEnF1qt=s64","userId":"09645300777368779770"}},"outputId":"c480092c-c0bb-4ce3-c750-71aff0662557"},"source":["    from sklearn import metrics\n","    from sklearn.svm import SVC\n","    model_SVM = SVC(probability=True)\n","    model_SVM.fit(x_train, y_train)\n","    y_pred_SVM = model_SVM.predict(x_test)\n","    print(\"SVM\")\n","    print(metrics.classification_report(y_test, y_pred_SVM))\n","    \n","    from sklearn.ensemble import RandomForestClassifier\n","    rf = RandomForestClassifier()\n","    rf.fit(x_train,y_train)\n","    y_pred_rf = rf.predict(x_test)\n","    print(\"random\")\n","    print(metrics.classification_report(y_test, y_pred_rf))\n","    \n","    from sklearn.linear_model import LogisticRegression\n","    LR = LogisticRegression()\n","    LR.fit(x_train,y_train)\n","    y_pred_LR = LR.predict(x_test)\n","    print(\"Logistic Regression\")\n","    print(metrics.classification_report(y_test, y_pred_LR ))\n","    \n","    from sklearn.neighbors import KNeighborsClassifier\n","    neigh = KNeighborsClassifier()\n","    neigh.fit(x_train,y_train)\n","    y_pred_KNN = neigh.predict(x_test)\n","    print(\"KNN\")\n","    print(metrics.classification_report(y_test, y_pred_KNN ))\n","    \n","    from sklearn.naive_bayes import GaussianNB\n","    naive = GaussianNB()\n","    naive.fit(x_train.toarray(),y_train)\n","    y_pred_naive = naive.predict(x_test.toarray())\n","    print(\"Naive Bayes\")\n","    print(metrics.classification_report(y_test, y_pred_naive ))\n","    \n","    from sklearn.ensemble import GradientBoostingClassifier\n","    gradient = GradientBoostingClassifier()\n","    gradient.fit(x_train,y_train)\n","    y_pred_gradient = gradient.predict(x_test)\n","    print(\"Gradient Boosting\")\n","    print(metrics.classification_report(y_test, y_pred_gradient ))\n","    \n","    from sklearn.tree import DecisionTreeClassifier\n","    decision = DecisionTreeClassifier()\n","    decision.fit(x_train,y_train)\n","    y_pred_decision = decision.predict(x_test)\n","    print(\"Decision Tree\")\n","    print(metrics.classification_report(y_test, y_pred_decision ))\n","\n","    from sklearn.ensemble import AdaBoostClassifier\n","    AdaBoostclf = AdaBoostClassifier()\n","    AdaBoostclf.fit(x_train,y_train)\n","    y_pred_AdaBoostclf = AdaBoostclf.predict(x_test)\n","    print(\"AdaBoost classifier\")\n","    print(metrics.classification_report(y_test, y_pred_AdaBoostclf ))\n","    "],"execution_count":14,"outputs":[{"output_type":"stream","text":["SVM\n","              precision    recall  f1-score   support\n","\n","           0       0.73      0.24      0.36       102\n","           1       0.78      0.54      0.64       237\n","           2       0.74      0.87      0.80       706\n","           3       0.83      0.79      0.81       141\n","           4       0.74      0.77      0.75       580\n","\n","    accuracy                           0.75      1766\n","   macro avg       0.76      0.64      0.67      1766\n","weighted avg       0.75      0.75      0.74      1766\n","\n","random\n","              precision    recall  f1-score   support\n","\n","           0       0.70      0.14      0.23       102\n","           1       0.83      0.29      0.43       237\n","           2       0.65      0.88      0.74       706\n","           3       0.80      0.67      0.73       141\n","           4       0.71      0.72      0.71       580\n","\n","    accuracy                           0.69      1766\n","   macro avg       0.74      0.54      0.57      1766\n","weighted avg       0.71      0.69      0.66      1766\n","\n"],"name":"stdout"},{"output_type":"stream","text":["/usr/local/lib/python3.7/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n","STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n","\n","Increase the number of iterations (max_iter) or scale the data as shown in:\n","    https://scikit-learn.org/stable/modules/preprocessing.html\n","Please also refer to the documentation for alternative solver options:\n","    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n","  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"],"name":"stderr"},{"output_type":"stream","text":["Logistic Regression\n","              precision    recall  f1-score   support\n","\n","           0       0.64      0.26      0.37       102\n","           1       0.77      0.51      0.61       237\n","           2       0.74      0.86      0.80       706\n","           3       0.84      0.76      0.80       141\n","           4       0.72      0.77      0.74       580\n","\n","    accuracy                           0.74      1766\n","   macro avg       0.74      0.63      0.66      1766\n","weighted avg       0.74      0.74      0.73      1766\n","\n","KNN\n","              precision    recall  f1-score   support\n","\n","           0       0.46      0.29      0.36       102\n","           1       0.48      0.49      0.48       237\n","           2       0.63      0.79      0.70       706\n","           3       0.65      0.62      0.64       141\n","           4       0.67      0.53      0.59       580\n","\n","    accuracy                           0.62      1766\n","   macro avg       0.58      0.54      0.55      1766\n","weighted avg       0.62      0.62      0.61      1766\n","\n","Naive Bayes\n","              precision    recall  f1-score   support\n","\n","           0       0.21      0.28      0.24       102\n","           1       0.28      0.58      0.38       237\n","           2       0.70      0.62      0.66       706\n","           3       0.68      0.61      0.64       141\n","           4       0.62      0.42      0.50       580\n","\n","    accuracy                           0.53      1766\n","   macro avg       0.50      0.50      0.48      1766\n","weighted avg       0.59      0.53      0.54      1766\n","\n","Gradient Boosting\n","              precision    recall  f1-score   support\n","\n","           0       0.67      0.16      0.25       102\n","           1       0.81      0.37      0.51       237\n","           2       0.63      0.89      0.74       706\n","           3       0.82      0.66      0.73       141\n","           4       0.74      0.67      0.70       580\n","\n","    accuracy                           0.69      1766\n","   macro avg       0.73      0.55      0.59      1766\n","weighted avg       0.71      0.69      0.67      1766\n","\n","Decision Tree\n","              precision    recall  f1-score   support\n","\n","           0       0.22      0.21      0.21       102\n","           1       0.41      0.38      0.40       237\n","           2       0.63      0.65      0.64       706\n","           3       0.59      0.56      0.58       141\n","           4       0.59      0.60      0.60       580\n","\n","    accuracy                           0.56      1766\n","   macro avg       0.49      0.48      0.48      1766\n","weighted avg       0.56      0.56      0.56      1766\n","\n","AdaBoost classifier\n","              precision    recall  f1-score   support\n","\n","           0       0.34      0.10      0.15       102\n","           1       0.81      0.18      0.29       237\n","           2       0.53      0.50      0.52       706\n","           3       0.61      0.61      0.61       141\n","           4       0.45      0.68      0.54       580\n","\n","    accuracy                           0.50      1766\n","   macro avg       0.55      0.41      0.42      1766\n","weighted avg       0.54      0.50      0.48      1766\n","\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"dutLuW-7NitW","executionInfo":{"status":"ok","timestamp":1624684748208,"user_tz":-330,"elapsed":29,"user":{"displayName":"Abhinav Kumar","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhN6WFfi4jSsnU9CzZnfDwqVV_KtgtD8yEnF1qt=s64","userId":"09645300777368779770"}}},"source":[""],"execution_count":14,"outputs":[]},{"cell_type":"code","metadata":{"id":"uSFmluVXNkL2","executionInfo":{"status":"ok","timestamp":1624684748209,"user_tz":-330,"elapsed":21,"user":{"displayName":"Abhinav Kumar","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhN6WFfi4jSsnU9CzZnfDwqVV_KtgtD8yEnF1qt=s64","userId":"09645300777368779770"}}},"source":[""],"execution_count":14,"outputs":[]},{"cell_type":"code","metadata":{"id":"aKJdPx2jNkO3"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"XbQ1Hm3TNkSP"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"bQANqfdsf0i6","executionInfo":{"status":"ok","timestamp":1624684807665,"user_tz":-330,"elapsed":1105,"user":{"displayName":"Abhinav Kumar","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhN6WFfi4jSsnU9CzZnfDwqVV_KtgtD8yEnF1qt=s64","userId":"09645300777368779770"}}},"source":["tfidf_test = tfidfvec.transform(data_test['text'])"],"execution_count":15,"outputs":[]},{"cell_type":"code","metadata":{"id":"RBDDmfD_URTa","executionInfo":{"status":"ok","timestamp":1624684837824,"user_tz":-330,"elapsed":29475,"user":{"displayName":"Abhinav Kumar","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhN6WFfi4jSsnU9CzZnfDwqVV_KtgtD8yEnF1qt=s64","userId":"09645300777368779770"}}},"source":["y_pred_model_SVM_prob = model_SVM.predict_proba(tfidf_test)"],"execution_count":16,"outputs":[]},{"cell_type":"code","metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"7VQ3hfA7gECM","executionInfo":{"status":"ok","timestamp":1624684837826,"user_tz":-330,"elapsed":12,"user":{"displayName":"Abhinav Kumar","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhN6WFfi4jSsnU9CzZnfDwqVV_KtgtD8yEnF1qt=s64","userId":"09645300777368779770"}},"outputId":"d72bbc7b-ff46-4285-a82f-5b670495cf1b"},"source":["y_pred_model_SVM_prob"],"execution_count":17,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array([[5.02168957e-01, 8.42647481e-02, 1.51468772e-01, 4.81105332e-02,\n","        2.13986990e-01],\n","       [4.69749344e-02, 1.37106408e-01, 3.08146152e-01, 2.19189985e-03,\n","        5.05580606e-01],\n","       [5.17165730e-02, 1.39418025e-01, 4.16065412e-01, 1.71490637e-02,\n","        3.75650926e-01],\n","       ...,\n","       [9.95890693e-02, 1.62487191e-01, 4.38871624e-01, 1.20002178e-03,\n","        2.97852093e-01],\n","       [9.88383717e-02, 4.00563448e-01, 3.18753952e-01, 6.64705865e-04,\n","        1.81179523e-01],\n","       [2.13842768e-01, 2.47631509e-01, 4.29130849e-01, 4.49199039e-04,\n","        1.08945674e-01]])"]},"metadata":{"tags":[]},"execution_count":17}]},{"cell_type":"code","metadata":{"id":"i_feFgnOiZM0","executionInfo":{"status":"ok","timestamp":1624684977869,"user_tz":-330,"elapsed":615,"user":{"displayName":"Abhinav Kumar","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhN6WFfi4jSsnU9CzZnfDwqVV_KtgtD8yEnF1qt=s64","userId":"09645300777368779770"}}},"source":["df_y_pred_model_SVM_prob= pd.DataFrame(y_pred_model_SVM_prob, columns = ['Mixed_feelings' ,'Negative' ,'Positive' ,'not-malayalam' ,'unknown_state'])"],"execution_count":18,"outputs":[]},{"cell_type":"code","metadata":{"id":"PujUzf51iZOG","executionInfo":{"status":"ok","timestamp":1624684979229,"user_tz":-330,"elapsed":696,"user":{"displayName":"Abhinav Kumar","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhN6WFfi4jSsnU9CzZnfDwqVV_KtgtD8yEnF1qt=s64","userId":"09645300777368779770"}}},"source":["df_y_pred_model_SVM_prob.to_csv('/content/drive/MyDrive/HASOC_2021/Sentiment_Dravidian/Malyalam/SVM_char_1-6_gram_Malayalam_74.csv',index = False)"],"execution_count":19,"outputs":[]},{"cell_type":"code","metadata":{"id":"k-XiNgH_i1Md","executionInfo":{"status":"ok","timestamp":1624684981037,"user_tz":-330,"elapsed":3,"user":{"displayName":"Abhinav Kumar","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhN6WFfi4jSsnU9CzZnfDwqVV_KtgtD8yEnF1qt=s64","userId":"09645300777368779770"}}},"source":["y_pred_LR_prob = LR.predict_proba(tfidf_test)"],"execution_count":20,"outputs":[]},{"cell_type":"code","metadata":{"id":"pS6nkrMbi1P_","executionInfo":{"status":"ok","timestamp":1624684982931,"user_tz":-330,"elapsed":3,"user":{"displayName":"Abhinav Kumar","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhN6WFfi4jSsnU9CzZnfDwqVV_KtgtD8yEnF1qt=s64","userId":"09645300777368779770"}}},"source":["df_y_pred_LR_prob= pd.DataFrame(y_pred_LR_prob, columns = ['Mixed_feelings' ,'Negative' ,'Positive' ,'not-malayalam' ,'unknown_state'])"],"execution_count":21,"outputs":[]},{"cell_type":"code","metadata":{"id":"ru0vsVTsiZT3","executionInfo":{"status":"ok","timestamp":1624684984209,"user_tz":-330,"elapsed":5,"user":{"displayName":"Abhinav Kumar","photoUrl":"https://lh3.googleusercontent.com/a-/AOh14GhN6WFfi4jSsnU9CzZnfDwqVV_KtgtD8yEnF1qt=s64","userId":"09645300777368779770"}}},"source":["df_y_pred_LR_prob.to_csv('/content/drive/MyDrive/HASOC_2021/Sentiment_Dravidian/Malyalam/LR_char_1-6_gram_Malayalam_73.csv', index = False)"],"execution_count":22,"outputs":[]},{"cell_type":"code","metadata":{"id":"7vXANa-i7HFG"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"OYQGvHNYnxGT"},"source":[""],"execution_count":null,"outputs":[]},{"cell_type":"code","metadata":{"id":"Ig9Rc2Lrny6y"},"source":[""],"execution_count":null,"outputs":[]}]}