topic_model_implementation_comparision.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "from nlpipe import NlPipe\n",
    "import numpy as np\n",
    "import os\n",
    "from tqdm.auto import tqdm\n",
    "import logging\n",
    "from threadpoolctl import threadpool_limits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "text df found. loading.\n"
     ]
    }
   ],
   "source": [
    "path = \"b_collection_extracted/\"\n",
    "\n",
    "stat_df = pd.read_pickle(f\"{path}stat_df\")\n",
    "if os.path.exists(f\"{path}text_df\"):\n",
    "    print(\"text df found. loading.\")\n",
    "    text_df = pd.read_pickle(f\"{path}text_df\")\n",
    "    texts = text_df.full_text.to_list()\n",
    "    thread_ids = text_df.thread_id.to_list()\n",
    "else:\n",
    "    thread_ids = stat_df.thread_id.to_list()\n",
    "    post_df = pd.read_pickle(f\"{path}post_df_extracted\")\n",
    "    thread_id_of_posts = np.array(post_df.thread_id, dtype=np.uint32)\n",
    "    texts = [\" \".join(post_df.full_string[thread_id_of_posts == thread_id].tolist()) for thread_id in thread_ids]\n",
    "    post_df = None\n",
    "    text_df = pd.DataFrame([thread_ids, texts]).transpose()\n",
    "    text_df.columns = ['thread_id', 'full_text']\n",
    "    text_df.to_pickle(f\"{path}text_df\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "326538 is limiting to 105583\n",
      "Found preprocessed data. Loading\n",
      "105583 preprocessed docs of 326538 docs loaded.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4aeb8930129c4ab0be584ae5662f4757",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating bag of words:   0%|          | 0/105583 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coherence LDA-Multicore (Gensim): 0.41049825837304993, coherence using MALLET:0.5191123715619884\n"
     ]
    }
   ],
   "source": [
    "nlp = NlPipe.NlPipe(texts, path=path, document_ids=thread_ids, no_processes=10)\n",
    "filter_array = np.logical_and(stat_df.language == 'en',\n",
    "                              stat_df.replies >= 10)\n",
    "print(f\"{len(filter_array)} is limiting to {sum(filter_array)}\")\n",
    "nlp.preprocess(load_existing=True, filter_loaded=filter_array)\n",
    "nlp.create_bag_of_words(filter_extremes=True, min_df=25, max_df=0.3)\n",
    "with threadpool_limits(limits=1, user_api='blas'):\n",
    "    nlp.create_lda_model(no_topics=20, passes=5, alpha='asymmetric', eta=0.01)\n",
    "coherence_normal = nlp.calculate_coherence()\n",
    "coherence_normal = coherence_normal.get_coherence()\n",
    "nlp.create_mallet_lda_model(no_topics=20)\n",
    "coherence_mallet = nlp.calculate_coherence()\n",
    "coherence_mallet = coherence_mallet.get_coherence()\n",
    "print(f\"Coherence LDA-Multicore (Gensim): {coherence_normal}, coherence using MALLET:{coherence_mallet}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}