hate_speech_make_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "labels = pd.read_csv('data/annotations_metadata.csv', index_col='file_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>subforum_id</th>\n",
       "      <th>num_contexts</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>file_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>12834217_1</th>\n",
       "      <td>572066</td>\n",
       "      <td>1346</td>\n",
       "      <td>0</td>\n",
       "      <td>noHate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12834217_2</th>\n",
       "      <td>572066</td>\n",
       "      <td>1346</td>\n",
       "      <td>0</td>\n",
       "      <td>noHate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12834217_3</th>\n",
       "      <td>572066</td>\n",
       "      <td>1346</td>\n",
       "      <td>0</td>\n",
       "      <td>noHate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12834217_4</th>\n",
       "      <td>572066</td>\n",
       "      <td>1346</td>\n",
       "      <td>0</td>\n",
       "      <td>hate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12834217_5</th>\n",
       "      <td>572066</td>\n",
       "      <td>1346</td>\n",
       "      <td>0</td>\n",
       "      <td>noHate</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            user_id  subforum_id  num_contexts   label\n",
       "file_id                                               \n",
       "12834217_1   572066         1346             0  noHate\n",
       "12834217_2   572066         1346             0  noHate\n",
       "12834217_3   572066         1346             0  noHate\n",
       "12834217_4   572066         1346             0    hate\n",
       "12834217_5   572066         1346             0  noHate"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "noHate      9507\n",
       "hate        1196\n",
       "relation     168\n",
       "idk/skip      73\n",
       "Name: label, dtype: int64"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels.label.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'12845244_1'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = 'data/sampled_test/'\n",
    "files = os.listdir(path)\n",
    "files.sort()\n",
    "files[0][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "user_id           574784\n",
       "subforum_id         1397\n",
       "num_contexts           0\n",
       "label           idk/skip\n",
       "Name: 12845244_1, dtype: object"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "labels.loc[files[0][:10]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(columns=['text', 'label'])\n",
    "i = 0\n",
    "path = 'data/sampled_test/'\n",
    "files = os.listdir(path)\n",
    "for fname in files:\n",
    "    with open(path + fname, 'r', encoding=\"utf-8\") as readfile:\n",
    "        line = readfile.readline()\n",
    "    df.loc[i] = {'text': line, 'label': labels.loc[fname[0:10]].label}\n",
    "    i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The white man has no future in Canada , that i...</td>\n",
       "      <td>idk/skip</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>I take the assumption that your Nation is so c...</td>\n",
       "      <td>noHate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>If you want to hear what a black `` intellectu...</td>\n",
       "      <td>hate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>I 've always considered teaching as one of the...</td>\n",
       "      <td>hate</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>And the sad thing is the white students at tho...</td>\n",
       "      <td>hate</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text     label\n",
       "0  The white man has no future in Canada , that i...  idk/skip\n",
       "1  I take the assumption that your Nation is so c...    noHate\n",
       "2  If you want to hear what a black `` intellectu...      hate\n",
       "3  I 've always considered teaching as one of the...      hate\n",
       "4  And the sad thing is the white students at tho...      hate"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(478, 2)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('hate-speech', encoding='utf-8', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "2bbb540255468386325b4206726795d516400b965540705b79d9ea24d0bda07d"
  },
  "kernelspec": {
   "display_name": "Python 3.8.8 64-bit ('base': conda)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}