getting_query_terms.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generating dicts of query terms for every contentious term"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### English"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# importing wordforms\n",
    "with open('en_cont_wordforms.json','r') as jf:\n",
    "    en_cont_wordforms = json.load(jf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# importing manually found wordforms\n",
    "with open('wordforms_added_manually.json','r') as jf:\n",
    "    additional_wordforms = json.load(jf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lemma_wordforms_dict = {}\n",
    "for term, lemmata in en_cont_wordforms.items():\n",
    "    list_of_wordforms = []\n",
    "    for list_of_lemmas in lemmata.values():\n",
    "        if len(list_of_lemmas) > 0:\n",
    "            for l in list_of_lemmas:\n",
    "                list_of_wordforms.extend([i.lower() for i in l['wordforms']]) # lowercase wordforms\n",
    "                lemma = l['lemma'].lower() # taking only 1 lemma per term (PoS doesn't matter)\n",
    "        else:\n",
    "            lemma = term # if there's no lemmata, take the initial term as lemma\n",
    "    lemma_wordforms_dict[lemma] = list_of_wordforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# adding manually defined wordforms to the dict\n",
    "\n",
    "for lemma, wordforms in lemma_wordforms_dict.items():\n",
    "    if lemma in additional_wordforms['en'].keys():\n",
    "        wordforms.extend(additional_wordforms['en'][lemma])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# getting only unique wordforms\n",
    "# for querying terms, we don't need compound comparatives and superlatives, capitalised forms,\n",
    "# nor the canonical forms (because we already have a lemma)\n",
    "\n",
    "cont_lemma_forms = {}\n",
    "for term, lemmata in lemma_wordforms_dict.items():\n",
    "    clean_lemmata = []\n",
    "    if len(lemmata) > 0:\n",
    "        clean_lemmata = set([l for l in lemmata if l != term and 'more ' not in l and 'most ' not in l])\n",
    "        cont_lemma_forms[term] = [i for i in clean_lemmata]\n",
    "    else:\n",
    "        cont_lemma_forms[term] = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# count how many query terms there are\n",
    "count_en_all = 0\n",
    "for lemma, forms in cont_lemma_forms.items():\n",
    "    count_en_all += 1 + len(forms)\n",
    "print(count_en_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# exporting file\n",
    "with open('query_terms_cont_en.json', 'w') as jf:\n",
    "    json.dump(cont_lemma_forms, jf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Dutch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# importing wordforms\n",
    "with open('nl_cont_wordforms.json','r') as jf:\n",
    "    nl_cont_wordforms = json.load(jf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lemma_wordforms_dict_nl = {}\n",
    "for term, lemmata in nl_cont_wordforms.items():\n",
    "    list_of_wordforms = []\n",
    "    for list_of_lemmas in lemmata.values():\n",
    "        if len(list_of_lemmas) > 0:\n",
    "            for l in list_of_lemmas:\n",
    "                list_of_wordforms.extend([i.lower() for i in l['wordforms']]) # lowercase wordforms\n",
    "                lemma = l['lemma'].lower() # taking only 1 lemma per term (PoS doesn't matter)\n",
    "        else:\n",
    "            lemma = term # if there's no lemmata, take the initial term as lemma\n",
    "    lemma_wordforms_dict_nl[lemma] = list_of_wordforms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# adding manually defined wordforms to the dict\n",
    "\n",
    "for lemma, wordforms in lemma_wordforms_dict_nl.items():\n",
    "    if lemma in additional_wordforms['nl'].keys():\n",
    "        wordforms.extend(additional_wordforms['nl'][lemma])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# getting only unique wordforms\n",
    "# for querying terms, we don't need capitalised forms,\n",
    "# nor the canonical forms (because we already have a lemma)\n",
    "\n",
    "cont_lemma_forms_nl = {}\n",
    "for term, lemmata in lemma_wordforms_dict_nl.items():\n",
    "    clean_lemmata = []\n",
    "    if len(lemmata) > 0:\n",
    "        clean_lemmata = set([l for l in lemmata if l != term])\n",
    "        cont_lemma_forms_nl[term] = [i for i in clean_lemmata]\n",
    "    else:\n",
    "        cont_lemma_forms_nl[term] = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# count how many query terms there are\n",
    "count_nl = 0\n",
    "for lemma, forms in cont_lemma_forms_nl.items():\n",
    "    count_nl += 1 + len(forms)\n",
    "print(count_nl)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# exporting file\n",
    "with open('query_terms_cont_nl.json', 'w') as jf:\n",
    "    json.dump(cont_lemma_forms_nl, jf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}