Cluster themes.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "793bace0-0b27-4a1a-9054-8b2d35e954d5",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/nathansmith/anaconda3/envs/chatbot/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from langchain_openai import OpenAIEmbeddings\n",
    "from graphdatascience import GraphDataScience\n",
    "from getpass import getpass\n",
    "import pandas as pd\n",
    "import os\n",
    "import numpy as np\n",
    "import nltk\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from IPython.display import display\n",
    "import string"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5555a013-1e44-4280-8f35-f734d017ff2c",
   "metadata": {},
   "source": [
    "# Set up connection to Neo4j"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8c2e5ff4-20fb-4c9b-92ab-7e778f3db6d0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      " ········\n"
     ]
    }
   ],
   "source": [
    "neo4j_password = getpass()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7c80d2d2-dd2b-48b5-a103-c7baabec3ca1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      " ········\n"
     ]
    }
   ],
   "source": [
    "openai_api_key = getpass()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "8e68f7ef-9ccc-4ccf-829a-1bd33654dfba",
   "metadata": {},
   "outputs": [],
   "source": [
    "neo4j_uri = \"neo4j+s://2fe3bf28.databases.neo4j.io\"\n",
    "neo4j_user = \"neo4j\"\n",
    "gds = GraphDataScience(neo4j_uri, auth=(neo4j_user, neo4j_password))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b692a6c7-c27a-401c-bdb3-afce3644db97",
   "metadata": {},
   "source": [
    "# Find themes with common stems"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "748f5617-e826-4e6c-bf0c-27c6f52ec2d1",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     /Users/nathansmith/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "nltk.download('wordnet')\n",
    "lemmatizer = WordNetLemmatizer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8f97d56a-e28e-4229-b41d-00e320b64e6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "theme_df = gds.run_cypher(\"\"\"\n",
    "    MATCH (t:Theme) \n",
    "    RETURN toLower(t.description) AS description \n",
    "    \"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7c51bbde-fa35-4226-80d5-3aa1e971b740",
   "metadata": {},
   "outputs": [],
   "source": [
    "theme_df['stem'] = theme_df['description'].map(lambda x: lemmatizer.lemmatize(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7d49b776-7a16-4d0e-b625-2d99bb007750",
   "metadata": {},
   "outputs": [],
   "source": [
    "table = str.maketrans(\"\", \"\", string.punctuation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "c62765e0-fcac-48bf-bf52-fe0927f056d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "theme_df['no_space_stem'] = theme_df['stem'].str.replace(\" \", \"\")\n",
    "theme_df['no_space_stem'] = theme_df['no_space_stem'].str.translate(table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "aa730c3b-3c62-4e87-ac8e-3e137654b7c8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>description</th>\n",
       "      <th>stem</th>\n",
       "      <th>no_space_stem</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>28937</th>\n",
       "      <td>haunted house</td>\n",
       "      <td>haunted house</td>\n",
       "      <td>hauntedhouse</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         description           stem no_space_stem\n",
       "28937  haunted house  haunted house  hauntedhouse"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "theme_df[theme_df['description'] == 'haunted house']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "abf06015-daf0-44d1-93b6-65628ef25c74",
   "metadata": {},
   "outputs": [],
   "source": [
    "grouped_themes = theme_df[['description', 'no_space_stem']].groupby('no_space_stem').apply(np.unique, include_groups=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "bb6a8595-12bf-44f9-9ec0-a649b98339ca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1533, 2)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shared_stems = grouped_themes[grouped_themes.map(len)>1]\n",
    "shared_stems_df = shared_stems.reset_index()\n",
    "shared_stems_df.columns = ['stem', 'descriptions']\n",
    "shared_stems_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "a1e81e30-b717-47a1-9939-cd46006caa73",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>stem</th>\n",
       "      <th>descriptions</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>774</th>\n",
       "      <td>lily</td>\n",
       "      <td>[lilies, lily]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>523</th>\n",
       "      <td>fly</td>\n",
       "      <td>[flies, fly]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1156</th>\n",
       "      <td>rumor</td>\n",
       "      <td>[rumor, rumors]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1069</th>\n",
       "      <td>punk</td>\n",
       "      <td>[punk, punks]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>294</th>\n",
       "      <td>copper</td>\n",
       "      <td>[copper, coppers]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1325</th>\n",
       "      <td>summer</td>\n",
       "      <td>[summer, summers]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1146</th>\n",
       "      <td>rockstar</td>\n",
       "      <td>[rock star, rockstar]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1440</th>\n",
       "      <td>undertaker</td>\n",
       "      <td>[undertaker, undertakers]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257</th>\n",
       "      <td>colonial</td>\n",
       "      <td>[colonial, colonials]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>american</td>\n",
       "      <td>[american, americans]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            stem               descriptions\n",
       "774         lily             [lilies, lily]\n",
       "523          fly               [flies, fly]\n",
       "1156       rumor            [rumor, rumors]\n",
       "1069        punk              [punk, punks]\n",
       "294       copper          [copper, coppers]\n",
       "1325      summer          [summer, summers]\n",
       "1146    rockstar      [rock star, rockstar]\n",
       "1440  undertaker  [undertaker, undertakers]\n",
       "257     colonial      [colonial, colonials]\n",
       "35      american      [american, americans]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shared_stems_df.sample(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "544974b6-d376-46e2-a431-567c0039eede",
   "metadata": {},
   "source": [
    "## Send stems shared by multiple themes to Neo4j"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "90a8e827-1d31-424e-889f-c4b92931d9b1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"CREATE CONSTRAINT stem_node_key IF NOT EXISTS FOR (s:Stem) REQUIRE s.stem IS NODE KEY\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "5a4a93a2-3b1e-4948-890e-17f6be108792",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>stemmedCount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3111</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   stemmedCount\n",
       "0          3111"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"UNWIND $data AS row\n",
    "MERGE (s:Stem {stem:row['stem']})\n",
    "WITH row, s\n",
    "UNWIND row['descriptions'] AS description\n",
    "MATCH (t:Theme)\n",
    "WHERE toLower(t.description) = description\n",
    "MERGE (t)-[:HAS_STEM]->(s)\n",
    "SET s.descriptions = row['descriptions']\n",
    "RETURN count(*) AS stemmedCount\"\"\",\n",
    "              {\"data\": shared_stems_df.to_dict(\"records\")})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7ae388cb-8244-4a3f-a104-4ac8897c7a74",
   "metadata": {},
   "source": [
    "## Aggregate theme vectors for themes that share a stem"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7fac48ce-592d-4a00-8caa-6d5aea1f4360",
   "metadata": {},
   "source": [
    "Set the stem's vector to whichever theme shows up most often in movies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "1f6a3805-7f19-46e5-b891-798a508030d0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"\n",
    "MATCH (s:Stem)\n",
    "CALL {\n",
    "WITH s\n",
    "MATCH (s)<-[:HAS_STEM]-(t)\n",
    "WITH s, t\n",
    "ORDER BY count{(t)<-[:HAS_THEME]-()} DESC\n",
    "WITH s, collect(t) AS themes\n",
    "CALL db.create.setNodeVectorProperty(s, \"embedding\", themes[0].embedding)\n",
    "}\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "d2c4756c-3f09-460d-a040-8309dffa2b23",
   "metadata": {},
   "outputs": [],
   "source": [
    "shared_stem_df = gds.run_cypher(\"\"\"\n",
    "MATCH (s:Stem)<-[:HAS_STEM]-(t)\n",
    "RETURN s.stem AS stem, collect(t.embedding) AS embeddings\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "ceebdb21-9ce4-4a00-b823-4ca66a78729b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_mean_vector(row):\n",
    "    mean_vector = np.array(row['embeddings']).mean(axis=0).tolist()\n",
    "    return mean_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "24853dc9-9286-4935-98de-a07da1003e46",
   "metadata": {},
   "outputs": [],
   "source": [
    "shared_stem_df['mean_embedding'] = shared_stem_df.apply(get_mean_vector, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "c2639c76-657a-4e89-80f6-6345a492bd5f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"\n",
    "    UNWIND $data AS row\n",
    "    MATCH (s:Stem {stem: row['stem']})\n",
    "    CALL db.create.setNodeVectorProperty(s, \"embedding\", row['mean_embedding'])\"\"\",\n",
    "                   {\"data\":shared_stem_df[['stem', 'mean_embedding']].to_dict(\"records\")})\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f633d72a-871d-46ce-a19f-156569385865",
   "metadata": {},
   "source": [
    "# Create projection\n",
    "Include stems and individual themes that do not share a stem with another theme.\n",
    "\n",
    "Use the theme embedding and the aggregated embedding for stems."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "3f591780-6653-42fd-b746-6acf9d095bbd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"\n",
    "MATCH (t:Stem|Theme)\n",
    "WHERE NOT EXISTS {(t)-[:HAS_STEM]->()}\n",
    "SET t:Groupable\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "5938f619-084a-468e-b5dc-129fafad89e9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"CREATE VECTOR INDEX groupbale_vectors IF NOT EXISTS \n",
    "                  FOR (t:Groupable)\n",
    "                  ON (t.embedding)\n",
    "                  OPTIONS {indexConfig: \n",
    "                      {`vector.dimensions`: 1536,\n",
    "                       `vector.similarity_function`: 'cosine'\n",
    "                       }}\n",
    "                       \"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "39686f77-1aa0-4c98-8816-2bed1cf013f7",
   "metadata": {},
   "outputs": [],
   "source": [
    "g_themes, result = gds.graph.cypher.project(\"\"\"\n",
    "MATCH (t:Groupable)\n",
    "RETURN gds.graph.project(\"themes\", \n",
    "t, null,\n",
    "{sourceNodeProperties: t{.embedding},\n",
    "targetNodeProperties: null}\n",
    ")\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "d77ec393-5071-4f42-9bfb-435c6da79eef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'relationshipCount': 0,\n",
       " 'graphName': 'themes',\n",
       " 'query': '\\nMATCH (t:Groupable)\\nRETURN gds.graph.project(******, \\nt, ******,\\n{sourceNodeProperties: t{.embedding},\\ntargetNodeProperties: ******}\\n)',\n",
       " 'projectMillis': 4917,\n",
       " 'configuration': {'readConcurrency': 4,\n",
       "  'undirectedRelationshipTypes': [],\n",
       "  'jobId': '75218b37-abf9-41fd-a6c2-567806a51835',\n",
       "  'logProgress': True,\n",
       "  'query': '\\nMATCH (t:Groupable)\\nRETURN gds.graph.project(******, \\nt, ******,\\n{sourceNodeProperties: t{.embedding},\\ntargetNodeProperties: ******}\\n)',\n",
       "  'inverseIndexedRelationshipTypes': [],\n",
       "  'creationTime': neo4j.time.DateTime(2024, 3, 25, 14, 2, 56, 569765655, tzinfo=<UTC>)},\n",
       " 'nodeCount': 38513}"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b74b34f4-5ffe-48f0-ae79-ff2074aabee3",
   "metadata": {},
   "source": [
    "# Test some theme pairs to find a score above which you think themes are probably duplicated\n",
    "Get a sense of the range of similarity values for top 5 nearest neighbors.\n",
    "\n",
    "Note that KNN returns cosine similarity normalized to the range [0, 1]. \n",
    "\n",
    "If you run gds.similarity.cosine() you get similarity in the range [-1,1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "eaf419ed-3562-4a3d-8f03-4f172f905c92",
   "metadata": {},
   "outputs": [],
   "source": [
    "knn_stats = gds.knn.stats(g_themes, topK=3, nodeProperties = \"embedding\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "afe4bf87-6667-47b1-bd53-38c365848f15",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'min': 0.6232261657714844,\n",
       " 'p5': 0.6851768493652344,\n",
       " 'max': 0.9789695739746094,\n",
       " 'p99': 0.9272689819335938,\n",
       " 'p1': 0.6668815612792969,\n",
       " 'p10': 0.6967124938964844,\n",
       " 'p90': 0.8597068786621094,\n",
       " 'p50': 0.7637672424316406,\n",
       " 'p25': 0.722442626953125,\n",
       " 'p75': 0.8146820068359375,\n",
       " 'p95': 0.8844757080078125,\n",
       " 'mean': 0.7717795717981278,\n",
       " 'p100': 0.9789695739746094,\n",
       " 'stdDev': 0.06212606247669753}"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "knn_stats['similarityDistribution']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b081a74d-db41-4d34-8d9b-b7a87da9374d",
   "metadata": {},
   "source": [
    "## Given a theme, check to see how many neighbors are duplicates and the similarity score for the first neighbor you would consider to be a different theme.\n",
    "\n",
    "db.index.vector.queryNodes() returns cosine similarity normalized to the range [0, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "d0277f0c-6600-4442-a7f3-3b739fce03bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "top_k_df = gds.run_cypher(\"\"\"\n",
    "MATCH (t1:Groupable) WHERE t1.description IN $themeList OR ANY(str IN $themeList WHERE str IN t1.descriptions)\n",
    "CALL db.index.vector.queryNodes(\"groupbale_vectors\", $k + 1, t1.embedding) YIELD node, score\n",
    "WITH t1, node AS t2, score\n",
    "WHERE t1 <> t2\n",
    "RETURN coalesce(t1.description, reduce(str=t1.descriptions[0], x in t1.descriptions[1..] | str + \", \" + x)) AS theme1,\n",
    "coalesce(t2.description, reduce(str=t2.descriptions[0], x in t2.descriptions[1..] | str + \", \" + x)) AS theme2,\n",
    "score\n",
    "ORDER BY theme1, score DESC\"\"\",\n",
    "               {\"k\": 20,\n",
    "                \"themeList\": ['friend', 'dance floor', 'fast food', 'Influential artist', 'antics', \n",
    "                              'ants', 'underwater', 'Africa', 'chatgpt', 'coming-of-age']})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "3f502ef5-6272-417f-a698-fd6eb276cc8e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Africa', 'Fast-food burger', 'Influential artist', 'antics',\n",
       "       'ants', 'chat-gpt, chatgpt', 'coming of age, coming-of-age',\n",
       "       'dance floor, dancefloor', 'friend, friends', 'underwater'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_k_df['theme1'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "ecd2bc75-2f94-4505-986e-af4f4dbde970",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>theme1</th>\n",
       "      <th>theme2</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>140</th>\n",
       "      <td>friend, friends</td>\n",
       "      <td>friend group</td>\n",
       "      <td>0.843112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141</th>\n",
       "      <td>friend, friends</td>\n",
       "      <td>friendship, friendships</td>\n",
       "      <td>0.836408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>142</th>\n",
       "      <td>friend, friends</td>\n",
       "      <td>Friendly</td>\n",
       "      <td>0.825685</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143</th>\n",
       "      <td>friend, friends</td>\n",
       "      <td>school friends</td>\n",
       "      <td>0.823544</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144</th>\n",
       "      <td>friend, friends</td>\n",
       "      <td>close friends</td>\n",
       "      <td>0.818371</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>145</th>\n",
       "      <td>friend, friends</td>\n",
       "      <td>partner</td>\n",
       "      <td>0.817516</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146</th>\n",
       "      <td>friend, friends</td>\n",
       "      <td>old friend</td>\n",
       "      <td>0.811989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147</th>\n",
       "      <td>friend, friends</td>\n",
       "      <td>true friend</td>\n",
       "      <td>0.811861</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>148</th>\n",
       "      <td>friend, friends</td>\n",
       "      <td>rich friends</td>\n",
       "      <td>0.809388</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>friend, friends</td>\n",
       "      <td>best friend</td>\n",
       "      <td>0.807411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150</th>\n",
       "      <td>friend, friends</td>\n",
       "      <td>old friends</td>\n",
       "      <td>0.807393</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151</th>\n",
       "      <td>friend, friends</td>\n",
       "      <td>college friends</td>\n",
       "      <td>0.805262</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>152</th>\n",
       "      <td>friend, friends</td>\n",
       "      <td>amigos</td>\n",
       "      <td>0.801439</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153</th>\n",
       "      <td>friend, friends</td>\n",
       "      <td>neighbor, neighbors</td>\n",
       "      <td>0.801292</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154</th>\n",
       "      <td>friend, friends</td>\n",
       "      <td>best friends</td>\n",
       "      <td>0.800639</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155</th>\n",
       "      <td>friend, friends</td>\n",
       "      <td>friend or foe</td>\n",
       "      <td>0.800298</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>156</th>\n",
       "      <td>friend, friends</td>\n",
       "      <td>buddies, buddy</td>\n",
       "      <td>0.798463</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157</th>\n",
       "      <td>friend, friends</td>\n",
       "      <td>friends-turned-foes</td>\n",
       "      <td>0.797568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>158</th>\n",
       "      <td>friend, friends</td>\n",
       "      <td>friends turned lovers</td>\n",
       "      <td>0.796343</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159</th>\n",
       "      <td>friend, friends</td>\n",
       "      <td>sibling, siblings</td>\n",
       "      <td>0.794062</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              theme1                   theme2     score\n",
       "140  friend, friends             friend group  0.843112\n",
       "141  friend, friends  friendship, friendships  0.836408\n",
       "142  friend, friends                 Friendly  0.825685\n",
       "143  friend, friends           school friends  0.823544\n",
       "144  friend, friends            close friends  0.818371\n",
       "145  friend, friends                  partner  0.817516\n",
       "146  friend, friends               old friend  0.811989\n",
       "147  friend, friends              true friend  0.811861\n",
       "148  friend, friends             rich friends  0.809388\n",
       "149  friend, friends              best friend  0.807411\n",
       "150  friend, friends              old friends  0.807393\n",
       "151  friend, friends          college friends  0.805262\n",
       "152  friend, friends                   amigos  0.801439\n",
       "153  friend, friends      neighbor, neighbors  0.801292\n",
       "154  friend, friends             best friends  0.800639\n",
       "155  friend, friends            friend or foe  0.800298\n",
       "156  friend, friends           buddies, buddy  0.798463\n",
       "157  friend, friends      friends-turned-foes  0.797568\n",
       "158  friend, friends    friends turned lovers  0.796343\n",
       "159  friend, friends        sibling, siblings  0.794062"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_k_df[top_k_df['theme1'] == 'friend, friends']\n",
    "# top 2 cutoff 0.825685"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "cd32818d-f47b-4857-a93c-7d20fa9ee877",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>theme1</th>\n",
       "      <th>theme2</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>120</th>\n",
       "      <td>dance floor, dancefloor</td>\n",
       "      <td>dance party</td>\n",
       "      <td>0.871256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121</th>\n",
       "      <td>dance floor, dancefloor</td>\n",
       "      <td>Dance music</td>\n",
       "      <td>0.838750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122</th>\n",
       "      <td>dance floor, dancefloor</td>\n",
       "      <td>dance, dances</td>\n",
       "      <td>0.838403</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123</th>\n",
       "      <td>dance floor, dancefloor</td>\n",
       "      <td>dance time</td>\n",
       "      <td>0.824521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124</th>\n",
       "      <td>dance floor, dancefloor</td>\n",
       "      <td>go-go dancer</td>\n",
       "      <td>0.820104</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125</th>\n",
       "      <td>dance floor, dancefloor</td>\n",
       "      <td>dancing</td>\n",
       "      <td>0.817867</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126</th>\n",
       "      <td>dance floor, dancefloor</td>\n",
       "      <td>dance routines</td>\n",
       "      <td>0.815567</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>127</th>\n",
       "      <td>dance floor, dancefloor</td>\n",
       "      <td>dance class</td>\n",
       "      <td>0.814394</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>128</th>\n",
       "      <td>dance floor, dancefloor</td>\n",
       "      <td>Dance diva</td>\n",
       "      <td>0.814090</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129</th>\n",
       "      <td>dance floor, dancefloor</td>\n",
       "      <td>dance studio</td>\n",
       "      <td>0.813564</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130</th>\n",
       "      <td>dance floor, dancefloor</td>\n",
       "      <td>Step dancing</td>\n",
       "      <td>0.811053</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>131</th>\n",
       "      <td>dance floor, dancefloor</td>\n",
       "      <td>erotic dance</td>\n",
       "      <td>0.807337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>dance floor, dancefloor</td>\n",
       "      <td>school dance</td>\n",
       "      <td>0.806545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>133</th>\n",
       "      <td>dance floor, dancefloor</td>\n",
       "      <td>dancer, dancers</td>\n",
       "      <td>0.806053</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134</th>\n",
       "      <td>dance floor, dancefloor</td>\n",
       "      <td>Grand dance</td>\n",
       "      <td>0.803411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>dance floor, dancefloor</td>\n",
       "      <td>disco</td>\n",
       "      <td>0.801380</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136</th>\n",
       "      <td>dance floor, dancefloor</td>\n",
       "      <td>dance theater</td>\n",
       "      <td>0.798271</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137</th>\n",
       "      <td>dance floor, dancefloor</td>\n",
       "      <td>DJ set</td>\n",
       "      <td>0.796932</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138</th>\n",
       "      <td>dance floor, dancefloor</td>\n",
       "      <td>break-dancing, breakdancing</td>\n",
       "      <td>0.795965</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139</th>\n",
       "      <td>dance floor, dancefloor</td>\n",
       "      <td>Dance Teacher</td>\n",
       "      <td>0.795948</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      theme1                       theme2     score\n",
       "120  dance floor, dancefloor                  dance party  0.871256\n",
       "121  dance floor, dancefloor                  Dance music  0.838750\n",
       "122  dance floor, dancefloor                dance, dances  0.838403\n",
       "123  dance floor, dancefloor                   dance time  0.824521\n",
       "124  dance floor, dancefloor                 go-go dancer  0.820104\n",
       "125  dance floor, dancefloor                      dancing  0.817867\n",
       "126  dance floor, dancefloor               dance routines  0.815567\n",
       "127  dance floor, dancefloor                  dance class  0.814394\n",
       "128  dance floor, dancefloor                   Dance diva  0.814090\n",
       "129  dance floor, dancefloor                 dance studio  0.813564\n",
       "130  dance floor, dancefloor                 Step dancing  0.811053\n",
       "131  dance floor, dancefloor                 erotic dance  0.807337\n",
       "132  dance floor, dancefloor                 school dance  0.806545\n",
       "133  dance floor, dancefloor              dancer, dancers  0.806053\n",
       "134  dance floor, dancefloor                  Grand dance  0.803411\n",
       "135  dance floor, dancefloor                        disco  0.801380\n",
       "136  dance floor, dancefloor                dance theater  0.798271\n",
       "137  dance floor, dancefloor                       DJ set  0.796932\n",
       "138  dance floor, dancefloor  break-dancing, breakdancing  0.795965\n",
       "139  dance floor, dancefloor                Dance Teacher  0.795948"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_k_df[top_k_df['theme1'] == 'dance floor, dancefloor']\n",
    "# top 0 cutoff 0.871256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "b86b6f4d-1db1-4b56-8cbb-8ce08e19d537",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>theme1</th>\n",
       "      <th>theme2</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>140</th>\n",
       "      <td>fast food</td>\n",
       "      <td>fast food restaurant</td>\n",
       "      <td>0.936906</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>141</th>\n",
       "      <td>fast food</td>\n",
       "      <td>Fast-food burger</td>\n",
       "      <td>0.845039</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>142</th>\n",
       "      <td>fast food</td>\n",
       "      <td>street food</td>\n",
       "      <td>0.804882</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>143</th>\n",
       "      <td>fast food</td>\n",
       "      <td>bad food</td>\n",
       "      <td>0.793561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>144</th>\n",
       "      <td>fast food</td>\n",
       "      <td>night food</td>\n",
       "      <td>0.779074</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>145</th>\n",
       "      <td>fast food</td>\n",
       "      <td>family restaurant</td>\n",
       "      <td>0.768764</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>146</th>\n",
       "      <td>fast food</td>\n",
       "      <td>burger, burgers</td>\n",
       "      <td>0.767863</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147</th>\n",
       "      <td>fast food</td>\n",
       "      <td>food service</td>\n",
       "      <td>0.767081</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>148</th>\n",
       "      <td>fast food</td>\n",
       "      <td>KFC</td>\n",
       "      <td>0.765815</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149</th>\n",
       "      <td>fast food</td>\n",
       "      <td>traditional food</td>\n",
       "      <td>0.763055</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>150</th>\n",
       "      <td>fast food</td>\n",
       "      <td>industrial food</td>\n",
       "      <td>0.761488</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>151</th>\n",
       "      <td>fast food</td>\n",
       "      <td>food</td>\n",
       "      <td>0.754798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>152</th>\n",
       "      <td>fast food</td>\n",
       "      <td>comfort food</td>\n",
       "      <td>0.754501</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>153</th>\n",
       "      <td>fast food</td>\n",
       "      <td>food culture</td>\n",
       "      <td>0.750905</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>154</th>\n",
       "      <td>fast food</td>\n",
       "      <td>fast lane</td>\n",
       "      <td>0.750737</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>155</th>\n",
       "      <td>fast food</td>\n",
       "      <td>French fries</td>\n",
       "      <td>0.746978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>156</th>\n",
       "      <td>fast food</td>\n",
       "      <td>snack, snacks</td>\n",
       "      <td>0.744306</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>157</th>\n",
       "      <td>fast food</td>\n",
       "      <td>french fry</td>\n",
       "      <td>0.743901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>158</th>\n",
       "      <td>fast food</td>\n",
       "      <td>family lunch</td>\n",
       "      <td>0.741885</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>159</th>\n",
       "      <td>fast food</td>\n",
       "      <td>Fast cash</td>\n",
       "      <td>0.740687</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        theme1                theme2     score\n",
       "140  fast food  fast food restaurant  0.936906\n",
       "141  fast food      Fast-food burger  0.845039\n",
       "142  fast food           street food  0.804882\n",
       "143  fast food              bad food  0.793561\n",
       "144  fast food            night food  0.779074\n",
       "145  fast food     family restaurant  0.768764\n",
       "146  fast food       burger, burgers  0.767863\n",
       "147  fast food          food service  0.767081\n",
       "148  fast food                   KFC  0.765815\n",
       "149  fast food      traditional food  0.763055\n",
       "150  fast food       industrial food  0.761488\n",
       "151  fast food                  food  0.754798\n",
       "152  fast food          comfort food  0.754501\n",
       "153  fast food          food culture  0.750905\n",
       "154  fast food             fast lane  0.750737\n",
       "155  fast food          French fries  0.746978\n",
       "156  fast food         snack, snacks  0.744306\n",
       "157  fast food            french fry  0.743901\n",
       "158  fast food          family lunch  0.741885\n",
       "159  fast food             Fast cash  0.740687"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_k_df[top_k_df['theme1'] == 'fast food']\n",
    "# top 2 cutoff 0.804882"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "dc2a1e54-b69f-45b7-9d45-1e444c242721",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>theme1</th>\n",
       "      <th>theme2</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>Influential artist</td>\n",
       "      <td>influential figure</td>\n",
       "      <td>0.858566</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>Influential artist</td>\n",
       "      <td>influential man</td>\n",
       "      <td>0.857857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>Influential artist</td>\n",
       "      <td>influential designer</td>\n",
       "      <td>0.839318</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>Influential artist</td>\n",
       "      <td>influential</td>\n",
       "      <td>0.835377</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>Influential artist</td>\n",
       "      <td>Prominent artists</td>\n",
       "      <td>0.832661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>Influential artist</td>\n",
       "      <td>Era-defining artists</td>\n",
       "      <td>0.813614</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>Influential artist</td>\n",
       "      <td>legendary artists</td>\n",
       "      <td>0.810524</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>Influential artist</td>\n",
       "      <td>visionary artist</td>\n",
       "      <td>0.802965</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>Influential artist</td>\n",
       "      <td>Art pioneers</td>\n",
       "      <td>0.791674</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>Influential artist</td>\n",
       "      <td>Music legend</td>\n",
       "      <td>0.787514</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>Influential artist</td>\n",
       "      <td>influencer</td>\n",
       "      <td>0.786168</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>Influential artist</td>\n",
       "      <td>visual artist</td>\n",
       "      <td>0.785451</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>Influential artist</td>\n",
       "      <td>Emerging artists</td>\n",
       "      <td>0.784419</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>Influential artist</td>\n",
       "      <td>Veteran artist</td>\n",
       "      <td>0.781525</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>Influential artist</td>\n",
       "      <td>Black artist</td>\n",
       "      <td>0.781424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>Influential artist</td>\n",
       "      <td>music artists</td>\n",
       "      <td>0.776276</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>Influential artist</td>\n",
       "      <td>international artists</td>\n",
       "      <td>0.775370</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>Influential artist</td>\n",
       "      <td>Female artist</td>\n",
       "      <td>0.774967</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>Influential artist</td>\n",
       "      <td>Iconic entertainer</td>\n",
       "      <td>0.773224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>Influential artist</td>\n",
       "      <td>eccentric artist</td>\n",
       "      <td>0.773069</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                theme1                 theme2     score\n",
       "40  Influential artist     influential figure  0.858566\n",
       "41  Influential artist        influential man  0.857857\n",
       "42  Influential artist   influential designer  0.839318\n",
       "43  Influential artist            influential  0.835377\n",
       "44  Influential artist      Prominent artists  0.832661\n",
       "45  Influential artist   Era-defining artists  0.813614\n",
       "46  Influential artist      legendary artists  0.810524\n",
       "47  Influential artist       visionary artist  0.802965\n",
       "48  Influential artist           Art pioneers  0.791674\n",
       "49  Influential artist           Music legend  0.787514\n",
       "50  Influential artist             influencer  0.786168\n",
       "51  Influential artist          visual artist  0.785451\n",
       "52  Influential artist       Emerging artists  0.784419\n",
       "53  Influential artist         Veteran artist  0.781525\n",
       "54  Influential artist           Black artist  0.781424\n",
       "55  Influential artist          music artists  0.776276\n",
       "56  Influential artist  international artists  0.775370\n",
       "57  Influential artist          Female artist  0.774967\n",
       "58  Influential artist     Iconic entertainer  0.773224\n",
       "59  Influential artist       eccentric artist  0.773069"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_k_df[top_k_df['theme1'] == 'Influential artist']\n",
    "# top 9 cutoff 0.787514"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "cd1bd3f2-3f1c-4563-8fde-5f3f240a5ee2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>theme1</th>\n",
       "      <th>theme2</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>antics</td>\n",
       "      <td>ants</td>\n",
       "      <td>0.854217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>antics</td>\n",
       "      <td>scope</td>\n",
       "      <td>0.829578</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>antics</td>\n",
       "      <td>aliases</td>\n",
       "      <td>0.827421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>antics</td>\n",
       "      <td>iteration</td>\n",
       "      <td>0.823729</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>antics</td>\n",
       "      <td>logic</td>\n",
       "      <td>0.822214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>antics</td>\n",
       "      <td>dream project</td>\n",
       "      <td>0.821021</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>antics</td>\n",
       "      <td>webs</td>\n",
       "      <td>0.819675</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>antics</td>\n",
       "      <td>ambiguous</td>\n",
       "      <td>0.815373</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>antics</td>\n",
       "      <td>genesis</td>\n",
       "      <td>0.813637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>antics</td>\n",
       "      <td>Elsewhere</td>\n",
       "      <td>0.809501</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>antics</td>\n",
       "      <td>reinterpret</td>\n",
       "      <td>0.807180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>antics</td>\n",
       "      <td>scheme, schemes</td>\n",
       "      <td>0.807126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>antics</td>\n",
       "      <td>dream venue</td>\n",
       "      <td>0.806595</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>antics</td>\n",
       "      <td>existence</td>\n",
       "      <td>0.803760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>antics</td>\n",
       "      <td>axe</td>\n",
       "      <td>0.803095</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>antics</td>\n",
       "      <td>Insensitive</td>\n",
       "      <td>0.801557</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>antics</td>\n",
       "      <td>changing</td>\n",
       "      <td>0.801284</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>antics</td>\n",
       "      <td>action, actions</td>\n",
       "      <td>0.800715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>antics</td>\n",
       "      <td>quest</td>\n",
       "      <td>0.800664</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>antics</td>\n",
       "      <td>target</td>\n",
       "      <td>0.799734</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    theme1           theme2     score\n",
       "40  antics             ants  0.854217\n",
       "41  antics            scope  0.829578\n",
       "42  antics          aliases  0.827421\n",
       "43  antics        iteration  0.823729\n",
       "44  antics            logic  0.822214\n",
       "45  antics    dream project  0.821021\n",
       "46  antics             webs  0.819675\n",
       "47  antics        ambiguous  0.815373\n",
       "48  antics          genesis  0.813637\n",
       "49  antics        Elsewhere  0.809501\n",
       "50  antics      reinterpret  0.807180\n",
       "51  antics  scheme, schemes  0.807126\n",
       "52  antics      dream venue  0.806595\n",
       "53  antics        existence  0.803760\n",
       "54  antics              axe  0.803095\n",
       "55  antics      Insensitive  0.801557\n",
       "56  antics         changing  0.801284\n",
       "57  antics  action, actions  0.800715\n",
       "58  antics            quest  0.800664\n",
       "59  antics           target  0.799734"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_k_df[top_k_df['theme1'] == 'antics']\n",
    "# top 0 cutoff 0.854217"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "4b4d6f82-2073-4575-9322-05c04c8a5831",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>theme1</th>\n",
       "      <th>theme2</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>ants</td>\n",
       "      <td>antics</td>\n",
       "      <td>0.854217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>ants</td>\n",
       "      <td>agent, agents</td>\n",
       "      <td>0.836309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>ants</td>\n",
       "      <td>rat, rats</td>\n",
       "      <td>0.814807</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>ants</td>\n",
       "      <td>webs</td>\n",
       "      <td>0.812752</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>ants</td>\n",
       "      <td>ads</td>\n",
       "      <td>0.807464</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>ants</td>\n",
       "      <td>pies</td>\n",
       "      <td>0.805520</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>ants</td>\n",
       "      <td>target</td>\n",
       "      <td>0.804613</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>ants</td>\n",
       "      <td>stocks</td>\n",
       "      <td>0.802675</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>ants</td>\n",
       "      <td>mutations</td>\n",
       "      <td>0.797931</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>ants</td>\n",
       "      <td>name, names</td>\n",
       "      <td>0.795486</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>ants</td>\n",
       "      <td>things</td>\n",
       "      <td>0.794117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>ants</td>\n",
       "      <td>crew</td>\n",
       "      <td>0.791735</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>ants</td>\n",
       "      <td>gun, guns</td>\n",
       "      <td>0.791468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>ants</td>\n",
       "      <td>genesis</td>\n",
       "      <td>0.791265</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>ants</td>\n",
       "      <td>charges</td>\n",
       "      <td>0.789297</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>ants</td>\n",
       "      <td>axe</td>\n",
       "      <td>0.787703</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>ants</td>\n",
       "      <td>aliases</td>\n",
       "      <td>0.786810</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>ants</td>\n",
       "      <td>teen, teens</td>\n",
       "      <td>0.786244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>ants</td>\n",
       "      <td>ease</td>\n",
       "      <td>0.785681</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>ants</td>\n",
       "      <td>contact</td>\n",
       "      <td>0.784929</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   theme1         theme2     score\n",
       "60   ants         antics  0.854217\n",
       "61   ants  agent, agents  0.836309\n",
       "62   ants      rat, rats  0.814807\n",
       "63   ants           webs  0.812752\n",
       "64   ants            ads  0.807464\n",
       "65   ants           pies  0.805520\n",
       "66   ants         target  0.804613\n",
       "67   ants         stocks  0.802675\n",
       "68   ants      mutations  0.797931\n",
       "69   ants    name, names  0.795486\n",
       "70   ants         things  0.794117\n",
       "71   ants           crew  0.791735\n",
       "72   ants      gun, guns  0.791468\n",
       "73   ants        genesis  0.791265\n",
       "74   ants        charges  0.789297\n",
       "75   ants            axe  0.787703\n",
       "76   ants        aliases  0.786810\n",
       "77   ants    teen, teens  0.786244\n",
       "78   ants           ease  0.785681\n",
       "79   ants        contact  0.784929"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_k_df[top_k_df['theme1'] == 'ants']\n",
    "# top 0 cutoff 0.854217"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "3ebffe7a-7b25-4acb-9824-6fbb0a2bc4e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>theme1</th>\n",
       "      <th>theme2</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>180</th>\n",
       "      <td>underwater</td>\n",
       "      <td>undersea</td>\n",
       "      <td>0.946337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>181</th>\n",
       "      <td>underwater</td>\n",
       "      <td>subaquatic</td>\n",
       "      <td>0.905288</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>182</th>\n",
       "      <td>underwater</td>\n",
       "      <td>underwater world</td>\n",
       "      <td>0.896511</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>183</th>\n",
       "      <td>underwater</td>\n",
       "      <td>undersea world</td>\n",
       "      <td>0.881088</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>184</th>\n",
       "      <td>underwater</td>\n",
       "      <td>underwater music</td>\n",
       "      <td>0.880904</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>185</th>\n",
       "      <td>underwater</td>\n",
       "      <td>underwater adventure</td>\n",
       "      <td>0.875346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>186</th>\n",
       "      <td>underwater</td>\n",
       "      <td>underwater exploration</td>\n",
       "      <td>0.873639</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>187</th>\n",
       "      <td>underwater</td>\n",
       "      <td>deep sea</td>\n",
       "      <td>0.857809</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>188</th>\n",
       "      <td>underwater</td>\n",
       "      <td>underwater ballet</td>\n",
       "      <td>0.847281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>189</th>\n",
       "      <td>underwater</td>\n",
       "      <td>submarine</td>\n",
       "      <td>0.846147</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>190</th>\n",
       "      <td>underwater</td>\n",
       "      <td>Underwater mining</td>\n",
       "      <td>0.836987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>191</th>\n",
       "      <td>underwater</td>\n",
       "      <td>deep water</td>\n",
       "      <td>0.833938</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>192</th>\n",
       "      <td>underwater</td>\n",
       "      <td>Under the Sea</td>\n",
       "      <td>0.827439</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>193</th>\n",
       "      <td>underwater</td>\n",
       "      <td>deep waters</td>\n",
       "      <td>0.825337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>194</th>\n",
       "      <td>underwater</td>\n",
       "      <td>Submersible</td>\n",
       "      <td>0.819398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>195</th>\n",
       "      <td>underwater</td>\n",
       "      <td>Submerged world</td>\n",
       "      <td>0.817633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>196</th>\n",
       "      <td>underwater</td>\n",
       "      <td>underground</td>\n",
       "      <td>0.809469</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>197</th>\n",
       "      <td>underwater</td>\n",
       "      <td>subterranean</td>\n",
       "      <td>0.798814</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>198</th>\n",
       "      <td>underwater</td>\n",
       "      <td>submerged village</td>\n",
       "      <td>0.794522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>199</th>\n",
       "      <td>underwater</td>\n",
       "      <td>diving within</td>\n",
       "      <td>0.792122</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         theme1                  theme2     score\n",
       "180  underwater                undersea  0.946337\n",
       "181  underwater              subaquatic  0.905288\n",
       "182  underwater        underwater world  0.896511\n",
       "183  underwater          undersea world  0.881088\n",
       "184  underwater        underwater music  0.880904\n",
       "185  underwater    underwater adventure  0.875346\n",
       "186  underwater  underwater exploration  0.873639\n",
       "187  underwater                deep sea  0.857809\n",
       "188  underwater       underwater ballet  0.847281\n",
       "189  underwater               submarine  0.846147\n",
       "190  underwater       Underwater mining  0.836987\n",
       "191  underwater              deep water  0.833938\n",
       "192  underwater           Under the Sea  0.827439\n",
       "193  underwater             deep waters  0.825337\n",
       "194  underwater             Submersible  0.819398\n",
       "195  underwater         Submerged world  0.817633\n",
       "196  underwater             underground  0.809469\n",
       "197  underwater            subterranean  0.798814\n",
       "198  underwater       submerged village  0.794522\n",
       "199  underwater           diving within  0.792122"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_k_df[top_k_df['theme1'] == 'underwater']\n",
    "# top 4 cutoff 0.880904"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "6f5cfb89-ad92-4373-97ae-54ee0ebe432b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>theme1</th>\n",
       "      <th>theme2</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Africa</td>\n",
       "      <td>Asia</td>\n",
       "      <td>0.829958</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Africa</td>\n",
       "      <td>Europe</td>\n",
       "      <td>0.793405</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Africa</td>\n",
       "      <td>African</td>\n",
       "      <td>0.792190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Africa</td>\n",
       "      <td>america, americas</td>\n",
       "      <td>0.780679</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Africa</td>\n",
       "      <td>Atlantic</td>\n",
       "      <td>0.775041</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Africa</td>\n",
       "      <td>Asian</td>\n",
       "      <td>0.750807</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Africa</td>\n",
       "      <td>African Union</td>\n",
       "      <td>0.750721</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Africa</td>\n",
       "      <td>Pan-African</td>\n",
       "      <td>0.749782</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Africa</td>\n",
       "      <td>African nation</td>\n",
       "      <td>0.748957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Africa</td>\n",
       "      <td>Horn of Africa</td>\n",
       "      <td>0.748596</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Africa</td>\n",
       "      <td>Trans-Saharan</td>\n",
       "      <td>0.740613</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Africa</td>\n",
       "      <td>Australia</td>\n",
       "      <td>0.739899</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Africa</td>\n",
       "      <td>authentic Africa</td>\n",
       "      <td>0.737924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Africa</td>\n",
       "      <td>Egypt</td>\n",
       "      <td>0.737744</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Africa</td>\n",
       "      <td>Arabia</td>\n",
       "      <td>0.734646</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Africa</td>\n",
       "      <td>Argentina</td>\n",
       "      <td>0.732920</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Africa</td>\n",
       "      <td>African coasts</td>\n",
       "      <td>0.732280</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Africa</td>\n",
       "      <td>European</td>\n",
       "      <td>0.732225</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Africa</td>\n",
       "      <td>Alexander</td>\n",
       "      <td>0.729888</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Africa</td>\n",
       "      <td>East Asia</td>\n",
       "      <td>0.723421</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    theme1             theme2     score\n",
       "0   Africa               Asia  0.829958\n",
       "1   Africa             Europe  0.793405\n",
       "2   Africa            African  0.792190\n",
       "3   Africa  america, americas  0.780679\n",
       "4   Africa           Atlantic  0.775041\n",
       "5   Africa              Asian  0.750807\n",
       "6   Africa      African Union  0.750721\n",
       "7   Africa        Pan-African  0.749782\n",
       "8   Africa     African nation  0.748957\n",
       "9   Africa     Horn of Africa  0.748596\n",
       "10  Africa      Trans-Saharan  0.740613\n",
       "11  Africa          Australia  0.739899\n",
       "12  Africa   authentic Africa  0.737924\n",
       "13  Africa              Egypt  0.737744\n",
       "14  Africa             Arabia  0.734646\n",
       "15  Africa          Argentina  0.732920\n",
       "16  Africa     African coasts  0.732280\n",
       "17  Africa           European  0.732225\n",
       "18  Africa          Alexander  0.729888\n",
       "19  Africa          East Asia  0.723421"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_k_df[top_k_df['theme1'] == 'Africa']\n",
    "# top 0 cutoff 0.829958"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "9bd95674-4c93-48eb-815c-82a4e169530e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>theme1</th>\n",
       "      <th>theme2</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>chat-gpt, chatgpt</td>\n",
       "      <td>GPT-3</td>\n",
       "      <td>0.820860</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>chat-gpt, chatgpt</td>\n",
       "      <td>Hangouts</td>\n",
       "      <td>0.725718</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>chat-gpt, chatgpt</td>\n",
       "      <td>G</td>\n",
       "      <td>0.721377</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>chat-gpt, chatgpt</td>\n",
       "      <td>CG</td>\n",
       "      <td>0.718669</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>chat-gpt, chatgpt</td>\n",
       "      <td>GeGeGe</td>\n",
       "      <td>0.713018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>chat-gpt, chatgpt</td>\n",
       "      <td>game show, gameshow</td>\n",
       "      <td>0.711933</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>chat-gpt, chatgpt</td>\n",
       "      <td>CGI</td>\n",
       "      <td>0.711643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>chat-gpt, chatgpt</td>\n",
       "      <td>competitive gaming</td>\n",
       "      <td>0.709213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>chat-gpt, chatgpt</td>\n",
       "      <td>Online Game</td>\n",
       "      <td>0.708129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>chat-gpt, chatgpt</td>\n",
       "      <td>GUI</td>\n",
       "      <td>0.706613</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>chat-gpt, chatgpt</td>\n",
       "      <td>GDR</td>\n",
       "      <td>0.706016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>chat-gpt, chatgpt</td>\n",
       "      <td>Gotchard</td>\n",
       "      <td>0.703900</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>chat-gpt, chatgpt</td>\n",
       "      <td>CGI gator</td>\n",
       "      <td>0.703468</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>chat-gpt, chatgpt</td>\n",
       "      <td>game, games</td>\n",
       "      <td>0.702814</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>chat-gpt, chatgpt</td>\n",
       "      <td>Classified gadget</td>\n",
       "      <td>0.702032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>chat-gpt, chatgpt</td>\n",
       "      <td>SMGP</td>\n",
       "      <td>0.701762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>chat-gpt, chatgpt</td>\n",
       "      <td>gamepads</td>\n",
       "      <td>0.701543</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>chat-gpt, chatgpt</td>\n",
       "      <td>genesis</td>\n",
       "      <td>0.701017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>chat-gpt, chatgpt</td>\n",
       "      <td>group</td>\n",
       "      <td>0.696931</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>chat-gpt, chatgpt</td>\n",
       "      <td>Global Probing</td>\n",
       "      <td>0.694316</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               theme1               theme2     score\n",
       "80  chat-gpt, chatgpt                GPT-3  0.820860\n",
       "81  chat-gpt, chatgpt             Hangouts  0.725718\n",
       "82  chat-gpt, chatgpt                    G  0.721377\n",
       "83  chat-gpt, chatgpt                   CG  0.718669\n",
       "84  chat-gpt, chatgpt               GeGeGe  0.713018\n",
       "85  chat-gpt, chatgpt  game show, gameshow  0.711933\n",
       "86  chat-gpt, chatgpt                  CGI  0.711643\n",
       "87  chat-gpt, chatgpt   competitive gaming  0.709213\n",
       "88  chat-gpt, chatgpt          Online Game  0.708129\n",
       "89  chat-gpt, chatgpt                  GUI  0.706613\n",
       "90  chat-gpt, chatgpt                  GDR  0.706016\n",
       "91  chat-gpt, chatgpt             Gotchard  0.703900\n",
       "92  chat-gpt, chatgpt            CGI gator  0.703468\n",
       "93  chat-gpt, chatgpt          game, games  0.702814\n",
       "94  chat-gpt, chatgpt    Classified gadget  0.702032\n",
       "95  chat-gpt, chatgpt                 SMGP  0.701762\n",
       "96  chat-gpt, chatgpt             gamepads  0.701543\n",
       "97  chat-gpt, chatgpt              genesis  0.701017\n",
       "98  chat-gpt, chatgpt                group  0.696931\n",
       "99  chat-gpt, chatgpt       Global Probing  0.694316"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_k_df[top_k_df['theme1'] == 'chat-gpt, chatgpt']\n",
    "# top 1 cutoff 0.725718"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "1fcb5e63-e208-4f0d-b58f-7536602635d7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>theme1</th>\n",
       "      <th>theme2</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>coming of age, coming-of-age</td>\n",
       "      <td>growing up, growing-up</td>\n",
       "      <td>0.826383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>coming of age, coming-of-age</td>\n",
       "      <td>adolescence</td>\n",
       "      <td>0.815145</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>coming of age, coming-of-age</td>\n",
       "      <td>puberty</td>\n",
       "      <td>0.797296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>coming of age, coming-of-age</td>\n",
       "      <td>boyhood</td>\n",
       "      <td>0.792982</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>coming of age, coming-of-age</td>\n",
       "      <td>post-adolescence</td>\n",
       "      <td>0.786747</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>coming of age, coming-of-age</td>\n",
       "      <td>lost youth</td>\n",
       "      <td>0.770889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>coming of age, coming-of-age</td>\n",
       "      <td>youthful adventure</td>\n",
       "      <td>0.768444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>coming of age, coming-of-age</td>\n",
       "      <td>Transition to adulthood</td>\n",
       "      <td>0.763974</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108</th>\n",
       "      <td>coming of age, coming-of-age</td>\n",
       "      <td>teenage angst</td>\n",
       "      <td>0.762358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>coming of age, coming-of-age</td>\n",
       "      <td>youth romance</td>\n",
       "      <td>0.759862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>110</th>\n",
       "      <td>coming of age, coming-of-age</td>\n",
       "      <td>broken adolescence</td>\n",
       "      <td>0.759854</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>coming of age, coming-of-age</td>\n",
       "      <td>teenage protagonists</td>\n",
       "      <td>0.757852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>112</th>\n",
       "      <td>coming of age, coming-of-age</td>\n",
       "      <td>Teenage adventure</td>\n",
       "      <td>0.757472</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113</th>\n",
       "      <td>coming of age, coming-of-age</td>\n",
       "      <td>young love</td>\n",
       "      <td>0.756827</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114</th>\n",
       "      <td>coming of age, coming-of-age</td>\n",
       "      <td>young protagonist</td>\n",
       "      <td>0.756616</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115</th>\n",
       "      <td>coming of age, coming-of-age</td>\n",
       "      <td>teenage dramas</td>\n",
       "      <td>0.755339</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116</th>\n",
       "      <td>coming of age, coming-of-age</td>\n",
       "      <td>youthful romance</td>\n",
       "      <td>0.755263</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>coming of age, coming-of-age</td>\n",
       "      <td>adulthood</td>\n",
       "      <td>0.752627</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118</th>\n",
       "      <td>coming of age, coming-of-age</td>\n",
       "      <td>Teen's journey</td>\n",
       "      <td>0.749123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119</th>\n",
       "      <td>coming of age, coming-of-age</td>\n",
       "      <td>new adulthood</td>\n",
       "      <td>0.749057</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           theme1                   theme2     score\n",
       "100  coming of age, coming-of-age   growing up, growing-up  0.826383\n",
       "101  coming of age, coming-of-age              adolescence  0.815145\n",
       "102  coming of age, coming-of-age                  puberty  0.797296\n",
       "103  coming of age, coming-of-age                  boyhood  0.792982\n",
       "104  coming of age, coming-of-age         post-adolescence  0.786747\n",
       "105  coming of age, coming-of-age               lost youth  0.770889\n",
       "106  coming of age, coming-of-age       youthful adventure  0.768444\n",
       "107  coming of age, coming-of-age  Transition to adulthood  0.763974\n",
       "108  coming of age, coming-of-age            teenage angst  0.762358\n",
       "109  coming of age, coming-of-age            youth romance  0.759862\n",
       "110  coming of age, coming-of-age       broken adolescence  0.759854\n",
       "111  coming of age, coming-of-age     teenage protagonists  0.757852\n",
       "112  coming of age, coming-of-age        Teenage adventure  0.757472\n",
       "113  coming of age, coming-of-age               young love  0.756827\n",
       "114  coming of age, coming-of-age        young protagonist  0.756616\n",
       "115  coming of age, coming-of-age           teenage dramas  0.755339\n",
       "116  coming of age, coming-of-age         youthful romance  0.755263\n",
       "117  coming of age, coming-of-age                adulthood  0.752627\n",
       "118  coming of age, coming-of-age           Teen's journey  0.749123\n",
       "119  coming of age, coming-of-age            new adulthood  0.749057"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_k_df[top_k_df['theme1'] == 'coming of age, coming-of-age']\n",
    "# top 3 cutoff 0.792982"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "76b051c8-e708-41c0-bedd-56514aa0f61a",
   "metadata": {},
   "source": [
    "# Based on the exploration above run KNN with a similarityCutoff you choose"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "d7953487-b712-4967-9f7d-6dc095daf02c",
   "metadata": {},
   "outputs": [],
   "source": [
    "similarity_cutoff = 0.83\n",
    "\n",
    "knn_result = gds.knn.mutate(g_themes,\n",
    "                            mutateRelationshipType = \"IS_SIMILAR\",\n",
    "                            mutateProperty = \"similarity\",\n",
    "                            nodeProperties = \"embedding\",\n",
    "                            topK = 2,\n",
    "                            sampleRate = 1.0,\n",
    "                            perturbationRate = 0.5,\n",
    "                            deltaThreshold = 0.00001,\n",
    "                            similarityCutoff = similarity_cutoff)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e944a057-f540-4817-acfb-8e8b1bfcb900",
   "metadata": {},
   "source": [
    "## Find themes that discovered no neighbors above the similarity threshold you selected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "939d4e66-5ab9-4de9-9c37-ee14f2c15923",
   "metadata": {},
   "outputs": [],
   "source": [
    "no_sim_df = gds.run_cypher(\"\"\"\n",
    "CALL gds.graph.relationships.stream(\"themes\", [\"IS_SIMILAR\"]) yield sourceNodeId, targetNodeId\n",
    "UNWIND [gds.util.asNode(sourceNodeId), gds.util.asNode(targetNodeId)] AS nodeWithSim\n",
    "WITH collect(DISTINCT nodeWithSim) AS nodesWithSim\n",
    "MATCH (t:Theme) WHERE NOT t IN nodesWithSim\n",
    "AND NOT EXISTS {(t)-[:HAS_STEM]->()}\n",
    "RETURN t.description AS description, count{ (t)<-[:HAS_THEME]-() } AS movieCount\n",
    "ORDER BY movieCount desc\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "2b7f8294-647d-4909-9aa2-c34197c82efa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>description</th>\n",
       "      <th>movieCount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>revenge</td>\n",
       "      <td>175</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>nature</td>\n",
       "      <td>148</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>art</td>\n",
       "      <td>79</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>love story</td>\n",
       "      <td>78</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>forbidden love</td>\n",
       "      <td>74</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28178</th>\n",
       "      <td>viagem</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28179</th>\n",
       "      <td>vibrant plea</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28180</th>\n",
       "      <td>vibraphonist</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28181</th>\n",
       "      <td>vicar scandal</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28182</th>\n",
       "      <td>vicarious thrills</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>28183 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             description  movieCount\n",
       "0                revenge         175\n",
       "1                 nature         148\n",
       "2                    art          79\n",
       "3             love story          78\n",
       "4         forbidden love          74\n",
       "...                  ...         ...\n",
       "28178             viagem           1\n",
       "28179       vibrant plea           1\n",
       "28180       vibraphonist           1\n",
       "28181      vicar scandal           1\n",
       "28182  vicarious thrills           1\n",
       "\n",
       "[28183 rows x 2 columns]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "no_sim_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "670d7f38-5ae4-4f4f-9be6-7cc926c7041b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4725"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "no_sim_df[no_sim_df['movieCount'] > 1]['description'].count()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4e2bbb54-2915-4f47-aed7-e86be293ddbf",
   "metadata": {},
   "source": [
    "## Examine theme pairs for high degree nodes\n",
    "For Stem nodes, turn their list of descriptions into a comma separated string before exporting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "68b99874-dc0b-4662-bbef-db334a729c9f",
   "metadata": {},
   "outputs": [],
   "source": [
    "similarity_result = gds.run_cypher(\"\"\"\n",
    "    CALL gds.graph.relationshipProperty.stream(\"themes\", \"similarity\", [\"IS_SIMILAR\"])\n",
    "    YIELD sourceNodeId, targetNodeId, propertyValue AS similarity\n",
    "    WITH gds.util.asNode(sourceNodeId) AS source, gds.util.asNode(targetNodeId) AS target, similarity\n",
    "    RETURN coalesce(source.description, reduce(s = source.descriptions[0], d in source.descriptions[1..] | s + \", \" + d)) AS source, \n",
    "    coalesce(target.description, reduce(s = target.descriptions[0], d in target.descriptions[1..] | s + \", \" + d)) AS target, similarity\n",
    "    ORDER BY similarity\"\"\")\n",
    "                    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e2d14be6-f586-4d4c-a6d8-1ba2063081d2",
   "metadata": {},
   "source": [
    "What are the nodes with the most inbound IS_SIMILAR relationships?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "a8b07bae-cd12-4413-8291-221be91c2dad",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "target\n",
       "Christmas magic          16\n",
       "Christmas spirit         14\n",
       "family relationships     13\n",
       "haunted mansion          12\n",
       "childhood                12\n",
       "marriage struggles       11\n",
       "surreal                  11\n",
       "aging queer              11\n",
       "Indigenous cultures      11\n",
       "filmmaking               11\n",
       "Wild journey             10\n",
       "missing mother           10\n",
       "elderly woman            10\n",
       "relationship troubles    10\n",
       "mother, mothers          10\n",
       "future world             10\n",
       "Gay professional         10\n",
       "crime gangs              10\n",
       "queer                    10\n",
       "Art space                10\n",
       "Name: source, dtype: int64"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "similarity_result.groupby(\"target\")['source'].count().sort_values(ascending=False).head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "e1c40d9a-31fa-4225-a4f8-c489cd212d41",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>target</th>\n",
       "      <th>similarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6126</th>\n",
       "      <td>Haunted train</td>\n",
       "      <td>haunted house</td>\n",
       "      <td>0.856283</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8272</th>\n",
       "      <td>haunted book</td>\n",
       "      <td>haunted house</td>\n",
       "      <td>0.869158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11382</th>\n",
       "      <td>haunted museum</td>\n",
       "      <td>haunted house</td>\n",
       "      <td>0.895871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11449</th>\n",
       "      <td>haunted hospital</td>\n",
       "      <td>haunted house</td>\n",
       "      <td>0.896431</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11925</th>\n",
       "      <td>haunted villa</td>\n",
       "      <td>haunted house</td>\n",
       "      <td>0.903460</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13448</th>\n",
       "      <td>haunted mansion</td>\n",
       "      <td>haunted house</td>\n",
       "      <td>0.944185</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 source         target  similarity\n",
       "6126      Haunted train  haunted house    0.856283\n",
       "8272       haunted book  haunted house    0.869158\n",
       "11382    haunted museum  haunted house    0.895871\n",
       "11449  haunted hospital  haunted house    0.896431\n",
       "11925     haunted villa  haunted house    0.903460\n",
       "13448   haunted mansion  haunted house    0.944185"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "similarity_result[similarity_result['target']==\"haunted house\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "cbc480e4-8c4b-4fcb-861d-3ddd3704e1e9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>target</th>\n",
       "      <th>similarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2472</th>\n",
       "      <td>hidden</td>\n",
       "      <td>hidden secrets</td>\n",
       "      <td>0.839962</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5796</th>\n",
       "      <td>shocking secrets</td>\n",
       "      <td>hidden secrets</td>\n",
       "      <td>0.854482</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7633</th>\n",
       "      <td>unsettling secrets</td>\n",
       "      <td>hidden secrets</td>\n",
       "      <td>0.865214</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7940</th>\n",
       "      <td>hidden feelings</td>\n",
       "      <td>hidden secrets</td>\n",
       "      <td>0.866977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8262</th>\n",
       "      <td>secrets exposed</td>\n",
       "      <td>hidden secrets</td>\n",
       "      <td>0.869080</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9915</th>\n",
       "      <td>forbidden secrets</td>\n",
       "      <td>hidden secrets</td>\n",
       "      <td>0.881313</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11526</th>\n",
       "      <td>past secrets</td>\n",
       "      <td>hidden secrets</td>\n",
       "      <td>0.897386</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11575</th>\n",
       "      <td>Dirty secrets</td>\n",
       "      <td>hidden secrets</td>\n",
       "      <td>0.898116</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12324</th>\n",
       "      <td>dark secrets</td>\n",
       "      <td>hidden secrets</td>\n",
       "      <td>0.909530</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12415</th>\n",
       "      <td>secret, secrets</td>\n",
       "      <td>hidden secrets</td>\n",
       "      <td>0.910885</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   source          target  similarity\n",
       "2472               hidden  hidden secrets    0.839962\n",
       "5796     shocking secrets  hidden secrets    0.854482\n",
       "7633   unsettling secrets  hidden secrets    0.865214\n",
       "7940      hidden feelings  hidden secrets    0.866977\n",
       "8262      secrets exposed  hidden secrets    0.869080\n",
       "9915    forbidden secrets  hidden secrets    0.881313\n",
       "11526        past secrets  hidden secrets    0.897386\n",
       "11575       Dirty secrets  hidden secrets    0.898116\n",
       "12324        dark secrets  hidden secrets    0.909530\n",
       "12415     secret, secrets  hidden secrets    0.910885"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "similarity_result[similarity_result['target']==\"hidden secrets\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "cf9f493b-7693-4dd5-a520-9757a72e3054",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>target</th>\n",
       "      <th>similarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4547</th>\n",
       "      <td>Christmas songs</td>\n",
       "      <td>Christmas magic</td>\n",
       "      <td>0.848337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7137</th>\n",
       "      <td>Christmas story</td>\n",
       "      <td>Christmas magic</td>\n",
       "      <td>0.862150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8239</th>\n",
       "      <td>Christmas competition</td>\n",
       "      <td>Christmas magic</td>\n",
       "      <td>0.868947</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8547</th>\n",
       "      <td>Christmas traditions</td>\n",
       "      <td>Christmas magic</td>\n",
       "      <td>0.871038</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9156</th>\n",
       "      <td>Xmas</td>\n",
       "      <td>Christmas magic</td>\n",
       "      <td>0.875206</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9240</th>\n",
       "      <td>Christmas mystery</td>\n",
       "      <td>Christmas magic</td>\n",
       "      <td>0.875824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9555</th>\n",
       "      <td>Christmas tradition</td>\n",
       "      <td>Christmas magic</td>\n",
       "      <td>0.878228</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9962</th>\n",
       "      <td>christmas</td>\n",
       "      <td>Christmas magic</td>\n",
       "      <td>0.881683</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10449</th>\n",
       "      <td>Christmas ambience</td>\n",
       "      <td>Christmas magic</td>\n",
       "      <td>0.885654</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10761</th>\n",
       "      <td>Christmas tree</td>\n",
       "      <td>Christmas magic</td>\n",
       "      <td>0.888673</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12092</th>\n",
       "      <td>Christmas decorations</td>\n",
       "      <td>Christmas magic</td>\n",
       "      <td>0.906152</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12522</th>\n",
       "      <td>Christmas celebration</td>\n",
       "      <td>Christmas magic</td>\n",
       "      <td>0.912869</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12667</th>\n",
       "      <td>Christmas celebrations</td>\n",
       "      <td>Christmas magic</td>\n",
       "      <td>0.916522</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12955</th>\n",
       "      <td>Christmas terror</td>\n",
       "      <td>Christmas magic</td>\n",
       "      <td>0.924765</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13087</th>\n",
       "      <td>Christmas love</td>\n",
       "      <td>Christmas magic</td>\n",
       "      <td>0.929875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13555</th>\n",
       "      <td>Christmas spirit</td>\n",
       "      <td>Christmas magic</td>\n",
       "      <td>0.952886</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       source           target  similarity\n",
       "4547          Christmas songs  Christmas magic    0.848337\n",
       "7137          Christmas story  Christmas magic    0.862150\n",
       "8239    Christmas competition  Christmas magic    0.868947\n",
       "8547     Christmas traditions  Christmas magic    0.871038\n",
       "9156                     Xmas  Christmas magic    0.875206\n",
       "9240        Christmas mystery  Christmas magic    0.875824\n",
       "9555      Christmas tradition  Christmas magic    0.878228\n",
       "9962                christmas  Christmas magic    0.881683\n",
       "10449      Christmas ambience  Christmas magic    0.885654\n",
       "10761          Christmas tree  Christmas magic    0.888673\n",
       "12092   Christmas decorations  Christmas magic    0.906152\n",
       "12522   Christmas celebration  Christmas magic    0.912869\n",
       "12667  Christmas celebrations  Christmas magic    0.916522\n",
       "12955        Christmas terror  Christmas magic    0.924765\n",
       "13087          Christmas love  Christmas magic    0.929875\n",
       "13555        Christmas spirit  Christmas magic    0.952886"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "similarity_result[similarity_result['target']==\"Christmas magic\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "d825583e-98ca-4a00-8832-fb1c6b9e50e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>target</th>\n",
       "      <th>similarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4243</th>\n",
       "      <td>meta-documentary</td>\n",
       "      <td>documentary video</td>\n",
       "      <td>0.846840</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7911</th>\n",
       "      <td>animated documentary</td>\n",
       "      <td>documentary video</td>\n",
       "      <td>0.866787</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9931</th>\n",
       "      <td>documentary crew</td>\n",
       "      <td>documentary video</td>\n",
       "      <td>0.881450</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11977</th>\n",
       "      <td>documentary interviews</td>\n",
       "      <td>documentary video</td>\n",
       "      <td>0.903989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13339</th>\n",
       "      <td>documentaries, documentary</td>\n",
       "      <td>documentary video</td>\n",
       "      <td>0.938533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13528</th>\n",
       "      <td>documentary film</td>\n",
       "      <td>documentary video</td>\n",
       "      <td>0.950835</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           source             target  similarity\n",
       "4243             meta-documentary  documentary video    0.846840\n",
       "7911         animated documentary  documentary video    0.866787\n",
       "9931             documentary crew  documentary video    0.881450\n",
       "11977      documentary interviews  documentary video    0.903989\n",
       "13339  documentaries, documentary  documentary video    0.938533\n",
       "13528            documentary film  documentary video    0.950835"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "similarity_result[similarity_result['target']==\"documentary video\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "949f8956-fbda-4f4b-aeb7-75e8351208f5",
   "metadata": {},
   "source": [
    "# Write weakly connected components with threshold at your KNN similarityCutoff or a higher value\n",
    "I'm bumping the threshold up to 0.875"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "3c5b7484-e622-418a-bde5-ed969fddae74",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "writeMillis                                                             41\n",
       "nodePropertiesWritten                                                 3605\n",
       "componentCount                                                       36237\n",
       "componentDistribution    {'min': 1, 'p5': 1, 'max': 29, 'p999': 7, 'p99...\n",
       "postProcessingMillis                                                    34\n",
       "preProcessingMillis                                                      0\n",
       "computeMillis                                                           23\n",
       "configuration            {'writeProperty': 'wccId', 'jobId': '9615c40e-...\n",
       "Name: 0, dtype: object"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.wcc.write(g_themes,\n",
    "              writeProperty = \"wccId\",\n",
    "              relationshipTypes = [\"IS_SIMILAR\"],\n",
    "              relationshipWeightProperty = \"similarity\",\n",
    "              threshold = 0.875,\n",
    "              minComponentSize=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a414f13c-794e-43fa-b228-99a4469e097d",
   "metadata": {},
   "source": [
    "## Examine the largest communities at the KNN value you selected"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "b26bf663-cc0d-4483-8a20-2320f850f7a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_colwidth', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "fcba8128-9164-4250-80b8-80771f601943",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>themeCount</th>\n",
       "      <th>themes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1964</td>\n",
       "      <td>29</td>\n",
       "      <td>[Christmas ambience, Christmas break, Christmas celebration, Christmas celebrations, Christmas competition, Christmas decorations, Christmas eve, Christmas gift, Christmas love, Christmas magic, Christmas market, Christmas memories, Christmas movie, Christmas mystery, Christmas nightmare, Christmas party, Christmas setting, Christmas spirit, Christmas terror, Christmas tradition, Christmas traditions, Christmas tree, Christmas wedding, Christmas wish, Christmas wishes, Holiday traditions, Holiday wishes, Xmas, christmas]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>653</td>\n",
       "      <td>25</td>\n",
       "      <td>[Russian Gas, Russian actor, Russian army, Russian cinema, Russian composer, Russian criminals, Russian culture, Russian defiance, Russian director, Russian forces, Russian literature, Russian mission, Russian mother, Russian occupation, Russian oligarch, Russian owner, Russian performers, Russian setting, Russian soldier, Russian spies, Russian students, Russian teacher, Russian team, Russian town, Russian troops]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2094</td>\n",
       "      <td>23</td>\n",
       "      <td>[Gay Agenda, Gay awakening, Gay conversion, Gay couple, Gay culture, Gay elder, Gay history, Gay killer, Gay life, Gay love, Gay professional, Gay relationship, Gay relationships, LGBTQ+ couple, Queer couple, Same-sex love, gay artist, gay identity, gay lord, gay man, gay rights, gay romance, same-sex relationships]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1167</td>\n",
       "      <td>18</td>\n",
       "      <td>[Horrifying nightmare, Surreal dream, Surreal fantasy, Surreal imagery, Surreal landscapes, bad dream, mystical landscape, nightmare begins, subconscious nightmares, surreal, surreal animation, surreal dreams, surreal landscape, surreal nightmare, surreal nightmares, surreal vision, surrealism, surrealist]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1681</td>\n",
       "      <td>17</td>\n",
       "      <td>[Dirty secret, Dirty secrets, Hidden desire, Secret desire, Secret passion, dark secret, dark secrets, dark truth, forbidden desires, forbidden secrets, hidden secrets, past secrets, secret desires, secret knowledge, secret living, secret past, secret relationship]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1495</td>\n",
       "      <td>16</td>\n",
       "      <td>[Countryside cottage, Countryside village, Rural community, Rural lifestyle, country life, countryside, countryside life, rural, rural areas, rural backdrop, rural communities, rural countryside, rural life, rural scenery, rural setting, rural world]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>50</td>\n",
       "      <td>15</td>\n",
       "      <td>[Unexplainable events, Weird events, enigmatic event, mysterious events, mysterious incident, strange, strange behavior, strange encounters, strange events, strange performance, strange phenomena, strange phenomenon, strange places, strange things, weird]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>203</td>\n",
       "      <td>13</td>\n",
       "      <td>[Weird incidents, bizarre incident, eerie encounters, mysterious encounters, serendipitous encounter, strange encounter, strange event, strange occurrences, surreal encounters, unexpected encounter, unexpected encounters, unexplained events, weird incident]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>72</td>\n",
       "      <td>13</td>\n",
       "      <td>[Crime Gang, Criminal family, Criminal group, armed gang, crime families, crime family, crime gangs, crime lords, crime syndicate, criminal empire, criminal gang, criminal groups, street gangs]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1112</td>\n",
       "      <td>12</td>\n",
       "      <td>[Elderly spinster, Elderly widow, elderly, elderly care, elderly gentleman, elderly lady, elderly life, elderly man, elderly mother, elderly woman, old man, older woman]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>364</td>\n",
       "      <td>12</td>\n",
       "      <td>[Deserted, Deserted resort, Desolate city, abandoned spaces, deserted village, desolate house, desolate landscape, desolate spaces, desolate town, desolate village, desolate world, desolation]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1355</td>\n",
       "      <td>12</td>\n",
       "      <td>[Relationship conflicts, Relationship issues, Relationship problems, Relationship tensions, Relationship turmoil, relationship challenges, relationship complications, relationship dilemma, relationship dynamics, relationship strife, relationship struggles, relationship troubles]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>3218</td>\n",
       "      <td>11</td>\n",
       "      <td>[Marital troubles, Marriage challenges, marital problems, marital strain, marital strife, marital struggle, marital struggles, marriage conflict, marriage strain, marriage struggles, marriage troubles]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>2748</td>\n",
       "      <td>11</td>\n",
       "      <td>[ailing mother, deceased mother, grieving mother, loss of mother, lost mother, missing mom, missing mother, mother's absence, mother's death, mother's loss, mother's passing]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>8107</td>\n",
       "      <td>11</td>\n",
       "      <td>['80s style, 1960s-1970s, 1970s, 1970s Britain, 1970s counterculture, 1970s era, 70s, 70s TV, 80s, 80s TV, seventies]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>1969</td>\n",
       "      <td>11</td>\n",
       "      <td>[Filmmaker persona, film photography, film production, filming, filmmaker relationship, filmmaker's journey, filmmaker's perspective, filmmaking, filmmaking aspirations, filmmaking dreams, filmmaking process]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>448</td>\n",
       "      <td>11</td>\n",
       "      <td>[Queer subculture, queer activism, queer community, queer culture, queer experience, queer identity, queer media, queer struggle, queer woman, queer women, queercore]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>26</td>\n",
       "      <td>11</td>\n",
       "      <td>[Demonic curse, Demonic evil, Demonic figure, Demonic presence, Demonic spirit, Supernatural possession, demonic, demonic darkness, demonic entity, demonic force, demonic possession]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>4579</td>\n",
       "      <td>10</td>\n",
       "      <td>[Company trip, Work travel, business trip, farewell trip, field trip, flight trip, foreign trip, friends trip, long trip, work trip]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>1532</td>\n",
       "      <td>10</td>\n",
       "      <td>[family conflict, family conflicts, family dysfunction, family issues, family problems, family struggle, family struggles, family suffering, family tension, family turmoil]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>5720</td>\n",
       "      <td>9</td>\n",
       "      <td>[Abandoned Mansions, Abandoned camp, Abandoned home, Abandoned town, abandoned campsite, abandoned homes, abandoned house, abandoned housing, abandoned place]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>2146</td>\n",
       "      <td>9</td>\n",
       "      <td>[Brotherly bonds, brotherly love, sibling bond, sibling bonds, sibling relationship, siblinghood, sisterhood bonds, sisterly bond, stepsister]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>4549</td>\n",
       "      <td>9</td>\n",
       "      <td>[Remote retreat, Secluded Village, Secluded farm, Secluded resort, Secluded retreat, isolated farm, secluded, secluded cabin, secluded house]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>[Isolated villages, dry village, isolated town, isolated village, remote village, rural farmhouse, rural town, rural village]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>7507</td>\n",
       "      <td>8</td>\n",
       "      <td>[Creepy Mansion, haunted guesthouse, haunted hospital, haunted house, haunted mansion, haunted museum, haunted town, haunted villa]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>1481</td>\n",
       "      <td>8</td>\n",
       "      <td>[Experimental films, art film, artist film, experimental cinema, experimental film, experimental filmmaker, experimental filmmakers, unfinished film]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>6217</td>\n",
       "      <td>8</td>\n",
       "      <td>[dark forces, evil forces, evil presence, evil spirit, malevolent figure, malevolent presence, malevolent spirit, sinister forces]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>10778</td>\n",
       "      <td>8</td>\n",
       "      <td>[digital age, digital environment, digital era, digital existence, digital frontier, digital revolution, digital technologies, digital world]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>487</td>\n",
       "      <td>8</td>\n",
       "      <td>[Dangerous adventures, Hazardous journey, Perilous adventures, dangerous adventure, dangerous journey, murderous journey, perilous journey, treacherous journey]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>1493</td>\n",
       "      <td>8</td>\n",
       "      <td>[aging queer, genderqueer, queer, queer art, queer artist, queer girl, queer life, queer youth]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       id  themeCount  \\\n",
       "0    1964          29   \n",
       "1     653          25   \n",
       "2    2094          23   \n",
       "3    1167          18   \n",
       "4    1681          17   \n",
       "5    1495          16   \n",
       "6      50          15   \n",
       "7     203          13   \n",
       "8      72          13   \n",
       "9    1112          12   \n",
       "10    364          12   \n",
       "11   1355          12   \n",
       "12   3218          11   \n",
       "13   2748          11   \n",
       "14   8107          11   \n",
       "15   1969          11   \n",
       "16    448          11   \n",
       "17     26          11   \n",
       "18   4579          10   \n",
       "19   1532          10   \n",
       "20   5720           9   \n",
       "21   2146           9   \n",
       "22   4549           9   \n",
       "23      1           8   \n",
       "24   7507           8   \n",
       "25   1481           8   \n",
       "26   6217           8   \n",
       "27  10778           8   \n",
       "28    487           8   \n",
       "29   1493           8   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            themes  \n",
       "0   [Christmas ambience, Christmas break, Christmas celebration, Christmas celebrations, Christmas competition, Christmas decorations, Christmas eve, Christmas gift, Christmas love, Christmas magic, Christmas market, Christmas memories, Christmas movie, Christmas mystery, Christmas nightmare, Christmas party, Christmas setting, Christmas spirit, Christmas terror, Christmas tradition, Christmas traditions, Christmas tree, Christmas wedding, Christmas wish, Christmas wishes, Holiday traditions, Holiday wishes, Xmas, christmas]  \n",
       "1                                                                                                               [Russian Gas, Russian actor, Russian army, Russian cinema, Russian composer, Russian criminals, Russian culture, Russian defiance, Russian director, Russian forces, Russian literature, Russian mission, Russian mother, Russian occupation, Russian oligarch, Russian owner, Russian performers, Russian setting, Russian soldier, Russian spies, Russian students, Russian teacher, Russian team, Russian town, Russian troops]  \n",
       "2                                                                                                                                                                                                                    [Gay Agenda, Gay awakening, Gay conversion, Gay couple, Gay culture, Gay elder, Gay history, Gay killer, Gay life, Gay love, Gay professional, Gay relationship, Gay relationships, LGBTQ+ couple, Queer couple, Same-sex love, gay artist, gay identity, gay lord, gay man, gay rights, gay romance, same-sex relationships]  \n",
       "3                                                                                                                                                                                                                              [Horrifying nightmare, Surreal dream, Surreal fantasy, Surreal imagery, Surreal landscapes, bad dream, mystical landscape, nightmare begins, subconscious nightmares, surreal, surreal animation, surreal dreams, surreal landscape, surreal nightmare, surreal nightmares, surreal vision, surrealism, surrealist]  \n",
       "4                                                                                                                                                                                                                                                                        [Dirty secret, Dirty secrets, Hidden desire, Secret desire, Secret passion, dark secret, dark secrets, dark truth, forbidden desires, forbidden secrets, hidden secrets, past secrets, secret desires, secret knowledge, secret living, secret past, secret relationship]  \n",
       "5                                                                                                                                                                                                                                                                                       [Countryside cottage, Countryside village, Rural community, Rural lifestyle, country life, countryside, countryside life, rural, rural areas, rural backdrop, rural communities, rural countryside, rural life, rural scenery, rural setting, rural world]  \n",
       "6                                                                                                                                                                                                                                                                                  [Unexplainable events, Weird events, enigmatic event, mysterious events, mysterious incident, strange, strange behavior, strange encounters, strange events, strange performance, strange phenomena, strange phenomenon, strange places, strange things, weird]  \n",
       "7                                                                                                                                                                                                                                                                                [Weird incidents, bizarre incident, eerie encounters, mysterious encounters, serendipitous encounter, strange encounter, strange event, strange occurrences, surreal encounters, unexpected encounter, unexpected encounters, unexplained events, weird incident]  \n",
       "8                                                                                                                                                                                                                                                                                                                                                [Crime Gang, Criminal family, Criminal group, armed gang, crime families, crime family, crime gangs, crime lords, crime syndicate, criminal empire, criminal gang, criminal groups, street gangs]  \n",
       "9                                                                                                                                                                                                                                                                                                                                                                        [Elderly spinster, Elderly widow, elderly, elderly care, elderly gentleman, elderly lady, elderly life, elderly man, elderly mother, elderly woman, old man, older woman]  \n",
       "10                                                                                                                                                                                                                                                                                                                                                [Deserted, Deserted resort, Desolate city, abandoned spaces, deserted village, desolate house, desolate landscape, desolate spaces, desolate town, desolate village, desolate world, desolation]  \n",
       "11                                                                                                                                                                                                                                                         [Relationship conflicts, Relationship issues, Relationship problems, Relationship tensions, Relationship turmoil, relationship challenges, relationship complications, relationship dilemma, relationship dynamics, relationship strife, relationship struggles, relationship troubles]  \n",
       "12                                                                                                                                                                                                                                                                                                                                       [Marital troubles, Marriage challenges, marital problems, marital strain, marital strife, marital struggle, marital struggles, marriage conflict, marriage strain, marriage struggles, marriage troubles]  \n",
       "13                                                                                                                                                                                                                                                                                                                                                                  [ailing mother, deceased mother, grieving mother, loss of mother, lost mother, missing mom, missing mother, mother's absence, mother's death, mother's loss, mother's passing]  \n",
       "14                                                                                                                                                                                                                                                                                                                                                                                                                           ['80s style, 1960s-1970s, 1970s, 1970s Britain, 1970s counterculture, 1970s era, 70s, 70s TV, 80s, 80s TV, seventies]  \n",
       "15                                                                                                                                                                                                                                                                                                                                [Filmmaker persona, film photography, film production, filming, filmmaker relationship, filmmaker's journey, filmmaker's perspective, filmmaking, filmmaking aspirations, filmmaking dreams, filmmaking process]  \n",
       "16                                                                                                                                                                                                                                                                                                                                                                          [Queer subculture, queer activism, queer community, queer culture, queer experience, queer identity, queer media, queer struggle, queer woman, queer women, queercore]  \n",
       "17                                                                                                                                                                                                                                                                                                                                                          [Demonic curse, Demonic evil, Demonic figure, Demonic presence, Demonic spirit, Supernatural possession, demonic, demonic darkness, demonic entity, demonic force, demonic possession]  \n",
       "18                                                                                                                                                                                                                                                                                                                                                                                                            [Company trip, Work travel, business trip, farewell trip, field trip, flight trip, foreign trip, friends trip, long trip, work trip]  \n",
       "19                                                                                                                                                                                                                                                                                                                                                                    [family conflict, family conflicts, family dysfunction, family issues, family problems, family struggle, family struggles, family suffering, family tension, family turmoil]  \n",
       "20                                                                                                                                                                                                                                                                                                                                                                                  [Abandoned Mansions, Abandoned camp, Abandoned home, Abandoned town, abandoned campsite, abandoned homes, abandoned house, abandoned housing, abandoned place]  \n",
       "21                                                                                                                                                                                                                                                                                                                                                                                                  [Brotherly bonds, brotherly love, sibling bond, sibling bonds, sibling relationship, siblinghood, sisterhood bonds, sisterly bond, stepsister]  \n",
       "22                                                                                                                                                                                                                                                                                                                                                                                                   [Remote retreat, Secluded Village, Secluded farm, Secluded resort, Secluded retreat, isolated farm, secluded, secluded cabin, secluded house]  \n",
       "23                                                                                                                                                                                                                                                                                                                                                                                                                   [Isolated villages, dry village, isolated town, isolated village, remote village, rural farmhouse, rural town, rural village]  \n",
       "24                                                                                                                                                                                                                                                                                                                                                                                                             [Creepy Mansion, haunted guesthouse, haunted hospital, haunted house, haunted mansion, haunted museum, haunted town, haunted villa]  \n",
       "25                                                                                                                                                                                                                                                                                                                                                                                           [Experimental films, art film, artist film, experimental cinema, experimental film, experimental filmmaker, experimental filmmakers, unfinished film]  \n",
       "26                                                                                                                                                                                                                                                                                                                                                                                                              [dark forces, evil forces, evil presence, evil spirit, malevolent figure, malevolent presence, malevolent spirit, sinister forces]  \n",
       "27                                                                                                                                                                                                                                                                                                                                                                                                   [digital age, digital environment, digital era, digital existence, digital frontier, digital revolution, digital technologies, digital world]  \n",
       "28                                                                                                                                                                                                                                                                                                                                                                                [Dangerous adventures, Hazardous journey, Perilous adventures, dangerous adventure, dangerous journey, murderous journey, perilous journey, treacherous journey]  \n",
       "29                                                                                                                                                                                                                                                                                                                                                                                                                                                 [aging queer, genderqueer, queer, queer art, queer artist, queer girl, queer life, queer youth]  "
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"\n",
    "MATCH (t:Theme)\n",
    "WHERE t.wccId IS NOT NULL\n",
    "WITH t.wccId AS id, collect(t.description) AS themes\n",
    "RETURN id, size(themes) AS themeCount, themes\n",
    "ORDER BY themeCount DESC\n",
    "LIMIT 30\"\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6028699d-a028-4579-a0ed-5ed795b83fb4",
   "metadata": {},
   "source": [
    "# Create undirected graph for Leiden\n",
    "\n",
    "The largest WCC communities are too big. For example, \"enchanted forest\" and \"buried body\" do not belog in the same community. We'll use Leiden to make smaller comminities.\n",
    "\n",
    "Leiden requires undirected relationships, so convert the directed IS_SIMILAR relationship to an undirected relationship called \"UNDIRECTED_SIMILAR\"."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "829621c3-cd37-427e-b999-5da7a9e50b44",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>inputRelationships</th>\n",
       "      <th>relationshipsWritten</th>\n",
       "      <th>mutateMillis</th>\n",
       "      <th>postProcessingMillis</th>\n",
       "      <th>preProcessingMillis</th>\n",
       "      <th>computeMillis</th>\n",
       "      <th>configuration</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>13736</td>\n",
       "      <td>16944</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>34</td>\n",
       "      <td>{'aggregation': 'MAX', 'relationshipType': 'IS_SIMILAR', 'jobId': 'bf7b8eaf-5c7f-4c5e-be0c-6f878df1fb42', 'logProgress': True, 'concurrency': 4, 'sudo': False, 'mutateRelationshipType': 'UNDIRECTED_SIMILAR'}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   inputRelationships  relationshipsWritten  mutateMillis  \\\n",
       "0               13736                 16944             0   \n",
       "\n",
       "   postProcessingMillis  preProcessingMillis  computeMillis  \\\n",
       "0                     0                    0             34   \n",
       "\n",
       "                                                                                                                                                                                                     configuration  \n",
       "0  {'aggregation': 'MAX', 'relationshipType': 'IS_SIMILAR', 'jobId': 'bf7b8eaf-5c7f-4c5e-be0c-6f878df1fb42', 'logProgress': True, 'concurrency': 4, 'sudo': False, 'mutateRelationshipType': 'UNDIRECTED_SIMILAR'}  "
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"\n",
    "call gds.graph.relationships.toUndirected(\"themes\", {relationshipType: \"IS_SIMILAR\", mutateRelationshipType: \"UNDIRECTED_SIMILAR\", aggregation: \"MAX\"})\"\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3eff02d9-249f-4617-b2a2-b83c5237aab8",
   "metadata": {},
   "source": [
    "## Transform relationship weights\n",
    "Because of our similarity cutoff in KNN, all of our relationship scores are between 0.811518 and 1.0. Apply a transformation to spread them out between 0.0 and 1.0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "8d08af09-0055-4086-aa0a-f0864d269faa",
   "metadata": {},
   "outputs": [],
   "source": [
    "gds.set_database(\"neo4j\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "b03a6967-76dc-4c08-8c53-4dc167379f18",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>graphName</th>\n",
       "      <th>relationshipCount</th>\n",
       "      <th>nodeCount</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>reweight_themes</td>\n",
       "      <td>33888</td>\n",
       "      <td>9319</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         graphName  relationshipCount  nodeCount\n",
       "0  reweight_themes              33888       9319"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"\n",
    "CALL gds.graph.relationshipProperties.stream($graphName, [\"similarity\"], [\"UNDIRECTED_SIMILAR\"])\n",
    "YIELD sourceNodeId, targetNodeId, propertyValue\n",
    "WITH sourceNodeId AS source, \n",
    "targetNodeId AS target,\n",
    "max(propertyValue) AS score\n",
    "WITH gds.graph.project(\"reweight_themes\",\n",
    "source,\n",
    "target,\n",
    "{relationshipType: \"IS_SIMILAR\",\n",
    "relationshipProperties: {similarity: (score-$similarityCutoff)/(1-$similarityCutoff)}},\n",
    "{undirectedRelationshipTypes: [\"IS_SIMILAR\"]}) AS g\n",
    "RETURN g.graphName AS graphName, g.relationshipCount AS relationshipCount, g.nodeCount AS nodeCount\"\"\",\n",
    "              {\"graphName\": g_themes.name(), \"similarityCutoff\": similarity_cutoff})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "4f214751-6a7c-4d12-8858-e0b34a4e7b6d",
   "metadata": {},
   "outputs": [],
   "source": [
    "g_themes2 = gds.graph.get(\"reweight_themes\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4b47cbf1-338c-494d-bfcb-a78ca11f165d",
   "metadata": {},
   "source": [
    "# Test Leiden with different values of gamma\n",
    "This function will run the Leiden algorithm with a value of gamma that you provide. It will return the size of the biggest Leiden communities and the themes in those communities. It will also check for the smallest communities that contain only one document."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "4f7bedf5-a7fc-4444-90fc-6639c1e31bd3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def test_leiden_gamma(gamma):\n",
    "    property_name = f\"leidenGamma{gamma}\"\n",
    "    try:\n",
    "        gds.graph.nodeProperties.drop(g_themes2, property_name)\n",
    "    except:\n",
    "        pass\n",
    "    leiden_result = gds.leiden.mutate(g_themes2,\n",
    "                      mutateProperty = property_name,\n",
    "                      relationshipTypes = [\"IS_SIMILAR\"],\n",
    "                      relationshipWeightProperty = \"similarity\",\n",
    "                      gamma=gamma)\n",
    "    biggest_communities = gds.run_cypher(\"\"\"\n",
    "        CALL gds.graph.nodeProperty.stream(\"reweight_themes\", $propName)\n",
    "        YIELD nodeId, propertyValue\n",
    "        WITH propertyValue, collect(gds.util.asNode(nodeId)) AS nodeList, count(*) as themeCount\n",
    "        ORDER BY themeCount desc\n",
    "        LIMIT 20\n",
    "        RETURN themeCount, [n in nodeList | coalesce(n.description, n.descriptions)] AS themes\"\"\",\n",
    "                                        {\"propName\": property_name})\n",
    "    stats_df = gds.run_cypher(\"\"\"\n",
    "        CALL gds.graph.nodeProperty.stream(\"reweight_themes\", \"leidenGamma1.0\")\n",
    "        YIELD nodeId, propertyValue\n",
    "        WITH gds.util.asNode(nodeId) as n, propertyValue AS leidenId\n",
    "        WITH leidenId, collect(n) AS nodeList\n",
    "        WHERE size(nodeList) = 1\n",
    "        WITH leidenId, nodeList[0] AS n\n",
    "        WITH leidenId,\n",
    "        COUNT{MATCH (n)<-[:HAS_THEME]-(d) RETURN DISTINCT d} AS themeCount,\n",
    "        COUNT{MATCH (n)<-[:HAS_STEM]-()<-[:HAS_THEME]-(d) RETURN DISTINCT d} AS stemCount\n",
    "        WHERE themeCount + stemCount = 1\n",
    "        RETURN count(*) AS singleDocCommunities\"\"\",\n",
    "                                           {\"propName\": property_name})\n",
    "    stats_df.loc[0,'community_count'] = leiden_result['communityCount']\n",
    "    stats_df.loc[0, 'max_community_size'] = leiden_result['communityDistribution']['max']\n",
    "    stats_df.loc[0, 'mean_community_size'] = leiden_result['communityDistribution']['mean']\n",
    "    stats_df.loc[0, 'median_community_size'] = leiden_result['communityDistribution']['p50']\n",
    "    stats_df.loc[0, 'gamma'] = gamma\n",
    "    return {\"biggest_communities\": biggest_communities, \"stats\": stats_df}\n",
    "                                           \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "788e5207-90d6-4e57-a01f-d783d97a0766",
   "metadata": {},
   "outputs": [],
   "source": [
    "gamma_tests = {gamma: test_leiden_gamma(gamma) for gamma in [1.0, 2.0, 4.0, 8.0, 16.0, 32.0, 64.0, 128.0, 256.0, 512.0, 1024.0]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "f74c7d73-903f-4525-a61c-d7c8e343a433",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>gamma</th>\n",
       "      <th>singleDocCommunities</th>\n",
       "      <th>community_count</th>\n",
       "      <th>max_community_size</th>\n",
       "      <th>mean_community_size</th>\n",
       "      <th>median_community_size</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2384.0</td>\n",
       "      <td>93.0</td>\n",
       "      <td>3.908977</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2398.0</td>\n",
       "      <td>87.0</td>\n",
       "      <td>3.886155</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2409.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>3.868410</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2439.0</td>\n",
       "      <td>53.0</td>\n",
       "      <td>3.820828</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>16.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2472.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>3.769822</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>32.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2527.0</td>\n",
       "      <td>33.0</td>\n",
       "      <td>3.687772</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>64.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2606.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>3.575979</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>128.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2720.0</td>\n",
       "      <td>19.0</td>\n",
       "      <td>3.426103</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>256.0</td>\n",
       "      <td>0</td>\n",
       "      <td>2872.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>3.244777</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>512.0</td>\n",
       "      <td>0</td>\n",
       "      <td>3146.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>2.962174</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1024.0</td>\n",
       "      <td>0</td>\n",
       "      <td>3634.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>2.564392</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    gamma  singleDocCommunities  community_count  max_community_size  \\\n",
       "0     1.0                     0           2384.0                93.0   \n",
       "0     2.0                     0           2398.0                87.0   \n",
       "0     4.0                     0           2409.0                65.0   \n",
       "0     8.0                     0           2439.0                53.0   \n",
       "0    16.0                     0           2472.0                40.0   \n",
       "0    32.0                     0           2527.0                33.0   \n",
       "0    64.0                     0           2606.0                25.0   \n",
       "0   128.0                     0           2720.0                19.0   \n",
       "0   256.0                     0           2872.0                15.0   \n",
       "0   512.0                     0           3146.0                12.0   \n",
       "0  1024.0                     0           3634.0                 9.0   \n",
       "\n",
       "   mean_community_size  median_community_size  \n",
       "0             3.908977                    2.0  \n",
       "0             3.886155                    2.0  \n",
       "0             3.868410                    2.0  \n",
       "0             3.820828                    2.0  \n",
       "0             3.769822                    2.0  \n",
       "0             3.687772                    2.0  \n",
       "0             3.575979                    2.0  \n",
       "0             3.426103                    2.0  \n",
       "0             3.244777                    2.0  \n",
       "0             2.962174                    2.0  \n",
       "0             2.564392                    2.0  "
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stats_df = pd.concat([r['stats'] for r in [gamma_tests[key] for key in gamma_tests.keys()]])\n",
    "stats_df[['gamma', 'singleDocCommunities', 'community_count', 'max_community_size', 'mean_community_size', 'median_community_size']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "990dfb02-5258-42ce-92e4-9030be2a14fc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gamma value: 1.0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>themeCount</th>\n",
       "      <th>themes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>93</td>\n",
       "      <td>[secret observation, secret relationship, dark stories, dark secrets, dark fantasies, secret romance, secret love, Uncovered secrets, Dirty secrets, secret mission, secret identity, secret task, unfulfilled desires, unresolved feelings, dark forest, dark path, dark walls, hidden secrets, shocking secrets, forbidden secrets, unsettling secrets, [secret, secrets], past secrets, hidden, secrets exposed, hidden feelings, Dark realities, Dark interest, Dark realm, dark presence, Dark pasts, Darker truth, dark characters, dark motives, dark history, Ancient secrets, inner darkness, dark place, darkness, Darkness beneath, lurking darkness, dark abyss, dark energy, dark force, dark, Deep dark, Dark Prison, dark moon, Dark Highway, forbidden room, forbidden desires, dark house, dark room, dark web, dark network, darknet, Old secret, Dirty secret, Secret passion, secret desires, Secret desire, invisibility, invisible, dark themes, dark time, dark twist, dark scheme, Darkness to light, Dark Light, secret recipe, secret daughters, secret identities, secret living, secret world, darkness vs. light, unresolved trauma, unresolved grief, dark thoughts, dark secret, dark truth, secret knowledge, Unspoken Desires, Hidden desire, Secret dish, Dark Void, children's secrets, buried truth, secret past, secret agenda, dark tunnel, secret online life, secret revealed, disturbing secret]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>87</td>\n",
       "      <td>[Women's work, women's community, women's crafts, estranged mother, estranged parents, missing mother, ailing father, missing father, ailing son, father's fight, late father, abusive father, Lonely father, Mom, [mother, mothers], grieving neighbor, grieving father, missing daughter, estranged daughter, Daughter's struggle, missing sister, Women's support, [mother's support, mothers' support], ailing mother, lost mother, Devoted mother, alcoholic mother, Irish mother, mother's departure, mother's sacrifice, mother's return, birth mother, mother's death, loss of mother, mother's passing, deceased mother, mother disappears, mother's debt, mother's loss, father's death, loss of father, dying father, dying parent, missing mom, deceased father, Fatherhood loss, mother's son, mother's life, Maternal, motherhood, mother's, separated, estranged, mother's love, mother's quest, Estranged son, father-daughter, [father, fathers], father-daughter relationship, abusive husband, father figure, father search, maternal love, new mother, motherhood struggles, mother-daughter, Mother-daughter vacation, estranged sisters, estranged friends, young mother, Great Mother, mother's sacrifices, father-son reconciliation, mother's absence, estrangement, Suicidal mother, grieving mother, Absent mother, mother's struggle, grief-stricken, grieving, Mothers and daughters, Struggling mother, mother's vow, postpartum anxiety, postpartum struggles, Irish life]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>75</td>\n",
       "      <td>[strange phenomenon, strange events, strange phenomena, strange performance, old stories, global stories, tall tales, untold stories, street tales, true stories, children's stories, funny stories, Strange creature, bizarre creature, strange creatures, Gruesome creature, true story, mysterious deaths, Mysterious disappearance, mysterious events, enigmatic, enigmatic key, enigmatic figure, [enigma, enigmas], enigmatic woman, enigmatic past, old couple, married couple, mysterious object, Mysterious phenomenon, [disappearance, disappearances], mystery creature, [mysteries, mystery], enigmatic castle, Enigmatic Writer, Unnatural occurrences, Unexplainable events, mysterious past, enigmatic event, mysterious incident, mysterious celebration, Strange happenings, bizarre events, Strange situations, Weird events, strange encounters, strange things, Children disappearing, strange couple, lesbian couple, Broken couple, Loving Couple, unsolved mystery, medical mystery, Misteri, strange behavior, Outrageous situation, Crazy situations, bizarre stories, crazy events, bizarre event, Inexplicable, strange places, Gruesome effects, Grisly, unfinished stories, bizarre, strange, children disappear, mutant creature, weird, strange object, strange town, strange terrain, unexplained]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>71</td>\n",
       "      <td>[Queer visions, Queer thoughts, queer art, Queer artists, Queer filmmaking, Queer creatives, Queer legacy, Queer visibility, Queer Hero, queer media, queer community, queer culture, Queer comedy, queer woman, queer identity, queer love, Transgender narrative, gender transition, queer, aging queer, genderqueer, queer youth, queer life, queer horror, queer history, queer girl, queercore, queer artist, Queer joy, queer nightlife, queerness, LGBTQIA+, Queer lovers, Queer Sexuality, Queer mom, Queer eroticism, queer role model, queer experience, queer women, gender expression, gender identity, gender neutral, gender affirmation, LGBTQIA+ experiences, LGBTQ history, Queer sensibilities, Queer experiences, Queer resistance, Queer desires, queer liberation, queer identities, Queer protagonists, Queer journey, gender dysphoria, sexuality, homosexuality, homosexual romance, homoerotic, queer romance, Queer subculture, Queer mecca, queer struggle, queer representations, queer activism, LGBTQI+ community, Queer struggles, Queer Scene, Queer genealogy, Femme identities, Queer existences, Queer utopias]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>68</td>\n",
       "      <td>[parents' home, family home, family agency, family provider, Family baggage, Family mystery, Christmas holidays, holiday magic, Holiday cheer, package holidays, familial bonds, Family bond, three families, [families, family], family secrets, Familial secrets, family memory, family ties, family traditions, family bonds, family roots, familial relationships, family relationships, Family antics, Family adventure, holiday spirit, holiday joy, holiday season, holiday gifts, family holiday, family secret, Family conversations, Family story, family wedding, family love, family life, Family pain, family values, family protection, family support, Winter holiday, [holiday, holidays], holiday special, family tradition, family business, winter break, Winter Movie, family bonding, Female bonding, family vacation home, family man, vacation friends, Family survival, family dynamics, Traditional values, conservative values, traditional family, family expectations, family reflections, family history, family move, family journey, family dignity, Conservative views, family visit, family legend, family divisions, Family culture]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   themeCount  \\\n",
       "0          93   \n",
       "1          87   \n",
       "2          75   \n",
       "3          71   \n",
       "4          68   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      themes  \n",
       "0                                                                  [secret observation, secret relationship, dark stories, dark secrets, dark fantasies, secret romance, secret love, Uncovered secrets, Dirty secrets, secret mission, secret identity, secret task, unfulfilled desires, unresolved feelings, dark forest, dark path, dark walls, hidden secrets, shocking secrets, forbidden secrets, unsettling secrets, [secret, secrets], past secrets, hidden, secrets exposed, hidden feelings, Dark realities, Dark interest, Dark realm, dark presence, Dark pasts, Darker truth, dark characters, dark motives, dark history, Ancient secrets, inner darkness, dark place, darkness, Darkness beneath, lurking darkness, dark abyss, dark energy, dark force, dark, Deep dark, Dark Prison, dark moon, Dark Highway, forbidden room, forbidden desires, dark house, dark room, dark web, dark network, darknet, Old secret, Dirty secret, Secret passion, secret desires, Secret desire, invisibility, invisible, dark themes, dark time, dark twist, dark scheme, Darkness to light, Dark Light, secret recipe, secret daughters, secret identities, secret living, secret world, darkness vs. light, unresolved trauma, unresolved grief, dark thoughts, dark secret, dark truth, secret knowledge, Unspoken Desires, Hidden desire, Secret dish, Dark Void, children's secrets, buried truth, secret past, secret agenda, dark tunnel, secret online life, secret revealed, disturbing secret]  \n",
       "1  [Women's work, women's community, women's crafts, estranged mother, estranged parents, missing mother, ailing father, missing father, ailing son, father's fight, late father, abusive father, Lonely father, Mom, [mother, mothers], grieving neighbor, grieving father, missing daughter, estranged daughter, Daughter's struggle, missing sister, Women's support, [mother's support, mothers' support], ailing mother, lost mother, Devoted mother, alcoholic mother, Irish mother, mother's departure, mother's sacrifice, mother's return, birth mother, mother's death, loss of mother, mother's passing, deceased mother, mother disappears, mother's debt, mother's loss, father's death, loss of father, dying father, dying parent, missing mom, deceased father, Fatherhood loss, mother's son, mother's life, Maternal, motherhood, mother's, separated, estranged, mother's love, mother's quest, Estranged son, father-daughter, [father, fathers], father-daughter relationship, abusive husband, father figure, father search, maternal love, new mother, motherhood struggles, mother-daughter, Mother-daughter vacation, estranged sisters, estranged friends, young mother, Great Mother, mother's sacrifices, father-son reconciliation, mother's absence, estrangement, Suicidal mother, grieving mother, Absent mother, mother's struggle, grief-stricken, grieving, Mothers and daughters, Struggling mother, mother's vow, postpartum anxiety, postpartum struggles, Irish life]  \n",
       "2                                                                                                                                                                         [strange phenomenon, strange events, strange phenomena, strange performance, old stories, global stories, tall tales, untold stories, street tales, true stories, children's stories, funny stories, Strange creature, bizarre creature, strange creatures, Gruesome creature, true story, mysterious deaths, Mysterious disappearance, mysterious events, enigmatic, enigmatic key, enigmatic figure, [enigma, enigmas], enigmatic woman, enigmatic past, old couple, married couple, mysterious object, Mysterious phenomenon, [disappearance, disappearances], mystery creature, [mysteries, mystery], enigmatic castle, Enigmatic Writer, Unnatural occurrences, Unexplainable events, mysterious past, enigmatic event, mysterious incident, mysterious celebration, Strange happenings, bizarre events, Strange situations, Weird events, strange encounters, strange things, Children disappearing, strange couple, lesbian couple, Broken couple, Loving Couple, unsolved mystery, medical mystery, Misteri, strange behavior, Outrageous situation, Crazy situations, bizarre stories, crazy events, bizarre event, Inexplicable, strange places, Gruesome effects, Grisly, unfinished stories, bizarre, strange, children disappear, mutant creature, weird, strange object, strange town, strange terrain, unexplained]  \n",
       "3                                                                                                                                                                                                                                                                                                                                                         [Queer visions, Queer thoughts, queer art, Queer artists, Queer filmmaking, Queer creatives, Queer legacy, Queer visibility, Queer Hero, queer media, queer community, queer culture, Queer comedy, queer woman, queer identity, queer love, Transgender narrative, gender transition, queer, aging queer, genderqueer, queer youth, queer life, queer horror, queer history, queer girl, queercore, queer artist, Queer joy, queer nightlife, queerness, LGBTQIA+, Queer lovers, Queer Sexuality, Queer mom, Queer eroticism, queer role model, queer experience, queer women, gender expression, gender identity, gender neutral, gender affirmation, LGBTQIA+ experiences, LGBTQ history, Queer sensibilities, Queer experiences, Queer resistance, Queer desires, queer liberation, queer identities, Queer protagonists, Queer journey, gender dysphoria, sexuality, homosexuality, homosexual romance, homoerotic, queer romance, Queer subculture, Queer mecca, queer struggle, queer representations, queer activism, LGBTQI+ community, Queer struggles, Queer Scene, Queer genealogy, Femme identities, Queer existences, Queer utopias]  \n",
       "4                                                                                                                                                                                                                                                                                                                                     [parents' home, family home, family agency, family provider, Family baggage, Family mystery, Christmas holidays, holiday magic, Holiday cheer, package holidays, familial bonds, Family bond, three families, [families, family], family secrets, Familial secrets, family memory, family ties, family traditions, family bonds, family roots, familial relationships, family relationships, Family antics, Family adventure, holiday spirit, holiday joy, holiday season, holiday gifts, family holiday, family secret, Family conversations, Family story, family wedding, family love, family life, Family pain, family values, family protection, family support, Winter holiday, [holiday, holidays], holiday special, family tradition, family business, winter break, Winter Movie, family bonding, Female bonding, family vacation home, family man, vacation friends, Family survival, family dynamics, Traditional values, conservative values, traditional family, family expectations, family reflections, family history, family move, family journey, family dignity, Conservative views, family visit, family legend, family divisions, Family culture]  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gamma value: 2.0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>themeCount</th>\n",
       "      <th>themes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>87</td>\n",
       "      <td>[Women's work, women's community, women's crafts, estranged mother, estranged parents, missing mother, ailing father, missing father, ailing son, father's fight, late father, abusive father, Lonely father, Mom, [mother, mothers], grieving neighbor, grieving father, missing daughter, estranged daughter, Daughter's struggle, missing sister, Women's support, [mother's support, mothers' support], ailing mother, lost mother, Devoted mother, alcoholic mother, Irish mother, mother's departure, mother's sacrifice, mother's return, birth mother, mother's death, loss of mother, mother's passing, deceased mother, mother disappears, mother's debt, mother's loss, father's death, loss of father, dying father, dying parent, missing mom, deceased father, Fatherhood loss, mother's son, mother's life, Maternal, motherhood, mother's, separated, estranged, mother's love, mother's quest, Estranged son, father-daughter, [father, fathers], father-daughter relationship, abusive husband, father figure, father search, maternal love, new mother, motherhood struggles, mother-daughter, Mother-daughter vacation, estranged sisters, estranged friends, young mother, Great Mother, mother's sacrifices, father-son reconciliation, mother's absence, estrangement, Suicidal mother, grieving mother, Absent mother, mother's struggle, grief-stricken, grieving, Mothers and daughters, Struggling mother, mother's vow, postpartum anxiety, postpartum struggles, Irish life]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>73</td>\n",
       "      <td>[strange phenomenon, strange events, strange phenomena, strange performance, old stories, global stories, tall tales, untold stories, street tales, true stories, children's stories, funny stories, Strange creature, bizarre creature, strange creatures, Gruesome creature, true story, mysterious deaths, Mysterious disappearance, mysterious events, enigmatic, enigmatic key, enigmatic figure, [enigma, enigmas], enigmatic woman, enigmatic past, old couple, married couple, mysterious object, Mysterious phenomenon, [disappearance, disappearances], mystery creature, [mysteries, mystery], enigmatic castle, Enigmatic Writer, Unnatural occurrences, Unexplainable events, mysterious past, enigmatic event, mysterious incident, mysterious celebration, Strange happenings, bizarre events, Strange situations, Weird events, strange encounters, strange things, strange couple, lesbian couple, Broken couple, Loving Couple, unsolved mystery, medical mystery, Misteri, strange behavior, Outrageous situation, Crazy situations, bizarre stories, crazy events, bizarre event, Inexplicable, strange places, Gruesome effects, Grisly, unfinished stories, bizarre, strange, mutant creature, weird, strange object, strange town, strange terrain, unexplained]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>65</td>\n",
       "      <td>[Queer visions, Queer thoughts, queer art, Queer artists, Queer filmmaking, Queer creatives, Queer legacy, Queer visibility, Queer Hero, queer media, queer community, queer culture, Queer comedy, queer woman, queer identity, queer love, queer, aging queer, genderqueer, queer youth, queer life, queer horror, queer history, queer girl, queercore, queer artist, Queer joy, queer nightlife, queerness, LGBTQIA+, Queer lovers, Queer Sexuality, Queer mom, Queer eroticism, queer role model, queer experience, queer women, gender neutral, LGBTQIA+ experiences, LGBTQ history, Queer sensibilities, Queer experiences, Queer resistance, Queer desires, queer liberation, queer identities, Queer protagonists, Queer journey, sexuality, homosexuality, homosexual romance, homoerotic, queer romance, Queer subculture, Queer mecca, queer struggle, queer representations, queer activism, LGBTQI+ community, Queer struggles, Queer Scene, Queer genealogy, Femme identities, Queer existences, Queer utopias]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>57</td>\n",
       "      <td>[Alien landscape, surreal landscape, mystical, nightmare begins, [nightmare, nightmares], mystical terror, supernatural terror, Surreal fantasy, Surreal dream, surreal, fantasy world, Surreal comedy, Surreal visions, surreal nightmare, Surreal building, Surreal associations, surreal flight, surreal dreams, shared subconscious, Subconsciousness, Subconscious world, mystical landscape, surreal animation, surrealism, dreamlike reality, Surreal imagery, surrealist, surreal vision, action adventure, surreal adventure, mysticism, imaginary world, Kafkaesque nightmare, Horrifying nightmare, surreal nightmares, bad dream, wild landscapes, wilderness, desolate landscapes, wild nature, extraordinary landscapes, wilderness captivity, Wilderness lodge, Artificial world, Mobile Nightmare, mystical brew, Psychedelic visions, Psychedelicized, mythological, Unfamiliar landscape, subconscious nightmares, Escalating nightmare, Surreal landscapes, contrasting landscapes, Surreal Voyage, turbulent dreams, morbid dreams]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>55</td>\n",
       "      <td>[secret observation, secret relationship, dark stories, dark secrets, dark fantasies, secret romance, secret love, Uncovered secrets, Dirty secrets, secret mission, secret identity, secret task, unfulfilled desires, unresolved feelings, hidden secrets, shocking secrets, forbidden secrets, unsettling secrets, [secret, secrets], past secrets, hidden, secrets exposed, hidden feelings, dark motives, dark history, Ancient secrets, forbidden room, forbidden desires, Old secret, Dirty secret, Secret passion, secret desires, Secret desire, invisibility, invisible, secret recipe, secret daughters, secret identities, secret living, secret world, unresolved trauma, unresolved grief, dark secret, dark truth, secret knowledge, Unspoken Desires, Hidden desire, Secret dish, children's secrets, buried truth, secret past, secret agenda, secret online life, secret revealed, disturbing secret]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   themeCount  \\\n",
       "0          87   \n",
       "1          73   \n",
       "2          65   \n",
       "3          57   \n",
       "4          55   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      themes  \n",
       "0  [Women's work, women's community, women's crafts, estranged mother, estranged parents, missing mother, ailing father, missing father, ailing son, father's fight, late father, abusive father, Lonely father, Mom, [mother, mothers], grieving neighbor, grieving father, missing daughter, estranged daughter, Daughter's struggle, missing sister, Women's support, [mother's support, mothers' support], ailing mother, lost mother, Devoted mother, alcoholic mother, Irish mother, mother's departure, mother's sacrifice, mother's return, birth mother, mother's death, loss of mother, mother's passing, deceased mother, mother disappears, mother's debt, mother's loss, father's death, loss of father, dying father, dying parent, missing mom, deceased father, Fatherhood loss, mother's son, mother's life, Maternal, motherhood, mother's, separated, estranged, mother's love, mother's quest, Estranged son, father-daughter, [father, fathers], father-daughter relationship, abusive husband, father figure, father search, maternal love, new mother, motherhood struggles, mother-daughter, Mother-daughter vacation, estranged sisters, estranged friends, young mother, Great Mother, mother's sacrifices, father-son reconciliation, mother's absence, estrangement, Suicidal mother, grieving mother, Absent mother, mother's struggle, grief-stricken, grieving, Mothers and daughters, Struggling mother, mother's vow, postpartum anxiety, postpartum struggles, Irish life]  \n",
       "1                                                                                                                                                                                                                    [strange phenomenon, strange events, strange phenomena, strange performance, old stories, global stories, tall tales, untold stories, street tales, true stories, children's stories, funny stories, Strange creature, bizarre creature, strange creatures, Gruesome creature, true story, mysterious deaths, Mysterious disappearance, mysterious events, enigmatic, enigmatic key, enigmatic figure, [enigma, enigmas], enigmatic woman, enigmatic past, old couple, married couple, mysterious object, Mysterious phenomenon, [disappearance, disappearances], mystery creature, [mysteries, mystery], enigmatic castle, Enigmatic Writer, Unnatural occurrences, Unexplainable events, mysterious past, enigmatic event, mysterious incident, mysterious celebration, Strange happenings, bizarre events, Strange situations, Weird events, strange encounters, strange things, strange couple, lesbian couple, Broken couple, Loving Couple, unsolved mystery, medical mystery, Misteri, strange behavior, Outrageous situation, Crazy situations, bizarre stories, crazy events, bizarre event, Inexplicable, strange places, Gruesome effects, Grisly, unfinished stories, bizarre, strange, mutant creature, weird, strange object, strange town, strange terrain, unexplained]  \n",
       "2                                                                                                                                                                                                                                                                                                                                                                                                                                                                             [Queer visions, Queer thoughts, queer art, Queer artists, Queer filmmaking, Queer creatives, Queer legacy, Queer visibility, Queer Hero, queer media, queer community, queer culture, Queer comedy, queer woman, queer identity, queer love, queer, aging queer, genderqueer, queer youth, queer life, queer horror, queer history, queer girl, queercore, queer artist, Queer joy, queer nightlife, queerness, LGBTQIA+, Queer lovers, Queer Sexuality, Queer mom, Queer eroticism, queer role model, queer experience, queer women, gender neutral, LGBTQIA+ experiences, LGBTQ history, Queer sensibilities, Queer experiences, Queer resistance, Queer desires, queer liberation, queer identities, Queer protagonists, Queer journey, sexuality, homosexuality, homosexual romance, homoerotic, queer romance, Queer subculture, Queer mecca, queer struggle, queer representations, queer activism, LGBTQI+ community, Queer struggles, Queer Scene, Queer genealogy, Femme identities, Queer existences, Queer utopias]  \n",
       "3                                                                                                                                                                                                                                                                                                                                                                                                                                                    [Alien landscape, surreal landscape, mystical, nightmare begins, [nightmare, nightmares], mystical terror, supernatural terror, Surreal fantasy, Surreal dream, surreal, fantasy world, Surreal comedy, Surreal visions, surreal nightmare, Surreal building, Surreal associations, surreal flight, surreal dreams, shared subconscious, Subconsciousness, Subconscious world, mystical landscape, surreal animation, surrealism, dreamlike reality, Surreal imagery, surrealist, surreal vision, action adventure, surreal adventure, mysticism, imaginary world, Kafkaesque nightmare, Horrifying nightmare, surreal nightmares, bad dream, wild landscapes, wilderness, desolate landscapes, wild nature, extraordinary landscapes, wilderness captivity, Wilderness lodge, Artificial world, Mobile Nightmare, mystical brew, Psychedelic visions, Psychedelicized, mythological, Unfamiliar landscape, subconscious nightmares, Escalating nightmare, Surreal landscapes, contrasting landscapes, Surreal Voyage, turbulent dreams, morbid dreams]  \n",
       "4                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    [secret observation, secret relationship, dark stories, dark secrets, dark fantasies, secret romance, secret love, Uncovered secrets, Dirty secrets, secret mission, secret identity, secret task, unfulfilled desires, unresolved feelings, hidden secrets, shocking secrets, forbidden secrets, unsettling secrets, [secret, secrets], past secrets, hidden, secrets exposed, hidden feelings, dark motives, dark history, Ancient secrets, forbidden room, forbidden desires, Old secret, Dirty secret, Secret passion, secret desires, Secret desire, invisibility, invisible, secret recipe, secret daughters, secret identities, secret living, secret world, unresolved trauma, unresolved grief, dark secret, dark truth, secret knowledge, Unspoken Desires, Hidden desire, Secret dish, children's secrets, buried truth, secret past, secret agenda, secret online life, secret revealed, disturbing secret]  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gamma value: 4.0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>themeCount</th>\n",
       "      <th>themes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>65</td>\n",
       "      <td>[Women's work, women's community, women's crafts, estranged mother, estranged parents, missing mother, Mom, [mother, mothers], missing daughter, estranged daughter, Daughter's struggle, missing sister, Women's support, [mother's support, mothers' support], ailing mother, lost mother, Devoted mother, alcoholic mother, Irish mother, mother's departure, mother's sacrifice, mother's return, birth mother, mother's death, loss of mother, mother's passing, deceased mother, mother disappears, mother's debt, mother's loss, father's death, missing mom, deceased father, mother's son, mother's life, Maternal, motherhood, mother's, separated, estranged, mother's love, mother's quest, Estranged son, maternal love, new mother, motherhood struggles, mother-daughter, Mother-daughter vacation, estranged sisters, estranged friends, young mother, Great Mother, mother's sacrifices, mother's absence, estrangement, Suicidal mother, grieving mother, Absent mother, mother's struggle, Mothers and daughters, Struggling mother, mother's vow, postpartum anxiety, postpartum struggles, Irish life]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>55</td>\n",
       "      <td>[secret observation, secret relationship, dark stories, dark secrets, dark fantasies, secret romance, secret love, Uncovered secrets, Dirty secrets, secret mission, secret identity, secret task, unfulfilled desires, unresolved feelings, hidden secrets, shocking secrets, forbidden secrets, unsettling secrets, [secret, secrets], past secrets, hidden, secrets exposed, hidden feelings, dark motives, dark history, Ancient secrets, forbidden room, forbidden desires, Old secret, Dirty secret, Secret passion, secret desires, Secret desire, invisibility, invisible, secret recipe, secret daughters, secret identities, secret living, secret world, unresolved trauma, unresolved grief, dark secret, dark truth, secret knowledge, Unspoken Desires, Hidden desire, Secret dish, children's secrets, buried truth, secret past, secret agenda, secret online life, secret revealed, disturbing secret]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>55</td>\n",
       "      <td>[Queer visions, Queer thoughts, queer art, Queer legacy, Queer visibility, Queer Hero, queer media, queer community, queer culture, Queer comedy, queer woman, queer identity, queer love, queer, aging queer, genderqueer, queer youth, queer life, queer horror, queer history, queer girl, queercore, queer artist, Queer joy, queer nightlife, queerness, LGBTQIA+, Queer lovers, Queer Sexuality, Queer mom, Queer eroticism, queer role model, queer experience, queer women, gender neutral, LGBTQIA+ experiences, LGBTQ history, queer liberation, Queer journey, sexuality, homosexuality, homosexual romance, homoerotic, queer romance, Queer subculture, Queer mecca, queer struggle, queer representations, queer activism, LGBTQI+ community, Queer struggles, Queer Scene, Queer genealogy, Queer existences, Queer utopias]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>54</td>\n",
       "      <td>[strange event, strange presence, strange encounter, weird incident, Weird incidents, bizarre incident, strange artifact, strange figure, Absurd encounters, unexpected encounters, Brutal crime, Brutal executions, brutal murders, Disturbing crime, Unusual occurrence, Bizarre phenomena, strange occurrences, Disturbing past, Disturbing events, Disturbing event, Eerie relationship, eerie encounters, spiritual encounters, mysterious encounters, surreal encounters, spiritual medium, Disturbing presence, uncanny presence, serendipitous encounter, unexpected encounter, unexpected friendship, Random encounters, First Encounter, Strange existence, Eccentric stranger, strange person, Disturbing truth, uncanny, gruesome deaths, unlikely friendship, strange murders, strange woman, strange bird, Unseen presence, Disturbing encounter, unsolved murders, unexplained events, alien encounter, disturbing book, Horrific crime, Strange company, strange action, ghostly presences, strange acquaintances]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>49</td>\n",
       "      <td>[Orphan son, orphan's journey, elderly inmates, elderly, elderly woman, Elderly spinster, elderly Dorothy, Elderly widow, elderly love, elderly lady, elderly maid, elderly mother, elderly student, older woman, elderly gentleman, elderly man, Widow's heart, [widow, widows], widow's journey, Widow's plight, Elderly uncle, Elderly father, Elderly daughter, elderly care, [senior, seniors], aging parents, elderly life, elderly parent, old age home, Aged care, Grieving widow, widower, Widowed father, [elder, elders], Elderly guardian, Widowhood, widowed mother, widowed, Fatherless journey, old man, grandfather, Older man, senior woman, Senior citizen, old lady, Elderly priest, grandson, elderly couple, elderly wrestler]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   themeCount  \\\n",
       "0          65   \n",
       "1          55   \n",
       "2          55   \n",
       "3          54   \n",
       "4          49   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        themes  \n",
       "0  [Women's work, women's community, women's crafts, estranged mother, estranged parents, missing mother, Mom, [mother, mothers], missing daughter, estranged daughter, Daughter's struggle, missing sister, Women's support, [mother's support, mothers' support], ailing mother, lost mother, Devoted mother, alcoholic mother, Irish mother, mother's departure, mother's sacrifice, mother's return, birth mother, mother's death, loss of mother, mother's passing, deceased mother, mother disappears, mother's debt, mother's loss, father's death, missing mom, deceased father, mother's son, mother's life, Maternal, motherhood, mother's, separated, estranged, mother's love, mother's quest, Estranged son, maternal love, new mother, motherhood struggles, mother-daughter, Mother-daughter vacation, estranged sisters, estranged friends, young mother, Great Mother, mother's sacrifices, mother's absence, estrangement, Suicidal mother, grieving mother, Absent mother, mother's struggle, Mothers and daughters, Struggling mother, mother's vow, postpartum anxiety, postpartum struggles, Irish life]  \n",
       "1                                                                                                                                                                                                      [secret observation, secret relationship, dark stories, dark secrets, dark fantasies, secret romance, secret love, Uncovered secrets, Dirty secrets, secret mission, secret identity, secret task, unfulfilled desires, unresolved feelings, hidden secrets, shocking secrets, forbidden secrets, unsettling secrets, [secret, secrets], past secrets, hidden, secrets exposed, hidden feelings, dark motives, dark history, Ancient secrets, forbidden room, forbidden desires, Old secret, Dirty secret, Secret passion, secret desires, Secret desire, invisibility, invisible, secret recipe, secret daughters, secret identities, secret living, secret world, unresolved trauma, unresolved grief, dark secret, dark truth, secret knowledge, Unspoken Desires, Hidden desire, Secret dish, children's secrets, buried truth, secret past, secret agenda, secret online life, secret revealed, disturbing secret]  \n",
       "2                                                                                                                                                                                                                                                                                  [Queer visions, Queer thoughts, queer art, Queer legacy, Queer visibility, Queer Hero, queer media, queer community, queer culture, Queer comedy, queer woman, queer identity, queer love, queer, aging queer, genderqueer, queer youth, queer life, queer horror, queer history, queer girl, queercore, queer artist, Queer joy, queer nightlife, queerness, LGBTQIA+, Queer lovers, Queer Sexuality, Queer mom, Queer eroticism, queer role model, queer experience, queer women, gender neutral, LGBTQIA+ experiences, LGBTQ history, queer liberation, Queer journey, sexuality, homosexuality, homosexual romance, homoerotic, queer romance, Queer subculture, Queer mecca, queer struggle, queer representations, queer activism, LGBTQI+ community, Queer struggles, Queer Scene, Queer genealogy, Queer existences, Queer utopias]  \n",
       "3                                                                                           [strange event, strange presence, strange encounter, weird incident, Weird incidents, bizarre incident, strange artifact, strange figure, Absurd encounters, unexpected encounters, Brutal crime, Brutal executions, brutal murders, Disturbing crime, Unusual occurrence, Bizarre phenomena, strange occurrences, Disturbing past, Disturbing events, Disturbing event, Eerie relationship, eerie encounters, spiritual encounters, mysterious encounters, surreal encounters, spiritual medium, Disturbing presence, uncanny presence, serendipitous encounter, unexpected encounter, unexpected friendship, Random encounters, First Encounter, Strange existence, Eccentric stranger, strange person, Disturbing truth, uncanny, gruesome deaths, unlikely friendship, strange murders, strange woman, strange bird, Unseen presence, Disturbing encounter, unsolved murders, unexplained events, alien encounter, disturbing book, Horrific crime, Strange company, strange action, ghostly presences, strange acquaintances]  \n",
       "4                                                                                                                                                                                                                                                                                                                                                                          [Orphan son, orphan's journey, elderly inmates, elderly, elderly woman, Elderly spinster, elderly Dorothy, Elderly widow, elderly love, elderly lady, elderly maid, elderly mother, elderly student, older woman, elderly gentleman, elderly man, Widow's heart, [widow, widows], widow's journey, Widow's plight, Elderly uncle, Elderly father, Elderly daughter, elderly care, [senior, seniors], aging parents, elderly life, elderly parent, old age home, Aged care, Grieving widow, widower, Widowed father, [elder, elders], Elderly guardian, Widowhood, widowed mother, widowed, Fatherless journey, old man, grandfather, Older man, senior woman, Senior citizen, old lady, Elderly priest, grandson, elderly couple, elderly wrestler]  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gamma value: 8.0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>themeCount</th>\n",
       "      <th>themes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>53</td>\n",
       "      <td>[Queer visions, Queer thoughts, queer art, Queer legacy, Queer visibility, Queer Hero, queer media, queer community, queer culture, Queer comedy, queer woman, queer identity, queer love, queer, aging queer, genderqueer, queer youth, queer life, queer horror, queer history, queer girl, queercore, queer artist, Queer joy, queer nightlife, queerness, Queer lovers, Queer Sexuality, Queer mom, Queer eroticism, queer role model, queer experience, queer women, gender neutral, LGBTQIA+ experiences, LGBTQ history, queer liberation, Queer journey, sexuality, homosexuality, homosexual romance, homoerotic, queer romance, Queer subculture, Queer mecca, queer struggle, queer representations, queer activism, Queer struggles, Queer Scene, Queer genealogy, Queer existences, Queer utopias]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>40</td>\n",
       "      <td>[parents' home, family home, family agency, family provider, three families, [families, family], family secrets, Familial secrets, family memory, family ties, family traditions, family bonds, family roots, familial relationships, family relationships, family holiday, family wedding, family love, family life, Family pain, family values, family protection, family support, family tradition, family business, family bonding, Female bonding, family vacation home, family man, vacation friends, Family survival, family dynamics, Traditional values, conservative values, traditional family, family history, family dignity, Conservative views, family legend, family divisions]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>36</td>\n",
       "      <td>[Dramedy, [comedies, comedy], drama, courtroom drama, Chaotic comedy, Farcical comedy, Hilarious chaos, Veteran comedian, comedy legend, comedy of errors, drama teacher, drama graduate, psychological thriller, psychological drama, [comedy rock, comedy-rock], Comedy awards, meta comedy, comedy roast, Comedy movie, comedy performance, comedy world, comedic love, comedy tour, musical comedy, historical drama, political drama, Gritty drama, Comedy showcase, psychological game, comedy special, folk comedy, live comedy, raw humor, flat humor, Comedy Cellar, action-comedy]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>36</td>\n",
       "      <td>[elderly inmates, elderly, elderly woman, Elderly spinster, elderly Dorothy, Elderly widow, elderly love, elderly lady, elderly maid, elderly mother, elderly student, older woman, elderly gentleman, elderly man, Elderly uncle, Elderly father, Elderly daughter, elderly care, [senior, seniors], aging parents, elderly life, elderly parent, old age home, Aged care, [elder, elders], Elderly guardian, old man, grandfather, Older man, senior woman, Senior citizen, old lady, Elderly priest, grandson, elderly couple, elderly wrestler]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>36</td>\n",
       "      <td>[strange phenomenon, strange events, strange phenomena, strange performance, mysterious deaths, mysterious events, enigmatic, enigmatic key, enigmatic figure, [enigma, enigmas], enigmatic woman, enigmatic past, mysterious object, Mysterious phenomenon, enigmatic castle, Enigmatic Writer, Unnatural occurrences, Unexplainable events, mysterious past, enigmatic event, mysterious incident, mysterious celebration, Weird events, strange encounters, strange things, strange couple, strange behavior, Inexplicable, strange places, bizarre, strange, weird, strange object, strange town, strange terrain, unexplained]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   themeCount  \\\n",
       "0          53   \n",
       "1          40   \n",
       "2          36   \n",
       "3          36   \n",
       "4          36   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           themes  \n",
       "0  [Queer visions, Queer thoughts, queer art, Queer legacy, Queer visibility, Queer Hero, queer media, queer community, queer culture, Queer comedy, queer woman, queer identity, queer love, queer, aging queer, genderqueer, queer youth, queer life, queer horror, queer history, queer girl, queercore, queer artist, Queer joy, queer nightlife, queerness, Queer lovers, Queer Sexuality, Queer mom, Queer eroticism, queer role model, queer experience, queer women, gender neutral, LGBTQIA+ experiences, LGBTQ history, queer liberation, Queer journey, sexuality, homosexuality, homosexual romance, homoerotic, queer romance, Queer subculture, Queer mecca, queer struggle, queer representations, queer activism, Queer struggles, Queer Scene, Queer genealogy, Queer existences, Queer utopias]  \n",
       "1                                                                                                                 [parents' home, family home, family agency, family provider, three families, [families, family], family secrets, Familial secrets, family memory, family ties, family traditions, family bonds, family roots, familial relationships, family relationships, family holiday, family wedding, family love, family life, Family pain, family values, family protection, family support, family tradition, family business, family bonding, Female bonding, family vacation home, family man, vacation friends, Family survival, family dynamics, Traditional values, conservative values, traditional family, family history, family dignity, Conservative views, family legend, family divisions]  \n",
       "2                                                                                                                                                                                                                    [Dramedy, [comedies, comedy], drama, courtroom drama, Chaotic comedy, Farcical comedy, Hilarious chaos, Veteran comedian, comedy legend, comedy of errors, drama teacher, drama graduate, psychological thriller, psychological drama, [comedy rock, comedy-rock], Comedy awards, meta comedy, comedy roast, Comedy movie, comedy performance, comedy world, comedic love, comedy tour, musical comedy, historical drama, political drama, Gritty drama, Comedy showcase, psychological game, comedy special, folk comedy, live comedy, raw humor, flat humor, Comedy Cellar, action-comedy]  \n",
       "3                                                                                                                                                                                                                                                             [elderly inmates, elderly, elderly woman, Elderly spinster, elderly Dorothy, Elderly widow, elderly love, elderly lady, elderly maid, elderly mother, elderly student, older woman, elderly gentleman, elderly man, Elderly uncle, Elderly father, Elderly daughter, elderly care, [senior, seniors], aging parents, elderly life, elderly parent, old age home, Aged care, [elder, elders], Elderly guardian, old man, grandfather, Older man, senior woman, Senior citizen, old lady, Elderly priest, grandson, elderly couple, elderly wrestler]  \n",
       "4                                                                                                                                                                             [strange phenomenon, strange events, strange phenomena, strange performance, mysterious deaths, mysterious events, enigmatic, enigmatic key, enigmatic figure, [enigma, enigmas], enigmatic woman, enigmatic past, mysterious object, Mysterious phenomenon, enigmatic castle, Enigmatic Writer, Unnatural occurrences, Unexplainable events, mysterious past, enigmatic event, mysterious incident, mysterious celebration, Weird events, strange encounters, strange things, strange couple, strange behavior, Inexplicable, strange places, bizarre, strange, weird, strange object, strange town, strange terrain, unexplained]  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gamma value: 16.0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>themeCount</th>\n",
       "      <th>themes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>40</td>\n",
       "      <td>[parents' home, family home, family agency, family provider, three families, [families, family], family secrets, Familial secrets, family memory, family ties, family traditions, family bonds, family roots, familial relationships, family relationships, family holiday, family wedding, family love, family life, Family pain, family values, family protection, family support, family tradition, family business, family bonding, Female bonding, family vacation home, family man, vacation friends, Family survival, family dynamics, Traditional values, conservative values, traditional family, family history, family dignity, Conservative views, family legend, family divisions]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>36</td>\n",
       "      <td>[strange phenomenon, strange events, strange phenomena, strange performance, mysterious deaths, mysterious events, enigmatic, enigmatic key, enigmatic figure, [enigma, enigmas], enigmatic woman, enigmatic past, mysterious object, Mysterious phenomenon, enigmatic castle, Enigmatic Writer, Unnatural occurrences, Unexplainable events, mysterious past, enigmatic event, mysterious incident, mysterious celebration, Weird events, strange encounters, strange things, strange couple, strange behavior, Inexplicable, strange places, bizarre, strange, weird, strange object, strange town, strange terrain, unexplained]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>32</td>\n",
       "      <td>[nightmare begins, [nightmare, nightmares], Surreal fantasy, Surreal dream, surreal, Surreal comedy, Surreal visions, surreal nightmare, Surreal building, Surreal associations, surreal flight, surreal dreams, surreal animation, surrealism, dreamlike reality, Surreal imagery, surrealist, surreal vision, action adventure, surreal adventure, Kafkaesque nightmare, Horrifying nightmare, surreal nightmares, bad dream, Mobile Nightmare, Psychedelic visions, Psychedelicized, subconscious nightmares, Escalating nightmare, Surreal Voyage, turbulent dreams, morbid dreams]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>32</td>\n",
       "      <td>[Dark realities, Dark interest, Dark realm, dark presence, Dark pasts, Darker truth, dark characters, inner darkness, dark place, darkness, Darkness beneath, lurking darkness, dark abyss, dark energy, dark force, dark, Deep dark, Dark Prison, dark moon, Dark Highway, dark house, dark room, dark themes, dark time, dark twist, dark scheme, Darkness to light, Dark Light, darkness vs. light, dark thoughts, Dark Void, dark tunnel]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>32</td>\n",
       "      <td>[broken family, family suffering, Parental Murder, family murder, family dysfunction, family struggles, family chaos, Family fragments, Interpersonal conflicts, family conflicts, family desperation, family problems, family struggle, family issues, family turmoil, family stability, family pressures, Family interference, Family rift, family disconnection, family tragedy, family death, Family deaths, family estrangement, family separation, Family collapse, personal problems, family trauma, family conflict, family tension, environmental trauma, emotional trauma]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   themeCount  \\\n",
       "0          40   \n",
       "1          36   \n",
       "2          32   \n",
       "3          32   \n",
       "4          32   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            themes  \n",
       "0  [parents' home, family home, family agency, family provider, three families, [families, family], family secrets, Familial secrets, family memory, family ties, family traditions, family bonds, family roots, familial relationships, family relationships, family holiday, family wedding, family love, family life, Family pain, family values, family protection, family support, family tradition, family business, family bonding, Female bonding, family vacation home, family man, vacation friends, Family survival, family dynamics, Traditional values, conservative values, traditional family, family history, family dignity, Conservative views, family legend, family divisions]  \n",
       "1                                                              [strange phenomenon, strange events, strange phenomena, strange performance, mysterious deaths, mysterious events, enigmatic, enigmatic key, enigmatic figure, [enigma, enigmas], enigmatic woman, enigmatic past, mysterious object, Mysterious phenomenon, enigmatic castle, Enigmatic Writer, Unnatural occurrences, Unexplainable events, mysterious past, enigmatic event, mysterious incident, mysterious celebration, Weird events, strange encounters, strange things, strange couple, strange behavior, Inexplicable, strange places, bizarre, strange, weird, strange object, strange town, strange terrain, unexplained]  \n",
       "2                                                                                                          [nightmare begins, [nightmare, nightmares], Surreal fantasy, Surreal dream, surreal, Surreal comedy, Surreal visions, surreal nightmare, Surreal building, Surreal associations, surreal flight, surreal dreams, surreal animation, surrealism, dreamlike reality, Surreal imagery, surrealist, surreal vision, action adventure, surreal adventure, Kafkaesque nightmare, Horrifying nightmare, surreal nightmares, bad dream, Mobile Nightmare, Psychedelic visions, Psychedelicized, subconscious nightmares, Escalating nightmare, Surreal Voyage, turbulent dreams, morbid dreams]  \n",
       "3                                                                                                                                                                                                                                                    [Dark realities, Dark interest, Dark realm, dark presence, Dark pasts, Darker truth, dark characters, inner darkness, dark place, darkness, Darkness beneath, lurking darkness, dark abyss, dark energy, dark force, dark, Deep dark, Dark Prison, dark moon, Dark Highway, dark house, dark room, dark themes, dark time, dark twist, dark scheme, Darkness to light, Dark Light, darkness vs. light, dark thoughts, Dark Void, dark tunnel]  \n",
       "4                                                                                                             [broken family, family suffering, Parental Murder, family murder, family dysfunction, family struggles, family chaos, Family fragments, Interpersonal conflicts, family conflicts, family desperation, family problems, family struggle, family issues, family turmoil, family stability, family pressures, Family interference, Family rift, family disconnection, family tragedy, family death, Family deaths, family estrangement, family separation, Family collapse, personal problems, family trauma, family conflict, family tension, environmental trauma, emotional trauma]  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gamma value: 32.0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>themeCount</th>\n",
       "      <th>themes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>33</td>\n",
       "      <td>[parents' home, family home, family agency, family provider, three families, [families, family], family ties, Family fragments, family bonds, family roots, familial relationships, family relationships, family holiday, family wedding, family love, family life, Family pain, family values, family protection, family support, Family interference, Family rift, family vacation home, family man, vacation friends, Family survival, family dynamics, Traditional values, conservative values, traditional family, family dignity, Conservative views, family divisions]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>32</td>\n",
       "      <td>[Dark realities, Dark interest, Dark realm, dark presence, Dark pasts, Darker truth, dark characters, inner darkness, dark place, darkness, Darkness beneath, lurking darkness, dark abyss, dark energy, dark force, dark, Deep dark, Dark Prison, dark moon, Dark Highway, dark house, dark room, dark themes, dark time, dark twist, dark scheme, Darkness to light, Dark Light, darkness vs. light, dark thoughts, Dark Void, dark tunnel]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>30</td>\n",
       "      <td>[Mom, [mother, mothers], Daughter's struggle, ailing mother, lost mother, Devoted mother, alcoholic mother, Irish mother, mother's departure, mother's sacrifice, mother's return, birth mother, Maternal, motherhood, mother's, mother's love, mother's quest, maternal love, new mother, motherhood struggles, mother-daughter, Mother-daughter vacation, young mother, Great Mother, mother's struggle, Mothers and daughters, Struggling mother, postpartum anxiety, postpartum struggles, Irish life]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>28</td>\n",
       "      <td>[Christmas magic, Christmas competition, christmas, Christmas traditions, Christmas celebration, Christmas decorations, Christmas spirit, Christmas mystery, Christmas story, Christmas tree, Xmas, Christmas ambience, Christmas terror, Christmas love, Christmas songs, Christmas nightmare, Christmas curse, Christmas wish, Christmas proposal, Christmas gift, Christmas carols, Christmas Eve wedding, Christmas break, Christmas Variety, Christmas setting, Christmas trees, Family Christmas, Christmas romance]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>28</td>\n",
       "      <td>[strange phenomenon, strange events, strange phenomena, strange performance, mysterious deaths, mysterious events, mysterious object, Mysterious phenomenon, Unnatural occurrences, Unexplainable events, mysterious past, enigmatic event, mysterious incident, mysterious celebration, Weird events, strange encounters, strange things, strange couple, strange behavior, Inexplicable, strange places, bizarre, strange, weird, strange object, strange town, strange terrain, unexplained]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   themeCount  \\\n",
       "0          33   \n",
       "1          32   \n",
       "2          30   \n",
       "3          28   \n",
       "4          28   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          themes  \n",
       "0  [parents' home, family home, family agency, family provider, three families, [families, family], family ties, Family fragments, family bonds, family roots, familial relationships, family relationships, family holiday, family wedding, family love, family life, Family pain, family values, family protection, family support, Family interference, Family rift, family vacation home, family man, vacation friends, Family survival, family dynamics, Traditional values, conservative values, traditional family, family dignity, Conservative views, family divisions]  \n",
       "1                                                                                                                                  [Dark realities, Dark interest, Dark realm, dark presence, Dark pasts, Darker truth, dark characters, inner darkness, dark place, darkness, Darkness beneath, lurking darkness, dark abyss, dark energy, dark force, dark, Deep dark, Dark Prison, dark moon, Dark Highway, dark house, dark room, dark themes, dark time, dark twist, dark scheme, Darkness to light, Dark Light, darkness vs. light, dark thoughts, Dark Void, dark tunnel]  \n",
       "2                                                                     [Mom, [mother, mothers], Daughter's struggle, ailing mother, lost mother, Devoted mother, alcoholic mother, Irish mother, mother's departure, mother's sacrifice, mother's return, birth mother, Maternal, motherhood, mother's, mother's love, mother's quest, maternal love, new mother, motherhood struggles, mother-daughter, Mother-daughter vacation, young mother, Great Mother, mother's struggle, Mothers and daughters, Struggling mother, postpartum anxiety, postpartum struggles, Irish life]  \n",
       "3                                                     [Christmas magic, Christmas competition, christmas, Christmas traditions, Christmas celebration, Christmas decorations, Christmas spirit, Christmas mystery, Christmas story, Christmas tree, Xmas, Christmas ambience, Christmas terror, Christmas love, Christmas songs, Christmas nightmare, Christmas curse, Christmas wish, Christmas proposal, Christmas gift, Christmas carols, Christmas Eve wedding, Christmas break, Christmas Variety, Christmas setting, Christmas trees, Family Christmas, Christmas romance]  \n",
       "4                                                                                [strange phenomenon, strange events, strange phenomena, strange performance, mysterious deaths, mysterious events, mysterious object, Mysterious phenomenon, Unnatural occurrences, Unexplainable events, mysterious past, enigmatic event, mysterious incident, mysterious celebration, Weird events, strange encounters, strange things, strange couple, strange behavior, Inexplicable, strange places, bizarre, strange, weird, strange object, strange town, strange terrain, unexplained]  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gamma value: 64.0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>themeCount</th>\n",
       "      <th>themes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>25</td>\n",
       "      <td>[indigenous peoples, Indigenous perspectives, Indigenous cultures, Indigenous scientists, indigenous culture, Aboriginal culture, indigenous community, Indigenous heritage, indigenous folklore, indigenous people, Indigenous identity, Indigenous trauma, Indigenous woman, Indigenous migration, Indigenous activism, Indigenous women, Indigenous sovereignty, Afro-Indigenous, Indigenous Australians, Indigenous roots, Indigenous history, indigenous territory, Indigenous language, intangible heritage, Indigenous beliefs]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>25</td>\n",
       "      <td>[Queer visions, Queer thoughts, queer art, queer, aging queer, genderqueer, queer youth, queer life, queer horror, queer history, queer girl, queer artist, queer nightlife, queerness, Queer lovers, Queer Sexuality, Queer mom, Queer eroticism, gender neutral, Queer mecca, queer representations, Queer struggles, Queer genealogy, Queer existences, Queer utopias]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>23</td>\n",
       "      <td>[art film, artist film, experimental film, Italian Filmmaker, experimental filmmaker, documentary filmmaker, experimental filmmakers, Experimental films, unfinished film, experimental cinema, [short film, short-film, shortfilm], Dedicated filmmaker, Missing filmmaker, Avant-garde cinema, budding filmmaker, Home film, original film, film within a film, silent film, flicker film, touching film, girl filmmaker, aspiring filmmaker]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>22</td>\n",
       "      <td>[Scary Tales, horror stories, teen horror, Real horror, [horror, horrors], body horror, psycho-horror, horror movie, horror convention, horror footage, Horror filming, horror film, Horror setting, Creeping horror, horror films, [horror comedy, horror-comedy], horror genre, Horror classic, Horror homage, horror realm, horror franchise, Horror legend]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>22</td>\n",
       "      <td>[Christmas magic, christmas, Christmas traditions, Christmas spirit, Christmas mystery, Christmas story, Christmas tree, Xmas, Christmas ambience, Christmas terror, Christmas love, Christmas songs, Christmas nightmare, Christmas curse, Christmas wish, Christmas proposal, Christmas gift, Christmas break, Christmas Variety, Christmas trees, Family Christmas, Christmas romance]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   themeCount  \\\n",
       "0          25   \n",
       "1          25   \n",
       "2          23   \n",
       "3          22   \n",
       "4          22   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   themes  \n",
       "0  [indigenous peoples, Indigenous perspectives, Indigenous cultures, Indigenous scientists, indigenous culture, Aboriginal culture, indigenous community, Indigenous heritage, indigenous folklore, indigenous people, Indigenous identity, Indigenous trauma, Indigenous woman, Indigenous migration, Indigenous activism, Indigenous women, Indigenous sovereignty, Afro-Indigenous, Indigenous Australians, Indigenous roots, Indigenous history, indigenous territory, Indigenous language, intangible heritage, Indigenous beliefs]  \n",
       "1                                                                                                                                                               [Queer visions, Queer thoughts, queer art, queer, aging queer, genderqueer, queer youth, queer life, queer horror, queer history, queer girl, queer artist, queer nightlife, queerness, Queer lovers, Queer Sexuality, Queer mom, Queer eroticism, gender neutral, Queer mecca, queer representations, Queer struggles, Queer genealogy, Queer existences, Queer utopias]  \n",
       "2                                                                                         [art film, artist film, experimental film, Italian Filmmaker, experimental filmmaker, documentary filmmaker, experimental filmmakers, Experimental films, unfinished film, experimental cinema, [short film, short-film, shortfilm], Dedicated filmmaker, Missing filmmaker, Avant-garde cinema, budding filmmaker, Home film, original film, film within a film, silent film, flicker film, touching film, girl filmmaker, aspiring filmmaker]  \n",
       "3                                                                                                                                                                         [Scary Tales, horror stories, teen horror, Real horror, [horror, horrors], body horror, psycho-horror, horror movie, horror convention, horror footage, Horror filming, horror film, Horror setting, Creeping horror, horror films, [horror comedy, horror-comedy], horror genre, Horror classic, Horror homage, horror realm, horror franchise, Horror legend]  \n",
       "4                                                                                                                                               [Christmas magic, christmas, Christmas traditions, Christmas spirit, Christmas mystery, Christmas story, Christmas tree, Xmas, Christmas ambience, Christmas terror, Christmas love, Christmas songs, Christmas nightmare, Christmas curse, Christmas wish, Christmas proposal, Christmas gift, Christmas break, Christmas Variety, Christmas trees, Family Christmas, Christmas romance]  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gamma value: 128.0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>themeCount</th>\n",
       "      <th>themes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>19</td>\n",
       "      <td>[queer, aging queer, genderqueer, queer youth, queer life, queer horror, queer history, queer girl, queer artist, queer nightlife, queerness, Queer lovers, Queer Sexuality, Queer mom, Queer eroticism, gender neutral, Queer mecca, Queer struggles, Queer genealogy]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>18</td>\n",
       "      <td>[1970, 1960s-1970s, 70s, 1970s, 1970s counterculture, 1970s Rome, 1970s era, seventies, 1970s Iran, 1970s Britain, 1960s, 1960s setting, 1960s NYC, 60s, 1976-1980, 60's, 70s glory, Eighties]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>18</td>\n",
       "      <td>[indigenous peoples, Indigenous perspectives, Indigenous cultures, Indigenous scientists, indigenous culture, Aboriginal culture, Indigenous heritage, indigenous folklore, Indigenous identity, Indigenous trauma, Indigenous woman, Indigenous migration, Indigenous Australians, Indigenous roots, Indigenous history, Indigenous language, intangible heritage, Indigenous beliefs]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>17</td>\n",
       "      <td>[Ancient identity, Ancient, Ancient power, ancient style, ancient world, ancient professions, ancient tradition, otherworldly powers, ancient powers, ancient curses, Ancient Crown, ancient origins, ancient myth, ancient literature, ancient spirit, ancient practice, Creation myth]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>17</td>\n",
       "      <td>[elderly inmates, elderly, elderly student, elderly gentleman, elderly man, elderly care, [senior, seniors], aging parents, elderly life, old age home, Aged care, old man, grandfather, Older man, grandson, elderly couple, elderly wrestler]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   themeCount  \\\n",
       "0          19   \n",
       "1          18   \n",
       "2          18   \n",
       "3          17   \n",
       "4          17   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                    themes  \n",
       "0                                                                                                                  [queer, aging queer, genderqueer, queer youth, queer life, queer horror, queer history, queer girl, queer artist, queer nightlife, queerness, Queer lovers, Queer Sexuality, Queer mom, Queer eroticism, gender neutral, Queer mecca, Queer struggles, Queer genealogy]  \n",
       "1                                                                                                                                                                                           [1970, 1960s-1970s, 70s, 1970s, 1970s counterculture, 1970s Rome, 1970s era, seventies, 1970s Iran, 1970s Britain, 1960s, 1960s setting, 1960s NYC, 60s, 1976-1980, 60's, 70s glory, Eighties]  \n",
       "2  [indigenous peoples, Indigenous perspectives, Indigenous cultures, Indigenous scientists, indigenous culture, Aboriginal culture, Indigenous heritage, indigenous folklore, Indigenous identity, Indigenous trauma, Indigenous woman, Indigenous migration, Indigenous Australians, Indigenous roots, Indigenous history, Indigenous language, intangible heritage, Indigenous beliefs]  \n",
       "3                                                                                                 [Ancient identity, Ancient, Ancient power, ancient style, ancient world, ancient professions, ancient tradition, otherworldly powers, ancient powers, ancient curses, Ancient Crown, ancient origins, ancient myth, ancient literature, ancient spirit, ancient practice, Creation myth]  \n",
       "4                                                                                                                                          [elderly inmates, elderly, elderly student, elderly gentleman, elderly man, elderly care, [senior, seniors], aging parents, elderly life, old age home, Aged care, old man, grandfather, Older man, grandson, elderly couple, elderly wrestler]  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gamma value: 256.0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>themeCount</th>\n",
       "      <th>themes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>15</td>\n",
       "      <td>[Twisted thriller, [twist, twists], Twisted love, twisted games, Unexpected turns, unexpected twist, twisted lesson, twisted, Strange twist, Twist ending, unexpected ending, gut-wrenching twists, time twist, harsh twist, twisted battle]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>14</td>\n",
       "      <td>[alien creature, mysterious creature, alien monster, classic monsters, [monster, monsters], giant monster, giant kraken, human monster, forest monster, miniature monsters, Monstro, deformed monster, animated monsters, Monster within]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>14</td>\n",
       "      <td>[Mom, [mother, mothers], birth mother, Maternal, motherhood, mother's, mother's love, maternal love, new mother, mother-daughter, Mother-daughter vacation, young mother, Great Mother, Mothers and daughters]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>14</td>\n",
       "      <td>[[comedies, comedy], comedy of errors, [comedy rock, comedy-rock], meta comedy, comedy roast, Comedy movie, comedy performance, comedy world, comedic love, comedy tour, musical comedy, folk comedy, live comedy, action-comedy]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>14</td>\n",
       "      <td>[1970, 1960s-1970s, 1970s, 1970s counterculture, 1970s Rome, 1970s era, 1970s Iran, 1970s Britain, 1960s, 1960s setting, 1960s NYC, 60s, 1976-1980, 60's]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   themeCount  \\\n",
       "0          15   \n",
       "1          14   \n",
       "2          14   \n",
       "3          14   \n",
       "4          14   \n",
       "\n",
       "                                                                                                                                                                                                                                         themes  \n",
       "0  [Twisted thriller, [twist, twists], Twisted love, twisted games, Unexpected turns, unexpected twist, twisted lesson, twisted, Strange twist, Twist ending, unexpected ending, gut-wrenching twists, time twist, harsh twist, twisted battle]  \n",
       "1     [alien creature, mysterious creature, alien monster, classic monsters, [monster, monsters], giant monster, giant kraken, human monster, forest monster, miniature monsters, Monstro, deformed monster, animated monsters, Monster within]  \n",
       "2                                [Mom, [mother, mothers], birth mother, Maternal, motherhood, mother's, mother's love, maternal love, new mother, mother-daughter, Mother-daughter vacation, young mother, Great Mother, Mothers and daughters]  \n",
       "3             [[comedies, comedy], comedy of errors, [comedy rock, comedy-rock], meta comedy, comedy roast, Comedy movie, comedy performance, comedy world, comedic love, comedy tour, musical comedy, folk comedy, live comedy, action-comedy]  \n",
       "4                                                                                     [1970, 1960s-1970s, 1970s, 1970s counterculture, 1970s Rome, 1970s era, 1970s Iran, 1970s Britain, 1960s, 1960s setting, 1960s NYC, 60s, 1976-1980, 60's]  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gamma value: 512.0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>themeCount</th>\n",
       "      <th>themes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>12</td>\n",
       "      <td>[ordinary life, routine life, structured life, home life, traditional life, easy life, monotonous life, quiet life, Busy life, modest life, simple life, Carefree life]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>11</td>\n",
       "      <td>[Haunted chapel, Haunted market, Haunted temple, Haunted terrace, Haunted carriage, Haunted manor, Haunted artifact, haunted hotel, Haunted pizzeria, haunted building, Gothic manor]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>11</td>\n",
       "      <td>[[comedies, comedy], Veteran comedian, comedy legend, comedy of errors, [comedy rock, comedy-rock], meta comedy, Comedy movie, comedy world, comedic love, comedy tour, musical comedy]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>10</td>\n",
       "      <td>[future world, future dystopia, future warfare, dystopian future, future catastrophe, Future crime, planetary future, future past, ancient future, chaotic future]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>10</td>\n",
       "      <td>[childhood, childhood friends, childhood roots, childhood game, childhood joy, childhood contemplation, childhood body, childhood photo, infância, childhood passion]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   themeCount  \\\n",
       "0          12   \n",
       "1          11   \n",
       "2          11   \n",
       "3          10   \n",
       "4          10   \n",
       "\n",
       "                                                                                                                                                                                    themes  \n",
       "0                  [ordinary life, routine life, structured life, home life, traditional life, easy life, monotonous life, quiet life, Busy life, modest life, simple life, Carefree life]  \n",
       "1    [Haunted chapel, Haunted market, Haunted temple, Haunted terrace, Haunted carriage, Haunted manor, Haunted artifact, haunted hotel, Haunted pizzeria, haunted building, Gothic manor]  \n",
       "2  [[comedies, comedy], Veteran comedian, comedy legend, comedy of errors, [comedy rock, comedy-rock], meta comedy, Comedy movie, comedy world, comedic love, comedy tour, musical comedy]  \n",
       "3                       [future world, future dystopia, future warfare, dystopian future, future catastrophe, Future crime, planetary future, future past, ancient future, chaotic future]  \n",
       "4                    [childhood, childhood friends, childhood roots, childhood game, childhood joy, childhood contemplation, childhood body, childhood photo, infância, childhood passion]  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gamma value: 1024.0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>themeCount</th>\n",
       "      <th>themes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>9</td>\n",
       "      <td>[novice dancer, professional dancer, go-go dancer, American dancer, Avant-garde dancer, dancer-choreographer, choreography, Dance diva, exotic dancer]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>9</td>\n",
       "      <td>[parents' home, family home, family agency, family provider, three families, [families, family], family vacation home, family man, vacation friends]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>9</td>\n",
       "      <td>[Wealth privilege, Wealthy man, wealthy, Wealthy lifestyles, Wealthy couple, Wealthy married woman, wealthy men, wealthy schoolkids, Wealth obsession]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8</td>\n",
       "      <td>[college memories, College move, college days, college enrollment, college sweetheart, college romance, college classmates, college fraternity]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>8</td>\n",
       "      <td>[unfinished film, [short film, short-film, shortfilm], Home film, original film, film within a film, silent film, flicker film, touching film]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   themeCount  \\\n",
       "0           9   \n",
       "1           9   \n",
       "2           9   \n",
       "3           8   \n",
       "4           8   \n",
       "\n",
       "                                                                                                                                                   themes  \n",
       "0  [novice dancer, professional dancer, go-go dancer, American dancer, Avant-garde dancer, dancer-choreographer, choreography, Dance diva, exotic dancer]  \n",
       "1    [parents' home, family home, family agency, family provider, three families, [families, family], family vacation home, family man, vacation friends]  \n",
       "2  [Wealth privilege, Wealthy man, wealthy, Wealthy lifestyles, Wealthy couple, Wealthy married woman, wealthy men, wealthy schoolkids, Wealth obsession]  \n",
       "3         [college memories, College move, college days, college enrollment, college sweetheart, college romance, college classmates, college fraternity]  \n",
       "4          [unfinished film, [short film, short-film, shortfilm], Home film, original film, film within a film, silent film, flicker film, touching film]  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for gamma, result in gamma_tests.items():\n",
    "    print(f\"Gamma value: {gamma}\")\n",
    "    display(result['biggest_communities'].head())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8eeee03b-9098-4395-88e6-0e4d8311c082",
   "metadata": {},
   "source": [
    "## For individual themes, see what other themes drop out of their communities as you increase gamma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "664ea39d-78f6-4e25-b48f-5110a5f00ba2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_proprety_df(property_name):\n",
    "    property_df = gds.graph.nodeProperties.stream(g_themes2,\n",
    "                                                  [property_name],\n",
    "                                                  db_node_properties = [\"description\", \"descriptions\"],\n",
    "                                                  separate_property_columns=True)\n",
    "    property_df.loc[pd.notna(property_df['descriptions']), 'descriptions'] = property_df.loc[pd.notna(property_df['descriptions']), 'descriptions'].map(lambda x: \", \".join(x))\n",
    "    property_df['description'] = property_df.apply(lambda row: row['description'] if row['description'] else row['descriptions'], axis=1)\n",
    "    property_df.drop('descriptions', axis=1, inplace=True)\n",
    "    return property_df\n",
    "\n",
    "property_dfs = {property_name: get_proprety_df(property_name)\n",
    "                for property_name in [f\"leidenGamma{gamma}\" for gamma in [32.0, 64.0, 128.0, 256.0, 512.0, 1024.0]]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "id": "b841dc7e-2205-4d56-a079-4c0949cb05d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_cluster_for_theme(property_name, theme):\n",
    "    theme_df = property_dfs[property_name]\n",
    "    result_list = []\n",
    "    if theme in theme_df['description'].to_list():\n",
    "        theme_cluster_id = theme_df.loc[theme_df['description']==theme, property_name].iloc[0]\n",
    "        result_list = theme_df.loc[theme_df[property_name]==theme_cluster_id, \"description\"].to_list()\n",
    "    return result_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "474c07ca-01b5-4038-9ed3-4ab1bbe7dee0",
   "metadata": {},
   "outputs": [],
   "source": [
    "def compare_gamma_for_theme(theme, gammas):\n",
    "    column_names = [f\"leidenGamma{gamma}\" for gamma in gammas]\n",
    "    lists = [get_cluster_for_theme(column, theme) for column in column_names]\n",
    "    output_df = pd.DataFrame()\n",
    "    output_df.loc[column_names[0], \"final_themes\"] = \", \".join(lists[0])\n",
    "    for i in range(1, len(gammas)):\n",
    "        output_df.loc[f\"{column_names[i-1]} to {column_names[i]}\", \"dropped_themes\"] = \", \".join([theme for theme in lists[i-1] if theme not in lists[i]])\n",
    "        output_df.loc[f\"{column_names[i-1]} to {column_names[i]}\", \"added_themes\"] = \", \".join([theme for theme in lists[i] if theme not in lists[i-1]])\n",
    "        output_df.loc[f\"{column_names[i-1]} to {column_names[i]}\", \"final_themes\"] = \", \".join(lists[i])\n",
    "    return output_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "id": "986ab496-d07c-4220-afd7-ce0535e9a7c6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>final_themes</th>\n",
       "      <th>dropped_themes</th>\n",
       "      <th>added_themes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>leidenGamma32.0</th>\n",
       "      <td>Christmas magic, Christmas competition, christmas, Christmas traditions, Christmas celebration, Christmas decorations, Christmas spirit, Christmas mystery, Christmas story, Christmas tree, Xmas, Christmas ambience, Christmas terror, Christmas love, Christmas songs, Christmas nightmare, Christmas curse, Christmas wish, Christmas proposal, Christmas gift, Christmas carols, Christmas Eve wedding, Christmas break, Christmas Variety, Christmas setting, Christmas trees, Family Christmas, Christmas romance</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma32.0 to leidenGamma64.0</th>\n",
       "      <td>Christmas magic, christmas, Christmas traditions, Christmas spirit, Christmas mystery, Christmas story, Christmas tree, Xmas, Christmas ambience, Christmas terror, Christmas love, Christmas songs, Christmas nightmare, Christmas curse, Christmas wish, Christmas proposal, Christmas gift, Christmas break, Christmas Variety, Christmas trees, Family Christmas, Christmas romance</td>\n",
       "      <td>Christmas competition, Christmas celebration, Christmas decorations, Christmas carols, Christmas Eve wedding, Christmas setting</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma64.0 to leidenGamma128.0</th>\n",
       "      <td>christmas, Xmas</td>\n",
       "      <td>Christmas magic, Christmas traditions, Christmas spirit, Christmas mystery, Christmas story, Christmas tree, Christmas ambience, Christmas terror, Christmas love, Christmas songs, Christmas nightmare, Christmas curse, Christmas wish, Christmas proposal, Christmas gift, Christmas break, Christmas Variety, Christmas trees, Family Christmas, Christmas romance</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma128.0 to leidenGamma256.0</th>\n",
       "      <td>christmas, Xmas</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma256.0 to leidenGamma512.0</th>\n",
       "      <td>christmas, Xmas</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma512.0 to leidenGamma1024.0</th>\n",
       "      <td>christmas, Xmas</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   final_themes  \\\n",
       "leidenGamma32.0                        Christmas magic, Christmas competition, christmas, Christmas traditions, Christmas celebration, Christmas decorations, Christmas spirit, Christmas mystery, Christmas story, Christmas tree, Xmas, Christmas ambience, Christmas terror, Christmas love, Christmas songs, Christmas nightmare, Christmas curse, Christmas wish, Christmas proposal, Christmas gift, Christmas carols, Christmas Eve wedding, Christmas break, Christmas Variety, Christmas setting, Christmas trees, Family Christmas, Christmas romance   \n",
       "leidenGamma32.0 to leidenGamma64.0                                                                                                                                      Christmas magic, christmas, Christmas traditions, Christmas spirit, Christmas mystery, Christmas story, Christmas tree, Xmas, Christmas ambience, Christmas terror, Christmas love, Christmas songs, Christmas nightmare, Christmas curse, Christmas wish, Christmas proposal, Christmas gift, Christmas break, Christmas Variety, Christmas trees, Family Christmas, Christmas romance   \n",
       "leidenGamma64.0 to leidenGamma128.0                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             christmas, Xmas   \n",
       "leidenGamma128.0 to leidenGamma256.0                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            christmas, Xmas   \n",
       "leidenGamma256.0 to leidenGamma512.0                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            christmas, Xmas   \n",
       "leidenGamma512.0 to leidenGamma1024.0                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           christmas, Xmas   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                               dropped_themes  \\\n",
       "leidenGamma32.0                                                                                                                                                                                                                                                                                                                                                                                           NaN   \n",
       "leidenGamma32.0 to leidenGamma64.0                                                                                                                                                                                                                                            Christmas competition, Christmas celebration, Christmas decorations, Christmas carols, Christmas Eve wedding, Christmas setting   \n",
       "leidenGamma64.0 to leidenGamma128.0    Christmas magic, Christmas traditions, Christmas spirit, Christmas mystery, Christmas story, Christmas tree, Christmas ambience, Christmas terror, Christmas love, Christmas songs, Christmas nightmare, Christmas curse, Christmas wish, Christmas proposal, Christmas gift, Christmas break, Christmas Variety, Christmas trees, Family Christmas, Christmas romance   \n",
       "leidenGamma128.0 to leidenGamma256.0                                                                                                                                                                                                                                                                                                                                                                            \n",
       "leidenGamma256.0 to leidenGamma512.0                                                                                                                                                                                                                                                                                                                                                                            \n",
       "leidenGamma512.0 to leidenGamma1024.0                                                                                                                                                                                                                                                                                                                                                                           \n",
       "\n",
       "                                      added_themes  \n",
       "leidenGamma32.0                                NaN  \n",
       "leidenGamma32.0 to leidenGamma64.0                  \n",
       "leidenGamma64.0 to leidenGamma128.0                 \n",
       "leidenGamma128.0 to leidenGamma256.0                \n",
       "leidenGamma256.0 to leidenGamma512.0                \n",
       "leidenGamma512.0 to leidenGamma1024.0               "
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compare_gamma_for_theme(\"christmas\", [32.0, 64.0, 128.0, 256.0, 512.0, 1024.0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "d1033f18-1106-4114-b03a-540b2b5f6a4f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>final_themes</th>\n",
       "      <th>dropped_themes</th>\n",
       "      <th>added_themes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>leidenGamma32.0</th>\n",
       "      <td>The Beatles, Beatles</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma32.0 to leidenGamma64.0</th>\n",
       "      <td>The Beatles, Beatles</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma64.0 to leidenGamma128.0</th>\n",
       "      <td>The Beatles, Beatles</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma128.0 to leidenGamma256.0</th>\n",
       "      <td>The Beatles, Beatles</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma256.0 to leidenGamma512.0</th>\n",
       "      <td>The Beatles, Beatles</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma512.0 to leidenGamma1024.0</th>\n",
       "      <td>The Beatles, Beatles</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               final_themes dropped_themes  \\\n",
       "leidenGamma32.0                        The Beatles, Beatles            NaN   \n",
       "leidenGamma32.0 to leidenGamma64.0     The Beatles, Beatles                  \n",
       "leidenGamma64.0 to leidenGamma128.0    The Beatles, Beatles                  \n",
       "leidenGamma128.0 to leidenGamma256.0   The Beatles, Beatles                  \n",
       "leidenGamma256.0 to leidenGamma512.0   The Beatles, Beatles                  \n",
       "leidenGamma512.0 to leidenGamma1024.0  The Beatles, Beatles                  \n",
       "\n",
       "                                      added_themes  \n",
       "leidenGamma32.0                                NaN  \n",
       "leidenGamma32.0 to leidenGamma64.0                  \n",
       "leidenGamma64.0 to leidenGamma128.0                 \n",
       "leidenGamma128.0 to leidenGamma256.0                \n",
       "leidenGamma256.0 to leidenGamma512.0                \n",
       "leidenGamma512.0 to leidenGamma1024.0               "
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compare_gamma_for_theme(\"Beatles\", [32.0, 64.0, 128.0, 256.0, 512.0, 1024.0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "fd16c902-b418-4847-a529-1163772def09",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>final_themes</th>\n",
       "      <th>dropped_themes</th>\n",
       "      <th>added_themes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>leidenGamma32.0</th>\n",
       "      <td>Family baggage, Family mystery, familial bonds, Family bond, Family antics, Family adventure, family secret, Family conversations, Family story, family expectations, family reflections, family move, family journey, family visit, Family culture</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma32.0 to leidenGamma64.0</th>\n",
       "      <td>Family baggage, Family mystery, familial bonds, Family bond, Family antics, Family adventure, family secret, Family conversations, Family story, family expectations, family reflections, family move, family journey, family visit, Family culture</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma64.0 to leidenGamma128.0</th>\n",
       "      <td>Family baggage, Family mystery, familial bonds, Family bond, Family antics, Family adventure, family secret, Family conversations, Family story, family expectations, family reflections, family move, family journey, family visit, Family culture</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma128.0 to leidenGamma256.0</th>\n",
       "      <td>Family baggage, Family mystery, familial bonds, Family bond, Family antics, family secret, Family conversations, Family story</td>\n",
       "      <td>Family adventure, family expectations, family reflections, family move, family journey, family visit, Family culture</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma256.0 to leidenGamma512.0</th>\n",
       "      <td>Family baggage, Family mystery, familial bonds, Family bond, Family antics, family secret, Family conversations, Family story</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma512.0 to leidenGamma1024.0</th>\n",
       "      <td>familial bonds, Family bond, Family conversations</td>\n",
       "      <td>Family baggage, Family mystery, Family antics, family secret, Family story</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                                                                                              final_themes  \\\n",
       "leidenGamma32.0                        Family baggage, Family mystery, familial bonds, Family bond, Family antics, Family adventure, family secret, Family conversations, Family story, family expectations, family reflections, family move, family journey, family visit, Family culture   \n",
       "leidenGamma32.0 to leidenGamma64.0     Family baggage, Family mystery, familial bonds, Family bond, Family antics, Family adventure, family secret, Family conversations, Family story, family expectations, family reflections, family move, family journey, family visit, Family culture   \n",
       "leidenGamma64.0 to leidenGamma128.0    Family baggage, Family mystery, familial bonds, Family bond, Family antics, Family adventure, family secret, Family conversations, Family story, family expectations, family reflections, family move, family journey, family visit, Family culture   \n",
       "leidenGamma128.0 to leidenGamma256.0                                                                                                                         Family baggage, Family mystery, familial bonds, Family bond, Family antics, family secret, Family conversations, Family story   \n",
       "leidenGamma256.0 to leidenGamma512.0                                                                                                                         Family baggage, Family mystery, familial bonds, Family bond, Family antics, family secret, Family conversations, Family story   \n",
       "leidenGamma512.0 to leidenGamma1024.0                                                                                                                                                                                                    familial bonds, Family bond, Family conversations   \n",
       "\n",
       "                                                                                                                                             dropped_themes  \\\n",
       "leidenGamma32.0                                                                                                                                         NaN   \n",
       "leidenGamma32.0 to leidenGamma64.0                                                                                                                            \n",
       "leidenGamma64.0 to leidenGamma128.0                                                                                                                           \n",
       "leidenGamma128.0 to leidenGamma256.0   Family adventure, family expectations, family reflections, family move, family journey, family visit, Family culture   \n",
       "leidenGamma256.0 to leidenGamma512.0                                                                                                                          \n",
       "leidenGamma512.0 to leidenGamma1024.0                                            Family baggage, Family mystery, Family antics, family secret, Family story   \n",
       "\n",
       "                                      added_themes  \n",
       "leidenGamma32.0                                NaN  \n",
       "leidenGamma32.0 to leidenGamma64.0                  \n",
       "leidenGamma64.0 to leidenGamma128.0                 \n",
       "leidenGamma128.0 to leidenGamma256.0                \n",
       "leidenGamma256.0 to leidenGamma512.0                \n",
       "leidenGamma512.0 to leidenGamma1024.0               "
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compare_gamma_for_theme(\"Family bond\", [32.0, 64.0, 128.0, 256.0, 512.0, 1024.0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "607fa52e-aaa5-4c9f-9935-9949de17ed79",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>final_themes</th>\n",
       "      <th>dropped_themes</th>\n",
       "      <th>added_themes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>leidenGamma32.0</th>\n",
       "      <td>Dark realities, Dark interest, Dark realm, dark presence, Dark pasts, Darker truth, dark characters, inner darkness, dark place, darkness, Darkness beneath, lurking darkness, dark abyss, dark energy, dark force, dark, Deep dark, Dark Prison, dark moon, Dark Highway, dark house, dark room, dark themes, dark time, dark twist, dark scheme, Darkness to light, Dark Light, darkness vs. light, dark thoughts, Dark Void, dark tunnel</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma32.0 to leidenGamma64.0</th>\n",
       "      <td>inner darkness, dark place, darkness, Darkness beneath, lurking darkness, dark abyss, dark energy, dark force, dark, Deep dark, dark moon, dark house, dark room, dark themes, dark time, dark twist, dark scheme, darkness vs. light, dark thoughts, dark tunnel</td>\n",
       "      <td>Dark realities, Dark interest, Dark realm, dark presence, Dark pasts, Darker truth, dark characters, Dark Prison, Dark Highway, Darkness to light, Dark Light, Dark Void</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma64.0 to leidenGamma128.0</th>\n",
       "      <td>inner darkness, dark place, darkness, Darkness beneath, lurking darkness, dark abyss, dark moon, dark house, dark room, dark themes, dark time, dark twist, dark scheme, darkness vs. light, dark thoughts, dark tunnel</td>\n",
       "      <td>dark energy, dark force, dark, Deep dark</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma128.0 to leidenGamma256.0</th>\n",
       "      <td>dark place, dark abyss, dark moon, dark house, dark room, dark themes, dark time, dark twist, dark scheme, dark thoughts, dark tunnel</td>\n",
       "      <td>inner darkness, darkness, Darkness beneath, lurking darkness, darkness vs. light</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma256.0 to leidenGamma512.0</th>\n",
       "      <td>dark place, dark abyss, dark house, dark room, dark twist, dark scheme</td>\n",
       "      <td>dark moon, dark themes, dark time, dark thoughts, dark tunnel</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma512.0 to leidenGamma1024.0</th>\n",
       "      <td>dark place, dark house, dark room, dark twist, dark scheme</td>\n",
       "      <td>dark abyss</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                      final_themes  \\\n",
       "leidenGamma32.0                        Dark realities, Dark interest, Dark realm, dark presence, Dark pasts, Darker truth, dark characters, inner darkness, dark place, darkness, Darkness beneath, lurking darkness, dark abyss, dark energy, dark force, dark, Deep dark, Dark Prison, dark moon, Dark Highway, dark house, dark room, dark themes, dark time, dark twist, dark scheme, Darkness to light, Dark Light, darkness vs. light, dark thoughts, Dark Void, dark tunnel   \n",
       "leidenGamma32.0 to leidenGamma64.0                                                                                                                                                                               inner darkness, dark place, darkness, Darkness beneath, lurking darkness, dark abyss, dark energy, dark force, dark, Deep dark, dark moon, dark house, dark room, dark themes, dark time, dark twist, dark scheme, darkness vs. light, dark thoughts, dark tunnel   \n",
       "leidenGamma64.0 to leidenGamma128.0                                                                                                                                                                                                                        inner darkness, dark place, darkness, Darkness beneath, lurking darkness, dark abyss, dark moon, dark house, dark room, dark themes, dark time, dark twist, dark scheme, darkness vs. light, dark thoughts, dark tunnel   \n",
       "leidenGamma128.0 to leidenGamma256.0                                                                                                                                                                                                                                                                                                         dark place, dark abyss, dark moon, dark house, dark room, dark themes, dark time, dark twist, dark scheme, dark thoughts, dark tunnel   \n",
       "leidenGamma256.0 to leidenGamma512.0                                                                                                                                                                                                                                                                                                                                                                        dark place, dark abyss, dark house, dark room, dark twist, dark scheme   \n",
       "leidenGamma512.0 to leidenGamma1024.0                                                                                                                                                                                                                                                                                                                                                                                   dark place, dark house, dark room, dark twist, dark scheme   \n",
       "\n",
       "                                                                                                                                                                                                 dropped_themes  \\\n",
       "leidenGamma32.0                                                                                                                                                                                             NaN   \n",
       "leidenGamma32.0 to leidenGamma64.0     Dark realities, Dark interest, Dark realm, dark presence, Dark pasts, Darker truth, dark characters, Dark Prison, Dark Highway, Darkness to light, Dark Light, Dark Void   \n",
       "leidenGamma64.0 to leidenGamma128.0                                                                                                                                    dark energy, dark force, dark, Deep dark   \n",
       "leidenGamma128.0 to leidenGamma256.0                                                                                           inner darkness, darkness, Darkness beneath, lurking darkness, darkness vs. light   \n",
       "leidenGamma256.0 to leidenGamma512.0                                                                                                              dark moon, dark themes, dark time, dark thoughts, dark tunnel   \n",
       "leidenGamma512.0 to leidenGamma1024.0                                                                                                                                                                dark abyss   \n",
       "\n",
       "                                      added_themes  \n",
       "leidenGamma32.0                                NaN  \n",
       "leidenGamma32.0 to leidenGamma64.0                  \n",
       "leidenGamma64.0 to leidenGamma128.0                 \n",
       "leidenGamma128.0 to leidenGamma256.0                \n",
       "leidenGamma256.0 to leidenGamma512.0                \n",
       "leidenGamma512.0 to leidenGamma1024.0               "
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compare_gamma_for_theme(\"dark place\", [32.0, 64.0, 128.0, 256.0, 512.0, 1024.0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "053fdccb-eadb-41a7-82b0-3ecfef951faa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>final_themes</th>\n",
       "      <th>dropped_themes</th>\n",
       "      <th>added_themes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>leidenGamma32.0</th>\n",
       "      <td>lgbtq identity, lgbtq+ identity, Gay dating, Gay community, lgbtq, lgbtq+, LGBTQ struggles, LGBTQIA+ community, LGBTQ2IA+, Gay/Lesbian, LGBTQ activism, LGBTQ+ pageant</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma32.0 to leidenGamma64.0</th>\n",
       "      <td>lgbtq identity, lgbtq+ identity, Gay dating, Gay community, lgbtq, lgbtq+, LGBTQ struggles, LGBTQIA+ community, LGBTQ2IA+, Gay/Lesbian, LGBTQ activism, LGBTQ+ pageant</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma64.0 to leidenGamma128.0</th>\n",
       "      <td>lgbtq identity, lgbtq+ identity, lgbtq, lgbtq+, LGBTQ struggles, LGBTQIA+ community, LGBTQ2IA+, Gay/Lesbian, LGBTQ activism, LGBTQ+ pageant</td>\n",
       "      <td>Gay dating, Gay community</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma128.0 to leidenGamma256.0</th>\n",
       "      <td>lgbtq identity, lgbtq+ identity, lgbtq, lgbtq+, LGBTQ struggles, LGBTQIA+ community, LGBTQ2IA+, Gay/Lesbian, LGBTQ activism, LGBTQ+ pageant</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma256.0 to leidenGamma512.0</th>\n",
       "      <td>lgbtq, lgbtq+, LGBTQIA+ community, LGBTQ2IA+, Gay/Lesbian, LGBTQ+ pageant</td>\n",
       "      <td>lgbtq identity, lgbtq+ identity, LGBTQ struggles, LGBTQ activism</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma512.0 to leidenGamma1024.0</th>\n",
       "      <td>lgbtq, lgbtq+, Gay/Lesbian, LGBTQ+ pageant</td>\n",
       "      <td>LGBTQIA+ community, LGBTQ2IA+</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                 final_themes  \\\n",
       "leidenGamma32.0                        lgbtq identity, lgbtq+ identity, Gay dating, Gay community, lgbtq, lgbtq+, LGBTQ struggles, LGBTQIA+ community, LGBTQ2IA+, Gay/Lesbian, LGBTQ activism, LGBTQ+ pageant   \n",
       "leidenGamma32.0 to leidenGamma64.0     lgbtq identity, lgbtq+ identity, Gay dating, Gay community, lgbtq, lgbtq+, LGBTQ struggles, LGBTQIA+ community, LGBTQ2IA+, Gay/Lesbian, LGBTQ activism, LGBTQ+ pageant   \n",
       "leidenGamma64.0 to leidenGamma128.0                               lgbtq identity, lgbtq+ identity, lgbtq, lgbtq+, LGBTQ struggles, LGBTQIA+ community, LGBTQ2IA+, Gay/Lesbian, LGBTQ activism, LGBTQ+ pageant   \n",
       "leidenGamma128.0 to leidenGamma256.0                              lgbtq identity, lgbtq+ identity, lgbtq, lgbtq+, LGBTQ struggles, LGBTQIA+ community, LGBTQ2IA+, Gay/Lesbian, LGBTQ activism, LGBTQ+ pageant   \n",
       "leidenGamma256.0 to leidenGamma512.0                                                                                                lgbtq, lgbtq+, LGBTQIA+ community, LGBTQ2IA+, Gay/Lesbian, LGBTQ+ pageant   \n",
       "leidenGamma512.0 to leidenGamma1024.0                                                                                                                              lgbtq, lgbtq+, Gay/Lesbian, LGBTQ+ pageant   \n",
       "\n",
       "                                                                                         dropped_themes  \\\n",
       "leidenGamma32.0                                                                                     NaN   \n",
       "leidenGamma32.0 to leidenGamma64.0                                                                        \n",
       "leidenGamma64.0 to leidenGamma128.0                                           Gay dating, Gay community   \n",
       "leidenGamma128.0 to leidenGamma256.0                                                                      \n",
       "leidenGamma256.0 to leidenGamma512.0   lgbtq identity, lgbtq+ identity, LGBTQ struggles, LGBTQ activism   \n",
       "leidenGamma512.0 to leidenGamma1024.0                                     LGBTQIA+ community, LGBTQ2IA+   \n",
       "\n",
       "                                      added_themes  \n",
       "leidenGamma32.0                                NaN  \n",
       "leidenGamma32.0 to leidenGamma64.0                  \n",
       "leidenGamma64.0 to leidenGamma128.0                 \n",
       "leidenGamma128.0 to leidenGamma256.0                \n",
       "leidenGamma256.0 to leidenGamma512.0                \n",
       "leidenGamma512.0 to leidenGamma1024.0               "
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compare_gamma_for_theme(\"lgbtq, lgbtq+\", [32.0, 64.0, 128.0, 256.0, 512.0, 1024.0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "abee2f29-8d5f-4b20-9bec-dabbe0412e2f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>final_themes</th>\n",
       "      <th>dropped_themes</th>\n",
       "      <th>added_themes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>leidenGamma32.0</th>\n",
       "      <td>best friend, strong friendship, childhood friendship, male friendship, intergenerational friendship, female friendship, girl friendships, unusual friendship, friendship, friendships, friendship bond</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma32.0 to leidenGamma64.0</th>\n",
       "      <td>best friend, strong friendship, male friendship, female friendship, girl friendships, unusual friendship, friendship, friendships, friendship bond</td>\n",
       "      <td>childhood friendship, intergenerational friendship</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma64.0 to leidenGamma128.0</th>\n",
       "      <td>best friend, strong friendship, male friendship, female friendship, girl friendships, unusual friendship, friendship, friendships, friendship bond</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma128.0 to leidenGamma256.0</th>\n",
       "      <td>best friend, strong friendship, unusual friendship, friendship, friendships, friendship bond</td>\n",
       "      <td>male friendship, female friendship, girl friendships</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma256.0 to leidenGamma512.0</th>\n",
       "      <td>best friend, strong friendship, unusual friendship, friendship, friendships, friendship bond</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma512.0 to leidenGamma1024.0</th>\n",
       "      <td>friendship, friendships, friendship bond</td>\n",
       "      <td>best friend, strong friendship, unusual friendship</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                                                 final_themes  \\\n",
       "leidenGamma32.0                        best friend, strong friendship, childhood friendship, male friendship, intergenerational friendship, female friendship, girl friendships, unusual friendship, friendship, friendships, friendship bond   \n",
       "leidenGamma32.0 to leidenGamma64.0                                                         best friend, strong friendship, male friendship, female friendship, girl friendships, unusual friendship, friendship, friendships, friendship bond   \n",
       "leidenGamma64.0 to leidenGamma128.0                                                        best friend, strong friendship, male friendship, female friendship, girl friendships, unusual friendship, friendship, friendships, friendship bond   \n",
       "leidenGamma128.0 to leidenGamma256.0                                                                                                             best friend, strong friendship, unusual friendship, friendship, friendships, friendship bond   \n",
       "leidenGamma256.0 to leidenGamma512.0                                                                                                             best friend, strong friendship, unusual friendship, friendship, friendships, friendship bond   \n",
       "leidenGamma512.0 to leidenGamma1024.0                                                                                                                                                                friendship, friendships, friendship bond   \n",
       "\n",
       "                                                                             dropped_themes  \\\n",
       "leidenGamma32.0                                                                         NaN   \n",
       "leidenGamma32.0 to leidenGamma64.0       childhood friendship, intergenerational friendship   \n",
       "leidenGamma64.0 to leidenGamma128.0                                                           \n",
       "leidenGamma128.0 to leidenGamma256.0   male friendship, female friendship, girl friendships   \n",
       "leidenGamma256.0 to leidenGamma512.0                                                          \n",
       "leidenGamma512.0 to leidenGamma1024.0    best friend, strong friendship, unusual friendship   \n",
       "\n",
       "                                      added_themes  \n",
       "leidenGamma32.0                                NaN  \n",
       "leidenGamma32.0 to leidenGamma64.0                  \n",
       "leidenGamma64.0 to leidenGamma128.0                 \n",
       "leidenGamma128.0 to leidenGamma256.0                \n",
       "leidenGamma256.0 to leidenGamma512.0                \n",
       "leidenGamma512.0 to leidenGamma1024.0               "
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compare_gamma_for_theme(\"friendship, friendships\", [32.0, 64.0, 128.0, 256.0, 512.0, 1024.0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "748ed6eb-abf4-43f0-85b0-adc35f11d9b0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>final_themes</th>\n",
       "      <th>dropped_themes</th>\n",
       "      <th>added_themes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>leidenGamma32.0</th>\n",
       "      <td>best friends, school friends, close friends, high school friends, high school, high-school, highschool, college friends, High school hierarchy, rich friends, friend group</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma32.0 to leidenGamma64.0</th>\n",
       "      <td>best friends, school friends, close friends, high school friends, high school, high-school, highschool, college friends, High school hierarchy, rich friends, friend group</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma64.0 to leidenGamma128.0</th>\n",
       "      <td>school friends, high school friends, high school, high-school, highschool, college friends, High school hierarchy, friend group</td>\n",
       "      <td>best friends, close friends, rich friends</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma128.0 to leidenGamma256.0</th>\n",
       "      <td>school friends, high school friends, high school, high-school, highschool, college friends, High school hierarchy, friend group</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma256.0 to leidenGamma512.0</th>\n",
       "      <td>school friends, high school friends, college friends, friend group</td>\n",
       "      <td>high school, high-school, highschool, High school hierarchy</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma512.0 to leidenGamma1024.0</th>\n",
       "      <td>school friends, high school friends, college friends, friend group</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                     final_themes  \\\n",
       "leidenGamma32.0                        best friends, school friends, close friends, high school friends, high school, high-school, highschool, college friends, High school hierarchy, rich friends, friend group   \n",
       "leidenGamma32.0 to leidenGamma64.0     best friends, school friends, close friends, high school friends, high school, high-school, highschool, college friends, High school hierarchy, rich friends, friend group   \n",
       "leidenGamma64.0 to leidenGamma128.0                                               school friends, high school friends, high school, high-school, highschool, college friends, High school hierarchy, friend group   \n",
       "leidenGamma128.0 to leidenGamma256.0                                              school friends, high school friends, high school, high-school, highschool, college friends, High school hierarchy, friend group   \n",
       "leidenGamma256.0 to leidenGamma512.0                                                                                                           school friends, high school friends, college friends, friend group   \n",
       "leidenGamma512.0 to leidenGamma1024.0                                                                                                          school friends, high school friends, college friends, friend group   \n",
       "\n",
       "                                                                                    dropped_themes  \\\n",
       "leidenGamma32.0                                                                                NaN   \n",
       "leidenGamma32.0 to leidenGamma64.0                                                                   \n",
       "leidenGamma64.0 to leidenGamma128.0                      best friends, close friends, rich friends   \n",
       "leidenGamma128.0 to leidenGamma256.0                                                                 \n",
       "leidenGamma256.0 to leidenGamma512.0   high school, high-school, highschool, High school hierarchy   \n",
       "leidenGamma512.0 to leidenGamma1024.0                                                                \n",
       "\n",
       "                                      added_themes  \n",
       "leidenGamma32.0                                NaN  \n",
       "leidenGamma32.0 to leidenGamma64.0                  \n",
       "leidenGamma64.0 to leidenGamma128.0                 \n",
       "leidenGamma128.0 to leidenGamma256.0                \n",
       "leidenGamma256.0 to leidenGamma512.0                \n",
       "leidenGamma512.0 to leidenGamma1024.0               "
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compare_gamma_for_theme(\"high school friends\", [32.0, 64.0, 128.0, 256.0, 512.0, 1024.0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "97397fe5-d8ce-4b49-a3a5-f8faaccd7322",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>final_themes</th>\n",
       "      <th>dropped_themes</th>\n",
       "      <th>added_themes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>leidenGamma32.0</th>\n",
       "      <td>Musician's journey, musician, musicians, Old musician, cult musician, busker, street musician, LA musician, guitarist, troubled musician</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma32.0 to leidenGamma64.0</th>\n",
       "      <td>Musician's journey, musician, musicians, Old musician, cult musician, busker, street musician, LA musician, guitarist, troubled musician</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma64.0 to leidenGamma128.0</th>\n",
       "      <td>Musician's journey, musician, musicians, Old musician, cult musician, busker, street musician, LA musician, guitarist, troubled musician</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma128.0 to leidenGamma256.0</th>\n",
       "      <td>Musician's journey, musician, musicians, Old musician, cult musician, busker, street musician, LA musician, guitarist, troubled musician</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma256.0 to leidenGamma512.0</th>\n",
       "      <td>Musician's journey, musician, musicians, Old musician, cult musician, busker, street musician, LA musician, guitarist, troubled musician</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma512.0 to leidenGamma1024.0</th>\n",
       "      <td>Musician's journey, musician, musicians, LA musician, guitarist</td>\n",
       "      <td>Old musician, cult musician, busker, street musician, troubled musician</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                   final_themes  \\\n",
       "leidenGamma32.0                        Musician's journey, musician, musicians, Old musician, cult musician, busker, street musician, LA musician, guitarist, troubled musician   \n",
       "leidenGamma32.0 to leidenGamma64.0     Musician's journey, musician, musicians, Old musician, cult musician, busker, street musician, LA musician, guitarist, troubled musician   \n",
       "leidenGamma64.0 to leidenGamma128.0    Musician's journey, musician, musicians, Old musician, cult musician, busker, street musician, LA musician, guitarist, troubled musician   \n",
       "leidenGamma128.0 to leidenGamma256.0   Musician's journey, musician, musicians, Old musician, cult musician, busker, street musician, LA musician, guitarist, troubled musician   \n",
       "leidenGamma256.0 to leidenGamma512.0   Musician's journey, musician, musicians, Old musician, cult musician, busker, street musician, LA musician, guitarist, troubled musician   \n",
       "leidenGamma512.0 to leidenGamma1024.0                                                                           Musician's journey, musician, musicians, LA musician, guitarist   \n",
       "\n",
       "                                                                                                dropped_themes  \\\n",
       "leidenGamma32.0                                                                                            NaN   \n",
       "leidenGamma32.0 to leidenGamma64.0                                                                               \n",
       "leidenGamma64.0 to leidenGamma128.0                                                                              \n",
       "leidenGamma128.0 to leidenGamma256.0                                                                             \n",
       "leidenGamma256.0 to leidenGamma512.0                                                                             \n",
       "leidenGamma512.0 to leidenGamma1024.0  Old musician, cult musician, busker, street musician, troubled musician   \n",
       "\n",
       "                                      added_themes  \n",
       "leidenGamma32.0                                NaN  \n",
       "leidenGamma32.0 to leidenGamma64.0                  \n",
       "leidenGamma64.0 to leidenGamma128.0                 \n",
       "leidenGamma128.0 to leidenGamma256.0                \n",
       "leidenGamma256.0 to leidenGamma512.0                \n",
       "leidenGamma512.0 to leidenGamma1024.0               "
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compare_gamma_for_theme(\"musician, musicians\", [32.0, 64.0, 128.0, 256.0, 512.0, 1024.0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "45bac8f5-a2a7-48e2-89aa-29afaa3be1b3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>final_themes</th>\n",
       "      <th>dropped_themes</th>\n",
       "      <th>added_themes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>leidenGamma32.0</th>\n",
       "      <td>horse, horses, horse dreams, horseback, white horses, real horse</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma32.0 to leidenGamma64.0</th>\n",
       "      <td>horse, horses, horse dreams, horseback, white horses, real horse</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma64.0 to leidenGamma128.0</th>\n",
       "      <td>horse, horses, horse dreams, horseback, white horses, real horse</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma128.0 to leidenGamma256.0</th>\n",
       "      <td>horse, horses, horse dreams, horseback, white horses, real horse</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma256.0 to leidenGamma512.0</th>\n",
       "      <td>horse, horses, horse dreams, horseback, white horses, real horse</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>leidenGamma512.0 to leidenGamma1024.0</th>\n",
       "      <td>horse, horses, horse dreams, horseback, white horses, real horse</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                           final_themes  \\\n",
       "leidenGamma32.0                        horse, horses, horse dreams, horseback, white horses, real horse   \n",
       "leidenGamma32.0 to leidenGamma64.0     horse, horses, horse dreams, horseback, white horses, real horse   \n",
       "leidenGamma64.0 to leidenGamma128.0    horse, horses, horse dreams, horseback, white horses, real horse   \n",
       "leidenGamma128.0 to leidenGamma256.0   horse, horses, horse dreams, horseback, white horses, real horse   \n",
       "leidenGamma256.0 to leidenGamma512.0   horse, horses, horse dreams, horseback, white horses, real horse   \n",
       "leidenGamma512.0 to leidenGamma1024.0  horse, horses, horse dreams, horseback, white horses, real horse   \n",
       "\n",
       "                                      dropped_themes added_themes  \n",
       "leidenGamma32.0                                  NaN          NaN  \n",
       "leidenGamma32.0 to leidenGamma64.0                                 \n",
       "leidenGamma64.0 to leidenGamma128.0                                \n",
       "leidenGamma128.0 to leidenGamma256.0                               \n",
       "leidenGamma256.0 to leidenGamma512.0                               \n",
       "leidenGamma512.0 to leidenGamma1024.0                              "
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "compare_gamma_for_theme(\"horse, horses\", [32.0, 64.0, 128.0, 256.0, 512.0, 1024.0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e61f2073-79d2-4187-aa19-a479a200f4a2",
   "metadata": {},
   "source": [
    "# Write best Leiden communities to Neo4j\n",
    "Based on the examples above, 512.0 seems like a good level for gamma.\n",
    "Rename the property LeidenGamma32.0 from the graph projection to leidenId in the on-disk graph.\n",
    "Drop any leidenId values from a previous run."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "4ab04a7e-5ad3-467a-b2e1-0c40afe8277b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "writeMillis                       59\n",
       "graphName            reweight_themes\n",
       "nodeProperties            [leidenId]\n",
       "propertiesWritten               9319\n",
       "Name: 0, dtype: object"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"MATCH (t:Theme|Stem) SET t.leidenId = null\")\n",
    "gds.graph.nodeProperties.write(g_themes2, {\"leidenGamma512.0\": \"leidenId\"})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "83fab864-146c-4cef-9584-b5d63b669d56",
   "metadata": {},
   "source": [
    "Create ThemeGroup nodes connecting themes that are duplicates."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "879ca314-ef1b-422d-92cf-8ca78636f227",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"CREATE CONSTRAINT theme_group_node_key IF NOT EXISTS FOR (n:ThemeGroup) REQUIRE n.id IS NODE KEY\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "d67d9406-a439-4674-bd09-000041dabb4a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"\n",
    "MATCH (t:Theme)\n",
    "WHERE t.leidenId IS NOT NULL\n",
    "WITH t ORDER BY t.description\n",
    "WITH t.leidenId AS id, collect(t) AS themeNodes, collect(t.description) AS descriptions\n",
    "MERGE (g:ThemeGroup {id:id}) \n",
    "SET g.descriptions = descriptions\n",
    "WITH g, themeNodes\n",
    "FOREACH (t in themeNodes | MERGE (t)-[:IN_GROUP]->(g))\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "fb9070f0-8880-4810-b653-c894574ac1ae",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"\n",
    "MATCH (s:Stem)\n",
    "WHERE s.leidenId IS NOT NULL\n",
    "CALL\n",
    "{\n",
    "WITH s\n",
    "MERGE (g:ThemeGroup {id:s.leidenId}) \n",
    "SET g.descriptions = coalesce(g.descriptions, []) + s.descriptions\n",
    "WITH s, g\n",
    "MATCH (s)<-[:HAS_STEM]-(t)\n",
    "MERGE (t)-[:IN_GROUP]->(g)\n",
    "}\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "03ec82f6-7af7-4e23-b303-7c43ec54adbd",
   "metadata": {},
   "source": [
    "Create ThemeGroup nodes for Theme nodes that are not already grouped in a theme group if the theme is referenced in more than one document."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "1e69e156-b9ff-40a3-883e-5769f9055bff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>singleThemeGroups</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>28183</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   singleThemeGroups\n",
       "0              28183"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"\n",
    "MATCH (g:ThemeGroup)\n",
    "WITH max(g.id) AS maxId\n",
    "MATCH (t:Theme&Groupable)\n",
    "WHERE NOT EXISTS {(t)-[:IN_GROUP]->()}\n",
    "WITH maxId, COLLECT(t) AS needsGroup\n",
    "UNWIND range(0, size(needsGroup)-1) AS idx\n",
    "WITH maxId, idx, needsGroup[idx] AS t\n",
    "MERGE (g:ThemeGroup {id:maxId + idx + 1})\n",
    "SET g.descriptions = [t.description]\n",
    "WITH g, t\n",
    "MERGE (t)-[:IN_GROUP]->(g)\n",
    "RETURN COUNT(*) AS singleThemeGroups\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "eb789db0-0e32-4d4e-9d77-f99b3a24ef73",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"\n",
    "MATCH (g:ThemeGroup)\n",
    "WITH max(g.id) AS maxId\n",
    "MATCH (s:Stem)\n",
    "WHERE s.leidenId IS NULL\n",
    "WITH maxId, COLLECT(s) AS needsGroup\n",
    "UNWIND range(0, size(needsGroup)-1) AS idx\n",
    "WITH maxId, idx, needsGroup[idx] AS s\n",
    "CALL\n",
    "{\n",
    "WITH s, maxId, idx\n",
    "MERGE (g:ThemeGroup {id:maxId + idx + 1}) \n",
    "SET g.descriptions = s.descriptions\n",
    "WITH s, g\n",
    "MATCH (s)<-[:HAS_STEM]-(t)\n",
    "MERGE (t)-[:IN_GROUP]->(g)\n",
    "}\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1e09c830-8760-47be-bb44-820883963f2b",
   "metadata": {},
   "source": [
    "For Theme nodes that are not already in a theme group and contain only one document, create ThemeGroup nodes and add a simple summary."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f16752f0-c215-4535-b158-c14c9a05e19d",
   "metadata": {},
   "outputs": [],
   "source": [
    "single_doc_df = gds.run_cypher(\"\"\"\n",
    "    MATCH (g:ThemeGroup)\n",
    "    WITH max(g.id) AS maxId\n",
    "    MATCH (t:Theme) WHERE NOT EXISTS {(t)-[:IN_GROUP]->()} \n",
    "    WITH maxId, COLLECT(t) AS needsGroup\n",
    "    UNWIND range(0, size(needsGroup)-1) AS idx\n",
    "    WITH maxId, idx, needsGroup[idx] AS t\n",
    "    CALL\n",
    "    {\n",
    "    WITH t, maxId, idx\n",
    "    MERGE (g:ThemeGroup {id:maxId + idx + 1})\n",
    "    SET g.descriptions = [t.description]\n",
    "    SET g.summary = \"A movie about \" + t.description\n",
    "    MERGE (t)-[:IN_GROUP]->(g)\n",
    "    RETURN g\n",
    "    } IN TRANSACTIONS OF 10000 ROWS\n",
    "    RETURN g.id AS groupId, g.summary AS summary\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "ce166298-032a-4fb0-a598-ec456bcbbc79",
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings = OpenAIEmbeddings(model=\"text-embedding-3-small\", openai_api_key=openai_api_key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "92e16a99-6315-42e3-a672-840ab58be150",
   "metadata": {},
   "outputs": [],
   "source": [
    "single_doc_df['embedding'] = embeddings.embed_documents(single_doc_df['summary'].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8956af97-caf9-4522-9657-a2c493dd014b",
   "metadata": {},
   "outputs": [],
   "source": [
    "gds.run_cypher(\"\"\"\n",
    "UNWIND $data AS row\n",
    "CALL\n",
    "{\n",
    "WITH row\n",
    "MATCH (g:ThemeGroup {id:row['groupId']})\n",
    "CALL db.create.setNodeVectorProperty(g, 'summaryEmbedding', row['embedding'])\n",
    "} IN TRANSACTIONS OF 10000 ROWS\n",
    "\"\"\", {\"data\": single_doc_df[['groupId', 'embedding']].to_dict(\"records\")})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "63c5ca03-852f-446b-a06c-a8ebfaf96428",
   "metadata": {},
   "source": [
    "## Test simple summaries for other groups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "a51f1d05-f580-4583-82a0-488443340f51",
   "metadata": {},
   "outputs": [],
   "source": [
    "needs_summary_df = gds.run_cypher(\"\"\"\n",
    "MATCH (g:ThemeGroup) WHERE g.summary IS NULL\n",
    "SET g.summary = \n",
    "    CASE WHEN size(g.descriptions) = 1 THEN \"Movies about \" + g.descriptions[0]\n",
    "    ELSE reduce(s = \"Movies about \" + g.descriptions[0], d in g.descriptions[1..-1] | s + \", \" + d) + \", and \" + g.descriptions[-1]\n",
    "    END\n",
    "RETURN g.id AS groupId, g.summary AS summary\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "38221f09-aeb6-4c68-ace5-d44bfdf7e08f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning: model not found. Using cl100k_base encoding.\n"
     ]
    }
   ],
   "source": [
    "needs_summary_df['embedding'] = embeddings.embed_documents(needs_summary_df['summary'].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "20c59419-1618-4d41-a04d-cfefb98a9585",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"\n",
    "UNWIND $data AS row\n",
    "CALL\n",
    "{\n",
    "WITH row\n",
    "MATCH (g:ThemeGroup {id:row['groupId']})\n",
    "CALL db.create.setNodeVectorProperty(g, 'summaryEmbedding', row['embedding'])\n",
    "} IN TRANSACTIONS OF 10000 ROWS\n",
    "\"\"\", {\"data\": needs_summary_df[['groupId', 'embedding']].to_dict(\"records\")})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7fc452c8-976c-47fa-ae97-17b7ae887609",
   "metadata": {},
   "source": [
    "# Clean up projections"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "id": "46132d91-3c81-4bbc-9a76-40e7f022bbe6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "graphName                                                                                                                                                                                                                                                                                                                                                                                                                        themes\n",
       "database                                                                                                                                                                                                                                                                                                                                                                                                                          neo4j\n",
       "databaseLocation                                                                                                                                                                                                                                                                                                                                                                                                                  local\n",
       "memoryUsage                                                                                                                                                                                                                                                                                                                                                                                                                            \n",
       "sizeInBytes                                                                                                                                                                                                                                                                                                                                                                                                                          -1\n",
       "nodeCount                                                                                                                                                                                                                                                                                                                                                                                                                         38513\n",
       "relationshipCount                                                                                                                                                                                                                                                                                                                                                                                                                 30680\n",
       "configuration                                           {'readConcurrency': 4, 'undirectedRelationshipTypes': [], 'jobId': '75218b37-abf9-41fd-a6c2-567806a51835', 'logProgress': True, 'query': '\n",
       "MATCH (t:Groupable)\n",
       "RETURN gds.graph.project(******, \n",
       "t, ******,\n",
       "{sourceNodeProperties: t{.embedding},\n",
       "targetNodeProperties: ******}\n",
       ")', 'inverseIndexedRelationshipTypes': [], 'creationTime': 2024-03-25T14:02:56.569765655+00:00}\n",
       "density                                                                                                                                                                                                                                                                                                                                                                                                                        0.000021\n",
       "creationTime                                                                                                                                                                                                                                                                                                                                                                                        2024-03-25T14:02:56.569765655+00:00\n",
       "modificationTime                                                                                                                                                                                                                                                                                                                                                                                    2024-03-25T14:07:29.050926585+00:00\n",
       "schema                                                                                                       {'graphProperties': {}, 'nodes': {'__ALL__': {'embedding': 'List of Float (DefaultValue(null), PERSISTENT)'}}, 'relationships': {'UNDIRECTED_SIMILAR': {'similarity': 'Float (DefaultValue(NaN), PERSISTENT, Aggregation.MAX)'}, 'IS_SIMILAR': {'similarity': 'Float (DefaultValue(NaN), PERSISTENT, Aggregation.NONE)'}}}\n",
       "schemaWithOrientation    {'graphProperties': {}, 'nodes': {'__ALL__': {'embedding': 'List of Float (DefaultValue(null), PERSISTENT)'}}, 'relationships': {'UNDIRECTED_SIMILAR': {'direction': 'UNDIRECTED', 'properties': {'similarity': 'Float (DefaultValue(NaN), PERSISTENT, Aggregation.MAX)'}}, 'IS_SIMILAR': {'direction': 'DIRECTED', 'properties': {'similarity': 'Float (DefaultValue(NaN), PERSISTENT, Aggregation.NONE)'}}}}\n",
       "Name: 0, dtype: object"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.graph.drop(g_themes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "015199da-29a3-44b9-a24a-db1e0c48192d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "graphName                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 reweight_themes\n",
       "database                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            neo4j\n",
       "databaseLocation                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    local\n",
       "memoryUsage                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              \n",
       "sizeInBytes                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            -1\n",
       "nodeCount                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                            9319\n",
       "relationshipCount                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   33888\n",
       "configuration                                                                                                                                                                                                                                                                                                                {'readConcurrency': 4, 'undirectedRelationshipTypes': ['IS_SIMILAR'], 'jobId': '1c40f369-b425-4923-940e-0a4bf09f0051', 'logProgress': True, 'query': '\n",
       "CALL gds.graph.relationshipProperties.stream($graphName, [******], [******])\n",
       "YIELD sourceNodeId, targetNodeId, propertyValue\n",
       "WITH sourceNodeId AS source, \n",
       "targetNodeId AS target,\n",
       "max(propertyValue) AS score\n",
       "WITH gds.graph.project(******,\n",
       "source,\n",
       "target,\n",
       "{relationshipType: ******,\n",
       "relationshipProperties: {similarity: (score-$similarityCutoff)/(******-$similarityCutoff)}},\n",
       "{undirectedRelationshipTypes: [******]}) AS g\n",
       "RETURN g.graphName AS graphName, g.relationshipCount AS relationshipCount, g.nodeCount AS nodeCount', 'inverseIndexedRelationshipTypes': [], 'creationTime': 2024-03-25T14:07:34.322646585+00:00}\n",
       "density                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           0.00039\n",
       "creationTime                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          2024-03-25T14:07:34.322646585+00:00\n",
       "modificationTime                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      2024-03-25T14:12:11.172995705+00:00\n",
       "schema                                                              {'graphProperties': {}, 'nodes': {'__ALL__': {'leidenGamma8.0': 'Integer (DefaultValue(-9223372036854775808), TRANSIENT)', 'leidenGamma2.0': 'Integer (DefaultValue(-9223372036854775808), TRANSIENT)', 'leidenGamma512.0': 'Integer (DefaultValue(-9223372036854775808), TRANSIENT)', 'leidenGamma32.0': 'Integer (DefaultValue(-9223372036854775808), TRANSIENT)', 'leidenGamma1024.0': 'Integer (DefaultValue(-9223372036854775808), TRANSIENT)', 'leidenGamma64.0': 'Integer (DefaultValue(-9223372036854775808), TRANSIENT)', 'leidenGamma16.0': 'Integer (DefaultValue(-9223372036854775808), TRANSIENT)', 'leidenGamma1.0': 'Integer (DefaultValue(-9223372036854775808), TRANSIENT)', 'leidenGamma4.0': 'Integer (DefaultValue(-9223372036854775808), TRANSIENT)', 'leidenGamma128.0': 'Integer (DefaultValue(-9223372036854775808), TRANSIENT)', 'leidenGamma256.0': 'Integer (DefaultValue(-9223372036854775808), TRANSIENT)'}}, 'relationships': {'IS_SIMILAR': {'similarity': 'Float (DefaultValue(NaN), TRANSIENT, Aggregation.NONE)'}}}\n",
       "schemaWithOrientation    {'graphProperties': {}, 'nodes': {'__ALL__': {'leidenGamma8.0': 'Integer (DefaultValue(-9223372036854775808), TRANSIENT)', 'leidenGamma2.0': 'Integer (DefaultValue(-9223372036854775808), TRANSIENT)', 'leidenGamma512.0': 'Integer (DefaultValue(-9223372036854775808), TRANSIENT)', 'leidenGamma32.0': 'Integer (DefaultValue(-9223372036854775808), TRANSIENT)', 'leidenGamma1024.0': 'Integer (DefaultValue(-9223372036854775808), TRANSIENT)', 'leidenGamma64.0': 'Integer (DefaultValue(-9223372036854775808), TRANSIENT)', 'leidenGamma16.0': 'Integer (DefaultValue(-9223372036854775808), TRANSIENT)', 'leidenGamma1.0': 'Integer (DefaultValue(-9223372036854775808), TRANSIENT)', 'leidenGamma4.0': 'Integer (DefaultValue(-9223372036854775808), TRANSIENT)', 'leidenGamma128.0': 'Integer (DefaultValue(-9223372036854775808), TRANSIENT)', 'leidenGamma256.0': 'Integer (DefaultValue(-9223372036854775808), TRANSIENT)'}}, 'relationships': {'IS_SIMILAR': {'direction': 'UNDIRECTED', 'properties': {'similarity': 'Float (DefaultValue(NaN), TRANSIENT, Aggregation.NONE)'}}}}\n",
       "Name: 0, dtype: object"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.graph.drop(g_themes2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "2028e6ba-8e52-4a89-85b0-fa5a77e731df",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"MATCH (g:Groupable) REMOVE g:Groupable\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "5fb99190-6495-48c9-a87e-9afb5885def6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"DROP INDEX groupbale_vectors\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "65f73e40-7696-4676-8c2f-c96c8303204c",
   "metadata": {},
   "source": [
    "# Add mean vector representation to each ThemeGroup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "c47616d9-4a80-455f-9306-8494b3c477e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "group_embeddings_df = gds.run_cypher(\"\"\"\n",
    "MATCH (g:ThemeGroup)<-[:IN_GROUP]-(t)\n",
    "RETURN g.id AS groupId, collect(t.embedding) AS embeddings\n",
    "\"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "id": "5217b0d4-c2ba-4adb-912b-ef4c4fdec83a",
   "metadata": {},
   "outputs": [],
   "source": [
    "group_embeddings_df['mean_embedding'] = group_embeddings_df.apply(get_mean_vector, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "id": "dbe2a4bb-198a-4d55-8e61-0ae06408bdcc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"\n",
    "    UNWIND $data AS row\n",
    "    CALL\n",
    "    {\n",
    "    WITH row\n",
    "    MATCH (g:ThemeGroup {id: row['groupId']})\n",
    "    CALL db.create.setNodeVectorProperty(g, \"meanEmbedding\", row['mean_embedding'])\n",
    "    } IN TRANSACTIONS OF 10000 ROWS\"\"\",\n",
    "                   {\"data\":group_embeddings_df[['groupId', 'mean_embedding']].to_dict(\"records\")})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "id": "ae4461fa-e89b-4915-a99d-ba5fc86fc32d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds.run_cypher(\"\"\"CREATE VECTOR INDEX theme_group_mean_vectors IF NOT EXISTS \n",
    "                  FOR (t:ThemeGroup)\n",
    "                  ON (t.meanEmbedding)\n",
    "                  OPTIONS {indexConfig: \n",
    "                      {`vector.dimensions`: 1536,\n",
    "                       `vector.similarity_function`: 'cosine'\n",
    "                       }}\n",
    "                       \"\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b81865e1-c2e8-4bbd-8a41-10c53ff2a3fd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "chatbot",
   "language": "python",
   "name": "chatbot"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}