examples/live_migration_demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "b782fbeb-1808-4141-85c9-51d673d52505",
   "metadata": {},
   "source": [
    "# Live GDS Graph Migration\n",
    "> powered by neo4j-arrow 🏹\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "886a65dc-55dd-4177-a603-db41b9e37626",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pyarrow==6.0.1 in ./venv/lib/python3.9/site-packages (6.0.1)\n",
      "Requirement already satisfied: networkx==2.5.1 in ./venv/lib/python3.9/site-packages (2.5.1)\n",
      "Requirement already satisfied: numpy>=1.16.6 in ./venv/lib/python3.9/site-packages (from pyarrow==6.0.1) (1.21.2)\n",
      "Requirement already satisfied: decorator<5,>=4.3 in ./venv/lib/python3.9/site-packages (from networkx==2.5.1) (4.4.2)\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Collecting git+https://github.com/voutilad/gds-python.git\n",
      "  Cloning https://github.com/voutilad/gds-python.git to /tmp/pip-req-build-b_k75bye\n",
      "  Running command git clone --filter=blob:none -q https://github.com/voutilad/gds-python.git /tmp/pip-req-build-b_k75bye\n",
      "  Resolved https://github.com/voutilad/gds-python.git to commit 951bba4fbea66ae0fbf16def453b657ce7f2a3b5\n",
      "  Installing build dependencies ... \u001B[?25ldone\n",
      "\u001B[?25h  Getting requirements to build wheel ... \u001B[?25ldone\n",
      "\u001B[?25h  Preparing metadata (pyproject.toml) ... \u001B[?25ldone\n",
      "\u001B[?25hCollecting neo4j<5.0.0,>=4.2.1\n",
      "  Using cached neo4j-4.4.1-py3-none-any.whl\n",
      "Collecting networkx==2.5.1\n",
      "  Using cached networkx-2.5.1-py3-none-any.whl (1.6 MB)\n",
      "Collecting decorator<5,>=4.3\n",
      "  Using cached decorator-4.4.2-py2.py3-none-any.whl (9.2 kB)\n",
      "Collecting pytz\n",
      "  Using cached pytz-2021.3-py2.py3-none-any.whl (503 kB)\n",
      "Building wheels for collected packages: gds-python\n",
      "  Building wheel for gds-python (pyproject.toml) ... \u001B[?25ldone\n",
      "\u001B[?25h  Created wheel for gds-python: filename=gds_python-0.0.3-py3-none-any.whl size=7435 sha256=ad9f0ebaf0a1f17f39f7f21e97cbd753025f27d50acfbbcedbecc56eab14a1b2\n",
      "  Stored in directory: /tmp/pip-ephem-wheel-cache-e3epj7j9/wheels/ce/cd/d5/6f9ca3f0f2423ebd6c66c6db4095f06c70a989a7c6bbe052ee\n",
      "Successfully built gds-python\n",
      "Installing collected packages: pytz, decorator, networkx, neo4j, gds-python\n",
      "  Attempting uninstall: pytz\n",
      "    Found existing installation: pytz 2021.3\n",
      "    Uninstalling pytz-2021.3:\n",
      "      Successfully uninstalled pytz-2021.3\n",
      "  Attempting uninstall: decorator\n",
      "    Found existing installation: decorator 4.4.2\n",
      "    Uninstalling decorator-4.4.2:\n",
      "      Successfully uninstalled decorator-4.4.2\n",
      "  Attempting uninstall: networkx\n",
      "    Found existing installation: networkx 2.5.1\n",
      "    Uninstalling networkx-2.5.1:\n",
      "      Successfully uninstalled networkx-2.5.1\n",
      "  Attempting uninstall: neo4j\n",
      "    Found existing installation: neo4j 4.4.1\n",
      "    Uninstalling neo4j-4.4.1:\n",
      "      Successfully uninstalled neo4j-4.4.1\n",
      "  Attempting uninstall: gds-python\n",
      "    Found existing installation: gds-python 0.0.3\n",
      "    Uninstalling gds-python-0.0.3:\n",
      "      Successfully uninstalled gds-python-0.0.3\n",
      "Successfully installed decorator-4.4.2 gds-python-0.0.3 neo4j-4.4.1 networkx-2.5.1 pytz-2021.3\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install pyarrow==6.0.1 networkx==2.5.1\n",
    "%pip install --force-reinstall git+https://github.com/voutilad/gds-python.git\n",
    "\n",
    "import neo4j_arrow as na\n",
    "from gds_python import GDS"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5d3c5080-6c9b-40bf-8def-f69c55bb35ac",
   "metadata": {},
   "source": [
    "## We'll be using 2 Neo4j instances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "2108ff20-2a9a-40bc-bc87-bee14e9a8f06",
   "metadata": {},
   "outputs": [],
   "source": [
    "SOURCE = 'voutila-arrow-test'\n",
    "TARGET = 'voutila-arrow-sink'\n",
    "GRAPH = 'movies'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b5cee5c-a703-4411-af77-4734f572bd16",
   "metadata": {},
   "source": [
    "- **SOURCE** is colocated with our Jupyter kernel in Montreal ⛄.\n",
    "- **TARGET** is in a different GCP region (somewhere in Iowa 🌽)."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f0ff6eb-2b66-4992-9baa-e95c521da57f",
   "metadata": {},
   "source": [
    "### Create our source Graph Projection\n",
    "\n",
    "We'll make a simple graph projection and run FastRP on it to get some additional data. Easy peasy:\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "198f4743-76e9-4595-b245-dcd60d2e80ef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'nodePropertiesWritten': 171,\n",
       "  'mutateMillis': 0,\n",
       "  'nodeCount': 171,\n",
       "  'createMillis': 0,\n",
       "  'computeMillis': 5,\n",
       "  'configuration': {'nodeSelfInfluence': 0,\n",
       "   'relationshipWeightProperty': None,\n",
       "   'propertyRatio': 0.0,\n",
       "   'concurrency': 58,\n",
       "   'normalizationStrength': 0.0,\n",
       "   'iterationWeights': [0.0, 1.0, 1.0],\n",
       "   'embeddingDimension': 256,\n",
       "   'nodeLabels': ['*'],\n",
       "   'sudo': False,\n",
       "   'relationshipTypes': ['*'],\n",
       "   'featureProperties': [],\n",
       "   'mutateProperty': 'fastRp',\n",
       "   'username': None}}]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds = GDS(f'bolt://{SOURCE}:7687', 'neo4j', 'password').connect()\n",
    "\n",
    "if gds.graph.exists(GRAPH)[0]['exists']:\n",
    "    gds.graph.drop(GRAPH)\n",
    "\n",
    "gds.graph.create(GRAPH,\n",
    "                 ['Movie', 'Person'],\n",
    "                 {\n",
    "                      'ACTED_IN': { 'orientation': 'UNDIRECTED' },\n",
    "                      'DIRECTED': { 'orientation': 'UNDIRECTED' },\n",
    "                      'PRODUCED': { 'orientation': 'UNDIRECTED' }\n",
    "                  },\n",
    "                 { 'readConcurrency': 58 })\n",
    "\n",
    "gds.fastRP.mutate(GRAPH, {\n",
    "    'embeddingDimension': 256,\n",
    "    'mutateProperty': 'fastRp',\n",
    "    'concurrency': 58\n",
    "})\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "334a6a1a-7e16-429e-a241-b86ec0cb078a",
   "metadata": {},
   "source": [
    "## Make Sure our Target is Blank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0ac5e400-f9b8-44a7-8c8b-d438c817f331",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "clean and ready!\n"
     ]
    }
   ],
   "source": [
    "gds_target = GDS(f'bolt://{TARGET}:7687', 'neo4j', 'password').connect()\n",
    "\n",
    "if gds_target.graph.exists(GRAPH)[0]['exists']:\n",
    "    gds_target.graph.drop(GRAPH)\n",
    "print('clean and ready!')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8cc04f88-2e36-4349-a4bf-d254d9b376fc",
   "metadata": {},
   "source": [
    "## Let's move a Graph!\n",
    "\n",
    "### First we'll create our `neo4j-arrow` clients"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "29c497d9-0980-4cc5-9230-363860ae85ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "source = na.Neo4jArrow('neo4j', 'password', (SOURCE, 9999), tls=True, verify_tls=False)\n",
    "target = na.Neo4jArrow('neo4j', 'password', (TARGET, 9999))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7665198f-aa8d-4f49-ae15-0017a6e0f63b",
   "metadata": {
    "tags": []
   },
   "source": [
    "### Now let's move the nodes..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a7878d9e-48b5-4bc0-8849-a471bd1c7e6f",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "wrote 171 rows, 0.17 MiB\n",
      "migrated 171 nodes (0.17 MiB)\n"
     ]
    }
   ],
   "source": [
    "nodes = source.stream(source.gds_nodes(GRAPH, properties=['fastRp']))\n",
    "ticket = target.gds_write_nodes(GRAPH)\n",
    "rows, nbytes = target.put(ticket, nodes)\n",
    "print(f'migrated {rows:,} nodes ({round(nbytes / (1 << 20), 2):,} MiB)')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dbf3641e-2f88-4978-908c-a7fc67c17507",
   "metadata": {},
   "source": [
    "## And now the relationships!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "36b462ca-6f58-419a-8b9b-fb27503e637f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "wrote 462 rows, 0.02 MiB\n",
      "migrated 462 relationships (0.02 MiB)\n"
     ]
    }
   ],
   "source": [
    "rels = source.stream(source.gds_relationships(GRAPH))\n",
    "ticket = target.gds_write_relationships(GRAPH)\n",
    "rows, nbytes = target.put(ticket, rels)\n",
    "print(f'migrated {rows:,} relationships ({round(nbytes / (1 << 20), 2):,} MiB)')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e27850c2-778f-4b1d-a2fb-becf6646bc77",
   "metadata": {},
   "source": [
    "## And now let's check our work!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "238c5b69-9dca-40a8-ba3a-76ae93cef6b7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'degreeDistribution': {'p99': 22,\n",
       "   'min': 0,\n",
       "   'max': 26,\n",
       "   'mean': 5.4035087719298245,\n",
       "   'p90': 14,\n",
       "   'p50': 4,\n",
       "   'p999': 26,\n",
       "   'p95': 16,\n",
       "   'p75': 8},\n",
       "  'graphName': 'movies',\n",
       "  'database': 'neo4j',\n",
       "  'memoryUsage': '304 MiB',\n",
       "  'sizeInBytes': 319152864,\n",
       "  'nodeProjection': None,\n",
       "  'relationshipProjection': None,\n",
       "  'nodeQuery': None,\n",
       "  'relationshipQuery': None,\n",
       "  'nodeCount': 171,\n",
       "  'relationshipCount': 462,\n",
       "  'nodeFilter': None,\n",
       "  'relationshipFilter': None,\n",
       "  'density': 0.015892672858617132,\n",
       "  'creationTime': neo4j.time.DateTime(2022, 1, 18, 23, 4, 26, 640153000, tzinfo=<StaticTzInfo 'Etc/UTC'>),\n",
       "  'modificationTime': neo4j.time.DateTime(2022, 1, 18, 23, 4, 24, 448935000, tzinfo=<StaticTzInfo 'Etc/UTC'>),\n",
       "  'schema': {'relationships': {'ACTED_IN': {}, 'PRODUCED': {}, 'DIRECTED': {}},\n",
       "   'nodes': {'Movie': {'fastRp': 'List of Float (DefaultValue(null), TRANSIENT)'},\n",
       "    'Person': {'fastRp': 'List of Float (DefaultValue(null), TRANSIENT)'}}}}]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gds_target.graph.list(GRAPH)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}