gcnns_perf_summary_table_no_rotations.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from groups.discrete_so2 import DiscreteSO2\n",
    "\n",
    "input_config = {\n",
    "    \"C\": 1,\n",
    "    \"H\": 28,\n",
    "    \"W\": 28\n",
    "}\n",
    "\n",
    "output_config = {\n",
    "    \"C\": 10\n",
    "}\n",
    "\n",
    "experiment_config = {\n",
    "    \"num_iters\": 1,\n",
    "    \"batch_size\": 32,\n",
    "    \"max_epochs\": 20,\n",
    "    \"log_every_n_steps\": 100,\n",
    "    \"patience\": 5,\n",
    "    \"rotate_train\": False,\n",
    "    \"rotate_test\": True,\n",
    "    \"seed\": 420,\n",
    "}\n",
    "\n",
    "\n",
    "def get_gcnn_hparams(order: int, gconv_layers: int, channels: int, kernel_size: int = 5):\n",
    "  return {\n",
    "      \"in_channels\": input_config[\"C\"],\n",
    "      \"out_channels\": output_config[\"C\"],\n",
    "      \"kernel_size\": kernel_size,\n",
    "      \"gconv_layers\": gconv_layers,\n",
    "      \"channels\": channels,\n",
    "      \"group\": DiscreteSO2(order=order)\n",
    "  }\n",
    "\n",
    "\n",
    "optimizer_hparams = {\n",
    "    \"lr\": 1e-2,\n",
    "    \"weight_decay\": 1e-4\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "metrics_base_path = \"results/no-rotations/gcnns\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from modules.MNISTModule import MNISTModule\n",
    "model_name = \"GCNN\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from model_saving_helpers import get_model_save_name\n",
    "from pathlib import Path\n",
    "import numpy as np\n",
    "\n",
    "from count_model_parameters import count_model_parameters\n",
    "\n",
    "\n",
    "def aggregate_model_stats(model_name, model_hparams, optimizer_hparams, experiment_config):\n",
    "  model_save_name = get_model_save_name(\n",
    "      model_name=model_name,\n",
    "      model_hparams=model_hparams,\n",
    "      optimizer_hparams=optimizer_hparams,\n",
    "      experiment_config=experiment_config\n",
    "  )\n",
    "  model = MNISTModule(model_name, model_hparams, optimizer_hparams).model\n",
    "  model_params = count_model_parameters(model)\n",
    "  million = pow(10, 3)\n",
    "\n",
    "  model_metrics_per_run = Path(\n",
    "      metrics_base_path).glob(f\"*{model_save_name}*.csv\")\n",
    "\n",
    "  model_metrics = pd.concat([pd.read_csv(experiment_metrics)\n",
    "                             for experiment_metrics in model_metrics_per_run])\n",
    "  test_metrics_subselection = model_metrics[[\n",
    "      \"test_loss\",\n",
    "      \"test_acc\",\n",
    "  ]].dropna()\n",
    "  test_metrics_means = test_metrics_subselection.mean().round(4)\n",
    "  test_metrics_stds = test_metrics_subselection.std().round(4)\n",
    "\n",
    "  test_acc_mean = test_metrics_means[\"test_acc\"]\n",
    "  test_acc_std = test_metrics_stds[\"test_acc\"]\n",
    "\n",
    "  report = pd.DataFrame({\n",
    "      \"Discretization Order\": [model_hparams[\"group\"].order.item()],\n",
    "      \"Group Conv Layers\": [model_hparams[\"gconv_layers\"]],\n",
    "      \"Channels\": [model_hparams[\"channels\"]],\n",
    "      \"Kernel Size\": [model_hparams[\"kernel_size\"]],\n",
    "      \"Params\": [f\"{np.round((model_params / million), 2)} K\"],\n",
    "      \"Test Accuracy\": [f\"{test_acc_mean} ± {test_acc_std}\"],\n",
    "  })\n",
    "\n",
    "  return report\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Discretization Order</th>\n",
       "      <th>Group Conv Layers</th>\n",
       "      <th>Channels</th>\n",
       "      <th>Kernel Size</th>\n",
       "      <th>Params</th>\n",
       "      <th>Test Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>5</td>\n",
       "      <td>51.77 K</td>\n",
       "      <td>0.834 ± 0.0024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>32</td>\n",
       "      <td>5</td>\n",
       "      <td>205.93 K</td>\n",
       "      <td>0.8401 ± 0.0128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>16</td>\n",
       "      <td>5</td>\n",
       "      <td>102.97 K</td>\n",
       "      <td>0.9216 ± 0.0137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>32</td>\n",
       "      <td>5</td>\n",
       "      <td>410.73 K</td>\n",
       "      <td>0.9294 ± 0.0083</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>5</td>\n",
       "      <td>102.97 K</td>\n",
       "      <td>0.8424 ± 0.0258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>32</td>\n",
       "      <td>5</td>\n",
       "      <td>410.73 K</td>\n",
       "      <td>0.8652 ± 0.0168</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>16</td>\n",
       "      <td>5</td>\n",
       "      <td>205.37 K</td>\n",
       "      <td>0.9369 ± 0.0034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>32</td>\n",
       "      <td>5</td>\n",
       "      <td>820.33 K</td>\n",
       "      <td>0.9429 ± 0.007</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Discretization Order  Group Conv Layers  Channels  Kernel Size    Params  \\\n",
       "0                     4                  2        16            5   51.77 K   \n",
       "0                     4                  2        32            5  205.93 K   \n",
       "0                     4                  4        16            5  102.97 K   \n",
       "0                     4                  4        32            5  410.73 K   \n",
       "0                     8                  2        16            5  102.97 K   \n",
       "0                     8                  2        32            5  410.73 K   \n",
       "0                     8                  4        16            5  205.37 K   \n",
       "0                     8                  4        32            5  820.33 K   \n",
       "\n",
       "     Test Accuracy  \n",
       "0   0.834 ± 0.0024  \n",
       "0  0.8401 ± 0.0128  \n",
       "0  0.9216 ± 0.0137  \n",
       "0  0.9294 ± 0.0083  \n",
       "0  0.8424 ± 0.0258  \n",
       "0  0.8652 ± 0.0168  \n",
       "0  0.9369 ± 0.0034  \n",
       "0   0.9429 ± 0.007  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from model_saving_helpers import get_model_save_name\n",
    "from pathlib import Path\n",
    "\n",
    "orders = [4, 8]\n",
    "convs = [2, 4]\n",
    "channels = [16, 32]\n",
    "\n",
    "all_models_stats = []\n",
    "\n",
    "for order in orders:\n",
    "  for conv in convs:\n",
    "    for channel in channels:\n",
    "      model_save_name = get_model_save_name(\n",
    "          model_name=model_name,\n",
    "          model_hparams=get_gcnn_hparams(order, conv, channel),\n",
    "          optimizer_hparams=optimizer_hparams,\n",
    "          experiment_config=experiment_config\n",
    "      )\n",
    "      model_metrics_per_run = Path(\n",
    "          metrics_base_path).glob(f\"*{model_save_name}*.csv\")\n",
    "      if len(list(model_metrics_per_run)):\n",
    "        all_models_stats.append(\n",
    "            aggregate_model_stats(\n",
    "                model_name=model_name,\n",
    "                model_hparams=get_gcnn_hparams(order, conv, channel),\n",
    "                optimizer_hparams=optimizer_hparams,\n",
    "                experiment_config=experiment_config\n",
    "            )\n",
    "        )\n",
    "\n",
    "all_models_stats = pd.concat(all_models_stats)\n",
    "\n",
    "all_models_stats\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{c|c|c|c|c|c}\n",
      "\\toprule\n",
      " Discretization Order &  Group Conv Layers &  Channels &  Kernel Size &   Params &   Test Accuracy \\\\\n",
      "\\midrule\n",
      "                    4 &                  2 &        16 &            5 &  51.77 K &  0.834 ± 0.0024 \\\\\n",
      "                    4 &                  2 &        32 &            5 & 205.93 K & 0.8401 ± 0.0128 \\\\\n",
      "                    4 &                  4 &        16 &            5 & 102.97 K & 0.9216 ± 0.0137 \\\\\n",
      "                    4 &                  4 &        32 &            5 & 410.73 K & 0.9294 ± 0.0083 \\\\\n",
      "                    8 &                  2 &        16 &            5 & 102.97 K & 0.8424 ± 0.0258 \\\\\n",
      "                    8 &                  2 &        32 &            5 & 410.73 K & 0.8652 ± 0.0168 \\\\\n",
      "                    8 &                  4 &        16 &            5 & 205.37 K & 0.9369 ± 0.0034 \\\\\n",
      "                    8 &                  4 &        32 &            5 & 820.33 K &  0.9429 ± 0.007 \\\\\n",
      "\\bottomrule\n",
      "\\end{tabular}\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/qw/52cxc2651nb6vmjvb6vb0gmr0000gn/T/ipykernel_24807/985651510.py:1: FutureWarning: In future versions `DataFrame.to_latex` is expected to utilise the base implementation of `Styler.to_latex` for formatting and rendering. The arguments signature may therefore change. It is recommended instead to use `DataFrame.style.to_latex` which also contains additional functionality.\n",
      "  print(all_models_stats.to_latex(\n"
     ]
    }
   ],
   "source": [
    "print(all_models_stats.to_latex(\n",
    "    column_format=\"c|c|c|c|c|c\",\n",
    "    index=False)\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "\n",
    "def get_max_model_stats(model_name, model_hparams, optimizer_hparams, experiment_config):\n",
    "  model_save_name = get_model_save_name(\n",
    "      model_name=model_name,\n",
    "      model_hparams=model_hparams,\n",
    "      optimizer_hparams=optimizer_hparams,\n",
    "      experiment_config=experiment_config\n",
    "  )\n",
    "  model = MNISTModule(model_name, model_hparams, optimizer_hparams).model\n",
    "  model_params = count_model_parameters(model)\n",
    "  million = pow(10, 3)\n",
    "\n",
    "  model_metrics_per_run = Path(\n",
    "      metrics_base_path).glob(f\"*{model_save_name}*.csv\")\n",
    "\n",
    "  all_metrics_files = []\n",
    "  for metrics in model_metrics_per_run:\n",
    "    run = re.search(\"-run=.\", str(metrics)).group(0)[-1]\n",
    "    read_metrics = pd.read_csv(metrics)\n",
    "    read_metrics[\"run\"] = int(run)\n",
    "    all_metrics_files.append(read_metrics.reset_index())\n",
    "\n",
    "  model_metrics = pd.concat(all_metrics_files)\n",
    "  test_metrics_subselection = model_metrics[[\n",
    "      \"run\",\n",
    "      \"test_loss\",\n",
    "      \"test_acc\",\n",
    "  ]].dropna()\n",
    "\n",
    "  best_acc_row = test_metrics_subselection[test_metrics_subselection.test_acc ==\n",
    "                                           test_metrics_subselection.test_acc.max()].reset_index()\n",
    "\n",
    "  report = pd.DataFrame({\n",
    "      \"Discretization Order\": [model_hparams[\"group\"].order.item()],\n",
    "      \"Group Conv Layers\": [model_hparams[\"gconv_layers\"]],\n",
    "      \"Channels\": [model_hparams[\"channels\"]],\n",
    "      \"Kernel Size\": [model_hparams[\"kernel_size\"]],\n",
    "      \"Params\": [f\"{np.round((model_params / million), 2)} K\"],\n",
    "      \"Best Run\": [best_acc_row[\"run\"].tolist()[0]],\n",
    "      \"Max Test Accuracy\": [best_acc_row[\"test_acc\"].tolist()[0]],\n",
    "  })\n",
    "\n",
    "  return report\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Discretization Order</th>\n",
       "      <th>Group Conv Layers</th>\n",
       "      <th>Channels</th>\n",
       "      <th>Kernel Size</th>\n",
       "      <th>Params</th>\n",
       "      <th>Best Run</th>\n",
       "      <th>Max Test Accuracy</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>5</td>\n",
       "      <td>51.77 K</td>\n",
       "      <td>3</td>\n",
       "      <td>0.8361</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>32</td>\n",
       "      <td>5</td>\n",
       "      <td>205.93 K</td>\n",
       "      <td>1</td>\n",
       "      <td>0.8547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>16</td>\n",
       "      <td>5</td>\n",
       "      <td>102.97 K</td>\n",
       "      <td>1</td>\n",
       "      <td>0.9358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>32</td>\n",
       "      <td>5</td>\n",
       "      <td>410.73 K</td>\n",
       "      <td>3</td>\n",
       "      <td>0.9362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>5</td>\n",
       "      <td>102.97 K</td>\n",
       "      <td>1</td>\n",
       "      <td>0.8700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>32</td>\n",
       "      <td>5</td>\n",
       "      <td>410.73 K</td>\n",
       "      <td>3</td>\n",
       "      <td>0.8845</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>16</td>\n",
       "      <td>5</td>\n",
       "      <td>205.37 K</td>\n",
       "      <td>2</td>\n",
       "      <td>0.9408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>8</td>\n",
       "      <td>4</td>\n",
       "      <td>32</td>\n",
       "      <td>5</td>\n",
       "      <td>820.33 K</td>\n",
       "      <td>2</td>\n",
       "      <td>0.9482</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Discretization Order  Group Conv Layers  Channels  Kernel Size    Params  \\\n",
       "0                     4                  2        16            5   51.77 K   \n",
       "0                     4                  2        32            5  205.93 K   \n",
       "0                     4                  4        16            5  102.97 K   \n",
       "0                     4                  4        32            5  410.73 K   \n",
       "0                     8                  2        16            5  102.97 K   \n",
       "0                     8                  2        32            5  410.73 K   \n",
       "0                     8                  4        16            5  205.37 K   \n",
       "0                     8                  4        32            5  820.33 K   \n",
       "\n",
       "   Best Run  Max Test Accuracy  \n",
       "0         3             0.8361  \n",
       "0         1             0.8547  \n",
       "0         1             0.9358  \n",
       "0         3             0.9362  \n",
       "0         1             0.8700  \n",
       "0         3             0.8845  \n",
       "0         2             0.9408  \n",
       "0         2             0.9482  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from model_saving_helpers import get_model_save_name\n",
    "from pathlib import Path\n",
    "\n",
    "orders = [4, 8]\n",
    "convs = [2, 4]\n",
    "channels = [16, 32]\n",
    "\n",
    "all_models_stats = []\n",
    "\n",
    "for order in orders:\n",
    "  for conv in convs:\n",
    "    for channel in channels:\n",
    "      model_save_name = get_model_save_name(\n",
    "          model_name=model_name,\n",
    "          model_hparams=get_gcnn_hparams(order, conv, channel),\n",
    "          optimizer_hparams=optimizer_hparams,\n",
    "          experiment_config=experiment_config\n",
    "      )\n",
    "      model_metrics_per_run = Path(\n",
    "          metrics_base_path).glob(f\"*{model_save_name}*.csv\")\n",
    "      if len(list(model_metrics_per_run)):\n",
    "        all_models_stats.append(\n",
    "            get_max_model_stats(\n",
    "                model_name=model_name,\n",
    "                model_hparams=get_gcnn_hparams(order, conv, channel),\n",
    "                optimizer_hparams=optimizer_hparams,\n",
    "                experiment_config=experiment_config\n",
    "            )\n",
    "        )\n",
    "\n",
    "all_models_stats = pd.concat(all_models_stats)\n",
    "\n",
    "all_models_stats\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "geometric-deep-learning-7fBP83ZS",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}