Features/new nn functionality

examples/all_configurations_benchmark/clean_results.ipynb

@@ -0,0 +1,193 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from copy import deepcopy\n",
+    "from datetime import datetime\n",
+    "from itertools import product\n",
+    "from pathlib import Path\n",
+    "\n",
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "from tqdm import tqdm"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "time_now = datetime.now().strftime(\"%Y-%m-%d\")\n",
+    "print(f\"Time: {time_now}\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "MODELS = [\"PyBoost\", \"DLinear_NN\", \"PatchTST_NN\", \"GPT4TS_NN\", \"TimesNet_NN\", \"TimeMixer_NN\", \"CycleNet_NN\"]\n",
+    "DATASETS = [\"ILI\"]\n",
+    "STRATEGY_TIMES = [\n",
+    "    \"FlatWideMIMOStrategy__model_horizon_NaN\",\n",
+    "    \"MIMOStrategy__model_horizon_NaN\",\n",
+    "    \"RecursiveStrategy__model_horizon_1.0\",\n",
+    "    \"RecursiveStrategy__model_horizon_6.0\",\n",
+    "]\n",
+    "DATETIMES = [\"False\", \"with_normalization_over_all\"]\n",
+    "IDS = [\"False\", \"with_le_normalization_over_all\"]\n",
+    "TRANSFORMER_NAMES = [\n",
+    "    \"NaN\",\n",
+    "    \"DifferenceNormalizer\",\n",
+    "    \"LastKnownNormalizer\",\n",
+    "]\n",
+    "TRANSFORMER_REGIMES = [\"NaN\", \"delta\"]\n",
+    "TRANSFORMER_TR_FEATURES = [\"True\"]\n",
+    "TRANSFORMER_TR_TARGETS = [\"True\"]\n",
+    "REGIMES = [\"multivariate\", \"global\"]\n",
+    "CI = [\"False\", \"True\"]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "df_path = f\"agg_results_{time_now}__normalized_True.csv\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "raw_df = pd.read_csv(df_path)\n",
+    "\n",
+    "# Convert bool to object\n",
+    "for col in raw_df.columns:\n",
+    "    if raw_df[col].dtype == bool:\n",
+    "        raw_df[col] = raw_df[col].astype(str)\n",
+    "        print(f\"Converted {col} to object\")\n",
+    "        \n",
+    "raw_df = raw_df.dropna(subset=['mae_test', 'rmse_test', 'fit_time_test',\n",
+    "       'forecast_time_test', 'mae_val', 'rmse_val', 'fit_time_val', 'forecast_time_val'])\n",
+    "# NaN to \"NaN\"\n",
+    "raw_df = raw_df.fillna(\"NaN\")\n",
+    "\n",
+    "raw_df = raw_df[raw_df[\"dateteime\"].isin(DATETIMES)]\n",
+    "raw_df = raw_df[raw_df[\"id\"].isin(IDS)]\n",
+    "\n",
+    "raw_df[\"tr_features\"] = raw_df[\"tr_features\"].str.replace(\"_metrics_test_v2.csv\", \"\", regex=False)\n",
+    "raw_df[\"tr_features\"] = raw_df[\"tr_features\"].str.replace(\"_metrics_test_v2_normalized.csv\", \"\", regex=False)\n",
+    "\n",
+    "# colname regime -> colname transormer_regime\n",
+    "# colname transformer -> colname transformer_name\n",
+    "# colname tr_target -> colname transformer_tr_target\n",
+    "# colname tr_features -> colname transformer_tr_features\n",
+    "raw_df = raw_df.rename(\n",
+    "    columns={\n",
+    "        \"dateteime\": \"datetime\",\n",
+    "        \"regime\": \"transformer_regime\",\n",
+    "        \"transformer\": \"transformer_name\",\n",
+    "        \"tr_target\": \"transformer_tr_target\",\n",
+    "        \"tr_features\": \"transformer_tr_features\",\n",
+    "    }\n",
+    ")\n",
+    "\n",
+    "# mult (\"False\" or \"True\") to regime (\"multivariate\" or \"global\")\n",
+    "raw_df[\"regime\"] = np.where(raw_df[\"mult\"] == \"False\", \"global\", \"multivariate\")\n",
+    "\n",
+    "# Concatenate strategy_time and model_horizon\n",
+    "raw_df[\"strategy_time\"] = (\n",
+    "    raw_df[\"strategy_time\"] + \"__model_horizon_\" + raw_df[\"model_hor\"].astype(str)\n",
+    ")\n",
+    "\n",
+    "# Drop unnecessary columns\n",
+    "raw_df = raw_df.drop(columns=[\"model_hor\", \"hor\", \"hist\", \"mult\"])\n",
+    "raw_df = raw_df.drop(columns=[\"dataset\", \"transformer_tr_target\", \"transformer_tr_features\"])\n",
+    "\n",
+    "raw_df[\"ci\"] = raw_df[\"ci\"].replace({\"True\": \"CI\", \"False\": \"CM\"})\n",
+    "raw_df[\"mode\"] = raw_df[\"regime\"] + \" \" + raw_df[\"ci\"]\n",
+    "raw_df[\"mode\"] = raw_df[\"mode\"].replace({\"global CI\": \"global\", \"global CM\": \"global\"})\n",
+    "raw_df.loc[\n",
+    "    (raw_df['model'] == 'PyBoost') & (raw_df['mode'] == 'multivariate CI'),\n",
+    "    'mode'\n",
+    "] = 'multivariate CM'\n",
+    "raw_df = raw_df.drop(columns=[\"regime\", \"ci\"])\n",
+    "\n",
+    "raw_df = raw_df.drop_duplicates(subset=[\"model\", \"strategy_time\", \"datetime\", \"id\", \"transformer_name\", \"mode\", \"transformer_regime\"])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Value counts for each column\n",
+    "cols_for_check = [\n",
+    "    col\n",
+    "    for col in raw_df.columns\n",
+    "    if col\n",
+    "    not in [\n",
+    "        \"mae_test\",\n",
+    "        \"rmse_test\",\n",
+    "        \"mape_test\",\n",
+    "        \"fit_time_test\",\n",
+    "        \"forecast_time_test\",\n",
+    "        \"mae_val\",\n",
+    "        \"rmse_val\",\n",
+    "        \"mape_val\",\n",
+    "        \"fit_time_val\",\n",
+    "        \"forecast_time_val\",\n",
+    "    ]\n",
+    "]\n",
+    "\n",
+    "for col in cols_for_check:\n",
+    "    print(\"Column: \", col)\n",
+    "    print(\"Type of data: \", raw_df[col].dtype)\n",
+    "    \n",
+    "    print(raw_df[col].value_counts(dropna=False))\n",
+    "    print()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "raw_df.to_csv(f\"{df_path[:-4]}_cleaned.csv\", index=False)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "py_3_10",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.10.14"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

examples/all_configurations_benchmark/constants.py

@@ -0,0 +1,82 @@
+all_models_params = {
+    "ILI": {
+        "DLinear": {
+            "moving_avg": 25,
+        },
+        "PatchTST": {
+            "e_layers": 3,
+            "n_heads": 4,
+            "d_model": 16,
+            "d_ff": 128,
+            "dropout": 0.3,
+            "head_dropout": 0,
+            "patch_len": 24,
+            "stride": 2,
+        },
+        "GPT4TS": {
+            "d_model": 768,
+            "patch_len": 24, 
+            "stride": 2,
+            "gpt_layers": 6,
+            "freeze": 1,    
+        },
+        "TimesNet": {
+            "e_layers": 2,
+            "d_model": 768,
+            "d_ff": 768,
+            "top_k": 5,    
+        },
+        "TimeMixer": {
+            "e_layers": 2,
+            "down_sampling_layers": 3,
+            "down_sampling_window": 2,
+            "d_model": 16,
+            "d_ff": 32,
+            "down_sampling_method": "avg",
+        },
+        "CycleNet": {
+            "model_type": 'linear',
+            "cycle_len": 24,
+        }
+    },
+    "ETTh1": {
+        "DLinear": {
+            "moving_avg": 25,
+        },
+        "PatchTST": {
+            "e_layers": 3,
+            "n_heads": 4,
+            "d_model": 16,
+            "d_ff": 128,
+            "dropout": 0.3,
+            "head_dropout": 0,
+            "patch_len": 16,
+            "stride": 8,
+        },
+        "GPT4TS": {
+            "d_model": 768,
+            "patch_len": 16, 
+            "stride": 8,
+            "gpt_layers": 6,
+            "freeze": 1,    
+        },
+        "TimesNet": {
+            "e_layers": 2,
+            "d_model": 16,
+            "d_ff": 32,
+            "top_k": 5,    
+        },
+        "TimeMixer": {
+            "e_layers": 2,
+            "down_sampling_layers": 3,
+            "down_sampling_window": 2,
+            "d_model": 16,
+            "d_ff": 32,
+            "down_sampling_method": "avg",
+        },
+        "CycleNet": {
+            "model_type": 'linear',
+            "cycle_len": 24,
+        }
+    },
+}