08_Hyperparameter_Optimization/24_Hyperparameter_Tuning_With_optuna.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9e0c293c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os \n",
    "os.environ['KMP_DUPLICATE_LIB_OK']='True'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "903f8943",
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "import math\n",
    "import random\n",
    "import subprocess\n",
    "import time\n",
    "from collections import OrderedDict\n",
    "from pathlib import Path\n",
    "\n",
    "import optuna\n",
    "optuna.logging.set_verbosity(optuna.logging.INFO)\n",
    "\n",
    "import mlflow\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import psutil\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.nn.init as init\n",
    "import torch.optim as optim\n",
    "import torch.optim.lr_scheduler as lr_scheduler\n",
    "import torch.utils.data as data\n",
    "from torch.utils.data import DataLoader\n",
    "from torch.utils.data.sampler import RandomSampler, SequentialSampler\n",
    "\n",
    "import torchvision\n",
    "from torchvision.datasets.mnist import MNIST\n",
    "import torchvision.transforms as transforms\n",
    "\n",
    "from thop import profile\n",
    "from tqdm.auto import tqdm\n",
    "from torchsummary import summary\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "13abf35c-ae3c-4ff4-8c4f-c5bbb27e87bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# torch.cuda.memory._record_memory_history()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1db77c57-1975-4f86-9188-59fcc8203174",
   "metadata": {},
   "outputs": [],
   "source": [
    "def increment_path(path, exist_ok=False, sep='', mkdir=True):\n",
    "    path = Path(path)\n",
    "    if path.exists() and not exist_ok:\n",
    "        path, suffix = (path.with_suffix(''), path.suffix) if path.is_file() else (path, '')\n",
    "        for n in range(2, 9999):\n",
    "            p = f'{path}{sep}{n}{suffix}'\n",
    "            if not os.path.exists(p):\n",
    "                break\n",
    "        path = Path(p)\n",
    "    if mkdir:\n",
    "        path.mkdir(parents=True, exist_ok=True)\n",
    "    return path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "94032296-1658-4b03-bb63-5c2283e00ce5",
   "metadata": {},
   "outputs": [],
   "source": [
    "EXPERIMENT_NAME = \"exp\"\n",
    "FILE = Path(os.getcwd()).resolve()\n",
    "SAVE_DIR = increment_path(FILE / \"artifacts\" / f\"{EXPERIMENT_NAME}\", exist_ok=False)\n",
    "(SAVE_DIR / 'checkpoints').mkdir(parents=True, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "15c8f025-3784-446b-b6a6-f9570e0bec97",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 22:30:04 INFO mlflow.tracking.fluent: Experiment with name 'exp11' does not exist. Creating a new experiment.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Experiment: artifact_location='file:///C:/Users/Hp/Desktop/mldev/2024/Git/mldevworld/Deep-Learning-Essentials/mlruns/11', creation_time=1710090004074, experiment_id='11', last_update_time=1710090004074, lifecycle_stage='active', name='exp11', tags={}>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "local_tracking_uri = f\"sqlite:///mlflow/mlflow.db\"\n",
    "mlflow.set_tracking_uri(local_tracking_uri)\n",
    "mlflow.set_experiment(SAVE_DIR.stem)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4b56f9ea-54b0-4175-972a-523290fdbe41",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random seed set to 42\n"
     ]
    }
   ],
   "source": [
    "def set_experiment_seed(seed=42):\n",
    "    \"\"\"\n",
    "    Set random seeds and CUDA-related flags for experiment reproducibility.\n",
    "    \"\"\"\n",
    "    # Set random seeds for reproducibility\n",
    "    torch.manual_seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    random.seed(seed)\n",
    "\n",
    "    # If using GPU, set random seed for CUDA operations\n",
    "    if torch.cuda.is_available():\n",
    "        torch.cuda.manual_seed(seed)\n",
    "        torch.backends.cudnn.deterministic = True\n",
    "        torch.backends.cudnn.benchmark = False  # Set to False for reproducibility\n",
    "\n",
    "    # Optionally log seed information\n",
    "    print(f\"Random seed set to {seed}\")\n",
    "\n",
    "set_experiment_seed()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "079ce688",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Download MNIST dataset\n",
    "data_train = MNIST(\n",
    "    \"./data/mnist\", download=True, \n",
    "    transform=transforms.Compose([\n",
    "        transforms.Resize((32, 32)),\n",
    "        transforms.RandomRotation(degrees=15),\n",
    "        transforms.RandomAffine(degrees=0, translate=(0.1, 0.1)),\n",
    "        transforms.RandomVerticalFlip(),\n",
    "        transforms.ColorJitter(brightness=0.2, contrast=0.2),\n",
    "        transforms.GaussianBlur(kernel_size=1),\n",
    "        transforms.ToTensor()\n",
    "    ])\n",
    ")\n",
    "        \n",
    "data_test = MNIST(\n",
    "    \"./data/mnist\", download=True, train=False,\n",
    "    transform=transforms.Compose([\n",
    "        transforms.Resize((32, 32)),\n",
    "        transforms.ToTensor()\n",
    "    ])\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "372c5b4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define hyperparameters\n",
    "N_CLASSES = 10\n",
    "# EPOCHS = 5\n",
    "# BATCH_SIZE = 512\n",
    "# DROPOUT_PROB = 0.2\n",
    "# USE_BATCHNORM = True\n",
    "# LABEL_SMOOTHING = 0.1\n",
    "# LR = 1.34E-03\n",
    "# WARMUP_PROPORTION = 0.1\n",
    "# WEIGHT_DECAY = 1e-4\n",
    "# ACCUMULATION_STEPS = 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "93918f18",
   "metadata": {},
   "outputs": [],
   "source": [
    "# plt.figure(figsize=(10,10))\n",
    "# for i in range(25):\n",
    "#     plt.subplot(5,5,i+1)\n",
    "#     plt.xticks([])\n",
    "#     plt.yticks([])\n",
    "#     plt.grid(False)\n",
    "#     plt.imshow(data_train[i][0].permute(1,2,0).numpy(), cmap=plt.cm.binary)\n",
    "#     plt.xlabel(data_train[i][1])\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ce264b0f-c32e-4907-bc34-c7644df7d2c7",
   "metadata": {},
   "source": [
    "`torch.utils.data.DataLoader` supports asynchronous data loading and data augmentation in separate worker subprocesses. The default setting for DataLoader is `num_workers=0`, which means that the data loading is synchronous and done in the main process. As a result the main training process has to wait for the data to be available to continue the execution.\r\n",
    "\r\n",
    "Settin`g num_workers >` 0 enables asynchronous data loading and overlap between the training and data loading. num_workers should be tuned depending on the workload, CPU, GPU, and location of training data.`\r\n",
    "\r\n",
    "DataLo`ader acce`pts pin_me`mory argument, which defaults` to F`alse. When using a GPU it’s better to `set pin_memory=`True, this instru`cts DataLo`ader to use pinned memory and enables faster and asynchronous memory copy from the host to the GPU."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "fb249649-163b-495a-bd21-94711d5ec411",
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_num_workers():\n",
    "    # Calculate the number of CPU cores\n",
    "    num_cpu_cores = os.cpu_count()\n",
    "\n",
    "    # Set a safe maximum multiplier value (e.g., 0.5) to avoid using all available resources\n",
    "    max_multiplier = 0.5\n",
    "\n",
    "    # Calculate the number of workers based on the available resources\n",
    "    multiplier = min(sum(psutil.cpu_percent(interval=1, percpu=True)) / 100.0, max_multiplier)\n",
    "    \n",
    "    # Ensure that num_workers is at least 1\n",
    "    num_workers = max(1, int(num_cpu_cores * multiplier))\n",
    "\n",
    "    return num_workers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5b752015",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_dataloaders(batch_size):\n",
    "    # Define train dataloader\n",
    "    train_dataloader = DataLoader(\n",
    "        data_train, batch_size=batch_size, \n",
    "        sampler=RandomSampler(data_train), \n",
    "        pin_memory=True, num_workers=calculate_num_workers()\n",
    "    )\n",
    "    \n",
    "    # Define test dataloader\n",
    "    test_dataloader = DataLoader(\n",
    "        data_test, batch_size=batch_size, \n",
    "        sampler=SequentialSampler(data_test),\n",
    "        pin_memory=True, num_workers=calculate_num_workers()\n",
    "    )\n",
    "\n",
    "    return train_dataloader, test_dataloader\n",
    "\n",
    "# train_dataloader, test_dataloader = get_dataloaders(BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6864f541-00b1-4851-a1da-288992d8d709",
   "metadata": {},
   "source": [
    "`torch.nn.Conv2d()` has `bias` parameter which defaults to `True` (the same is true for `Conv1d` and `Conv3d` ).\n",
    "\n",
    "If a `nn.Conv2d` layer is directly followed by a `nn.BatchNorm2d` layer, then the bias in the convolution is not needed, instead use `nn.Conv2d(..., bias=False, ....)`. Bias is not needed because in the first step `BatchNorm` subtracts the mean, which effectively cancels out the effect of bias.\n",
    "\n",
    "This is also applicable to 1d and 3d convolutions as long as `BatchNorm`(or other normalization layer) normalizes on the same dimension as convolution’s bias."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "35af50e0-5861-444c-ab61-8581c61f9af0",
   "metadata": {},
   "outputs": [],
   "source": [
    "class LeNet(nn.Module):\n",
    "    def __init__(self, use_batchnorm, dropout_prob):\n",
    "        super(LeNet, self).__init__()\n",
    "\n",
    "        self.use_batchnorm = use_batchnorm\n",
    "                \n",
    "        self.features1 = nn.Sequential(\n",
    "            nn.Conv2d(1, 6, kernel_size=(5, 5), bias=not use_batchnorm),\n",
    "            self._get_norm_layer(6),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(kernel_size=(2, 2), stride=2),\n",
    "            nn.Dropout2d(p=dropout_prob)\n",
    "        )\n",
    "        \n",
    "        self.features2 = nn.Sequential(\n",
    "            nn.Conv2d(6, 16, kernel_size=(5, 5), bias=not use_batchnorm),\n",
    "            self._get_norm_layer(16),\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(kernel_size=(2, 2), stride=2),\n",
    "            nn.Dropout2d(p=dropout_prob)\n",
    "        )\n",
    "\n",
    "        self.features3 = nn.Sequential(\n",
    "            nn.Conv2d(16, 120, kernel_size=(5, 5), bias=not use_batchnorm),\n",
    "            self._get_norm_layer(120),\n",
    "            nn.ReLU(),\n",
    "            nn.Dropout2d(p=dropout_prob)\n",
    "        )\n",
    "\n",
    "        self.classifier = nn.Sequential(\n",
    "            nn.Linear(120, 84, bias=True),\n",
    "            nn.ReLU(),\n",
    "            nn.Dropout(p=dropout_prob),\n",
    "            nn.Linear(84, 10, bias=True)\n",
    "        )\n",
    "\n",
    "        # Initialize layers\n",
    "        self.apply(self._initialize_weights)\n",
    "\n",
    "    def _get_norm_layer(self, channels):\n",
    "        if self.use_batchnorm:\n",
    "            return nn.BatchNorm2d(channels)\n",
    "        else:\n",
    "            return nn.Identity()\n",
    "\n",
    "    def _initialize_weights(self, m):\n",
    "        if isinstance(m, nn.Conv2d) or isinstance(m, nn.Linear):\n",
    "            init.kaiming_uniform_(m.weight)\n",
    "            if m.bias is not None:\n",
    "                init.constant_(m.bias, 0)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = self.features1(x)\n",
    "        x = self.features2(x)\n",
    "        x = self.features3(x)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        x = self.classifier(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "621c6db2-e8a3-42cc-ab36-5694abdf5764",
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "31f61bca",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_model(use_batchnorm, dropout_prob):\n",
    "    # Instantiate the model\n",
    "    model = LeNet(use_batchnorm, dropout_prob)\n",
    "    model = model.to(device)\n",
    "    \n",
    "    # Test with a random input\n",
    "    # summary(model, input_size=(1, 32, 32), batch_size=BATCH_SIZE)\n",
    "    return model\n",
    "\n",
    "# model = get_model(USE_BATCHNORM, DROPOUT_PROB)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6d186351-babd-4d4e-886e-d9338ca71640",
   "metadata": {},
   "source": [
    "#### 1. Cross Entropy Loss\n",
    "The standard cross-entropy loss for classification tasks is given by:\n",
    "\n",
    "$$ \\text{Traditional Cross Entropy Loss: } H(y, \\hat{y}) = - \\sum_i y_i \\log(\\hat{y}_i) $$\n",
    "\n",
    " - $y_i$ is a binary indicator of whether class $i$ is the correct classification.  \n",
    " - $p_i$ is the predicted probability of class $i$.\n",
    "\n",
    "#### 2. Label Smoothed Cross Entropy\n",
    "Label Smoothing Cross Entropy Loss introduces a modification to the target distribution:\n",
    "\n",
    " $$ \\text{Label Smoothed Cross Entropy Loss} = - \\sum_i \\left( (1 - \\text{smoothing}) \\cdot 1_{\\{y_i\\}} + \\frac{\\text{smoothing}}{C-1} \\cdot 1_{\\{1 - y_i\\}} \\right) \\cdot \\log(p_i) $$\n",
    "\n",
    "Where:\n",
    "- $1_{\\{y_i\\}}$ is a binary indicator of whether class $i$ is the correct classification.\n",
    "- $p_i$ is the predicted probability of class $i$.\n",
    "- $C$ is the number of classes.\n",
    "- $\\text{smoothing}$  is the smoothing factor.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "5556bcc8-3ff2-4169-b316-0a7b17aa1d2e",
   "metadata": {},
   "outputs": [],
   "source": [
    "class LabelSmoothedCrossEntropy(nn.Module):\n",
    "    def __init__(self, num_classes, smoothing):\n",
    "        super(LabelSmoothedCrossEntropy, self).__init__()\n",
    "        self.num_classes = num_classes\n",
    "        self.smoothing = smoothing\n",
    "        self.confidence = 1.0 - smoothing\n",
    "\n",
    "    def forward(self, input_logits, target):\n",
    "        log_probs = F.log_softmax(input_logits, dim=-1)\n",
    "        true_dist = torch.zeros_like(log_probs)\n",
    "        true_dist.fill_(self.smoothing / (self.num_classes - 1))\n",
    "        true_dist.scatter_(1, target.data.unsqueeze(1), self.confidence)\n",
    "        loss = -torch.sum(true_dist * log_probs) / input_logits.size(0)\n",
    "        return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "262d50c6-de05-416f-8ad7-14792edf403c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_loss(label_smoothing):\n",
    "    # Initialize loss\n",
    "    # criterion = nn.CrossEntropyLoss()\n",
    "    return LabelSmoothedCrossEntropy(num_classes=N_CLASSES, smoothing=label_smoothing)\n",
    "\n",
    "# criterion = get_loss(label_smoothing)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1365bb52-8cf9-414e-bb7e-d87c073b933b",
   "metadata": {},
   "source": [
    "#### Adam w/ Weight Decay"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "3412026a-4b8b-4070-9781-6efb964d3446",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_optimizer(model, lr, weight_decay):\n",
    "    # Initialize optimizer\n",
    "    # optimizer = optim.Adam(model.parameters(), lr=LR)\n",
    "    optimizer = optim.AdamW(model.parameters(), lr=lr, weight_decay=weight_decay)\n",
    "    return optimizer\n",
    "\n",
    "# optimizer =  get_optimizer(LR, WEIGHT_DECAY)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7f78d5cf-5ad7-4d10-be31-cb8d498227c4",
   "metadata": {},
   "source": [
    "#### Cosine Annealing With Warmup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "6ebc561a-59b6-4848-92f0-bb32d4682d1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_lr_lambda(initial_lr, warmup_steps, total_steps):\n",
    "    def lr_lambda(current_step):\n",
    "        if current_step < warmup_steps:\n",
    "            return initial_lr + (1.0 - initial_lr) * float(current_step) / float(max(1, warmup_steps))\n",
    "        else:\n",
    "            return max(0.0, 0.5 * (1.0 + math.cos(math.pi * (current_step - warmup_steps) / float(total_steps - warmup_steps))))\n",
    "    return lr_lambda"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "b45993ef-a7a2-4154-9eeb-d10e5e003e4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_cosine_with_warmup_scheduler(**kwargs):\n",
    "    train_dataloader = kwargs['train_dataloader']\n",
    "    optimizer = kwargs['optimizer']\n",
    "    \n",
    "    len_dataloader = len(train_dataloader)\n",
    "    num_update_steps_per_epoch = max(len_dataloader // kwargs['accumulation_steps'], 1) \n",
    "    num_examples = len(train_dataloader.dataset)\n",
    "    max_steps = math.ceil(kwargs['epochs'] * num_update_steps_per_epoch)\n",
    "    num_warmup_steps = math.ceil(max_steps * kwargs['warmup_proportion'])\n",
    "    lr_lambda = get_lr_lambda(kwargs['lr'], num_warmup_steps, max_steps)\n",
    "    scheduler = lr_scheduler.LambdaLR(optimizer, lr_lambda=lr_lambda)\n",
    "    return scheduler\n",
    "\n",
    "# scheduler = get_cosine_with_warmup_scheduler()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ab52c285-645c-4c4f-a051-565c8eb72c94",
   "metadata": {},
   "source": [
    "#### Adaptive Gradient Clipping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "d4216258-a9d0-4eda-9aa0-e91543d2f4d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def unitwise_norm(x, norm_type=2.0):\n",
    "    if x.ndim <= 1:\n",
    "        return x.norm(norm_type)\n",
    "    else:\n",
    "        return x.norm(norm_type, dim=tuple(range(1, x.ndim)), keepdim=True)\n",
    "\n",
    "\n",
    "def adaptive_clip_grad(parameters, clip_factor=0.01, eps=1e-3, norm_type=2.0):\n",
    "    if isinstance(parameters, torch.Tensor):\n",
    "        parameters = [parameters]\n",
    "    for p in parameters:\n",
    "        if p.grad is None:\n",
    "            continue\n",
    "        p_data = p.detach()\n",
    "        g_data = p.grad.detach()\n",
    "        max_norm = unitwise_norm(p_data, norm_type=norm_type).clamp_(min=eps).mul_(clip_factor)\n",
    "        grad_norm = unitwise_norm(g_data, norm_type=norm_type)\n",
    "        clipped_grad = g_data * (max_norm / grad_norm.clamp(min=1e-6))\n",
    "        new_grads = torch.where(grad_norm < max_norm, g_data, clipped_grad)\n",
    "        p.grad.detach().copy_(new_grads)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "465ce651-83e8-4526-a9fc-bf799b70446b",
   "metadata": {},
   "source": [
    "#### Logging Model Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "e8aa3340-60e2-4583-8867-d197fb467d33",
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_flops(model, input_size):\n",
    "    input_tensor = torch.randn(*input_size).to(device)\n",
    "    flops, params = profile(model, inputs=(input_tensor,))\n",
    "    return flops\n",
    "\n",
    "def calculate_disk_size(model):\n",
    "    # Calculate model disk size in bytes\n",
    "    disk_size_bytes = sum(p.numel() * p.element_size() for p in model.parameters())\n",
    "    # Convert bytes to megabytes\n",
    "    disk_size_mb = disk_size_bytes / (1024 * 1024)\n",
    "    return disk_size_mb\n",
    "\n",
    "def get_model_metrics(model, train_dataloader):\n",
    "    num_params = sum(p.numel() for p in model.parameters())\n",
    "    num_trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
    "\n",
    "    # Set an example input size (adjust based on your actual input size)\n",
    "    input_size = tuple(next(iter(train_dataloader))[0][0].unsqueeze(0).shape)\n",
    "\n",
    "    flops = calculate_flops(model, input_size)\n",
    "    disk_size = calculate_disk_size(model)\n",
    "\n",
    "    return {\n",
    "        \"num_params\": num_params,\n",
    "        \"num_trainable_params\": num_trainable_params,\n",
    "        \"flops\": flops,\n",
    "        \"disk_size\": disk_size\n",
    "    }"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0be3441b-14b2-4251-9bc4-b61b60263606",
   "metadata": {},
   "source": [
    "#### Logging GPU Usage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "02adfbae-1213-4960-b91a-0abf1079256a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_gpu_usage():\n",
    "    try:\n",
    "        # Run nvidia-smi command to get GPU utilization information\n",
    "        result = subprocess.run([\"nvidia-smi\", \"--query-gpu=utilization.gpu\", \"--format=csv,nounits,noheader\"], stdout=subprocess.PIPE, text=True)\n",
    "        gpu_usage = int(result.stdout.strip())\n",
    "        return gpu_usage\n",
    "    except Exception as e:\n",
    "        print(f\"Error getting GPU usage: {e}\")\n",
    "        return None"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6297eb60-ff83-4f9a-abbb-47919289ecc4",
   "metadata": {},
   "source": [
    "#### Train\n",
    "\n",
    "##### Gradient Penalty\n",
    "Its primary purpose is to encourage smoothness in the learned representations of the model by penalizing sharp changes or oscillations in the model's gradients.\n",
    "\n",
    "##### Gradient Accumulation\n",
    "Gradient accumulation adds gradients over an effective batch of size `batch_per_iter * iters_to_accumulate` (`* num_procs` if distributed). The scale should be calibrated for the effective batch, which means inf/NaN checking, step skipping if inf/NaN grads are found, and scale updates should occur at effective-batch granularity. Also, grads should remain scaled, and the scale factor should remain constant, while grads for a given effective batch are accumulated. If grads are unscaled (or the scale factor changes) before accumulation is complete, the next backward pass will add scaled grads to unscaled grads (or grads scaled by a different factor) after which it’s impossible to recover the accumulated unscaled grads `step` must apply.\n",
    "\n",
    "##### Automatic Mixed Precision Training\n",
    "Automatic Mixed Precision (AMP) training is a technique used in deep learning to accelerate training by using a combination of lower-precision and higher-precision numerical representations. The primary idea behind AMP is to use lower-precision data types (such as float16) for some parts of the model computation, while maintaining higher precision (such as float32) for critical numerical stability aspects."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "34cd9acf",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(epoch, train_dataloader, model, criterion, optimizer, scheduler, accumulation_steps):\n",
    "    model.train()\n",
    "    total_loss = 0.0\n",
    "    \n",
    "    # Create a GradScaler once at the beginning of training.\n",
    "    scaler = torch.cuda.amp.GradScaler()\n",
    "    \n",
    "    optimizer.zero_grad()\n",
    "    with tqdm(total=len(train_dataloader), desc=f\"Epoch {epoch}/Training\", unit=\"batch\") as pbar:\n",
    "        for i, (images, labels) in enumerate(train_dataloader):\n",
    "            images, labels = images.to(device), labels.to(device)\n",
    "\n",
    "            # Runs the forward pass with autocasting.\n",
    "            with torch.autocast(device_type=device.type, dtype=torch.float16, enabled=True):\n",
    "                output = model(images)\n",
    "                loss = criterion(output, labels)\n",
    "\n",
    "            # # Scales the loss for autograd.grad's backward pass, producing scaled_grad_params\n",
    "            # scaled_grad_params = torch.autograd.grad(outputs=scaler.scale(loss), inputs=model.parameters(), create_graph=True)\n",
    "    \n",
    "            # # Creates unscaled grad_params before computing the penalty. scaled_grad_params are\n",
    "            # # not owned by any optimizer, so ordinary division is used instead of scaler.unscale_:\n",
    "            # inv_scale = 1. / scaler.get_scale()\n",
    "            # grad_params = [p * inv_scale for p in scaled_grad_params]\n",
    "    \n",
    "            # # Computes the penalty term and adds it to the loss\n",
    "            # with torch.autocast(device_type=device.type, dtype=torch.float16, enabled=True):\n",
    "            #     grad_norm = 0\n",
    "            #     for grad in grad_params:\n",
    "            #         grad_norm += grad.pow(2).sum()\n",
    "            #     grad_norm = grad_norm.sqrt()\n",
    "            #     loss = loss + grad_norm\n",
    "\n",
    "            # Scale the loss by accumulation steps\n",
    "            if accumulation_steps > 1:\n",
    "                with torch.autocast(device_type=device.type, dtype=torch.float16, enabled=True):\n",
    "                    loss = loss / accumulation_steps\n",
    "                \n",
    "            # Applies scaling to the backward call as usual.\n",
    "            # Accumulates leaf gradients that are correctly scaled.\n",
    "            scaler.scale(loss).backward()\n",
    "\n",
    "            if (i + 1) % accumulation_steps == 0:\n",
    "                # Unscales the gradients of optimizer's assigned params in-place\n",
    "                scaler.unscale_(optimizer)\n",
    "                # torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=5.0)\n",
    "                adaptive_clip_grad(model.parameters())\n",
    "\n",
    "                # optimizer's gradients are already unscaled, so scaler.step does not unscale them,\n",
    "                # although it still skips optimizer.step() if the gradients contain infs or NaNs.\n",
    "                scaler.step(optimizer)\n",
    "\n",
    "                mlflow.log_metric(\"lr\", optimizer.param_groups[0][\"lr\"])\n",
    "                scheduler.step()\n",
    "\n",
    "                # Updates the scale for next iteration.\n",
    "                scaler.update()\n",
    "                \n",
    "                for param in model.parameters():\n",
    "                    param.grad = None\n",
    "            \n",
    "            total_loss += loss.item()\n",
    "            pbar.update(1)\n",
    "            pbar.set_postfix(loss=total_loss / (i+1), lr=f\"{optimizer.param_groups[0]['lr']:.5f}\")\n",
    "\n",
    "    avg_loss = total_loss / len(train_dataloader.dataset)\n",
    "    mlflow.log_metric(\"train_loss\", avg_loss)\n",
    "        \n",
    "    print(f\"[Train][Epoch {epoch}] Average Loss: {avg_loss:.5f}, Updated Learning Rate: {optimizer.param_groups[0]['lr']}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a217af40-7a4e-4a1e-8cfc-133fc220090f",
   "metadata": {},
   "source": [
    "#### Eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "65850723-b84a-4915-8429-644a1805ba97",
   "metadata": {},
   "outputs": [],
   "source": [
    "class BestLossTracker:\n",
    "    def __init__(self):\n",
    "        self.best_loss = float('inf')\n",
    "    \n",
    "    def update_best_loss(self, avg_loss):\n",
    "        if avg_loss < self.best_loss:\n",
    "            self.best_loss = avg_loss\n",
    "            return True\n",
    "        return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "c2ca1f2b-086a-4160-a0a6-f0abb7c43100",
   "metadata": {},
   "outputs": [],
   "source": [
    "class EarlyStopping:\n",
    "    def __init__(self, patience=5, delta=0.0):\n",
    "        self.patience = patience\n",
    "        self.delta = delta\n",
    "        self.counter = 0\n",
    "        self.best_loss = float('inf')\n",
    "        self.early_stop = False\n",
    "\n",
    "    def update_best_loss(self, val_loss):\n",
    "        if val_loss < self.best_loss - self.delta:\n",
    "            self.best_loss = val_loss\n",
    "            self.counter = 0\n",
    "        else:\n",
    "            self.counter += 1\n",
    "            if self.counter >= self.patience:\n",
    "                self.early_stop = True\n",
    "\n",
    "        return not self.early_stop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "de2f38e6",
   "metadata": {},
   "outputs": [],
   "source": [
    "def test(epoch, best_loss_tracker, test_dataloader, model, criterion):\n",
    "    model.eval()\n",
    "    total_correct = 0\n",
    "    total_loss = 0.0\n",
    "\n",
    "    with tqdm(total=len(test_dataloader), desc=f\"Epoch {epoch}/Testing\", unit=\"batch\") as pbar: \n",
    "        with torch.no_grad():\n",
    "            for i, (images, labels) in enumerate(test_dataloader):\n",
    "                images, labels = images.to(device), labels.to(device)\n",
    "                \n",
    "                output = model(images)                \n",
    "                loss = criterion(output, labels)\n",
    "\n",
    "                total_loss += loss.item()\n",
    "                pbar.update(1)\n",
    "                pbar.set_postfix(loss=total_loss / (i+1))\n",
    "                \n",
    "                pred = output.detach().max(1)[1]\n",
    "                total_correct += pred.eq(labels.view_as(pred)).sum()\n",
    "            \n",
    "    avg_loss = total_loss / len(test_dataloader.dataset)\n",
    "    accuracy = total_correct / len(test_dataloader.dataset)\n",
    "\n",
    "    mlflow.log_metric(\"val_loss\", avg_loss)\n",
    "    mlflow.log_metric(\"val_accuracy\", accuracy)\n",
    "    \n",
    "    if best_loss_tracker.update_best_loss(avg_loss):\n",
    "        checkpoint_name = f\"checkpoint_best_epoch_{epoch}.pth\"\n",
    "        checkpoint_path = str(SAVE_DIR / \"checkpoints\" / checkpoint_name)\n",
    "        torch.save(model.state_dict(), checkpoint_path)\n",
    "        print(f\"[Test][Epoch {epoch}] New best model found! Saving checkpoint. Loss: {avg_loss:.5f}, Accuracy: {accuracy:.3f}\")\n",
    "    else:\n",
    "        print(f\"[Test][Epoch {epoch}] Loss: {avg_loss:.5f}, Accuracy: {accuracy:.3f}\")\n",
    "    \n",
    "    return avg_loss"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2e589a1d-848f-4045-813c-662ef287e348",
   "metadata": {},
   "source": [
    "#### Run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "0c3cd024",
   "metadata": {},
   "outputs": [],
   "source": [
    "def run(run_name, **kwargs):\n",
    "    print(kwargs)\n",
    "    with mlflow.start_run(run_name=run_name):\n",
    "        mlflow.log_param(\"torch_version\", torch.__version__)\n",
    "        mlflow.log_param(\"mlflow_version\", mlflow.__version__)\n",
    "\n",
    "        hyper_params = {\n",
    "            \"accumulation_steps\": kwargs['accumulation_steps'], \n",
    "            \"batch_size\": kwargs['batch_size'], \n",
    "            \"batchnorm\": kwargs['batchnorm'],\n",
    "            \"dropout\":kwargs['dropout_prob'],\n",
    "            \"epochs\": kwargs['epochs'], \n",
    "            \"label_smoothing\": kwargs['label_smoothing'],\n",
    "            \"start_lr\": kwargs['lr'],\n",
    "            \"warmup_proportion\": kwargs['warmup_proportion'],\n",
    "            \"weight_decay\": kwargs['weight_decay']\n",
    "        }\n",
    "\n",
    "        mlflow.log_params(hyper_params)\n",
    "        mlflow.set_tags({\"model\": \"lenet\", \"dataset\": \"mnist\"})\n",
    "\n",
    "        train_dataloader, test_dataloader = get_dataloaders(kwargs['batch_size'])\n",
    "        model = get_model(kwargs['dropout_prob'], kwargs['batchnorm'])\n",
    "        criterion = get_loss(kwargs['label_smoothing'])\n",
    "        optimizer = get_optimizer(model, kwargs['lr'], kwargs['weight_decay'])\n",
    "        \n",
    "        scheduler_kwargs = {\n",
    "            'epochs': kwargs['epochs'],\n",
    "            'train_dataloader':train_dataloader,\n",
    "            'optimizer': optimizer,\n",
    "            'lr':kwargs['lr'],\n",
    "            'accumulation_steps': kwargs['accumulation_steps'],\n",
    "            'warmup_proportion': kwargs['warmup_proportion']\n",
    "        }\n",
    "        scheduler = get_cosine_with_warmup_scheduler(**scheduler_kwargs)\n",
    "        \n",
    "        mlflow.pytorch.log_model(model, \"model\")\n",
    "        mlflow.log_param(\"start_time\", datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\"))\n",
    "\n",
    "        model_metrics = get_model_metrics(model, train_dataloader)\n",
    "        mlflow.log_param(\"num_params\", model_metrics['num_params'])\n",
    "        mlflow.log_param(\"num_trainable_params\", model_metrics['num_trainable_params'])\n",
    "        mlflow.log_param(\"flops\", model_metrics['flops'])\n",
    "        mlflow.log_param(\"disk_size\", model_metrics['disk_size'])\n",
    "\n",
    "        best_loss_tracker = BestLossTracker()\n",
    "        early_stopping = EarlyStopping(patience=1, delta=1e-6)\n",
    "\n",
    "        start_time = time.time()\n",
    "        for epoch in range(kwargs['epochs']):\n",
    "            train(epoch, train_dataloader, model, criterion, optimizer, scheduler, kwargs['accumulation_steps'])\n",
    "            val_loss = test(epoch, best_loss_tracker, test_dataloader, model, criterion)\n",
    "            \n",
    "            if get_gpu_usage() is not None:\n",
    "                mlflow.log_metric(\"gpu\", get_gpu_usage(), step=epoch)\n",
    "\n",
    "            if not early_stopping.update_best_loss(val_loss):\n",
    "                print(\"Early stopping triggered.\")\n",
    "                break\n",
    "\n",
    "\n",
    "        total_time = time.time() - start_time\n",
    "        avg_epoch_time = total_time / kwargs['epochs']\n",
    "        avg_batch_time = avg_epoch_time / len(train_dataloader)\n",
    "\n",
    "        mlflow.log_metric(\"total_training_time\", total_time)\n",
    "        mlflow.log_metric(\"average_epoch_time\", avg_epoch_time)\n",
    "        mlflow.log_metric(\"average_batch_time\", avg_batch_time)\n",
    "        mlflow.log_param(\"end_time\", datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\"))\n",
    "\n",
    "    # torch.cuda.memory._dump_snapshot(\"snapshot.pickle\")\n",
    "    return val_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "8a212d0e-9eb7-4715-94e8-80ea280c61ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "def objective(trial):\n",
    "    EPOCHS = 5\n",
    "    accumulation_steps = trial.suggest_int('accumulation_steps', 1, 3)  # Accumulation steps\n",
    "    batchnorm = trial.suggest_categorical('batchnorm', [True, False])  # Use batch normalization\n",
    "    batch_size = trial.suggest_int('batch_size', 256, 2048, log=True)  # Batch size\n",
    "    dropout_prob = trial.suggest_float('dropout_prob', 0.0, 0.4)  # Dropout probability\n",
    "    label_smoothing = trial.suggest_float('label_smoothing', 0.0, 0.25)  # Label smoothing\n",
    "    lr = trial.suggest_float('lr', 1e-3, 1e-1, log=True)  # Learning rate\n",
    "    warmup_proportion = trial.suggest_float('warmup_proportion', 0.0, 0.1)  # Warmup proportion\n",
    "    weight_decay = trial.suggest_float('weight_decay', 1e-6, 1e-3, log=True)  # Weight decay\n",
    "\n",
    "    run_name = f\"run_{trial.number}\"  # Create a unique run_name based on the trial number\n",
    "    \n",
    "    # Update the model and optimizer with the sampled hyperparameters\n",
    "    kwargs = {\n",
    "        'epochs': EPOCHS,\n",
    "        'accumulation_steps': accumulation_steps,\n",
    "        'batchnorm': batchnorm,\n",
    "        'batch_size': batch_size,\n",
    "        'dropout_prob': dropout_prob,\n",
    "        'label_smoothing': label_smoothing,\n",
    "        'lr': lr,\n",
    "        'warmup_proportion': warmup_proportion,\n",
    "        'weight_decay': weight_decay\n",
    "    }\n",
    "    \n",
    "    # Train and evaluate the model\n",
    "    val_loss = run(run_name=run_name, **kwargs)\n",
    "\n",
    "    # Return the validation loss for optimization\n",
    "    return val_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "83991d05-0990-4004-8272-d942b771c985",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 22:30:04,648] A new study created in memory with name: no-name-105c6438-58f0-4288-beb6-8d0e3d0c0428\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'epochs': 5, 'accumulation_steps': 1, 'batchnorm': True, 'batch_size': 1221, 'dropout_prob': 0.1451223008122237, 'label_smoothing': 0.2069263707464151, 'lr': 0.0014513999543911257, 'warmup_proportion': 0.04213229508497701, 'weight_decay': 0.0006887755240006057}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 22:30:07 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 22:30:11 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6a96e9f8922c45ee944847b6bfdb9889",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/50 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00192, Updated Learning Rate: 0.001358111449214277\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1bf663ad26b948c7bb5bba6a0a0ed15c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/9 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.51094, Accuracy: 0.104\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c34a6e40b63e4d6ca5504d777a138c52",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/50 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00192, Updated Learning Rate: 0.0010089120347935119\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e76bee08c7cb4f738ce69a82e69fece4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/9 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 22:31:02,752] Trial 0 finished with value: 74.44430390625 and parameters: {'accumulation_steps': 1, 'batchnorm': True, 'batch_size': 1221, 'dropout_prob': 0.1451223008122237, 'label_smoothing': 0.2069263707464151, 'lr': 0.0014513999543911257, 'warmup_proportion': 0.04213229508497701, 'weight_decay': 0.0006887755240006057}. Best is trial 0 with value: 74.44430390625.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] Loss: 74.44430, Accuracy: 0.103\n",
      "Early stopping triggered.\n",
      "{'epochs': 5, 'accumulation_steps': 2, 'batchnorm': True, 'batch_size': 306, 'dropout_prob': 0.3243738591345986, 'label_smoothing': 0.09768230864805996, 'lr': 0.009730465780858325, 'warmup_proportion': 0.039214878621064514, 'weight_decay': 6.365078881414862e-05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 22:31:04 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 22:31:07 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "db49bb2af1284e9da129ca752a2b7080",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/197 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00378, Updated Learning Rate: 0.009084057769771836\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7f5160be1e114d3682c78db3a0e68b4f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/33 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 609.40280, Accuracy: 0.155\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f1d38b681d7945169d6fd787d0bbf7a7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/197 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00378, Updated Learning Rate: 0.00673458553816387\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a77b5670f42b4617b1f20e1c6d4f59d8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/33 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 608.46390, Accuracy: 0.155\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dbd1a0e04e1a4d5da15341499c708444",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/197 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00378, Updated Learning Rate: 0.0036112541194416154\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "55595c71e40b4a8096427ee3a6890789",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/33 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 22:34:23,417] Trial 1 finished with value: 609.92189375 and parameters: {'accumulation_steps': 2, 'batchnorm': True, 'batch_size': 306, 'dropout_prob': 0.3243738591345986, 'label_smoothing': 0.09768230864805996, 'lr': 0.009730465780858325, 'warmup_proportion': 0.039214878621064514, 'weight_decay': 6.365078881414862e-05}. Best is trial 0 with value: 74.44430390625.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] Loss: 609.92189, Accuracy: 0.156\n",
      "Early stopping triggered.\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1455, 'dropout_prob': 0.3003211659471703, 'label_smoothing': 0.16495847481123055, 'lr': 0.0021027838170086114, 'warmup_proportion': 0.07511850493740019, 'weight_decay': 2.9647313463561168e-05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 22:34:25 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 22:34:28 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "960f1628057948919b33b56c78305519",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/42 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00048, Updated Learning Rate: 0.002022751373419413\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ec008697c4fd4334a7ead371cc8bf870",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00095, Accuracy: 0.746\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1f592d4e62094788b1fc1462b3ac69d8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/42 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00034, Updated Learning Rate: 0.0015470146232984935\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "408703b0a07f43a7824063ead004efbb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00080, Accuracy: 0.887\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aa485251ff1e436bbf5518438cbb9033",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/42 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00029, Updated Learning Rate: 0.0008462755225090491\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9b6a78c044a84138b909f81b84ffa68b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00075, Accuracy: 0.921\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "88d969630935435ab3ce9cc03b81f231",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/42 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00027, Updated Learning Rate: 0.00023865497264947906\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "71879dd43e3e4f4a9cc4d4d6865b78db",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00073, Accuracy: 0.932\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "739871257dbf489e88f9165f050a44ce",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/42 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00027, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3893bd115d8b4cd29cf33a8d3934de67",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 22:37:44,600] Trial 2 finished with value: 0.0007258852958679199 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1455, 'dropout_prob': 0.3003211659471703, 'label_smoothing': 0.16495847481123055, 'lr': 0.0021027838170086114, 'warmup_proportion': 0.07511850493740019, 'weight_decay': 2.9647313463561168e-05}. Best is trial 2 with value: 0.0007258852958679199.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00073, Accuracy: 0.934\n",
      "{'epochs': 5, 'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 439, 'dropout_prob': 0.030600731469720577, 'label_smoothing': 0.055868582104746434, 'lr': 0.026147846541169317, 'warmup_proportion': 0.07288919886552776, 'weight_decay': 8.069520934441288e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 22:37:46 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 22:37:49 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bc811442103d48a0aaa1e1c772b17ccc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/137 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00103, Updated Learning Rate: 0.024963918865667283\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ab1ce6824a4144e3977a155f6dbaafcc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/23 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00125, Accuracy: 0.940\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9c708daf215d4b8dbfd9dd6aae207807",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/137 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00059, Updated Learning Rate: 0.018922061049586894\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b035faac1cbd43e582b46c4795b81db3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/23 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00100, Accuracy: 0.977\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "43f8be04a96745beb7ab3dcd122c36c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/137 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00054, Updated Learning Rate: 0.01029196536405235\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4a85975a26fc46c4b990fad4301a79d9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/23 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00094, Accuracy: 0.984\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2fcb83b7fb20425daa7ed9678920da50",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/137 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00051, Updated Learning Rate: 0.0028930938864643566\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "00fd48d8b6474a19acfd24df4c1d7830",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/23 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00094, Accuracy: 0.981\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6b629ec5201b4fc19ce962409e6692b7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/137 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00050, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6e86694b5c884fea92d826d30604688d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/23 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 22:41:08,743] Trial 3 finished with value: 0.0009276029706001282 and parameters: {'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 439, 'dropout_prob': 0.030600731469720577, 'label_smoothing': 0.055868582104746434, 'lr': 0.026147846541169317, 'warmup_proportion': 0.07288919886552776, 'weight_decay': 8.069520934441288e-06}. Best is trial 2 with value: 0.0007258852958679199.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00093, Accuracy: 0.983\n",
      "{'epochs': 5, 'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 545, 'dropout_prob': 0.19918007956039283, 'label_smoothing': 0.11565733474080886, 'lr': 0.05043128858476219, 'warmup_proportion': 0.07199384103602492, 'weight_decay': 3.1704728283556938e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 22:41:10 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 22:41:13 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "56e271e6f9134d7cb8f904630f333b26",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/111 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00100, Updated Learning Rate: 0.048123181438563506\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7644a7d7a18840558291f423e4376a0d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/19 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00150, Accuracy: 0.956\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3b5dfd5fcde34eeca3a1b6aed459a965",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/111 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00072, Updated Learning Rate: 0.036455224119113096\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fec26184b85a4e6ab576b41caa5e2507",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/19 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00135, Accuracy: 0.971\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "24def386775643b0b5cd26402271b38b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/111 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00068, Updated Learning Rate: 0.01982118376387369\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7c3c90ec862e4dc79c9f385bf41f1c47",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/19 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00131, Accuracy: 0.976\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1178e04c6c1145ebae7d9792ed7345d5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/111 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00066, Updated Learning Rate: 0.005570625650037887\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5349ccf7271d413497ef9d4d042eae20",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/19 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00128, Accuracy: 0.983\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "63b3628133bf4acaa682488071197212",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/111 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00065, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "54ba8f26f81048698792e2fb8ddd03f4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/19 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 22:43:16,743] Trial 4 finished with value: 0.0012740315675735473 and parameters: {'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 545, 'dropout_prob': 0.19918007956039283, 'label_smoothing': 0.11565733474080886, 'lr': 0.05043128858476219, 'warmup_proportion': 0.07199384103602492, 'weight_decay': 3.1704728283556938e-06}. Best is trial 2 with value: 0.0007258852958679199.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00127, Accuracy: 0.981\n",
      "{'epochs': 5, 'accumulation_steps': 1, 'batchnorm': True, 'batch_size': 529, 'dropout_prob': 0.07592134919382998, 'label_smoothing': 0.08181435903512166, 'lr': 0.03126306236108547, 'warmup_proportion': 0.08372642835652444, 'weight_decay': 4.000280229405524e-05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 22:43:18 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 22:43:21 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "79c149aac343474a83aa8caf39d5f4a4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/114 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00437, Updated Learning Rate: 0.030046034750972717\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "81bcf13303054d95bb8d08386bc83c91",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/19 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 721.78963, Accuracy: 0.094\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "46cb7ee7ac0d4353ab1d0fb8d9edcf7b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/114 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00437, Updated Learning Rate: 0.022953472326570905\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bfcecc1b8049449481a56302c88efdb4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/19 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 22:44:28,254] Trial 5 finished with value: 1165.336025 and parameters: {'accumulation_steps': 1, 'batchnorm': True, 'batch_size': 529, 'dropout_prob': 0.07592134919382998, 'label_smoothing': 0.08181435903512166, 'lr': 0.03126306236108547, 'warmup_proportion': 0.08372642835652444, 'weight_decay': 4.000280229405524e-05}. Best is trial 2 with value: 0.0007258852958679199.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] Loss: 1165.33603, Accuracy: 0.094\n",
      "Early stopping triggered.\n",
      "{'epochs': 5, 'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1030, 'dropout_prob': 0.3754555768400405, 'label_smoothing': 0.13575912327829695, 'lr': 0.01610731483882752, 'warmup_proportion': 0.06889986899514466, 'weight_decay': 0.0006701132602394994}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 22:44:30 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 22:44:33 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "68162350640e48f79f2c25689b57acb4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/59 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00121, Updated Learning Rate: 0.01535491641644112\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "11fad8c391a7478e8eb0bfd85cf3094e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00088, Accuracy: 0.946\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d493d90dd9b941bca5ae1433d5ee033d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/59 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00085, Updated Learning Rate: 0.011619149342767022\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d821a5163f294bf695397ef7a91e4997",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00079, Accuracy: 0.975\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7b110b55415949a2936a6c12b913d4bf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/59 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00081, Updated Learning Rate: 0.006313031219334113\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a17b8d81351d4bc29a82c04c742b2082",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00077, Accuracy: 0.983\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2b0780a9be07410d85c371c12b003af2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/59 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00079, Updated Learning Rate: 0.0017735377971463725\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dfd8a0cca87148208c6828cacac4244d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00077, Accuracy: 0.980\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "638cb2391b854c92833eaee2cb056399",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/59 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00078, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1208978f88a24d9a8a5a4894aeb79bf4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 22:47:12,219] Trial 6 finished with value: 0.0007622253298759461 and parameters: {'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1030, 'dropout_prob': 0.3754555768400405, 'label_smoothing': 0.13575912327829695, 'lr': 0.01610731483882752, 'warmup_proportion': 0.06889986899514466, 'weight_decay': 0.0006701132602394994}. Best is trial 2 with value: 0.0007258852958679199.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00076, Accuracy: 0.980\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1933, 'dropout_prob': 0.01661403758630624, 'label_smoothing': 0.04202568800819467, 'lr': 0.007319596288072455, 'warmup_proportion': 0.07692142464279984, 'weight_decay': 2.683958287613536e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 22:47:14 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 22:47:17 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6f6f6357862941be9d75bf2eb98d428d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/32 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00033, Updated Learning Rate: 0.007016606362974937\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "df6057fa58444a49b9aa54ff83eee9c8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00047, Accuracy: 0.850\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2263b104cd584aa5a86d6a461b0ff594",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/32 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00016, Updated Learning Rate: 0.005343543315260655\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7f5555f9d06c4249a06b990b54b7910a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00035, Accuracy: 0.906\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6d55767f982240b8aa305d3e47c1a9cd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/32 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00012, Updated Learning Rate: 0.002915190205073188\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3b1e8c45dc204c21a5666fc56c17cb45",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00029, Accuracy: 0.940\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "67866dd2de6d4b79823a5db172d7a704",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/32 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00010, Updated Learning Rate: 0.0008208514020082457\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9fa3bdd1eaac4e6b96543e054ec40416",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00027, Accuracy: 0.949\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c394bd3876cc4b2a915cad443b3bc642",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/32 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00010, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ae50153972f842afa650d30f9b457581",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 22:49:58,341] Trial 7 finished with value: 0.00027190839052200315 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1933, 'dropout_prob': 0.01661403758630624, 'label_smoothing': 0.04202568800819467, 'lr': 0.007319596288072455, 'warmup_proportion': 0.07692142464279984, 'weight_decay': 2.683958287613536e-06}. Best is trial 7 with value: 0.00027190839052200315.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00027, Accuracy: 0.950\n",
      "{'epochs': 5, 'accumulation_steps': 2, 'batchnorm': True, 'batch_size': 383, 'dropout_prob': 0.24245313725793186, 'label_smoothing': 0.15437927876340252, 'lr': 0.012435366248203749, 'warmup_proportion': 0.06288194564868863, 'weight_decay': 0.0001984218589233328}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 22:50:00 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 22:50:03 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dc8b004ac6324605b66b1a91ee8813a6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/157 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00301, Updated Learning Rate: 0.011799567071853628\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7b8673f2088f4f4bb9419d2f005a9f41",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/27 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 1198.77133, Accuracy: 0.089\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "20c93c11b66549b389ddefe2ff462368",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/157 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00301, Updated Learning Rate: 0.008884397288523672\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4afcd348480e4d75a17793ca3da8f02c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/27 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 22:51:10,388] Trial 8 finished with value: 1205.289178125 and parameters: {'accumulation_steps': 2, 'batchnorm': True, 'batch_size': 383, 'dropout_prob': 0.24245313725793186, 'label_smoothing': 0.15437927876340252, 'lr': 0.012435366248203749, 'warmup_proportion': 0.06288194564868863, 'weight_decay': 0.0001984218589233328}. Best is trial 7 with value: 0.00027190839052200315.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] Loss: 1205.28918, Accuracy: 0.088\n",
      "Early stopping triggered.\n",
      "{'epochs': 5, 'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 514, 'dropout_prob': 0.09840609444828657, 'label_smoothing': 0.15437088084394157, 'lr': 0.001582783755459807, 'warmup_proportion': 0.0026520366753583604, 'weight_decay': 1.4807182991689228e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 22:51:12 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 22:51:15 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cbacded707a24828a93cd5de914dca0f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/117 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00147, Updated Learning Rate: 0.0014356624092111024\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e18faa69fc5948528fc9af3f35ac2034",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00208, Accuracy: 0.908\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "67c121eca7144712bf0cf43cb502ffb2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/117 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00105, Updated Learning Rate: 0.001040849272872664\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9b105d857a4249f29b5657ab67a10b4d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00190, Accuracy: 0.949\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d605fd2e28cf4e40951b30f1198d8ec2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/117 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00099, Updated Learning Rate: 0.0005501133680549705\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "daf2c0e265a540c5a81eaf22bad361e7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00186, Accuracy: 0.955\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9c74f46a2dc4407e96675bb863afcf55",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/117 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00096, Updated Learning Rate: 0.0001521552404177181\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e90014f216bc45859fd6bf4334ad86bd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00183, Accuracy: 0.960\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fcb9af6b8ed44ad39ef0dab67ccfb4a1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/117 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00095, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "545895cc98e34e55bf023c8bb9b6bc68",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/20 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 22:53:18,356] Trial 9 finished with value: 0.0018298095524311067 and parameters: {'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 514, 'dropout_prob': 0.09840609444828657, 'label_smoothing': 0.15437088084394157, 'lr': 0.001582783755459807, 'warmup_proportion': 0.0026520366753583604, 'weight_decay': 1.4807182991689228e-06}. Best is trial 7 with value: 0.00027190839052200315.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00183, Accuracy: 0.958\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1981, 'dropout_prob': 0.010637250040331109, 'label_smoothing': 0.0011370928764577112, 'lr': 0.004429305809869871, 'warmup_proportion': 0.09301320716693588, 'weight_decay': 7.299487103407476e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 22:53:20 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 22:53:23 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e4342e27df4346e897e4b3a9324a6d93",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/31 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00033, Updated Learning Rate: 0.004295745897304371\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "38da2e46e2914e609b5f846d792c5270",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00039, Accuracy: 0.791\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f3a4a0aa0c634ebc8a4d8e1e0fbf4b0e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/31 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00015, Updated Learning Rate: 0.0033219793574024038\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4c3b44454ad144caa1f83ffb39a991ce",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00026, Accuracy: 0.859\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "95a1b629c3264e42bf323bc52d72e5c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/31 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00009, Updated Learning Rate: 0.0018300824638282107\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7178399ca3af4cdea99b9d08e25e0984",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00014, Accuracy: 0.926\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "12cf300d40ed4edc80e7f0f09f1b684f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/31 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00007, Updated Learning Rate: 0.0005181303536722262\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "26a0b7ad43d745f4b484272e0a0cbe71",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00011, Accuracy: 0.945\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cf36bde845ad42ae819f16d502405af4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/31 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00006, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b750023ff2d447c5bf5b85e55b7e60f0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 22:56:38,730] Trial 10 finished with value: 0.00010674896910786629 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1981, 'dropout_prob': 0.010637250040331109, 'label_smoothing': 0.0011370928764577112, 'lr': 0.004429305809869871, 'warmup_proportion': 0.09301320716693588, 'weight_decay': 7.299487103407476e-06}. Best is trial 10 with value: 0.00010674896910786629.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00011, Accuracy: 0.946\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 2004, 'dropout_prob': 0.010782012196208663, 'label_smoothing': 0.0024582737850606, 'lr': 0.004736243327261055, 'warmup_proportion': 0.09874362627349086, 'weight_decay': 8.001240986920908e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 22:56:40 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 22:56:43 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e95a6615422348ecb9921705b164bc25",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00033, Updated Learning Rate: 0.004593428116067383\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3008f76952384b1c9d51c2b0d2285f62",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00046, Accuracy: 0.718\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "83797e9d449541f5aaf5df99c919eb31",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00015, Updated Learning Rate: 0.0035521824954457912\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "be8fd1963bc543399d9ecb2d6895d946",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00019, Accuracy: 0.891\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c0d77f9e96d14e879fa1063a091c6537",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00009, Updated Learning Rate: 0.0019569016522475073\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6406aabf9d724ac1a5e957cc5212c49e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00013, Accuracy: 0.930\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2a77a10b574d484f907ea47e0299dbe2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00007, Updated Learning Rate: 0.0005540352225766926\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e45c346aeb2348dd9a4a950b59dd31c8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00011, Accuracy: 0.936\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d9a730f553f14edab89ed59b7195bf27",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00007, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d24330a6ba864dcd8755ad584965fdb9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 22:58:57,002] Trial 11 finished with value: 0.00010795635730028152 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 2004, 'dropout_prob': 0.010782012196208663, 'label_smoothing': 0.0024582737850606, 'lr': 0.004736243327261055, 'warmup_proportion': 0.09874362627349086, 'weight_decay': 8.001240986920908e-06}. Best is trial 10 with value: 0.00010674896910786629.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00011, Accuracy: 0.939\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 2030, 'dropout_prob': 0.098927511782571, 'label_smoothing': 0.0009805456348906938, 'lr': 0.004033572769646186, 'warmup_proportion': 0.0999264569274631, 'weight_decay': 1.0877697810098024e-05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 22:58:59 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 22:59:02 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8cc39c9d0b5a4861a6158bc974aa0b0f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00034, Updated Learning Rate: 0.003911945668342843\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8ee74a3b20074c65a61eba70ee3a3f3f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00041, Accuracy: 0.741\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "df25138e3e074e898259c6eafbf31b10",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00015, Updated Learning Rate: 0.0030251795772346397\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4c1b6411b0264af1adc9a56f011c12b2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00019, Accuracy: 0.886\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f059f120ddf34df3b96ae1fa3870944d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00009, Updated Learning Rate: 0.0016665751043550866\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "06dca4d43758433094d2305d3aa680bf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00013, Accuracy: 0.924\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "52d5b69a350d40a6a3451efb70f601b4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00007, Updated Learning Rate: 0.0004718383817713501\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "feb808f402424446832bd00679b83f29",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00010, Accuracy: 0.940\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8e4507fea32c404c9244d44fc19fe63e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00006, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7472dce7affc46449538d3da50678183",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 23:01:09,441] Trial 12 finished with value: 9.675593301653863e-05 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 2030, 'dropout_prob': 0.098927511782571, 'label_smoothing': 0.0009805456348906938, 'lr': 0.004033572769646186, 'warmup_proportion': 0.0999264569274631, 'weight_decay': 1.0877697810098024e-05}. Best is trial 12 with value: 9.675593301653863e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00010, Accuracy: 0.943\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 868, 'dropout_prob': 0.11599471913971118, 'label_smoothing': 0.010739719762420634, 'lr': 0.004361838616412535, 'warmup_proportion': 0.09800669471991089, 'weight_decay': 1.0436728592947257e-05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 23:01:11 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 23:01:14 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e96704d2520440b78b670806f6d92104",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/70 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00059, Updated Learning Rate: 0.004240236115167705\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "02bcd091e2f442d9b889861f11b2cfb5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/12 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00051, Accuracy: 0.888\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1d5c00b187014bb7840ac295bb6fb2a0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/70 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00019, Updated Learning Rate: 0.0032905249432587974\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "97e35662a4fc47c88b3444ca1237fbe8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/12 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00032, Accuracy: 0.943\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "062f451ea54d4f34bab099f13d963802",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/70 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00013, Updated Learning Rate: 0.0018167728796290904\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "04aa74b943344850ae4b33e3a1cfe81f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/12 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00027, Accuracy: 0.956\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "630955517e5d4f638f623f7b71fef7f5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/70 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00012, Updated Learning Rate: 0.0005149986909490671\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a6c5e171ccdb46e784ce32e4619618b7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/12 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00025, Accuracy: 0.964\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9c648328c7014f4bb7dbd464a660c6ed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/70 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00011, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ebfce704864b4b208e29b83e3b9a09e8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/12 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 23:03:16,832] Trial 13 finished with value: 0.00025169426053762436 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 868, 'dropout_prob': 0.11599471913971118, 'label_smoothing': 0.010739719762420634, 'lr': 0.004361838616412535, 'warmup_proportion': 0.09800669471991089, 'weight_decay': 1.0436728592947257e-05}. Best is trial 12 with value: 9.675593301653863e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] Loss: 0.00025, Accuracy: 0.962\n",
      "Early stopping triggered.\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1445, 'dropout_prob': 0.1745769427699067, 'label_smoothing': 0.03880282952132687, 'lr': 0.0034334122336878415, 'warmup_proportion': 0.019289416776569458, 'weight_decay': 1.4075059784864738e-05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 23:03:18 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 23:03:21 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e0d1a7fb28f04d039131016ea766c447",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/42 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00043, Updated Learning Rate: 0.0031762790743964675\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c9bc9ad4dd8e4ffeb390555d0cd6c3fd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00061, Accuracy: 0.784\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fff8629249494780befed684a1bead11",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/42 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00022, Updated Learning Rate: 0.002336851894846296\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "04f26af64f7b405cb2de3cf4973d8b84",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00039, Accuracy: 0.911\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "904c538287e2460980425b084943513b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/42 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00016, Updated Learning Rate: 0.001246907187833379\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2b3b26c7cf1a499fb5412bdfc0183833",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00033, Accuracy: 0.942\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "694db4af822a43e78613df99fc107f31",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/42 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00014, Updated Learning Rate: 0.00034674506142510825\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e5e39c74e0204ef6b1fbef53303b1501",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00032, Accuracy: 0.945\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4a26a6e1b97c4f3092551a8b6d68eb9c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/42 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00014, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "247b587d6b984533babe4ca37f708f15",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 23:06:38,870] Trial 14 finished with value: 0.0003169318586587906 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1445, 'dropout_prob': 0.1745769427699067, 'label_smoothing': 0.03880282952132687, 'lr': 0.0034334122336878415, 'warmup_proportion': 0.019289416776569458, 'weight_decay': 1.4075059784864738e-05}. Best is trial 12 with value: 9.675593301653863e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00032, Accuracy: 0.947\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 768, 'dropout_prob': 0.06100809238217579, 'label_smoothing': 0.06658468571810651, 'lr': 0.0027114201825804746, 'warmup_proportion': 0.08855932290236279, 'weight_decay': 0.00010666774678723855}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 23:06:40 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 23:06:43 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "67d63a5549e4436fb8bb29c605a9343b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/79 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00072, Updated Learning Rate: 0.0026183319454460476\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c0bb587bbeda412e96d66c80e2f7b660",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/14 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00106, Accuracy: 0.881\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d62499f101be430fad9a7391c43fb907",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/79 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00036, Updated Learning Rate: 0.0020126205959436786\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "77f74bc482db4fc8a1b5b9ad11c03aed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/14 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00084, Accuracy: 0.943\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "79cece6433d54da2badd834192d8dc3a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/79 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00030, Updated Learning Rate: 0.001104512298243044\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e834b427237048b5b173674fd34ace12",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/14 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00078, Accuracy: 0.958\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6e4ba2fd5115436dbca0ae09a480cb8e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/79 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00028, Updated Learning Rate: 0.00031203839133615234\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ebf441b211dc4c9d84c2393ddbb852b2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/14 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00077, Accuracy: 0.961\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6d26bbc668f8426f876e8005e481c963",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/79 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00028, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b4bfbe2381d14e83a788f942e300cfc3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/14 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 23:12:00,204] Trial 15 finished with value: 0.000761160358786583 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 768, 'dropout_prob': 0.06100809238217579, 'label_smoothing': 0.06658468571810651, 'lr': 0.0027114201825804746, 'warmup_proportion': 0.08855932290236279, 'weight_decay': 0.00010666774678723855}. Best is trial 12 with value: 9.675593301653863e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00076, Accuracy: 0.962\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1580, 'dropout_prob': 0.12167555371301995, 'label_smoothing': 0.24853464412941306, 'lr': 0.0010749075521783032, 'warmup_proportion': 0.055695902600475605, 'weight_decay': 3.6520396057506486e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 23:12:02 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 23:12:05 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e38c83b364f44b188f886f127027bc26",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00049, Updated Learning Rate: 0.0010216828962820747\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "908fb6fcee7e4603a42062596b5e2408",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00128, Accuracy: 0.568\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b698a80323084563a74a1aac86b3d743",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00039, Updated Learning Rate: 0.0007706462300615636\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8e5d53d6454d45658da81a5c1ec06e24",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00111, Accuracy: 0.757\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ffc5ddbf26f74e798986476a2073b131",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00036, Updated Learning Rate: 0.00041785905987544577\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "66a185aecce14523a96f0dfc931e8c83",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00104, Accuracy: 0.821\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c758714f5fe64869bbd285d29a1fcdaf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00034, Updated Learning Rate: 0.00011725549357132985\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "61243125e57b460891deea2307f4dc65",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00102, Accuracy: 0.846\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0efaf978ba3141f7bd64e9e77e1a2c09",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00034, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6a4243370f304d0db8320cf118cb78dc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 23:15:15,776] Trial 16 finished with value: 0.0010118022322654725 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1580, 'dropout_prob': 0.12167555371301995, 'label_smoothing': 0.24853464412941306, 'lr': 0.0010749075521783032, 'warmup_proportion': 0.055695902600475605, 'weight_decay': 3.6520396057506486e-06}. Best is trial 12 with value: 9.675593301653863e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00101, Accuracy: 0.850\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1066, 'dropout_prob': 0.0646171911795538, 'label_smoothing': 0.023495019799586958, 'lr': 0.006120359080901847, 'warmup_proportion': 0.08926686602248009, 'weight_decay': 1.4057358549679689e-05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 23:15:17 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 23:15:20 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "87f765c18eea4acf8e3b6c631817a3f7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/57 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00049, Updated Learning Rate: 0.0059184365268458525\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f3f795fcbbd943bb8829896e72ad9835",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00066, Accuracy: 0.834\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2e97592483e441aeae57ec40de8ddc15",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/57 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00018, Updated Learning Rate: 0.004557886025053932\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d79eb1eb4e8644eab45a7c5fe2ca5315",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00038, Accuracy: 0.939\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aabaac14ad6340d4b7e04d1051dcea59",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/57 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00014, Updated Learning Rate: 0.002504338165291719\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2dafcbd2625e43ee8c77cbed9f6d0c43",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00032, Accuracy: 0.959\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c6c4487a4a4c4b96974749bb5513d96c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/57 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00013, Updated Learning Rate: 0.0007079812894745393\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cea693c94643469eb343fe0aa9964c62",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00030, Accuracy: 0.964\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a8108199953740c589ae4dee9f76987d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/57 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00012, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8d38d967d31d4a5ebed6f1f323c28f79",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 23:18:31,369] Trial 17 finished with value: 0.0003036852046847343 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1066, 'dropout_prob': 0.0646171911795538, 'label_smoothing': 0.023495019799586958, 'lr': 0.006120359080901847, 'warmup_proportion': 0.08926686602248009, 'weight_decay': 1.4057358549679689e-05}. Best is trial 12 with value: 9.675593301653863e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00030, Accuracy: 0.963\n",
      "Early stopping triggered.\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': True, 'batch_size': 1700, 'dropout_prob': 0.23929617064990163, 'label_smoothing': 0.004612073534762902, 'lr': 0.002748894015773769, 'warmup_proportion': 0.0246652558177621, 'weight_decay': 2.4878399651784663e-05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 23:18:33 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 23:18:36 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6aca9887067543fbaaf212ec5123c5c2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/36 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00046, Updated Learning Rate: 0.0025521517898568326\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "437059ce95ed44aabd01fe41cce934f2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.10945, Accuracy: 0.094\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0d1ef8d5deae47dcbe58d19eb09a85fb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/36 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00046, Updated Learning Rate: 0.0018831822879986008\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb1bcd377da440948518ee2f38c6265f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 23:19:55,386] Trial 18 finished with value: 5.086788623046875 and parameters: {'accumulation_steps': 3, 'batchnorm': True, 'batch_size': 1700, 'dropout_prob': 0.23929617064990163, 'label_smoothing': 0.004612073534762902, 'lr': 0.002748894015773769, 'warmup_proportion': 0.0246652558177621, 'weight_decay': 2.4878399651784663e-05}. Best is trial 12 with value: 9.675593301653863e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] Loss: 5.08679, Accuracy: 0.095\n",
      "Early stopping triggered.\n",
      "{'epochs': 5, 'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 2040, 'dropout_prob': 0.1572000276998347, 'label_smoothing': 0.08098572401468691, 'lr': 0.08016730598073825, 'warmup_proportion': 0.09877022658132287, 'weight_decay': 1.076057855022966e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 23:19:57 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 23:20:00 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "efb7eaebee8c42a7abd658db99eebae8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00034, Updated Learning Rate: 0.07802746723395483\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1abb826c79ad489c972faa748484e73f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00077, Accuracy: 0.711\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "74784409fded45b8aba808260466460f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00019, Updated Learning Rate: 0.060665574382727704\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "617c53e195454642a1a15be62ae5c963",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00032, Accuracy: 0.954\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "be67c6f3fd5240a2aa8d4fd146e4afa5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00017, Updated Learning Rate: 0.03353489151275318\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e932865668ba47a5b9c7c9a7bb96a947",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00029, Accuracy: 0.968\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "63ac8f28f14944f98915d3991647c7f6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00016, Updated Learning Rate: 0.009512444970094696\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d8e302b58f4440cd957d54e86bc14980",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00029, Accuracy: 0.972\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ae75464e31fa45aa839f6416faca4dd1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00016, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "838e1e464b9045ef95edf74a8d3bf7ce",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 23:22:39,092] Trial 19 finished with value: 0.00028162769079208375 and parameters: {'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 2040, 'dropout_prob': 0.1572000276998347, 'label_smoothing': 0.08098572401468691, 'lr': 0.08016730598073825, 'warmup_proportion': 0.09877022658132287, 'weight_decay': 1.076057855022966e-06}. Best is trial 12 with value: 9.675593301653863e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00028, Accuracy: 0.972\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1240, 'dropout_prob': 0.0020425564571031896, 'label_smoothing': 0.029830217046906832, 'lr': 0.017110783300254073, 'warmup_proportion': 0.08534882634013566, 'weight_decay': 4.815358890666891e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 23:22:41 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 23:22:44 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0221bc3f6c4543ac850a0250c32e981e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/49 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00036, Updated Learning Rate: 0.016477040757577038\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7ec1a4aa2ef64733abea4c51191dd83b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/9 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00057, Accuracy: 0.884\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fcd394fcebfa41c28c43ad4ecf7d966d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/49 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00015, Updated Learning Rate: 0.012618784086308362\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "424edb1e37c045cfb7d69fbaf06f833c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/9 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00034, Accuracy: 0.956\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a953b2e41b064030bfdadf7f2ea27a05",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/49 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00012, Updated Learning Rate: 0.006908892992680061\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d34cbf5bea764ba99b2b9375c6af53a4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/9 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00031, Accuracy: 0.963\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7f034204b0d34c1a8db29d8340a151b5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/49 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00011, Updated Learning Rate: 0.001949289654512278\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b02292d490434370a2067bd4bda1448c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/9 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00028, Accuracy: 0.970\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "073c9766497b4284813c3f07024e0aed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/49 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00011, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b939275fe27848f1830a8fe5cd505827",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/9 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 23:28:01,748] Trial 20 finished with value: 0.00028082954883575437 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1240, 'dropout_prob': 0.0020425564571031896, 'label_smoothing': 0.029830217046906832, 'lr': 0.017110783300254073, 'warmup_proportion': 0.08534882634013566, 'weight_decay': 4.815358890666891e-06}. Best is trial 12 with value: 9.675593301653863e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00028, Accuracy: 0.971\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 2037, 'dropout_prob': 0.043102425960477274, 'label_smoothing': 0.0024346970621232098, 'lr': 0.005206968960316317, 'warmup_proportion': 0.09987999931887383, 'weight_decay': 6.732090754703305e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 23:28:03 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 23:28:06 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "635ab5f8d6054b7dad51abcd51c6ffb7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00032, Updated Learning Rate: 0.005049959634493417\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e8e4b1736a91481da51976bfb2a10e35",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00041, Accuracy: 0.727\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cb56dfb9c623427a805c869e89315235",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00014, Updated Learning Rate: 0.003905226720237238\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "80171638c50e49bd924c22700fd0e53d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00017, Accuracy: 0.897\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d62b94c6a7b24d31891d293c07734287",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00008, Updated Learning Rate: 0.002151394144594559\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c11351b579c345c48259673e1ccdc424",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00013, Accuracy: 0.921\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5ddbc9f54faa42608e20434d49340ba6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00007, Updated Learning Rate: 0.0006090996613865004\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "efdbf485101c46ec9de7b69560f3a2d9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00011, Accuracy: 0.941\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bd09d42436e04eacb89d2f8a599a9074",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00006, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "32939d3f208b49249fd9761934eebd25",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 23:30:41,624] Trial 21 finished with value: 0.00010402768701314926 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 2037, 'dropout_prob': 0.043102425960477274, 'label_smoothing': 0.0024346970621232098, 'lr': 0.005206968960316317, 'warmup_proportion': 0.09987999931887383, 'weight_decay': 6.732090754703305e-06}. Best is trial 12 with value: 9.675593301653863e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00010, Accuracy: 0.941\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1648, 'dropout_prob': 0.06756314823371665, 'label_smoothing': 0.0015061153643863, 'lr': 0.008288598375033563, 'warmup_proportion': 0.0918871087536229, 'weight_decay': 6.4270072447862914e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 23:30:43 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 23:30:46 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "495ba3de85764d039691743f457a6e9d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/37 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00035, Updated Learning Rate: 0.008038666552355337\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "632b35e71e0a463bacf66edde8962a6c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00081, Accuracy: 0.653\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "94a23117a9f24170998265a750dcf685",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/37 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00012, Updated Learning Rate: 0.0062164487812751724\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2d5246c9ddf842a88d942abfbdf06815",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00016, Accuracy: 0.932\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "19862e0b52bd4558bbd4c17457ea7596",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/37 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00007, Updated Learning Rate: 0.0034246491858979534\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "90c179325ac34ed9998ba28d4fc15b71",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00013, Accuracy: 0.944\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e24b4a13d17d4df1af7c3956609cd695",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/37 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00006, Updated Learning Rate: 0.0009695818242970546\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0242c15e46864836a39c4e70210b4cce",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00010, Accuracy: 0.958\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1011d709397b482281427e1a8ed27c01",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/37 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00005, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c031c7a62db5420a905e9674c60cf5bf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 23:34:01,966] Trial 22 finished with value: 0.00010083994939923287 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1648, 'dropout_prob': 0.06756314823371665, 'label_smoothing': 0.0015061153643863, 'lr': 0.008288598375033563, 'warmup_proportion': 0.0918871087536229, 'weight_decay': 6.4270072447862914e-06}. Best is trial 12 with value: 9.675593301653863e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00010, Accuracy: 0.958\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1628, 'dropout_prob': 0.05221313037960226, 'label_smoothing': 0.05623074102371584, 'lr': 0.0075028331514503015, 'warmup_proportion': 0.0835192644421218, 'weight_decay': 1.9443394981967603e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 23:34:04 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 23:34:06 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "21926248daea4c8e9933310cd42a8f28",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/37 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00038, Updated Learning Rate: 0.007276595049428016\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "de0de57bc9824ee2b26ffc7a0b636d8c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00056, Accuracy: 0.855\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "34a9b386a97e4924aca83124ecb5b0b5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/37 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00018, Updated Learning Rate: 0.005627124863587726\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aec8fdd5f75941d59cda4240111dd1fd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00044, Accuracy: 0.924\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fe034d856e004b3390b6afebae914389",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/37 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00014, Updated Learning Rate: 0.0030999899236809634\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4f0ddf6555f44531b4422304737e10af",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00038, Accuracy: 0.951\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "108f33dbd9354206883ef17850be3f2a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/37 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00013, Updated Learning Rate: 0.0008776647540664734\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8d14caf88a8b499d9c983898dbc4a338",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00036, Accuracy: 0.953\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cd1921df51a54d14ab5930972416599c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/37 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00013, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5b4b2ec8e1fd4bcfbd365eabdb62fff0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 23:37:19,769] Trial 23 finished with value: 0.0003589511275291443 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1628, 'dropout_prob': 0.05221313037960226, 'label_smoothing': 0.05623074102371584, 'lr': 0.0075028331514503015, 'warmup_proportion': 0.0835192644421218, 'weight_decay': 1.9443394981967603e-06}. Best is trial 12 with value: 9.675593301653863e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00036, Accuracy: 0.958\n",
      "{'epochs': 5, 'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 1272, 'dropout_prob': 0.11274565342183124, 'label_smoothing': 0.024893962219692096, 'lr': 0.009804392909208334, 'warmup_proportion': 0.09437146866757967, 'weight_decay': 2.284525115373476e-05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 23:37:21 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 23:37:24 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "186f600f26c74267bf958f806fa10338",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/48 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00054, Updated Learning Rate: 0.009508754288638545\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "76507ad5e94d423393c1792134ee0bcb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00044, Accuracy: 0.891\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ff72183cad2c45209cbf57855105e08b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/48 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00020, Updated Learning Rate: 0.00735329468190625\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4aeadc6be72b4f2f9013effc1ff2e8e5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00032, Accuracy: 0.936\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3c09d189f366410cbb6758f17be08f8b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/48 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00016, Updated Learning Rate: 0.0040509389736968675\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7aecf3d4631644a3bcc7bd8c637e070c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00024, Accuracy: 0.964\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "617f61184b9744a4b728bac9cdbbf984",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/48 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00015, Updated Learning Rate: 0.0011468961014770883\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "42b7309e98b543a1bb94f50a144d1a76",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00023, Accuracy: 0.970\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fc7ef43b6cac4b78bcb2c2b0a9abbdc2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/48 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00014, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5ab723251aed40bfbf377163ac265cbe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 23:39:59,557] Trial 24 finished with value: 0.00022974773198366165 and parameters: {'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 1272, 'dropout_prob': 0.11274565342183124, 'label_smoothing': 0.024893962219692096, 'lr': 0.009804392909208334, 'warmup_proportion': 0.09437146866757967, 'weight_decay': 2.284525115373476e-05}. Best is trial 12 with value: 9.675593301653863e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00023, Accuracy: 0.971\n",
      "Early stopping triggered.\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1766, 'dropout_prob': 0.08781859752580784, 'label_smoothing': 0.04455715317948723, 'lr': 0.0063503817633186466, 'warmup_proportion': 0.05775277708566127, 'weight_decay': 5.341495764784108e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 23:40:01 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 23:40:04 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a647bfc182714b9a8dd27f98bb1d037a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00033, Updated Learning Rate: 0.0060597415290448335\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bb91f286c46a4bfa84965c4104be8dab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00049, Accuracy: 0.808\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "410ae6f0be304ecf980334819651a897",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00015, Updated Learning Rate: 0.004590495244526808\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0f2def545de54de39d1ab81972f5461f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00036, Accuracy: 0.914\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "495a7e24283243549392d53f86f085e6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00012, Updated Learning Rate: 0.0024959125065767873\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ebaf541bff524e34a962d9c07ad4f422",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00029, Accuracy: 0.947\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7bab14348f754b59bf1f3c0b8e5d9baf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00011, Updated Learning Rate: 0.0007014613453475075\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "280a4e6a1ba44021955f32f1a7a96bb3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00028, Accuracy: 0.955\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "56fa9ff14b8d49d29ed007de44ac33fd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00010, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2f5d833f57354cedaf315cef3fcac4d5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 23:42:09,472] Trial 25 finished with value: 0.00027549923062324523 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1766, 'dropout_prob': 0.08781859752580784, 'label_smoothing': 0.04455715317948723, 'lr': 0.0063503817633186466, 'warmup_proportion': 0.05775277708566127, 'weight_decay': 5.341495764784108e-06}. Best is trial 12 with value: 9.675593301653863e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00028, Accuracy: 0.956\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': True, 'batch_size': 1424, 'dropout_prob': 0.1415047682362926, 'label_smoothing': 0.018320995057560397, 'lr': 0.014579658590104416, 'warmup_proportion': 0.07899151783148109, 'weight_decay': 1.6320845160803314e-05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 23:42:11 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 23:42:14 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "81cade1ff59840bd928fd9fcceac6919",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/43 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00055, Updated Learning Rate: 0.014024753376252124\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "74eb794c8e0f4c36a63b8449ce9e61e9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.17604, Accuracy: 0.042\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "55b7d1905ffd4e6ea3c663d459be0853",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/43 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00055, Updated Learning Rate: 0.010726231036758384\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f53f884ab7354bb29365213f9a62c22e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 23:43:11,303] Trial 26 finished with value: 15.5095416015625 and parameters: {'accumulation_steps': 3, 'batchnorm': True, 'batch_size': 1424, 'dropout_prob': 0.1415047682362926, 'label_smoothing': 0.018320995057560397, 'lr': 0.014579658590104416, 'warmup_proportion': 0.07899151783148109, 'weight_decay': 1.6320845160803314e-05}. Best is trial 12 with value: 9.675593301653863e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] Loss: 15.50954, Accuracy: 0.043\n",
      "Early stopping triggered.\n",
      "{'epochs': 5, 'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 907, 'dropout_prob': 0.036209372369698706, 'label_smoothing': 0.08262128613839981, 'lr': 0.022817272088603183, 'warmup_proportion': 0.09091127314537721, 'weight_decay': 5.874084922159233e-05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 23:43:13 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 23:43:16 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a43c7f1cefe843b3b720bb55dd68d118",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/67 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00067, Updated Learning Rate: 0.02209221190146309\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "57db1cd562e44580b18c44d552fe81a4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/12 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00081, Accuracy: 0.947\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ec1480e994ad42a6bb06c325af90ff32",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/67 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00038, Updated Learning Rate: 0.017043374236268834\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f4c6dc37090e46d5ad5bc4991846a164",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/12 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00069, Accuracy: 0.973\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2ef2dbce9ab644ccaf83147de146649f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/67 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00035, Updated Learning Rate: 0.009374926651923401\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2933367330f948f7bf8e0949a7ccf97c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/12 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00067, Accuracy: 0.976\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1d2d3dac65694ca8ae4e917820418866",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/67 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00033, Updated Learning Rate: 0.0026519578308219515\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "38bdd3c4a6de4b4a9c2ebd50b339c142",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/12 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00065, Accuracy: 0.979\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b7f1cba664fb46e68c1a0c95cc625268",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/67 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00033, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "10d4a59eb4284f7084343691dd7ed38e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/12 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 23:45:19,815] Trial 27 finished with value: 0.0006539548456668854 and parameters: {'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 907, 'dropout_prob': 0.036209372369698706, 'label_smoothing': 0.08262128613839981, 'lr': 0.022817272088603183, 'warmup_proportion': 0.09091127314537721, 'weight_decay': 5.874084922159233e-05}. Best is trial 12 with value: 9.675593301653863e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00065, Accuracy: 0.980\n",
      "Early stopping triggered.\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 627, 'dropout_prob': 0.19216664626620464, 'label_smoothing': 0.022847485709152373, 'lr': 0.0034850293045288654, 'warmup_proportion': 0.0649826500583231, 'weight_decay': 5.096795642752917e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 23:45:21 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 23:45:24 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5fa42c9ac8e1458a951cc176d905a476",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/96 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00079, Updated Learning Rate: 0.003316992140252004\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "257dec20b84e4441ae378a7107422f50",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/16 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00089, Accuracy: 0.870\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0d894acbfc2c47f5ae0002ef0a7b3a0f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/96 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00028, Updated Learning Rate: 0.0025056488979424162\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a8dca4761c994a98b20b7acff0558770",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/16 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00053, Accuracy: 0.954\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1f420af6e74f46c0a22cd4b08909c6cd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/96 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00022, Updated Learning Rate: 0.0013598874048829925\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5f77a15ae5e04764b8527acc7a576a1e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/16 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00047, Accuracy: 0.966\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ab4c59aa5c9b4e0cb301353e1ac5a0ba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/96 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00020, Updated Learning Rate: 0.0003817996015881341\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cfa7e0680ed74e3d93c03ea489364092",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/16 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00046, Accuracy: 0.967\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "341506d9188340faa38fea754176bd98",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/96 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00019, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "874519e45d334e15baab9e365f21e414",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/16 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 23:47:28,162] Trial 28 finished with value: 0.00045484486371278764 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 627, 'dropout_prob': 0.19216664626620464, 'label_smoothing': 0.022847485709152373, 'lr': 0.0034850293045288654, 'warmup_proportion': 0.0649826500583231, 'weight_decay': 5.096795642752917e-06}. Best is trial 12 with value: 9.675593301653863e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00045, Accuracy: 0.969\n",
      "{'epochs': 5, 'accumulation_steps': 1, 'batchnorm': True, 'batch_size': 1332, 'dropout_prob': 0.13901221535739477, 'label_smoothing': 0.22455100499862526, 'lr': 0.0019519278708419181, 'warmup_proportion': 0.045089317675399726, 'weight_decay': 2.51052890415051e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 23:47:30 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 23:47:33 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c1db0303572c4a92986028ea2cd0e335",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/46 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00176, Updated Learning Rate: 0.001831477299897316\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6c57cd4f2fcb463c82e8090acceb3a3a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.49440, Accuracy: 0.085\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9915e0b6432b42c0af1a15f6b7e6c4a0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/46 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00176, Updated Learning Rate: 0.0013639513849464857\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "04dd29ee3a544e2c8aee2c9c21b78eee",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 23:48:28,697] Trial 29 finished with value: 55.203887109375 and parameters: {'accumulation_steps': 1, 'batchnorm': True, 'batch_size': 1332, 'dropout_prob': 0.13901221535739477, 'label_smoothing': 0.22455100499862526, 'lr': 0.0019519278708419181, 'warmup_proportion': 0.045089317675399726, 'weight_decay': 2.51052890415051e-06}. Best is trial 12 with value: 9.675593301653863e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] Loss: 55.20389, Accuracy: 0.085\n",
      "Early stopping triggered.\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1089, 'dropout_prob': 0.04587775289362337, 'label_smoothing': 0.18777729858412195, 'lr': 0.005361404692473498, 'warmup_proportion': 0.08123509116173881, 'weight_decay': 1.7947846074654357e-05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 23:48:30 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 23:48:33 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3ae67cc934a347d5b16b2299b343829b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/56 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00058, Updated Learning Rate: 0.00516706034304647\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "53e5ad4ccc6b410f8d9d36475f428b27",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00128, Accuracy: 0.849\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7c64d2cd1c0f47f682f483a3d8344349",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/56 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00039, Updated Learning Rate: 0.003961326984566366\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ccfd5fae53ab49c2aa6426ca15710c36",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00110, Accuracy: 0.935\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b48b38824ea340b2a324c1f123b6b9f1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/56 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00036, Updated Learning Rate: 0.0021703204531964056\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5eb238ea6fe24d7dad4a5ee7d638a8d9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00106, Accuracy: 0.952\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bf25f703dbe34ef3ae4632be72d3e1d1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/56 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00035, Updated Learning Rate: 0.0006125694917918013\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "856b164bd6eb4d4c9c50caed2b47fac1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00104, Accuracy: 0.957\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d7bde67614ef426d9ce850705372b925",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/56 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00035, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e0006ed9a3c94ed8abed9836bd27f346",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 23:50:41,001] Trial 30 finished with value: 0.0010404990196228028 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1089, 'dropout_prob': 0.04587775289362337, 'label_smoothing': 0.18777729858412195, 'lr': 0.005361404692473498, 'warmup_proportion': 0.08123509116173881, 'weight_decay': 1.7947846074654357e-05}. Best is trial 12 with value: 9.675593301653863e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00104, Accuracy: 0.958\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1774, 'dropout_prob': 0.038819814958578946, 'label_smoothing': 8.582147054318995e-05, 'lr': 0.003325357117078681, 'warmup_proportion': 0.0936041066135404, 'weight_decay': 9.709687768548523e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 23:50:43 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 23:50:46 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5483353a96e64a2495dc5c279fbedadb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00039, Updated Learning Rate: 0.003240653199952516\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ae7605a1d925480cbe6c029edd7937ef",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00056, Accuracy: 0.694\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b5ea9ae42e8d4ec6bb09ce46897681e8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00019, Updated Learning Rate: 0.0025245987667753913\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1428a3cfada745038957d98dd32779ae",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00023, Accuracy: 0.889\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3ee6a86d688f496f87b68e615cc3c278",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00011, Updated Learning Rate: 0.0013973152351222115\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "61a7ab356e4e4b51b7e396c71ad2fcfb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00017, Accuracy: 0.906\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8e13488c416142bb821bd479b249a73f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00009, Updated Learning Rate: 0.0003966386900725418\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "749f2a611dab4f2483689bcda58749af",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00013, Accuracy: 0.934\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1f224cc932774c418fb188673bfb24c6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00008, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fff5c740d6cb4e5c8650123281ba0565",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 23:52:55,302] Trial 31 finished with value: 0.00012797627449035644 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1774, 'dropout_prob': 0.038819814958578946, 'label_smoothing': 8.582147054318995e-05, 'lr': 0.003325357117078681, 'warmup_proportion': 0.0936041066135404, 'weight_decay': 9.709687768548523e-06}. Best is trial 12 with value: 9.675593301653863e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00013, Accuracy: 0.935\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1832, 'dropout_prob': 0.07985584476619423, 'label_smoothing': 0.0004311194470224544, 'lr': 0.008730687529148204, 'warmup_proportion': 0.0993370192538024, 'weight_decay': 7.09143369245701e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 23:52:57 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 23:53:00 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b29eb9e6ed534621a16c6e5f7f4ae026",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/33 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00033, Updated Learning Rate: 0.008508298352020341\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "07c5ed7d82b44f6aab5b78980adb1aa6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00041, Accuracy: 0.758\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "299d3b879bbc44599c738c49d36a9e02",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/33 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00011, Updated Learning Rate: 0.006628305530250007\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "79ad6803c55540d59ca435661e2c2674",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00017, Accuracy: 0.910\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "118c3aee7f3f4349944d48477741e671",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/33 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00007, Updated Learning Rate: 0.0036686353579634587\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b29ea916f7d440bfbe6ace64bd87bd12",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00010, Accuracy: 0.946\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "adad440ddc0943c7b0856f6b9f567b4e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/33 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00005, Updated Learning Rate: 0.0010413703981472506\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "24ff0ae3d90f4260be7e5f8453658371",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00008, Accuracy: 0.958\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c2477044c88645b5811ab30c436bb85c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/33 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00005, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "18bd7cb00a084b7fbca132a57ff83fbd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 23:55:13,177] Trial 32 finished with value: 7.672419399023056e-05 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1832, 'dropout_prob': 0.07985584476619423, 'label_smoothing': 0.0004311194470224544, 'lr': 0.008730687529148204, 'warmup_proportion': 0.0993370192538024, 'weight_decay': 7.09143369245701e-06}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00008, Accuracy: 0.960\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1548, 'dropout_prob': 0.07647214091928793, 'label_smoothing': 0.03544435836714206, 'lr': 0.008694217893494775, 'warmup_proportion': 0.0974848667225722, 'weight_decay': 5.426841681649178e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 23:55:15 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 23:55:18 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1b327403b7dc40ed935b1ad610b70c95",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/39 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00038, Updated Learning Rate: 0.008466660525609682\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "663bf55d01d44ffab42bdc9f1a43759c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00083, Accuracy: 0.662\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1551d69b2b8c4e96ac3314fd7579fd67",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/39 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00015, Updated Learning Rate: 0.006588277731811744\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3dc6ca31f32d4846ac700943d34c73cf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00033, Accuracy: 0.938\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8b0d905a4c7c423e9b535df503d4045e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/39 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00012, Updated Learning Rate: 0.003643824982110209\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "748e993adc6b4986b438a9e91ddb7c10",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00029, Accuracy: 0.952\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "801bdfa80d5e4beda54fb748a9ce83e2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/39 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00010, Updated Learning Rate: 0.0010339074580306639\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "23f5a2bca7eb444d9eae154e898c3ec7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00028, Accuracy: 0.957\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c931067308e846b486aa5fb9ef9f6bad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/39 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00010, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2d37abeb9a144ca490c1a793e06f60b7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 23:57:26,360] Trial 33 finished with value: 0.00027399299442768096 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1548, 'dropout_prob': 0.07647214091928793, 'label_smoothing': 0.03544435836714206, 'lr': 0.008694217893494775, 'warmup_proportion': 0.0974848667225722, 'weight_decay': 5.426841681649178e-06}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00027, Accuracy: 0.958\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 264, 'dropout_prob': 0.23296697356891147, 'label_smoothing': 0.05313215822264636, 'lr': 0.010847393682350926, 'warmup_proportion': 0.09953744407804783, 'weight_decay': 3.498010111471705e-05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 23:57:28 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 23:57:31 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1e0057d586e848a986234f1c774ef83d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/228 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00126, Updated Learning Rate: 0.010520304740207887\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3a41605f800d4e4d96bc38db2279147a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00197, Accuracy: 0.947\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ec5cf8b6268049ca9325e8a62cb82cc3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/228 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00065, Updated Learning Rate: 0.008135545261763195\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b6ce8ddfa4e741f798b7fde0f19a04e2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00161, Accuracy: 0.977\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "081ea4822d15494e8c63f8794ce3c7e5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/228 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00059, Updated Learning Rate: 0.004481881768487458\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7a79a86784d04ebeaa0ab4b3b1c364d8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00159, Accuracy: 0.977\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b6f4374f228841a2910e7e2823ba5fe1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/228 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00056, Updated Learning Rate: 0.0012689040148310458\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "54efcbdb3b7348cea09ca58560bc2ab2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00152, Accuracy: 0.981\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "210af51f59ad49cea7c59f3dc76b5c6d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/228 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00055, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9b8055ce683c40318a39ccac01806061",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-10 23:59:42,962] Trial 34 finished with value: 0.0015111255675554274 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 264, 'dropout_prob': 0.23296697356891147, 'label_smoothing': 0.05313215822264636, 'lr': 0.010847393682350926, 'warmup_proportion': 0.09953744407804783, 'weight_decay': 3.498010111471705e-05}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00151, Accuracy: 0.981\n",
      "{'epochs': 5, 'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 1781, 'dropout_prob': 0.10166356875940774, 'label_smoothing': 0.10803373833328106, 'lr': 0.007007906757396097, 'warmup_proportion': 0.08724099897949009, 'weight_decay': 1.1230029514109688e-05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/10 23:59:45 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/10 23:59:48 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d9fb34c84c3f4e24b1229c9dd082734f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00050, Updated Learning Rate: 0.006774321035049212\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cefca7ae250f41df855b908f9b6e31d9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00056, Accuracy: 0.883\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dfe3969c6c204808adf475090f905403",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00028, Updated Learning Rate: 0.005214500045280918\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "318e0d95ba14498d92fbee3776275621",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00048, Accuracy: 0.944\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3e5b34b10abc4a25a73931d910fe6f1f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00024, Updated Learning Rate: 0.00286423723811967\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6082ec209325425da34d258eb958f157",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00044, Accuracy: 0.961\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d654f24497a34129a7b0e8303c65a794",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00023, Updated Learning Rate: 0.0008095865873835523\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b16a4e2d674a40d5ac6b3f382826d38e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00043, Accuracy: 0.966\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "69da7b6faf264787aeff403c7f473e94",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00023, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "31e0c65b96ee46e78cb92a83c01f8415",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 00:02:02,063] Trial 35 finished with value: 0.00043155853748321534 and parameters: {'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 1781, 'dropout_prob': 0.10166356875940774, 'label_smoothing': 0.10803373833328106, 'lr': 0.007007906757396097, 'warmup_proportion': 0.08724099897949009, 'weight_decay': 1.1230029514109688e-05}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] Loss: 0.00043, Accuracy: 0.968\n",
      "Early stopping triggered.\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': True, 'batch_size': 1426, 'dropout_prob': 0.08395748942188344, 'label_smoothing': 0.01615457580074229, 'lr': 0.002306260145502806, 'warmup_proportion': 0.0297121833508251, 'weight_decay': 3.3762365241971748e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 00:02:04 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 00:02:07 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "81b32268a47e4d5abe5d05967c0fe171",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/43 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00055, Updated Learning Rate: 0.0021562452712540607\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "47ef2bd871f247ae987e44ed3d5b6a7e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.13175, Accuracy: 0.088\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a2d93d64882d4cd98855ecfb24872d6a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/43 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00055, Updated Learning Rate: 0.0016006504590575811\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cfc678dda05f4e4cb7786753d59936ce",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 00:03:07,551] Trial 36 finished with value: 11.78404296875 and parameters: {'accumulation_steps': 3, 'batchnorm': True, 'batch_size': 1426, 'dropout_prob': 0.08395748942188344, 'label_smoothing': 0.01615457580074229, 'lr': 0.002306260145502806, 'warmup_proportion': 0.0297121833508251, 'weight_decay': 3.3762365241971748e-06}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] Loss: 11.78404, Accuracy: 0.088\n",
      "Early stopping triggered.\n",
      "{'epochs': 5, 'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 1802, 'dropout_prob': 0.16059651826575091, 'label_smoothing': 0.01557981237452252, 'lr': 0.01999277302338321, 'warmup_proportion': 0.07495341872655466, 'weight_decay': 5.194808328343723e-05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 00:03:09 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 00:03:12 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7f2c11c0de99402598b226cdabe0512a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00036, Updated Learning Rate: 0.019192856613316222\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "13718b55d2c64245b246c67d46bb465a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00029, Accuracy: 0.890\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "093131efc4f8420085f45b9e35b728f4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00012, Updated Learning Rate: 0.01464193896038047\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "49ff5dcb81c8428e88f8c69b58bc807a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00016, Accuracy: 0.959\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8b19398823254ab0bd5f91e4ba79a755",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00009, Updated Learning Rate: 0.007996852364437004\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "40d9dc2235eb42cebe01d3e3eee90673",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00014, Accuracy: 0.970\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eb8fc890852b4081a09ebb387e9b2418",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00008, Updated Learning Rate: 0.0022531359193882494\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "310b3223e15a4330a54f187d8f1f670d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00013, Accuracy: 0.970\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "36934922103d4e44bbcf268d20b4c0f6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00008, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4b2606269a174b3b94bcfeecc2b1c13e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 00:05:27,844] Trial 37 finished with value: 0.00013149097561836241 and parameters: {'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 1802, 'dropout_prob': 0.16059651826575091, 'label_smoothing': 0.01557981237452252, 'lr': 0.01999277302338321, 'warmup_proportion': 0.07495341872655466, 'weight_decay': 5.194808328343723e-05}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00013, Accuracy: 0.972\n",
      "Early stopping triggered.\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1582, 'dropout_prob': 0.30096110826918593, 'label_smoothing': 0.06437037250910484, 'lr': 0.03679458845079387, 'warmup_proportion': 0.09206875423876412, 'weight_decay': 6.42267565924399e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 00:05:29 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 00:05:33 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7c85234b87224646bfd78945ce688c01",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00031, Updated Learning Rate: 0.03568509585142964\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "22aef8bddd14415ebdb82f5dbd553f83",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00060, Accuracy: 0.894\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7c50691acd6f419eb2954e8e46549390",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00016, Updated Learning Rate: 0.027595941338095405\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6b29907028804da6b7a7d49850301468",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00041, Accuracy: 0.951\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cddd0d1191a74251bc89e8ff5c1d7a71",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00013, Updated Learning Rate: 0.015202637609154421\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2d8c2823bbf045b58bd04c5d62665de6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00037, Accuracy: 0.961\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "52ef418fde564e968d60a7734dade579",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00012, Updated Learning Rate: 0.004304149215606746\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "606ce3d1d3b7482dbcbe1db6e7e7e907",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00036, Accuracy: 0.967\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2b35f970f72043f4a93d1f45775ef177",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00012, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0d07df8ee87d44eca638955cca8e111b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 00:07:43,364] Trial 38 finished with value: 0.000354938668012619 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1582, 'dropout_prob': 0.30096110826918593, 'label_smoothing': 0.06437037250910484, 'lr': 0.03679458845079387, 'warmup_proportion': 0.09206875423876412, 'weight_decay': 6.42267565924399e-06}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00035, Accuracy: 0.969\n",
      "{'epochs': 5, 'accumulation_steps': 2, 'batchnorm': True, 'batch_size': 1187, 'dropout_prob': 0.026865211857513366, 'label_smoothing': 0.03506366320822358, 'lr': 0.012377821684166056, 'warmup_proportion': 0.0698320698345305, 'weight_decay': 1.8792414435923372e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 00:07:45 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 00:07:48 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8165e03ee2cf450889d901d03df7ebaf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/51 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00098, Updated Learning Rate: 0.011805814196892668\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "918c65b0fbb14d5b9ca53ab41bbcbe38",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/9 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.65865, Accuracy: 0.091\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "636eb51f5fab4baca65d88c35ee52884",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/51 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00098, Updated Learning Rate: 0.008938716378801113\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e7a15a2e3403424b91305467dc7f291c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/9 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 00:08:46,611] Trial 39 finished with value: 99.87796640625 and parameters: {'accumulation_steps': 2, 'batchnorm': True, 'batch_size': 1187, 'dropout_prob': 0.026865211857513366, 'label_smoothing': 0.03506366320822358, 'lr': 0.012377821684166056, 'warmup_proportion': 0.0698320698345305, 'weight_decay': 1.8792414435923372e-06}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] Loss: 99.87797, Accuracy: 0.090\n",
      "Early stopping triggered.\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1869, 'dropout_prob': 0.3807695268299693, 'label_smoothing': 0.04888299119121421, 'lr': 0.008039197764394468, 'warmup_proportion': 0.0841086859038431, 'weight_decay': 0.00010763741035381019}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 00:08:48 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 00:08:51 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "57c16dd87edd456e8a5980cc41511133",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/33 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00033, Updated Learning Rate: 0.007756927405566921\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6948e19539b845df847e206af716ff55",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00058, Accuracy: 0.766\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d6f898342ae14072bca5065fd3c13318",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/33 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00015, Updated Learning Rate: 0.005956055412110898\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "63252fe1f02e410083584badd0f85b74",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00038, Accuracy: 0.899\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d625049728554e8b910a7e5c1c382753",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/33 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00012, Updated Learning Rate: 0.003266401159543807\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "671a27ae3db84582a4157de9e9df658c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00029, Accuracy: 0.955\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0c9042df287847d8b2d2223c8bebb9cd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/33 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00011, Updated Learning Rate: 0.0009224447128175059\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b88973faa8a14e22bb47bdd17a9de9ca",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00028, Accuracy: 0.958\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "37b45c430ebb430f85ef6c4dba9fd985",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/33 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00010, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e2b3c030ab3a41a0b5e07065cd59928d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 00:11:07,490] Trial 40 finished with value: 0.00028276586532592775 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1869, 'dropout_prob': 0.3807695268299693, 'label_smoothing': 0.04888299119121421, 'lr': 0.008039197764394468, 'warmup_proportion': 0.0841086859038431, 'weight_decay': 0.00010763741035381019}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00028, Accuracy: 0.957\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1994, 'dropout_prob': 0.06300913813620652, 'label_smoothing': 0.0015262399516501582, 'lr': 0.0042939261444058475, 'warmup_proportion': 0.09303702943224616, 'weight_decay': 6.740337290041167e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 00:11:09 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 00:11:12 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cd5a5396c42746479580efdf29a35e95",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/31 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00034, Updated Learning Rate: 0.004164448428251854\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2c37fe1a9d9341e9ae0ba876d0deb570",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00053, Accuracy: 0.748\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "24ea164eb6d748b588092f95937e1ac5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/31 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00016, Updated Learning Rate: 0.0032204446083043854\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1bcb11f09b204f8da2ad709b98d07870",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00025, Accuracy: 0.870\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "743167729d0349d3b781591a0e3fcaec",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/31 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00009, Updated Learning Rate: 0.001774146847196692\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "492dc2d5764a49a697849191c5339835",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00016, Accuracy: 0.924\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "54a49f428f6a48efa44557462248ac54",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/31 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00007, Updated Learning Rate: 0.0005022939411602251\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "000ccc623fa84124b84b83ab289d2454",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00013, Accuracy: 0.939\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "14a41ba8e4d04101bb1ed551d34db760",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/31 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00007, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9202a4c850ff413aad4676dfc79f008d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 00:13:26,963] Trial 41 finished with value: 0.00012302398681640625 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1994, 'dropout_prob': 0.06300913813620652, 'label_smoothing': 0.0015262399516501582, 'lr': 0.0042939261444058475, 'warmup_proportion': 0.09303702943224616, 'weight_decay': 6.740337290041167e-06}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00012, Accuracy: 0.942\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 2024, 'dropout_prob': 0.023551620618388114, 'label_smoothing': 0.01261610623746187, 'lr': 0.005442453785293932, 'warmup_proportion': 0.0937276168480096, 'weight_decay': 8.680970962764236e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 00:13:29 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 00:13:32 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a4945dd9f85b4f3e8d2549c5ebc08a99",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00033, Updated Learning Rate: 0.005278343723151488\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "243a4b2bf94049f28b13349b15dab752",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00041, Accuracy: 0.760\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "57341fe2442c4742833ee067abf87334",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00015, Updated Learning Rate: 0.004081840338970449\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "928900a03ba04a89acc13a4da3b2772a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00027, Accuracy: 0.852\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4a1489d5041c451ca6f1dd119ac43129",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00010, Updated Learning Rate: 0.002248690801720577\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d85e0fa255a249149293bdc9487e75f7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00016, Accuracy: 0.934\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1d0c2494893647df908ea9c3c33ad16a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00008, Updated Learning Rate: 0.0006366461530688342\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c172b0c40c7a45b8bbf51dfae3611d68",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00014, Accuracy: 0.943\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ad92789112cc45648406b9af63efbe11",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00007, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "58686979610e4aa49b0105b0571827c0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 00:15:45,459] Trial 42 finished with value: 0.0001395293802022934 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 2024, 'dropout_prob': 0.023551620618388114, 'label_smoothing': 0.01261610623746187, 'lr': 0.005442453785293932, 'warmup_proportion': 0.0937276168480096, 'weight_decay': 8.680970962764236e-06}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00014, Accuracy: 0.947\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1617, 'dropout_prob': 7.066229373137167e-05, 'label_smoothing': 0.0013270250577958771, 'lr': 0.004101979102293477, 'warmup_proportion': 0.0870300233605791, 'weight_decay': 4.549711062958117e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 00:15:47 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 00:15:50 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "50a53ca72b384d25bb39a9e094f40c8b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00041, Updated Learning Rate: 0.003978289297668331\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e944275fc619434ea33871898d7c93a8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00050, Accuracy: 0.812\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6adb22799ddc4cd7be60d0481a27d768",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00018, Updated Learning Rate: 0.0030764843267201076\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d680f7f5869d4a9f8578ff9ffaa54197",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00026, Accuracy: 0.885\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "42c8db3ac7da4f73a17c2f62c9922e9a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00010, Updated Learning Rate: 0.0016948389531761924\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e511e3bbe57d44ba9cbe2555f60ef698",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00017, Accuracy: 0.926\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4a3b3ba4f38b481a8799ac17f1551cab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00008, Updated Learning Rate: 0.00047984040259569207\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e4ec810e5173495eae8364baab987fee",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00015, Accuracy: 0.940\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "43873c4fca134de5bb0b6ffb650288c0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00008, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b0a76370d4f9480b8cc58aa129196ab0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 00:18:03,804] Trial 43 finished with value: 0.00013843236565589905 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1617, 'dropout_prob': 7.066229373137167e-05, 'label_smoothing': 0.0013270250577958771, 'lr': 0.004101979102293477, 'warmup_proportion': 0.0870300233605791, 'weight_decay': 4.549711062958117e-06}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00014, Accuracy: 0.945\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1858, 'dropout_prob': 0.04617589712207457, 'label_smoothing': 0.031571706419335266, 'lr': 0.009926887702573158, 'warmup_proportion': 0.09922103951289431, 'weight_decay': 2.142312975023968e-05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 00:18:05 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 00:18:09 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eb6cbdda8a6b462cadc8d431c59f0aee",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/33 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00032, Updated Learning Rate: 0.009674028763314874\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a68cfd2cc99746f0a59af3064441e902",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00045, Accuracy: 0.816\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "da436fcc9997459589f8bc53977a0b84",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/33 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00013, Updated Learning Rate: 0.007536456257020114\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "560861a0a53f432ea3f4536229c8541e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00034, Accuracy: 0.896\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d131733623f644f8b4e7324c65bbb491",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/33 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00010, Updated Learning Rate: 0.004171278733617169\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "da06cc931a4c4e93846bb7a8be722861",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00024, Accuracy: 0.949\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "03bf78867be640f380224a0e7c4a4da5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/33 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00009, Updated Learning Rate: 0.0011840495911322835\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b6a098a392ad4ca69a74c5ddb984053e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00022, Accuracy: 0.960\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "333e806bbb5143ee8e7cf2a3996b9523",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/33 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00008, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0c69ccfb59ce43e38a8b7e99c8c746d2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 00:20:31,478] Trial 44 finished with value: 0.0002163852334022522 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1858, 'dropout_prob': 0.04617589712207457, 'label_smoothing': 0.031571706419335266, 'lr': 0.009926887702573158, 'warmup_proportion': 0.09922103951289431, 'weight_decay': 2.142312975023968e-05}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00022, Accuracy: 0.963\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1325, 'dropout_prob': 0.3478812348423442, 'label_smoothing': 0.011990221435607917, 'lr': 0.0047353062484082, 'warmup_proportion': 0.07934852987075283, 'weight_decay': 1.0603360840417003e-05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 00:20:33 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 00:20:36 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "28ee875384ad446ca8f091b819e1d26d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/46 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00042, Updated Learning Rate: 0.004539291327768795\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "50485da18ca3448fb7a376a31f8edcc5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00046, Accuracy: 0.839\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eccf94fdeb7e4b7b833c2fe2715e9a21",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/46 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00016, Updated Learning Rate: 0.0034569275481253397\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "26d759506a074022b0e7b3cb8c4c1dc0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00029, Accuracy: 0.911\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a8d84d25f0334b47b15c18522ffe4be7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/46 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00011, Updated Learning Rate: 0.001885939859261922\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b154c0e857144b24ad6198f60aa28101",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00022, Accuracy: 0.944\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e2a12d76389c47e98d28bff04f79e3d3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/46 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00009, Updated Learning Rate: 0.000531037863287386\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "24c6e647125e499098164e7c594ccd86",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00020, Accuracy: 0.955\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3e747b5a0888428cb6ee63a86f5b44dd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/46 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00009, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ba43691996764f928e327e8eff8cd24c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 00:23:00,941] Trial 45 finished with value: 0.00019673039466142655 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1325, 'dropout_prob': 0.3478812348423442, 'label_smoothing': 0.011990221435607917, 'lr': 0.0047353062484082, 'warmup_proportion': 0.07934852987075283, 'weight_decay': 1.0603360840417003e-05}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00020, Accuracy: 0.958\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 379, 'dropout_prob': 0.12532478127071403, 'label_smoothing': 0.023207273543586954, 'lr': 0.0027047054771084504, 'warmup_proportion': 0.09530365432486898, 'weight_decay': 3.7370275617275377e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 00:23:03 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 00:23:08 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0ac9c02dc3ec4a4bbfcf66dd98542ad5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/159 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00127, Updated Learning Rate: 0.0026204247964557964\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2b72823a0d494f23a64287b01a5efd9d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/27 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00117, Accuracy: 0.917\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d6038d8417bd49239677fc1736c40622",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/159 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00043, Updated Learning Rate: 0.0020233910508486476\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b88f6b877e5e4bc9a512d01bad61180d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/27 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00088, Accuracy: 0.956\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c44a33e72cd842fe93b457231eb1b90c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/159 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00035, Updated Learning Rate: 0.0011136298769680022\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "40df816766c14d038106fdada2344daa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/27 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00079, Accuracy: 0.966\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bcdceff175b6453e8ba33cb99a48d569",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/159 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00032, Updated Learning Rate: 0.0003151219433363187\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5649c1c4fb97423798b0d7af2d9ddc4d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/27 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00078, Accuracy: 0.966\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d995db94b5b241aca9b148aae02a28fe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/159 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00031, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "45b108b8d8434ff5b5b719ef6d0a1a31",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/27 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 00:26:15,167] Trial 46 finished with value: 0.0007615103632211685 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 379, 'dropout_prob': 0.12532478127071403, 'label_smoothing': 0.023207273543586954, 'lr': 0.0027047054771084504, 'warmup_proportion': 0.09530365432486898, 'weight_decay': 3.7370275617275377e-06}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00076, Accuracy: 0.969\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1456, 'dropout_prob': 0.09776517374201914, 'label_smoothing': 0.12424701195091421, 'lr': 0.0014485607060082983, 'warmup_proportion': 9.547112115332324e-05, 'weight_decay': 7.479046812158262e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 00:26:17 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 00:26:22 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7d2294c958ab49c38de95dc831b39e5e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/42 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00048, Updated Learning Rate: 0.0013253499652461545\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c4f8c102fcc1499db3c04e2af562cbac",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00094, Accuracy: 0.710\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7c41c95948984b63a2d85ffdcbb38710",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/42 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00033, Updated Learning Rate: 0.0009668270767212436\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ca514642511e4c4a8640a7de5e94467e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00074, Accuracy: 0.854\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "56d6460695ee49f1a8f7eac533bc9319",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/42 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00028, Updated Learning Rate: 0.000513046929023582\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c93c7085ec6a4de88e7f41da122102d5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00068, Accuracy: 0.893\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "48b19a7f5cec4653a4c3a22d65400a38",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/42 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00026, Updated Learning Rate: 0.00014222612501729937\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d33173348952471e8b705ab95ee022dc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00066, Accuracy: 0.908\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "32fc6235aa6e408fa85c01e5cae30cba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/42 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00026, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e29472b6c9b3434a82d969e444b99caa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00066, Accuracy: 0.909\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 00:29:46,076] Trial 47 finished with value: 0.0006585891544818878 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1456, 'dropout_prob': 0.09776517374201914, 'label_smoothing': 0.12424701195091421, 'lr': 0.0014485607060082983, 'warmup_proportion': 9.547112115332324e-05, 'weight_decay': 7.479046812158262e-06}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'epochs': 5, 'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1697, 'dropout_prob': 0.07082147003631134, 'label_smoothing': 0.1821572554676943, 'lr': 0.00590311909569932, 'warmup_proportion': 0.08903678862653412, 'weight_decay': 0.0005623841211419445}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 00:29:48 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 00:29:53 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "57ab4d8ec7c0480387665aa91c3b50d2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/36 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00098, Updated Learning Rate: 0.005707417402432405\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "285390fa31c4436bb0540402fbe1b3b0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00075, Accuracy: 0.860\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "11570c8f1ed7480e8a56f8148e958462",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/36 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00066, Updated Learning Rate: 0.004394374618739986\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a4e72482f6424584b7916009f81d78bc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00061, Accuracy: 0.957\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "51689961bbb04d81ae298b16b3347be4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/36 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00063, Updated Learning Rate: 0.0024141466705884674\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7a396459623c484d8e50c089470e39b8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00059, Accuracy: 0.968\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2ca9b04e7ec44791838f04e6ef19bf66",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/36 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00062, Updated Learning Rate: 0.0006824286708925685\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ca241e2287cd49f59d656aa39fd67c86",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00059, Accuracy: 0.968\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e848b4b4d1bc4170b2e0ed3ba36ecb00",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/36 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00061, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "adc6318f6917492e867368dff6a4df10",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00059, Accuracy: 0.969\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 00:33:36,409] Trial 48 finished with value: 0.000585165399312973 and parameters: {'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1697, 'dropout_prob': 0.07082147003631134, 'label_smoothing': 0.1821572554676943, 'lr': 0.00590311909569932, 'warmup_proportion': 0.08903678862653412, 'weight_decay': 0.0005623841211419445}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 968, 'dropout_prob': 0.020115059950453644, 'label_smoothing': 0.009697344358271373, 'lr': 0.003738260280127671, 'warmup_proportion': 0.008456759014071169, 'weight_decay': 1.2713170342649703e-05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 00:33:38 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 00:33:45 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a6b67537af8644e4a64419d5f63cbd5c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/62 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00046, Updated Learning Rate: 0.00340868886407908\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "215f2b17e92547bba8c4574fe96b08c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/11 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00047, Accuracy: 0.888\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "818074fff3604b0fbe345040810e68a2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/62 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00017, Updated Learning Rate: 0.0024804627281497216\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1b4ce8e666e3456d89d5b5b7a8225feb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/11 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00034, Accuracy: 0.928\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "539408a1fa294ef4abc4140eeec156d6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/62 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00013, Updated Learning Rate: 0.0013141473173386906\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6fc47dbd8170448fb2d96b6dda552fa6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/11 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00026, Accuracy: 0.953\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "02c7309b3e9949609366b1ae4ce633eb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/62 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00011, Updated Learning Rate: 0.00036397523081556696\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a13d5de77de345ecabdbf9b85064db86",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/11 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00025, Accuracy: 0.956\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "22b1b66876df4da7a4bbbe416f76c39b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/62 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00011, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "613c7a87983043c4b596d1168b4a9b1f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/11 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 00:37:03,662] Trial 49 finished with value: 0.0002470990613102913 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 968, 'dropout_prob': 0.020115059950453644, 'label_smoothing': 0.009697344358271373, 'lr': 0.003738260280127671, 'warmup_proportion': 0.008456759014071169, 'weight_decay': 1.2713170342649703e-05}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00025, Accuracy: 0.957\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 2048, 'dropout_prob': 0.2194195323254399, 'label_smoothing': 0.07100021036648316, 'lr': 0.013326781784477762, 'warmup_proportion': 0.07443025547725267, 'weight_decay': 2.3754504934027695e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 00:37:05 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 00:37:09 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eb8269bd68044d9aa7373d602812249f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00029, Updated Learning Rate: 0.012775128324948887\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9fa8ea74b2264ec9809c2ac047d359be",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00059, Accuracy: 0.697\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "41251a44192447c4bd172918bd9d8974",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00015, Updated Learning Rate: 0.009728984074494178\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3dd2bbacbd3f4276ac33a7749b01416c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00034, Accuracy: 0.940\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "77abaa691230409696e2df4713ce8067",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00012, Updated Learning Rate: 0.005307683947892726\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "47514e4b49d546a7870dd170b8425e46",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00030, Accuracy: 0.947\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ced02e0af7c648a2a1708e1bbafd2b5a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00011, Updated Learning Rate: 0.0014945233427521838\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "83c85594baab4a33b634a48a8486865a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00029, Accuracy: 0.959\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "156214af3e5743cca45a75561f0937f1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00011, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "41dfd35efb9d48a5aade4e8284dc4c4e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 00:40:20,251] Trial 50 finished with value: 0.0002853821396827698 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 2048, 'dropout_prob': 0.2194195323254399, 'label_smoothing': 0.07100021036648316, 'lr': 0.013326781784477762, 'warmup_proportion': 0.07443025547725267, 'weight_decay': 2.3754504934027695e-06}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00029, Accuracy: 0.962\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1910, 'dropout_prob': 0.011244867264315426, 'label_smoothing': 0.0006663369310054537, 'lr': 0.00455823515995647, 'warmup_proportion': 0.09907346973852363, 'weight_decay': 8.197211303389555e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 00:40:22 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 00:40:28 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2266219dbd3540ad8f1633ea16b1e719",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/32 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00034, Updated Learning Rate: 0.00442078755178722\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "74f3a3fca4ce41238e02bfe2762a12dc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00049, Accuracy: 0.757\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1e18083ee8df489c9ffe2ed11607f385",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/32 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00015, Updated Learning Rate: 0.0034186763699673526\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0e3dca05def049aea848a806d18c3ce9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00024, Accuracy: 0.876\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "48a7191f15824948abfe3c8034c3c1ef",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/32 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00009, Updated Learning Rate: 0.00188335296552635\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5705f655e50047ff9966af5f3e916e76",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00016, Accuracy: 0.919\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0ced78a5d0014981b295ccc804bf5597",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/32 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00007, Updated Learning Rate: 0.0005332122226211353\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b637fc47669f45beb6d8973e900ab733",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00013, Accuracy: 0.935\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ae80b81630c24df6bd8f51c4324866c8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/32 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00007, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "374e08e45b61435c90d790585bccdb80",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 00:43:16,631] Trial 51 finished with value: 0.0001220807082951069 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1910, 'dropout_prob': 0.011244867264315426, 'label_smoothing': 0.0006663369310054537, 'lr': 0.00455823515995647, 'warmup_proportion': 0.09907346973852363, 'weight_decay': 8.197211303389555e-06}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00012, Accuracy: 0.937\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1531, 'dropout_prob': 0.056588318406691596, 'label_smoothing': 0.010387616273347925, 'lr': 0.005124348014480555, 'warmup_proportion': 0.09488668384971018, 'weight_decay': 3.8123538957438616e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 00:43:18 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 00:43:23 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0f1d29e1b1754c8da7cf999b92504a4a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/40 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00041, Updated Learning Rate: 0.004990226330323675\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "381e25cae51b495aaf2246a32ad66a20",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00042, Accuracy: 0.824\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4139b6f69d0f4a778e01ce3c9c931c99",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/40 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00016, Updated Learning Rate: 0.003883112699431709\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "89c73f5e799c4e6d9ece67b8c22c974e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00025, Accuracy: 0.914\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ebf451770e0c40edb4d7d2f9188c9549",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/40 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00010, Updated Learning Rate: 0.0021476603808333475\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e1852da949354623a131f464cf5fd8a8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00018, Accuracy: 0.949\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e2884be407d24c2b9cdba5d1331709e0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/40 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00008, Updated Learning Rate: 0.0006093822002874161\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f417af31486b41d9bc73ec534bc6e40d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00016, Accuracy: 0.959\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d0f7e634b6144b3e9fbc5277aa0edc04",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/40 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00008, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7c30c9a6f5434546b8e018c23699ad83",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 00:46:13,275] Trial 52 finished with value: 0.00015829007774591445 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1531, 'dropout_prob': 0.056588318406691596, 'label_smoothing': 0.010387616273347925, 'lr': 0.005124348014480555, 'warmup_proportion': 0.09488668384971018, 'weight_decay': 3.8123538957438616e-06}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00016, Accuracy: 0.959\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1871, 'dropout_prob': 0.03663925950208341, 'label_smoothing': 0.03999437996109294, 'lr': 0.006910177816613813, 'warmup_proportion': 0.090355641954154, 'weight_decay': 7.88921137557113e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 00:46:15 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 00:46:19 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d1cd0d6e3efe4a7a8709f25454cd752f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/33 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00034, Updated Learning Rate: 0.006667549331903977\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5104489790364fe5a1473eff0ffbc27f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00084, Accuracy: 0.610\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6ae53d9a51c74853a66b1bb536150f69",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/33 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00016, Updated Learning Rate: 0.005119590684231843\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ef062f52b8064922a542ee85c944d342",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00039, Accuracy: 0.881\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "788177cdb524431f9e23f68fa7803b5f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/33 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00012, Updated Learning Rate: 0.0028076698066578022\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e2cd5fa79c6948d1b1d985105e345379",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00027, Accuracy: 0.950\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "98e5f2691f59439ca5b7fc96c4947eee",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/33 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00010, Updated Learning Rate: 0.0007928971494886908\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6c810ff42d79430fad744aaa1001331b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00026, Accuracy: 0.956\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "feee062824e848a0aea1ebb8f03953c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/33 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00010, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9fae9bd19e564e47bd0d660f6016421e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 00:49:20,285] Trial 53 finished with value: 0.00025623578429222107 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1871, 'dropout_prob': 0.03663925950208341, 'label_smoothing': 0.03999437996109294, 'lr': 0.006910177816613813, 'warmup_proportion': 0.090355641954154, 'weight_decay': 7.88921137557113e-06}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00026, Accuracy: 0.957\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1685, 'dropout_prob': 0.013904941686217556, 'label_smoothing': 0.02435521708040058, 'lr': 0.0028505332450974937, 'warmup_proportion': 0.09604114467978811, 'weight_decay': 2.8687222661720064e-05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 00:49:22 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 00:49:26 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b7c6cc3a4dd5417592a5f7590db6ffc0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/36 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00042, Updated Learning Rate: 0.002764579150410259\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "38aa295bc74a464da4b4499dee6df0b0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00061, Accuracy: 0.715\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7bb855675de84275a1af47d10eafcb3e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/36 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00021, Updated Learning Rate: 0.0021378999338231204\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3bb28cb9d3ec4d8ebfdd7e0e60f5069b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00032, Accuracy: 0.880\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bf15d8821b2c4e4c83151604cbeb9721",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/36 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00014, Updated Learning Rate: 0.0011777716708536567\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "66c764916c9e47d59e9a8ca5f1d5d63c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00025, Accuracy: 0.926\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8057cf69b59f480e98a0c9bd8154ea2a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/36 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00012, Updated Learning Rate: 0.0003334490463823251\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c5dde11cd9074ddcae266e1631f8d088",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00023, Accuracy: 0.938\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cad5ab0734734eafa316b6c0ea774571",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/36 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00011, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f739f52623fe4c64a2eab00286f144f3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 00:52:25,480] Trial 54 finished with value: 0.00022504575848579405 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1685, 'dropout_prob': 0.013904941686217556, 'label_smoothing': 0.02435521708040058, 'lr': 0.0028505332450974937, 'warmup_proportion': 0.09604114467978811, 'weight_decay': 2.8687222661720064e-05}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00023, Accuracy: 0.941\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': True, 'batch_size': 1156, 'dropout_prob': 0.08782180972457004, 'label_smoothing': 0.009620404991221522, 'lr': 0.009128015822471544, 'warmup_proportion': 0.08259579236844573, 'weight_decay': 1.287457297760135e-05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 00:52:27 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 00:52:31 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2f3ffea47a284e769b2b8b2da48a84b9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/52 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00067, Updated Learning Rate: 0.008823763177095587\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a1d72c58e7bf401a83b3088a651609ef",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/9 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 1.01059, Accuracy: 0.109\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "026d157f272244b794439b23e094bfde",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/52 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00066, Updated Learning Rate: 0.006792047977717192\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "731a2bd828d64d85b1b77ff8bc60211d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/9 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 00:53:50,428] Trial 55 finished with value: 157.5256296875 and parameters: {'accumulation_steps': 3, 'batchnorm': True, 'batch_size': 1156, 'dropout_prob': 0.08782180972457004, 'label_smoothing': 0.009620404991221522, 'lr': 0.009128015822471544, 'warmup_proportion': 0.08259579236844573, 'weight_decay': 1.287457297760135e-05}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] Loss: 157.52563, Accuracy: 0.109\n",
      "Early stopping triggered.\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 2045, 'dropout_prob': 0.05196682931493239, 'label_smoothing': 0.030693617693221605, 'lr': 0.0031330422076305464, 'warmup_proportion': 0.08724502881929494, 'weight_decay': 1.607234374958887e-05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 00:53:52 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 00:53:56 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "103760fdddd34b008a5f3cee206cff2e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00036, Updated Learning Rate: 0.0030385694253758813\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8a3ecd4e31a446d1bcfd0d8d2d5c8a6e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00053, Accuracy: 0.726\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4135bd57707343d48a3c0144c418c862",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00020, Updated Learning Rate: 0.0023497816557229096\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "51c1a11be5d3413f80c74bb0a44f0dcc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00030, Accuracy: 0.876\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2051a8c6767949cda1c0f9b8470caa80",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00013, Updated Learning Rate: 0.0012944975688609628\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1f17e3ad0a7649f688bc2ce545dc2fe3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00026, Accuracy: 0.903\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d7bd3777b0004dae9c34e43edb6f95f9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00011, Updated Learning Rate: 0.0003664963172089757\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ce8ebb571168491fab99493a10cfb666",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00023, Accuracy: 0.925\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "05fe40eefc4e46fc960060ceb149d468",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/30 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00011, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "89d4c29eaa8444359aee8d20b2f345ef",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/5 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 00:56:39,241] Trial 56 finished with value: 0.0002262932389974594 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 2045, 'dropout_prob': 0.05196682931493239, 'label_smoothing': 0.030693617693221605, 'lr': 0.0031330422076305464, 'warmup_proportion': 0.08724502881929494, 'weight_decay': 1.607234374958887e-05}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00023, Accuracy: 0.929\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1507, 'dropout_prob': 0.2765682978738866, 'label_smoothing': 0.14089553425116288, 'lr': 0.01104412946332945, 'warmup_proportion': 0.09041731837318381, 'weight_decay': 6.227068425213167e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 00:56:41 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 00:56:44 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0c465c3fe2244db487d26cb1f9e8f483",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/40 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00038, Updated Learning Rate: 0.010664963390497722\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "30345a8a09b14d9abd15a67d07ac652e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00077, Accuracy: 0.880\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eab6ab93403e405b8650c205e3baee3c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/40 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00024, Updated Learning Rate: 0.0081977860034261\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "31e45dc0d90149baaebf55522542b970",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00065, Accuracy: 0.939\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0c57a9e1f15a4117a033dcc40b710035",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/40 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00022, Updated Learning Rate: 0.004498888403208114\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "be2a6d2300bd4c8aabaab9e0fc692646",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00061, Accuracy: 0.955\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9d83876190484a9590bc8ca48d59ddb6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/40 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00021, Updated Learning Rate: 0.0012709916425294745\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "65ad1e61396241e6b6261d672cdc9c46",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00060, Accuracy: 0.959\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "676e54ef741744d3999cfe2d8cef8764",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/40 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00020, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ef54af8ba297493db9d9323fa15eb3d3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 00:59:04,654] Trial 57 finished with value: 0.0005987521529197693 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1507, 'dropout_prob': 0.2765682978738866, 'label_smoothing': 0.14089553425116288, 'lr': 0.01104412946332945, 'warmup_proportion': 0.09041731837318381, 'weight_decay': 6.227068425213167e-06}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00060, Accuracy: 0.962\n",
      "{'epochs': 5, 'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 1702, 'dropout_prob': 0.10959292744083032, 'label_smoothing': 0.019128781297432347, 'lr': 0.00234632539556916, 'warmup_proportion': 0.09946743538298781, 'weight_decay': 2.018304379925148e-05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 00:59:06 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 00:59:10 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "be3a07541b5b42938b7799c5de6414f9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/36 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00061, Updated Learning Rate: 0.0022755750278740387\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d401cce500c24c33accdeee4656eba40",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00053, Accuracy: 0.739\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "da371288cceb41b1ac069c281e8e6072",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/36 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00026, Updated Learning Rate: 0.00175974404667687\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ef566f9919a24933a1c5c7707013e94c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00028, Accuracy: 0.896\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3dbf52c934fc42d8bac6180d738f0ba3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/36 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00017, Updated Learning Rate: 0.000969445133207468\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a18a4676ea3844cbaad00e5f33b8d601",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00021, Accuracy: 0.935\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "99b1473ea28d4cf18c767e4bd913d1eb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/36 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00015, Updated Learning Rate: 0.0002744679322722333\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3046ad8774d74944ab51f8a61d1aff22",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00020, Accuracy: 0.944\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8e45650f46214d1e9bcf54a60594f3b5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/36 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00014, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dc69ee044dd34af297c2d79620877ed5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 01:01:37,337] Trial 58 finished with value: 0.00019985833466053008 and parameters: {'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 1702, 'dropout_prob': 0.10959292744083032, 'label_smoothing': 0.019128781297432347, 'lr': 0.00234632539556916, 'warmup_proportion': 0.09946743538298781, 'weight_decay': 2.018304379925148e-05}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00020, Accuracy: 0.945\n",
      "Early stopping triggered.\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1351, 'dropout_prob': 0.07246236468130336, 'label_smoothing': 0.04429345289365813, 'lr': 0.007724008852157527, 'warmup_proportion': 0.032653827346873505, 'weight_decay': 4.250467799400843e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 01:01:39 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 01:01:42 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0212a285ba164a348b1f85434f1dd031",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/45 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00040, Updated Learning Rate: 0.0072065983685909146\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "310401e186d943e48bbee4a9a6d0ecd2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00063, Accuracy: 0.820\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3d8bb7a58ca44b4f8c65864df60cea5b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/45 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00018, Updated Learning Rate: 0.005339929535659754\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e32a704aeca7440faa39bae40d9294e5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00041, Accuracy: 0.940\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "03ac12f089644c5ca264d745015bbc52",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/45 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00014, Updated Learning Rate: 0.0028624441283393487\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6abbbb7728474790b8d9ba80f9d13034",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00036, Accuracy: 0.956\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "53be1c22570e4a7881753874db0a2ac6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/45 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00013, Updated Learning Rate: 0.0007980703144296423\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "acd51aa1dace4328a9d1501c67801a81",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00035, Accuracy: 0.957\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "80f78202204b4c19ab164d7b0cccf52a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/45 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00013, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fa95b41551b44c7395032753d58c74bf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 01:04:23,693] Trial 59 finished with value: 0.0003443235069513321 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1351, 'dropout_prob': 0.07246236468130336, 'label_smoothing': 0.04429345289365813, 'lr': 0.007724008852157527, 'warmup_proportion': 0.032653827346873505, 'weight_decay': 4.250467799400843e-06}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00034, Accuracy: 0.958\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': True, 'batch_size': 798, 'dropout_prob': 0.039162771049601114, 'label_smoothing': 0.09652444906963403, 'lr': 0.006144901520183269, 'warmup_proportion': 0.05173860508328992, 'weight_decay': 3.0318844957758715e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 01:04:25 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 01:04:30 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b94a44db096d438292165c7973cd2ef1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/76 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00097, Updated Learning Rate: 0.0057987963175680874\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8dd4f9862d484fb4a862e8fa8bee53fc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/13 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 10.58357, Accuracy: 0.055\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "28414f76c574477d9c511b676c315a46",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/76 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00097, Updated Learning Rate: 0.004342024854334814\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a2d37e558bc24b80aae324924326c140",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/13 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 01:05:50,931] Trial 60 finished with value: 358.36845625 and parameters: {'accumulation_steps': 3, 'batchnorm': True, 'batch_size': 798, 'dropout_prob': 0.039162771049601114, 'label_smoothing': 0.09652444906963403, 'lr': 0.006144901520183269, 'warmup_proportion': 0.05173860508328992, 'weight_decay': 3.0318844957758715e-06}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] Loss: 358.36846, Accuracy: 0.055\n",
      "Early stopping triggered.\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1908, 'dropout_prob': 0.012608585144554835, 'label_smoothing': 0.0025294074673229416, 'lr': 0.0052701089239719475, 'warmup_proportion': 0.09681524166760228, 'weight_decay': 8.858445654611237e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 01:05:53 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 01:05:56 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d369056c183b451b8f1983c90381f23e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/32 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00035, Updated Learning Rate: 0.0051111956952831755\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aebaef58fe4b4b1fb7292220283e0ee5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00059, Accuracy: 0.684\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5d106a2ecc1e4839a444bf418722e1fa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/32 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00015, Updated Learning Rate: 0.003952581692978961\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d184f77162a648faadd0d1961ac2a821",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00022, Accuracy: 0.891\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7b3210d25da44b24bec5f3e33aa17a91",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/32 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00009, Updated Learning Rate: 0.002177482056608996\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3fe87418eb534094abd9b8156ebbd377",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00015, Accuracy: 0.931\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7c0f29e7be1a413ebf3b972ac514603d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/32 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00007, Updated Learning Rate: 0.0006164856340657506\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bf55d089729546b6996c9ed5c6cf16d0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00013, Accuracy: 0.944\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "525cf65254134c9d9f6e53744951c3aa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/32 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00006, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "485ea292943943759499835ecd23fe04",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 01:08:34,872] Trial 61 finished with value: 0.00012276450395584107 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1908, 'dropout_prob': 0.012608585144554835, 'label_smoothing': 0.0025294074673229416, 'lr': 0.0052701089239719475, 'warmup_proportion': 0.09681524166760228, 'weight_decay': 8.858445654611237e-06}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00012, Accuracy: 0.946\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1951, 'dropout_prob': 0.0027115179379815224, 'label_smoothing': 0.000477595046984889, 'lr': 0.004023982723839898, 'warmup_proportion': 0.09658910194567755, 'weight_decay': 6.046506885341318e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 01:08:36 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 01:08:40 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "98f6532d3d304115aa74a5e66ff788a0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/31 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00034, Updated Learning Rate: 0.0039026447978011154\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1a3cd157720e41c7b938c870565368ba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00047, Accuracy: 0.775\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "af5ecbc40a2644cf9562c6b8e228920c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/31 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00015, Updated Learning Rate: 0.0030179870428799238\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3f00e30999b9435cad4da8e966a19496",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00022, Accuracy: 0.899\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c9fa9972cfa646da964ebad6e911a3f1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/31 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00009, Updated Learning Rate: 0.0016626127284409447\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "460ee19335954001aad5ad685423cdd2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00014, Accuracy: 0.934\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f4d30521b813436d98e5e601d90e7f93",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/31 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00007, Updated Learning Rate: 0.00047071655951778777\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cd79bd896f1041699b0f9b7582ace7f4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00012, Accuracy: 0.942\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5db91b3570e64d02bc93aaf4bbffbf2e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/31 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00007, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7ff4f91325944aa2a98db9df28c81a77",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 01:11:15,263] Trial 62 finished with value: 0.00011712319999933243 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1951, 'dropout_prob': 0.0027115179379815224, 'label_smoothing': 0.000477595046984889, 'lr': 0.004023982723839898, 'warmup_proportion': 0.09658910194567755, 'weight_decay': 6.046506885341318e-06}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00012, Accuracy: 0.943\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 637, 'dropout_prob': 0.030826453233809826, 'label_smoothing': 0.028211290714924897, 'lr': 0.0037588123069648774, 'warmup_proportion': 0.092119320736955, 'weight_decay': 5.88969001213547e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 01:11:17 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 01:11:20 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "602d4c8a0fdd4fd6bdbf47852a46e7dd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/95 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00076, Updated Learning Rate: 0.0036389717304675957\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9c5e0e2909804bd7a0426768213e76bd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/16 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00077, Accuracy: 0.913\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4c97d1bf3fb0454f95750ee6811cd8ec",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/95 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00028, Updated Learning Rate: 0.0028069085353067255\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "48b1e82cf4fb4d658ed533005f82d5a7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/16 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00057, Accuracy: 0.959\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9287d0c356064b759e8ce86b4d7e23db",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/95 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00023, Updated Learning Rate: 0.0015438252266511646\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "21c83c55dfd443c5a5dc0344fbf63dc3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/16 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00051, Accuracy: 0.969\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dc9aad50ab674d07ac497273b4a8dce8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/95 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00021, Updated Learning Rate: 0.00043668990476516365\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9f0d27d51dd741eaa2bb95504eb5e2fa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/16 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00051, Accuracy: 0.967\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "16112ce2772d4da38836769d9b299ca3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/95 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00020, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "218296e58134474e8794ed026cef8e06",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/16 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 01:13:49,050] Trial 63 finished with value: 0.0005039022266864776 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 637, 'dropout_prob': 0.030826453233809826, 'label_smoothing': 0.028211290714924897, 'lr': 0.0037588123069648774, 'warmup_proportion': 0.092119320736955, 'weight_decay': 5.88969001213547e-06}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00050, Accuracy: 0.969\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1800, 'dropout_prob': 0.0033103471174727754, 'label_smoothing': 0.007991713259979227, 'lr': 0.0042001047633916455, 'warmup_proportion': 0.095778733744986, 'weight_decay': 1.1783698450478562e-05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 01:13:51 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 01:13:55 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b276379024d1486ba0a22044bf838f77",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00037, Updated Learning Rate: 0.0040931191635676855\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f3c9f10d84bb442bab12e69ce2cf7f20",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00048, Accuracy: 0.752\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "61c93e31f6af424997f74c6727d16b55",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00017, Updated Learning Rate: 0.003188703929429756\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bf58b2bfb26c4fa6a1cb83909619e2d5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00024, Accuracy: 0.893\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0f4c459fdc8e44ad9632feeb6f854511",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00010, Updated Learning Rate: 0.0017648842420125714\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2e9ab7c3a35e481cb138b072a083e44e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00018, Accuracy: 0.927\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "05cfc67bbbfc45ddbb0f928bb908590d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00008, Updated Learning Rate: 0.0005009759832900641\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e986a1928c5c40bb9143b14b8efb759b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00016, Accuracy: 0.940\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "63730ea28c104e06ab6f01d126fa0ef3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00008, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "33d2b63897884f908162a826ce9dea21",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00015, Accuracy: 0.943\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 01:16:42,567] Trial 64 finished with value: 0.00015349487066268922 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1800, 'dropout_prob': 0.0033103471174727754, 'label_smoothing': 0.007991713259979227, 'lr': 0.0042001047633916455, 'warmup_proportion': 0.095778733744986, 'weight_decay': 1.1783698450478562e-05}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1654, 'dropout_prob': 0.024467625090603715, 'label_smoothing': 0.018970775969641546, 'lr': 0.0018394782514119623, 'warmup_proportion': 0.08603705445054774, 'weight_decay': 1.2310562074035852e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 01:16:44 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 01:16:48 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "906eeece90e144a2a87abeee470ff127",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/37 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00043, Updated Learning Rate: 0.0017840111951799747\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "84c8c78f9a2c44a69262fa03985f74b8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00084, Accuracy: 0.674\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "64312728bae745119f799cfffc86cce7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/37 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00027, Updated Learning Rate: 0.0013796086885589716\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "99474562f68b48a99145c295206dcab4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00048, Accuracy: 0.845\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ed571d3f2ccd40c3bc0a6befe7e8026a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/37 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00019, Updated Learning Rate: 0.000760028102598162\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "673ab72da1304cc1818af78e816014bf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00037, Accuracy: 0.885\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2455c45b6fbe46e7bfebc6a491db9b64",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/37 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00016, Updated Learning Rate: 0.0002151780793398069\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ae071e84d2e549c38e1d7d3e272108cd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00033, Accuracy: 0.902\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7272114cfa8949539055dd4e08c9b575",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/37 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00015, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4f9101d465fe48769bbc587b81a71224",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 01:19:16,788] Trial 65 finished with value: 0.0003228820353746414 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1654, 'dropout_prob': 0.024467625090603715, 'label_smoothing': 0.018970775969641546, 'lr': 0.0018394782514119623, 'warmup_proportion': 0.08603705445054774, 'weight_decay': 1.2310562074035852e-06}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00032, Accuracy: 0.903\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1893, 'dropout_prob': 0.05285432067544883, 'label_smoothing': 0.008691567191083804, 'lr': 0.006503340994859891, 'warmup_proportion': 0.09019985121159034, 'weight_decay': 1.5346723340406505e-05}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 01:19:18 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 01:19:22 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6766a542060a4ddfba98625efd30e359",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/32 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00034, Updated Learning Rate: 0.00630724126909211\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "48e74fbbf8824ad2a16cf2d357a041ed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00063, Accuracy: 0.645\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb296e978eab43078281a0c1f0a468a4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/32 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00014, Updated Learning Rate: 0.004877505746144918\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a7493dd049604a8684edd8351ee5ad1f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00026, Accuracy: 0.878\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4bef80535766457796d5321138b7070b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/32 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00009, Updated Learning Rate: 0.002687023841177915\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "090769c94d794094b5e62447eaa07f1d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00017, Accuracy: 0.930\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d749ba90d6ba427a831aaba563aedd6b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/32 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00007, Updated Learning Rate: 0.0007607463820198131\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5097705200f4485491b7b6e1815c2b38",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00015, Accuracy: 0.946\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "22107b9c9d434c09a7cc8e2a4fe7b0e4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/32 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00007, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4e4d4ce6078d41b6b8c1134037ad90d2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 01:21:57,539] Trial 66 finished with value: 0.0001488666906952858 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1893, 'dropout_prob': 0.05285432067544883, 'label_smoothing': 0.008691567191083804, 'lr': 0.006503340994859891, 'warmup_proportion': 0.09019985121159034, 'weight_decay': 1.5346723340406505e-05}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00015, Accuracy: 0.947\n",
      "{'epochs': 5, 'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1784, 'dropout_prob': 0.079996902208114, 'label_smoothing': 0.01655763892667384, 'lr': 0.0030763152399638977, 'warmup_proportion': 0.07811644606779555, 'weight_decay': 4.9313713889771346e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 01:21:59 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 01:22:03 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b24e30d261c94653b171eab5385192b6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00041, Updated Learning Rate: 0.0029683004066310015\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dd79950bad7549ef8a7abfb87ca02b20",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00055, Accuracy: 0.762\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "633f003e0faf48c69cda07e4f708e5f9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00021, Updated Learning Rate: 0.0022791707047558024\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e06d046952844399b7980b512387a782",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00037, Accuracy: 0.833\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "65106f5d67de4074a4396703ccd31c35",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00013, Updated Learning Rate: 0.0012499356231096819\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6a3746bc9b7a43e98dffc89ab72211be",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00025, Accuracy: 0.904\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "43977337e0b74ef68761e16568906cf6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00011, Updated Learning Rate: 0.0003529868043093675\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e999c7cf68cf4307aa92c9ce45e6861c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00021, Accuracy: 0.932\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a7a907c5ba21423b9e9398ec0cb56c85",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00010, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d331e162463c4635a9efbc1d6527366f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 01:24:39,123] Trial 67 finished with value: 0.00020747385025024414 and parameters: {'accumulation_steps': 3, 'batchnorm': False, 'batch_size': 1784, 'dropout_prob': 0.079996902208114, 'label_smoothing': 0.01655763892667384, 'lr': 0.0030763152399638977, 'warmup_proportion': 0.07811644606779555, 'weight_decay': 4.9313713889771346e-06}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00021, Accuracy: 0.934\n",
      "{'epochs': 5, 'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 1930, 'dropout_prob': 0.06617749925302711, 'label_smoothing': 0.0006590912614831109, 'lr': 0.004851705448029092, 'warmup_proportion': 0.0994046091609695, 'weight_decay': 2.8768462739162755e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 01:24:41 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 01:24:44 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "553c22580ec648048758b3223e52576d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/32 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00046, Updated Learning Rate: 0.00470540862788441\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "daf94b84afea48baa2ad7801c462e208",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00052, Accuracy: 0.706\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4ac952d6ff944282979d59633f1d47db",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/32 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00017, Updated Learning Rate: 0.003638779086021819\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "826b492134444a15a0435eadc9620907",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00018, Accuracy: 0.911\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8010df4bdef54196824d4cc005145748",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/32 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00009, Updated Learning Rate: 0.002004607819201061\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3a29e099bb4a43fcb8cf665f2ffbeac0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00011, Accuracy: 0.949\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7a02a68344034495b53d6ac53e3d68c6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/32 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00007, Updated Learning Rate: 0.0005675417249581675\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c41d318431084e239600625783cfc6de",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00010, Accuracy: 0.956\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3a4d19dde70e414eb7f8dc7b0cccb7c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/32 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00007, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ba27c0c8fd5a4ef5bfeba4380bbd5816",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00009, Accuracy: 0.956\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 01:27:20,984] Trial 68 finished with value: 9.192864671349526e-05 and parameters: {'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 1930, 'dropout_prob': 0.06617749925302711, 'label_smoothing': 0.0006590912614831109, 'lr': 0.004851705448029092, 'warmup_proportion': 0.0994046091609695, 'weight_decay': 2.8768462739162755e-06}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'epochs': 5, 'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 1579, 'dropout_prob': 0.13009080475648893, 'label_smoothing': 0.037833809788747036, 'lr': 0.00859070775167085, 'warmup_proportion': 0.09983366211202707, 'weight_decay': 1.7629226701210268e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 01:27:23 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 01:27:26 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4a9bda02ca0b4a9db2f1cdfbab4e61cd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00047, Updated Learning Rate: 0.008355252895502966\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "efd261294ae442caa2c0f0625bd91e37",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00044, Accuracy: 0.894\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ee4691f1eb474c239fb96a65546a48cf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00019, Updated Learning Rate: 0.006488695550587501\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9be8de158fd041368033cc1c42a8f0a1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00030, Accuracy: 0.950\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bffd031d2ac840818e050a6d13eb47d0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00015, Updated Learning Rate: 0.003584241392108344\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3d5a0772bd7947388bdbca94f1b33459",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00027, Accuracy: 0.965\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a829979205f8416c8b38fb0056fba2d0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00014, Updated Learning Rate: 0.0010162881032573206\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dc0c26d7887a4278ad6f4dc4c27211d7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00026, Accuracy: 0.972\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d249d6d969524f0a8d913b469bf02fe2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00014, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3d4be77bbfbc4769b62f6d4cf05720ca",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 01:29:58,767] Trial 69 finished with value: 0.00025559661388397216 and parameters: {'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 1579, 'dropout_prob': 0.13009080475648893, 'label_smoothing': 0.037833809788747036, 'lr': 0.00859070775167085, 'warmup_proportion': 0.09983366211202707, 'weight_decay': 1.7629226701210268e-06}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00026, Accuracy: 0.971\n",
      "Early stopping triggered.\n",
      "{'epochs': 5, 'accumulation_steps': 2, 'batchnorm': True, 'batch_size': 472, 'dropout_prob': 0.09513497578138688, 'label_smoothing': 0.025634104988118273, 'lr': 0.00497057921772258, 'warmup_proportion': 0.09246544247975522, 'weight_decay': 2.5797672441499613e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 01:30:00 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 01:30:04 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "07b9dc8d43f246939fca171ae7e273f5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/128 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00246, Updated Learning Rate: 0.004803895484118803\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "99a3975cc92741be8e8ff35c1b67a1eb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/22 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 437.32650, Accuracy: 0.057\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3c552fb6cd094eadac39f2364a3a34c4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/128 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00245, Updated Learning Rate: 0.0036967174775946144\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "68dd8a30c56a443b842f6c29ccdb34cd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/22 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 01:31:09,757] Trial 70 finished with value: 499.1525765625 and parameters: {'accumulation_steps': 2, 'batchnorm': True, 'batch_size': 472, 'dropout_prob': 0.09513497578138688, 'label_smoothing': 0.025634104988118273, 'lr': 0.00497057921772258, 'warmup_proportion': 0.09246544247975522, 'weight_decay': 2.5797672441499613e-06}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] Loss: 499.15258, Accuracy: 0.057\n",
      "Early stopping triggered.\n",
      "{'epochs': 5, 'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 1942, 'dropout_prob': 0.061320308395246374, 'label_smoothing': 0.005485139548162462, 'lr': 0.003817606155518297, 'warmup_proportion': 0.09569847139182544, 'weight_decay': 3.043452993656249e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 01:31:11 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 01:31:15 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1b146d9d952b4e70b8b9ac170c8e33ec",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/31 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00046, Updated Learning Rate: 0.003715705992208584\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bfe69d0582fb47c6be9d62a74eb81fcb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00035, Accuracy: 0.853\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a6a3d87e72c24235b6e247dea1d45650",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/31 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00018, Updated Learning Rate: 0.0028889241986901756\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "320374da1dc64f2aa0b6ce1abbdbc65f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00020, Accuracy: 0.908\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e197a72a07674c9999c77f53e4215c01",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/31 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00011, Updated Learning Rate: 0.0015969478666837682\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5fa88ed2bf444c3f98d5042660ed227a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00013, Accuracy: 0.948\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bc99b241f26b4d62b030aeb73b6447a6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/31 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00009, Updated Learning Rate: 0.00045298726242076667\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "596ddf8d0cc34d49b3ddbd5071ffa0a0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00012, Accuracy: 0.957\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9334435464e941f7857fb107863efc8b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/31 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00008, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "538f1e3ca9b3469aa0fdc087374036a2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 01:34:26,906] Trial 71 finished with value: 0.000112393319606781 and parameters: {'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 1942, 'dropout_prob': 0.061320308395246374, 'label_smoothing': 0.005485139548162462, 'lr': 0.003817606155518297, 'warmup_proportion': 0.09569847139182544, 'weight_decay': 3.043452993656249e-06}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00011, Accuracy: 0.958\n",
      "{'epochs': 5, 'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 1694, 'dropout_prob': 0.06302729759580003, 'label_smoothing': 0.008858948102414693, 'lr': 0.005827351205432264, 'warmup_proportion': 0.09649740402444897, 'weight_decay': 3.231662262492706e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 01:34:29 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 01:34:33 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6a255f4eb7304fdba2891314799c2cb0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/36 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00047, Updated Learning Rate: 0.005651635065952415\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "543abb685521400ebb902eb59a9d52ad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00039, Accuracy: 0.794\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "71ae3e9cd4e34081bed708c3507e4f29",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/36 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00015, Updated Learning Rate: 0.004370513404074198\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cad03b2eed1445958f079cbaf176409a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00016, Accuracy: 0.945\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aab5cf19be5e4a05ba761d7d16f46ffe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/36 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00011, Updated Learning Rate: 0.0024077211439918807\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6d5ab693c9344013b26102b49f3f8cc4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00013, Accuracy: 0.954\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a2f8aac25d9c452b86425000c8bfa413",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/36 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00009, Updated Learning Rate: 0.0006816705982040993\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1539d025fad040e98ec0e00b38d05dcf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00012, Accuracy: 0.962\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3dedbac785e94a5c9c846a8120921fb0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/36 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00009, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "67dd19270a9b400096e4aa5af31b4a26",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 01:37:43,205] Trial 72 finished with value: 0.00011883199065923691 and parameters: {'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 1694, 'dropout_prob': 0.06302729759580003, 'label_smoothing': 0.008858948102414693, 'lr': 0.005827351205432264, 'warmup_proportion': 0.09649740402444897, 'weight_decay': 3.231662262492706e-06}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00012, Accuracy: 0.965\n",
      "{'epochs': 5, 'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 1956, 'dropout_prob': 0.06849107920029406, 'label_smoothing': 0.015629455811888, 'lr': 0.003529193382400453, 'warmup_proportion': 0.08127852216203038, 'weight_decay': 4.045510746461231e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 01:37:45 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 01:37:49 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8f9acd336c294d66b4e2dc7d3d7fe117",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/31 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00047, Updated Learning Rate: 0.003410034101843251\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e3829df0e80f4c99b3799bb6fbc2190d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00041, Accuracy: 0.824\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7d9f2168e3a94a92b8b47e9a3a9ac3a3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/31 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00021, Updated Learning Rate: 0.0026232573440025126\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6dbff219383d4f25afe12794351fcb0f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00026, Accuracy: 0.902\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "23bf7e5b7079409495d799931a20cc19",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/31 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00013, Updated Learning Rate: 0.0014403529000514695\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9132f13871254a40b898a5424f81b63b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00019, Accuracy: 0.943\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb8d5c8f9bb2444fab724bd03f497a92",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/31 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00011, Updated Learning Rate: 0.0004070325115759073\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d2c00ab0124448ce8af81db2ac5bb404",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00017, Accuracy: 0.952\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4467fdac6c634cc6a194bc41c7d9cc97",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/31 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00011, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6b5ba2de3fdc43218aa977d197baf6a2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00017, Accuracy: 0.953\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 01:40:27,655] Trial 73 finished with value: 0.0001704613596200943 and parameters: {'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 1956, 'dropout_prob': 0.06849107920029406, 'label_smoothing': 0.015629455811888, 'lr': 0.003529193382400453, 'warmup_proportion': 0.08127852216203038, 'weight_decay': 4.045510746461231e-06}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'epochs': 5, 'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 1769, 'dropout_prob': 0.10661053906196688, 'label_smoothing': 0.005536829671775631, 'lr': 0.007440382344339986, 'warmup_proportion': 0.09392746585129103, 'weight_decay': 2.1655362852746594e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 01:40:29 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 01:40:34 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b63c06167a3b466f9853b10a549127ce",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00043, Updated Learning Rate: 0.007192381458397058\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "12329f4e1d7e4457951eadfa080fdc23",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00037, Accuracy: 0.816\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0a756e747fbd438fa53852656cc523fe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00014, Updated Learning Rate: 0.0055362999843742485\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e3a39c443c444398bb2feae0a3751b77",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00014, Accuracy: 0.939\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b2518ac71cd743cc9cc7d7eadea9ac84",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00009, Updated Learning Rate: 0.003040996536378743\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ed77c5ae8f774c8f83acdfd8d4eaf25a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00011, Accuracy: 0.958\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "65c376d47d234e61ac3aff1bb5319d64",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00008, Updated Learning Rate: 0.0008595482159670207\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "07f8730b112c41c4bdf314233fabc0bb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00010, Accuracy: 0.962\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4e70166e9b474b6d8198934e32be7758",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00007, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "05820e56a4fa4281b0ef7753802fb60e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 01:43:38,828] Trial 74 finished with value: 9.842046275734901e-05 and parameters: {'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 1769, 'dropout_prob': 0.10661053906196688, 'label_smoothing': 0.005536829671775631, 'lr': 0.007440382344339986, 'warmup_proportion': 0.09392746585129103, 'weight_decay': 2.1655362852746594e-06}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00010, Accuracy: 0.963\n",
      "{'epochs': 5, 'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 1751, 'dropout_prob': 0.1128891922628885, 'label_smoothing': 0.019951781839721707, 'lr': 0.007049327095982044, 'warmup_proportion': 0.09994578514027992, 'weight_decay': 1.5452988117834795e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 01:43:40 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 01:43:44 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "40499866284642a2a0148babdca1cd27",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/35 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00050, Updated Learning Rate: 0.006858351102874662\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0ec702a8703b457aad38818c5436facd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00083, Accuracy: 0.614\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6d6b55eaa7444760a15d999a3d2cd0c4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/35 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00018, Updated Learning Rate: 0.005328886121278733\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a67e52740055414e9f2d7c9dde10945f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00028, Accuracy: 0.898\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c457fdfeaee54f16bcc0179b6984beaf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/35 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00013, Updated Learning Rate: 0.0029445229954320018\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b0124c02020d43d0a5e43374023d46de",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00019, Accuracy: 0.953\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fc3ac3cf98c24387ae214b000eda2830",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/35 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00011, Updated Learning Rate: 0.0008350489487544695\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ddbd7c93ce254df1a7ca85c2c15d0571",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00018, Accuracy: 0.961\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e977e378ca8b4ce2b4fff17c0c3429a0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/35 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00011, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d35de7da27a64021bcac73592ac36d5c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 01:46:27,999] Trial 75 finished with value: 0.00017520414441823959 and parameters: {'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 1751, 'dropout_prob': 0.1128891922628885, 'label_smoothing': 0.019951781839721707, 'lr': 0.007049327095982044, 'warmup_proportion': 0.09994578514027992, 'weight_decay': 1.5452988117834795e-06}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00018, Accuracy: 0.960\n",
      "Early stopping triggered.\n",
      "{'epochs': 5, 'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 1478, 'dropout_prob': 0.045181094193727775, 'label_smoothing': 0.030958704137732985, 'lr': 0.016093282539320387, 'warmup_proportion': 0.08469556387315039, 'weight_decay': 1.2497648289007854e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 01:46:30 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 01:46:33 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "85e4324dd6fb48a4b5affde70d4b38d8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/41 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00044, Updated Learning Rate: 0.015520009560906883\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "05a4dbbfda2d4520b2137868db88c4f8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00051, Accuracy: 0.855\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "04a0d7125b4a4afeaf78d9031b5bda8b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/41 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00018, Updated Learning Rate: 0.01190852598666804\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f03c597a0f6548918fb5853d246a1b04",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00027, Accuracy: 0.948\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "35022271554544a0aa51c0054d0f548c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/41 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00015, Updated Learning Rate: 0.006527937544952325\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6319fa2f6248447eab40702c86336897",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00023, Accuracy: 0.966\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "61ed084765764c67a03d6f05d1fd16a2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/41 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00013, Updated Learning Rate: 0.0018430576363793608\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "848b24a3850b44e485fc299ac992e1c7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00023, Accuracy: 0.968\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4558f4f41b85492c93060d0b912cc655",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/41 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00013, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "35155f30d557499fa2683a1f0ae05a1e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 01:49:03,701] Trial 76 finished with value: 0.00022508572340011596 and parameters: {'accumulation_steps': 2, 'batchnorm': False, 'batch_size': 1478, 'dropout_prob': 0.045181094193727775, 'label_smoothing': 0.030958704137732985, 'lr': 0.016093282539320387, 'warmup_proportion': 0.08469556387315039, 'weight_decay': 1.2497648289007854e-06}. Best is trial 32 with value: 7.672419399023056e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00023, Accuracy: 0.971\n",
      "{'epochs': 5, 'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1818, 'dropout_prob': 0.09099453479465104, 'label_smoothing': 0.00041236524150670356, 'lr': 0.01128750591525955, 'warmup_proportion': 0.08838801161035545, 'weight_decay': 9.558261298622317e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 01:49:05 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 01:49:09 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8fee3f26836143d3a695edf23a29bc60",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00064, Updated Learning Rate: 0.010911273708698299\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "df0223f382714154bb132cdb8ededb1b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00025, Accuracy: 0.871\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "82732c6edb0c4657a512f1bc33505edf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00015, Updated Learning Rate: 0.008398898864359246\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "feae5d66985448aca2e217da78fc8ffb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00013, Accuracy: 0.928\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f15fcd2cb66f4ccda7cfa6be052d4258",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00016, Updated Learning Rate: 0.00461337399129369\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5591b8b7aeb140b0ae9628ca167087aa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00006, Accuracy: 0.968\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "76b76a563640453887a83271a5a75f18",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00011, Updated Learning Rate: 0.0013039861559747826\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5c057de27fa64b55a0c432c35ac72468",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00005, Accuracy: 0.971\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a1a3b230e1a94487b2508f0aa8733bd8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/34 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00010, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9b2d57d99ac245749d4cdbb15f8e8bd8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/6 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 01:51:54,048] Trial 77 finished with value: 4.9139298126101495e-05 and parameters: {'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1818, 'dropout_prob': 0.09099453479465104, 'label_smoothing': 0.00041236524150670356, 'lr': 0.01128750591525955, 'warmup_proportion': 0.08838801161035545, 'weight_decay': 9.558261298622317e-06}. Best is trial 77 with value: 4.9139298126101495e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00005, Accuracy: 0.974\n",
      "{'epochs': 5, 'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1393, 'dropout_prob': 0.09019808641768803, 'label_smoothing': 0.00027204710879494927, 'lr': 0.011100221041937443, 'warmup_proportion': 0.0640075239753762, 'weight_decay': 2.048117691844822e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 01:51:56 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 01:51:59 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7bd3abbbc40141f491ea761d186f25b6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/44 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00064, Updated Learning Rate: 0.01056108328800287\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "99259a43ce024d95b9520e2b0053d551",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00014, Accuracy: 0.947\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "41a3958724764c49a57b285050d2620c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/44 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00016, Updated Learning Rate: 0.007974621251909293\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "75a4bb16593541c4b988a23cb4ec6766",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00007, Accuracy: 0.971\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9fe1ea79bcb54a3081fb1531022a86ce",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/44 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00012, Updated Learning Rate: 0.004326943704797737\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "75c7ead05eae41c8a3e4251b02628666",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00006, Accuracy: 0.973\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ae85bfe7eaec4d18b03491014b0359e8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/44 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00010, Updated Learning Rate: 0.0012146498874086558\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8b46f80e29014a78ac4a95c56588892c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00005, Accuracy: 0.980\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f6edf82ee64945a18fb339e06a104611",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/44 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00009, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b33e79ddbf4b4a24a9ec751919ec81e2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 01:54:54,440] Trial 78 finished with value: 5.441967248916626e-05 and parameters: {'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1393, 'dropout_prob': 0.09019808641768803, 'label_smoothing': 0.00027204710879494927, 'lr': 0.011100221041937443, 'warmup_proportion': 0.0640075239753762, 'weight_decay': 2.048117691844822e-06}. Best is trial 77 with value: 4.9139298126101495e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] Loss: 0.00005, Accuracy: 0.978\n",
      "Early stopping triggered.\n",
      "{'epochs': 5, 'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1383, 'dropout_prob': 0.16556461055700467, 'label_smoothing': 0.01424757139731482, 'lr': 0.011290478236395417, 'warmup_proportion': 0.08885295093197397, 'weight_decay': 2.191987990444896e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 01:54:56 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 01:54:59 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a21a0acc15fb429595e8ebbaa3a63c11",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/44 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00073, Updated Learning Rate: 0.010894049707514465\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "636040c068f941e8abff7784a5a5fe54",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00024, Accuracy: 0.950\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d7b5ca902bb7416b860b26626bcef902",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/44 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00025, Updated Learning Rate: 0.008364853804572182\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8d8ce08ec00f465b858ee0e4e9910cc7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00018, Accuracy: 0.968\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5289b932b7d340c782029c437b3c921b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/44 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00020, Updated Learning Rate: 0.0045874267910790655\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e0a8d9e896254287ab142b6220ff386a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00017, Accuracy: 0.967\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a9da983bf2284b7e937bc02f7d347223",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/44 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00018, Updated Learning Rate: 0.0012955076189904557\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7de59e434f774416b193068c225e89de",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00016, Accuracy: 0.973\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0915daaa8f2c453bb4452ba04f30764d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/44 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00017, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a1f6858a8433456391d4a1836f7d18d8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 01:57:04,504] Trial 79 finished with value: 0.00015299172848463058 and parameters: {'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1383, 'dropout_prob': 0.16556461055700467, 'label_smoothing': 0.01424757139731482, 'lr': 0.011290478236395417, 'warmup_proportion': 0.08885295093197397, 'weight_decay': 2.191987990444896e-06}. Best is trial 77 with value: 4.9139298126101495e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00015, Accuracy: 0.979\n",
      "{'epochs': 5, 'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1590, 'dropout_prob': 0.1484471288141354, 'label_smoothing': 0.006203593404504267, 'lr': 0.014213681860476713, 'warmup_proportion': 0.0401314436840497, 'weight_decay': 2.1401542058879345e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 01:57:06 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 01:57:13 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e37bc6119dda4208a503bd74124bbc6d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00054, Updated Learning Rate: 0.013281889510164898\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "333d502c4ff84e0a9443e759b76c4923",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00018, Accuracy: 0.940\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a0f027bc26444dccb817ad63f02ee99e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00018, Updated Learning Rate: 0.009854819965336385\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a55673aaef8249cb9e60e395b91e54cc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00013, Accuracy: 0.962\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a823eeb28ade49cbbcf82401ee6f98e3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00014, Updated Learning Rate: 0.005287211845276396\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3018a563881d40d7ac72c48dcddfbc77",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00011, Accuracy: 0.970\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "602aa2780f4c40e1a8b1efe6fb9a321c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00012, Updated Learning Rate: 0.0014748340735943153\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4698cd91950f4fe6923545d5fd5627b5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00009, Accuracy: 0.981\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5d4bca88bd894d42924779fa61275e49",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00011, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5431116e29a9497984a91bf4bd155650",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 01:59:36,993] Trial 80 finished with value: 8.808303549885749e-05 and parameters: {'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1590, 'dropout_prob': 0.1484471288141354, 'label_smoothing': 0.006203593404504267, 'lr': 0.014213681860476713, 'warmup_proportion': 0.0401314436840497, 'weight_decay': 2.1401542058879345e-06}. Best is trial 77 with value: 4.9139298126101495e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00009, Accuracy: 0.981\n",
      "Early stopping triggered.\n",
      "{'epochs': 5, 'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1278, 'dropout_prob': 0.1330949043780857, 'label_smoothing': 0.0001751482485685057, 'lr': 0.015027348719658981, 'warmup_proportion': 0.04111802111306783, 'weight_decay': 1.4156428486034392e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 01:59:39 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 01:59:42 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ecabcd7a23014f24bcf022927e1d8ff3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/47 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00062, Updated Learning Rate: 0.014046776117654605\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d0322c480bc94d2ba75967e17bfb9093",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00017, Accuracy: 0.935\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "db0b6362c29149548d8fe0253a4360df",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/47 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00017, Updated Learning Rate: 0.010425340285788945\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0613a61533d14d3395c101fb6aa7cc40",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00016, Accuracy: 0.939\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d00ef41288cb4dd5b7bab85af3d451fb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/47 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00012, Updated Learning Rate: 0.00559433811810013\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dd64baa12a6d4b2187602a8391e2f9e4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00006, Accuracy: 0.974\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5e389adf9bbb4b2da810ba44d3c6f32f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/47 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00010, Updated Learning Rate: 0.0015606679809931434\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "411a282b68e74d6bbd4b99b6ed1c5823",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00005, Accuracy: 0.978\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c874bd92cd94420f87328ab7fd946727",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/47 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00009, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "58d9cf2994fa40f08d4a8cca13f707d0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 02:01:48,723] Trial 81 finished with value: 5.434997007250786e-05 and parameters: {'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1278, 'dropout_prob': 0.1330949043780857, 'label_smoothing': 0.0001751482485685057, 'lr': 0.015027348719658981, 'warmup_proportion': 0.04111802111306783, 'weight_decay': 1.4156428486034392e-06}. Best is trial 77 with value: 4.9139298126101495e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] Loss: 0.00005, Accuracy: 0.978\n",
      "Early stopping triggered.\n",
      "{'epochs': 5, 'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1240, 'dropout_prob': 0.15101485052790634, 'label_smoothing': 0.006079533570475484, 'lr': 0.019555245558801237, 'warmup_proportion': 0.041549708806068016, 'weight_decay': 1.5861836771535662e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 02:01:50 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 02:01:53 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f6a001788b2a4ffd980668d25060769a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/49 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00060, Updated Learning Rate: 0.01831016267197363\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7a70c31e12d54d92ac66929c0053d42b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/9 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00022, Accuracy: 0.935\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "98f9178ce38442b8bce6ef5dc82ce013",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/49 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00021, Updated Learning Rate: 0.013610124432732927\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "747963dda94d4527b013039abe06c2e0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/9 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00016, Accuracy: 0.964\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9dd925c365c2438786d8c792c4dc1a37",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/49 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00017, Updated Learning Rate: 0.0073104431348988775\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "559a444c77554dfba4b63c38dc393616",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/9 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00013, Accuracy: 0.972\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "334d4a84bcf24f4ebf59a73d6121abe3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/49 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00014, Updated Learning Rate: 0.0020405348546964036\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ff7c6cbb7a44473eac2e88fd73bcd5b2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/9 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00011, Accuracy: 0.979\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6dfc35e473fb4baf85c4eb4683a0b05d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/49 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00013, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ec35f8e782604a648d1797f9cd9d1394",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/9 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 02:04:06,773] Trial 82 finished with value: 0.0001076039381325245 and parameters: {'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1240, 'dropout_prob': 0.15101485052790634, 'label_smoothing': 0.006079533570475484, 'lr': 0.019555245558801237, 'warmup_proportion': 0.041549708806068016, 'weight_decay': 1.5861836771535662e-06}. Best is trial 77 with value: 4.9139298126101495e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00011, Accuracy: 0.979\n",
      "{'epochs': 5, 'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1299, 'dropout_prob': 0.10640080167532477, 'label_smoothing': 0.021862240952618444, 'lr': 0.015570401585579645, 'warmup_proportion': 0.03610467147737083, 'weight_decay': 1.0176268466175279e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 02:04:08 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 02:04:11 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bab5453828e84e738f0f0f56a5619f11",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/47 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00062, Updated Learning Rate: 0.01450927331762368\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "982836181716446e880b77569237896d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00030, Accuracy: 0.937\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2c33366b569c4dceb873fd3aefa27afc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/47 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00028, Updated Learning Rate: 0.010739454438713714\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0bbddfc161d141debd1417fba934c069",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00022, Accuracy: 0.971\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "113649e46f144ff3bca2ad540d60b9ff",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/47 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00024, Updated Learning Rate: 0.005752825749528555\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ae6fc02466704e98ac6127c694a86021",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00020, Accuracy: 0.975\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1d9e6041fbc94f049e9cc3e3ab0be950",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/47 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00022, Updated Learning Rate: 0.0016032998237472435\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cf56972b679b47758443c4151386522f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00020, Accuracy: 0.975\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4852c83f94d046d8940e735168578b43",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/47 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00021, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "23bcab90fe6143c5a0d9a953e01d41b4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 02:06:12,862] Trial 83 finished with value: 0.00018890149295330047 and parameters: {'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1299, 'dropout_prob': 0.10640080167532477, 'label_smoothing': 0.021862240952618444, 'lr': 0.015570401585579645, 'warmup_proportion': 0.03610467147737083, 'weight_decay': 1.0176268466175279e-06}. Best is trial 77 with value: 4.9139298126101495e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00019, Accuracy: 0.978\n",
      "{'epochs': 5, 'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1609, 'dropout_prob': 0.12931721102481436, 'label_smoothing': 8.83890730090731e-07, 'lr': 0.028147621640854794, 'warmup_proportion': 0.04584907346713653, 'weight_decay': 2.1415275778699216e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 02:06:14 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 02:06:17 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "97b6064338dc44ef9bc3651bc9769f22",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00045, Updated Learning Rate: 0.026402075025726137\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9fc8d0e8e6c14320b48587c9e29aa690",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00012, Accuracy: 0.945\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5414f50f73c447f28b6b1a00c65f4bc1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00013, Updated Learning Rate: 0.019656468746473504\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b066c0c6c78a4294b049550631001b3b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00010, Accuracy: 0.960\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6b3b75fa9fbb40318c07c42b9b84374c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00010, Updated Learning Rate: 0.010569064550042183\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "32b1208b48df4b6b8b7b2a7c286fd492",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00006, Accuracy: 0.973\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8d39105b74b6482da32ad6e3636c1ce5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00008, Updated Learning Rate: 0.0029518216325127794\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f126bb31bf9a4a8fa5a77d8f8e890154",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00005, Accuracy: 0.977\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9c83a02c4958408993cae8c4d45f7c27",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00007, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5fa6b86c765b40cf957c8783af23e11d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 02:08:22,236] Trial 84 finished with value: 4.6606420353055e-05 and parameters: {'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1609, 'dropout_prob': 0.12931721102481436, 'label_smoothing': 8.83890730090731e-07, 'lr': 0.028147621640854794, 'warmup_proportion': 0.04584907346713653, 'weight_decay': 2.1415275778699216e-06}. Best is trial 84 with value: 4.6606420353055e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] Loss: 0.00005, Accuracy: 0.978\n",
      "Early stopping triggered.\n",
      "{'epochs': 5, 'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1412, 'dropout_prob': 0.1344738440515692, 'label_smoothing': 0.012565473661815428, 'lr': 0.028152168453605124, 'warmup_proportion': 0.04487391523795066, 'weight_decay': 1.3023121195979873e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 02:08:24 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 02:08:27 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "643fd94133b64a4a8d78fe050315fce2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/43 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00057, Updated Learning Rate: 0.026390208778437323\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7c75ebd3abc1431e9d6295f135139e4b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00028, Accuracy: 0.926\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6edeac04cc564c35b68f26e5e2777642",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/43 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00021, Updated Learning Rate: 0.019636660795594955\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2a8e6e5365a949ac83b8f067385bc4a6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00016, Accuracy: 0.972\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "17af7cec4dd9411a8cf32a3267f311df",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/43 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00018, Updated Learning Rate: 0.01055461487496293\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5ede6bf93f1147b899654129f8b62557",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00015, Accuracy: 0.973\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f7e60c68c02546dba9d383588b815b79",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/43 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00016, Updated Learning Rate: 0.0029471883615642147\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8e8c074a0e214d03b071b8ca7f3a855f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00013, Accuracy: 0.980\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5705c72ad1af419cbc66be09383d09da",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/43 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00015, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a53eaf59088740499bf9f6a75fb136ce",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 02:10:29,759] Trial 85 finished with value: 0.00013243851214647293 and parameters: {'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1412, 'dropout_prob': 0.1344738440515692, 'label_smoothing': 0.012565473661815428, 'lr': 0.028152168453605124, 'warmup_proportion': 0.04487391523795066, 'weight_decay': 1.3023121195979873e-06}. Best is trial 84 with value: 4.6606420353055e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] Loss: 0.00013, Accuracy: 0.979\n",
      "Early stopping triggered.\n",
      "{'epochs': 5, 'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1127, 'dropout_prob': 0.17956987689599113, 'label_smoothing': 0.007105180504132021, 'lr': 0.06745294746314857, 'warmup_proportion': 0.049586309260943755, 'weight_decay': 1.970315313608002e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 02:10:31 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 02:10:34 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f1a3a658855c4eb6b975235d4480bdbb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/54 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00056, Updated Learning Rate: 0.06347056808693587\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0b4ea9f06b294baa95e9c9e6bde40768",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/9 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00022, Accuracy: 0.953\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b233497732f247bc8afc98e012937bae",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/54 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00023, Updated Learning Rate: 0.04739383426331222\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9404be3c4bfc49dd83f8a9e78757f8d1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/9 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00018, Accuracy: 0.961\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ea46aaa7434c4e03bf0a471b2a3c8437",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/54 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00018, Updated Learning Rate: 0.025531609076773664\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "25cecc33f1dd4be682dd86c13485afce",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/9 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00013, Accuracy: 0.977\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cc774f8d8be1414c91caddc035ad879f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/54 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00016, Updated Learning Rate: 0.007138328648845122\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c1a7369090f94aa4b0b79e9869efb4fb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/9 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00011, Accuracy: 0.981\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "69316cbc6f2445fcbaed9dc7f65a9015",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/54 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00014, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dd2103c35e2e46e18b659be97179ce1c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/9 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 02:12:36,772] Trial 86 finished with value: 0.00010937424078583718 and parameters: {'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1127, 'dropout_prob': 0.17956987689599113, 'label_smoothing': 0.007105180504132021, 'lr': 0.06745294746314857, 'warmup_proportion': 0.049586309260943755, 'weight_decay': 1.970315313608002e-06}. Best is trial 84 with value: 4.6606420353055e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00011, Accuracy: 0.982\n",
      "{'epochs': 5, 'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1014, 'dropout_prob': 0.1226996516311242, 'label_smoothing': 0.0005011892549716046, 'lr': 0.036302127256330224, 'warmup_proportion': 0.06364242819916421, 'weight_decay': 2.0680402907745966e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 02:12:38 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 02:12:41 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a21911433253453c9b20fe1fc19fb32e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/60 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00063, Updated Learning Rate: 0.03450460687645781\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ced8cc1eb95047b29505bd4866d62f7d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00024, Accuracy: 0.931\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a6aee35c4e114a31a457951772aeb893",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/60 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00021, Updated Learning Rate: 0.0260265149841141\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "37e95e5347b347dfb1a90716f7f90380",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00015, Accuracy: 0.956\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0ce0190ff1e04be0aa1a7fa348f8d0da",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/60 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00015, Updated Learning Rate: 0.014112071997325323\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7849884c3de24861a73a0c7b6fe8e173",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00010, Accuracy: 0.970\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "312cc16f8b844894aac3768e3623add4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/60 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00012, Updated Learning Rate: 0.003959990643385249\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e7618a9548d94f3f8dc1d60219c99bfb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00007, Accuracy: 0.978\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a358d36ad8ea407db05f9186e43461f0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/60 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00010, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eff942e52c8b44d6af5f7cc4f5d79cd9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 02:15:02,289] Trial 87 finished with value: 6.671954840421677e-05 and parameters: {'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1014, 'dropout_prob': 0.1226996516311242, 'label_smoothing': 0.0005011892549716046, 'lr': 0.036302127256330224, 'warmup_proportion': 0.06364242819916421, 'weight_decay': 2.0680402907745966e-06}. Best is trial 84 with value: 4.6606420353055e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00007, Accuracy: 0.979\n",
      "{'epochs': 5, 'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1027, 'dropout_prob': 0.11985673508254018, 'label_smoothing': 0.23850194634012017, 'lr': 0.03715924455621446, 'warmup_proportion': 0.06002000726607524, 'weight_decay': 2.88094601225188e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 02:15:04 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 02:15:07 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d23e67071f504207b3c74607a2a0d3f0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/59 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00142, Updated Learning Rate: 0.035186480363961256\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "097dfd522e48401382b800a303276ebd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00122, Accuracy: 0.951\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ce957cf2fc1b44529af854d5eb69d946",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/59 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00119, Updated Learning Rate: 0.026437014495890917\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "52ec72fc752a4359ad4621832596d462",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00116, Accuracy: 0.966\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3d65c1b2dd794e32b279691918679ab2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/59 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00116, Updated Learning Rate: 0.014298663650893\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f6d96425173c4534b6cf2132766cc28f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00114, Accuracy: 0.976\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b60ff408bf844d35b3e31f75666b8ff6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/59 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00114, Updated Learning Rate: 0.00400668574700645\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8d508a58a4d04018a1bb9ef49ea32239",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00113, Accuracy: 0.979\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6bdc4672630e4d98a4dfb9c4fd5161bc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/59 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00113, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "02d83b6081f84486aeeb2b5e1a89185f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 02:17:05,768] Trial 88 finished with value: 0.0011234356999397278 and parameters: {'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1027, 'dropout_prob': 0.11985673508254018, 'label_smoothing': 0.23850194634012017, 'lr': 0.03715924455621446, 'warmup_proportion': 0.06002000726607524, 'weight_decay': 2.88094601225188e-06}. Best is trial 84 with value: 4.6606420353055e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00112, Accuracy: 0.979\n",
      "{'epochs': 5, 'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1542, 'dropout_prob': 0.14588098218295464, 'label_smoothing': 0.000730932224792607, 'lr': 0.04105490292483175, 'warmup_proportion': 0.051794173761270816, 'weight_decay': 1.3914106054110367e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 02:17:07 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 02:17:10 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d68f1a3975074a1aa783c45e9a37207b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/39 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00045, Updated Learning Rate: 0.03875347218736715\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "77e4dfa928c64a808d82a52d7baa8fe5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00033, Accuracy: 0.877\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f6d89208bb1f4105b410e1420b8d3097",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/39 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00014, Updated Learning Rate: 0.029025869932212806\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c3f2ed7a4f384e7f8720b8a5cfcbb438",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00009, Accuracy: 0.962\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b12d816f1a7049abb7fe3f18870d818f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/39 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00010, Updated Learning Rate: 0.015667281313263864\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0fbe6adc12f145a4b4cc29ac1cf644af",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00005, Accuracy: 0.979\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "50dbfba433b74d8c96e3da139931529d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/39 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00008, Updated Learning Rate: 0.004385221592016308\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "be38ad3d6f294c30b6e4972fce7af51d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 02:18:46,714] Trial 89 finished with value: 5.446032136678696e-05 and parameters: {'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1542, 'dropout_prob': 0.14588098218295464, 'label_smoothing': 0.000730932224792607, 'lr': 0.04105490292483175, 'warmup_proportion': 0.051794173761270816, 'weight_decay': 1.3914106054110367e-06}. Best is trial 84 with value: 4.6606420353055e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] Loss: 0.00005, Accuracy: 0.979\n",
      "Early stopping triggered.\n",
      "{'epochs': 5, 'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1218, 'dropout_prob': 0.14544103903577663, 'label_smoothing': 0.0004250191535122064, 'lr': 0.04895058704616252, 'warmup_proportion': 0.04656483952580117, 'weight_decay': 1.4051783210808624e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 02:18:48 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 02:18:51 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "85423457a10a4a6eb9f0baeb17805983",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/50 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00052, Updated Learning Rate: 0.045935596105459664\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0389930c33d843f0b8504dae4804f3d0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/9 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00084, Accuracy: 0.798\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5199c9214ac34aee82a13bb9d103e07e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/50 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00018, Updated Learning Rate: 0.03421339177895887\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2cb6f79e3c84496882ae0a0a6757a229",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/9 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00008, Accuracy: 0.976\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ffcfadd8ddfb4f04a1201e0dfd7f6238",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/50 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00012, Updated Learning Rate: 0.018401053313675093\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "61746f997ec1415f84227dbb9fb57384",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/9 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00007, Accuracy: 0.976\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e9dd27be87a14456a4be1bdc40d3cf94",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/50 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00010, Updated Learning Rate: 0.005139978616208111\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "babb533f34eb4c86b42e3ee9e84a5ff1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/9 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 02:21:30,761] Trial 90 finished with value: 7.076191119849682e-05 and parameters: {'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1218, 'dropout_prob': 0.14544103903577663, 'label_smoothing': 0.0004250191535122064, 'lr': 0.04895058704616252, 'warmup_proportion': 0.04656483952580117, 'weight_decay': 1.4051783210808624e-06}. Best is trial 84 with value: 4.6606420353055e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] Loss: 0.00007, Accuracy: 0.974\n",
      "Early stopping triggered.\n",
      "{'epochs': 5, 'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1174, 'dropout_prob': 0.14599915242657968, 'label_smoothing': 0.0003123981787481759, 'lr': 0.03531056475481153, 'warmup_proportion': 0.052252020555818984, 'weight_decay': 1.4004933112543e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 02:21:32 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 02:21:35 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ee825b25bc7e4d7fac81b8338e10b426",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/52 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00055, Updated Learning Rate: 0.03327210818152333\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "46704939587c41659afb06bf2105eb58",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/9 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00020, Accuracy: 0.933\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "04d1c14c6c534367ae9340c11e3a68cb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/52 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00018, Updated Learning Rate: 0.02487750467840632\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3b5dfdb76ac848278cb73105ae9e95b6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/9 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00009, Accuracy: 0.970\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b93723ae15f049918000ce7bc5f68597",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/52 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00013, Updated Learning Rate: 0.013413258897067517\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c3baf55411f14e6ea7dca75bd87aea23",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/9 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 02:22:49,533] Trial 91 finished with value: 0.00015124109983444213 and parameters: {'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1174, 'dropout_prob': 0.14599915242657968, 'label_smoothing': 0.0003123981787481759, 'lr': 0.03531056475481153, 'warmup_proportion': 0.052252020555818984, 'weight_decay': 1.4004933112543e-06}. Best is trial 84 with value: 4.6606420353055e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] Loss: 0.00015, Accuracy: 0.949\n",
      "Early stopping triggered.\n",
      "{'epochs': 5, 'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1007, 'dropout_prob': 0.1753022615660737, 'label_smoothing': 0.015025790070462843, 'lr': 0.046274419795688616, 'warmup_proportion': 0.03816183693898994, 'weight_decay': 1.7230050771506873e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 02:22:51 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 02:22:54 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b85c71b46e694431851a42188709942a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/60 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00065, Updated Learning Rate: 0.04317462144207024\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6b2bc923e79640308c5d63bc55d9aa6f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00032, Accuracy: 0.940\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d8d75480c29e4272b9898482aca9b50b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/60 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00030, Updated Learning Rate: 0.031991436796902896\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8d5bf2dde331496a8c310fab320110ab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00022, Accuracy: 0.974\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "be0250c418b449ef9077d105c4b49704",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/60 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00025, Updated Learning Rate: 0.01714885932574765\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bc188d7663ee4a4ba826a6f6e5a8cd4b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00020, Accuracy: 0.977\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "03af05f15d914ac881ba24bf87f7132e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/60 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00022, Updated Learning Rate: 0.004781227140370101\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e5d40312e70f4fe2b88804d61b887f7b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00018, Accuracy: 0.983\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3d073d450a914e9e88af64104622e676",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/60 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00021, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8a8f07711929474788b423ef77d592e0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 02:24:54,990] Trial 92 finished with value: 0.00017207796722650527 and parameters: {'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1007, 'dropout_prob': 0.1753022615660737, 'label_smoothing': 0.015025790070462843, 'lr': 0.046274419795688616, 'warmup_proportion': 0.03816183693898994, 'weight_decay': 1.7230050771506873e-06}. Best is trial 84 with value: 4.6606420353055e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00017, Accuracy: 0.983\n",
      "{'epochs': 5, 'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1245, 'dropout_prob': 0.19163072717145666, 'label_smoothing': 0.00579984198477846, 'lr': 0.05087704994205054, 'warmup_proportion': 0.04765431539396159, 'weight_decay': 1.131120556720482e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 02:24:57 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 02:24:59 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0a07cd27598d47e4b38c3074cb28ce6f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/49 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00053, Updated Learning Rate: 0.047776582105869714\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7a07002010da443da07043e0a3485fe2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/9 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00022, Accuracy: 0.943\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "24d79232201d41ce9062d0ae2a062e90",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/49 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00022, Updated Learning Rate: 0.03560744552818607\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c7d7bb24844f4b799b6fda92cd1362d0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/9 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00020, Accuracy: 0.943\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c4cac2531e8a42159f4d5821de20b7c8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/49 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00016, Updated Learning Rate: 0.019158742790386096\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4825c5f31bca4c3fa9ae878c670a5d35",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/9 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00010, Accuracy: 0.981\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "25b9c3f717024b899b2e985d09682630",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/49 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00014, Updated Learning Rate: 0.005352871527382155\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b2b54c0d7a46419d972080f157070dd7",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/9 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00010, Accuracy: 0.982\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3dff687d770544d380e8ccb92c3bb6c0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/49 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00012, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c3c9a5a8f512461daa1ef68828adfabc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/9 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 02:27:35,975] Trial 93 finished with value: 0.00010051075294613838 and parameters: {'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1245, 'dropout_prob': 0.19163072717145666, 'label_smoothing': 0.00579984198477846, 'lr': 0.05087704994205054, 'warmup_proportion': 0.04765431539396159, 'weight_decay': 1.131120556720482e-06}. Best is trial 84 with value: 4.6606420353055e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00010, Accuracy: 0.982\n",
      "{'epochs': 5, 'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1526, 'dropout_prob': 0.15507945728762118, 'label_smoothing': 0.026144513538877725, 'lr': 0.0472792832862768, 'warmup_proportion': 0.04355204138242804, 'weight_decay': 1.466634472382698e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 02:27:38 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 02:27:40 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9aa40867bd50458f86fc5df88585f81c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/40 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00053, Updated Learning Rate: 0.044272255269991664\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "65c6011199704206ba7053843b5bf4d0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00033, Accuracy: 0.912\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "34103234b1964305820d5cf5ada8f10b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/40 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00025, Updated Learning Rate: 0.03291017770274255\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ed76e30299b6451a994d25cb0b868cbe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00021, Accuracy: 0.963\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8d2227c84156411f83955a853ff25f62",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/40 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00021, Updated Learning Rate: 0.017677885831472763\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "80ad5864a1e547779049692ef3d3f4b5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00018, Accuracy: 0.978\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "96b668a843c44ce5ba0a5343b264e387",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/40 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00020, Updated Learning Rate: 0.004934476207253098\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "02289ee188874ec3a38754ec01fe5401",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00017, Accuracy: 0.982\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2fd622cbea4c4b4fbc35dea89003d211",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/40 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00019, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7a1f432236fd485ea5c045a4e67a97e2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 02:29:41,127] Trial 94 finished with value: 0.00017635799646377565 and parameters: {'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1526, 'dropout_prob': 0.15507945728762118, 'label_smoothing': 0.026144513538877725, 'lr': 0.0472792832862768, 'warmup_proportion': 0.04355204138242804, 'weight_decay': 1.466634472382698e-06}. Best is trial 84 with value: 4.6606420353055e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] Loss: 0.00018, Accuracy: 0.979\n",
      "Early stopping triggered.\n",
      "{'epochs': 5, 'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1077, 'dropout_prob': 0.12335619173633919, 'label_smoothing': 0.014040408215465799, 'lr': 0.02553775723719236, 'warmup_proportion': 0.06354969074060182, 'weight_decay': 2.2122013936583115e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 02:29:43 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 02:29:45 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "887df06487924ea38c2da261a28d00bc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/56 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00070, Updated Learning Rate: 0.024235010059777563\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ebbe44c64bb54f9e8f6c336e3532606a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00031, Accuracy: 0.935\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f7d2d7eea68e44809fb207f04178bdf3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/56 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00027, Updated Learning Rate: 0.018249894535788666\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d179f6887f2449a38e61b81bf258fe29",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00022, Accuracy: 0.970\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5e3c820869e542098c5e17d7b88a580c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/56 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00023, Updated Learning Rate: 0.00988490388057156\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "56a5eb86b2e64b43ba8296f8b7569278",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00019, Accuracy: 0.980\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d7e83245eb71429dbc1b4ef9ace469e5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/56 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00020, Updated Learning Rate: 0.002772144538273329\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f4ba367cf46d41aeb8d88a1361b3f8cf",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00017, Accuracy: 0.982\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f920839e21644958bf98cf9cfe65ff19",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/56 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00019, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dd654070980a4877bd7cdfdd0320871f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/10 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 02:31:47,560] Trial 95 finished with value: 0.00017212972193956375 and parameters: {'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1077, 'dropout_prob': 0.12335619173633919, 'label_smoothing': 0.014040408215465799, 'lr': 0.02553775723719236, 'warmup_proportion': 0.06354969074060182, 'weight_decay': 2.2122013936583115e-06}. Best is trial 84 with value: 4.6606420353055e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00017, Accuracy: 0.983\n",
      "{'epochs': 5, 'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1357, 'dropout_prob': 0.1405084330328094, 'label_smoothing': 0.020636429042130255, 'lr': 0.01949452294128528, 'warmup_proportion': 0.05750315146067203, 'weight_decay': 1.967939745099535e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 02:31:49 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 02:31:52 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5269fcdd671142aa8719ba94f466fd0d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/45 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00061, Updated Learning Rate: 0.018418981755779756\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c2e6fb3d8a8145118cf142c03d648131",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00028, Accuracy: 0.945\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6c1da141fb3b449c98e33de61387587e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/45 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00026, Updated Learning Rate: 0.013808361089906403\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "40d2ee13e9294c04ba89e2d69f7b0f85",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00021, Accuracy: 0.969\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cbab07b6a0b84f6fb4f05c0b6b049a60",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/45 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00022, Updated Learning Rate: 0.007457765766610055\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ebe173198fd24017b2dab8eb439a0d58",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00021, Accuracy: 0.971\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "529da706e20a4be98030d071f2f7fce6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/45 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00020, Updated Learning Rate: 0.0020881029189415055\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cb5a92f8def54c929f9b3b057b0ea899",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00019, Accuracy: 0.975\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "61f519ebe2a847d7a78c4f9e0f8b3a53",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/45 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00020, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "67cf1225e7c64d87b62bea2ea3c5ae8e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 02:35:13,603] Trial 96 finished with value: 0.00018337733000516892 and parameters: {'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1357, 'dropout_prob': 0.1405084330328094, 'label_smoothing': 0.020636429042130255, 'lr': 0.01949452294128528, 'warmup_proportion': 0.05750315146067203, 'weight_decay': 1.967939745099535e-06}. Best is trial 84 with value: 4.6606420353055e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00018, Accuracy: 0.979\n",
      "{'epochs': 5, 'accumulation_steps': 1, 'batchnorm': True, 'batch_size': 1585, 'dropout_prob': 0.09148700728392728, 'label_smoothing': 0.010905647870567836, 'lr': 0.04146692352903154, 'warmup_proportion': 0.041519777841391346, 'weight_decay': 2.545948198339146e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 02:35:15 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 02:35:18 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7166460b285f492698004fd2b3eda563",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00146, Updated Learning Rate: 0.038748517241723464\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "27042da60d54473f9d9359c11513affb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.07844, Accuracy: 0.117\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "692950c2ad904f71918d1d499cac2f40",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/38 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00146, Updated Learning Rate: 0.028750401894901523\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "dfabba52464c4f918997d433ee44261e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/7 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 02:36:14,017] Trial 97 finished with value: 4.390048291015625 and parameters: {'accumulation_steps': 1, 'batchnorm': True, 'batch_size': 1585, 'dropout_prob': 0.09148700728392728, 'label_smoothing': 0.010905647870567836, 'lr': 0.04146692352903154, 'warmup_proportion': 0.041519777841391346, 'weight_decay': 2.545948198339146e-06}. Best is trial 84 with value: 4.6606420353055e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] Loss: 4.39005, Accuracy: 0.117\n",
      "Early stopping triggered.\n",
      "{'epochs': 5, 'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 915, 'dropout_prob': 0.13049371377289745, 'label_smoothing': 0.00592473410669996, 'lr': 0.06597907541445124, 'warmup_proportion': 0.06772061854474638, 'weight_decay': 1.0118648237154745e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 02:36:16 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 02:36:19 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c54dea8c18b1406faae74f200f44064f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/66 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00072, Updated Learning Rate: 0.06283648848352925\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f7fd858d03cc4eb89d892647d9da57ff",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/11 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00030, Accuracy: 0.935\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b43f6bac00494f10be56bdd0b7d4a363",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/66 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00030, Updated Learning Rate: 0.047498379911508154\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1a293a351c2a46d6baadfc3246f20a4a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/11 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00024, Accuracy: 0.952\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b0ee43e367ee42c6a87b8e613240e743",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/66 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00023, Updated Learning Rate: 0.025789798473896224\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d3ae1b8ecf3f47d588145351171494bc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/11 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00018, Accuracy: 0.968\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c5f620781ab649a98b31475ae9afa00e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/66 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00019, Updated Learning Rate: 0.007242444016319718\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6f6697b9c23d45aeb3094888f53505d1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/11 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00014, Accuracy: 0.977\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "51e93605528c4acfa7a7d373dff6ae8f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/66 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00017, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0c94d9a0cc5a4dc1a3ce7e8e148b0c34",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/11 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 02:38:21,643] Trial 98 finished with value: 0.00012755308225750924 and parameters: {'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 915, 'dropout_prob': 0.13049371377289745, 'label_smoothing': 0.00592473410669996, 'lr': 0.06597907541445124, 'warmup_proportion': 0.06772061854474638, 'weight_decay': 1.0118648237154745e-06}. Best is trial 84 with value: 4.6606420353055e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00013, Accuracy: 0.982\n",
      "{'epochs': 5, 'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1287, 'dropout_prob': 0.16847003099338972, 'label_smoothing': 0.03425209877627631, 'lr': 0.05775113013921722, 'warmup_proportion': 0.0540230239045516, 'weight_decay': 1.5959801171624955e-06}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024/03/11 02:38:23 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n",
      "2024/03/11 02:38:26 WARNING mlflow.utils.requirements_utils: Found torch version (2.2.0+cu118) contains a local version label (+cu118). MLflow logged a pip requirement for this package as 'torch==2.2.0' without the local version label to make it installable from PyPI. To specify pip requirements containing local version labels, please use `conda_env` or `pip_requirements`.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO] Register count_convNd() for <class 'torch.nn.modules.conv.Conv2d'>.\n",
      "[INFO] Register count_normalization() for <class 'torch.nn.modules.batchnorm.BatchNorm2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.activation.ReLU'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.pooling.MaxPool2d'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.container.Sequential'>.\n",
      "[INFO] Register count_linear() for <class 'torch.nn.modules.linear.Linear'>.\n",
      "[INFO] Register zero_ops() for <class 'torch.nn.modules.dropout.Dropout'>.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2666b13ec3274b86ac1c5f51973770c4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Training:   0%|          | 0/47 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 0] Average Loss: 0.00062, Updated Learning Rate: 0.05447276270308113\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5fec814674a34ad0a2e9be4d7203432b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 0/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 0] New best model found! Saving checkpoint. Loss: 0.00035, Accuracy: 0.952\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eafa2b4d3609423a9b326790c5183cad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Training:   0%|          | 0/47 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 1] Average Loss: 0.00033, Updated Learning Rate: 0.04076944636553103\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "038906b9874b4002a9d294b167e63db0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 1/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 1] New best model found! Saving checkpoint. Loss: 0.00028, Accuracy: 0.970\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "79d1bdd7155d41e984f8f8324cdf2b3f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Training:   0%|          | 0/47 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 2] Average Loss: 0.00029, Updated Learning Rate: 0.021995713609575716\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bc0e452c350f4eefb6e532d8c96f2318",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 2/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 2] New best model found! Saving checkpoint. Loss: 0.00028, Accuracy: 0.965\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0f5536044a3f44468b0a14f84712e85f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Training:   0%|          | 0/47 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 3] Average Loss: 0.00027, Updated Learning Rate: 0.006154893242034694\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6f22bae4743246109c162ce3d310c463",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 3/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 3] New best model found! Saving checkpoint. Loss: 0.00024, Accuracy: 0.977\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "abd6a08b33a84f10988644ec2b7da83e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Training:   0%|          | 0/47 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Train][Epoch 4] Average Loss: 0.00026, Updated Learning Rate: 0.0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "813323c8303b4160963168d249a8c0b9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch 4/Testing:   0%|          | 0/8 [00:00<?, ?batch/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-03-11 02:40:27,976] Trial 99 finished with value: 0.00023607323169708253 and parameters: {'accumulation_steps': 1, 'batchnorm': False, 'batch_size': 1287, 'dropout_prob': 0.16847003099338972, 'label_smoothing': 0.03425209877627631, 'lr': 0.05775113013921722, 'warmup_proportion': 0.0540230239045516, 'weight_decay': 1.5959801171624955e-06}. Best is trial 84 with value: 4.6606420353055e-05.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Test][Epoch 4] New best model found! Saving checkpoint. Loss: 0.00024, Accuracy: 0.980\n"
     ]
    }
   ],
   "source": [
    "study = optuna.create_study(direction='minimize', sampler=optuna.samplers.TPESampler())  # Change 'minimize' to 'maximize' if you're maximizing a metric\n",
    "study.optimize(objective, n_trials=100)  # Adjust the number of trials as needed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "555bbc83-0f0d-45c4-a0e3-e7ea6f441d8c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best trial:\n",
      "Value: 4.6606420353055e-05\n",
      "Params: \n",
      "    accumulation_steps: 1\n",
      "    batchnorm: False\n",
      "    batch_size: 1609\n",
      "    dropout_prob: 0.12931721102481436\n",
      "    label_smoothing: 8.83890730090731e-07\n",
      "    lr: 0.028147621640854794\n",
      "    warmup_proportion: 0.04584907346713653\n",
      "    weight_decay: 2.1415275778699216e-06\n"
     ]
    }
   ],
   "source": [
    "# Print the best hyperparameters and the corresponding value\n",
    "print('Best trial:')\n",
    "trial = study.best_trial\n",
    "\n",
    "print('Value: {}'.format(trial.value))\n",
    "print('Params: ')\n",
    "for key, value in trial.params.items():\n",
    "    print('    {}: {}'.format(key, value))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a37418a7-255d-48b4-a4b5-6334f99fc0dd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}