try1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "CUDA_VISIBLE_DEVICES=1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/data1/debajyoti/colie/colenv/lib/python3.8/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import os\n",
    "import logging\n",
    "import numpy as np\n",
    "import random\n",
    "from tqdm import tqdm\n",
    "import time\n",
    "import pandas as pd\n",
    "\n",
    "from transformers import LongformerModel, AutoTokenizer, LongformerForSequenceClassification, LongformerForMultipleChoice\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import f1_score, accuracy_score, classification_report\n",
    "\n",
    "logging.basicConfig(filename=f'./logs/train_{time.asctime().replace(\" \",\"_\")}.log', filemode='w', level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')\n",
    "\n",
    "# Create a logger object\n",
    "logger = logging.getLogger()\n",
    "logger.setLevel(logging.INFO)\n",
    "\n",
    "# Create a stream handler to print log messages to the console\n",
    "console_handler = logging.StreamHandler()\n",
    "console_handler.setLevel(logging.INFO)\n",
    "formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')\n",
    "console_handler.setFormatter(formatter)\n",
    "logger.addHandler(console_handler)\n",
    "\n",
    "torch.manual_seed(40)\n",
    "np.random.seed(40)\n",
    "random.seed(40)\n",
    "torch.cuda.manual_seed(40)\n",
    "torch.backends.cudnn.deterministic = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BOOK_id</th>\n",
       "      <th>Epoch</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>31873_1.txt</td>\n",
       "      <td>Viktorian</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>31873_2.txt</td>\n",
       "      <td>Viktorian</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>31873_3.txt</td>\n",
       "      <td>Viktorian</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>31873_4.txt</td>\n",
       "      <td>Viktorian</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>31873_5.txt</td>\n",
       "      <td>Viktorian</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36252</th>\n",
       "      <td>36919_48.txt</td>\n",
       "      <td>Modernism</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36253</th>\n",
       "      <td>36919_49.txt</td>\n",
       "      <td>Modernism</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36254</th>\n",
       "      <td>36919_50.txt</td>\n",
       "      <td>Modernism</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36255</th>\n",
       "      <td>36919_51.txt</td>\n",
       "      <td>Modernism</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36256</th>\n",
       "      <td>36919_52.txt</td>\n",
       "      <td>Modernism</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>36257 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            BOOK_id      Epoch\n",
       "0       31873_1.txt  Viktorian\n",
       "1       31873_2.txt  Viktorian\n",
       "2       31873_3.txt  Viktorian\n",
       "3       31873_4.txt  Viktorian\n",
       "4       31873_5.txt  Viktorian\n",
       "...             ...        ...\n",
       "36252  36919_48.txt  Modernism\n",
       "36253  36919_49.txt  Modernism\n",
       "36254  36919_50.txt  Modernism\n",
       "36255  36919_51.txt  Modernism\n",
       "36256  36919_52.txt  Modernism\n",
       "\n",
       "[36257 rows x 2 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Define the path to the CSV file\n",
    "train_csv_file = \"/data1/debajyoti/colie/train.csv\"\n",
    "val_csv_file = \"/data1/debajyoti/colie/valid.csv\"\n",
    "\n",
    "# Read the CSV file\n",
    "train_labels = pd.read_csv(train_csv_file)\n",
    "val_labels = pd.read_csv(val_csv_file)\n",
    "val_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'27993_1.txt'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_labels.BOOK_id[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                text      label\n",
      "0  rifle; Ivan's was a double-barrelled shot-gun ...  Viktorian\n",
      "1  upon the track of the bear. After following it...  Viktorian\n",
      "2  to pull him out with their hands--even had the...  Viktorian\n",
      "3  a slight sparkle of scientific conceit, \"this ...  Viktorian\n",
      "4  bears with a white ring round their necks? Yes...  Viktorian                                                 text      label\n",
      "0  kind good morning, and returned her hearty emb...  Viktorian\n",
      "1  sky, and of the moon, which clothed the old pi...  Viktorian\n",
      "2  left Rome for Augsburg, my mind being much exc...  Viktorian\n",
      "3  thoughts some of the old melodies he knew by h...  Viktorian\n",
      "4  \"But,\" said Henry, \"is it not possible that th...  Viktorian\n",
      "(546210, 2) (36257, 2)\n"
     ]
    }
   ],
   "source": [
    "# Define the path to the train folder\n",
    "train_folder = \"/data1/debajyoti/colie/train/train/\"\n",
    "# Define the path to the validation folder\n",
    "val_folder = \"/data1/debajyoti/colie/valid/valid/\"\n",
    "\n",
    "\n",
    "\n",
    "def create_df(folder, label):\n",
    "    # Initialize empty lists to store the data\n",
    "    text_data = []\n",
    "    labels = []\n",
    "    for index in label.index:\n",
    "        # filename = df_labels.BOOK_id[index]\n",
    "        # print(filename)\n",
    "        # print(df_labels['BOOK_id'][index], df_labels['Epoch'][index])\n",
    "        file_name = label['BOOK_id'][index]  # Assuming 'File Name' is the column name for the file names in the CSV\n",
    "\n",
    "        # Construct the file path\n",
    "        file_path = os.path.join(folder, file_name)\n",
    "\n",
    "        # Read the text from the file\n",
    "        with open(file_path, 'r', encoding='ISO-8859-1') as file:\n",
    "            text = file.read()\n",
    "\n",
    "        # Append the text and label to the respective lists\n",
    "        text_data.append(text)\n",
    "        labels.append(label['Epoch'][index].strip())  # Assuming 'Label' is the column name for the labels in the CSV\n",
    "        # break\n",
    "    return text_data, labels\n",
    "\n",
    "train_data, train_label = create_df(train_folder, train_labels)\n",
    "val_data, val_label = create_df(val_folder, val_labels)\n",
    "\n",
    "# Create a dataframe from the lists\n",
    "train = pd.DataFrame({'text': train_data, 'label': train_label})\n",
    "val = pd.DataFrame({'text': val_data, 'label': val_label})\n",
    "print(train.head(), val.head())\n",
    "print(train.shape, val.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_dic = {'Romanticism':0,\n",
    "            'Viktorian':1,\n",
    "            'Modernism':2,\n",
    "            'PostModernism':3,\n",
    "            'OurDays':4}\n",
    "train['label'] = train['label'].map(label_dic)\n",
    "val['label'] = val['label'].map(label_dic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "483268    1128\n",
      "483267    1068\n",
      "521384    1065\n",
      "483265    1034\n",
      "81542     1020\n",
      "          ... \n",
      "470405       1\n",
      "130188       1\n",
      "217335       1\n",
      "351867       1\n",
      "368135       1\n",
      "Name: text, Length: 546210, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# Length of text\n",
    "def length (txt):\n",
    "    length = len(txt.split())\n",
    "    return length\n",
    "\n",
    "txt_length = train['text'].apply(lambda x: length(x))\n",
    "print(txt_length.sort_values(ascending = False))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "label\n",
       "1    16938\n",
       "2    14848\n",
       "3     1713\n",
       "4     1600\n",
       "0     1158\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val['label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model = LongformerModel.from_pretrained(\"allenai/longformer-base-4096\")\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"allenai/longformer-base-4096\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/22759 [00:00<?, ?it/s]"
     ]
    }
   ],
   "source": [
    "max_length= 1200\n",
    "class CustomDataset(Dataset):\n",
    "    def __init__(self, tokenizer, df):\n",
    "        # Initialize thetokenizer\n",
    "        self.tokenizer = tokenizer\n",
    "\n",
    "        self.df = df\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.df)\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        # Get the text and label from the dataframe\n",
    "        text = self.df.iloc[index]['text']\n",
    "        label = self.df.iloc[index]['label']\n",
    "\n",
    "        # Tokenize the text and convert it to input IDs\n",
    "        inputs = self.tokenizer(\n",
    "            text,\n",
    "            None,\n",
    "            add_special_tokens=False,\n",
    "            max_length=max_length,\n",
    "            padding='max_length',\n",
    "            truncation=True,\n",
    "            return_token_type_ids=False,\n",
    "            return_attention_mask=True,\n",
    "            return_tensors='pt',\n",
    "        )\n",
    "\n",
    "\n",
    "        # Return the input IDs and label as PyTorch tensors\n",
    "        return {\n",
    "            'input_ids': inputs['input_ids'][0],\n",
    "            'attention_mask': inputs['attention_mask'][0],\n",
    "            # 'token_type_ids': inputs['token_type_ids'][0],\n",
    "            'label': torch.tensor(label, dtype=torch.int64),\n",
    "        }\n",
    "\n",
    "# datasetclass = CustomDataset(tokenizer, train)\n",
    "train_dataset = CustomDataset(tokenizer, train)\n",
    "val_dataset = CustomDataset(tokenizer, val)\n",
    "\n",
    "# DataLoader\n",
    "batch_size = 24\n",
    "train_dataloader = tqdm(DataLoader(train_dataset, batch_size=batch_size, shuffle=True))\n",
    "val_dataloader = tqdm(DataLoader(val_dataset, batch_size=batch_size, shuffle=True))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TransformerModel(nn.Module):\n",
    "    def __init__(self, num_labels):\n",
    "        super(TransformerModel, self).__init__()\n",
    "        \n",
    "        self.Longformer = LongformerModel.from_pretrained(\"allenai/longformer-base-4096\")\n",
    "        # self.xlnet.resize_token_embeddings(num_tokens)\n",
    "        # self.transformer_encoder = TransformerEncoder(TransformerEncoderLayer(d_model=hidden_size, nhead=num_heads), num_layers=num_layers)\n",
    "        #self.transformer_decoder = TransformerDecoder(TransformerDecoderLayer(d_model=hidden_size, nhead=num_heads), num_layers=num_layers)\n",
    "        #self.transformer = Transformer(nhead=16, num_encoder_layers=6, num_decoder_layers = 6)\n",
    "        self.decoder = nn.Linear(self.Longformer.config.hidden_size, num_labels) \n",
    "        # self.fc1 = nn.Linear(num_tokens, 2)\n",
    "        # self.fc2 = nn.Linear(num_tokens, 2)\n",
    "        # self.fc3 = nn.Linear(num_tokens, 5)\n",
    "        # self.num_classes = num_classes\n",
    "        # self.classifiers = nn.ModuleList([nn.Linear(self.roberta.config.hidden_size, num_classes[i]) for i in range(len(num_classes))])\n",
    "        # self.classifiers = nn.ModuleList([nn.Linear(num_tokens, num_classes[i]) for i in range(len(num_classes))])\n",
    "        # self.tanh = nn.Tanh()\n",
    "\n",
    "    def forward(self, input_ids, attention_mask):  # src = [bsz, seq_len]\n",
    "        long_output = self.Longformer(input_ids=input_ids).pooler_output\n",
    "        # print(long_output.shape)\n",
    "        # roberta_outputs = self.roberta(input_ids=input_ids, attention_mask=attention_mask)\n",
    "        # last_hidden_state = outputs.last_hidden_state # Shape: (batch_size, sequence_len, hidden_size)\n",
    "        # src_embedded = last_hidden_state\n",
    "        # src_embedded = self.roberta.embeddings(src) # Use RoBERTa model to embed source sequence output: [bsz, seq_len, features,i.e. hidden_dim] [20, 100, 768]\n",
    "        # print(\"shape of roberta embeddings:\", src_embedded.shape)\n",
    "        #tgt_embedded = self.roberta.embeddings(tgt) # Use RoBERTa model to embed target sequence\n",
    "        # src_embedded = src_embedded # output: [bsz, seq_len, features] \n",
    "        # src_embedded = torch.cat([t1,t2,t3, src_embedded],1)\n",
    "\n",
    "        # t1 = torch.cat(src_embedded.size(0) * [t1])\n",
    "        # t2 = torch.cat(src_embedded.size(0) * [t2])\n",
    "        # t3 = torch.cat(src_embedded.size(0) * [t3])\n",
    "        # t = torch.stack([t1,t2,t3], dim=1)\n",
    "        # task_embedded = torch.cat([t, src_embedded],1)  # output shape: [bsz, seq_len, features] [8, 203, 768]\n",
    "\n",
    "        # memory = self.transformer_encoder(src_embedded)  # output shape: [bsz, seq_len, features] [8, 203, 768]\n",
    "        # print(\"shape after transformer encoder layer:\", memory.shape)\n",
    "        #output = self.transformer_decoder(tgt_embedded, memory)\n",
    "        #print(\"shape after transformer decoder layer:\", output.shape)\n",
    "\n",
    "        output = self.decoder(long_output)  # output shape: [bsz, seq_len, vocab_size] [8, 203, 50k]\n",
    "        # print(\"shape after transformer decoder layer:\", output.shape, output.dtype)\n",
    "        # task1_output = self.fc1(output[:,0,:])\n",
    "        # task2_output = self.fc2(output[:,1,:])\n",
    "        # task3_output = self.fc3(output[:,2,:num_classes])\n",
    "        # ae_output = output[:,len(self.num_classes):,:]\n",
    "        # ae_output = output[:,:,:]\n",
    "        # print(\"shape after final linear layer:\", output.shape)\n",
    "        # task_logits = [classifier(pooled_output) for classifier in self.classifiers]\n",
    "        # task_logits = []\n",
    "\n",
    "        # pooled_outputs = [output[:,i,:] for i in range(len(self.num_classes))] # output shape : [bsz, 1, vocab_size]\n",
    "\n",
    "        # for classifier, pooled_output in zip(self.classifiers, pooled_outputs):\n",
    "        #     # pooled_output = self.tanh(pooled_output)\n",
    "        #     logits = classifier(pooled_output)\n",
    "        #     task_logits.append(logits)\n",
    "        \n",
    "        return output\n",
    "\n",
    "device = torch.device('cuda:1' if torch.cuda.is_available() else 'cpu')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of the model checkpoint at allenai/longformer-base-4096 were not used when initializing LongformerModel: ['lm_head.layer_norm.bias', 'lm_head.decoder.weight', 'lm_head.layer_norm.weight', 'lm_head.bias', 'lm_head.dense.bias', 'lm_head.dense.weight']\n",
      "- This IS expected if you are initializing LongformerModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing LongformerModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
     ]
    }
   ],
   "source": [
    "num_labels = 5\n",
    "\n",
    "model = TransformerModel(num_labels).to(device)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# num_epochs = 5\n",
    "learning_rate = 2e-5\n",
    "class_weights = torch.tensor([0.35, 0.03, 0.03, 0.25, 0.34])\n",
    "\n",
    "# Set optimizer and learning rate scheduler\n",
    "criterion = nn.CrossEntropyLoss(weight=class_weights).to(device)\n",
    "optimizer = torch.optim.AdamW(model.parameters(), lr=learning_rate)\n",
    "scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 1.0, gamma=0.95)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_labels(logit, targets):\n",
    "    \"\"\"\n",
    "    Calculate accuracy and macro F1 score for each class\n",
    "    \"\"\"\n",
    "    # pos = list(task_dict.keys()).index(task_name)\n",
    "    # mask = torch.arange(targets.shape[0]).to(device)\n",
    "    # task_idx = mask[targets[:,pos] != 99]\n",
    "    output = logit\n",
    "    true_label = targets\n",
    "    # print(\"shapes for label:\", output.shape, true_label.shape)\n",
    "    pred_label = torch.argmax(output, dim=1).flatten().tolist()\n",
    "    true_label = true_label.flatten().tolist()\n",
    "\n",
    "\n",
    "    return pred_label, true_label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "current_train_loss = []\n",
    "\n",
    "def train(model: nn.Module) -> None:\n",
    "    model.train()  # turn on train mode\n",
    "    total_loss = 0.\n",
    "    log_interval = 1\n",
    "    start_time = time.time()\n",
    "    num_batches = len(train_dataset) // batch_size\n",
    "    for batch, i in enumerate(train_dataloader):\n",
    "        data, mask, targets = i.values()\n",
    "        data = data.to(device)\n",
    "        mask = mask.to(device)\n",
    "        targets = targets.to(device)\n",
    "        # print(data.dtype)        \n",
    "        # print(data.shape)\n",
    "        # task_logits, ae_output = model(data)\n",
    "        output = model(data, mask)\n",
    "        # t1_out, t2_out, t3_out, auto_output = model(data, t1, t2, t3)\n",
    "        # loss = custom_loss(logits_task1, logits_task2, logits_task3, targets)\n",
    "        # print(\"shape:\", data.shape, targets.flatten().shape)\n",
    "        # print(\"datatype:\", data.dtype, targets.flatten().dtype)\n",
    "        loss = criterion(output, targets.flatten())\n",
    "\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        # torch.nn.utils.clip_grad_norm_(model.parameters(), 0.5)\n",
    "        optimizer.step()\n",
    "\n",
    "        total_loss += loss.item()\n",
    "        if batch % log_interval == 0 and batch > 0:\n",
    "            lr = scheduler.get_last_lr()[0]\n",
    "            ms_per_batch = (time.time() - start_time) * 1000 / log_interval\n",
    "            cur_loss = total_loss / log_interval\n",
    "            # ppl = np.exp(cur_loss)\n",
    "            print(f'| epoch {epoch:3d} | {batch:5d}/{num_batches:5d} batches | '\n",
    "                    f'lr {lr:02.7f} | ms/batch {ms_per_batch:5.2f} | '\n",
    "                    f'loss {cur_loss:5.5f}')\n",
    "            total_loss = 0\n",
    "            start_time = time.time()\n",
    "        \n",
    "        if batch == 100:\n",
    "            break\n",
    "    current_train_loss.append(cur_loss)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(model: nn.Module) -> float:\n",
    "    model.eval()  # turn on evaluation mode\n",
    "    total_loss = 0.\n",
    "    # src_mask = generate_square_subsequent_mask(bptt).to(device)\n",
    "    with torch.no_grad():\n",
    "        predictions = []\n",
    "        true_labels = []\n",
    "        for batch, i in enumerate(val_dataloader):\n",
    "            data, mask, targets = i.values()\n",
    "            data = data.to(device)\n",
    "            mask = mask.to(device)\n",
    "            targets = targets.to(device)\n",
    "            seq_len = data.size(1)\n",
    "            # logits_task1, logits_task2, logits_task3, ae_output = model(data, mask)\n",
    "            # task_logits, ae_output = model(data)\n",
    "            # task_logits, ae_output = model(data, mask)\n",
    "            output = model(data, mask)\n",
    "            # t1_out, t2_out, t3_out, auto_output = model(data, t1, t2, t3)\n",
    "            # loss = custom_loss(logits_task1, logits_task2, logits_task3, targets)\n",
    "            # loss = custom_loss(logits_task1, logits_task2, logits_task3, ae_output, data, targets)\n",
    "            loss = criterion(output, targets.flatten())\n",
    "\n",
    "            total_loss += seq_len * loss.item()\n",
    "\n",
    "            #get the labels for classification report\n",
    "            pred_label, true_label = get_labels(output, targets)\n",
    "            predictions.extend(pred_label)\n",
    "            true_labels.extend(true_label)\n",
    "            # if batch == 100:\n",
    "            #     break\n",
    "\n",
    "    # Compute overall classification report\n",
    "    logging.info(f\"\\n Scores:\")\n",
    "    logging.info(f\"\\n {classification_report(true_labels, predictions)}\")\n",
    "    return total_loss / (len(val_dataset) - 1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-07-17 11:26:35,082 - INFO - #########################################################################################\n",
      "2023-07-17 11:26:35,083 - INFO - \n",
      " DESCRIPTION-> \n",
      " logic: longformer + linear_layer + loss_reweighting(100 batches), model: allenai/longformer-base-4096, lr:2e-05, max_seq_length: 1200\n",
      "2023-07-17 11:26:35,084 - INFO - #########################################################################################\n"
     ]
    }
   ],
   "source": [
    "logging.info(f\"#\"* 89)\n",
    "logging.info(f\"\\n DESCRIPTION-> \\n logic: longformer + linear_layer + loss_reweighting(100 batches), model: {tokenizer.name_or_path}, lr:{learning_rate}, max_seq_length: {max_length}\")\n",
    "logging.info('#' * 89)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 2/22759 [00:10<29:54:27,  4.73s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |     1/22758 batches | lr 0.0000200 | ms/batch 6687.19 | loss 3.31908\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 3/22759 [00:13<24:43:11,  3.91s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |     2/22758 batches | lr 0.0000200 | ms/batch 2934.17 | loss 1.57182\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 4/22759 [00:16<22:13:32,  3.52s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |     3/22758 batches | lr 0.0000200 | ms/batch 2911.35 | loss 1.62768\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 5/22759 [00:19<20:51:32,  3.30s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |     4/22758 batches | lr 0.0000200 | ms/batch 2916.93 | loss 1.60275\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 6/22759 [00:21<20:01:54,  3.17s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |     5/22758 batches | lr 0.0000200 | ms/batch 2915.44 | loss 1.63506\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 7/22759 [00:24<19:30:47,  3.09s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |     6/22758 batches | lr 0.0000200 | ms/batch 2918.64 | loss 1.59265\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 8/22759 [00:27<19:09:20,  3.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |     7/22758 batches | lr 0.0000200 | ms/batch 2910.22 | loss 1.60107\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 9/22759 [00:30<18:55:36,  3.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |     8/22758 batches | lr 0.0000200 | ms/batch 2915.50 | loss 1.57812\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 10/22759 [00:33<18:47:11,  2.97s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |     9/22758 batches | lr 0.0000200 | ms/batch 2923.52 | loss 1.66095\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 11/22759 [00:36<18:39:57,  2.95s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    10/22758 batches | lr 0.0000200 | ms/batch 2910.73 | loss 1.62084\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 12/22759 [00:39<18:34:31,  2.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    11/22758 batches | lr 0.0000200 | ms/batch 2907.30 | loss 1.60792\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 13/22759 [00:42<18:31:40,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    12/22758 batches | lr 0.0000200 | ms/batch 2915.10 | loss 1.59776\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 14/22759 [00:45<18:29:12,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    13/22758 batches | lr 0.0000200 | ms/batch 2911.31 | loss 1.60637\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 15/22759 [00:48<18:27:35,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    14/22758 batches | lr 0.0000200 | ms/batch 2912.11 | loss 1.55568\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 16/22759 [00:51<18:26:25,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    15/22758 batches | lr 0.0000200 | ms/batch 2911.92 | loss 1.55268\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 17/22759 [00:53<18:26:52,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    16/22758 batches | lr 0.0000200 | ms/batch 2923.27 | loss 1.56571\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 18/22759 [00:56<18:25:44,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    17/22758 batches | lr 0.0000200 | ms/batch 2910.53 | loss 1.54393\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 19/22759 [00:59<18:27:58,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    18/22758 batches | lr 0.0000200 | ms/batch 2936.62 | loss 1.54037\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 20/22759 [01:02<18:29:54,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    19/22758 batches | lr 0.0000200 | ms/batch 2940.67 | loss 1.70009\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 21/22759 [01:05<18:30:36,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    20/22758 batches | lr 0.0000200 | ms/batch 2935.19 | loss 1.62295\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 22/22759 [01:08<18:31:11,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    21/22758 batches | lr 0.0000200 | ms/batch 2936.04 | loss 1.47377\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 23/22759 [01:11<18:31:01,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    22/22758 batches | lr 0.0000200 | ms/batch 2931.17 | loss 1.58164\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 24/22759 [01:14<18:35:39,  2.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    23/22758 batches | lr 0.0000200 | ms/batch 2972.83 | loss 1.57515\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 25/22759 [01:17<18:33:39,  2.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    24/22758 batches | lr 0.0000200 | ms/batch 2926.31 | loss 1.46694\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 26/22759 [01:20<18:32:03,  2.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    25/22758 batches | lr 0.0000200 | ms/batch 2924.85 | loss 1.48548\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 27/22759 [01:23<18:32:18,  2.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    26/22758 batches | lr 0.0000200 | ms/batch 2937.65 | loss 1.87018\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 28/22759 [01:26<18:29:53,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    27/22758 batches | lr 0.0000200 | ms/batch 2914.90 | loss 1.58744\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 29/22759 [01:29<18:28:43,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    28/22758 batches | lr 0.0000200 | ms/batch 2919.50 | loss 1.52318\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 30/22759 [01:32<18:27:46,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    29/22758 batches | lr 0.0000200 | ms/batch 2918.75 | loss 1.58006\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 31/22759 [01:35<18:31:02,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    30/22758 batches | lr 0.0000200 | ms/batch 2953.27 | loss 1.71528\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 32/22759 [01:37<18:28:41,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    31/22758 batches | lr 0.0000200 | ms/batch 2912.71 | loss 1.49429\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 33/22759 [01:40<18:27:03,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    32/22758 batches | lr 0.0000200 | ms/batch 2913.02 | loss 1.67318\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 34/22759 [01:43<18:27:17,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    33/22758 batches | lr 0.0000200 | ms/batch 2925.09 | loss 1.80831\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 35/22759 [01:46<18:29:08,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    34/22758 batches | lr 0.0000200 | ms/batch 2939.98 | loss 1.75395\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 36/22759 [01:49<18:28:44,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    35/22758 batches | lr 0.0000200 | ms/batch 2925.45 | loss 1.71122\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 37/22759 [01:52<18:29:09,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    36/22758 batches | lr 0.0000200 | ms/batch 2931.60 | loss 1.52372\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 38/22759 [01:55<18:32:53,  2.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    37/22758 batches | lr 0.0000200 | ms/batch 2962.04 | loss 1.64488\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 39/22759 [01:58<18:31:08,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    38/22758 batches | lr 0.0000200 | ms/batch 2923.72 | loss 1.63901\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 40/22759 [02:01<18:30:31,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    39/22758 batches | lr 0.0000200 | ms/batch 2929.29 | loss 1.56547\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 41/22759 [02:04<18:29:05,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    40/22758 batches | lr 0.0000200 | ms/batch 2920.64 | loss 1.62322\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 42/22759 [02:07<18:28:39,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    41/22758 batches | lr 0.0000200 | ms/batch 2925.61 | loss 1.58416\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 43/22759 [02:10<18:28:51,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    42/22758 batches | lr 0.0000200 | ms/batch 2930.24 | loss 1.58975\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 44/22759 [02:13<18:29:34,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    43/22758 batches | lr 0.0000200 | ms/batch 2935.48 | loss 1.61509\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 45/22759 [02:16<18:31:00,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    44/22758 batches | lr 0.0000200 | ms/batch 2943.81 | loss 1.67200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 46/22759 [02:18<18:29:29,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    45/22758 batches | lr 0.0000200 | ms/batch 2921.67 | loss 1.52760\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 47/22759 [02:21<18:28:00,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    46/22758 batches | lr 0.0000200 | ms/batch 2918.07 | loss 1.58483\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 48/22759 [02:24<18:34:50,  2.95s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    47/22758 batches | lr 0.0000200 | ms/batch 2987.68 | loss 1.68172\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 49/22759 [02:27<18:32:45,  2.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    48/22758 batches | lr 0.0000200 | ms/batch 2927.29 | loss 1.58837\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 50/22759 [02:30<18:30:39,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    49/22758 batches | lr 0.0000200 | ms/batch 2921.66 | loss 1.54884\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 51/22759 [02:33<18:29:26,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    50/22758 batches | lr 0.0000200 | ms/batch 2923.93 | loss 1.67857\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 52/22759 [02:36<18:28:46,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    51/22758 batches | lr 0.0000200 | ms/batch 2925.23 | loss 1.64099\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 53/22759 [02:39<18:28:33,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    52/22758 batches | lr 0.0000200 | ms/batch 2928.35 | loss 1.64869\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 54/22759 [02:42<18:27:52,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    53/22758 batches | lr 0.0000200 | ms/batch 2922.43 | loss 1.55526\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 55/22759 [02:45<18:28:36,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    54/22758 batches | lr 0.0000200 | ms/batch 2934.45 | loss 1.63556\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 56/22759 [02:48<18:27:51,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    55/22758 batches | lr 0.0000200 | ms/batch 2923.50 | loss 1.49992\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 57/22759 [02:51<18:27:38,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    56/22758 batches | lr 0.0000200 | ms/batch 2926.28 | loss 1.60646\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 58/22759 [02:54<18:26:52,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    57/22758 batches | lr 0.0000200 | ms/batch 2921.03 | loss 1.61651\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 59/22759 [02:57<18:33:25,  2.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    58/22758 batches | lr 0.0000200 | ms/batch 2983.54 | loss 1.53537\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 60/22759 [03:00<18:31:39,  2.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    59/22758 batches | lr 0.0000200 | ms/batch 2927.74 | loss 1.66742\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 61/22759 [03:02<18:29:15,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    60/22758 batches | lr 0.0000200 | ms/batch 2917.45 | loss 1.59098\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 62/22759 [03:05<18:27:25,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    61/22758 batches | lr 0.0000200 | ms/batch 2916.40 | loss 1.58944\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 63/22759 [03:08<18:26:26,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    62/22758 batches | lr 0.0000200 | ms/batch 2919.14 | loss 1.39825\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 64/22759 [03:11<18:25:18,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    63/22758 batches | lr 0.0000200 | ms/batch 2915.38 | loss 1.66447\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 65/22759 [03:14<18:24:39,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    64/22758 batches | lr 0.0000200 | ms/batch 2916.77 | loss 1.64497\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 66/22759 [03:17<18:29:04,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    65/22758 batches | lr 0.0000200 | ms/batch 2959.88 | loss 1.50000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 67/22759 [03:20<18:27:35,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    66/22758 batches | lr 0.0000200 | ms/batch 2919.66 | loss 1.61260\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 68/22759 [03:23<18:26:11,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    67/22758 batches | lr 0.0000200 | ms/batch 2916.63 | loss 1.61354\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 69/22759 [03:26<18:25:11,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    68/22758 batches | lr 0.0000200 | ms/batch 2916.53 | loss 1.41785\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 70/22759 [03:29<18:24:53,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    69/22758 batches | lr 0.0000200 | ms/batch 2920.23 | loss 1.58308\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 71/22759 [03:32<18:24:28,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    70/22758 batches | lr 0.0000200 | ms/batch 2918.46 | loss 1.59955\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 72/22759 [03:35<18:26:34,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    71/22758 batches | lr 0.0000200 | ms/batch 2939.68 | loss 1.49589\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 73/22759 [03:38<18:25:55,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    72/22758 batches | lr 0.0000200 | ms/batch 2921.06 | loss 1.55118\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 74/22759 [03:40<18:25:02,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    73/22758 batches | lr 0.0000200 | ms/batch 2917.44 | loss 1.48561\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 75/22759 [03:43<18:24:21,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    74/22758 batches | lr 0.0000200 | ms/batch 2916.97 | loss 1.37814\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 76/22759 [03:46<18:24:42,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    75/22758 batches | lr 0.0000200 | ms/batch 2924.62 | loss 1.67272\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 77/22759 [03:49<18:24:43,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    76/22758 batches | lr 0.0000200 | ms/batch 2922.58 | loss 1.76804\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 78/22759 [03:52<18:23:46,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    77/22758 batches | lr 0.0000200 | ms/batch 2914.26 | loss 1.64887\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 79/22759 [03:55<18:23:28,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    78/22758 batches | lr 0.0000200 | ms/batch 2917.62 | loss 1.51614\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 80/22759 [03:58<18:28:43,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    79/22758 batches | lr 0.0000200 | ms/batch 2965.81 | loss 1.67785\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 81/22759 [04:01<18:27:13,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    80/22758 batches | lr 0.0000200 | ms/batch 2920.36 | loss 1.28234\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 82/22759 [04:04<18:25:24,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    81/22758 batches | lr 0.0000200 | ms/batch 2913.81 | loss 1.28396\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 83/22759 [04:07<18:24:23,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    82/22758 batches | lr 0.0000200 | ms/batch 2916.11 | loss 1.62336\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 84/22759 [04:10<18:23:45,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    83/22758 batches | lr 0.0000200 | ms/batch 2916.90 | loss 1.69428\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 85/22759 [04:13<18:23:03,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    84/22758 batches | lr 0.0000200 | ms/batch 2914.80 | loss 1.60492\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 86/22759 [04:16<18:22:24,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    85/22758 batches | lr 0.0000200 | ms/batch 2913.51 | loss 1.52354\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 87/22759 [04:18<18:22:40,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    86/22758 batches | lr 0.0000200 | ms/batch 2920.08 | loss 1.54339\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 88/22759 [04:21<18:22:56,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    87/22758 batches | lr 0.0000200 | ms/batch 2920.68 | loss 1.67685\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 89/22759 [04:24<18:22:52,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    88/22758 batches | lr 0.0000200 | ms/batch 2918.80 | loss 1.57377\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 90/22759 [04:27<18:22:23,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    89/22758 batches | lr 0.0000200 | ms/batch 2914.96 | loss 1.53873\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 91/22759 [04:30<18:22:45,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    90/22758 batches | lr 0.0000200 | ms/batch 2921.40 | loss 1.75393\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 92/22759 [04:33<18:22:32,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    91/22758 batches | lr 0.0000200 | ms/batch 2917.38 | loss 1.66978\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 93/22759 [04:36<18:22:22,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    92/22758 batches | lr 0.0000200 | ms/batch 2917.27 | loss 1.55546\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 94/22759 [04:39<18:24:58,  2.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    93/22758 batches | lr 0.0000200 | ms/batch 2941.38 | loss 1.80391\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 95/22759 [04:42<18:24:41,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    94/22758 batches | lr 0.0000200 | ms/batch 2922.94 | loss 1.51705\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 96/22759 [04:45<18:24:25,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    95/22758 batches | lr 0.0000200 | ms/batch 2922.56 | loss 1.59760\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 97/22759 [04:48<18:23:42,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    96/22758 batches | lr 0.0000200 | ms/batch 2918.00 | loss 1.43044\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 98/22759 [04:51<18:23:21,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    97/22758 batches | lr 0.0000200 | ms/batch 2919.44 | loss 1.54504\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 99/22759 [04:54<18:22:19,  2.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    98/22758 batches | lr 0.0000200 | ms/batch 2912.53 | loss 1.54301\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 100/22759 [04:56<18:20:25,  2.91s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |    99/22758 batches | lr 0.0000200 | ms/batch 2902.35 | loss 1.58384\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 100/22759 [04:59<18:52:31,  3.00s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   1 |   100/22758 batches | lr 0.0000200 | ms/batch 2971.93 | loss 1.54353\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "100%|██████████| 1511/1511 [24:05<00:00,  1.05it/s]\n",
      "2023-07-17 11:50:37,143 - INFO - \n",
      " Scores:\n",
      "/data1/debajyoti/colie/colenv/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1469: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/data1/debajyoti/colie/colenv/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1469: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/data1/debajyoti/colie/colenv/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1469: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "2023-07-17 11:50:37,205 - INFO - \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.00      0.00      0.00      1158\n",
      "           1       0.50      0.02      0.03     16938\n",
      "           2       0.43      0.85      0.57     14848\n",
      "           3       0.00      0.00      0.00      1713\n",
      "           4       0.04      0.15      0.06      1600\n",
      "\n",
      "    accuracy                           0.36     36257\n",
      "   macro avg       0.19      0.20      0.13     36257\n",
      "weighted avg       0.41      0.36      0.25     36257\n",
      "\n",
      "2023-07-17 11:50:37,206 - INFO - -----------------------------------------------------------------------------------------\n",
      "2023-07-17 11:50:37,207 - INFO - | end of epoch   1 | time: 1442.11s | valid loss 80.29001\n",
      "2023-07-17 11:50:37,208 - INFO - -----------------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   2 |     1/22758 batches | lr 0.0000190 | ms/batch 5853.72 | loss 3.25321\n",
      "| epoch   2 |     2/22758 batches | lr 0.0000190 | ms/batch 2944.87 | loss 1.68745\n",
      "| epoch   2 |     3/22758 batches | lr 0.0000190 | ms/batch 2970.22 | loss 1.76145\n",
      "| epoch   2 |     4/22758 batches | lr 0.0000190 | ms/batch 2919.10 | loss 1.71872\n",
      "| epoch   2 |     5/22758 batches | lr 0.0000190 | ms/batch 2920.87 | loss 1.59743\n",
      "| epoch   2 |     6/22758 batches | lr 0.0000190 | ms/batch 2924.06 | loss 1.55215\n",
      "| epoch   2 |     7/22758 batches | lr 0.0000190 | ms/batch 2923.04 | loss 1.66996\n",
      "| epoch   2 |     8/22758 batches | lr 0.0000190 | ms/batch 2925.49 | loss 1.54641\n",
      "| epoch   2 |     9/22758 batches | lr 0.0000190 | ms/batch 2931.90 | loss 1.55562\n",
      "| epoch   2 |    10/22758 batches | lr 0.0000190 | ms/batch 2921.41 | loss 1.67077\n",
      "| epoch   2 |    11/22758 batches | lr 0.0000190 | ms/batch 2924.22 | loss 1.63249\n",
      "| epoch   2 |    12/22758 batches | lr 0.0000190 | ms/batch 2916.35 | loss 1.59587\n",
      "| epoch   2 |    13/22758 batches | lr 0.0000190 | ms/batch 2925.97 | loss 1.61726\n",
      "| epoch   2 |    14/22758 batches | lr 0.0000190 | ms/batch 2920.74 | loss 1.58173\n",
      "| epoch   2 |    15/22758 batches | lr 0.0000190 | ms/batch 2921.04 | loss 1.63384\n",
      "| epoch   2 |    16/22758 batches | lr 0.0000190 | ms/batch 2957.60 | loss 1.58218\n",
      "| epoch   2 |    17/22758 batches | lr 0.0000190 | ms/batch 2918.72 | loss 1.63519\n",
      "| epoch   2 |    18/22758 batches | lr 0.0000190 | ms/batch 2924.15 | loss 1.44839\n",
      "| epoch   2 |    19/22758 batches | lr 0.0000190 | ms/batch 2919.28 | loss 1.57810\n",
      "| epoch   2 |    20/22758 batches | lr 0.0000190 | ms/batch 2924.57 | loss 1.57799\n",
      "| epoch   2 |    21/22758 batches | lr 0.0000190 | ms/batch 2920.79 | loss 1.63315\n",
      "| epoch   2 |    22/22758 batches | lr 0.0000190 | ms/batch 2924.33 | loss 1.58563\n",
      "| epoch   2 |    23/22758 batches | lr 0.0000190 | ms/batch 2920.52 | loss 1.56901\n",
      "| epoch   2 |    24/22758 batches | lr 0.0000190 | ms/batch 2927.36 | loss 1.56515\n",
      "| epoch   2 |    25/22758 batches | lr 0.0000190 | ms/batch 2933.02 | loss 1.71568\n",
      "| epoch   2 |    26/22758 batches | lr 0.0000190 | ms/batch 2922.63 | loss 1.60335\n",
      "| epoch   2 |    27/22758 batches | lr 0.0000190 | ms/batch 2926.45 | loss 1.59353\n",
      "| epoch   2 |    28/22758 batches | lr 0.0000190 | ms/batch 2922.18 | loss 1.55808\n",
      "| epoch   2 |    29/22758 batches | lr 0.0000190 | ms/batch 2933.63 | loss 1.58561\n",
      "| epoch   2 |    30/22758 batches | lr 0.0000190 | ms/batch 3029.67 | loss 1.55330\n",
      "| epoch   2 |    31/22758 batches | lr 0.0000190 | ms/batch 2922.57 | loss 1.59622\n",
      "| epoch   2 |    32/22758 batches | lr 0.0000190 | ms/batch 2921.91 | loss 1.61277\n",
      "| epoch   2 |    33/22758 batches | lr 0.0000190 | ms/batch 2918.45 | loss 1.54064\n",
      "| epoch   2 |    34/22758 batches | lr 0.0000190 | ms/batch 2918.07 | loss 1.51875\n",
      "| epoch   2 |    35/22758 batches | lr 0.0000190 | ms/batch 2920.00 | loss 1.56098\n",
      "| epoch   2 |    36/22758 batches | lr 0.0000190 | ms/batch 2920.80 | loss 1.65615\n",
      "| epoch   2 |    37/22758 batches | lr 0.0000190 | ms/batch 2932.47 | loss 1.66607\n",
      "| epoch   2 |    38/22758 batches | lr 0.0000190 | ms/batch 2924.71 | loss 1.62426\n",
      "| epoch   2 |    39/22758 batches | lr 0.0000190 | ms/batch 2922.14 | loss 1.57541\n",
      "| epoch   2 |    40/22758 batches | lr 0.0000190 | ms/batch 2920.55 | loss 1.50824\n",
      "| epoch   2 |    41/22758 batches | lr 0.0000190 | ms/batch 2919.04 | loss 1.63703\n",
      "| epoch   2 |    42/22758 batches | lr 0.0000190 | ms/batch 2919.17 | loss 1.51803\n",
      "| epoch   2 |    43/22758 batches | lr 0.0000190 | ms/batch 2920.42 | loss 1.62937\n",
      "| epoch   2 |    44/22758 batches | lr 0.0000190 | ms/batch 2917.58 | loss 1.55978\n",
      "| epoch   2 |    45/22758 batches | lr 0.0000190 | ms/batch 2918.73 | loss 1.46141\n",
      "| epoch   2 |    46/22758 batches | lr 0.0000190 | ms/batch 2926.20 | loss 1.44556\n",
      "| epoch   2 |    47/22758 batches | lr 0.0000190 | ms/batch 2917.75 | loss 1.52663\n",
      "| epoch   2 |    48/22758 batches | lr 0.0000190 | ms/batch 2923.13 | loss 1.56062\n",
      "| epoch   2 |    49/22758 batches | lr 0.0000190 | ms/batch 2919.31 | loss 1.44911\n",
      "| epoch   2 |    50/22758 batches | lr 0.0000190 | ms/batch 2917.79 | loss 1.51574\n",
      "| epoch   2 |    51/22758 batches | lr 0.0000190 | ms/batch 2959.60 | loss 1.50776\n",
      "| epoch   2 |    52/22758 batches | lr 0.0000190 | ms/batch 2919.29 | loss 1.52290\n",
      "| epoch   2 |    53/22758 batches | lr 0.0000190 | ms/batch 2915.22 | loss 1.58944\n",
      "| epoch   2 |    54/22758 batches | lr 0.0000190 | ms/batch 2925.43 | loss 1.66639\n",
      "| epoch   2 |    55/22758 batches | lr 0.0000190 | ms/batch 3047.60 | loss 1.65723\n",
      "| epoch   2 |    56/22758 batches | lr 0.0000190 | ms/batch 3006.11 | loss 1.51757\n",
      "| epoch   2 |    57/22758 batches | lr 0.0000190 | ms/batch 3053.63 | loss 1.57923\n",
      "| epoch   2 |    58/22758 batches | lr 0.0000190 | ms/batch 2977.50 | loss 1.65471\n",
      "| epoch   2 |    59/22758 batches | lr 0.0000190 | ms/batch 2920.30 | loss 1.47315\n",
      "| epoch   2 |    60/22758 batches | lr 0.0000190 | ms/batch 2926.33 | loss 1.56289\n",
      "| epoch   2 |    61/22758 batches | lr 0.0000190 | ms/batch 2920.99 | loss 1.64952\n",
      "| epoch   2 |    62/22758 batches | lr 0.0000190 | ms/batch 2919.59 | loss 1.51246\n",
      "| epoch   2 |    63/22758 batches | lr 0.0000190 | ms/batch 2954.03 | loss 1.42884\n",
      "| epoch   2 |    64/22758 batches | lr 0.0000190 | ms/batch 2934.44 | loss 1.79299\n",
      "| epoch   2 |    65/22758 batches | lr 0.0000190 | ms/batch 2927.28 | loss 1.52062\n",
      "| epoch   2 |    66/22758 batches | lr 0.0000190 | ms/batch 2925.16 | loss 1.49001\n",
      "| epoch   2 |    67/22758 batches | lr 0.0000190 | ms/batch 2922.48 | loss 1.94511\n",
      "| epoch   2 |    68/22758 batches | lr 0.0000190 | ms/batch 2926.17 | loss 1.61056\n",
      "| epoch   2 |    69/22758 batches | lr 0.0000190 | ms/batch 2926.64 | loss 1.73582\n",
      "| epoch   2 |    70/22758 batches | lr 0.0000190 | ms/batch 2950.28 | loss 1.64497\n",
      "| epoch   2 |    71/22758 batches | lr 0.0000190 | ms/batch 2920.26 | loss 1.78548\n",
      "| epoch   2 |    72/22758 batches | lr 0.0000190 | ms/batch 2951.98 | loss 1.77613\n",
      "| epoch   2 |    73/22758 batches | lr 0.0000190 | ms/batch 2909.97 | loss 1.61197\n",
      "| epoch   2 |    74/22758 batches | lr 0.0000190 | ms/batch 2922.05 | loss 1.64279\n",
      "| epoch   2 |    75/22758 batches | lr 0.0000190 | ms/batch 2918.43 | loss 1.58512\n",
      "| epoch   2 |    76/22758 batches | lr 0.0000190 | ms/batch 2918.36 | loss 1.51570\n",
      "| epoch   2 |    77/22758 batches | lr 0.0000190 | ms/batch 2966.78 | loss 1.57225\n",
      "| epoch   2 |    78/22758 batches | lr 0.0000190 | ms/batch 2923.61 | loss 1.59777\n",
      "| epoch   2 |    79/22758 batches | lr 0.0000190 | ms/batch 2925.03 | loss 1.54222\n",
      "| epoch   2 |    80/22758 batches | lr 0.0000190 | ms/batch 2921.15 | loss 1.59656\n",
      "| epoch   2 |    81/22758 batches | lr 0.0000190 | ms/batch 2922.05 | loss 1.59658\n",
      "| epoch   2 |    82/22758 batches | lr 0.0000190 | ms/batch 2916.84 | loss 1.59890\n",
      "| epoch   2 |    83/22758 batches | lr 0.0000190 | ms/batch 2924.89 | loss 1.53156\n",
      "| epoch   2 |    84/22758 batches | lr 0.0000190 | ms/batch 2922.85 | loss 1.56313\n",
      "| epoch   2 |    85/22758 batches | lr 0.0000190 | ms/batch 2925.64 | loss 1.52041\n",
      "| epoch   2 |    86/22758 batches | lr 0.0000190 | ms/batch 2923.37 | loss 1.70752\n",
      "| epoch   2 |    87/22758 batches | lr 0.0000190 | ms/batch 2921.11 | loss 1.59431\n",
      "| epoch   2 |    88/22758 batches | lr 0.0000190 | ms/batch 2927.42 | loss 1.74894\n",
      "| epoch   2 |    89/22758 batches | lr 0.0000190 | ms/batch 2920.31 | loss 1.55249\n",
      "| epoch   2 |    90/22758 batches | lr 0.0000190 | ms/batch 2978.05 | loss 1.61896\n",
      "| epoch   2 |    91/22758 batches | lr 0.0000190 | ms/batch 2911.88 | loss 1.59597\n",
      "| epoch   2 |    92/22758 batches | lr 0.0000190 | ms/batch 2925.35 | loss 1.59339\n",
      "| epoch   2 |    93/22758 batches | lr 0.0000190 | ms/batch 2919.19 | loss 1.52883\n",
      "| epoch   2 |    94/22758 batches | lr 0.0000190 | ms/batch 2912.84 | loss 1.60748\n",
      "| epoch   2 |    95/22758 batches | lr 0.0000190 | ms/batch 2916.63 | loss 1.66917\n",
      "| epoch   2 |    96/22758 batches | lr 0.0000190 | ms/batch 2920.79 | loss 1.54039\n",
      "| epoch   2 |    97/22758 batches | lr 0.0000190 | ms/batch 2967.04 | loss 1.67349\n",
      "| epoch   2 |    98/22758 batches | lr 0.0000190 | ms/batch 2920.21 | loss 1.51436\n",
      "| epoch   2 |    99/22758 batches | lr 0.0000190 | ms/batch 2920.86 | loss 1.52551\n",
      "| epoch   2 |   100/22758 batches | lr 0.0000190 | ms/batch 2921.22 | loss 1.49208\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-07-17 12:14:36,253 - INFO - \n",
      " Scores:\n",
      "/data1/debajyoti/colie/colenv/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1469: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/data1/debajyoti/colie/colenv/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1469: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "/data1/debajyoti/colie/colenv/lib/python3.8/site-packages/sklearn/metrics/_classification.py:1469: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "2023-07-17 12:14:36,304 - INFO - \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.00      0.00      0.00      1158\n",
      "           1       0.52      0.61      0.56     16938\n",
      "           2       0.47      0.02      0.04     14848\n",
      "           3       0.00      0.00      0.00      1713\n",
      "           4       0.05      0.49      0.09      1600\n",
      "\n",
      "    accuracy                           0.31     36257\n",
      "   macro avg       0.21      0.22      0.14     36257\n",
      "weighted avg       0.44      0.31      0.28     36257\n",
      "\n",
      "2023-07-17 12:14:36,305 - INFO - -----------------------------------------------------------------------------------------\n",
      "2023-07-17 12:14:36,306 - INFO - | end of epoch   2 | time: 1438.45s | valid loss 79.39922\n",
      "2023-07-17 12:14:36,306 - INFO - -----------------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   3 |     1/22758 batches | lr 0.0000181 | ms/batch 5847.81 | loss 3.15688\n",
      "| epoch   3 |     2/22758 batches | lr 0.0000181 | ms/batch 2909.91 | loss 1.63082\n",
      "| epoch   3 |     3/22758 batches | lr 0.0000181 | ms/batch 2912.65 | loss 1.53810\n",
      "| epoch   3 |     4/22758 batches | lr 0.0000181 | ms/batch 2918.40 | loss 1.54178\n",
      "| epoch   3 |     5/22758 batches | lr 0.0000181 | ms/batch 2910.59 | loss 1.64238\n",
      "| epoch   3 |     6/22758 batches | lr 0.0000181 | ms/batch 2926.31 | loss 1.50475\n",
      "| epoch   3 |     7/22758 batches | lr 0.0000181 | ms/batch 2921.47 | loss 1.52614\n",
      "| epoch   3 |     8/22758 batches | lr 0.0000181 | ms/batch 2918.80 | loss 1.57445\n",
      "| epoch   3 |     9/22758 batches | lr 0.0000181 | ms/batch 2918.49 | loss 1.55922\n",
      "| epoch   3 |    10/22758 batches | lr 0.0000181 | ms/batch 2917.41 | loss 1.50498\n",
      "| epoch   3 |    11/22758 batches | lr 0.0000181 | ms/batch 2925.17 | loss 1.38305\n",
      "| epoch   3 |    12/22758 batches | lr 0.0000181 | ms/batch 2915.53 | loss 1.47678\n",
      "| epoch   3 |    13/22758 batches | lr 0.0000181 | ms/batch 2918.78 | loss 1.72357\n",
      "| epoch   3 |    14/22758 batches | lr 0.0000181 | ms/batch 2912.26 | loss 1.44761\n",
      "| epoch   3 |    15/22758 batches | lr 0.0000181 | ms/batch 2917.39 | loss 1.60449\n",
      "| epoch   3 |    16/22758 batches | lr 0.0000181 | ms/batch 2916.57 | loss 1.61454\n",
      "| epoch   3 |    17/22758 batches | lr 0.0000181 | ms/batch 2912.05 | loss 1.66955\n",
      "| epoch   3 |    18/22758 batches | lr 0.0000181 | ms/batch 2911.60 | loss 1.56605\n",
      "| epoch   3 |    19/22758 batches | lr 0.0000181 | ms/batch 2915.76 | loss 1.56544\n",
      "| epoch   3 |    20/22758 batches | lr 0.0000181 | ms/batch 2982.85 | loss 1.46674\n",
      "| epoch   3 |    21/22758 batches | lr 0.0000181 | ms/batch 2917.80 | loss 1.53745\n",
      "| epoch   3 |    22/22758 batches | lr 0.0000181 | ms/batch 2909.76 | loss 1.49237\n",
      "| epoch   3 |    23/22758 batches | lr 0.0000181 | ms/batch 2923.01 | loss 1.44497\n",
      "| epoch   3 |    24/22758 batches | lr 0.0000181 | ms/batch 2918.88 | loss 1.56439\n",
      "| epoch   3 |    25/22758 batches | lr 0.0000181 | ms/batch 2919.24 | loss 1.37758\n",
      "| epoch   3 |    26/22758 batches | lr 0.0000181 | ms/batch 2925.91 | loss 1.61742\n",
      "| epoch   3 |    27/22758 batches | lr 0.0000181 | ms/batch 2966.95 | loss 1.38901\n",
      "| epoch   3 |    28/22758 batches | lr 0.0000181 | ms/batch 2925.03 | loss 1.52983\n",
      "| epoch   3 |    29/22758 batches | lr 0.0000181 | ms/batch 2923.03 | loss 1.47961\n",
      "| epoch   3 |    30/22758 batches | lr 0.0000181 | ms/batch 2917.91 | loss 1.44704\n",
      "| epoch   3 |    31/22758 batches | lr 0.0000181 | ms/batch 2913.97 | loss 1.50570\n",
      "| epoch   3 |    32/22758 batches | lr 0.0000181 | ms/batch 2920.85 | loss 1.46198\n",
      "| epoch   3 |    33/22758 batches | lr 0.0000181 | ms/batch 2920.66 | loss 1.73434\n",
      "| epoch   3 |    34/22758 batches | lr 0.0000181 | ms/batch 2941.40 | loss 1.55391\n",
      "| epoch   3 |    35/22758 batches | lr 0.0000181 | ms/batch 2921.67 | loss 1.79627\n",
      "| epoch   3 |    36/22758 batches | lr 0.0000181 | ms/batch 2926.88 | loss 1.58538\n",
      "| epoch   3 |    37/22758 batches | lr 0.0000181 | ms/batch 2920.04 | loss 1.38304\n",
      "| epoch   3 |    38/22758 batches | lr 0.0000181 | ms/batch 2918.04 | loss 1.36146\n",
      "| epoch   3 |    39/22758 batches | lr 0.0000181 | ms/batch 2921.65 | loss 1.52950\n",
      "| epoch   3 |    40/22758 batches | lr 0.0000181 | ms/batch 2923.49 | loss 1.57919\n",
      "| epoch   3 |    41/22758 batches | lr 0.0000181 | ms/batch 2914.62 | loss 1.36009\n",
      "| epoch   3 |    42/22758 batches | lr 0.0000181 | ms/batch 2917.20 | loss 1.48048\n",
      "| epoch   3 |    43/22758 batches | lr 0.0000181 | ms/batch 2915.17 | loss 1.56418\n",
      "| epoch   3 |    44/22758 batches | lr 0.0000181 | ms/batch 2923.46 | loss 1.89011\n",
      "| epoch   3 |    45/22758 batches | lr 0.0000181 | ms/batch 2917.28 | loss 1.59274\n",
      "| epoch   3 |    46/22758 batches | lr 0.0000181 | ms/batch 2919.06 | loss 1.40377\n",
      "| epoch   3 |    47/22758 batches | lr 0.0000181 | ms/batch 2921.61 | loss 1.63858\n",
      "| epoch   3 |    48/22758 batches | lr 0.0000181 | ms/batch 2957.00 | loss 1.47718\n",
      "| epoch   3 |    49/22758 batches | lr 0.0000181 | ms/batch 2919.62 | loss 1.64984\n",
      "| epoch   3 |    50/22758 batches | lr 0.0000181 | ms/batch 2920.82 | loss 1.51310\n",
      "| epoch   3 |    51/22758 batches | lr 0.0000181 | ms/batch 2924.60 | loss 1.58750\n",
      "| epoch   3 |    52/22758 batches | lr 0.0000181 | ms/batch 2916.98 | loss 1.46619\n",
      "| epoch   3 |    53/22758 batches | lr 0.0000181 | ms/batch 2920.53 | loss 1.42657\n",
      "| epoch   3 |    54/22758 batches | lr 0.0000181 | ms/batch 2922.18 | loss 1.53785\n",
      "| epoch   3 |    55/22758 batches | lr 0.0000181 | ms/batch 2945.22 | loss 1.38951\n",
      "| epoch   3 |    56/22758 batches | lr 0.0000181 | ms/batch 2921.19 | loss 1.44011\n",
      "| epoch   3 |    57/22758 batches | lr 0.0000181 | ms/batch 2922.81 | loss 1.35530\n",
      "| epoch   3 |    58/22758 batches | lr 0.0000181 | ms/batch 2920.26 | loss 1.71748\n",
      "| epoch   3 |    59/22758 batches | lr 0.0000181 | ms/batch 2916.60 | loss 1.59301\n",
      "| epoch   3 |    60/22758 batches | lr 0.0000181 | ms/batch 2924.15 | loss 1.36594\n",
      "| epoch   3 |    61/22758 batches | lr 0.0000181 | ms/batch 2924.08 | loss 1.40677\n",
      "| epoch   3 |    62/22758 batches | lr 0.0000181 | ms/batch 2922.40 | loss 1.56663\n",
      "| epoch   3 |    63/22758 batches | lr 0.0000181 | ms/batch 2920.30 | loss 1.56858\n",
      "| epoch   3 |    64/22758 batches | lr 0.0000181 | ms/batch 2918.37 | loss 1.53524\n",
      "| epoch   3 |    65/22758 batches | lr 0.0000181 | ms/batch 2917.09 | loss 1.42833\n",
      "| epoch   3 |    66/22758 batches | lr 0.0000181 | ms/batch 2924.42 | loss 1.66907\n",
      "| epoch   3 |    67/22758 batches | lr 0.0000181 | ms/batch 2986.35 | loss 1.61507\n",
      "| epoch   3 |    68/22758 batches | lr 0.0000181 | ms/batch 2921.59 | loss 1.75089\n",
      "| epoch   3 |    69/22758 batches | lr 0.0000181 | ms/batch 2955.84 | loss 1.51126\n",
      "| epoch   3 |    70/22758 batches | lr 0.0000181 | ms/batch 2908.88 | loss 1.33859\n",
      "| epoch   3 |    71/22758 batches | lr 0.0000181 | ms/batch 2921.16 | loss 1.43512\n",
      "| epoch   3 |    72/22758 batches | lr 0.0000181 | ms/batch 2923.37 | loss 1.31086\n",
      "| epoch   3 |    73/22758 batches | lr 0.0000181 | ms/batch 2921.99 | loss 1.45546\n",
      "| epoch   3 |    74/22758 batches | lr 0.0000181 | ms/batch 2919.91 | loss 1.48639\n",
      "| epoch   3 |    75/22758 batches | lr 0.0000181 | ms/batch 2927.12 | loss 1.75483\n",
      "| epoch   3 |    76/22758 batches | lr 0.0000181 | ms/batch 2923.69 | loss 1.64053\n",
      "| epoch   3 |    77/22758 batches | lr 0.0000181 | ms/batch 2918.73 | loss 1.41932\n",
      "| epoch   3 |    78/22758 batches | lr 0.0000181 | ms/batch 2928.41 | loss 1.52940\n",
      "| epoch   3 |    79/22758 batches | lr 0.0000181 | ms/batch 2923.79 | loss 1.37159\n",
      "| epoch   3 |    80/22758 batches | lr 0.0000181 | ms/batch 2921.82 | loss 1.34167\n",
      "| epoch   3 |    81/22758 batches | lr 0.0000181 | ms/batch 2919.88 | loss 1.48419\n",
      "| epoch   3 |    82/22758 batches | lr 0.0000181 | ms/batch 2921.50 | loss 1.53377\n",
      "| epoch   3 |    83/22758 batches | lr 0.0000181 | ms/batch 2961.44 | loss 1.44180\n",
      "| epoch   3 |    84/22758 batches | lr 0.0000181 | ms/batch 2921.16 | loss 1.23676\n",
      "| epoch   3 |    85/22758 batches | lr 0.0000181 | ms/batch 2927.73 | loss 1.43824\n",
      "| epoch   3 |    86/22758 batches | lr 0.0000181 | ms/batch 2928.25 | loss 1.36435\n",
      "| epoch   3 |    87/22758 batches | lr 0.0000181 | ms/batch 2921.36 | loss 1.25808\n",
      "| epoch   3 |    88/22758 batches | lr 0.0000181 | ms/batch 2921.35 | loss 1.33136\n",
      "| epoch   3 |    89/22758 batches | lr 0.0000181 | ms/batch 2922.59 | loss 1.54166\n",
      "| epoch   3 |    90/22758 batches | lr 0.0000181 | ms/batch 2953.08 | loss 1.48986\n",
      "| epoch   3 |    91/22758 batches | lr 0.0000181 | ms/batch 2922.99 | loss 1.60105\n",
      "| epoch   3 |    92/22758 batches | lr 0.0000181 | ms/batch 2928.44 | loss 1.50791\n",
      "| epoch   3 |    93/22758 batches | lr 0.0000181 | ms/batch 2916.99 | loss 1.74217\n",
      "| epoch   3 |    94/22758 batches | lr 0.0000181 | ms/batch 2921.73 | loss 1.17630\n",
      "| epoch   3 |    95/22758 batches | lr 0.0000181 | ms/batch 2920.51 | loss 1.31468\n",
      "| epoch   3 |    96/22758 batches | lr 0.0000181 | ms/batch 2921.35 | loss 1.23870\n",
      "| epoch   3 |    97/22758 batches | lr 0.0000181 | ms/batch 2919.86 | loss 1.95631\n",
      "| epoch   3 |    98/22758 batches | lr 0.0000181 | ms/batch 2921.77 | loss 1.33808\n",
      "| epoch   3 |    99/22758 batches | lr 0.0000181 | ms/batch 2925.02 | loss 1.47596\n",
      "| epoch   3 |   100/22758 batches | lr 0.0000181 | ms/batch 2926.59 | loss 1.74982\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-07-17 12:39:04,186 - INFO - \n",
      " Scores:\n",
      "2023-07-17 12:39:04,237 - INFO - \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.14      0.40      0.21      1158\n",
      "           1       0.66      0.44      0.53     16938\n",
      "           2       0.70      0.06      0.12     14848\n",
      "           3       0.22      0.04      0.07      1713\n",
      "           4       0.04      0.54      0.08      1600\n",
      "\n",
      "    accuracy                           0.27     36257\n",
      "   macro avg       0.35      0.30      0.20     36257\n",
      "weighted avg       0.61      0.27      0.31     36257\n",
      "\n",
      "2023-07-17 12:39:04,238 - INFO - -----------------------------------------------------------------------------------------\n",
      "2023-07-17 12:39:04,239 - INFO - | end of epoch   3 | time: 1441.80s | valid loss 74.36805\n",
      "2023-07-17 12:39:04,240 - INFO - -----------------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   4 |     1/22758 batches | lr 0.0000171 | ms/batch 5832.39 | loss 2.93668\n",
      "| epoch   4 |     2/22758 batches | lr 0.0000171 | ms/batch 2917.87 | loss 1.31364\n",
      "| epoch   4 |     3/22758 batches | lr 0.0000171 | ms/batch 2919.47 | loss 1.47680\n",
      "| epoch   4 |     4/22758 batches | lr 0.0000171 | ms/batch 2915.70 | loss 1.43661\n",
      "| epoch   4 |     5/22758 batches | lr 0.0000171 | ms/batch 2929.09 | loss 1.74241\n",
      "| epoch   4 |     6/22758 batches | lr 0.0000171 | ms/batch 2915.97 | loss 1.20938\n",
      "| epoch   4 |     7/22758 batches | lr 0.0000171 | ms/batch 2920.95 | loss 1.22711\n",
      "| epoch   4 |     8/22758 batches | lr 0.0000171 | ms/batch 2918.13 | loss 1.40864\n",
      "| epoch   4 |     9/22758 batches | lr 0.0000171 | ms/batch 2919.30 | loss 1.91864\n",
      "| epoch   4 |    10/22758 batches | lr 0.0000171 | ms/batch 2921.78 | loss 1.45985\n",
      "| epoch   4 |    11/22758 batches | lr 0.0000171 | ms/batch 2920.68 | loss 1.19569\n",
      "| epoch   4 |    12/22758 batches | lr 0.0000171 | ms/batch 2949.60 | loss 1.33313\n",
      "| epoch   4 |    13/22758 batches | lr 0.0000171 | ms/batch 2920.15 | loss 1.31977\n",
      "| epoch   4 |    14/22758 batches | lr 0.0000171 | ms/batch 2922.63 | loss 1.53330\n",
      "| epoch   4 |    15/22758 batches | lr 0.0000171 | ms/batch 2917.14 | loss 1.09443\n",
      "| epoch   4 |    16/22758 batches | lr 0.0000171 | ms/batch 2922.24 | loss 1.48229\n",
      "| epoch   4 |    17/22758 batches | lr 0.0000171 | ms/batch 2920.16 | loss 1.63295\n",
      "| epoch   4 |    18/22758 batches | lr 0.0000171 | ms/batch 2917.00 | loss 1.37978\n",
      "| epoch   4 |    19/22758 batches | lr 0.0000171 | ms/batch 2942.14 | loss 1.84992\n",
      "| epoch   4 |    20/22758 batches | lr 0.0000171 | ms/batch 2915.09 | loss 1.49135\n",
      "| epoch   4 |    21/22758 batches | lr 0.0000171 | ms/batch 2921.89 | loss 1.57286\n",
      "| epoch   4 |    22/22758 batches | lr 0.0000171 | ms/batch 2920.35 | loss 1.23200\n",
      "| epoch   4 |    23/22758 batches | lr 0.0000171 | ms/batch 2920.17 | loss 1.73166\n",
      "| epoch   4 |    24/22758 batches | lr 0.0000171 | ms/batch 2917.94 | loss 1.47548\n",
      "| epoch   4 |    25/22758 batches | lr 0.0000171 | ms/batch 2924.51 | loss 1.47062\n",
      "| epoch   4 |    26/22758 batches | lr 0.0000171 | ms/batch 2921.55 | loss 1.59540\n",
      "| epoch   4 |    27/22758 batches | lr 0.0000171 | ms/batch 2923.57 | loss 1.35709\n",
      "| epoch   4 |    28/22758 batches | lr 0.0000171 | ms/batch 2913.29 | loss 1.40509\n",
      "| epoch   4 |    29/22758 batches | lr 0.0000171 | ms/batch 2921.69 | loss 1.24571\n",
      "| epoch   4 |    30/22758 batches | lr 0.0000171 | ms/batch 2926.65 | loss 1.50539\n",
      "| epoch   4 |    31/22758 batches | lr 0.0000171 | ms/batch 2919.60 | loss 1.27261\n",
      "| epoch   4 |    32/22758 batches | lr 0.0000171 | ms/batch 2923.63 | loss 1.52196\n",
      "| epoch   4 |    33/22758 batches | lr 0.0000171 | ms/batch 2922.52 | loss 1.51880\n",
      "| epoch   4 |    34/22758 batches | lr 0.0000171 | ms/batch 2923.25 | loss 1.51340\n",
      "| epoch   4 |    35/22758 batches | lr 0.0000171 | ms/batch 2921.18 | loss 1.47755\n",
      "| epoch   4 |    36/22758 batches | lr 0.0000171 | ms/batch 2920.45 | loss 1.31996\n",
      "| epoch   4 |    37/22758 batches | lr 0.0000171 | ms/batch 2920.68 | loss 1.40113\n",
      "| epoch   4 |    38/22758 batches | lr 0.0000171 | ms/batch 2925.84 | loss 1.17085\n",
      "| epoch   4 |    39/22758 batches | lr 0.0000171 | ms/batch 2920.65 | loss 1.17866\n",
      "| epoch   4 |    40/22758 batches | lr 0.0000171 | ms/batch 2920.23 | loss 0.93130\n",
      "| epoch   4 |    41/22758 batches | lr 0.0000171 | ms/batch 2946.65 | loss 1.40835\n",
      "| epoch   4 |    42/22758 batches | lr 0.0000171 | ms/batch 2927.72 | loss 1.30336\n",
      "| epoch   4 |    43/22758 batches | lr 0.0000171 | ms/batch 2923.47 | loss 1.14329\n",
      "| epoch   4 |    44/22758 batches | lr 0.0000171 | ms/batch 2921.05 | loss 1.32864\n",
      "| epoch   4 |    45/22758 batches | lr 0.0000171 | ms/batch 2925.47 | loss 1.38285\n",
      "| epoch   4 |    46/22758 batches | lr 0.0000171 | ms/batch 2919.76 | loss 1.66575\n",
      "| epoch   4 |    47/22758 batches | lr 0.0000171 | ms/batch 2927.36 | loss 1.34547\n",
      "| epoch   4 |    48/22758 batches | lr 0.0000171 | ms/batch 2994.27 | loss 1.09071\n",
      "| epoch   4 |    49/22758 batches | lr 0.0000171 | ms/batch 2916.73 | loss 1.96101\n",
      "| epoch   4 |    50/22758 batches | lr 0.0000171 | ms/batch 2923.16 | loss 1.50231\n",
      "| epoch   4 |    51/22758 batches | lr 0.0000171 | ms/batch 2907.84 | loss 1.24203\n",
      "| epoch   4 |    52/22758 batches | lr 0.0000171 | ms/batch 2911.40 | loss 1.26960\n",
      "| epoch   4 |    53/22758 batches | lr 0.0000171 | ms/batch 2911.00 | loss 1.79891\n",
      "| epoch   4 |    54/22758 batches | lr 0.0000171 | ms/batch 2909.77 | loss 1.54049\n",
      "| epoch   4 |    55/22758 batches | lr 0.0000171 | ms/batch 2918.12 | loss 1.45295\n",
      "| epoch   4 |    56/22758 batches | lr 0.0000171 | ms/batch 2921.36 | loss 1.41038\n",
      "| epoch   4 |    57/22758 batches | lr 0.0000171 | ms/batch 2919.25 | loss 1.47005\n",
      "| epoch   4 |    58/22758 batches | lr 0.0000171 | ms/batch 2923.26 | loss 1.17115\n",
      "| epoch   4 |    59/22758 batches | lr 0.0000171 | ms/batch 2925.01 | loss 1.77830\n",
      "| epoch   4 |    60/22758 batches | lr 0.0000171 | ms/batch 2925.95 | loss 1.55844\n",
      "| epoch   4 |    61/22758 batches | lr 0.0000171 | ms/batch 2917.44 | loss 1.74334\n",
      "| epoch   4 |    62/22758 batches | lr 0.0000171 | ms/batch 2919.23 | loss 1.32384\n",
      "| epoch   4 |    63/22758 batches | lr 0.0000171 | ms/batch 2929.15 | loss 1.32947\n",
      "| epoch   4 |    64/22758 batches | lr 0.0000171 | ms/batch 2920.57 | loss 1.26632\n",
      "| epoch   4 |    65/22758 batches | lr 0.0000171 | ms/batch 2921.38 | loss 1.68187\n",
      "| epoch   4 |    66/22758 batches | lr 0.0000171 | ms/batch 2927.88 | loss 1.37645\n",
      "| epoch   4 |    67/22758 batches | lr 0.0000171 | ms/batch 2915.26 | loss 1.38370\n",
      "| epoch   4 |    68/22758 batches | lr 0.0000171 | ms/batch 2907.53 | loss 1.28164\n",
      "| epoch   4 |    69/22758 batches | lr 0.0000171 | ms/batch 2957.94 | loss 1.26011\n",
      "| epoch   4 |    70/22758 batches | lr 0.0000171 | ms/batch 2911.08 | loss 1.47631\n",
      "| epoch   4 |    71/22758 batches | lr 0.0000171 | ms/batch 2907.94 | loss 1.62840\n",
      "| epoch   4 |    72/22758 batches | lr 0.0000171 | ms/batch 2912.30 | loss 1.69686\n",
      "| epoch   4 |    73/22758 batches | lr 0.0000171 | ms/batch 2907.04 | loss 1.15752\n",
      "| epoch   4 |    74/22758 batches | lr 0.0000171 | ms/batch 2919.76 | loss 1.28067\n",
      "| epoch   4 |    75/22758 batches | lr 0.0000171 | ms/batch 2930.38 | loss 1.14535\n",
      "| epoch   4 |    76/22758 batches | lr 0.0000171 | ms/batch 2925.65 | loss 1.44986\n",
      "| epoch   4 |    77/22758 batches | lr 0.0000171 | ms/batch 2923.27 | loss 1.34310\n",
      "| epoch   4 |    78/22758 batches | lr 0.0000171 | ms/batch 2926.37 | loss 1.56565\n",
      "| epoch   4 |    79/22758 batches | lr 0.0000171 | ms/batch 2924.47 | loss 1.27203\n",
      "| epoch   4 |    80/22758 batches | lr 0.0000171 | ms/batch 2926.45 | loss 1.49209\n",
      "| epoch   4 |    81/22758 batches | lr 0.0000171 | ms/batch 2911.53 | loss 1.27312\n",
      "| epoch   4 |    82/22758 batches | lr 0.0000171 | ms/batch 2911.68 | loss 1.38519\n",
      "| epoch   4 |    83/22758 batches | lr 0.0000171 | ms/batch 2909.82 | loss 2.28596\n",
      "| epoch   4 |    84/22758 batches | lr 0.0000171 | ms/batch 2937.72 | loss 1.80830\n",
      "| epoch   4 |    85/22758 batches | lr 0.0000171 | ms/batch 2931.72 | loss 1.72624\n",
      "| epoch   4 |    86/22758 batches | lr 0.0000171 | ms/batch 2919.27 | loss 1.25959\n",
      "| epoch   4 |    87/22758 batches | lr 0.0000171 | ms/batch 2909.85 | loss 1.18038\n",
      "| epoch   4 |    88/22758 batches | lr 0.0000171 | ms/batch 2926.00 | loss 1.53608\n",
      "| epoch   4 |    89/22758 batches | lr 0.0000171 | ms/batch 2922.93 | loss 1.64450\n",
      "| epoch   4 |    90/22758 batches | lr 0.0000171 | ms/batch 2916.07 | loss 1.21613\n",
      "| epoch   4 |    91/22758 batches | lr 0.0000171 | ms/batch 2951.69 | loss 1.42160\n",
      "| epoch   4 |    92/22758 batches | lr 0.0000171 | ms/batch 2908.89 | loss 1.72600\n",
      "| epoch   4 |    93/22758 batches | lr 0.0000171 | ms/batch 2914.77 | loss 1.72807\n",
      "| epoch   4 |    94/22758 batches | lr 0.0000171 | ms/batch 2920.43 | loss 1.61077\n",
      "| epoch   4 |    95/22758 batches | lr 0.0000171 | ms/batch 2923.29 | loss 1.35118\n",
      "| epoch   4 |    96/22758 batches | lr 0.0000171 | ms/batch 2920.73 | loss 1.41680\n",
      "| epoch   4 |    97/22758 batches | lr 0.0000171 | ms/batch 2921.98 | loss 1.40574\n",
      "| epoch   4 |    98/22758 batches | lr 0.0000171 | ms/batch 2973.27 | loss 1.38036\n",
      "| epoch   4 |    99/22758 batches | lr 0.0000171 | ms/batch 2911.84 | loss 1.30426\n",
      "| epoch   4 |   100/22758 batches | lr 0.0000171 | ms/batch 2909.27 | loss 1.41998\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-07-17 13:03:11,414 - INFO - \n",
      " Scores:\n",
      "2023-07-17 13:03:11,468 - INFO - \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.08      0.81      0.15      1158\n",
      "           1       0.51      0.43      0.47     16938\n",
      "           2       0.71      0.45      0.55     14848\n",
      "           3       0.39      0.10      0.16      1713\n",
      "           4       0.09      0.04      0.06      1600\n",
      "\n",
      "    accuracy                           0.42     36257\n",
      "   macro avg       0.36      0.37      0.28     36257\n",
      "weighted avg       0.55      0.42      0.46     36257\n",
      "\n",
      "2023-07-17 13:03:11,469 - INFO - -----------------------------------------------------------------------------------------\n",
      "2023-07-17 13:03:11,470 - INFO - | end of epoch   4 | time: 1435.76s | valid loss 73.16209\n",
      "2023-07-17 13:03:11,471 - INFO - -----------------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   5 |     1/22758 batches | lr 0.0000163 | ms/batch 5862.14 | loss 2.99125\n",
      "| epoch   5 |     2/22758 batches | lr 0.0000163 | ms/batch 2920.59 | loss 1.31630\n",
      "| epoch   5 |     3/22758 batches | lr 0.0000163 | ms/batch 2941.11 | loss 1.45659\n",
      "| epoch   5 |     4/22758 batches | lr 0.0000163 | ms/batch 2923.52 | loss 1.33540\n",
      "| epoch   5 |     5/22758 batches | lr 0.0000163 | ms/batch 2919.42 | loss 1.64563\n",
      "| epoch   5 |     6/22758 batches | lr 0.0000163 | ms/batch 2918.11 | loss 1.48746\n",
      "| epoch   5 |     7/22758 batches | lr 0.0000163 | ms/batch 2910.01 | loss 1.87122\n",
      "| epoch   5 |     8/22758 batches | lr 0.0000163 | ms/batch 2907.34 | loss 1.30860\n",
      "| epoch   5 |     9/22758 batches | lr 0.0000163 | ms/batch 2911.46 | loss 1.25273\n",
      "| epoch   5 |    10/22758 batches | lr 0.0000163 | ms/batch 2946.29 | loss 1.36217\n",
      "| epoch   5 |    11/22758 batches | lr 0.0000163 | ms/batch 2923.79 | loss 1.50565\n",
      "| epoch   5 |    12/22758 batches | lr 0.0000163 | ms/batch 2908.73 | loss 2.03124\n",
      "| epoch   5 |    13/22758 batches | lr 0.0000163 | ms/batch 2920.81 | loss 1.19807\n",
      "| epoch   5 |    14/22758 batches | lr 0.0000163 | ms/batch 2919.21 | loss 1.34160\n",
      "| epoch   5 |    15/22758 batches | lr 0.0000163 | ms/batch 2919.56 | loss 1.37561\n",
      "| epoch   5 |    16/22758 batches | lr 0.0000163 | ms/batch 2921.67 | loss 1.38781\n",
      "| epoch   5 |    17/22758 batches | lr 0.0000163 | ms/batch 3003.52 | loss 1.31474\n",
      "| epoch   5 |    18/22758 batches | lr 0.0000163 | ms/batch 2922.39 | loss 1.53334\n",
      "| epoch   5 |    19/22758 batches | lr 0.0000163 | ms/batch 2922.02 | loss 1.42169\n",
      "| epoch   5 |    20/22758 batches | lr 0.0000163 | ms/batch 2926.16 | loss 1.45779\n",
      "| epoch   5 |    21/22758 batches | lr 0.0000163 | ms/batch 2918.93 | loss 1.12448\n",
      "| epoch   5 |    22/22758 batches | lr 0.0000163 | ms/batch 2921.49 | loss 1.58350\n",
      "| epoch   5 |    23/22758 batches | lr 0.0000163 | ms/batch 2923.17 | loss 1.40725\n",
      "| epoch   5 |    24/22758 batches | lr 0.0000163 | ms/batch 2929.92 | loss 1.19743\n",
      "| epoch   5 |    25/22758 batches | lr 0.0000163 | ms/batch 2917.94 | loss 1.42588\n",
      "| epoch   5 |    26/22758 batches | lr 0.0000163 | ms/batch 2909.84 | loss 1.42738\n",
      "| epoch   5 |    27/22758 batches | lr 0.0000163 | ms/batch 2922.05 | loss 1.58803\n",
      "| epoch   5 |    28/22758 batches | lr 0.0000163 | ms/batch 2921.17 | loss 1.08896\n",
      "| epoch   5 |    29/22758 batches | lr 0.0000163 | ms/batch 2931.04 | loss 1.09464\n",
      "| epoch   5 |    30/22758 batches | lr 0.0000163 | ms/batch 2932.67 | loss 1.20374\n",
      "| epoch   5 |    31/22758 batches | lr 0.0000163 | ms/batch 2975.34 | loss 1.49497\n",
      "| epoch   5 |    32/22758 batches | lr 0.0000163 | ms/batch 2923.47 | loss 1.20651\n",
      "| epoch   5 |    33/22758 batches | lr 0.0000163 | ms/batch 2939.63 | loss 1.31421\n",
      "| epoch   5 |    34/22758 batches | lr 0.0000163 | ms/batch 3119.63 | loss 1.61761\n",
      "| epoch   5 |    35/22758 batches | lr 0.0000163 | ms/batch 3071.09 | loss 1.13487\n",
      "| epoch   5 |    36/22758 batches | lr 0.0000163 | ms/batch 3111.15 | loss 1.21461\n",
      "| epoch   5 |    37/22758 batches | lr 0.0000163 | ms/batch 2993.52 | loss 1.73146\n",
      "| epoch   5 |    38/22758 batches | lr 0.0000163 | ms/batch 2921.82 | loss 1.31364\n",
      "| epoch   5 |    39/22758 batches | lr 0.0000163 | ms/batch 2926.41 | loss 1.13787\n",
      "| epoch   5 |    40/22758 batches | lr 0.0000163 | ms/batch 2917.09 | loss 1.48516\n",
      "| epoch   5 |    41/22758 batches | lr 0.0000163 | ms/batch 2924.72 | loss 1.39676\n",
      "| epoch   5 |    42/22758 batches | lr 0.0000163 | ms/batch 2920.40 | loss 1.43286\n",
      "| epoch   5 |    43/22758 batches | lr 0.0000163 | ms/batch 2922.80 | loss 1.32992\n",
      "| epoch   5 |    44/22758 batches | lr 0.0000163 | ms/batch 2934.13 | loss 1.42850\n",
      "| epoch   5 |    45/22758 batches | lr 0.0000163 | ms/batch 2927.69 | loss 1.44816\n",
      "| epoch   5 |    46/22758 batches | lr 0.0000163 | ms/batch 2921.83 | loss 1.45370\n",
      "| epoch   5 |    47/22758 batches | lr 0.0000163 | ms/batch 2920.17 | loss 1.33557\n",
      "| epoch   5 |    48/22758 batches | lr 0.0000163 | ms/batch 2925.59 | loss 1.29109\n",
      "| epoch   5 |    49/22758 batches | lr 0.0000163 | ms/batch 2926.04 | loss 1.25750\n",
      "| epoch   5 |    50/22758 batches | lr 0.0000163 | ms/batch 2923.31 | loss 1.10669\n",
      "| epoch   5 |    51/22758 batches | lr 0.0000163 | ms/batch 2923.01 | loss 1.02926\n",
      "| epoch   5 |    52/22758 batches | lr 0.0000163 | ms/batch 2926.35 | loss 1.19544\n",
      "| epoch   5 |    53/22758 batches | lr 0.0000163 | ms/batch 2918.41 | loss 1.40770\n",
      "| epoch   5 |    54/22758 batches | lr 0.0000163 | ms/batch 2931.91 | loss 1.96455\n",
      "| epoch   5 |    55/22758 batches | lr 0.0000163 | ms/batch 2922.92 | loss 1.38217\n",
      "| epoch   5 |    56/22758 batches | lr 0.0000163 | ms/batch 2956.49 | loss 1.66042\n",
      "| epoch   5 |    57/22758 batches | lr 0.0000163 | ms/batch 2919.87 | loss 1.55089\n",
      "| epoch   5 |    58/22758 batches | lr 0.0000163 | ms/batch 2922.01 | loss 1.16514\n",
      "| epoch   5 |    59/22758 batches | lr 0.0000163 | ms/batch 2920.13 | loss 0.90330\n",
      "| epoch   5 |    60/22758 batches | lr 0.0000163 | ms/batch 2925.28 | loss 1.25688\n",
      "| epoch   5 |    61/22758 batches | lr 0.0000163 | ms/batch 2920.58 | loss 1.37559\n",
      "| epoch   5 |    62/22758 batches | lr 0.0000163 | ms/batch 2925.69 | loss 1.46221\n",
      "| epoch   5 |    63/22758 batches | lr 0.0000163 | ms/batch 2941.41 | loss 1.32887\n",
      "| epoch   5 |    64/22758 batches | lr 0.0000163 | ms/batch 2940.08 | loss 1.26864\n",
      "| epoch   5 |    65/22758 batches | lr 0.0000163 | ms/batch 2925.80 | loss 1.59950\n",
      "| epoch   5 |    66/22758 batches | lr 0.0000163 | ms/batch 2921.35 | loss 1.26610\n",
      "| epoch   5 |    67/22758 batches | lr 0.0000163 | ms/batch 2922.50 | loss 1.19837\n",
      "| epoch   5 |    68/22758 batches | lr 0.0000163 | ms/batch 2925.08 | loss 1.06263\n",
      "| epoch   5 |    69/22758 batches | lr 0.0000163 | ms/batch 2927.97 | loss 1.32456\n",
      "| epoch   5 |    70/22758 batches | lr 0.0000163 | ms/batch 2981.59 | loss 1.55633\n",
      "| epoch   5 |    71/22758 batches | lr 0.0000163 | ms/batch 2923.71 | loss 1.76751\n",
      "| epoch   5 |    72/22758 batches | lr 0.0000163 | ms/batch 2922.01 | loss 1.76984\n",
      "| epoch   5 |    73/22758 batches | lr 0.0000163 | ms/batch 2921.95 | loss 2.17559\n",
      "| epoch   5 |    74/22758 batches | lr 0.0000163 | ms/batch 2925.71 | loss 1.18157\n",
      "| epoch   5 |    75/22758 batches | lr 0.0000163 | ms/batch 2919.62 | loss 1.55027\n",
      "| epoch   5 |    76/22758 batches | lr 0.0000163 | ms/batch 2921.61 | loss 0.97803\n",
      "| epoch   5 |    77/22758 batches | lr 0.0000163 | ms/batch 2920.62 | loss 1.27605\n",
      "| epoch   5 |    78/22758 batches | lr 0.0000163 | ms/batch 2915.89 | loss 1.29518\n",
      "| epoch   5 |    79/22758 batches | lr 0.0000163 | ms/batch 2920.56 | loss 1.23428\n",
      "| epoch   5 |    80/22758 batches | lr 0.0000163 | ms/batch 2921.18 | loss 1.32375\n",
      "| epoch   5 |    81/22758 batches | lr 0.0000163 | ms/batch 2926.57 | loss 1.46392\n",
      "| epoch   5 |    82/22758 batches | lr 0.0000163 | ms/batch 2920.97 | loss 1.34917\n",
      "| epoch   5 |    83/22758 batches | lr 0.0000163 | ms/batch 2915.97 | loss 1.37747\n",
      "| epoch   5 |    84/22758 batches | lr 0.0000163 | ms/batch 2958.75 | loss 1.70304\n",
      "| epoch   5 |    85/22758 batches | lr 0.0000163 | ms/batch 2919.06 | loss 1.35377\n",
      "| epoch   5 |    86/22758 batches | lr 0.0000163 | ms/batch 2919.86 | loss 1.57142\n",
      "| epoch   5 |    87/22758 batches | lr 0.0000163 | ms/batch 2920.30 | loss 1.44438\n",
      "| epoch   5 |    88/22758 batches | lr 0.0000163 | ms/batch 2924.71 | loss 1.11619\n",
      "| epoch   5 |    89/22758 batches | lr 0.0000163 | ms/batch 2920.84 | loss 0.82509\n",
      "| epoch   5 |    90/22758 batches | lr 0.0000163 | ms/batch 2918.59 | loss 1.51732\n",
      "| epoch   5 |    91/22758 batches | lr 0.0000163 | ms/batch 2936.31 | loss 1.47233\n",
      "| epoch   5 |    92/22758 batches | lr 0.0000163 | ms/batch 2919.90 | loss 1.09301\n",
      "| epoch   5 |    93/22758 batches | lr 0.0000163 | ms/batch 2917.66 | loss 1.54679\n",
      "| epoch   5 |    94/22758 batches | lr 0.0000163 | ms/batch 2917.05 | loss 1.30666\n",
      "| epoch   5 |    95/22758 batches | lr 0.0000163 | ms/batch 2923.08 | loss 1.49748\n",
      "| epoch   5 |    96/22758 batches | lr 0.0000163 | ms/batch 2920.99 | loss 1.04174\n",
      "| epoch   5 |    97/22758 batches | lr 0.0000163 | ms/batch 2923.65 | loss 1.32744\n",
      "| epoch   5 |    98/22758 batches | lr 0.0000163 | ms/batch 2928.75 | loss 1.81726\n",
      "| epoch   5 |    99/22758 batches | lr 0.0000163 | ms/batch 2923.55 | loss 1.52395\n",
      "| epoch   5 |   100/22758 batches | lr 0.0000163 | ms/batch 2913.78 | loss 1.32975\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-07-17 13:27:18,116 - INFO - \n",
      " Scores:\n",
      "2023-07-17 13:27:18,178 - INFO - \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.14      0.44      0.21      1158\n",
      "           1       0.68      0.51      0.58     16938\n",
      "           2       0.62      0.56      0.59     14848\n",
      "           3       0.12      0.34      0.18      1713\n",
      "           4       0.04      0.04      0.04      1600\n",
      "\n",
      "    accuracy                           0.50     36257\n",
      "   macro avg       0.32      0.38      0.32     36257\n",
      "weighted avg       0.59      0.50      0.53     36257\n",
      "\n",
      "2023-07-17 13:27:18,179 - INFO - -----------------------------------------------------------------------------------------\n",
      "2023-07-17 13:27:18,180 - INFO - | end of epoch   5 | time: 1439.19s | valid loss 72.45588\n",
      "2023-07-17 13:27:18,181 - INFO - -----------------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   6 |     1/22758 batches | lr 0.0000155 | ms/batch 5856.61 | loss 3.16991\n",
      "| epoch   6 |     2/22758 batches | lr 0.0000155 | ms/batch 2920.38 | loss 1.28221\n",
      "| epoch   6 |     3/22758 batches | lr 0.0000155 | ms/batch 2921.18 | loss 1.10255\n",
      "| epoch   6 |     4/22758 batches | lr 0.0000155 | ms/batch 2917.20 | loss 1.85213\n",
      "| epoch   6 |     5/22758 batches | lr 0.0000155 | ms/batch 2922.84 | loss 1.35475\n",
      "| epoch   6 |     6/22758 batches | lr 0.0000155 | ms/batch 2990.14 | loss 1.16973\n",
      "| epoch   6 |     7/22758 batches | lr 0.0000155 | ms/batch 2916.05 | loss 1.42471\n",
      "| epoch   6 |     8/22758 batches | lr 0.0000155 | ms/batch 2924.55 | loss 1.54204\n",
      "| epoch   6 |     9/22758 batches | lr 0.0000155 | ms/batch 2924.57 | loss 1.36597\n",
      "| epoch   6 |    10/22758 batches | lr 0.0000155 | ms/batch 2922.74 | loss 1.50469\n",
      "| epoch   6 |    11/22758 batches | lr 0.0000155 | ms/batch 2929.85 | loss 1.52680\n",
      "| epoch   6 |    12/22758 batches | lr 0.0000155 | ms/batch 2920.41 | loss 1.52535\n",
      "| epoch   6 |    13/22758 batches | lr 0.0000155 | ms/batch 2930.49 | loss 1.02867\n",
      "| epoch   6 |    14/22758 batches | lr 0.0000155 | ms/batch 2928.46 | loss 1.49209\n",
      "| epoch   6 |    15/22758 batches | lr 0.0000155 | ms/batch 2922.90 | loss 1.47624\n",
      "| epoch   6 |    16/22758 batches | lr 0.0000155 | ms/batch 2917.35 | loss 1.08706\n",
      "| epoch   6 |    17/22758 batches | lr 0.0000155 | ms/batch 2918.43 | loss 1.08123\n",
      "| epoch   6 |    18/22758 batches | lr 0.0000155 | ms/batch 2929.35 | loss 1.54940\n",
      "| epoch   6 |    19/22758 batches | lr 0.0000155 | ms/batch 2925.52 | loss 1.63543\n",
      "| epoch   6 |    20/22758 batches | lr 0.0000155 | ms/batch 2943.82 | loss 1.45291\n",
      "| epoch   6 |    21/22758 batches | lr 0.0000155 | ms/batch 2921.43 | loss 1.22282\n",
      "| epoch   6 |    22/22758 batches | lr 0.0000155 | ms/batch 2922.36 | loss 1.71436\n",
      "| epoch   6 |    23/22758 batches | lr 0.0000155 | ms/batch 2920.37 | loss 1.39505\n",
      "| epoch   6 |    24/22758 batches | lr 0.0000155 | ms/batch 2918.34 | loss 1.41192\n",
      "| epoch   6 |    25/22758 batches | lr 0.0000155 | ms/batch 2919.59 | loss 1.54430\n",
      "| epoch   6 |    26/22758 batches | lr 0.0000155 | ms/batch 2922.44 | loss 1.52547\n",
      "| epoch   6 |    27/22758 batches | lr 0.0000155 | ms/batch 2973.25 | loss 1.55850\n",
      "| epoch   6 |    28/22758 batches | lr 0.0000155 | ms/batch 2916.47 | loss 1.28802\n",
      "| epoch   6 |    29/22758 batches | lr 0.0000155 | ms/batch 2920.95 | loss 1.17560\n",
      "| epoch   6 |    30/22758 batches | lr 0.0000155 | ms/batch 2925.52 | loss 1.68484\n",
      "| epoch   6 |    31/22758 batches | lr 0.0000155 | ms/batch 2919.28 | loss 1.44903\n",
      "| epoch   6 |    32/22758 batches | lr 0.0000155 | ms/batch 2923.30 | loss 1.35905\n",
      "| epoch   6 |    33/22758 batches | lr 0.0000155 | ms/batch 2927.66 | loss 1.60062\n",
      "| epoch   6 |    34/22758 batches | lr 0.0000155 | ms/batch 2931.16 | loss 1.55922\n",
      "| epoch   6 |    35/22758 batches | lr 0.0000155 | ms/batch 2924.53 | loss 0.90345\n",
      "| epoch   6 |    36/22758 batches | lr 0.0000155 | ms/batch 2923.74 | loss 1.11865\n",
      "| epoch   6 |    37/22758 batches | lr 0.0000155 | ms/batch 2920.29 | loss 1.42344\n",
      "| epoch   6 |    38/22758 batches | lr 0.0000155 | ms/batch 2929.68 | loss 1.03705\n",
      "| epoch   6 |    39/22758 batches | lr 0.0000155 | ms/batch 2924.99 | loss 1.56138\n",
      "| epoch   6 |    40/22758 batches | lr 0.0000155 | ms/batch 2923.77 | loss 1.27949\n",
      "| epoch   6 |    41/22758 batches | lr 0.0000155 | ms/batch 2920.83 | loss 1.64647\n",
      "| epoch   6 |    42/22758 batches | lr 0.0000155 | ms/batch 2920.50 | loss 1.36091\n",
      "| epoch   6 |    43/22758 batches | lr 0.0000155 | ms/batch 2920.08 | loss 1.86836\n",
      "| epoch   6 |    44/22758 batches | lr 0.0000155 | ms/batch 2919.76 | loss 1.05390\n",
      "| epoch   6 |    45/22758 batches | lr 0.0000155 | ms/batch 2922.74 | loss 1.48021\n",
      "| epoch   6 |    46/22758 batches | lr 0.0000155 | ms/batch 2938.78 | loss 1.21596\n",
      "| epoch   6 |    47/22758 batches | lr 0.0000155 | ms/batch 2916.78 | loss 1.34944\n",
      "| epoch   6 |    48/22758 batches | lr 0.0000155 | ms/batch 2984.30 | loss 1.59613\n",
      "| epoch   6 |    49/22758 batches | lr 0.0000155 | ms/batch 2920.94 | loss 1.41789\n",
      "| epoch   6 |    50/22758 batches | lr 0.0000155 | ms/batch 2927.22 | loss 1.35702\n",
      "| epoch   6 |    51/22758 batches | lr 0.0000155 | ms/batch 2925.37 | loss 1.34134\n",
      "| epoch   6 |    52/22758 batches | lr 0.0000155 | ms/batch 2925.57 | loss 1.62879\n",
      "| epoch   6 |    53/22758 batches | lr 0.0000155 | ms/batch 2917.82 | loss 1.20003\n",
      "| epoch   6 |    54/22758 batches | lr 0.0000155 | ms/batch 2918.59 | loss 1.68179\n",
      "| epoch   6 |    55/22758 batches | lr 0.0000155 | ms/batch 2958.86 | loss 0.82791\n",
      "| epoch   6 |    56/22758 batches | lr 0.0000155 | ms/batch 2919.93 | loss 1.13235\n",
      "| epoch   6 |    57/22758 batches | lr 0.0000155 | ms/batch 2919.13 | loss 1.86061\n",
      "| epoch   6 |    58/22758 batches | lr 0.0000155 | ms/batch 2923.36 | loss 1.58422\n",
      "| epoch   6 |    59/22758 batches | lr 0.0000155 | ms/batch 2912.69 | loss 1.48352\n",
      "| epoch   6 |    60/22758 batches | lr 0.0000155 | ms/batch 2918.81 | loss 1.42208\n",
      "| epoch   6 |    61/22758 batches | lr 0.0000155 | ms/batch 2922.10 | loss 1.13135\n",
      "| epoch   6 |    62/22758 batches | lr 0.0000155 | ms/batch 2938.18 | loss 1.41873\n",
      "| epoch   6 |    63/22758 batches | lr 0.0000155 | ms/batch 2918.69 | loss 1.27204\n",
      "| epoch   6 |    64/22758 batches | lr 0.0000155 | ms/batch 2920.35 | loss 1.28469\n",
      "| epoch   6 |    65/22758 batches | lr 0.0000155 | ms/batch 2917.27 | loss 1.76218\n",
      "| epoch   6 |    66/22758 batches | lr 0.0000155 | ms/batch 2919.78 | loss 1.05159\n",
      "| epoch   6 |    67/22758 batches | lr 0.0000155 | ms/batch 2920.24 | loss 1.59510\n",
      "| epoch   6 |    68/22758 batches | lr 0.0000155 | ms/batch 2918.86 | loss 1.28076\n",
      "| epoch   6 |    69/22758 batches | lr 0.0000155 | ms/batch 2947.74 | loss 1.23925\n",
      "| epoch   6 |    70/22758 batches | lr 0.0000155 | ms/batch 2918.06 | loss 1.75683\n",
      "| epoch   6 |    71/22758 batches | lr 0.0000155 | ms/batch 2923.12 | loss 1.10647\n",
      "| epoch   6 |    72/22758 batches | lr 0.0000155 | ms/batch 2921.41 | loss 1.12044\n",
      "| epoch   6 |    73/22758 batches | lr 0.0000155 | ms/batch 2918.70 | loss 1.44376\n",
      "| epoch   6 |    74/22758 batches | lr 0.0000155 | ms/batch 2922.81 | loss 1.04504\n",
      "| epoch   6 |    75/22758 batches | lr 0.0000155 | ms/batch 2919.47 | loss 1.59700\n",
      "| epoch   6 |    76/22758 batches | lr 0.0000155 | ms/batch 2992.36 | loss 1.01195\n",
      "| epoch   6 |    77/22758 batches | lr 0.0000155 | ms/batch 2917.94 | loss 1.60772\n",
      "| epoch   6 |    78/22758 batches | lr 0.0000155 | ms/batch 2921.71 | loss 2.08444\n",
      "| epoch   6 |    79/22758 batches | lr 0.0000155 | ms/batch 2924.62 | loss 1.99368\n",
      "| epoch   6 |    80/22758 batches | lr 0.0000155 | ms/batch 2928.16 | loss 1.72844\n",
      "| epoch   6 |    81/22758 batches | lr 0.0000155 | ms/batch 2923.57 | loss 2.10838\n",
      "| epoch   6 |    82/22758 batches | lr 0.0000155 | ms/batch 2920.11 | loss 1.54794\n",
      "| epoch   6 |    83/22758 batches | lr 0.0000155 | ms/batch 2943.74 | loss 1.25138\n",
      "| epoch   6 |    84/22758 batches | lr 0.0000155 | ms/batch 2921.69 | loss 1.32952\n",
      "| epoch   6 |    85/22758 batches | lr 0.0000155 | ms/batch 2919.17 | loss 1.44523\n",
      "| epoch   6 |    86/22758 batches | lr 0.0000155 | ms/batch 2925.70 | loss 1.20392\n",
      "| epoch   6 |    87/22758 batches | lr 0.0000155 | ms/batch 2923.97 | loss 1.56335\n",
      "| epoch   6 |    88/22758 batches | lr 0.0000155 | ms/batch 2923.05 | loss 1.32565\n",
      "| epoch   6 |    89/22758 batches | lr 0.0000155 | ms/batch 2928.03 | loss 1.43011\n",
      "| epoch   6 |    90/22758 batches | lr 0.0000155 | ms/batch 2925.00 | loss 1.28246\n",
      "| epoch   6 |    91/22758 batches | lr 0.0000155 | ms/batch 2922.27 | loss 1.43664\n",
      "| epoch   6 |    92/22758 batches | lr 0.0000155 | ms/batch 2924.73 | loss 1.20410\n",
      "| epoch   6 |    93/22758 batches | lr 0.0000155 | ms/batch 2921.29 | loss 1.54588\n",
      "| epoch   6 |    94/22758 batches | lr 0.0000155 | ms/batch 2922.27 | loss 1.61032\n",
      "| epoch   6 |    95/22758 batches | lr 0.0000155 | ms/batch 2924.70 | loss 1.09404\n",
      "| epoch   6 |    96/22758 batches | lr 0.0000155 | ms/batch 2996.52 | loss 1.13079\n",
      "| epoch   6 |    97/22758 batches | lr 0.0000155 | ms/batch 2919.36 | loss 1.82930\n",
      "| epoch   6 |    98/22758 batches | lr 0.0000155 | ms/batch 2921.52 | loss 1.42868\n",
      "| epoch   6 |    99/22758 batches | lr 0.0000155 | ms/batch 2919.42 | loss 1.30950\n",
      "| epoch   6 |   100/22758 batches | lr 0.0000155 | ms/batch 2918.86 | loss 1.27829\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-07-17 13:51:30,961 - INFO - \n",
      " Scores:\n",
      "2023-07-17 13:51:31,017 - INFO - \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.13      0.50      0.21      1158\n",
      "           1       0.72      0.42      0.53     16938\n",
      "           2       0.75      0.48      0.59     14848\n",
      "           3       0.30      0.23      0.26      1713\n",
      "           4       0.04      0.30      0.08      1600\n",
      "\n",
      "    accuracy                           0.44     36257\n",
      "   macro avg       0.39      0.39      0.33     36257\n",
      "weighted avg       0.66      0.44      0.51     36257\n",
      "\n",
      "2023-07-17 13:51:31,019 - INFO - -----------------------------------------------------------------------------------------\n",
      "2023-07-17 13:51:31,020 - INFO - | end of epoch   6 | time: 1441.78s | valid loss 70.27165\n",
      "2023-07-17 13:51:31,021 - INFO - -----------------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   7 |     1/22758 batches | lr 0.0000147 | ms/batch 5852.09 | loss 2.31261\n",
      "| epoch   7 |     2/22758 batches | lr 0.0000147 | ms/batch 2920.14 | loss 0.85204\n",
      "| epoch   7 |     3/22758 batches | lr 0.0000147 | ms/batch 2922.74 | loss 1.60093\n",
      "| epoch   7 |     4/22758 batches | lr 0.0000147 | ms/batch 2983.99 | loss 1.31920\n",
      "| epoch   7 |     5/22758 batches | lr 0.0000147 | ms/batch 2920.26 | loss 1.68314\n",
      "| epoch   7 |     6/22758 batches | lr 0.0000147 | ms/batch 2923.26 | loss 1.05793\n",
      "| epoch   7 |     7/22758 batches | lr 0.0000147 | ms/batch 2925.82 | loss 1.36122\n",
      "| epoch   7 |     8/22758 batches | lr 0.0000147 | ms/batch 2917.36 | loss 0.85137\n",
      "| epoch   7 |     9/22758 batches | lr 0.0000147 | ms/batch 2922.16 | loss 1.18954\n",
      "| epoch   7 |    10/22758 batches | lr 0.0000147 | ms/batch 2919.98 | loss 1.64406\n",
      "| epoch   7 |    11/22758 batches | lr 0.0000147 | ms/batch 2955.63 | loss 0.69318\n",
      "| epoch   7 |    12/22758 batches | lr 0.0000147 | ms/batch 2929.41 | loss 1.64020\n",
      "| epoch   7 |    13/22758 batches | lr 0.0000147 | ms/batch 2922.20 | loss 1.29704\n",
      "| epoch   7 |    14/22758 batches | lr 0.0000147 | ms/batch 2930.09 | loss 1.36455\n",
      "| epoch   7 |    15/22758 batches | lr 0.0000147 | ms/batch 2921.74 | loss 1.69370\n",
      "| epoch   7 |    16/22758 batches | lr 0.0000147 | ms/batch 2924.50 | loss 1.50481\n",
      "| epoch   7 |    17/22758 batches | lr 0.0000147 | ms/batch 2925.07 | loss 1.45967\n",
      "| epoch   7 |    18/22758 batches | lr 0.0000147 | ms/batch 3012.34 | loss 1.82389\n",
      "| epoch   7 |    19/22758 batches | lr 0.0000147 | ms/batch 2920.80 | loss 1.50555\n",
      "| epoch   7 |    20/22758 batches | lr 0.0000147 | ms/batch 2924.52 | loss 1.70007\n",
      "| epoch   7 |    21/22758 batches | lr 0.0000147 | ms/batch 2919.41 | loss 2.00417\n",
      "| epoch   7 |    22/22758 batches | lr 0.0000147 | ms/batch 2919.17 | loss 0.93298\n",
      "| epoch   7 |    23/22758 batches | lr 0.0000147 | ms/batch 2918.59 | loss 1.56224\n",
      "| epoch   7 |    24/22758 batches | lr 0.0000147 | ms/batch 2922.64 | loss 1.32090\n",
      "| epoch   7 |    25/22758 batches | lr 0.0000147 | ms/batch 2941.64 | loss 1.85719\n",
      "| epoch   7 |    26/22758 batches | lr 0.0000147 | ms/batch 2918.85 | loss 1.06669\n",
      "| epoch   7 |    27/22758 batches | lr 0.0000147 | ms/batch 2918.81 | loss 1.20345\n",
      "| epoch   7 |    28/22758 batches | lr 0.0000147 | ms/batch 2921.79 | loss 0.70469\n",
      "| epoch   7 |    29/22758 batches | lr 0.0000147 | ms/batch 2923.97 | loss 1.57336\n",
      "| epoch   7 |    30/22758 batches | lr 0.0000147 | ms/batch 2921.80 | loss 1.36804\n",
      "| epoch   7 |    31/22758 batches | lr 0.0000147 | ms/batch 2921.21 | loss 1.22599\n",
      "| epoch   7 |    32/22758 batches | lr 0.0000147 | ms/batch 2924.80 | loss 1.26110\n",
      "| epoch   7 |    33/22758 batches | lr 0.0000147 | ms/batch 2925.74 | loss 1.24532\n",
      "| epoch   7 |    34/22758 batches | lr 0.0000147 | ms/batch 2920.75 | loss 1.31620\n",
      "| epoch   7 |    35/22758 batches | lr 0.0000147 | ms/batch 2922.23 | loss 1.40319\n",
      "| epoch   7 |    36/22758 batches | lr 0.0000147 | ms/batch 2912.72 | loss 1.36830\n",
      "| epoch   7 |    37/22758 batches | lr 0.0000147 | ms/batch 2924.55 | loss 1.00105\n",
      "| epoch   7 |    38/22758 batches | lr 0.0000147 | ms/batch 2934.55 | loss 1.16731\n",
      "| epoch   7 |    39/22758 batches | lr 0.0000147 | ms/batch 3010.37 | loss 1.27302\n",
      "| epoch   7 |    40/22758 batches | lr 0.0000147 | ms/batch 2924.37 | loss 0.94194\n",
      "| epoch   7 |    41/22758 batches | lr 0.0000147 | ms/batch 2920.56 | loss 1.21156\n",
      "| epoch   7 |    42/22758 batches | lr 0.0000147 | ms/batch 2923.36 | loss 1.23413\n",
      "| epoch   7 |    43/22758 batches | lr 0.0000147 | ms/batch 2925.04 | loss 1.27544\n",
      "| epoch   7 |    44/22758 batches | lr 0.0000147 | ms/batch 2920.98 | loss 1.20095\n",
      "| epoch   7 |    45/22758 batches | lr 0.0000147 | ms/batch 2921.85 | loss 1.13671\n",
      "| epoch   7 |    46/22758 batches | lr 0.0000147 | ms/batch 2948.95 | loss 1.38341\n",
      "| epoch   7 |    47/22758 batches | lr 0.0000147 | ms/batch 2923.01 | loss 1.75579\n",
      "| epoch   7 |    48/22758 batches | lr 0.0000147 | ms/batch 2927.67 | loss 0.92670\n",
      "| epoch   7 |    49/22758 batches | lr 0.0000147 | ms/batch 2932.45 | loss 1.39765\n",
      "| epoch   7 |    50/22758 batches | lr 0.0000147 | ms/batch 2930.85 | loss 1.57528\n",
      "| epoch   7 |    51/22758 batches | lr 0.0000147 | ms/batch 2921.66 | loss 1.47640\n",
      "| epoch   7 |    52/22758 batches | lr 0.0000147 | ms/batch 2920.80 | loss 1.33082\n",
      "| epoch   7 |    53/22758 batches | lr 0.0000147 | ms/batch 2940.61 | loss 1.28409\n",
      "| epoch   7 |    54/22758 batches | lr 0.0000147 | ms/batch 2922.16 | loss 1.16289\n",
      "| epoch   7 |    55/22758 batches | lr 0.0000147 | ms/batch 2925.59 | loss 0.96121\n",
      "| epoch   7 |    56/22758 batches | lr 0.0000147 | ms/batch 2935.62 | loss 1.21260\n",
      "| epoch   7 |    57/22758 batches | lr 0.0000147 | ms/batch 2922.14 | loss 1.41037\n",
      "| epoch   7 |    58/22758 batches | lr 0.0000147 | ms/batch 2924.90 | loss 1.80177\n",
      "| epoch   7 |    59/22758 batches | lr 0.0000147 | ms/batch 2947.03 | loss 1.30896\n",
      "| epoch   7 |    60/22758 batches | lr 0.0000147 | ms/batch 2921.94 | loss 1.20439\n",
      "| epoch   7 |    61/22758 batches | lr 0.0000147 | ms/batch 2917.17 | loss 1.38489\n",
      "| epoch   7 |    62/22758 batches | lr 0.0000147 | ms/batch 2923.46 | loss 1.14565\n",
      "| epoch   7 |    63/22758 batches | lr 0.0000147 | ms/batch 2920.80 | loss 1.37784\n",
      "| epoch   7 |    64/22758 batches | lr 0.0000147 | ms/batch 2924.01 | loss 1.26992\n",
      "| epoch   7 |    65/22758 batches | lr 0.0000147 | ms/batch 2925.58 | loss 1.30672\n",
      "| epoch   7 |    66/22758 batches | lr 0.0000147 | ms/batch 2924.67 | loss 1.24318\n",
      "| epoch   7 |    67/22758 batches | lr 0.0000147 | ms/batch 2924.01 | loss 1.07944\n",
      "| epoch   7 |    68/22758 batches | lr 0.0000147 | ms/batch 2922.80 | loss 1.01923\n",
      "| epoch   7 |    69/22758 batches | lr 0.0000147 | ms/batch 2919.32 | loss 0.86246\n",
      "| epoch   7 |    70/22758 batches | lr 0.0000147 | ms/batch 2924.87 | loss 1.08213\n",
      "| epoch   7 |    71/22758 batches | lr 0.0000147 | ms/batch 2928.15 | loss 1.57676\n",
      "| epoch   7 |    72/22758 batches | lr 0.0000147 | ms/batch 2920.73 | loss 1.38887\n",
      "| epoch   7 |    73/22758 batches | lr 0.0000147 | ms/batch 2973.57 | loss 1.42570\n",
      "| epoch   7 |    74/22758 batches | lr 0.0000147 | ms/batch 2925.09 | loss 0.99494\n",
      "| epoch   7 |    75/22758 batches | lr 0.0000147 | ms/batch 2914.33 | loss 1.04093\n",
      "| epoch   7 |    76/22758 batches | lr 0.0000147 | ms/batch 2908.93 | loss 1.42519\n",
      "| epoch   7 |    77/22758 batches | lr 0.0000147 | ms/batch 2909.95 | loss 1.39231\n",
      "| epoch   7 |    78/22758 batches | lr 0.0000147 | ms/batch 2923.22 | loss 0.95867\n",
      "| epoch   7 |    79/22758 batches | lr 0.0000147 | ms/batch 2927.28 | loss 1.15045\n",
      "| epoch   7 |    80/22758 batches | lr 0.0000147 | ms/batch 2936.38 | loss 1.14843\n",
      "| epoch   7 |    81/22758 batches | lr 0.0000147 | ms/batch 2918.90 | loss 1.49077\n",
      "| epoch   7 |    82/22758 batches | lr 0.0000147 | ms/batch 2912.68 | loss 1.03719\n",
      "| epoch   7 |    83/22758 batches | lr 0.0000147 | ms/batch 2922.06 | loss 1.96064\n",
      "| epoch   7 |    84/22758 batches | lr 0.0000147 | ms/batch 2925.84 | loss 0.83337\n",
      "| epoch   7 |    85/22758 batches | lr 0.0000147 | ms/batch 2920.49 | loss 1.23962\n",
      "| epoch   7 |    86/22758 batches | lr 0.0000147 | ms/batch 2920.14 | loss 1.08374\n",
      "| epoch   7 |    87/22758 batches | lr 0.0000147 | ms/batch 2921.57 | loss 1.39035\n",
      "| epoch   7 |    88/22758 batches | lr 0.0000147 | ms/batch 2948.00 | loss 1.21513\n",
      "| epoch   7 |    89/22758 batches | lr 0.0000147 | ms/batch 2923.29 | loss 0.91620\n",
      "| epoch   7 |    90/22758 batches | lr 0.0000147 | ms/batch 2916.34 | loss 1.13169\n",
      "| epoch   7 |    91/22758 batches | lr 0.0000147 | ms/batch 2923.06 | loss 1.66823\n",
      "| epoch   7 |    92/22758 batches | lr 0.0000147 | ms/batch 2926.86 | loss 1.18722\n",
      "| epoch   7 |    93/22758 batches | lr 0.0000147 | ms/batch 2927.39 | loss 1.39113\n",
      "| epoch   7 |    94/22758 batches | lr 0.0000147 | ms/batch 2919.70 | loss 0.85685\n",
      "| epoch   7 |    95/22758 batches | lr 0.0000147 | ms/batch 2940.05 | loss 1.61216\n",
      "| epoch   7 |    96/22758 batches | lr 0.0000147 | ms/batch 2926.56 | loss 1.51983\n",
      "| epoch   7 |    97/22758 batches | lr 0.0000147 | ms/batch 2923.85 | loss 1.22894\n",
      "| epoch   7 |    98/22758 batches | lr 0.0000147 | ms/batch 2928.43 | loss 1.16027\n",
      "| epoch   7 |    99/22758 batches | lr 0.0000147 | ms/batch 2925.02 | loss 1.60119\n",
      "| epoch   7 |   100/22758 batches | lr 0.0000147 | ms/batch 2926.03 | loss 1.54696\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-07-17 14:15:35,252 - INFO - \n",
      " Scores:\n",
      "2023-07-17 14:15:35,303 - INFO - \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.15      0.47      0.22      1158\n",
      "           1       0.63      0.71      0.66     16938\n",
      "           2       0.72      0.57      0.64     14848\n",
      "           3       0.33      0.22      0.27      1713\n",
      "           4       0.09      0.03      0.04      1600\n",
      "\n",
      "    accuracy                           0.59     36257\n",
      "   macro avg       0.38      0.40      0.37     36257\n",
      "weighted avg       0.61      0.59      0.59     36257\n",
      "\n",
      "2023-07-17 14:15:35,305 - INFO - -----------------------------------------------------------------------------------------\n",
      "2023-07-17 14:15:35,306 - INFO - | end of epoch   7 | time: 1436.54s | valid loss 69.13249\n",
      "2023-07-17 14:15:35,306 - INFO - -----------------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   8 |     1/22758 batches | lr 0.0000140 | ms/batch 5863.47 | loss 2.78418\n",
      "| epoch   8 |     2/22758 batches | lr 0.0000140 | ms/batch 2919.72 | loss 1.33664\n",
      "| epoch   8 |     3/22758 batches | lr 0.0000140 | ms/batch 2905.22 | loss 1.28363\n",
      "| epoch   8 |     4/22758 batches | lr 0.0000140 | ms/batch 2961.36 | loss 1.28963\n",
      "| epoch   8 |     5/22758 batches | lr 0.0000140 | ms/batch 2920.38 | loss 1.24685\n",
      "| epoch   8 |     6/22758 batches | lr 0.0000140 | ms/batch 2919.79 | loss 1.24316\n",
      "| epoch   8 |     7/22758 batches | lr 0.0000140 | ms/batch 2920.87 | loss 1.44340\n",
      "| epoch   8 |     8/22758 batches | lr 0.0000140 | ms/batch 2922.06 | loss 1.46030\n",
      "| epoch   8 |     9/22758 batches | lr 0.0000140 | ms/batch 2916.56 | loss 1.06249\n",
      "| epoch   8 |    10/22758 batches | lr 0.0000140 | ms/batch 2914.32 | loss 1.30905\n",
      "| epoch   8 |    11/22758 batches | lr 0.0000140 | ms/batch 2961.82 | loss 1.49124\n",
      "| epoch   8 |    12/22758 batches | lr 0.0000140 | ms/batch 2922.53 | loss 1.08213\n",
      "| epoch   8 |    13/22758 batches | lr 0.0000140 | ms/batch 2921.68 | loss 1.51413\n",
      "| epoch   8 |    14/22758 batches | lr 0.0000140 | ms/batch 2922.53 | loss 1.21815\n",
      "| epoch   8 |    15/22758 batches | lr 0.0000140 | ms/batch 2923.58 | loss 1.33872\n",
      "| epoch   8 |    16/22758 batches | lr 0.0000140 | ms/batch 2923.17 | loss 1.27809\n",
      "| epoch   8 |    17/22758 batches | lr 0.0000140 | ms/batch 2986.64 | loss 1.54227\n",
      "| epoch   8 |    18/22758 batches | lr 0.0000140 | ms/batch 3053.18 | loss 1.26058\n",
      "| epoch   8 |    19/22758 batches | lr 0.0000140 | ms/batch 3108.01 | loss 1.28930\n",
      "| epoch   8 |    20/22758 batches | lr 0.0000140 | ms/batch 3072.23 | loss 1.06296\n",
      "| epoch   8 |    21/22758 batches | lr 0.0000140 | ms/batch 2919.15 | loss 1.33443\n",
      "| epoch   8 |    22/22758 batches | lr 0.0000140 | ms/batch 2979.33 | loss 0.89516\n",
      "| epoch   8 |    23/22758 batches | lr 0.0000140 | ms/batch 2921.59 | loss 1.22649\n",
      "| epoch   8 |    24/22758 batches | lr 0.0000140 | ms/batch 2920.88 | loss 1.89379\n",
      "| epoch   8 |    25/22758 batches | lr 0.0000140 | ms/batch 2923.01 | loss 1.13563\n",
      "| epoch   8 |    26/22758 batches | lr 0.0000140 | ms/batch 2926.81 | loss 1.22692\n",
      "| epoch   8 |    27/22758 batches | lr 0.0000140 | ms/batch 2926.25 | loss 1.30880\n",
      "| epoch   8 |    28/22758 batches | lr 0.0000140 | ms/batch 2924.98 | loss 1.09541\n",
      "| epoch   8 |    29/22758 batches | lr 0.0000140 | ms/batch 2990.48 | loss 1.04919\n",
      "| epoch   8 |    30/22758 batches | lr 0.0000140 | ms/batch 2933.60 | loss 0.96116\n",
      "| epoch   8 |    31/22758 batches | lr 0.0000140 | ms/batch 2925.87 | loss 1.09997\n",
      "| epoch   8 |    32/22758 batches | lr 0.0000140 | ms/batch 2921.62 | loss 1.00521\n",
      "| epoch   8 |    33/22758 batches | lr 0.0000140 | ms/batch 2922.53 | loss 1.47688\n",
      "| epoch   8 |    34/22758 batches | lr 0.0000140 | ms/batch 2926.77 | loss 1.30193\n",
      "| epoch   8 |    35/22758 batches | lr 0.0000140 | ms/batch 2921.97 | loss 0.84780\n",
      "| epoch   8 |    36/22758 batches | lr 0.0000140 | ms/batch 2939.32 | loss 1.29220\n",
      "| epoch   8 |    37/22758 batches | lr 0.0000140 | ms/batch 2931.23 | loss 1.05916\n",
      "| epoch   8 |    38/22758 batches | lr 0.0000140 | ms/batch 2919.42 | loss 1.56709\n",
      "| epoch   8 |    39/22758 batches | lr 0.0000140 | ms/batch 2933.02 | loss 1.67435\n",
      "| epoch   8 |    40/22758 batches | lr 0.0000140 | ms/batch 2931.13 | loss 1.53799\n",
      "| epoch   8 |    41/22758 batches | lr 0.0000140 | ms/batch 2922.91 | loss 1.29154\n",
      "| epoch   8 |    42/22758 batches | lr 0.0000140 | ms/batch 3002.07 | loss 1.31696\n",
      "| epoch   8 |    43/22758 batches | lr 0.0000140 | ms/batch 2924.32 | loss 1.13919\n",
      "| epoch   8 |    44/22758 batches | lr 0.0000140 | ms/batch 2921.27 | loss 0.86139\n",
      "| epoch   8 |    45/22758 batches | lr 0.0000140 | ms/batch 2925.08 | loss 1.46962\n",
      "| epoch   8 |    46/22758 batches | lr 0.0000140 | ms/batch 2919.37 | loss 1.19759\n",
      "| epoch   8 |    47/22758 batches | lr 0.0000140 | ms/batch 2916.45 | loss 1.45692\n",
      "| epoch   8 |    48/22758 batches | lr 0.0000140 | ms/batch 2920.12 | loss 2.06146\n",
      "| epoch   8 |    49/22758 batches | lr 0.0000140 | ms/batch 2984.19 | loss 1.38199\n",
      "| epoch   8 |    50/22758 batches | lr 0.0000140 | ms/batch 2926.74 | loss 1.04536\n",
      "| epoch   8 |    51/22758 batches | lr 0.0000140 | ms/batch 2921.56 | loss 1.66305\n",
      "| epoch   8 |    52/22758 batches | lr 0.0000140 | ms/batch 2921.02 | loss 1.21148\n",
      "| epoch   8 |    53/22758 batches | lr 0.0000140 | ms/batch 2925.40 | loss 1.33259\n",
      "| epoch   8 |    54/22758 batches | lr 0.0000140 | ms/batch 2923.92 | loss 1.31216\n",
      "| epoch   8 |    55/22758 batches | lr 0.0000140 | ms/batch 2928.04 | loss 1.44525\n",
      "| epoch   8 |    56/22758 batches | lr 0.0000140 | ms/batch 2968.86 | loss 1.42176\n",
      "| epoch   8 |    57/22758 batches | lr 0.0000140 | ms/batch 2925.36 | loss 1.54160\n",
      "| epoch   8 |    58/22758 batches | lr 0.0000140 | ms/batch 2921.95 | loss 1.15418\n",
      "| epoch   8 |    59/22758 batches | lr 0.0000140 | ms/batch 2928.22 | loss 1.36981\n",
      "| epoch   8 |    60/22758 batches | lr 0.0000140 | ms/batch 2924.10 | loss 1.19390\n",
      "| epoch   8 |    61/22758 batches | lr 0.0000140 | ms/batch 2921.59 | loss 0.94185\n",
      "| epoch   8 |    62/22758 batches | lr 0.0000140 | ms/batch 3032.62 | loss 1.10686\n",
      "| epoch   8 |    63/22758 batches | lr 0.0000140 | ms/batch 2924.81 | loss 1.38283\n",
      "| epoch   8 |    64/22758 batches | lr 0.0000140 | ms/batch 2929.25 | loss 1.67351\n",
      "| epoch   8 |    65/22758 batches | lr 0.0000140 | ms/batch 2923.78 | loss 1.21469\n",
      "| epoch   8 |    66/22758 batches | lr 0.0000140 | ms/batch 2928.36 | loss 1.28189\n",
      "| epoch   8 |    67/22758 batches | lr 0.0000140 | ms/batch 2920.73 | loss 1.59388\n",
      "| epoch   8 |    68/22758 batches | lr 0.0000140 | ms/batch 2923.60 | loss 1.13878\n",
      "| epoch   8 |    69/22758 batches | lr 0.0000140 | ms/batch 2935.01 | loss 1.18861\n",
      "| epoch   8 |    70/22758 batches | lr 0.0000140 | ms/batch 2920.23 | loss 0.99472\n",
      "| epoch   8 |    71/22758 batches | lr 0.0000140 | ms/batch 2922.02 | loss 1.11069\n",
      "| epoch   8 |    72/22758 batches | lr 0.0000140 | ms/batch 2921.19 | loss 1.38791\n",
      "| epoch   8 |    73/22758 batches | lr 0.0000140 | ms/batch 2923.86 | loss 0.87673\n",
      "| epoch   8 |    74/22758 batches | lr 0.0000140 | ms/batch 2923.12 | loss 0.91188\n",
      "| epoch   8 |    75/22758 batches | lr 0.0000140 | ms/batch 2926.75 | loss 1.51873\n",
      "| epoch   8 |    76/22758 batches | lr 0.0000140 | ms/batch 2933.27 | loss 1.46867\n",
      "| epoch   8 |    77/22758 batches | lr 0.0000140 | ms/batch 2920.12 | loss 1.31840\n",
      "| epoch   8 |    78/22758 batches | lr 0.0000140 | ms/batch 2923.11 | loss 1.56772\n",
      "| epoch   8 |    79/22758 batches | lr 0.0000140 | ms/batch 2933.56 | loss 1.65597\n",
      "| epoch   8 |    80/22758 batches | lr 0.0000140 | ms/batch 2923.65 | loss 0.90782\n",
      "| epoch   8 |    81/22758 batches | lr 0.0000140 | ms/batch 2922.03 | loss 1.32651\n",
      "| epoch   8 |    82/22758 batches | lr 0.0000140 | ms/batch 2926.44 | loss 1.76795\n",
      "| epoch   8 |    83/22758 batches | lr 0.0000140 | ms/batch 2926.55 | loss 0.87114\n",
      "| epoch   8 |    84/22758 batches | lr 0.0000140 | ms/batch 2927.74 | loss 1.24298\n",
      "| epoch   8 |    85/22758 batches | lr 0.0000140 | ms/batch 2922.21 | loss 1.11066\n",
      "| epoch   8 |    86/22758 batches | lr 0.0000140 | ms/batch 2927.43 | loss 1.44426\n",
      "| epoch   8 |    87/22758 batches | lr 0.0000140 | ms/batch 2920.88 | loss 1.39283\n",
      "| epoch   8 |    88/22758 batches | lr 0.0000140 | ms/batch 2921.51 | loss 1.44111\n",
      "| epoch   8 |    89/22758 batches | lr 0.0000140 | ms/batch 2922.99 | loss 0.84525\n",
      "| epoch   8 |    90/22758 batches | lr 0.0000140 | ms/batch 2918.18 | loss 1.63755\n",
      "| epoch   8 |    91/22758 batches | lr 0.0000140 | ms/batch 2924.70 | loss 1.07350\n",
      "| epoch   8 |    92/22758 batches | lr 0.0000140 | ms/batch 2920.61 | loss 1.19059\n",
      "| epoch   8 |    93/22758 batches | lr 0.0000140 | ms/batch 2923.97 | loss 1.31857\n",
      "| epoch   8 |    94/22758 batches | lr 0.0000140 | ms/batch 2921.10 | loss 1.28915\n",
      "| epoch   8 |    95/22758 batches | lr 0.0000140 | ms/batch 2921.87 | loss 1.38633\n",
      "| epoch   8 |    96/22758 batches | lr 0.0000140 | ms/batch 2926.01 | loss 1.57234\n",
      "| epoch   8 |    97/22758 batches | lr 0.0000140 | ms/batch 2951.03 | loss 1.44054\n",
      "| epoch   8 |    98/22758 batches | lr 0.0000140 | ms/batch 2919.29 | loss 1.09939\n",
      "| epoch   8 |    99/22758 batches | lr 0.0000140 | ms/batch 2928.49 | loss 1.68911\n",
      "| epoch   8 |   100/22758 batches | lr 0.0000140 | ms/batch 2922.01 | loss 1.20823\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-07-17 14:39:46,372 - INFO - \n",
      " Scores:\n",
      "2023-07-17 14:39:46,423 - INFO - \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.13      0.62      0.22      1158\n",
      "           1       0.51      0.74      0.60     16938\n",
      "           2       0.78      0.25      0.38     14848\n",
      "           3       0.33      0.18      0.23      1713\n",
      "           4       0.07      0.03      0.04      1600\n",
      "\n",
      "    accuracy                           0.48     36257\n",
      "   macro avg       0.36      0.36      0.30     36257\n",
      "weighted avg       0.58      0.48      0.46     36257\n",
      "\n",
      "2023-07-17 14:39:46,424 - INFO - -----------------------------------------------------------------------------------------\n",
      "2023-07-17 14:39:46,425 - INFO - | end of epoch   8 | time: 1440.15s | valid loss 72.12461\n",
      "2023-07-17 14:39:46,426 - INFO - -----------------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch   9 |     1/22758 batches | lr 0.0000133 | ms/batch 5847.78 | loss 2.69972\n",
      "| epoch   9 |     2/22758 batches | lr 0.0000133 | ms/batch 2924.67 | loss 1.27642\n",
      "| epoch   9 |     3/22758 batches | lr 0.0000133 | ms/batch 2919.94 | loss 1.49561\n",
      "| epoch   9 |     4/22758 batches | lr 0.0000133 | ms/batch 2921.12 | loss 1.06143\n",
      "| epoch   9 |     5/22758 batches | lr 0.0000133 | ms/batch 2996.94 | loss 1.36295\n",
      "| epoch   9 |     6/22758 batches | lr 0.0000133 | ms/batch 2929.97 | loss 1.54357\n",
      "| epoch   9 |     7/22758 batches | lr 0.0000133 | ms/batch 2927.00 | loss 1.31199\n",
      "| epoch   9 |     8/22758 batches | lr 0.0000133 | ms/batch 2921.94 | loss 1.06850\n",
      "| epoch   9 |     9/22758 batches | lr 0.0000133 | ms/batch 2926.84 | loss 1.66200\n",
      "| epoch   9 |    10/22758 batches | lr 0.0000133 | ms/batch 2926.14 | loss 1.23944\n",
      "| epoch   9 |    11/22758 batches | lr 0.0000133 | ms/batch 2927.37 | loss 1.54754\n",
      "| epoch   9 |    12/22758 batches | lr 0.0000133 | ms/batch 2940.66 | loss 1.44903\n",
      "| epoch   9 |    13/22758 batches | lr 0.0000133 | ms/batch 2928.54 | loss 1.59029\n",
      "| epoch   9 |    14/22758 batches | lr 0.0000133 | ms/batch 2908.84 | loss 1.04291\n",
      "| epoch   9 |    15/22758 batches | lr 0.0000133 | ms/batch 2919.83 | loss 1.27345\n",
      "| epoch   9 |    16/22758 batches | lr 0.0000133 | ms/batch 2920.15 | loss 1.09278\n",
      "| epoch   9 |    17/22758 batches | lr 0.0000133 | ms/batch 2924.18 | loss 0.89446\n",
      "| epoch   9 |    18/22758 batches | lr 0.0000133 | ms/batch 2918.65 | loss 1.41423\n",
      "| epoch   9 |    19/22758 batches | lr 0.0000133 | ms/batch 2967.84 | loss 1.86701\n",
      "| epoch   9 |    20/22758 batches | lr 0.0000133 | ms/batch 2925.53 | loss 1.28046\n",
      "| epoch   9 |    21/22758 batches | lr 0.0000133 | ms/batch 2919.04 | loss 0.71061\n",
      "| epoch   9 |    22/22758 batches | lr 0.0000133 | ms/batch 2919.62 | loss 1.35987\n",
      "| epoch   9 |    23/22758 batches | lr 0.0000133 | ms/batch 2923.44 | loss 1.65243\n",
      "| epoch   9 |    24/22758 batches | lr 0.0000133 | ms/batch 2920.97 | loss 1.85067\n",
      "| epoch   9 |    25/22758 batches | lr 0.0000133 | ms/batch 2921.68 | loss 1.28031\n",
      "| epoch   9 |    26/22758 batches | lr 0.0000133 | ms/batch 3025.31 | loss 1.14942\n",
      "| epoch   9 |    27/22758 batches | lr 0.0000133 | ms/batch 2923.54 | loss 1.46060\n",
      "| epoch   9 |    28/22758 batches | lr 0.0000133 | ms/batch 2919.74 | loss 1.35721\n",
      "| epoch   9 |    29/22758 batches | lr 0.0000133 | ms/batch 2926.43 | loss 1.80738\n",
      "| epoch   9 |    30/22758 batches | lr 0.0000133 | ms/batch 2921.05 | loss 1.23741\n",
      "| epoch   9 |    31/22758 batches | lr 0.0000133 | ms/batch 2921.66 | loss 1.69992\n",
      "| epoch   9 |    32/22758 batches | lr 0.0000133 | ms/batch 2924.06 | loss 1.24601\n",
      "| epoch   9 |    33/22758 batches | lr 0.0000133 | ms/batch 2926.28 | loss 1.10351\n",
      "| epoch   9 |    34/22758 batches | lr 0.0000133 | ms/batch 2910.62 | loss 1.28950\n",
      "| epoch   9 |    35/22758 batches | lr 0.0000133 | ms/batch 2926.54 | loss 2.33033\n",
      "| epoch   9 |    36/22758 batches | lr 0.0000133 | ms/batch 2925.98 | loss 0.96236\n",
      "| epoch   9 |    37/22758 batches | lr 0.0000133 | ms/batch 2918.80 | loss 1.53778\n",
      "| epoch   9 |    38/22758 batches | lr 0.0000133 | ms/batch 2920.30 | loss 1.42343\n",
      "| epoch   9 |    39/22758 batches | lr 0.0000133 | ms/batch 2921.61 | loss 1.13236\n",
      "| epoch   9 |    40/22758 batches | lr 0.0000133 | ms/batch 2950.84 | loss 1.10181\n",
      "| epoch   9 |    41/22758 batches | lr 0.0000133 | ms/batch 2924.44 | loss 1.06239\n",
      "| epoch   9 |    42/22758 batches | lr 0.0000133 | ms/batch 2923.49 | loss 1.21526\n",
      "| epoch   9 |    43/22758 batches | lr 0.0000133 | ms/batch 2921.08 | loss 1.45858\n",
      "| epoch   9 |    44/22758 batches | lr 0.0000133 | ms/batch 2923.76 | loss 1.63758\n",
      "| epoch   9 |    45/22758 batches | lr 0.0000133 | ms/batch 2924.20 | loss 1.04686\n",
      "| epoch   9 |    46/22758 batches | lr 0.0000133 | ms/batch 2920.03 | loss 1.40971\n",
      "| epoch   9 |    47/22758 batches | lr 0.0000133 | ms/batch 2983.65 | loss 1.36613\n",
      "| epoch   9 |    48/22758 batches | lr 0.0000133 | ms/batch 2923.64 | loss 1.40041\n",
      "| epoch   9 |    49/22758 batches | lr 0.0000133 | ms/batch 2927.43 | loss 1.39216\n",
      "| epoch   9 |    50/22758 batches | lr 0.0000133 | ms/batch 2923.67 | loss 1.42144\n",
      "| epoch   9 |    51/22758 batches | lr 0.0000133 | ms/batch 2923.62 | loss 1.56418\n",
      "| epoch   9 |    52/22758 batches | lr 0.0000133 | ms/batch 2922.34 | loss 1.28724\n",
      "| epoch   9 |    53/22758 batches | lr 0.0000133 | ms/batch 2920.15 | loss 1.15350\n",
      "| epoch   9 |    54/22758 batches | lr 0.0000133 | ms/batch 2940.59 | loss 1.14993\n",
      "| epoch   9 |    55/22758 batches | lr 0.0000133 | ms/batch 2922.86 | loss 1.27230\n",
      "| epoch   9 |    56/22758 batches | lr 0.0000133 | ms/batch 2922.04 | loss 1.27987\n",
      "| epoch   9 |    57/22758 batches | lr 0.0000133 | ms/batch 2919.23 | loss 1.29536\n",
      "| epoch   9 |    58/22758 batches | lr 0.0000133 | ms/batch 2919.97 | loss 1.61708\n",
      "| epoch   9 |    59/22758 batches | lr 0.0000133 | ms/batch 2927.56 | loss 1.32937\n",
      "| epoch   9 |    60/22758 batches | lr 0.0000133 | ms/batch 2923.69 | loss 1.49621\n",
      "| epoch   9 |    61/22758 batches | lr 0.0000133 | ms/batch 2938.37 | loss 1.16373\n",
      "| epoch   9 |    62/22758 batches | lr 0.0000133 | ms/batch 2919.29 | loss 1.25629\n",
      "| epoch   9 |    63/22758 batches | lr 0.0000133 | ms/batch 2918.61 | loss 1.19165\n",
      "| epoch   9 |    64/22758 batches | lr 0.0000133 | ms/batch 2922.04 | loss 1.47650\n",
      "| epoch   9 |    65/22758 batches | lr 0.0000133 | ms/batch 2916.03 | loss 1.44304\n",
      "| epoch   9 |    66/22758 batches | lr 0.0000133 | ms/batch 2921.09 | loss 1.02134\n",
      "| epoch   9 |    67/22758 batches | lr 0.0000133 | ms/batch 2920.68 | loss 1.41471\n",
      "| epoch   9 |    68/22758 batches | lr 0.0000133 | ms/batch 2978.40 | loss 1.29339\n",
      "| epoch   9 |    69/22758 batches | lr 0.0000133 | ms/batch 2931.72 | loss 1.63908\n",
      "| epoch   9 |    70/22758 batches | lr 0.0000133 | ms/batch 2919.44 | loss 1.74591\n",
      "| epoch   9 |    71/22758 batches | lr 0.0000133 | ms/batch 2921.60 | loss 1.29251\n",
      "| epoch   9 |    72/22758 batches | lr 0.0000133 | ms/batch 2922.65 | loss 0.89800\n",
      "| epoch   9 |    73/22758 batches | lr 0.0000133 | ms/batch 2922.47 | loss 1.63191\n",
      "| epoch   9 |    74/22758 batches | lr 0.0000133 | ms/batch 2921.47 | loss 1.42742\n",
      "| epoch   9 |    75/22758 batches | lr 0.0000133 | ms/batch 2947.16 | loss 1.37249\n",
      "| epoch   9 |    76/22758 batches | lr 0.0000133 | ms/batch 2922.78 | loss 1.57515\n",
      "| epoch   9 |    77/22758 batches | lr 0.0000133 | ms/batch 2923.64 | loss 1.04111\n",
      "| epoch   9 |    78/22758 batches | lr 0.0000133 | ms/batch 2921.00 | loss 1.29281\n",
      "| epoch   9 |    79/22758 batches | lr 0.0000133 | ms/batch 2917.57 | loss 2.20787\n",
      "| epoch   9 |    80/22758 batches | lr 0.0000133 | ms/batch 2919.90 | loss 1.46980\n",
      "| epoch   9 |    81/22758 batches | lr 0.0000133 | ms/batch 2926.93 | loss 1.47794\n",
      "| epoch   9 |    82/22758 batches | lr 0.0000133 | ms/batch 2921.17 | loss 1.04107\n",
      "| epoch   9 |    83/22758 batches | lr 0.0000133 | ms/batch 2924.71 | loss 1.75424\n",
      "| epoch   9 |    84/22758 batches | lr 0.0000133 | ms/batch 2920.78 | loss 1.93806\n",
      "| epoch   9 |    85/22758 batches | lr 0.0000133 | ms/batch 2914.74 | loss 0.83796\n",
      "| epoch   9 |    86/22758 batches | lr 0.0000133 | ms/batch 2926.52 | loss 1.10611\n",
      "| epoch   9 |    87/22758 batches | lr 0.0000133 | ms/batch 2925.21 | loss 1.24930\n",
      "| epoch   9 |    88/22758 batches | lr 0.0000133 | ms/batch 2926.28 | loss 1.25401\n",
      "| epoch   9 |    89/22758 batches | lr 0.0000133 | ms/batch 2990.58 | loss 1.18254\n",
      "| epoch   9 |    90/22758 batches | lr 0.0000133 | ms/batch 2919.45 | loss 1.77532\n",
      "| epoch   9 |    91/22758 batches | lr 0.0000133 | ms/batch 2919.02 | loss 1.42306\n",
      "| epoch   9 |    92/22758 batches | lr 0.0000133 | ms/batch 2909.41 | loss 1.56236\n",
      "| epoch   9 |    93/22758 batches | lr 0.0000133 | ms/batch 2907.31 | loss 1.75630\n",
      "| epoch   9 |    94/22758 batches | lr 0.0000133 | ms/batch 2906.40 | loss 0.98920\n",
      "| epoch   9 |    95/22758 batches | lr 0.0000133 | ms/batch 2906.24 | loss 1.61717\n",
      "| epoch   9 |    96/22758 batches | lr 0.0000133 | ms/batch 2969.50 | loss 1.01496\n",
      "| epoch   9 |    97/22758 batches | lr 0.0000133 | ms/batch 2930.02 | loss 1.19163\n",
      "| epoch   9 |    98/22758 batches | lr 0.0000133 | ms/batch 2924.88 | loss 1.41098\n",
      "| epoch   9 |    99/22758 batches | lr 0.0000133 | ms/batch 2916.97 | loss 1.21496\n",
      "| epoch   9 |   100/22758 batches | lr 0.0000133 | ms/batch 2920.73 | loss 0.88413\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-07-17 15:03:46,722 - INFO - \n",
      " Scores:\n",
      "2023-07-17 15:03:46,788 - INFO - \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.14      0.63      0.23      1158\n",
      "           1       0.72      0.47      0.57     16938\n",
      "           2       0.64      0.67      0.65     14848\n",
      "           3       0.24      0.32      0.28      1713\n",
      "           4       0.05      0.06      0.05      1600\n",
      "\n",
      "    accuracy                           0.53     36257\n",
      "   macro avg       0.36      0.43      0.36     36257\n",
      "weighted avg       0.61      0.53      0.56     36257\n",
      "\n",
      "2023-07-17 15:03:46,790 - INFO - -----------------------------------------------------------------------------------------\n",
      "2023-07-17 15:03:46,790 - INFO - | end of epoch   9 | time: 1440.36s | valid loss 67.39701\n",
      "2023-07-17 15:03:46,791 - INFO - -----------------------------------------------------------------------------------------\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "| epoch  10 |     1/22758 batches | lr 0.0000126 | ms/batch 5934.34 | loss 3.28735\n",
      "| epoch  10 |     2/22758 batches | lr 0.0000126 | ms/batch 2911.77 | loss 1.33608\n",
      "| epoch  10 |     3/22758 batches | lr 0.0000126 | ms/batch 2909.76 | loss 1.57771\n",
      "| epoch  10 |     4/22758 batches | lr 0.0000126 | ms/batch 2909.54 | loss 1.13079\n",
      "| epoch  10 |     5/22758 batches | lr 0.0000126 | ms/batch 2921.00 | loss 1.05130\n",
      "| epoch  10 |     6/22758 batches | lr 0.0000126 | ms/batch 2926.40 | loss 1.14101\n",
      "| epoch  10 |     7/22758 batches | lr 0.0000126 | ms/batch 2917.56 | loss 1.42674\n",
      "| epoch  10 |     8/22758 batches | lr 0.0000126 | ms/batch 2922.09 | loss 1.44864\n",
      "| epoch  10 |     9/22758 batches | lr 0.0000126 | ms/batch 2905.75 | loss 1.31752\n",
      "| epoch  10 |    10/22758 batches | lr 0.0000126 | ms/batch 2905.81 | loss 1.21934\n",
      "| epoch  10 |    11/22758 batches | lr 0.0000126 | ms/batch 2905.83 | loss 1.03424\n",
      "| epoch  10 |    12/22758 batches | lr 0.0000126 | ms/batch 2926.93 | loss 1.42170\n",
      "| epoch  10 |    13/22758 batches | lr 0.0000126 | ms/batch 2927.44 | loss 1.23243\n",
      "| epoch  10 |    14/22758 batches | lr 0.0000126 | ms/batch 2917.82 | loss 1.45305\n",
      "| epoch  10 |    15/22758 batches | lr 0.0000126 | ms/batch 2960.85 | loss 1.52561\n",
      "| epoch  10 |    16/22758 batches | lr 0.0000126 | ms/batch 2917.94 | loss 1.23911\n",
      "| epoch  10 |    17/22758 batches | lr 0.0000126 | ms/batch 2920.48 | loss 1.08070\n",
      "| epoch  10 |    18/22758 batches | lr 0.0000126 | ms/batch 2922.60 | loss 1.35322\n",
      "| epoch  10 |    19/22758 batches | lr 0.0000126 | ms/batch 2920.69 | loss 1.42164\n",
      "| epoch  10 |    20/22758 batches | lr 0.0000126 | ms/batch 2921.11 | loss 1.18051\n",
      "| epoch  10 |    21/22758 batches | lr 0.0000126 | ms/batch 2922.98 | loss 1.22659\n",
      "| epoch  10 |    22/22758 batches | lr 0.0000126 | ms/batch 2919.91 | loss 1.58050\n",
      "| epoch  10 |    23/22758 batches | lr 0.0000126 | ms/batch 2918.62 | loss 1.18471\n",
      "| epoch  10 |    24/22758 batches | lr 0.0000126 | ms/batch 2922.20 | loss 1.22390\n",
      "| epoch  10 |    25/22758 batches | lr 0.0000126 | ms/batch 2911.48 | loss 1.09113\n",
      "| epoch  10 |    26/22758 batches | lr 0.0000126 | ms/batch 2917.73 | loss 0.97000\n",
      "| epoch  10 |    27/22758 batches | lr 0.0000126 | ms/batch 2907.05 | loss 1.04314\n",
      "| epoch  10 |    28/22758 batches | lr 0.0000126 | ms/batch 2909.32 | loss 1.30421\n",
      "| epoch  10 |    29/22758 batches | lr 0.0000126 | ms/batch 2921.39 | loss 1.36572\n",
      "| epoch  10 |    30/22758 batches | lr 0.0000126 | ms/batch 2906.20 | loss 1.18078\n",
      "| epoch  10 |    31/22758 batches | lr 0.0000126 | ms/batch 2911.92 | loss 1.37311\n",
      "| epoch  10 |    32/22758 batches | lr 0.0000126 | ms/batch 2919.91 | loss 1.17462\n",
      "| epoch  10 |    33/22758 batches | lr 0.0000126 | ms/batch 2912.13 | loss 0.94479\n",
      "| epoch  10 |    34/22758 batches | lr 0.0000126 | ms/batch 2910.67 | loss 0.94226\n",
      "| epoch  10 |    35/22758 batches | lr 0.0000126 | ms/batch 2912.00 | loss 1.68844\n",
      "| epoch  10 |    36/22758 batches | lr 0.0000126 | ms/batch 2917.07 | loss 1.56436\n",
      "| epoch  10 |    37/22758 batches | lr 0.0000126 | ms/batch 2974.04 | loss 1.51492\n",
      "| epoch  10 |    38/22758 batches | lr 0.0000126 | ms/batch 2921.97 | loss 0.94637\n",
      "| epoch  10 |    39/22758 batches | lr 0.0000126 | ms/batch 2921.12 | loss 1.43488\n",
      "| epoch  10 |    40/22758 batches | lr 0.0000126 | ms/batch 2925.56 | loss 1.28184\n",
      "| epoch  10 |    41/22758 batches | lr 0.0000126 | ms/batch 2920.62 | loss 0.73485\n",
      "| epoch  10 |    42/22758 batches | lr 0.0000126 | ms/batch 2921.36 | loss 0.81219\n",
      "| epoch  10 |    43/22758 batches | lr 0.0000126 | ms/batch 2920.42 | loss 1.25610\n",
      "| epoch  10 |    44/22758 batches | lr 0.0000126 | ms/batch 2931.02 | loss 1.74772\n",
      "| epoch  10 |    45/22758 batches | lr 0.0000126 | ms/batch 2907.31 | loss 1.22777\n",
      "| epoch  10 |    46/22758 batches | lr 0.0000126 | ms/batch 2904.41 | loss 1.55454\n",
      "| epoch  10 |    47/22758 batches | lr 0.0000126 | ms/batch 2910.23 | loss 1.23641\n",
      "| epoch  10 |    48/22758 batches | lr 0.0000126 | ms/batch 2907.82 | loss 1.35441\n",
      "| epoch  10 |    49/22758 batches | lr 0.0000126 | ms/batch 2914.84 | loss 1.07942\n",
      "| epoch  10 |    50/22758 batches | lr 0.0000126 | ms/batch 2910.76 | loss 1.28089\n",
      "| epoch  10 |    51/22758 batches | lr 0.0000126 | ms/batch 2935.98 | loss 1.54324\n",
      "| epoch  10 |    52/22758 batches | lr 0.0000126 | ms/batch 2915.47 | loss 1.44010\n",
      "| epoch  10 |    53/22758 batches | lr 0.0000126 | ms/batch 2916.33 | loss 1.38671\n",
      "| epoch  10 |    54/22758 batches | lr 0.0000126 | ms/batch 2905.04 | loss 1.32179\n",
      "| epoch  10 |    55/22758 batches | lr 0.0000126 | ms/batch 2910.14 | loss 0.72054\n",
      "| epoch  10 |    56/22758 batches | lr 0.0000126 | ms/batch 2908.48 | loss 1.31086\n",
      "| epoch  10 |    57/22758 batches | lr 0.0000126 | ms/batch 2908.34 | loss 1.16701\n",
      "| epoch  10 |    58/22758 batches | lr 0.0000126 | ms/batch 3001.75 | loss 0.91323\n",
      "| epoch  10 |    59/22758 batches | lr 0.0000126 | ms/batch 2919.19 | loss 1.11308\n",
      "| epoch  10 |    60/22758 batches | lr 0.0000126 | ms/batch 2924.59 | loss 1.56976\n",
      "| epoch  10 |    61/22758 batches | lr 0.0000126 | ms/batch 2923.53 | loss 1.38259\n",
      "| epoch  10 |    62/22758 batches | lr 0.0000126 | ms/batch 2919.80 | loss 1.56578\n",
      "| epoch  10 |    63/22758 batches | lr 0.0000126 | ms/batch 2924.52 | loss 1.13568\n",
      "| epoch  10 |    64/22758 batches | lr 0.0000126 | ms/batch 2919.88 | loss 1.11117\n",
      "| epoch  10 |    65/22758 batches | lr 0.0000126 | ms/batch 2944.99 | loss 1.35527\n",
      "| epoch  10 |    66/22758 batches | lr 0.0000126 | ms/batch 2921.79 | loss 1.48482\n",
      "| epoch  10 |    67/22758 batches | lr 0.0000126 | ms/batch 2922.67 | loss 0.91773\n",
      "| epoch  10 |    68/22758 batches | lr 0.0000126 | ms/batch 2929.07 | loss 1.16747\n",
      "| epoch  10 |    69/22758 batches | lr 0.0000126 | ms/batch 2927.24 | loss 1.22876\n",
      "| epoch  10 |    70/22758 batches | lr 0.0000126 | ms/batch 2924.16 | loss 1.19635\n",
      "| epoch  10 |    71/22758 batches | lr 0.0000126 | ms/batch 2924.96 | loss 1.00312\n",
      "| epoch  10 |    72/22758 batches | lr 0.0000126 | ms/batch 2929.77 | loss 1.12595\n",
      "| epoch  10 |    73/22758 batches | lr 0.0000126 | ms/batch 2922.33 | loss 1.25041\n",
      "| epoch  10 |    74/22758 batches | lr 0.0000126 | ms/batch 2919.78 | loss 1.26307\n",
      "| epoch  10 |    75/22758 batches | lr 0.0000126 | ms/batch 2920.37 | loss 1.29456\n",
      "| epoch  10 |    76/22758 batches | lr 0.0000126 | ms/batch 2919.49 | loss 1.71764\n",
      "| epoch  10 |    77/22758 batches | lr 0.0000126 | ms/batch 2923.14 | loss 2.15354\n",
      "| epoch  10 |    78/22758 batches | lr 0.0000126 | ms/batch 2926.96 | loss 1.51118\n",
      "| epoch  10 |    79/22758 batches | lr 0.0000126 | ms/batch 2961.71 | loss 1.20554\n",
      "| epoch  10 |    80/22758 batches | lr 0.0000126 | ms/batch 2923.28 | loss 0.86386\n",
      "| epoch  10 |    81/22758 batches | lr 0.0000126 | ms/batch 2923.98 | loss 1.04890\n",
      "| epoch  10 |    82/22758 batches | lr 0.0000126 | ms/batch 2922.04 | loss 1.29123\n",
      "| epoch  10 |    83/22758 batches | lr 0.0000126 | ms/batch 2923.78 | loss 1.19654\n",
      "| epoch  10 |    84/22758 batches | lr 0.0000126 | ms/batch 2923.83 | loss 0.98426\n",
      "| epoch  10 |    85/22758 batches | lr 0.0000126 | ms/batch 2921.61 | loss 1.21723\n",
      "| epoch  10 |    86/22758 batches | lr 0.0000126 | ms/batch 2965.52 | loss 1.40562\n",
      "| epoch  10 |    87/22758 batches | lr 0.0000126 | ms/batch 2919.02 | loss 1.14102\n",
      "| epoch  10 |    88/22758 batches | lr 0.0000126 | ms/batch 2917.77 | loss 0.93716\n",
      "| epoch  10 |    89/22758 batches | lr 0.0000126 | ms/batch 2919.69 | loss 0.84527\n",
      "| epoch  10 |    90/22758 batches | lr 0.0000126 | ms/batch 2919.06 | loss 1.74472\n",
      "| epoch  10 |    91/22758 batches | lr 0.0000126 | ms/batch 2922.30 | loss 1.16462\n",
      "| epoch  10 |    92/22758 batches | lr 0.0000126 | ms/batch 2925.93 | loss 1.63774\n",
      "| epoch  10 |    93/22758 batches | lr 0.0000126 | ms/batch 2942.95 | loss 1.57986\n",
      "| epoch  10 |    94/22758 batches | lr 0.0000126 | ms/batch 2917.58 | loss 1.43880\n",
      "| epoch  10 |    95/22758 batches | lr 0.0000126 | ms/batch 2921.66 | loss 1.09883\n",
      "| epoch  10 |    96/22758 batches | lr 0.0000126 | ms/batch 2917.74 | loss 1.74623\n",
      "| epoch  10 |    97/22758 batches | lr 0.0000126 | ms/batch 2923.92 | loss 1.61017\n",
      "| epoch  10 |    98/22758 batches | lr 0.0000126 | ms/batch 2919.75 | loss 0.77783\n",
      "| epoch  10 |    99/22758 batches | lr 0.0000126 | ms/batch 2923.05 | loss 1.21298\n",
      "| epoch  10 |   100/22758 batches | lr 0.0000126 | ms/batch 2934.84 | loss 1.00381\n"
     ]
    }
   ],
   "source": [
    "best_val_loss = float('inf')\n",
    "current_val_loss = []   # for plotting graph of val_loss\n",
    "epochs = 40\n",
    "early_stop_thresh = 3\n",
    "\n",
    "tempdir = '/data1/debajyoti/colie/.temp/'\n",
    "best_model_params_path = os.path.join(tempdir, f\"best_model_params_{time.asctime().replace(' ','_')}.pt\")\n",
    "\n",
    "for epoch in range(1, epochs + 1):\n",
    "    epoch_start_time = time.time()\n",
    "    train(model)\n",
    "    val_loss = evaluate(model)\n",
    "    current_val_loss.append(val_loss)\n",
    "    # val_ppl = np.exp(val_loss)\n",
    "    elapsed = time.time() - epoch_start_time\n",
    "    logging.info('-' * 89)\n",
    "    logging.info(f'| end of epoch {epoch:3d} | time: {elapsed:5.2f}s | '\n",
    "        f'valid loss {val_loss:5.5f}')\n",
    "    logging.info('-' * 89)\n",
    "\n",
    "    if val_loss < best_val_loss:\n",
    "        best_val_loss = val_loss\n",
    "        best_epoch = epoch\n",
    "        torch.save(model.state_dict(), best_model_params_path)\n",
    "    elif epoch - best_epoch > early_stop_thresh:\n",
    "        logging.info(\"Early stopped training at epoch %d\" % epoch)\n",
    "        break  # terminate the training loop\n",
    "\n",
    "    scheduler.step()\n",
    "model.load_state_dict(torch.load(best_model_params_path)) # load best model states\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "(1/43)/((1/30) + (1/377) + (1/327) + (1/43) + (1/31))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "env",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}