NLP_proj.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuClass": "premium"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "gpuClass": "premium"
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "mXnSpQuqlhcm",
        "outputId": "f83b12fe-5c61-406c-c51b-ce26ce391bd1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "\n",
            "cuda\n",
            "\n",
            "\n",
            "\n",
            "Epoch 0/15\n",
            "3.9173085689544678\n",
            "1.8183770179748535\n",
            "1.8926398754119873\n",
            "2.014730215072632\n",
            "1.8306968212127686\n",
            "1.8531278371810913\n",
            "1.8489058017730713\n",
            "1.8255528211593628\n",
            "1.6332751512527466\n",
            "1.7383685111999512\n",
            "1.4514012336730957\n",
            "1.6570998430252075\n",
            "1.7615340948104858\n",
            "1.6897836923599243\n",
            "1.7192317247390747\n",
            "1.845238447189331\n",
            "1.8756437301635742\n",
            "1.6406128406524658\n",
            "1.59213125705719\n",
            "1.579964280128479\n",
            "1.671155333518982\n",
            "1.5308290719985962\n",
            "1.8954477310180664\n",
            "1.7528046369552612\n",
            "1.6376148462295532\n",
            "2.0387399196624756\n",
            "1.5845390558242798\n",
            "1.6814318895339966\n",
            "1.7465016841888428\n",
            "1.8286787271499634\n",
            "1.9464972019195557\n",
            "1.7064290046691895\n",
            "1.4980711936950684\n",
            "1.5216928720474243\n",
            "1.7230327129364014\n",
            "1.8511635065078735\n",
            "1.7285088300704956\n",
            "1.5686490535736084\n",
            "1.7890788316726685\n",
            "1.5574474334716797\n",
            "1.6661704778671265\n",
            "1.5839061737060547\n",
            "1.5996133089065552\n",
            "1.6513259410858154\n",
            "1.7753080129623413\n",
            "1.6596182584762573\n",
            "1.756629467010498\n",
            "1.646897315979004\n",
            "1.7504825592041016\n",
            "1.7946631908416748\n",
            "1.6473487615585327\n",
            "1.8052258491516113\n",
            "1.7671951055526733\n",
            "1.7849910259246826\n",
            "1.682629108428955\n",
            "1.6376478672027588\n",
            "1.6502857208251953\n",
            "1.6706340312957764\n",
            "1.6066704988479614\n",
            "1.5232281684875488\n",
            "1.797587275505066\n",
            "1.6954169273376465\n",
            "1.7272695302963257\n",
            "The avg loss is 1.7151631134033203\n",
            "0\n",
            "10\n",
            "20\n",
            "30\n",
            "40\n",
            "50\n",
            "60\n",
            "70\n",
            "80\n",
            "90\n",
            "100\n",
            "110\n",
            "120\n",
            "The wider bleu is 26.74\n",
            "[26.74]\n",
            "[1.7151631134033203]\n",
            "Epoch 1/15\n",
            "1.8827896118164062\n",
            "1.6372253894805908\n",
            "1.6616066694259644\n",
            "1.6744457483291626\n",
            "1.5788556337356567\n",
            "1.4950103759765625\n",
            "1.5421546697616577\n",
            "1.7811013460159302\n",
            "1.8253896236419678\n",
            "1.5741232633590698\n",
            "1.6179457902908325\n",
            "1.5942925214767456\n",
            "1.7229938507080078\n",
            "1.9284157752990723\n",
            "1.6646775007247925\n",
            "1.6581666469573975\n",
            "1.4906952381134033\n",
            "1.8416060209274292\n",
            "1.644217610359192\n",
            "1.6843918561935425\n",
            "1.8022271394729614\n",
            "1.6462019681930542\n",
            "1.7974557876586914\n",
            "1.7622343301773071\n",
            "1.4893165826797485\n",
            "1.6642742156982422\n",
            "1.5955461263656616\n",
            "1.6671502590179443\n",
            "1.5694751739501953\n",
            "1.6139732599258423\n",
            "1.5528324842453003\n",
            "1.6213268041610718\n",
            "1.8233051300048828\n",
            "1.5882997512817383\n",
            "1.8333536386489868\n",
            "1.8553786277770996\n",
            "1.6903681755065918\n",
            "1.8707118034362793\n",
            "1.4686317443847656\n",
            "1.5901864767074585\n",
            "1.6622247695922852\n",
            "2.01312518119812\n",
            "1.6125043630599976\n",
            "1.4480088949203491\n",
            "1.610660433769226\n",
            "1.5622527599334717\n",
            "1.7684626579284668\n",
            "1.8776288032531738\n",
            "1.579434871673584\n",
            "1.726554274559021\n",
            "1.7130368947982788\n",
            "1.5844266414642334\n",
            "1.6616790294647217\n",
            "1.4656922817230225\n",
            "1.510291576385498\n",
            "1.3977892398834229\n",
            "1.5918734073638916\n",
            "1.9220914840698242\n",
            "1.8419570922851562\n",
            "1.7040213346481323\n",
            "1.5598626136779785\n",
            "1.6853221654891968\n",
            "1.815009593963623\n",
            "The avg loss is 1.6764829674720765\n",
            "0\n",
            "10\n",
            "20\n",
            "30\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-2-52b1720b6117>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m    230\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    231\u001b[0m \u001b[0;31m# if _name_ == \"_main_\":\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 232\u001b[0;31m \u001b[0mmain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-2-52b1720b6117>\u001b[0m in \u001b[0;36mmain\u001b[0;34m()\u001b[0m\n\u001b[1;32m    221\u001b[0m       \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34mf\"optimizer_epoch_{epoch+1}.pt\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    222\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 223\u001b[0;31m           \u001b[0mwide_score\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcalc_bleu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtokenizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbeams\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m4\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    224\u001b[0m           \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"The wider bleu is {wide_score}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    225\u001b[0m           \u001b[0mwide_scores\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwide_score\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-2-52b1720b6117>\u001b[0m in \u001b[0;36mcalc_bleu\u001b[0;34m(epoch, model, tokenizer, val_loader, beams, task_prefix)\u001b[0m\n\u001b[1;32m     94\u001b[0m                 input_ids, attention_mask = encoding.input_ids.to(device, non_blocking=True), encoding.attention_mask.to(\n\u001b[1;32m     95\u001b[0m                     device, non_blocking=True)\n\u001b[0;32m---> 96\u001b[0;31m                 outputs_ids = model.generate(input_ids, attention_mask=attention_mask,\n\u001b[0m\u001b[1;32m     97\u001b[0m                                              max_length=1000, num_beams=beams)\n\u001b[1;32m     98\u001b[0m                 \u001b[0moutputs_ids\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moutputs_ids\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtolist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/torch/autograd/grad_mode.py\u001b[0m in \u001b[0;36mdecorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     25\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclone\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     28\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/transformers/generation/utils.py\u001b[0m in \u001b[0;36mgenerate\u001b[0;34m(self, inputs, generation_config, logits_processor, stopping_criteria, prefix_allowed_tokens_fn, synced_gpus, **kwargs)\u001b[0m\n\u001b[1;32m   1488\u001b[0m             )\n\u001b[1;32m   1489\u001b[0m             \u001b[0;31m# 13. run beam search\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1490\u001b[0;31m             return self.beam_search(\n\u001b[0m\u001b[1;32m   1491\u001b[0m                 \u001b[0minput_ids\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1492\u001b[0m                 \u001b[0mbeam_scorer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/transformers/generation/utils.py\u001b[0m in \u001b[0;36mbeam_search\u001b[0;34m(self, input_ids, beam_scorer, logits_processor, stopping_criteria, max_length, pad_token_id, eos_token_id, output_attentions, output_hidden_states, output_scores, return_dict_in_generate, synced_gpus, **model_kwargs)\u001b[0m\n\u001b[1;32m   2747\u001b[0m             \u001b[0mmodel_inputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprepare_inputs_for_generation\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_ids\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mmodel_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2748\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2749\u001b[0;31m             outputs = self(\n\u001b[0m\u001b[1;32m   2750\u001b[0m                 \u001b[0;34m**\u001b[0m\u001b[0mmodel_inputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2751\u001b[0m                 \u001b[0mreturn_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1192\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m   1193\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1194\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1195\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1196\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/transformers/models/t5/modeling_t5.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input_ids, attention_mask, decoder_input_ids, decoder_attention_mask, head_mask, decoder_head_mask, cross_attn_head_mask, encoder_outputs, past_key_values, inputs_embeds, decoder_inputs_embeds, labels, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m   1702\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1703\u001b[0m         \u001b[0;31m# Decode\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1704\u001b[0;31m         decoder_outputs = self.decoder(\n\u001b[0m\u001b[1;32m   1705\u001b[0m             \u001b[0minput_ids\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdecoder_input_ids\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1706\u001b[0m             \u001b[0mattention_mask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdecoder_attention_mask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1192\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m   1193\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1194\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1195\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1196\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/transformers/models/t5/modeling_t5.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input_ids, attention_mask, encoder_hidden_states, encoder_attention_mask, inputs_embeds, head_mask, cross_attn_head_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m   1072\u001b[0m                 )\n\u001b[1;32m   1073\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1074\u001b[0;31m                 layer_outputs = layer_module(\n\u001b[0m\u001b[1;32m   1075\u001b[0m                     \u001b[0mhidden_states\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1076\u001b[0m                     \u001b[0mattention_mask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mextended_attention_mask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1192\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m   1193\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1194\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1195\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1196\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/transformers/models/t5/modeling_t5.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, hidden_states, attention_mask, position_bias, encoder_hidden_states, encoder_attention_mask, encoder_decoder_position_bias, layer_head_mask, cross_attn_layer_head_mask, past_key_value, use_cache, output_attentions, return_dict)\u001b[0m\n\u001b[1;32m    717\u001b[0m                 \u001b[0mquery_length\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    718\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 719\u001b[0;31m             cross_attention_outputs = self.layer[1](\n\u001b[0m\u001b[1;32m    720\u001b[0m                 \u001b[0mhidden_states\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    721\u001b[0m                 \u001b[0mkey_value_states\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mencoder_hidden_states\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1192\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m   1193\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1194\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1195\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1196\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/transformers/models/t5/modeling_t5.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, hidden_states, key_value_states, attention_mask, position_bias, layer_head_mask, past_key_value, use_cache, query_length, output_attentions)\u001b[0m\n\u001b[1;32m    632\u001b[0m     ):\n\u001b[1;32m    633\u001b[0m         \u001b[0mnormed_hidden_states\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayer_norm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhidden_states\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 634\u001b[0;31m         attention_output = self.EncDecAttention(\n\u001b[0m\u001b[1;32m    635\u001b[0m             \u001b[0mnormed_hidden_states\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    636\u001b[0m             \u001b[0mmask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattention_mask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1192\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m   1193\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1194\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1195\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1196\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/transformers/models/t5/modeling_t5.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, hidden_states, mask, key_value_states, position_bias, past_key_value, layer_head_mask, query_length, use_cache, output_attentions)\u001b[0m\n\u001b[1;32m    528\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    529\u001b[0m         \u001b[0;31m# compute scores\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 530\u001b[0;31m         scores = torch.matmul(\n\u001b[0m\u001b[1;32m    531\u001b[0m             \u001b[0mquery_states\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey_states\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtranspose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    532\u001b[0m         )  # equivalent of torch.einsum(\"bnqd,bnkd->bnqk\", query_states, key_states), compatible with onnx op>9\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ],
      "source": [
        "import torch\n",
        "from torch.cuda.amp import GradScaler\n",
        "from torch.utils.data import Dataset\n",
        "from transformers import T5ForConditionalGeneration, T5Tokenizer\n",
        "from torch.optim import AdamW\n",
        "from torch.utils.data import DataLoader\n",
        "from project_evaluate import calculate_score\n",
        "import torch.nn as nn\n",
        "import gc\n",
        "import math\n",
        "\n",
        "HF_HUB_DISABLE_SYMLINKS_WARNING = False\n",
        "device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
        "torch.backends.cudnn.benchmark = True\n",
        "\n",
        "class TrainExpDataset(Dataset):\n",
        "    def __init__(self, path):\n",
        "        with open(path, \"r\", encoding=\"utf-8\") as f:\n",
        "            text = f.read()\n",
        "        sentences = text.split(\"\\n\\n\")\n",
        "        splitted = [sentence.split(\"\\nEnglish:\\n\") for sentence in sentences]\n",
        "        self.german_sents = [sentence[0][8:] for sentence in splitted if sentence[0] != \"\"]\n",
        "        self.english_sents = []\n",
        "        self.roots = []\n",
        "        self.modifiers = []\n",
        "        for sentence in splitted:\n",
        "          if sentence[0] != \"\":\n",
        "            english_sent, other = sentence[1].split(\"Roots in English: \")\n",
        "            self.english_sents.append(english_sent)\n",
        "            try:\n",
        "              roots, modifiers = other.split(\"\\n\")\n",
        "            except:\n",
        "              print(other, sentence)\n",
        "            self.roots.append(roots)\n",
        "            modifiers = modifiers.replace(\"Modifiers in English: (\", \"\")\n",
        "            modifiers = modifiers.replace(\"), (\", \"; \")\n",
        "            self.modifiers.append(modifiers[:-1])\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.german_sents)\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        return self.german_sents[index], self.english_sents[index], self.roots[index], self.modifiers[index]\n",
        "\n",
        "class ValExpDataset(Dataset):\n",
        "    def __init__(self, path):\n",
        "        with open(path, \"r\", encoding=\"utf-8\") as f:\n",
        "            text = f.read()\n",
        "        sentences = text.split(\"\\n\\n\")\n",
        "        splitted = [sentence.split(\"\\nRoots in English:\") for sentence in sentences]\n",
        "        self.german_sents = [sentence[0][8:] for sentence in splitted if sentence[0] != \"\"]\n",
        "        self.roots = []\n",
        "        self.modifiers = []\n",
        "        for sentence in splitted:\n",
        "            if len(sentence) == 1:\n",
        "              break\n",
        "            roots, modifiers = sentence[1].split(\"\\n\")\n",
        "            self.roots.append(str(roots.split(\", \")))\n",
        "            modifiers = modifiers.replace(\"Modifiers in English: (\", \"\")\n",
        "            modifiers = modifiers.split(\"), (\")\n",
        "            some = []\n",
        "            for mod in modifiers:\n",
        "              some.append(mod.split(\", \"))\n",
        "            some[-1][-1] = some[-1][-1][:-1]\n",
        "            self.modifiers.append(str(some))\n",
        "\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.german_sents)\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        return self.german_sents[index], self.roots[index], self.modifiers[index]\n",
        "\n",
        "def calc_bleu(epoch, model, tokenizer, val_loader, beams=1, task_prefix=\"translate German to English: \"):\n",
        "    with open(f\"predict{epoch}.labeled\", \"w\", encoding=\"utf-8\") as f:\n",
        "        for i, (input_sequences, english_roots, english_modifiers\n",
        "            ) in enumerate(val_loader):\n",
        "        \n",
        "            # english_roots = [root.replace(\",\", \"\") for root in english_roots]\n",
        "            # english_modifiers = [mod.replace(\",\", \"\").replace(\";\", \"\") for mod in english_modifiers]\n",
        "            english_hints = [f\"[{root + mod}]\" for root, mod in zip(english_roots, english_modifiers)]\n",
        "            if i % 10 == 0:\n",
        "                print(i)\n",
        "            encoding = tokenizer(\n",
        "            [task_prefix +\" ||| roots: \"+ root + \" ||| modifiers: \"+ mod +\" ||| \"+ \" german: \" + sequence for sequence, root, mod in zip(\n",
        "                input_sequences, english_roots, english_modifiers)],  \n",
        "            padding=\"longest\",\n",
        "            max_length=1000,\n",
        "            truncation=True,\n",
        "            return_tensors=\"pt\",\n",
        "            )\n",
        "\n",
        "            with torch.no_grad():\n",
        "                input_ids, attention_mask = encoding.input_ids.to(device, non_blocking=True), encoding.attention_mask.to(\n",
        "                    device, non_blocking=True)\n",
        "                outputs_ids = model.generate(input_ids, attention_mask=attention_mask,\n",
        "                                             max_length=1000, num_beams=beams)\n",
        "                outputs_ids = outputs_ids.tolist()\n",
        "                decoded_outputs = tokenizer.batch_decode(outputs_ids, skip_spaciel_tokens=True)\n",
        "                truncated_outputs = []\n",
        "                for output in decoded_outputs:\n",
        "                    output = output[6:]\n",
        "                    if \"</s>\" in output:\n",
        "                        truncated_outputs.append(output[:output.index(\"</s>\")])\n",
        "                    else:\n",
        "                        truncated_outputs.append(output)\n",
        "                #output_text = tokenizer.decode(outputs_ids[0][1:], skip_spaciel_tokens=True)\n",
        "                #output_text = tokenizer.decode(outputs_ids, skip_spaciel_tokens=True)\n",
        "            for inp, out in zip(input_sequences, truncated_outputs):\n",
        "              f.write(\"German:\\n\")\n",
        "              f.write(inp)\n",
        "              f.write(\"\\nEnglish:\\n\")\n",
        "              f.write(out)\n",
        "              f.write(\"\\n\\n\")\n",
        "    return calculate_score(f\"predict{epoch}.labeled\", \"val.labeled\")\n",
        "\n",
        "\n",
        "def train_epoch(tokenizer, data_loader, model, optimizer, scaler\n",
        "                , task_prefix=\"translate German to English with hints: \"):\n",
        "    losses = []\n",
        "    for i, (input_sequences, target_sequences, english_roots, english_modifiers\n",
        "            ) in enumerate(data_loader):\n",
        "        # english_roots = [root.replace(\",\", \"\") for root in english_roots]\n",
        "        # english_modifiers = [mod.replace(\",\", \"\").replace(\";\", \"\") for mod in english_modifiers]\n",
        "        english_hints = [f\"[{root + mod}]\" for root, mod in zip(english_roots, english_modifiers)]\n",
        "\n",
        "        encoding = tokenizer(\n",
        "            [task_prefix +\" ||| roots: \"+ root + \" ||| modifiers: \"+ mod +\" ||| \"+ \" german: \" + sequence for sequence, root, mod in zip(\n",
        "                input_sequences, english_roots, english_modifiers)],                \n",
        "            padding=\"longest\",\n",
        "            max_length=1000,\n",
        "            truncation=True,\n",
        "            return_tensors=\"pt\",\n",
        "        )\n",
        "        input_ids, attention_mask = encoding.input_ids.to(device, non_blocking=True), encoding.attention_mask.to(device,\n",
        "                                                                                                                 non_blocking=True)\n",
        "\n",
        "        # encode the targets\n",
        "        target_encoding = tokenizer(\n",
        "            target_sequences,\n",
        "            padding=\"longest\",\n",
        "            max_length=1000,\n",
        "            truncation=True,\n",
        "            return_tensors=\"pt\",\n",
        "        )\n",
        "        labels = target_encoding.input_ids.to(device, non_blocking=True)\n",
        "\n",
        "        # replace padding token id's of the labels by -100 so it's ignored by the loss\n",
        "        labels[labels == tokenizer.pad_token_id] = -100\n",
        "\n",
        "        # forward pass\n",
        "        with torch.cuda.amp.autocast():\n",
        "            loss = model(input_ids=input_ids, attention_mask=attention_mask, labels=labels).loss\n",
        "        if not math.isnan(loss.item()):\n",
        "          losses.append(loss.item())\n",
        "        if i % 20 == 0:\n",
        "          print(loss.item())\n",
        "        #loss.backward()\n",
        "        scaler.scale(loss).backward()\n",
        "        if True or (i+1) % 2 == 0:\n",
        "            #optimizer.step()\n",
        "            scaler.step(optimizer)\n",
        "            scaler.update()\n",
        "            #optimizer.zero_grad(set_to_none=True)\n",
        "    return sum(losses) / len(losses)\n",
        "\n",
        "\n",
        "def main():\n",
        "    torch.cuda.empty_cache()\n",
        "    gc.collect()\n",
        "    print(\"\\n\\n\")\n",
        "    print(device)\n",
        "    print(\"\\n\\n\")\n",
        "    model = T5ForConditionalGeneration.from_pretrained(\"t5-base\")\n",
        "    model.to(device)\n",
        "    optimizer = AdamW(model.parameters(), lr=1e-4)\n",
        "    torch.save(model.state_dict(), f'weights{0}.pkl')\n",
        "    #torch.save(optimizer.state_dict(), f\"optimizer_epoch_{0}.pt\")\n",
        "    tokenizer = T5Tokenizer.from_pretrained(\"t5-base\")\n",
        "\n",
        "    #batch_size = 16\n",
        "    batch_size = 8\n",
        "\n",
        "    # train_dataset = TrainExpDataset(\"train.extra_labeled\")\n",
        "    # train_loader = DataLoader(train_dataset, batch_size=batch_size, pin_memory=True, num_workers=4,\n",
        "    #                           shuffle=True)\n",
        "\n",
        "    val_dataset = ValExpDataset(\"val.unlabeled\")\n",
        "    #val_loader = DataLoader(val_dataset, batch_size=20, pin_memory=True)\n",
        "    val_loader = DataLoader(val_dataset, batch_size=8, pin_memory=True)\n",
        "\n",
        "    #model.eval()\n",
        "    #calc_bleu(0, model, tokenizer, val_loader)\n",
        "\n",
        "    model.train()\n",
        "    scaler = GradScaler()\n",
        "    epochs = 15\n",
        "\n",
        "    wide_scores = []\n",
        "    avg_losses = []\n",
        "    #config = T5Config.from_pretrained('t5-base')\n",
        "    #config.fp16 = False\n",
        "    for epoch in range(epochs):\n",
        "      #torch.cuda.empty_cache()\n",
        "      #gc.collect() \n",
        "      #model = T5ForConditionalGeneration(config=config)\n",
        "      #model = T5ForConditionalGeneration.from_pretrained(\"t5-base\")\n",
        "      #model.to(device) \n",
        "      #model.load_state_dict(torch.load(f'weights{epoch}.pkl'))\n",
        "      #optimizer = AdamW(model.parameters(), lr=1e-4)\n",
        "      #optimizer.load_state_dict(torch.load(f\"optimizer_epoch_{epoch}.pt\"))\n",
        "      train_dataset = TrainExpDataset(f\"train.extra_labeled{epoch}\")\n",
        "      train_loader = DataLoader(train_dataset, batch_size=batch_size, pin_memory=True,\n",
        "                            shuffle=True)\n",
        "      print(f'Epoch {epoch}/{epochs}')\n",
        "      model.train()\n",
        "      avg_loss = train_epoch(tokenizer, train_loader, model, optimizer, scaler)\n",
        "      print(f\"The avg loss is {avg_loss}\")\n",
        "      avg_losses.append(avg_loss)\n",
        "      model.eval()\n",
        "      torch.save(optimizer.state_dict(), f\"optimizer_epoch_{epoch+1}.pt\")\n",
        "      with torch.no_grad():\n",
        "          wide_score = calc_bleu(epoch, model, tokenizer, val_loader, beams=4)\n",
        "          print(f\"The wider bleu is {wide_score}\")\n",
        "          wide_scores.append(wide_score)\n",
        "          torch.save(model.state_dict(), f'weights{epoch+1}.pkl')\n",
        "\n",
        "      print(wide_scores)\n",
        "      print(avg_losses)\n",
        "\n",
        "# if _name_ == \"_main_\":\n",
        "main()"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "from torch.cuda.amp import GradScaler\n",
        "from torch.utils.data import Dataset\n",
        "from transformers import T5ForConditionalGeneration, T5Tokenizer\n",
        "from torch.optim import AdamW\n",
        "from torch.utils.data import DataLoader\n",
        "from project_evaluate import calculate_score\n",
        "import math\n",
        "\n",
        "HF_HUB_DISABLE_SYMLINKS_WARNING = False\n",
        "device = torch.device(\"cuda\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
        "torch.backends.cudnn.benchmark = True\n",
        "\n",
        "class ValExpDataset(Dataset):\n",
        "    def __init__(self, path):\n",
        "        with open(path, \"r\", encoding=\"utf-8\") as f:\n",
        "            text = f.read()\n",
        "        sentences = text.split(\"\\n\\n\")\n",
        "        splitted = [sentence.split(\"\\nRoots in English:\") for sentence in sentences]\n",
        "        self.german_sents = [sentence[0][8:] for sentence in splitted if sentence[0] != \"\"]\n",
        "        self.roots = []\n",
        "        self.modifiers = []\n",
        "        for sentence in splitted:\n",
        "            if len(sentence) == 1:\n",
        "              break\n",
        "            roots, modifiers = sentence[1].split(\"\\n\")\n",
        "            self.roots.append(str(roots.split(\", \")))\n",
        "            modifiers = modifiers.replace(\"Modifiers in English: (\", \"\")\n",
        "            modifiers = modifiers.split(\"), (\")\n",
        "            some = []\n",
        "            for mod in modifiers:\n",
        "              some.append(mod.split(\", \"))\n",
        "            some[-1][-1] = some[-1][-1][:-1]\n",
        "            self.modifiers.append(str(some))\n",
        "\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.german_sents)\n",
        "\n",
        "    def __getitem__(self, index):\n",
        "        return self.german_sents[index], self.roots[index], self.modifiers[index]\n",
        "\n",
        "def calc_bleu(epoch, model, tokenizer, val_loader, task_prefix=\"translate German to English: \"):\n",
        "    with open(f\"predict{epoch}.labeled\", \"w\", encoding=\"utf-8\") as f:\n",
        "        for i, (input_sequences, english_roots, english_modifiers\n",
        "            ) in enumerate(val_loader):\n",
        "        \n",
        "            # english_roots = [root.replace(\",\", \"\") for root in english_roots]\n",
        "            # english_modifiers = [mod.replace(\",\", \"\").replace(\";\", \"\") for mod in english_modifiers]\n",
        "            english_hints = [f\"[{root + mod}]\" for root, mod in zip(english_roots, english_modifiers)]\n",
        "            if i % 10 == 0:\n",
        "                print(i)\n",
        "            encoding = tokenizer(\n",
        "            [task_prefix +\" ||| roots: \"+ root + \" ||| modifiers: \"+ mod +\" ||| \"+ \" german: \" + sequence for sequence, root, mod in zip(\n",
        "                input_sequences, english_roots, english_modifiers)],  \n",
        "            padding=\"longest\",\n",
        "            max_length=1000,\n",
        "            truncation=True,\n",
        "            return_tensors=\"pt\",\n",
        "            )\n",
        "\n",
        "            with torch.no_grad():\n",
        "                input_ids, attention_mask = encoding.input_ids.to(device, non_blocking=True), encoding.attention_mask.to(\n",
        "                    device, non_blocking=True)\n",
        "                outputs_ids = model.generate(input_ids, attention_mask=attention_mask,\n",
        "                                             max_length=1000,num_beams=3)\n",
        "                outputs_ids = outputs_ids.tolist()\n",
        "                decoded_outputs = tokenizer.batch_decode(outputs_ids, skip_spaciel_tokens=True)\n",
        "                truncated_outputs = []\n",
        "                for output in decoded_outputs:\n",
        "                    output = output[6:]\n",
        "                    if \"</s>\" in output:\n",
        "                        truncated_outputs.append(output[:output.index(\"</s>\")])\n",
        "                    else:\n",
        "                        truncated_outputs.append(output)\n",
        "                #output_text = tokenizer.decode(outputs_ids[0][1:], skip_spaciel_tokens=True)\n",
        "                #output_text = tokenizer.decode(outputs_ids, skip_spaciel_tokens=True)\n",
        "            for inp, out in zip(input_sequences, truncated_outputs):\n",
        "              f.write(\"German:\\n\")\n",
        "              f.write(inp)\n",
        "              f.write(\"\\nEnglish:\\n\")\n",
        "              f.write(out)\n",
        "              f.write(\"\\n\\n\")\n",
        "    return calculate_score(f\"predict{epoch}.labeled\", \"val.labeled\")\n",
        "\n",
        "def main():\n",
        "    print(\"\\n\\n\")\n",
        "    print(device)\n",
        "    print(\"\\n\\n\")\n",
        "    model = T5ForConditionalGeneration.from_pretrained(\"t5-base\")\n",
        "    model.load_state_dict(torch.load(\"weights13.pkl\"))\n",
        "    model.to(device)\n",
        "    tokenizer = T5Tokenizer.from_pretrained(\"t5-base\")\n",
        "\n",
        "    batch_size = 8\n",
        "    val_dataset = ValExpDataset(\"val.unlabeled\")\n",
        "    val_loader = DataLoader(val_dataset, batch_size=8, pin_memory=True)\n",
        "    model.eval()\n",
        "    bleu = calc_bleu(0, model, tokenizer, val_loader)\n",
        "    print(bleu)\n",
        "      \n",
        "# if _name_ == \"_main_\":\n",
        "main()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "3R7BA6bStdv_",
        "outputId": "c9f223fd-5db3-4c4a-e9f4-bf4af2550cd9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\n",
            "\n",
            "cuda\n",
            "\n",
            "\n",
            "\n",
            "Epoch 0/15\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.9/dist-packages/transformers/models/t5/tokenization_t5.py:163: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5.\n",
            "For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`.\n",
            "- Be aware that you SHOULD NOT rely on t5-base automatically truncating your input to 512 when padding/encoding.\n",
            "- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding.\n",
            "- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value.\n",
            "  warnings.warn(\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "2.856860876083374\n",
            "1.9917386770248413\n",
            "2.0872743129730225\n",
            "1.8690707683563232\n",
            "1.5727643966674805\n",
            "1.788523554801941\n",
            "1.6146329641342163\n",
            "1.5288859605789185\n",
            "1.7197693586349487\n",
            "1.5664262771606445\n",
            "1.7692183256149292\n",
            "1.6935234069824219\n",
            "1.6563704013824463\n",
            "1.7148308753967285\n",
            "1.6467167139053345\n",
            "1.656943678855896\n",
            "1.5903360843658447\n",
            "1.5002624988555908\n",
            "1.4455012083053589\n",
            "1.5768861770629883\n",
            "1.6330647468566895\n",
            "1.580560564994812\n",
            "1.6009604930877686\n",
            "1.5403094291687012\n",
            "1.7366607189178467\n",
            "1.378781795501709\n",
            "1.504751443862915\n",
            "1.3752506971359253\n",
            "1.5961337089538574\n",
            "1.4685829877853394\n",
            "1.5168734788894653\n",
            "1.3688346147537231\n",
            "1.4661755561828613\n",
            "1.3573142290115356\n",
            "1.2625622749328613\n",
            "1.4265047311782837\n",
            "1.356203317642212\n",
            "1.3743501901626587\n",
            "1.2766581773757935\n",
            "1.3651083707809448\n",
            "1.615344762802124\n",
            "1.931228518486023\n",
            "1.350174069404602\n",
            "1.5812652111053467\n",
            "1.2525349855422974\n",
            "1.4267679452896118\n",
            "1.3822673559188843\n",
            "1.307924509048462\n",
            "1.18259859085083\n",
            "1.1129114627838135\n",
            "1.361085057258606\n",
            "1.2357499599456787\n",
            "1.4924968481063843\n",
            "1.479532241821289\n",
            "1.550001621246338\n",
            "1.4467613697052002\n",
            "1.2663670778274536\n",
            "1.3324494361877441\n",
            "1.4133387804031372\n",
            "1.4133696556091309\n",
            "1.6094810962677002\n",
            "1.3481941223144531\n",
            "1.4741190671920776\n",
            "0\n",
            "10\n",
            "20\n",
            "30\n",
            "40\n",
            "50\n",
            "60\n",
            "70\n",
            "80\n",
            "90\n",
            "100\n",
            "110\n",
            "120\n",
            "[35.62]\n",
            "[1.514411053943634]\n",
            "Epoch 1/15\n",
            "1.2043166160583496\n",
            "1.1079602241516113\n",
            "1.1481355428695679\n",
            "1.1750837564468384\n",
            "1.2312722206115723\n",
            "1.3518000841140747\n",
            "1.2039130926132202\n",
            "1.046372890472412\n",
            "1.237728238105774\n",
            "1.3021881580352783\n",
            "1.30644690990448\n",
            "1.1979748010635376\n",
            "1.3617647886276245\n",
            "1.317750096321106\n",
            "1.3378013372421265\n",
            "1.2071397304534912\n",
            "1.337941288948059\n",
            "1.2436949014663696\n",
            "1.2755646705627441\n",
            "1.3509174585342407\n",
            "1.4383565187454224\n",
            "1.4272222518920898\n",
            "1.1473219394683838\n",
            "1.2535845041275024\n",
            "1.259732961654663\n",
            "1.4987751245498657\n",
            "0.9652889966964722\n",
            "1.19859778881073\n",
            "1.0882569551467896\n",
            "1.183676838874817\n",
            "1.3617942333221436\n",
            "1.1894996166229248\n",
            "1.1270298957824707\n",
            "1.2161266803741455\n",
            "1.3807374238967896\n",
            "1.132705807685852\n",
            "1.1701698303222656\n",
            "1.3515965938568115\n",
            "1.09407377243042\n",
            "1.3561036586761475\n",
            "1.340600848197937\n",
            "1.2398022413253784\n",
            "1.4361140727996826\n",
            "1.2125197649002075\n",
            "1.2599653005599976\n",
            "1.334496259689331\n",
            "1.1334493160247803\n",
            "1.252022624015808\n",
            "1.2680188417434692\n",
            "1.1056865453720093\n",
            "1.255480408668518\n",
            "1.2182095050811768\n",
            "1.211086630821228\n",
            "1.28273606300354\n",
            "1.3037351369857788\n",
            "1.3966835737228394\n",
            "1.1564674377441406\n",
            "1.0337700843811035\n",
            "1.1379358768463135\n",
            "1.0833278894424438\n",
            "1.1429039239883423\n",
            "1.5029078722000122\n",
            "1.4236613512039185\n",
            "0\n",
            "10\n",
            "20\n",
            "30\n",
            "40\n",
            "50\n",
            "60\n",
            "70\n",
            "80\n",
            "90\n",
            "100\n",
            "110\n",
            "120\n",
            "[35.62, 37.07]\n",
            "[1.514411053943634, 1.2403838464260102]\n",
            "Epoch 2/15\n",
            "1.1354436874389648\n",
            "1.3427999019622803\n",
            "1.0581327676773071\n",
            "1.1683878898620605\n",
            "1.2261571884155273\n",
            "1.150458812713623\n",
            "1.1353834867477417\n",
            "1.214322805404663\n",
            "1.1593432426452637\n",
            "1.2426077127456665\n",
            "0.8989672064781189\n",
            "1.1914178133010864\n",
            "1.0474776029586792\n",
            "1.1152796745300293\n",
            "1.0294122695922852\n",
            "1.1525771617889404\n",
            "1.1201211214065552\n",
            "1.0794363021850586\n",
            "1.1905463933944702\n",
            "1.1201515197753906\n",
            "1.1320133209228516\n",
            "1.080031394958496\n",
            "1.1738696098327637\n",
            "1.3764339685440063\n",
            "0.9505805969238281\n",
            "0.8927780985832214\n",
            "nan\n",
            "nan\n",
            "1.267228603363037\n",
            "1.180363416671753\n",
            "0.9960023164749146\n",
            "nan\n",
            "1.1269508600234985\n",
            "1.2753509283065796\n",
            "1.1850148439407349\n",
            "1.3527905941009521\n",
            "1.0874855518341064\n",
            "1.0565483570098877\n",
            "1.123153567314148\n",
            "1.0812031030654907\n",
            "1.2466713190078735\n",
            "1.1707704067230225\n",
            "1.131432294845581\n",
            "1.0624699592590332\n",
            "1.1862620115280151\n",
            "0.9822898507118225\n",
            "nan\n",
            "1.15511953830719\n",
            "1.151154637336731\n",
            "1.0737096071243286\n",
            "1.0526881217956543\n",
            "1.0227787494659424\n",
            "1.2842402458190918\n",
            "1.2746825218200684\n",
            "1.1575971841812134\n",
            "1.0937851667404175\n",
            "1.1644104719161987\n",
            "1.3047316074371338\n",
            "1.1855309009552002\n",
            "1.1536836624145508\n",
            "1.0250333547592163\n",
            "1.2129576206207275\n",
            "1.119208812713623\n",
            "0\n",
            "10\n",
            "20\n",
            "30\n",
            "40\n",
            "50\n",
            "60\n",
            "70\n",
            "80\n",
            "90\n",
            "100\n",
            "110\n",
            "120\n",
            "[35.62, 37.07, 37.69]\n",
            "[1.514411053943634, 1.2403838464260102, 1.1372041819594236]\n",
            "Epoch 3/15\n",
            "1.3232990503311157\n",
            "1.3524621725082397\n",
            "1.2671140432357788\n",
            "0.9254850149154663\n",
            "1.162297248840332\n",
            "1.0243680477142334\n",
            "1.0258872509002686\n",
            "1.1517175436019897\n",
            "1.0399712324142456\n",
            "1.0701242685317993\n",
            "1.3173649311065674\n",
            "nan\n",
            "1.0628907680511475\n",
            "1.1937419176101685\n",
            "nan\n",
            "1.2873425483703613\n",
            "0.924365222454071\n",
            "1.4637893438339233\n",
            "1.0262423753738403\n",
            "1.1063201427459717\n",
            "1.1487873792648315\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "nan\n",
            "0\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-3-f8131ce298e0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m    208\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    209\u001b[0m \u001b[0;31m# if _name_ == \"_main_\":\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 210\u001b[0;31m \u001b[0mmain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m<ipython-input-3-f8131ce298e0>\u001b[0m in \u001b[0;36mmain\u001b[0;34m()\u001b[0m\n\u001b[1;32m    200\u001b[0m       \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    201\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 202\u001b[0;31m           \u001b[0mwide_score\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcalc_bleu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtokenizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_loader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    203\u001b[0m           \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msave\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34mf'weights{epoch}.pkl'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    204\u001b[0m           \u001b[0mwide_scores\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mwide_score\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-3-f8131ce298e0>\u001b[0m in \u001b[0;36mcalc_bleu\u001b[0;34m(epoch, model, tokenizer, val_loader, task_prefix)\u001b[0m\n\u001b[1;32m     92\u001b[0m                 input_ids, attention_mask = encoding.input_ids.to(device, non_blocking=True), encoding.attention_mask.to(\n\u001b[1;32m     93\u001b[0m                     device, non_blocking=True)\n\u001b[0;32m---> 94\u001b[0;31m                 outputs_ids = model.generate(input_ids, attention_mask=attention_mask,\n\u001b[0m\u001b[1;32m     95\u001b[0m                                              max_length=1000,num_beams=3)\n\u001b[1;32m     96\u001b[0m                 \u001b[0moutputs_ids\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moutputs_ids\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtolist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/torch/autograd/grad_mode.py\u001b[0m in \u001b[0;36mdecorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     25\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclone\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     28\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/transformers/generation/utils.py\u001b[0m in \u001b[0;36mgenerate\u001b[0;34m(self, inputs, generation_config, logits_processor, stopping_criteria, prefix_allowed_tokens_fn, synced_gpus, **kwargs)\u001b[0m\n\u001b[1;32m   1488\u001b[0m             )\n\u001b[1;32m   1489\u001b[0m             \u001b[0;31m# 13. run beam search\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1490\u001b[0;31m             return self.beam_search(\n\u001b[0m\u001b[1;32m   1491\u001b[0m                 \u001b[0minput_ids\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1492\u001b[0m                 \u001b[0mbeam_scorer\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/transformers/generation/utils.py\u001b[0m in \u001b[0;36mbeam_search\u001b[0;34m(self, input_ids, beam_scorer, logits_processor, stopping_criteria, max_length, pad_token_id, eos_token_id, output_attentions, output_hidden_states, output_scores, return_dict_in_generate, synced_gpus, **model_kwargs)\u001b[0m\n\u001b[1;32m   2747\u001b[0m             \u001b[0mmodel_inputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprepare_inputs_for_generation\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_ids\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mmodel_kwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2748\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2749\u001b[0;31m             outputs = self(\n\u001b[0m\u001b[1;32m   2750\u001b[0m                 \u001b[0;34m**\u001b[0m\u001b[0mmodel_inputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2751\u001b[0m                 \u001b[0mreturn_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1192\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m   1193\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1194\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1195\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1196\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/transformers/models/t5/modeling_t5.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input_ids, attention_mask, decoder_input_ids, decoder_attention_mask, head_mask, decoder_head_mask, cross_attn_head_mask, encoder_outputs, past_key_values, inputs_embeds, decoder_inputs_embeds, labels, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m   1702\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1703\u001b[0m         \u001b[0;31m# Decode\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1704\u001b[0;31m         decoder_outputs = self.decoder(\n\u001b[0m\u001b[1;32m   1705\u001b[0m             \u001b[0minput_ids\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdecoder_input_ids\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1706\u001b[0m             \u001b[0mattention_mask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdecoder_attention_mask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1192\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m   1193\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1194\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1195\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1196\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/transformers/models/t5/modeling_t5.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input_ids, attention_mask, encoder_hidden_states, encoder_attention_mask, inputs_embeds, head_mask, cross_attn_head_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m   1072\u001b[0m                 )\n\u001b[1;32m   1073\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1074\u001b[0;31m                 layer_outputs = layer_module(\n\u001b[0m\u001b[1;32m   1075\u001b[0m                     \u001b[0mhidden_states\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1076\u001b[0m                     \u001b[0mattention_mask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mextended_attention_mask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1192\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m   1193\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1194\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1195\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1196\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/transformers/models/t5/modeling_t5.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, hidden_states, attention_mask, position_bias, encoder_hidden_states, encoder_attention_mask, encoder_decoder_position_bias, layer_head_mask, cross_attn_layer_head_mask, past_key_value, use_cache, output_attentions, return_dict)\u001b[0m\n\u001b[1;32m    691\u001b[0m             \u001b[0mself_attn_past_key_value\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcross_attn_past_key_value\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    692\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 693\u001b[0;31m         self_attention_outputs = self.layer[0](\n\u001b[0m\u001b[1;32m    694\u001b[0m             \u001b[0mhidden_states\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    695\u001b[0m             \u001b[0mattention_mask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattention_mask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1192\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m   1193\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1194\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1195\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1196\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/transformers/models/t5/modeling_t5.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, hidden_states, attention_mask, position_bias, layer_head_mask, past_key_value, use_cache, output_attentions)\u001b[0m\n\u001b[1;32m    598\u001b[0m     ):\n\u001b[1;32m    599\u001b[0m         \u001b[0mnormed_hidden_states\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayer_norm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhidden_states\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 600\u001b[0;31m         attention_output = self.SelfAttention(\n\u001b[0m\u001b[1;32m    601\u001b[0m             \u001b[0mnormed_hidden_states\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    602\u001b[0m             \u001b[0mmask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattention_mask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1192\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m   1193\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1194\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1195\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1196\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/transformers/models/t5/modeling_t5.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, hidden_states, mask, key_value_states, position_bias, past_key_value, layer_head_mask, query_length, use_cache, output_attentions)\u001b[0m\n\u001b[1;32m    517\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    518\u001b[0m         \u001b[0;31m# get query states\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 519\u001b[0;31m         \u001b[0mquery_states\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mq\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhidden_states\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# (batch_size, n_heads, seq_length, dim_per_head)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    520\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    521\u001b[0m         \u001b[0;31m# get key/value states\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1192\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m   1193\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1194\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1195\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1196\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.9/dist-packages/torch/nn/modules/linear.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    112\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    113\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mTensor\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 114\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlinear\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mweight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbias\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    115\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    116\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mextra_repr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import T5Config\n",
        "config = T5Config.from_pretrained('t5-base')\n",
        "config.fp16 = False\n",
        "model = T5ForConditionalGeneration(config=config)"
      ],
      "metadata": {
        "id": "ZP-aus67driT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = T5ForConditionalGeneration.from_pretrained('t5-base')"
      ],
      "metadata": {
        "id": "nIedqeide84N"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers --upgrade"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_U5c7-FVfuGN",
        "outputId": "c28bde6b-15ac-436b-ad2b-a92de55358a5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: transformers in /usr/local/lib/python3.9/dist-packages (4.27.2)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.9/dist-packages (from transformers) (4.65.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.9/dist-packages (from transformers) (23.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (1.22.4)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.9/dist-packages (from transformers) (2.27.1)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.9/dist-packages (from transformers) (0.13.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from transformers) (3.10.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (2022.10.31)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.11.0 in /usr/local/lib/python3.9/dist-packages (from transformers) (0.13.3)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (4.5.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (3.4)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (2.0.12)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (2022.12.7)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (1.26.15)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.config"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6m_9KQ00e5DW",
        "outputId": "23883daa-b337-4262-bbef-0ca6797ee10a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "T5Config {\n",
              "  \"_name_or_path\": \"t5-base\",\n",
              "  \"architectures\": [\n",
              "    \"T5ForConditionalGeneration\"\n",
              "  ],\n",
              "  \"d_ff\": 3072,\n",
              "  \"d_kv\": 64,\n",
              "  \"d_model\": 768,\n",
              "  \"decoder_start_token_id\": 0,\n",
              "  \"dense_act_fn\": \"relu\",\n",
              "  \"dropout_rate\": 0.1,\n",
              "  \"eos_token_id\": 1,\n",
              "  \"feed_forward_proj\": \"relu\",\n",
              "  \"initializer_factor\": 1.0,\n",
              "  \"is_encoder_decoder\": true,\n",
              "  \"is_gated_act\": false,\n",
              "  \"layer_norm_epsilon\": 1e-06,\n",
              "  \"model_type\": \"t5\",\n",
              "  \"n_positions\": 512,\n",
              "  \"num_decoder_layers\": 12,\n",
              "  \"num_heads\": 12,\n",
              "  \"num_layers\": 12,\n",
              "  \"output_past\": true,\n",
              "  \"pad_token_id\": 0,\n",
              "  \"relative_attention_max_distance\": 128,\n",
              "  \"relative_attention_num_buckets\": 32,\n",
              "  \"task_specific_params\": {\n",
              "    \"summarization\": {\n",
              "      \"early_stopping\": true,\n",
              "      \"length_penalty\": 2.0,\n",
              "      \"max_length\": 200,\n",
              "      \"min_length\": 30,\n",
              "      \"no_repeat_ngram_size\": 3,\n",
              "      \"num_beams\": 4,\n",
              "      \"prefix\": \"summarize: \"\n",
              "    },\n",
              "    \"translation_en_to_de\": {\n",
              "      \"early_stopping\": true,\n",
              "      \"max_length\": 300,\n",
              "      \"num_beams\": 4,\n",
              "      \"prefix\": \"translate English to German: \"\n",
              "    },\n",
              "    \"translation_en_to_fr\": {\n",
              "      \"early_stopping\": true,\n",
              "      \"max_length\": 300,\n",
              "      \"num_beams\": 4,\n",
              "      \"prefix\": \"translate English to French: \"\n",
              "    },\n",
              "    \"translation_en_to_ro\": {\n",
              "      \"early_stopping\": true,\n",
              "      \"max_length\": 300,\n",
              "      \"num_beams\": 4,\n",
              "      \"prefix\": \"translate English to Romanian: \"\n",
              "    }\n",
              "  },\n",
              "  \"transformers_version\": \"4.27.2\",\n",
              "  \"use_cache\": true,\n",
              "  \"vocab_size\": 32128\n",
              "}"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "help(T5ForConditionalGeneration.from_pretrained)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bMsm-zy3dubI",
        "outputId": "ffe2ce15-3d49-4d35-942c-ac4e347d05b8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Help on method from_pretrained in module transformers.modeling_utils:\n",
            "\n",
            "from_pretrained(pretrained_model_name_or_path: Union[str, os.PathLike, NoneType], *model_args, **kwargs) method of builtins.type instance\n",
            "    Instantiate a pretrained pytorch model from a pre-trained model configuration.\n",
            "    \n",
            "    The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated). To train\n",
            "    the model, you should first set it back in training mode with `model.train()`.\n",
            "    \n",
            "    The warning *Weights from XXX not initialized from pretrained model* means that the weights of XXX do not come\n",
            "    pretrained with the rest of the model. It is up to you to train those weights with a downstream fine-tuning\n",
            "    task.\n",
            "    \n",
            "    The warning *Weights from XXX not used in YYY* means that the layer XXX is not used by YYY, therefore those\n",
            "    weights are discarded.\n",
            "    \n",
            "    Parameters:\n",
            "        pretrained_model_name_or_path (`str` or `os.PathLike`, *optional*):\n",
            "            Can be either:\n",
            "    \n",
            "                - A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.\n",
            "                  Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced under a\n",
            "                  user or organization name, like `dbmdz/bert-base-german-cased`.\n",
            "                - A path to a *directory* containing model weights saved using\n",
            "                  [`~PreTrainedModel.save_pretrained`], e.g., `./my_model_directory/`.\n",
            "                - A path or url to a *tensorflow index checkpoint file* (e.g, `./tf_model/model.ckpt.index`). In\n",
            "                  this case, `from_tf` should be set to `True` and a configuration object should be provided as\n",
            "                  `config` argument. This loading path is slower than converting the TensorFlow checkpoint in a\n",
            "                  PyTorch model using the provided conversion scripts and loading the PyTorch model afterwards.\n",
            "                - A path or url to a model folder containing a *flax checkpoint file* in *.msgpack* format (e.g,\n",
            "                  `./flax_model/` containing `flax_model.msgpack`). In this case, `from_flax` should be set to\n",
            "                  `True`.\n",
            "                - `None` if you are both providing the configuration and state dictionary (resp. with keyword\n",
            "                  arguments `config` and `state_dict`).\n",
            "        model_args (sequence of positional arguments, *optional*):\n",
            "            All remaining positional arguments will be passed to the underlying model's `__init__` method.\n",
            "        config (`Union[PretrainedConfig, str, os.PathLike]`, *optional*):\n",
            "            Can be either:\n",
            "    \n",
            "                - an instance of a class derived from [`PretrainedConfig`],\n",
            "                - a string or path valid as input to [`~PretrainedConfig.from_pretrained`].\n",
            "    \n",
            "            Configuration for the model to use instead of an automatically loaded configuration. Configuration can\n",
            "            be automatically loaded when:\n",
            "    \n",
            "                - The model is a model provided by the library (loaded with the *model id* string of a pretrained\n",
            "                  model).\n",
            "                - The model was saved using [`~PreTrainedModel.save_pretrained`] and is reloaded by supplying the\n",
            "                  save directory.\n",
            "                - The model is loaded by supplying a local directory as `pretrained_model_name_or_path` and a\n",
            "                  configuration JSON file named *config.json* is found in the directory.\n",
            "        state_dict (`Dict[str, torch.Tensor]`, *optional*):\n",
            "            A state dictionary to use instead of a state dictionary loaded from saved weights file.\n",
            "    \n",
            "            This option can be used if you want to create a model from a pretrained configuration but load your own\n",
            "            weights. In this case though, you should check if using [`~PreTrainedModel.save_pretrained`] and\n",
            "            [`~PreTrainedModel.from_pretrained`] is not a simpler option.\n",
            "        cache_dir (`Union[str, os.PathLike]`, *optional*):\n",
            "            Path to a directory in which a downloaded pretrained model configuration should be cached if the\n",
            "            standard cache should not be used.\n",
            "        from_tf (`bool`, *optional*, defaults to `False`):\n",
            "            Load the model weights from a TensorFlow checkpoint save file (see docstring of\n",
            "            `pretrained_model_name_or_path` argument).\n",
            "        from_flax (`bool`, *optional*, defaults to `False`):\n",
            "            Load the model weights from a Flax checkpoint save file (see docstring of\n",
            "            `pretrained_model_name_or_path` argument).\n",
            "        ignore_mismatched_sizes (`bool`, *optional*, defaults to `False`):\n",
            "            Whether or not to raise an error if some of the weights from the checkpoint do not have the same size\n",
            "            as the weights of the model (if for instance, you are instantiating a model with 10 labels from a\n",
            "            checkpoint with 3 labels).\n",
            "        force_download (`bool`, *optional*, defaults to `False`):\n",
            "            Whether or not to force the (re-)download of the model weights and configuration files, overriding the\n",
            "            cached versions if they exist.\n",
            "        resume_download (`bool`, *optional*, defaults to `False`):\n",
            "            Whether or not to delete incompletely received files. Will attempt to resume the download if such a\n",
            "            file exists.\n",
            "        proxies (`Dict[str, str]`, *optional*):\n",
            "            A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',\n",
            "            'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.\n",
            "        output_loading_info(`bool`, *optional*, defaults to `False`):\n",
            "            Whether ot not to also return a dictionary containing missing keys, unexpected keys and error messages.\n",
            "        local_files_only(`bool`, *optional*, defaults to `False`):\n",
            "            Whether or not to only look at local files (i.e., do not try to download the model).\n",
            "        use_auth_token (`str` or `bool`, *optional*):\n",
            "            The token to use as HTTP bearer authorization for remote files. If `True`, or not specified, will use\n",
            "            the token generated when running `huggingface-cli login` (stored in `~/.huggingface`).\n",
            "        revision (`str`, *optional*, defaults to `\"main\"`):\n",
            "            The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a\n",
            "            git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any\n",
            "            identifier allowed by git.\n",
            "    \n",
            "            <Tip>\n",
            "    \n",
            "            To test a pull request you made on the Hub, you can pass `revision=\"refs/pr/<pr_number>\".\n",
            "    \n",
            "            </Tip>\n",
            "    \n",
            "        mirror (`str`, *optional*):\n",
            "            Mirror source to accelerate downloads in China. If you are from China and have an accessibility\n",
            "            problem, you can set this option to resolve it. Note that we do not guarantee the timeliness or safety.\n",
            "            Please refer to the mirror site for more information.\n",
            "        _fast_init(`bool`, *optional*, defaults to `True`):\n",
            "            Whether or not to disable fast initialization.\n",
            "    \n",
            "            <Tip warning={true}>\n",
            "    \n",
            "            One should only disable *_fast_init* to ensure backwards compatibility with `transformers.__version__ <\n",
            "            4.6.0` for seeded model initialization. This argument will be removed at the next major version. See\n",
            "            [pull request 11471](https://github.com/huggingface/transformers/pull/11471) for more information.\n",
            "    \n",
            "            </Tip>\n",
            "    \n",
            "        > Parameters for big model inference\n",
            "    \n",
            "        low_cpu_mem_usage(`bool`, *optional*):\n",
            "            Tries to not use more than 1x model size in CPU memory (including peak memory) while loading the model.\n",
            "            This is an experimental feature and a subject to change at any moment.\n",
            "        torch_dtype (`str` or `torch.dtype`, *optional*):\n",
            "            Override the default `torch.dtype` and load the model under a specific `dtype`. The different options\n",
            "            are:\n",
            "    \n",
            "            1. `torch.float16` or `torch.bfloat16` or `torch.float`: load in a specified\n",
            "              `dtype`, ignoring the model's `config.torch_dtype` if one exists. If not specified\n",
            "              - the model will get loaded in `torch.float` (fp32).\n",
            "    \n",
            "            2. `\"auto\"` - A `torch_dtype` entry in the `config.json` file of the model will be\n",
            "              attempted to be used. If this entry isn't found then next check the `dtype` of the first weight in\n",
            "              the checkpoint that's of a floating point type and use that as `dtype`. This will load the model\n",
            "              using the `dtype` it was saved in at the end of the training. It can't be used as an indicator of how\n",
            "              the model was trained. Since it could be trained in one of half precision dtypes, but saved in fp32.\n",
            "    \n",
            "            <Tip>\n",
            "    \n",
            "            For some models the `dtype` they were trained in is unknown - you may try to check the model's paper or\n",
            "            reach out to the authors and ask them to add this information to the model's card and to insert the\n",
            "            `torch_dtype` entry in `config.json` on the hub.\n",
            "    \n",
            "            </Tip>\n",
            "    \n",
            "        device_map (`str` or `Dict[str, Union[int, str, torch.device]]`, *optional*):\n",
            "            A map that specifies where each submodule should go. It doesn't need to be refined to each\n",
            "            parameter/buffer name, once a given module name is inside, every submodule of it will be sent to the\n",
            "            same device.\n",
            "    \n",
            "            To have Accelerate compute the most optimized `device_map` automatically, set `device_map=\"auto\"`. For\n",
            "            more information about each option see [designing a device\n",
            "            map](https://hf.co/docs/accelerate/main/en/usage_guides/big_modeling#designing-a-device-map).\n",
            "        max_memory (`Dict`, *optional*):\n",
            "            A dictionary device identifier to maximum memory. Will default to the maximum memory available for each\n",
            "            GPU and the available CPU RAM if unset.\n",
            "        offload_folder (`str` or `os.PathLike`, *optional*):\n",
            "            If the `device_map` contains any value `\"disk\"`, the folder where we will offload weights.\n",
            "        offload_state_dict (`bool`, *optional*):\n",
            "            If `True`, will temporarily offload the CPU state dict to the hard drive to avoid getting out of CPU\n",
            "            RAM if the weight of the CPU state dict + the biggest shard of the checkpoint does not fit. Defaults to\n",
            "            `True` when there is some disk offload.\n",
            "        load_in_8bit (`bool`, *optional*, defaults to `False`):\n",
            "            If `True`, will convert the loaded model into mixed-8bit quantized model. To use this feature please\n",
            "            install `bitsandbytes` compiled with your CUDA version by running `pip install -i\n",
            "            https://test.pypi.org/simple/ bitsandbytes-cudaXXX` where XXX is your CUDA version (e.g. 11.6 = 116).\n",
            "            Make also sure that you have enough GPU RAM to store half of the model size since the 8bit modules are\n",
            "            not compiled and adapted for CPUs.\n",
            "        quantization_config (`Dict`, *optional*):\n",
            "            A dictionary of configuration parameters for the `bitsandbytes` library and loading the model using\n",
            "            advanced features such as offloading in fp32 on CPU or on disk.\n",
            "        subfolder (`str`, *optional*, defaults to `\"\"`):\n",
            "            In case the relevant files are located inside a subfolder of the model repo on huggingface.co, you can\n",
            "            specify the folder name here.\n",
            "        variant (`str`, *optional*):\n",
            "            If specified load weights from `variant` filename, *e.g.* pytorch_model.<variant>.bin. `variant` is\n",
            "            ignored when using `from_tf` or `from_flax`.\n",
            "    \n",
            "        kwargs (remaining dictionary of keyword arguments, *optional*):\n",
            "            Can be used to update the configuration object (after it being loaded) and initiate the model (e.g.,\n",
            "            `output_attentions=True`). Behaves differently depending on whether a `config` is provided or\n",
            "            automatically loaded:\n",
            "    \n",
            "                - If a configuration is provided with `config`, `**kwargs` will be directly passed to the\n",
            "                  underlying model's `__init__` method (we assume all relevant updates to the configuration have\n",
            "                  already been done)\n",
            "                - If a configuration is not provided, `kwargs` will be first passed to the configuration class\n",
            "                  initialization function ([`~PretrainedConfig.from_pretrained`]). Each key of `kwargs` that\n",
            "                  corresponds to a configuration attribute will be used to override said attribute with the\n",
            "                  supplied `kwargs` value. Remaining keys that do not correspond to any configuration attribute\n",
            "                  will be passed to the underlying model's `__init__` function.\n",
            "    \n",
            "    <Tip>\n",
            "    \n",
            "    Activate the special [\"offline-mode\"](https://huggingface.co/transformers/installation.html#offline-mode) to\n",
            "    use this method in a firewalled environment.\n",
            "    \n",
            "    </Tip>\n",
            "    \n",
            "    Examples:\n",
            "    \n",
            "    ```python\n",
            "    >>> from transformers import BertConfig, BertModel\n",
            "    \n",
            "    >>> # Download model and configuration from huggingface.co and cache.\n",
            "    >>> model = BertModel.from_pretrained(\"bert-base-uncased\")\n",
            "    >>> # Model was saved using *save_pretrained('./test/saved_model/')* (for example purposes, not runnable).\n",
            "    >>> model = BertModel.from_pretrained(\"./test/saved_model/\")\n",
            "    >>> # Update configuration during loading.\n",
            "    >>> model = BertModel.from_pretrained(\"bert-base-uncased\", output_attentions=True)\n",
            "    >>> assert model.config.output_attentions == True\n",
            "    >>> # Loading from a TF checkpoint file instead of a PyTorch model (slower, for example purposes, not runnable).\n",
            "    >>> config = BertConfig.from_json_file(\"./tf_model/my_tf_model_config.json\")\n",
            "    >>> model = BertModel.from_pretrained(\"./tf_model/my_tf_checkpoint.ckpt.index\", from_tf=True, config=config)\n",
            "    >>> # Loading from a Flax checkpoint file instead of a PyTorch model (slower)\n",
            "    >>> model = BertModel.from_pretrained(\"bert-base-uncased\", from_flax=True)\n",
            "    ```\n",
            "    \n",
            "    * `low_cpu_mem_usage` algorithm:\n",
            "    \n",
            "    This is an experimental function that loads the model using ~1x model size CPU memory\n",
            "    \n",
            "    Here is how it works:\n",
            "    \n",
            "    1. save which state_dict keys we have\n",
            "    2. drop state_dict before the model is created, since the latter takes 1x model size CPU memory\n",
            "    3. after the model has been instantiated switch to the meta device all params/buffers that\n",
            "    are going to be replaced from the loaded state_dict\n",
            "    4. load state_dict 2nd time\n",
            "    5. replace the params/buffers from the state_dict\n",
            "    \n",
            "    Currently, it can't handle deepspeed ZeRO stage 3 and ignores loading errors\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import locale\n",
        "locale.getpreferredencoding = lambda: \"UTF-8\""
      ],
      "metadata": {
        "id": "zMl1s34YvzY4"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers\n",
        "!pip install evaluate\n",
        "!pip install sentencepiece\n",
        "!pip install sacrebleu"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EwZ1TO-up5wC",
        "outputId": "6ad7d4e8-8e04-4e7a-b34f-471d09abb00d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.27.2-py3-none-any.whl (6.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.8/6.8 MB\u001b[0m \u001b[31m52.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.9/dist-packages (from transformers) (2.27.1)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1\n",
            "  Downloading tokenizers-0.13.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m104.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (1.22.4)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.9/dist-packages (from transformers) (4.65.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (2022.10.31)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from transformers) (3.10.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.9/dist-packages (from transformers) (23.0)\n",
            "Collecting huggingface-hub<1.0,>=0.11.0\n",
            "  Downloading huggingface_hub-0.13.3-py3-none-any.whl (199 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m199.8/199.8 KB\u001b[0m \u001b[31m24.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (4.5.0)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (2.0.12)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (2022.12.7)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (1.26.15)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (3.4)\n",
            "Installing collected packages: tokenizers, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.13.3 tokenizers-0.13.2 transformers-4.27.2\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting evaluate\n",
            "  Downloading evaluate-0.4.0-py3-none-any.whl (81 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m81.4/81.4 KB\u001b[0m \u001b[31m3.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting dill\n",
            "  Downloading dill-0.3.6-py3-none-any.whl (110 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m110.5/110.5 KB\u001b[0m \u001b[31m5.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from evaluate) (1.22.4)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.9/dist-packages (from evaluate) (23.0)\n",
            "Requirement already satisfied: fsspec[http]>=2021.05.0 in /usr/local/lib/python3.9/dist-packages (from evaluate) (2023.3.0)\n",
            "Collecting multiprocess\n",
            "  Downloading multiprocess-0.70.14-py39-none-any.whl (132 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m132.9/132.9 KB\u001b[0m \u001b[31m16.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.9/dist-packages (from evaluate) (1.4.4)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.9/dist-packages (from evaluate) (4.65.0)\n",
            "Collecting xxhash\n",
            "  Downloading xxhash-3.2.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (212 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m212.2/212.2 KB\u001b[0m \u001b[31m24.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting responses<0.19\n",
            "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.9/dist-packages (from evaluate) (2.27.1)\n",
            "Requirement already satisfied: huggingface-hub>=0.7.0 in /usr/local/lib/python3.9/dist-packages (from evaluate) (0.13.3)\n",
            "Collecting datasets>=2.0.0\n",
            "  Downloading datasets-2.10.1-py3-none-any.whl (469 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m469.0/469.0 KB\u001b[0m \u001b[31m22.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from datasets>=2.0.0->evaluate) (6.0)\n",
            "Collecting aiohttp\n",
            "  Downloading aiohttp-3.8.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.0 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/1.0 MB\u001b[0m \u001b[31m46.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.9/dist-packages (from datasets>=2.0.0->evaluate) (9.0.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.9/dist-packages (from huggingface-hub>=0.7.0->evaluate) (4.5.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from huggingface-hub>=0.7.0->evaluate) (3.10.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->evaluate) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->evaluate) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->evaluate) (3.4)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests>=2.19.0->evaluate) (1.26.15)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.9/dist-packages (from pandas->evaluate) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.9/dist-packages (from pandas->evaluate) (2022.7.1)\n",
            "Collecting yarl<2.0,>=1.0\n",
            "  Downloading yarl-1.8.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (264 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m264.6/264.6 KB\u001b[0m \u001b[31m30.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting aiosignal>=1.1.2\n",
            "  Downloading aiosignal-1.3.1-py3-none-any.whl (7.6 kB)\n",
            "Collecting frozenlist>=1.1.1\n",
            "  Downloading frozenlist-1.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl (158 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m158.8/158.8 KB\u001b[0m \u001b[31m21.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting async-timeout<5.0,>=4.0.0a3\n",
            "  Downloading async_timeout-4.0.2-py3-none-any.whl (5.8 kB)\n",
            "Collecting multidict<7.0,>=4.5\n",
            "  Downloading multidict-6.0.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (114 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m114.2/114.2 KB\u001b[0m \u001b[31m15.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.9/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (22.2.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.9/dist-packages (from python-dateutil>=2.8.1->pandas->evaluate) (1.16.0)\n",
            "Installing collected packages: xxhash, multidict, frozenlist, dill, async-timeout, yarl, responses, multiprocess, aiosignal, aiohttp, datasets, evaluate\n",
            "Successfully installed aiohttp-3.8.4 aiosignal-1.3.1 async-timeout-4.0.2 datasets-2.10.1 dill-0.3.6 evaluate-0.4.0 frozenlist-1.3.3 multidict-6.0.4 multiprocess-0.70.14 responses-0.18.0 xxhash-3.2.0 yarl-1.8.2\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting sentencepiece\n",
            "  Downloading sentencepiece-0.1.97-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m17.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: sentencepiece\n",
            "Successfully installed sentencepiece-0.1.97\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting sacrebleu\n",
            "  Downloading sacrebleu-2.3.1-py3-none-any.whl (118 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m118.9/118.9 KB\u001b[0m \u001b[31m3.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting colorama\n",
            "  Downloading colorama-0.4.6-py2.py3-none-any.whl (25 kB)\n",
            "Requirement already satisfied: tabulate>=0.8.9 in /usr/local/lib/python3.9/dist-packages (from sacrebleu) (0.8.10)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.9/dist-packages (from sacrebleu) (2022.10.31)\n",
            "Collecting portalocker\n",
            "  Downloading portalocker-2.7.0-py2.py3-none-any.whl (15 kB)\n",
            "Requirement already satisfied: lxml in /usr/local/lib/python3.9/dist-packages (from sacrebleu) (4.9.2)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from sacrebleu) (1.22.4)\n",
            "Installing collected packages: portalocker, colorama, sacrebleu\n",
            "Successfully installed colorama-0.4.6 portalocker-2.7.0 sacrebleu-2.3.1\n"
          ]
        }
      ]
    }
  ]
}